From 3c46cc9c3572d4bf70026559d95eb96f49096759 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 09:59:24 -0500 Subject: [PATCH 001/160] [usb_uart] Fix uint32_t format specifier warning in pl2303 (#18310) --- esphome/components/usb_uart/pl2303.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/usb_uart/pl2303.cpp b/esphome/components/usb_uart/pl2303.cpp index 3c7ecd9a83..c56f43f75a 100644 --- a/esphome/components/usb_uart/pl2303.cpp +++ b/esphome/components/usb_uart/pl2303.cpp @@ -292,8 +292,8 @@ bool USBUartTypePL2303::config_step(USBUartChannel *channel, uint8_t step, bool // Data bits line_coding[6] = channel->get_data_bits(); - ESP_LOGD(TAG, "PL2303: SET_LINE_REQUEST baud=%u stop=%u parity=%u data=%u", baud, line_coding[4], line_coding[5], - line_coding[6]); + ESP_LOGD(TAG, "PL2303: SET_LINE_REQUEST baud=%" PRIu32 " stop=%u parity=%u data=%u", baud, line_coding[4], + line_coding[5], line_coding[6]); std::vector lc_vec(line_coding, line_coding + 7); this->config_transfer_(SET_LINE_REQUEST_TYPE, SET_LINE_REQUEST, 0, iface, lc_vec); From 1a01c34ec4fed020264e69e78f08ad3f29af8bad Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:33:45 +0000 Subject: [PATCH 002/160] Bump aioesphomeapi from 45.10.0 to 45.10.1 (#18318) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9c231bd0fe..85a0f55263 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.10.0 +aioesphomeapi==45.10.1 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From f2121130f971b63fd6776dcd00d8befe0fea2aee Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Wed, 12 Aug 2026 12:49:54 -0400 Subject: [PATCH 003/160] [sendspin] Bump sendspin-cpp to v0.7.2 (#18316) --- esphome/components/sendspin/__init__.py | 2 +- esphome/idf_component.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index bd889c2c92..082639374f 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -234,7 +234,7 @@ async def to_code(config: ConfigType) -> None: psram.request_external_task_stack() # sendspin-cpp library - esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.7.1") + esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.7.2") cg.add_define("USE_SENDSPIN", True) # for MDNS diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 6a9d7171ec..aff1a6819f 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -98,7 +98,7 @@ dependencies: esp32async/asynctcp: version: 3.4.91 sendspin/sendspin-cpp: - version: 0.7.1 + version: 0.7.2 lvgl/lvgl: version: 9.5.0 fastled/FastLED: From 99677390e04468438ec2a908bf431a6f16a63bae Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:27:14 -0500 Subject: [PATCH 004/160] Bump bundled esphome-device-builder to 1.9.6 (#18328) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a4f5d3c3a6..d7ae2cd4ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.9.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.9.6 RUN \ platformio settings set enable_telemetry No \ From 905485b6738213f33e1663ae3ab56cf8a3281289 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 18:49:34 -0500 Subject: [PATCH 005/160] [ld2420] Fix out-of-bounds read when device reports unknown command error (#18322) --- esphome/components/ld2420/ld2420.cpp | 9 ++++++++- esphome/components/ld2420/ld2420.h | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/esphome/components/ld2420/ld2420.cpp b/esphome/components/ld2420/ld2420.cpp index ae622cda28..f71bec7e5f 100644 --- a/esphome/components/ld2420/ld2420.cpp +++ b/esphome/components/ld2420/ld2420.cpp @@ -746,7 +746,14 @@ void LD2420Component::set_reg_value(uint16_t reg, uint16_t value) { this->send_cmd_from_array(cmd_frame); } -void LD2420Component::handle_cmd_error(uint8_t error) { ESP_LOGE(TAG, "Command failed: %s", ERR_MESSAGE[error]); } +void LD2420Component::handle_cmd_error(uint16_t error) { + if (error < std::size(ERR_MESSAGE)) { + ESP_LOGE(TAG, "Command failed: %s", ERR_MESSAGE[error]); + } else { + // The error word comes from the device reply frame; unknown codes must not index ERR_MESSAGE + ESP_LOGE(TAG, "Command failed: error 0x%04X", error); + } +} int LD2420Component::get_gate_threshold_(uint8_t gate) { uint8_t error; diff --git a/esphome/components/ld2420/ld2420.h b/esphome/components/ld2420/ld2420.h index ae44b16065..977ee2eccc 100644 --- a/esphome/components/ld2420/ld2420.h +++ b/esphome/components/ld2420/ld2420.h @@ -108,7 +108,7 @@ class LD2420Component final : public Component, public uart::UARTDevice { float get_setup_priority() const override; int send_cmd_from_array(CmdFrameT cmd_frame); void report_gate_data(); - void handle_cmd_error(uint8_t error); + void handle_cmd_error(uint16_t error); void set_operating_mode(const char *state); void auto_calibrate_sensitivity(); void update_radar_data(uint16_t const *gate_energy, uint8_t sample_number); From 787a909aa49df808ed1d55f77d9dea0e60e6e4ab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 19:11:32 -0500 Subject: [PATCH 006/160] [core] Don't block logs startup on MQTT IP discovery when addresses are known (#18313) --- esphome/__main__.py | 144 +++++++++---- esphome/api_client.py | 87 +++++++- esphome/mqtt.py | 91 ++++++-- tests/unit_tests/test_api_client.py | 323 +++++++++++++++++++++++++++- tests/unit_tests/test_main.py | 171 +++++++++++---- tests/unit_tests/test_mqtt.py | 262 ++++++++++++++++++++++ 6 files changed, 982 insertions(+), 96 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 0ac5898268..1262a4525e 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -10,7 +10,7 @@ from pathlib import Path import re import sys import time -from typing import Protocol +from typing import TYPE_CHECKING, Protocol # Note: Do not import modules from esphome.components here, as this would # cause them to be loaded before external components are processed, resulting @@ -71,6 +71,9 @@ from esphome.util import ( safe_print, ) +if TYPE_CHECKING: + import threading + # Keep expensive imports (zeroconf, writer, yaml_util, etc.) out of this # module's top level. Every `esphome` invocation — including fast paths # like `esphome version` — pays the cost of what's imported here before @@ -567,11 +570,48 @@ def has_name_add_mac_suffix() -> bool: def mqtt_get_ip( - config: ConfigType, username: str, password: str, client_id: str + config: ConfigType, + username: str, + password: str, + client_id: str, + stop_event: "threading.Event | None" = None, ) -> list[str]: from esphome import mqtt - return mqtt.get_esphome_device_ip(config, username, password, client_id) + return mqtt.get_esphome_device_ip( + config, username, password, client_id, stop_event=stop_event + ) + + +def _add_network_device(device: str, network_devices: list[str]) -> None: + """Append a device to the list, expanding it through ``CORE.address_cache``. + + If the hostname is already in the address cache (e.g. populated by mDNS + discovery), substitute the cached IPs so aioesphomeapi doesn't open its + own Zeroconf to re-resolve it. Duplicates are dropped. + """ + if CORE.address_cache and (cached := CORE.address_cache.get_addresses(device)): + network_devices.extend(addr for addr in cached if addr not in network_devices) + elif device not in network_devices: + network_devices.append(device) + + +def _split_network_devices(devices: list[str]) -> tuple[list[str], bool]: + """Split the device list into direct addresses and an MQTT-lookup flag. + + Direct addresses are expanded through ``CORE.address_cache`` and deduped + the same way ``_resolve_network_devices`` does; MQTT/MQTTIP magic strings + are not resolved, only reported via the returned bool so the caller can + defer the broker lookup. + """ + network_devices: list[str] = [] + has_mqtt_lookup = False + for device in devices: + if get_port_type(device) in _MQTT_PORT_TYPES: + has_mqtt_lookup = True + else: + _add_network_device(device, network_devices) + return network_devices, has_mqtt_lookup def _resolve_network_devices( @@ -604,40 +644,44 @@ def _resolve_network_devices( if port_type in _MQTT_PORT_TYPES: # Only resolve MQTT once, even if multiple MQTT entries if not mqtt_resolved: - try: - mqtt_ips = mqtt_get_ip( - config, args.username, args.password, args.client_id - ) - # pylint can't infer mqtt_get_ip's return through its - # lazy ``from esphome import mqtt`` import, so it flags - # the genexpr below. - network_devices.extend( - addr - for addr in mqtt_ips # pylint: disable=not-an-iterable - if addr not in network_devices - ) - except EsphomeError as err: - _LOGGER.warning( - "MQTT IP discovery failed (%s), will try other devices if available", - err, - ) + mqtt_ips = _mqtt_get_ip_or_warn( + config, args.username, args.password, args.client_id + ) + network_devices.extend( + addr for addr in mqtt_ips if addr not in network_devices + ) mqtt_resolved = True continue - # If the hostname is already in the address cache (e.g. populated by - # mDNS discovery), substitute the cached IPs so aioesphomeapi doesn't - # open its own Zeroconf to re-resolve it. - if CORE.address_cache and (cached := CORE.address_cache.get_addresses(device)): - network_devices.extend( - addr for addr in cached if addr not in network_devices - ) - elif device not in network_devices: - # Regular network address or IP - add if not already present - network_devices.append(device) + _add_network_device(device, network_devices) return network_devices +def _mqtt_get_ip_or_warn( + config: ConfigType, + username: str, + password: str, + client_id: str, + stop_event: "threading.Event | None" = None, +) -> list[str]: + """Look up the device IP via MQTT, returning [] with a warning on failure. + + This owns the failure policy for MQTT IP discovery on paths that have + other addresses to fall back on: a broker problem must not abort the + operation. Also used as the deferred resolver handed to ``run_logs``, + where it runs in a worker thread. + """ + try: + return mqtt_get_ip(config, username, password, client_id, stop_event=stop_event) + except EsphomeError as err: + _LOGGER.warning( + "MQTT IP discovery failed (%s), will try other devices if available", + err, + ) + return [] + + def run_miniterm(config: ConfigType, port: str, args) -> int: from datetime import datetime @@ -1438,17 +1482,37 @@ def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int return run_miniterm(config, port, args) # Check if we should use API for logging - # Resolve MQTT magic strings to actual IP addresses - if has_api() and ( - network_devices := _resolve_network_devices(devices, config, args) - ): - from esphome.api_client import run_logs + if has_api(): + network_devices, has_mqtt_lookup = _split_network_devices(devices) + mqtt_resolver = None + if has_mqtt_lookup: + if network_devices: + # Addresses are already known, so don't block startup on the + # MQTT broker lookup; hand it to run_logs as a deferred + # resolver that runs in the background and feeds discovered + # addresses into the running log client, keeping MQTT as a + # fallback for when the known addresses are stale (e.g. DHCP + # reassigned the IP). + mqtt_resolver = functools.partial( + _mqtt_get_ip_or_warn, + config, + args.username, + args.password, + args.client_id, + ) + else: + # The MQTT lookup is the only way to find the device; resolve + # it up front since the client needs an address to start with. + network_devices = _resolve_network_devices(devices, config, args) + if network_devices: + from esphome.api_client import run_logs - return run_logs( - config, - network_devices, - subscribe_states=_should_subscribe_states(args), - ) + return run_logs( + config, + network_devices, + subscribe_states=_should_subscribe_states(args), + mqtt_resolver=mqtt_resolver, + ) if port_type in (PortType.NETWORK, PortType.MQTT) and has_mqtt_logging(): from esphome import mqtt diff --git a/esphome/api_client.py b/esphome/api_client.py index a75f219b17..fb41075de8 100644 --- a/esphome/api_client.py +++ b/esphome/api_client.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio from contextlib import suppress import logging +import threading from typing import TYPE_CHECKING, Any import warnings @@ -20,6 +21,8 @@ from esphome.stacktrace import LogLineProcessor from esphome.util import safe_print if TYPE_CHECKING: + from collections.abc import Callable + from aioesphomeapi.api_pb2 import ( SubscribeLogsResponse, # pylint: disable=no-name-in-module ) @@ -32,8 +35,18 @@ async def async_run_logs( config: dict[str, Any], addresses: list[str], subscribe_states: bool = True, + mqtt_resolver: Callable[[threading.Event], list[str]] | None = None, ) -> None: - """Run the logs command in the event loop.""" + """Run the logs command in the event loop. + + If ``mqtt_resolver`` is given, it is called in a worker thread (paho-mqtt + has no asyncio support on Windows) concurrently with the connection + attempts to ``addresses``, and any addresses it discovers are fed into + the running client. It owns its own failure handling (returning [] when + discovery fails) and must honor the ``threading.Event`` it is passed so + teardown is not delayed by the lookup's wait window; the initial broker + connect itself is only bounded by the socket timeout. + """ from datetime import datetime conf = config["api"] @@ -60,6 +73,41 @@ async def async_run_logs( # Decoder resolution policy lives in LogLineProcessor. processor = LogLineProcessor(config, CORE.target_platform) + mqtt_task: asyncio.Task[None] | None = None + mqtt_stop_event = threading.Event() + + def _cancel_mqtt_discovery() -> None: + """Stop the broker lookup once a connection has been established. + + Its answer is only useful while still disconnected: after that it + either duplicates the connected address or arrives too late to + matter, so don't keep an idle broker session open for it. + """ + mqtt_stop_event.set() + if mqtt_task is not None and not mqtt_task.done(): + mqtt_task.cancel() + + async def _resolve_mqtt_addresses() -> None: + """Discover the device address via the MQTT broker in the background.""" + try: + mqtt_ips = await asyncio.to_thread(mqtt_resolver, mqtt_stop_event) + if not mqtt_ips: + _LOGGER.debug( + "MQTT discovery %s", + "aborted" if mqtt_stop_event.is_set() else "found no addresses", + ) + return + if cli.add_addresses(mqtt_ips): + _LOGGER.info("Discovered address(es) via MQTT: %s", ", ".join(mqtt_ips)) + else: + _LOGGER.debug( + "MQTT-discovered address(es) already known: %s", ", ".join(mqtt_ips) + ) + except Exception: # pylint: disable=broad-except + # A background task failure would otherwise stay invisible for + # the whole session and only re-raise at teardown + _LOGGER.exception("MQTT address discovery failed") + def on_log(msg: SubscribeLogsResponse) -> None: """Handle a new log message.""" time_ = datetime.now().astimezone() @@ -98,20 +146,53 @@ async def async_run_logs( # A top-level ``deep_sleep:`` block means the device is only awake # briefly; cap the reconnect backoff so a wake window is not missed. deep_sleep="deep_sleep" in config, + on_connect=_cancel_mqtt_discovery if mqtt_resolver is not None else None, ) try: + # Don't start (or keep) the broker lookup if a connection already + # succeeded; the stop event doubles as the not-needed-anymore latch + # and get_esphome_device_ip returns immediately when it is set. + if mqtt_resolver is not None and not mqtt_stop_event.is_set(): + mqtt_task = asyncio.create_task(_resolve_mqtt_addresses()) await asyncio.Event().wait() finally: - await stop() + try: + if mqtt_task is not None: + # Unblock the worker thread first so it can't hold up + # loop.shutdown_default_executor() for the full lookup timeout. + mqtt_stop_event.set() + # Give the worker a moment to exit through its own error + # handling; cancelling first would race out a late failure. + done, _ = await asyncio.wait([mqtt_task], timeout=1.0) + if not done: + mqtt_task.cancel() + # return_exceptions keeps a CancelledError from the cancel() + # above from re-raising here and jumping over the stop() below. + # The task handles Exception itself, so only a BaseException + # escape (e.g. SystemExit from the worker) can land here. + (result,) = await asyncio.gather(mqtt_task, return_exceptions=True) + if isinstance(result, BaseException) and not isinstance( + result, asyncio.CancelledError + ): + _LOGGER.error("MQTT address discovery failed", exc_info=result) + finally: + # Must run even if a second cancellation lands mid-cleanup above + await stop() def run_logs( config: dict[str, Any], addresses: list[str], subscribe_states: bool = True, + mqtt_resolver: Callable[[threading.Event], list[str]] | None = None, ) -> None: """Run the logs command.""" with suppress(KeyboardInterrupt): asyncio.run( - async_run_logs(config, addresses, subscribe_states=subscribe_states) + async_run_logs( + config, + addresses, + subscribe_states=subscribe_states, + mqtt_resolver=mqtt_resolver, + ) ) diff --git a/esphome/mqtt.py b/esphome/mqtt.py index 3198de9d21..62deafb09a 100644 --- a/esphome/mqtt.py +++ b/esphome/mqtt.py @@ -6,6 +6,7 @@ from pathlib import Path import ssl import tempfile import time +from typing import TYPE_CHECKING import paho.mqtt.client as mqtt @@ -31,6 +32,9 @@ from esphome.helpers import get_int_env, get_str_env from esphome.types import ConfigType from esphome.util import safe_print +if TYPE_CHECKING: + import threading + _LOGGER = logging.getLogger(__name__) @@ -164,6 +168,7 @@ def get_esphome_device_ip( password: str | None = None, client_id: str | None = None, timeout: float = 25, + stop_event: "threading.Event | None" = None, ) -> list[str]: if CONF_MQTT not in config: raise EsphomeError( @@ -182,55 +187,113 @@ def get_esphome_device_ip( dev_name = config[CONF_ESPHOME][CONF_NAME] dev_ip = None + failed = False topic = "esphome/discover/" + dev_name _LOGGER.info("Starting looking for IP in topic %s", topic) def on_message(client, userdata, msg): - nonlocal dev_ip + nonlocal dev_ip, failed time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") if len(payload) > 0: message = time_ + " " + payload _LOGGER.debug(message) - data = json.loads(payload) + try: + data = json.loads(payload) + except ValueError: + data = None + if not isinstance(data, dict): + # A raise in this handler would kill paho's network thread + _LOGGER.warning("Ignoring unparsable discovery payload") + return if "name" not in data or data["name"] != dev_name: _LOGGER.warning("Wrong device answer") return - dev_ip = [] + addresses = [] key = "ip" n = 0 while key in data: - dev_ip.append(data[key]) + value = data[key] + if ( + isinstance(value, str) + and (value := value.strip()) + and value.isprintable() + ): + addresses.append(value) + else: + # repr-escaped and truncated: must not forge log lines + _LOGGER.warning( + "Ignoring invalid address in discovery answer: %s", + repr(value)[:100], + ) n = n + 1 key = "ip" + str(n) - if dev_ip: - client.disconnect() + if not addresses: + _LOGGER.warning("Device answer did not include an IP address") + failed = True + return + + dev_ip = addresses + failed = False # a complete answer wins over an earlier empty one + client.disconnect() def on_connect(client, userdata, flags, return_code): topic = "esphome/ping/" + dev_name _LOGGER.info("Send discover via MQTT broker topic: %s", topic) client.publish(topic, None, retain=False) + if stop_event is not None and stop_event.is_set(): + # Teardown already started; don't open a broker connection at all + return [] + + def on_disconnect(client, userdata, result_code): + nonlocal failed + if result_code != 0: + _LOGGER.warning("Disconnected from MQTT broker (%s)", result_code) + failed = True + mqtt_client = prepare( config, [topic], on_message, on_connect, username, password, client_id ) + # Discovery is one-shot; prepare()'s reconnect-forever on_disconnect runs + # on the network thread and would make loop_stop() below join forever. + mqtt_client.on_disconnect = on_disconnect - mqtt_client.loop_start() - while timeout > 0: - if dev_ip is not None: - break - timeout -= 0.250 - time.sleep(0.250) - mqtt_client.loop_stop() + if stop_event is None: + import threading + + stop_event = threading.Event() # never set; wait() below is a plain sleep + stopped = stop_event.is_set() # teardown may have started during connect + try: + if not stopped: + mqtt_client.loop_start() + while timeout > 0: + if dev_ip is not None or failed: + break + if stop_event.wait(0.250): + stopped = True + break + timeout -= 0.250 + finally: + # A cleanup failure must not replace the discovery result or its + # EsphomeError; a second disconnect after on_message's is harmless. + try: + mqtt_client.disconnect() + except Exception: # pylint: disable=broad-except + _LOGGER.debug("Error disconnecting from MQTT broker", exc_info=True) + mqtt_client.loop_stop() # only signals and joins; does not raise if dev_ip is None: + if stopped: + # Aborted by the caller, not a failure; stay quiet + return [] raise EsphomeError("Failed to find IP via MQTT") - _LOGGER.info("Found IP: %s", dev_ip) + _LOGGER.info("Found IP via MQTT broker: %s", ", ".join(dev_ip)) return dev_ip diff --git a/tests/unit_tests/test_api_client.py b/tests/unit_tests/test_api_client.py index 19ed83abe1..405567d84f 100644 --- a/tests/unit_tests/test_api_client.py +++ b/tests/unit_tests/test_api_client.py @@ -56,7 +56,7 @@ async def test_async_run_logs_full_flow(caplog) -> None: with ( patch.object(api_client, "async_run", mock_run), - patch.object(api_client, "APIClient") as mock_client, + patch.object(api_client, "APIClient", autospec=True) as mock_client, patch.object(api_client, "safe_print", printed.append), ): task = asyncio.get_running_loop().create_task( @@ -163,3 +163,324 @@ async def test_async_run_logs_passes_deep_sleep( await api_client.async_run_logs(config, ["1.2.3.4"]) assert mock_run.call_args.kwargs["deep_sleep"] is expected_deep_sleep + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_feeds_addresses(caplog) -> None: + """Addresses discovered via MQTT are fed into the running client.""" + caplog.set_level("INFO", logger="esphome.api_client") + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + fed = asyncio.Event() + + def resolver(stop_event): + return ["10.0.0.9", "10.0.0.10"] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + mock_client.return_value.add_addresses.side_effect = lambda addrs: ( + fed.set() or True + ) + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + async with asyncio.timeout(1): + await fed.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_called_once_with( + ["10.0.0.9", "10.0.0.10"] + ) + assert "Discovered address(es) via MQTT" in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_no_addresses_keeps_running() -> None: + """A resolver returning nothing (failed lookup) leaves the session running.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + + def resolver(stop_event): + # The resolver owns failure handling; a failed lookup returns [] + resolver_ran.set() + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + await asyncio.sleep(0) + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_not_called() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_stopped_on_teardown() -> None: + """Teardown sets the resolver's stop event so the thread exits promptly.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + captured_event: threading.Event | None = None + resolver_started = threading.Event() + + def resolver(stop_event): + nonlocal captured_event + captured_event = stop_event + resolver_started.set() + # Simulate a slow broker lookup that only ends via the stop event. + stop_event.wait(timeout=5) + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_started.wait, 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert captured_event is not None + assert captured_event.is_set() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_crash_still_stops_cleanly(caplog) -> None: + """A resolver raising unexpectedly must not skip stop() at teardown.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + + def resolver(stop_event): + resolver_ran.set() + raise RuntimeError("resolver blew up") + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + await asyncio.sleep(0.05) + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert "MQTT address discovery failed" in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_connect_cancels_mqtt_discovery() -> None: + """A successful connection stops the in-flight broker lookup.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + captured_event: threading.Event | None = None + resolver_started = threading.Event() + + def resolver(stop_event): + nonlocal captured_event + captured_event = stop_event + resolver_started.set() + stop_event.wait(timeout=5) + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)) as mock_run, + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_started.wait, 1) + + # The runner reports a successful connection + on_connect = mock_run.call_args.kwargs["on_connect"] + on_connect() + await asyncio.sleep(0.05) + + assert captured_event is not None + assert captured_event.is_set() + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_not_called() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_connect_before_discovery_skips_lookup() -> None: + """A connection during async_run startup prevents the lookup from starting.""" + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver = Mock(name="resolver") + + async def fake_async_run(*args, **kwargs): + # Connection succeeds before async_run even returns + kwargs["on_connect"]() + return stop + + with ( + patch.object(api_client, "async_run", AsyncMock(side_effect=fake_async_run)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.sleep(0.05) + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + resolver.assert_not_called() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_duplicate_addresses_logged(caplog) -> None: + """A discovery the client rejects as already known leaves a debug trace.""" + import threading + + caplog.set_level("DEBUG", logger="esphome.api_client") + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + fed = threading.Event() + + def resolver(stop_event): + return ["1.2.3.4"] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + mock_client.return_value.add_addresses.side_effect = lambda addrs: ( + fed.set() or False + ) + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(fed.wait, 1) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_called_once_with(["1.2.3.4"]) + assert "MQTT-discovered address(es) already known: 1.2.3.4" in caplog.text + assert "Discovered address(es) via MQTT" not in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_base_exception_escape_logged_at_teardown(caplog) -> None: + """A BaseException escaping the worker is reported, and stop() still runs.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + + class WorkerEscape(BaseException): + """Not an Exception, so the task-level guard must not catch it.""" + + def resolver(stop_event): + resolver_ran.set() + raise WorkerEscape("worker bailed") + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert "MQTT address discovery failed" in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_stubborn_worker_cancelled_at_teardown() -> None: + """A worker that ignores the stop event is cancelled after the grace period.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + release = threading.Event() + + def resolver(stop_event): + resolver_ran.set() + # Ignore stop_event entirely; only the test releases us + release.wait(timeout=10) + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + release.set() + + stop.assert_awaited_once() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 23bfdbcd69..a40341e194 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -25,6 +25,7 @@ from esphome.__main__ import ( _make_crystal_freq_callback, _redact_with_legacy_fallback, _resolve_network_devices, + _split_network_devices, _unresolved_default_error, _validate_bootloader_binary, _validate_partition_table_binary, @@ -2879,7 +2880,9 @@ def test_upload_program_ota_with_mqtt_resolution( assert exit_code == 0 assert host == "192.168.1.100" - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) expected_firmware = ( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) @@ -2926,7 +2929,9 @@ def test_upload_program_ota_with_mqtt_empty_broker( assert exit_code == 0 assert host == "192.168.1.50" # Verify MQTT was attempted but failed gracefully - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify we fell back to the IP address expected_firmware = ( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" @@ -3015,7 +3020,10 @@ def test_show_logs_api( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100", "192.168.1.101"], subscribe_states=True + CORE.config, + ["192.168.1.100", "192.168.1.101"], + subscribe_states=True, + mqtt_resolver=None, ) @@ -3042,7 +3050,7 @@ def test_show_logs_api_no_states( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=False + CORE.config, ["192.168.1.100"], subscribe_states=False, mqtt_resolver=None ) @@ -3069,7 +3077,7 @@ def test_show_logs_api_with_fqdn_mdns_disabled( assert result == 0 # Should use the FQDN directly, not try MQTT lookup mock_run_logs.assert_called_once_with( - CORE.config, ["device.example.com"], subscribe_states=True + CORE.config, ["device.example.com"], subscribe_states=True, mqtt_resolver=None ) @@ -3097,9 +3105,44 @@ def test_show_logs_api_with_mqtt_fallback( result = show_logs(CORE.config, args, devices) assert result == 0 - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None + ) mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.200"], subscribe_states=True + CORE.config, ["192.168.1.200"], subscribe_states=True, mqtt_resolver=None + ) + + +@patch("esphome.mqtt.show_logs") +def test_show_logs_api_mqtt_only_resolve_failure_falls_back_to_mqtt_logs( + mock_mqtt_show_logs: Mock, + mock_mqtt_get_ip: Mock, +) -> None: + """With no addresses at all after a failed MQTT lookup, MQTT logging is used.""" + setup_core( + config={ + "logger": {}, + CONF_API: {}, + CONF_MQTT: {CONF_BROKER: "mqtt.local"}, + }, + platform=PLATFORM_ESP32, + ) + mock_mqtt_show_logs.return_value = 0 + mock_mqtt_get_ip.side_effect = EsphomeError("Failed to find IP via MQTT") + + args = MockArgs( + topic="esphome/logs", username="user", password="pass", client_id="client" + ) + devices = ["MQTT", "MQTTIP"] + + result = show_logs(CORE.config, args, devices) + + assert result == 0 + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None + ) + mock_mqtt_show_logs.assert_called_once_with( + CORE.config, "esphome/logs", "user", "pass", "client" ) @@ -3466,7 +3509,9 @@ def test_mqtt_get_ip() -> None: result = mqtt_get_ip(config, "user", "pass", "client-id") assert result == ["192.168.1.100", "192.168.1.101"] - mock_get_ip.assert_called_once_with(config, "user", "pass", "client-id") + mock_get_ip.assert_called_once_with( + config, "user", "pass", "client-id", stop_event=None + ) def test_has_resolvable_address() -> None: @@ -3847,6 +3892,37 @@ def test_resolve_network_devices_keeps_uncached_hosts(tmp_path: Path) -> None: assert result == ["unknown.local", "192.168.1.50"] +def test_split_network_devices_direct_only(tmp_path: Path) -> None: + """Direct addresses pass through deduped, with no MQTT flag.""" + setup_core(tmp_path=tmp_path) + + assert _split_network_devices(["192.168.1.50", "device.local", "192.168.1.50"]) == ( + ["192.168.1.50", "device.local"], + False, + ) + + +def test_split_network_devices_mqtt_only(tmp_path: Path) -> None: + """MQTT magic strings produce no direct addresses, only the flag.""" + setup_core(tmp_path=tmp_path) + + assert _split_network_devices(["MQTTIP", "MQTT"]) == ([], True) + + +def test_split_network_devices_expands_cached_mdns_hosts(tmp_path: Path) -> None: + """Hostnames in ``CORE.address_cache`` are expanded like _resolve_network_devices.""" + setup_core(tmp_path=tmp_path) + CORE.address_cache = AddressCache( + mdns_cache={ + "device-abc123.local": ["10.0.0.1", "10.0.0.2"], + } + ) + + assert _split_network_devices( + ["device-abc123.local", "MQTTIP", "192.168.1.50", "device-abc123.local"] + ) == (["10.0.0.1", "10.0.0.2", "192.168.1.50"], True) + + def test_await_discovery_timeout_returns_empty( caplog: pytest.LogCaptureFixture, ) -> None: @@ -5022,7 +5098,9 @@ def test_upload_program_ota_static_ip_with_mqttip( assert host == "192.168.1.100" # Verify MQTT was resolved - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with both IPs expected_firmware = ( @@ -5069,7 +5147,9 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( assert host == "192.168.2.50" # Verify MQTT was only resolved once despite multiple MQTT magic strings - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with all unique IPs expected_firmware = ( @@ -5116,7 +5196,9 @@ def test_upload_program_ota_mqttip_deduplication( assert host == "192.168.1.100" # Verify MQTT was resolved - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with deduplicated IPs (only one instance of 192.168.1.100) # Note: Current implementation doesn't dedupe, so we'll get the IP twice @@ -5136,7 +5218,9 @@ def test_show_logs_api_static_ip_with_mqttip( This tests the scenario where a device has manual_ip (static IP) configured and MQTT is also configured. The devices list contains both the static IP - and "MQTTIP" magic string. + and "MQTTIP" magic string. The MQTT lookup must not block startup; it is + handed to run_logs as a deferred resolver instead (issue #18311), while + still being reachable as a fallback for a stale static IP. """ setup_core( config={ @@ -5157,12 +5241,19 @@ def test_show_logs_api_static_ip_with_mqttip( assert result == 0 - # Verify MQTT was resolved - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") + # The broker must not be contacted before run_logs starts + mock_mqtt_get_ip.assert_not_called() - # Verify run_logs was called with both IPs - mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100", "192.168.2.50"], subscribe_states=True + # run_logs gets the static IP immediately plus a deferred MQTT resolver + mock_run_logs.assert_called_once() + assert mock_run_logs.call_args.args == (CORE.config, ["192.168.1.100"]) + assert mock_run_logs.call_args.kwargs["subscribe_states"] is True + resolver = mock_run_logs.call_args.kwargs["mqtt_resolver"] + + # Invoking the resolver performs the MQTT lookup (the #11260 fallback) + assert resolver(None) == ["192.168.2.50"] + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None ) @@ -5171,7 +5262,7 @@ def test_show_logs_api_multiple_mqttip_resolves_once( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, ) -> None: - """Test that MQTT resolution only happens once for show_logs with multiple MQTT magic strings.""" + """Test that multiple MQTT magic strings collapse into one deferred resolver.""" setup_core( config={ "logger": {}, @@ -5191,16 +5282,16 @@ def test_show_logs_api_multiple_mqttip_resolves_once( assert result == 0 - # Verify MQTT was only resolved once despite multiple MQTT magic strings - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") + # Note: "MQTT" is a different magic string from "MQTTIP", but both defer + # to the same single resolver; the broker is not contacted eagerly + mock_mqtt_get_ip.assert_not_called() + mock_run_logs.assert_called_once() + assert mock_run_logs.call_args.args == (CORE.config, ["192.168.1.100"]) - # Verify run_logs was called with all unique IPs (MQTT strings replaced with IPs) - # Note: "MQTT" is a different magic string from "MQTTIP", but both trigger MQTT resolution - # The _resolve_network_devices helper filters out both after first resolution - mock_run_logs.assert_called_once_with( - CORE.config, - ["192.168.2.50", "192.168.2.51", "192.168.1.100"], - subscribe_states=True, + resolver = mock_run_logs.call_args.kwargs["mqtt_resolver"] + assert resolver(None) == ["192.168.2.50", "192.168.2.51"] + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None ) @@ -5238,7 +5329,9 @@ def test_upload_program_ota_mqtt_timeout_fallback( assert host == "192.168.1.100" # Verify MQTT was attempted - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with only the static IP (MQTT failed) expected_firmware = ( @@ -5254,7 +5347,7 @@ def test_show_logs_api_mqtt_timeout_fallback( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, ) -> None: - """Test show_logs falls back to other devices when MQTT times out.""" + """Test show_logs proceeds with the static IP when MQTT times out.""" setup_core( config={ "logger": {}, @@ -5273,15 +5366,17 @@ def test_show_logs_api_mqtt_timeout_fallback( result = show_logs(CORE.config, args, devices) - # Should succeed using the static IP even though MQTT failed + # Logs start on the static IP without waiting for the broker assert result == 0 + mock_run_logs.assert_called_once() + assert mock_run_logs.call_args.args == (CORE.config, ["192.168.1.100"]) - # Verify MQTT was attempted - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") - - # Verify run_logs was called with only the static IP (MQTT failed) - mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=True + # The deferred resolver owns the failure policy: it logs a warning and + # returns no addresses so the session keeps running on the known ones + resolver = mock_run_logs.call_args.kwargs["mqtt_resolver"] + assert resolver(None) == [] + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None ) @@ -6764,7 +6859,7 @@ def test_command_run_passes_no_states_to_show_logs( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=False + CORE.config, ["192.168.1.100"], subscribe_states=False, mqtt_resolver=None ) @@ -6805,7 +6900,7 @@ def test_command_run_defaults_subscribe_states_true( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=True + CORE.config, ["192.168.1.100"], subscribe_states=True, mqtt_resolver=None ) diff --git a/tests/unit_tests/test_mqtt.py b/tests/unit_tests/test_mqtt.py index 4c2c34dff1..1ae10d0eb5 100644 --- a/tests/unit_tests/test_mqtt.py +++ b/tests/unit_tests/test_mqtt.py @@ -2,6 +2,11 @@ from __future__ import annotations +import json +import threading +import time +from unittest.mock import MagicMock, patch + import pytest from esphome.const import CONF_BROKER, CONF_ESPHOME, CONF_MQTT, CONF_NAME @@ -89,3 +94,260 @@ def test_get_esphome_device_ip_missing_name() -> None: match="Cannot discover IP via MQTT as the config does not include the device name:", ): get_esphome_device_ip(config) + + +def _discovery_config() -> dict: + return { + CONF_MQTT: { + CONF_BROKER: "mqtt.local", + }, + CONF_ESPHOME: { + CONF_NAME: "test-device", + }, + } + + +def _deliver_on_loop_start(mock_prepare, client, payload: bytes) -> None: + """Deliver a discovery answer as soon as the network loop starts.""" + + def deliver(*args, **kwargs): + msg = MagicMock() + msg.payload = payload + mock_prepare.call_args.args[2](client, None, msg) + + client.loop_start.side_effect = deliver + + +def test_get_esphome_device_ip_success() -> None: + """A device answer on the discovery topic returns its IPs.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, + client, + json.dumps( + {"name": "test-device", "ip": "10.0.0.5", "ip1": "10.0.0.6"} + ).encode(), + ) + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5", "10.0.0.6"] + client.loop_stop.assert_called_once_with() + # Once from on_message on receiving the answer, once from the finally + assert client.disconnect.call_count == 2 + + +def test_get_esphome_device_ip_preset_stop_event_skips_lookup() -> None: + """A stop event set before the call returns [] without touching the broker.""" + stop_event = threading.Event() + stop_event.set() + + with patch("esphome.mqtt.prepare") as mock_prepare: + result = get_esphome_device_ip(_discovery_config(), stop_event=stop_event) + + assert result == [] + mock_prepare.assert_not_called() + + +def test_get_esphome_device_ip_stop_event_aborts_wait() -> None: + """A stop event set mid-wait exits quietly with no addresses.""" + stop_event = threading.Event() + client = MagicMock() + # Simulate teardown starting right after the network loop spins up + client.loop_start.side_effect = stop_event.set + + start = time.monotonic() + with patch("esphome.mqtt.prepare", return_value=client): + result = get_esphome_device_ip(_discovery_config(), stop_event=stop_event) + + # An abort is not a failure and must be nowhere near the 25s timeout + assert result == [] + assert time.monotonic() - start < 5 + client.disconnect.assert_called_once_with() + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_timeout_raises() -> None: + """No answer within the timeout raises EsphomeError (default stop event path).""" + client = MagicMock() + with ( + patch("esphome.mqtt.prepare", return_value=client), + pytest.raises(EsphomeError, match="Failed to find IP via MQTT"), + ): + get_esphome_device_ip(_discovery_config(), timeout=0.25) + + client.disconnect.assert_called_once_with() + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_stop_during_connect_skips_wait() -> None: + """A stop event set while the broker connect is in flight still cleans up.""" + stop_event = threading.Event() + client = MagicMock() + + def prepare_and_stop(*args): + stop_event.set() + return client + + with patch("esphome.mqtt.prepare", side_effect=prepare_and_stop): + result = get_esphome_device_ip(_discovery_config(), stop_event=stop_event) + + assert result == [] + client.loop_start.assert_not_called() + client.disconnect.assert_called_once_with() + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_replaces_reconnect_handler( + caplog: pytest.LogCaptureFixture, +) -> None: + """The one-shot discovery client must not inherit the reconnect-forever + handler, which would make loop_stop() join the network thread forever; + its replacement still reports a broker-initiated disconnect.""" + client = MagicMock() + prepare_handler = MagicMock() + client.on_disconnect = prepare_handler + + with ( + patch("esphome.mqtt.prepare", return_value=client), + pytest.raises(EsphomeError, match="Failed to find IP via MQTT"), + ): + get_esphome_device_ip(_discovery_config(), timeout=0.25) + + assert client.on_disconnect is not prepare_handler + client.on_disconnect(client, None, 0) + assert "Disconnected from MQTT broker" not in caplog.text + client.on_disconnect(client, None, 5) + assert "Disconnected from MQTT broker (5)" in caplog.text + + +def test_get_esphome_device_ip_answer_without_ip_fails_fast( + caplog: pytest.LogCaptureFixture, +) -> None: + """A device answer with no IP fields fails promptly, not at the timeout.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, client, json.dumps({"name": "test-device"}).encode() + ) + + start = time.monotonic() + with pytest.raises(EsphomeError, match="Failed to find IP via MQTT"): + get_esphome_device_ip(_discovery_config(), timeout=5) + + assert time.monotonic() - start < 1 + assert "Device answer did not include an IP address" in caplog.text + + +@pytest.mark.parametrize("payload", [b"not json {", b"123", b"null"]) +def test_get_esphome_device_ip_unparsable_payload_ignored( + caplog: pytest.LogCaptureFixture, + payload: bytes, +) -> None: + """Garbage on the discovery topic must not kill paho's network thread.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start(mock_prepare, client, payload) + + with pytest.raises(EsphomeError, match="Failed to find IP via MQTT"): + get_esphome_device_ip(_discovery_config(), timeout=0) + + assert "Ignoring unparsable discovery payload" in caplog.text + + +def test_get_esphome_device_ip_broker_disconnect_fails_fast( + caplog: pytest.LogCaptureFixture, +) -> None: + """A broker-initiated disconnect aborts the wait instead of timing out.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client): + + def drop_connection(*args, **kwargs): + client.on_disconnect(client, None, 5) + + client.loop_start.side_effect = drop_connection + + start = time.monotonic() + with pytest.raises(EsphomeError, match="Failed to find IP via MQTT"): + get_esphome_device_ip(_discovery_config(), timeout=5) + + assert time.monotonic() - start < 1 + assert "Disconnected from MQTT broker (5)" in caplog.text + + +def test_get_esphome_device_ip_sends_discovery_ping() -> None: + """Connecting publishes the discovery ping for the device.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + + def connect_then_answer(*args, **kwargs): + on_connect = mock_prepare.call_args.args[3] + on_connect(client, None, None, 0) + msg = MagicMock() + msg.payload = json.dumps({"name": "test-device", "ip": "10.0.0.5"}).encode() + mock_prepare.call_args.args[2](client, None, msg) + + client.loop_start.side_effect = connect_then_answer + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5"] + client.publish.assert_called_once_with( + "esphome/ping/test-device", None, retain=False + ) + + +def test_get_esphome_device_ip_disconnect_error_does_not_mask_result( + caplog: pytest.LogCaptureFixture, +) -> None: + """A cleanup failure must not replace the discovery result.""" + client = MagicMock() + # First disconnect (from on_message) succeeds; the finally's fails + client.disconnect.side_effect = [None, OSError("socket already closed")] + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, + client, + json.dumps({"name": "test-device", "ip": "10.0.0.5"}).encode(), + ) + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5"] + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_invalid_address_values_skipped( + caplog: pytest.LogCaptureFixture, +) -> None: + """Non-string or non-printable ip values are skipped, valid ones kept.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, + client, + json.dumps( + { + "name": "test-device", + "ip": 1234, + "ip1": "x\n[00:00:00][I][forged] fake line", + "ip2": " 10.0.0.5 ", + } + ).encode(), + ) + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5"] + assert caplog.text.count("Ignoring invalid address in discovery answer") == 2 + assert "forged" not in "".join( + r.getMessage() for r in caplog.records if "Found IP" in r.getMessage() + ) From 8e624b4117ab7c7cec32b6303efaa9cfe50462cd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 19:14:50 -0500 Subject: [PATCH 007/160] [core] Retry framework downloads on transient network errors (#18330) --- esphome/framework_helpers.py | 259 ++++++++++++++------- tests/unit_tests/test_framework_helpers.py | 201 +++++++++++++++- 2 files changed, 373 insertions(+), 87 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 6ed608b171..86d5e4eaea 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -25,9 +25,13 @@ _LOGGER = logging.getLogger(__name__) # Attempts per mirror URL before falling through to the next mirror; only # mid-stream drops retry (resuming when the server gave a validator), -# connect errors move on immediately. +# connect errors move on to the next mirror immediately. _MIRROR_ATTEMPTS = 3 +# Passes over the whole mirror list when a transient network error is in +# the mix; matches git.py's _NETWORK_MAX_ATTEMPTS (3 tries, 2s/4s backoff). +_MIRROR_SWEEP_ATTEMPTS = 3 + def get_project_link_flags() -> list[str]: """Return the sorted -Wl, linker flags from the current build.""" @@ -887,37 +891,51 @@ def _failure_reason(e: Exception) -> str: return str(e).split(" for url: ", maxsplit=1)[0] or repr(e) -def download_from_mirrors( - mirrors: list[str], - substitutions: dict[str, str], - target: io.RawIOBase | IO[bytes] | PathType, - timeout: int = 30, -) -> str: +def _spent_attempts_error(e: Exception, attempts: int) -> Exception: + """Wrap a failure whose mirror already consumed download attempts, so + the sweep classifies it as permanent.""" + from esphome.core import EsphomeError + + err = EsphomeError(f"failed after {attempts} attempts: {_failure_reason(e)}") + err.__cause__ = e + return err + + +def _is_transient_download_error(e: Exception) -> bool: + """Return True when a download failure is worth retrying. + + Connection-level failures and HTTP 429/5xx are transient. Other HTTP + errors, local errors, and exhausted-attempts EsphomeError wrappers + (their per-mirror retries are already spent) are permanent. """ - Download file from multiple mirrors with substitution support. + # Imported lazily: requests is a heavy import (~85ms) and is only + # needed when actually downloading, never during config validation. + import requests - Args: - mirrors: list of mirror URLs - substitutions: Dictionary of substitutions to apply to URLs - target: Target file path or file-like object - timeout: Download timeout in seconds + if isinstance(e, requests.exceptions.HTTPError): + resp = e.response + return resp is not None and (resp.status_code == 429 or resp.status_code >= 500) + return isinstance( + e, + ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, + ), + ) - Returns: - The source URL. - Mirror URL templates that reference a substitution not present in - ``substitutions`` are skipped, so callers can offer templates that only - apply to some downloads. +def _try_mirrors_once( + urls: list[str], + path_target: Path | None, + f: IO[bytes] | None, + timeout: int, + failures: list[tuple[str, Exception]], +) -> str | None: + """Single pass over the resolved mirror ``urls``, one try per URL. - A path target downloads through ``download_with_resume``, so an - interrupted download resumes on the next esphome run; a file-like target - only resumes mid-stream drops within this call. - - Raises: - ValueError: If mirrors list is empty. - EsphomeError: If all download attempts fail; the message lists every - attempted URL with its individual failure reason. Also raised if - no template matched the provided substitutions. + Returns the source URL on success, or None with each URL's exception + appended to ``failures``. """ # Imported lazily: requests is a heavy import (~85ms) and is only # needed when actually downloading, never during config validation. @@ -925,43 +943,7 @@ def download_from_mirrors( from esphome.core import EsphomeError - ensure_happy_eyeballs() - - # 1. Classify the target: filesystem path or open file object - path_target: Path | None = None - f: IO[bytes] | None = None - if isinstance(target, (str, os.PathLike)): - path_target = Path(target) - elif isinstance(target, (io.RawIOBase, io.IOBase)): - f = target - else: - raise TypeError( - f"target must be str, Path, or file-like object: {type(target)}" - ) - - # 2. Try each mirror in order - failures: list[tuple[str, Exception]] = [] - skipped: list[tuple[str, str]] = [] - - for mirror in mirrors: - # 3. Apply substitutions to URL - try: - url = mirror.format(**substitutions) - except KeyError as e: - # The template references a substitution not provided for - # this download (e.g. SHORT_VERSION only exists for x.y.0 - # versions) - expected, the template just doesn't apply. - _LOGGER.debug("Skipping mirror %s: %s not available", mirror, e) - skipped.append((mirror, f"not applicable ({e.args[0]} not available)")) - continue - except (IndexError, ValueError) as e: - # A malformed template (unbalanced braces, bad format spec) - # is an authoring error, not an expected fallthrough - warn - # even if a later mirror succeeds. - _LOGGER.warning("Skipping malformed mirror URL template %s: %r", mirror, e) - skipped.append((mirror, f"skipped ({e!r})")) - continue - + for url in urls: _LOGGER.debug("Trying to download from %s", url) # Path targets delegate to download_with_resume so a partial @@ -986,14 +968,14 @@ def download_from_mirrors( failures.append((url, e)) continue - # 4. Download; mid-stream failures retry the same mirror with - # resume (see download_with_resume) instead of starting over. - # There is no checksum to verify a resumed file against, so a - # stitch is only trusted when the server proves consistency: the - # If-Range validator guarantees 206 only for unchanged content, - # and the expected total length (when the first response carried - # one) guards against short or shifted bodies. Without a - # validator the retry restarts from zero. + # File-like targets download here; mid-stream failures retry the + # same mirror with resume (see download_with_resume) instead of + # starting over. There is no checksum to verify a resumed file + # against, so a stitch is only trusted when the server proves + # consistency: the If-Range validator guarantees 206 only for + # unchanged content, and the expected total length (when the first + # response carried one) guards against short or shifted bodies. + # Without a validator the retry restarts from zero. offset = 0 expected_total = 0 validator = None @@ -1001,9 +983,12 @@ def download_from_mirrors( try: resp, offset = _open_ranged(url, offset, timeout, validator) except (requests.RequestException, OSError) as e: - # Connect/HTTP error, no bytes flowed — next mirror. + # Connect/HTTP error, no bytes flowed — next mirror. Wrap + # when earlier attempts were already spent on this mirror. _LOGGER.debug("Failed to download %s: %s", url, str(e)) - failures.append((url, e)) + failures.append( + (url, _spent_attempts_error(e, attempt + 1) if attempt else e) + ) break try: @@ -1031,7 +1016,7 @@ def download_from_mirrors( _LOGGER.debug("Downloaded successfully from: %s", url) - # 5. Reset file pointer and return + # Reset file pointer and return f.seek(0) return url @@ -1054,16 +1039,124 @@ def download_from_mirrors( ) offset = 0 if attempt == _MIRROR_ATTEMPTS - 1: - failures.append((url, e)) + failures.append((url, _spent_attempts_error(e, _MIRROR_ATTEMPTS))) - # 6. Report every attempted URL if all mirrors failed. Falling back - # past an early mirror is normal (e.g. only one of the framework URL - # templates matches a given version's tag), so raising only the last - # error would hide the failure that actually matters. - if failures: - attempts = "".join( - f"\n {url}\n {_failure_reason(e)}" for url, e in failures + return None + + +def download_from_mirrors( + mirrors: list[str], + substitutions: dict[str, str], + target: io.RawIOBase | IO[bytes] | PathType, + timeout: int = 30, +) -> str: + """ + Download file from multiple mirrors with substitution support. + + Args: + mirrors: list of mirror URLs + substitutions: Dictionary of substitutions to apply to URLs + target: Target file path or file-like object + timeout: Download timeout in seconds + + Returns: + The source URL. + + Mirror URL templates that reference a substitution not present in + ``substitutions`` are skipped, so callers can offer templates that only + apply to some downloads. + + A path target downloads through ``download_with_resume``, so an + interrupted download resumes on the next esphome run; a file-like target + only resumes mid-stream drops within this call. + + When every mirror fails and at least one failure is transient (dropped + connection, timeout, HTTP 429/5xx), the whole list is retried with a + short backoff; permanent failures (e.g. 404) raise immediately. + + Raises: + ValueError: If mirrors list is empty. + EsphomeError: If all download attempts fail; the message lists every + attempted URL with its individual failure reason. Also raised if + no template matched the provided substitutions. + """ + from esphome.core import EsphomeError + + ensure_happy_eyeballs() + + # 1. Classify the target: filesystem path or open file object + path_target: Path | None = None + f: IO[bytes] | None = None + if isinstance(target, (str, os.PathLike)): + path_target = Path(target) + elif isinstance(target, (io.RawIOBase, io.IOBase)): + f = target + else: + raise TypeError( + f"target must be str, Path, or file-like object: {type(target)}" ) + + # 2. Resolve the mirror templates (invariant across retry sweeps) + urls: list[str] = [] + skipped: list[tuple[str, str]] = [] + for mirror in mirrors: + try: + urls.append(mirror.format(**substitutions)) + except KeyError as e: + # The template references a substitution not provided for + # this download (e.g. SHORT_VERSION only exists for x.y.0 + # versions) - expected, the template just doesn't apply. + _LOGGER.debug("Skipping mirror %s: %s not available", mirror, e) + skipped.append((mirror, f"not applicable ({e.args[0]} not available)")) + except (IndexError, ValueError) as e: + # A malformed template (unbalanced braces, bad format spec) + # is an authoring error, not an expected fallthrough - warn + # even if a later mirror succeeds. + _LOGGER.warning("Skipping malformed mirror URL template %s: %r", mirror, e) + skipped.append((mirror, f"skipped ({e!r})")) + + # 3. Sweep the mirror list, retrying transient failures with backoff: + # a single pass keeps mirror failover fast, re-sweeping keeps one + # network blip from failing the build when only one mirror applies. + failures: list[tuple[str, Exception]] = [] + for sweep in range(1, _MIRROR_SWEEP_ATTEMPTS + 1): + sweep_failures: list[tuple[str, Exception]] = [] + if ( + url := _try_mirrors_once(urls, path_target, f, timeout, sweep_failures) + ) is not None: + return url + failures.extend(sweep_failures) + # Permanent failures (404, verification mismatch) won't heal; + # only retry when a transient error is in the mix (as git.py does). + transient = next( + ((u, e) for u, e in sweep_failures if _is_transient_download_error(e)), + None, + ) + if transient is None: + break + if sweep < _MIRROR_SWEEP_ATTEMPTS: + delay = 2**sweep + _LOGGER.warning( + "Download of %s failed (%s); retrying in %d seconds (attempt %d/%d)", + transient[0], + _failure_reason(transient[1]), + delay, + sweep + 1, + _MIRROR_SWEEP_ATTEMPTS, + ) + time.sleep(delay) + + # 4. Report every attempted URL if all mirrors failed. failures spans + # all sweeps (deduplicated by URL and reason), so neither an early + # mirror's failure nor an earlier sweep's failure mode is hidden. + if failures: + seen: set[tuple[str, str]] = set() + attempts = "" + for url, e in failures: + reason = _failure_reason(e) + if (url, reason) not in seen: + seen.add((url, reason)) + attempts += f"\n {url}\n {reason}" attempts += "".join(f"\n {mirror}\n {reason}" for mirror, reason in skipped) raise EsphomeError( f"Failed to download from all mirrors:{attempts}" diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 08751879c2..7451ee9b39 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -12,7 +12,7 @@ from pathlib import Path import subprocess import sys import tarfile -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, call, patch import zipfile import pytest @@ -23,6 +23,7 @@ from esphome.core import EsphomeError from esphome.framework_helpers import ( _7z_extract_all, _detect_archive_root, + _is_transient_download_error, _rename_with_retry, _tar_extract_all, _zip_extract_all, @@ -515,16 +516,23 @@ class TestArchiveExtractAll: # --------------------------------------------------------------------------- -def _mock_response(content: bytes, ok: bool = True) -> MagicMock: +def _mock_response( + content: bytes, ok: bool = True, status: int | None = None +) -> MagicMock: + """A fake requests response. The HTTPError carries the response (as + ``raise_for_status`` on a real response) so the transient classifier + can see its ``status``; failures default to a permanent 404.""" + if status is None: + status = 200 if ok else 404 r = MagicMock() r.__enter__.return_value = r r.__exit__.return_value = False - r.status_code = 200 + r.status_code = status r.ok = ok if ok: r.raise_for_status.return_value = None else: - r.raise_for_status.side_effect = req.HTTPError("503") + r.raise_for_status.side_effect = req.HTTPError(str(status), response=r) r.headers = {"content-length": "0"} # suppress ProgressBar r.iter_content.return_value = [content] if content else [] return r @@ -1419,6 +1427,191 @@ class TestDownloadFromMirrors: assert target.exists() assert target.read_bytes() == b"" + @pytest.mark.parametrize("target_kind", ["path", "file-like"]) + def test_transient_failure_retries_mirror_sweep( + self, tmp_path: Path, target_kind: str + ) -> None: + """A transient connect error on the only applicable mirror retries the + whole mirror list with backoff instead of failing the build.""" + target = tmp_path / "idf.tar.xz" if target_kind == "path" else io.BytesIO() + with ( + patch( + "requests.get", + side_effect=[ + req.ConnectionError("Remote end closed connection"), + _mock_response(b"data"), + ], + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + ): + url = download_from_mirrors(["https://mirror1.com/f"], {}, target) + assert url == "https://mirror1.com/f" + data = target.read_bytes() if target_kind == "path" else target.getvalue() + assert data == b"data" + assert mock_get.call_count == 2 + mock_sleep.assert_called_once_with(2) + + def test_permanent_failure_does_not_retry_sweep(self, tmp_path: Path) -> None: + """An HTTP 404 will not heal on its own; fail after a single pass.""" + with ( + patch( + "requests.get", return_value=_mock_response(b"", ok=False, status=404) + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="all mirrors"), + ): + download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin") + assert mock_get.call_count == 1 + mock_sleep.assert_not_called() + + def test_transient_failure_exhausts_sweeps(self, tmp_path: Path) -> None: + """A persistent transient error gives up after the configured number + of passes, with 2s/4s backoff, and still lists the attempted URL.""" + with ( + patch("requests.get", side_effect=req.ConnectionError("down")) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="all mirrors") as ei, + ): + download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin") + assert mock_get.call_count == 3 + assert mock_sleep.call_args_list == [call(2), call(4)] + assert "https://mirror1.com/f" in str(ei.value) + + def test_mixed_permanent_and_transient_retries_sweep(self, tmp_path: Path) -> None: + """One mirror 404s permanently while another hits a transient error; + the transient failure makes the whole list worth another pass.""" + dest = tmp_path / "out.bin" + with ( + patch( + "requests.get", + side_effect=[ + _mock_response(b"", ok=False, status=404), + req.ConnectionError("down"), + _mock_response(b"", ok=False, status=404), + _mock_response(b"data"), + ], + ), + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + ): + url = download_from_mirrors( + ["https://mirror1.com/f", "https://mirror2.com/f"], {}, dest + ) + assert url == "https://mirror2.com/f" + assert dest.read_bytes() == b"data" + mock_sleep.assert_called_once_with(2) + + def test_http_5xx_retries_sweep(self, tmp_path: Path) -> None: + """A real 5xx (response attached to the HTTPError) is transient.""" + dest = tmp_path / "out.bin" + with ( + patch( + "requests.get", + side_effect=[ + _mock_response(b"", ok=False, status=503), + _mock_response(b"data"), + ], + ), + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + ): + url = download_from_mirrors(["https://mirror1.com/f"], {}, dest) + assert url == "https://mirror1.com/f" + assert dest.read_bytes() == b"data" + mock_sleep.assert_called_once_with(2) + + def test_error_reports_failure_modes_from_all_sweeps(self, tmp_path: Path) -> None: + """A failure mode that changes between sweeps stays in the final + error; the first failure (the one that started the retries) is + chained as the cause.""" + with ( + patch( + "requests.get", + side_effect=[ + req.ConnectionError("dropped by middlebox"), + _mock_response(b"", ok=False, status=404), + ], + ), + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="all mirrors") as ei, + ): + download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin") + assert "dropped by middlebox" in str(ei.value) + assert "404" in str(ei.value) + assert isinstance(ei.value.__cause__, req.ConnectionError) + mock_sleep.assert_called_once_with(2) + + def test_exhausted_mid_stream_attempts_not_swept(self) -> None: + """A file-like mirror that spent all its mid-stream attempts is not + retried again at the sweep level (unlike a path target, it has no + part file to resume from on a later sweep).""" + buf = io.BytesIO() + with ( + patch( + "requests.get", + side_effect=[_interrupted_response(b"1234") for _ in range(3)], + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="failed after 3 attempts"), + ): + download_from_mirrors(["https://mirror1.com/f"], {}, buf) + assert mock_get.call_count == 3 + mock_sleep.assert_not_called() + + def test_mid_stream_drop_then_connect_error_not_swept(self) -> None: + """A connect error on a later attempt (after a mid-stream drop spent + one) also counts as spent budget and does not re-arm the sweep.""" + buf = io.BytesIO() + with ( + patch( + "requests.get", + side_effect=[ + _interrupted_response(b"1234"), + req.ConnectionError("down"), + ], + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="failed after 2 attempts"), + ): + download_from_mirrors(["https://mirror1.com/f"], {}, buf) + assert mock_get.call_count == 2 + mock_sleep.assert_not_called() + + +def _http_error(status: int) -> req.HTTPError: + """An HTTPError carrying a response with the given status, as raised by + ``raise_for_status`` on a real response.""" + resp = MagicMock() + resp.status_code = status + return req.HTTPError(str(status), response=resp) + + +class TestIsTransientDownloadError: + def test_connection_errors_are_transient(self) -> None: + assert _is_transient_download_error(req.ConnectionError("reset")) + assert _is_transient_download_error(req.Timeout("timed out")) + assert _is_transient_download_error( + req.exceptions.ChunkedEncodingError("dropped") + ) + + def test_http_statuses(self) -> None: + assert not _is_transient_download_error(_http_error(404)) + assert not _is_transient_download_error(_http_error(403)) + assert _is_transient_download_error(_http_error(429)) + assert _is_transient_download_error(_http_error(503)) + + def test_http_error_without_response_is_permanent(self) -> None: + assert not _is_transient_download_error(req.HTTPError("boom")) + + def test_exhausted_resume_attempts_are_permanent(self) -> None: + """download_with_resume already spent its own resume attempts; its + EsphomeError wrapper is not retried again at the sweep level.""" + wrapped = EsphomeError("Failed to download after 3 attempts") + wrapped.__cause__ = req.ConnectionError("down") + assert not _is_transient_download_error(wrapped) + + def test_unrelated_errors_are_permanent(self) -> None: + assert not _is_transient_download_error(OSError("disk full")) + assert not _is_transient_download_error(EsphomeError("size mismatch")) + def test_importing_framework_helpers_does_not_import_requests() -> None: """Importing framework_helpers must not drag in requests. From 7569a7b5ced61c4a2826fac165e64fceee1baccd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 09:59:24 -0500 Subject: [PATCH 008/160] [usb_uart] Fix uint32_t format specifier warning in pl2303 (#18310) --- esphome/components/usb_uart/pl2303.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/usb_uart/pl2303.cpp b/esphome/components/usb_uart/pl2303.cpp index 3c7ecd9a83..c56f43f75a 100644 --- a/esphome/components/usb_uart/pl2303.cpp +++ b/esphome/components/usb_uart/pl2303.cpp @@ -292,8 +292,8 @@ bool USBUartTypePL2303::config_step(USBUartChannel *channel, uint8_t step, bool // Data bits line_coding[6] = channel->get_data_bits(); - ESP_LOGD(TAG, "PL2303: SET_LINE_REQUEST baud=%u stop=%u parity=%u data=%u", baud, line_coding[4], line_coding[5], - line_coding[6]); + ESP_LOGD(TAG, "PL2303: SET_LINE_REQUEST baud=%" PRIu32 " stop=%u parity=%u data=%u", baud, line_coding[4], + line_coding[5], line_coding[6]); std::vector lc_vec(line_coding, line_coding + 7); this->config_transfer_(SET_LINE_REQUEST_TYPE, SET_LINE_REQUEST, 0, iface, lc_vec); From 89489b1f0dabee05c18ce8327386dd4a660d79a5 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:33:45 +0000 Subject: [PATCH 009/160] Bump aioesphomeapi from 45.10.0 to 45.10.1 (#18318) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9c231bd0fe..85a0f55263 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.10.0 +aioesphomeapi==45.10.1 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 83cff59fddca496cdb60d66d1ea8525c62c620f6 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Wed, 12 Aug 2026 12:49:54 -0400 Subject: [PATCH 010/160] [sendspin] Bump sendspin-cpp to v0.7.2 (#18316) --- esphome/components/sendspin/__init__.py | 2 +- esphome/idf_component.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index bd889c2c92..082639374f 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -234,7 +234,7 @@ async def to_code(config: ConfigType) -> None: psram.request_external_task_stack() # sendspin-cpp library - esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.7.1") + esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.7.2") cg.add_define("USE_SENDSPIN", True) # for MDNS diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 6a9d7171ec..aff1a6819f 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -98,7 +98,7 @@ dependencies: esp32async/asynctcp: version: 3.4.91 sendspin/sendspin-cpp: - version: 0.7.1 + version: 0.7.2 lvgl/lvgl: version: 9.5.0 fastled/FastLED: From 48d6368ff9e4b3ba18b663eb84f686e5ee84065e Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:27:14 -0500 Subject: [PATCH 011/160] Bump bundled esphome-device-builder to 1.9.6 (#18328) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a4f5d3c3a6..d7ae2cd4ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.9.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.9.6 RUN \ platformio settings set enable_telemetry No \ From a14ea0e8fabce6ad71a3386dd5fa1ecb9edb1166 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 18:49:34 -0500 Subject: [PATCH 012/160] [ld2420] Fix out-of-bounds read when device reports unknown command error (#18322) --- esphome/components/ld2420/ld2420.cpp | 9 ++++++++- esphome/components/ld2420/ld2420.h | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/esphome/components/ld2420/ld2420.cpp b/esphome/components/ld2420/ld2420.cpp index ae622cda28..f71bec7e5f 100644 --- a/esphome/components/ld2420/ld2420.cpp +++ b/esphome/components/ld2420/ld2420.cpp @@ -746,7 +746,14 @@ void LD2420Component::set_reg_value(uint16_t reg, uint16_t value) { this->send_cmd_from_array(cmd_frame); } -void LD2420Component::handle_cmd_error(uint8_t error) { ESP_LOGE(TAG, "Command failed: %s", ERR_MESSAGE[error]); } +void LD2420Component::handle_cmd_error(uint16_t error) { + if (error < std::size(ERR_MESSAGE)) { + ESP_LOGE(TAG, "Command failed: %s", ERR_MESSAGE[error]); + } else { + // The error word comes from the device reply frame; unknown codes must not index ERR_MESSAGE + ESP_LOGE(TAG, "Command failed: error 0x%04X", error); + } +} int LD2420Component::get_gate_threshold_(uint8_t gate) { uint8_t error; diff --git a/esphome/components/ld2420/ld2420.h b/esphome/components/ld2420/ld2420.h index ae44b16065..977ee2eccc 100644 --- a/esphome/components/ld2420/ld2420.h +++ b/esphome/components/ld2420/ld2420.h @@ -108,7 +108,7 @@ class LD2420Component final : public Component, public uart::UARTDevice { float get_setup_priority() const override; int send_cmd_from_array(CmdFrameT cmd_frame); void report_gate_data(); - void handle_cmd_error(uint8_t error); + void handle_cmd_error(uint16_t error); void set_operating_mode(const char *state); void auto_calibrate_sensitivity(); void update_radar_data(uint16_t const *gate_energy, uint8_t sample_number); From c1a326f32e710d26e0671c56b1a74e85a37a5822 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 19:11:32 -0500 Subject: [PATCH 013/160] [core] Don't block logs startup on MQTT IP discovery when addresses are known (#18313) --- esphome/__main__.py | 144 +++++++++---- esphome/api_client.py | 87 +++++++- esphome/mqtt.py | 91 ++++++-- tests/unit_tests/test_api_client.py | 323 +++++++++++++++++++++++++++- tests/unit_tests/test_main.py | 171 +++++++++++---- tests/unit_tests/test_mqtt.py | 262 ++++++++++++++++++++++ 6 files changed, 982 insertions(+), 96 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 0ac5898268..1262a4525e 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -10,7 +10,7 @@ from pathlib import Path import re import sys import time -from typing import Protocol +from typing import TYPE_CHECKING, Protocol # Note: Do not import modules from esphome.components here, as this would # cause them to be loaded before external components are processed, resulting @@ -71,6 +71,9 @@ from esphome.util import ( safe_print, ) +if TYPE_CHECKING: + import threading + # Keep expensive imports (zeroconf, writer, yaml_util, etc.) out of this # module's top level. Every `esphome` invocation — including fast paths # like `esphome version` — pays the cost of what's imported here before @@ -567,11 +570,48 @@ def has_name_add_mac_suffix() -> bool: def mqtt_get_ip( - config: ConfigType, username: str, password: str, client_id: str + config: ConfigType, + username: str, + password: str, + client_id: str, + stop_event: "threading.Event | None" = None, ) -> list[str]: from esphome import mqtt - return mqtt.get_esphome_device_ip(config, username, password, client_id) + return mqtt.get_esphome_device_ip( + config, username, password, client_id, stop_event=stop_event + ) + + +def _add_network_device(device: str, network_devices: list[str]) -> None: + """Append a device to the list, expanding it through ``CORE.address_cache``. + + If the hostname is already in the address cache (e.g. populated by mDNS + discovery), substitute the cached IPs so aioesphomeapi doesn't open its + own Zeroconf to re-resolve it. Duplicates are dropped. + """ + if CORE.address_cache and (cached := CORE.address_cache.get_addresses(device)): + network_devices.extend(addr for addr in cached if addr not in network_devices) + elif device not in network_devices: + network_devices.append(device) + + +def _split_network_devices(devices: list[str]) -> tuple[list[str], bool]: + """Split the device list into direct addresses and an MQTT-lookup flag. + + Direct addresses are expanded through ``CORE.address_cache`` and deduped + the same way ``_resolve_network_devices`` does; MQTT/MQTTIP magic strings + are not resolved, only reported via the returned bool so the caller can + defer the broker lookup. + """ + network_devices: list[str] = [] + has_mqtt_lookup = False + for device in devices: + if get_port_type(device) in _MQTT_PORT_TYPES: + has_mqtt_lookup = True + else: + _add_network_device(device, network_devices) + return network_devices, has_mqtt_lookup def _resolve_network_devices( @@ -604,40 +644,44 @@ def _resolve_network_devices( if port_type in _MQTT_PORT_TYPES: # Only resolve MQTT once, even if multiple MQTT entries if not mqtt_resolved: - try: - mqtt_ips = mqtt_get_ip( - config, args.username, args.password, args.client_id - ) - # pylint can't infer mqtt_get_ip's return through its - # lazy ``from esphome import mqtt`` import, so it flags - # the genexpr below. - network_devices.extend( - addr - for addr in mqtt_ips # pylint: disable=not-an-iterable - if addr not in network_devices - ) - except EsphomeError as err: - _LOGGER.warning( - "MQTT IP discovery failed (%s), will try other devices if available", - err, - ) + mqtt_ips = _mqtt_get_ip_or_warn( + config, args.username, args.password, args.client_id + ) + network_devices.extend( + addr for addr in mqtt_ips if addr not in network_devices + ) mqtt_resolved = True continue - # If the hostname is already in the address cache (e.g. populated by - # mDNS discovery), substitute the cached IPs so aioesphomeapi doesn't - # open its own Zeroconf to re-resolve it. - if CORE.address_cache and (cached := CORE.address_cache.get_addresses(device)): - network_devices.extend( - addr for addr in cached if addr not in network_devices - ) - elif device not in network_devices: - # Regular network address or IP - add if not already present - network_devices.append(device) + _add_network_device(device, network_devices) return network_devices +def _mqtt_get_ip_or_warn( + config: ConfigType, + username: str, + password: str, + client_id: str, + stop_event: "threading.Event | None" = None, +) -> list[str]: + """Look up the device IP via MQTT, returning [] with a warning on failure. + + This owns the failure policy for MQTT IP discovery on paths that have + other addresses to fall back on: a broker problem must not abort the + operation. Also used as the deferred resolver handed to ``run_logs``, + where it runs in a worker thread. + """ + try: + return mqtt_get_ip(config, username, password, client_id, stop_event=stop_event) + except EsphomeError as err: + _LOGGER.warning( + "MQTT IP discovery failed (%s), will try other devices if available", + err, + ) + return [] + + def run_miniterm(config: ConfigType, port: str, args) -> int: from datetime import datetime @@ -1438,17 +1482,37 @@ def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int return run_miniterm(config, port, args) # Check if we should use API for logging - # Resolve MQTT magic strings to actual IP addresses - if has_api() and ( - network_devices := _resolve_network_devices(devices, config, args) - ): - from esphome.api_client import run_logs + if has_api(): + network_devices, has_mqtt_lookup = _split_network_devices(devices) + mqtt_resolver = None + if has_mqtt_lookup: + if network_devices: + # Addresses are already known, so don't block startup on the + # MQTT broker lookup; hand it to run_logs as a deferred + # resolver that runs in the background and feeds discovered + # addresses into the running log client, keeping MQTT as a + # fallback for when the known addresses are stale (e.g. DHCP + # reassigned the IP). + mqtt_resolver = functools.partial( + _mqtt_get_ip_or_warn, + config, + args.username, + args.password, + args.client_id, + ) + else: + # The MQTT lookup is the only way to find the device; resolve + # it up front since the client needs an address to start with. + network_devices = _resolve_network_devices(devices, config, args) + if network_devices: + from esphome.api_client import run_logs - return run_logs( - config, - network_devices, - subscribe_states=_should_subscribe_states(args), - ) + return run_logs( + config, + network_devices, + subscribe_states=_should_subscribe_states(args), + mqtt_resolver=mqtt_resolver, + ) if port_type in (PortType.NETWORK, PortType.MQTT) and has_mqtt_logging(): from esphome import mqtt diff --git a/esphome/api_client.py b/esphome/api_client.py index a75f219b17..fb41075de8 100644 --- a/esphome/api_client.py +++ b/esphome/api_client.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio from contextlib import suppress import logging +import threading from typing import TYPE_CHECKING, Any import warnings @@ -20,6 +21,8 @@ from esphome.stacktrace import LogLineProcessor from esphome.util import safe_print if TYPE_CHECKING: + from collections.abc import Callable + from aioesphomeapi.api_pb2 import ( SubscribeLogsResponse, # pylint: disable=no-name-in-module ) @@ -32,8 +35,18 @@ async def async_run_logs( config: dict[str, Any], addresses: list[str], subscribe_states: bool = True, + mqtt_resolver: Callable[[threading.Event], list[str]] | None = None, ) -> None: - """Run the logs command in the event loop.""" + """Run the logs command in the event loop. + + If ``mqtt_resolver`` is given, it is called in a worker thread (paho-mqtt + has no asyncio support on Windows) concurrently with the connection + attempts to ``addresses``, and any addresses it discovers are fed into + the running client. It owns its own failure handling (returning [] when + discovery fails) and must honor the ``threading.Event`` it is passed so + teardown is not delayed by the lookup's wait window; the initial broker + connect itself is only bounded by the socket timeout. + """ from datetime import datetime conf = config["api"] @@ -60,6 +73,41 @@ async def async_run_logs( # Decoder resolution policy lives in LogLineProcessor. processor = LogLineProcessor(config, CORE.target_platform) + mqtt_task: asyncio.Task[None] | None = None + mqtt_stop_event = threading.Event() + + def _cancel_mqtt_discovery() -> None: + """Stop the broker lookup once a connection has been established. + + Its answer is only useful while still disconnected: after that it + either duplicates the connected address or arrives too late to + matter, so don't keep an idle broker session open for it. + """ + mqtt_stop_event.set() + if mqtt_task is not None and not mqtt_task.done(): + mqtt_task.cancel() + + async def _resolve_mqtt_addresses() -> None: + """Discover the device address via the MQTT broker in the background.""" + try: + mqtt_ips = await asyncio.to_thread(mqtt_resolver, mqtt_stop_event) + if not mqtt_ips: + _LOGGER.debug( + "MQTT discovery %s", + "aborted" if mqtt_stop_event.is_set() else "found no addresses", + ) + return + if cli.add_addresses(mqtt_ips): + _LOGGER.info("Discovered address(es) via MQTT: %s", ", ".join(mqtt_ips)) + else: + _LOGGER.debug( + "MQTT-discovered address(es) already known: %s", ", ".join(mqtt_ips) + ) + except Exception: # pylint: disable=broad-except + # A background task failure would otherwise stay invisible for + # the whole session and only re-raise at teardown + _LOGGER.exception("MQTT address discovery failed") + def on_log(msg: SubscribeLogsResponse) -> None: """Handle a new log message.""" time_ = datetime.now().astimezone() @@ -98,20 +146,53 @@ async def async_run_logs( # A top-level ``deep_sleep:`` block means the device is only awake # briefly; cap the reconnect backoff so a wake window is not missed. deep_sleep="deep_sleep" in config, + on_connect=_cancel_mqtt_discovery if mqtt_resolver is not None else None, ) try: + # Don't start (or keep) the broker lookup if a connection already + # succeeded; the stop event doubles as the not-needed-anymore latch + # and get_esphome_device_ip returns immediately when it is set. + if mqtt_resolver is not None and not mqtt_stop_event.is_set(): + mqtt_task = asyncio.create_task(_resolve_mqtt_addresses()) await asyncio.Event().wait() finally: - await stop() + try: + if mqtt_task is not None: + # Unblock the worker thread first so it can't hold up + # loop.shutdown_default_executor() for the full lookup timeout. + mqtt_stop_event.set() + # Give the worker a moment to exit through its own error + # handling; cancelling first would race out a late failure. + done, _ = await asyncio.wait([mqtt_task], timeout=1.0) + if not done: + mqtt_task.cancel() + # return_exceptions keeps a CancelledError from the cancel() + # above from re-raising here and jumping over the stop() below. + # The task handles Exception itself, so only a BaseException + # escape (e.g. SystemExit from the worker) can land here. + (result,) = await asyncio.gather(mqtt_task, return_exceptions=True) + if isinstance(result, BaseException) and not isinstance( + result, asyncio.CancelledError + ): + _LOGGER.error("MQTT address discovery failed", exc_info=result) + finally: + # Must run even if a second cancellation lands mid-cleanup above + await stop() def run_logs( config: dict[str, Any], addresses: list[str], subscribe_states: bool = True, + mqtt_resolver: Callable[[threading.Event], list[str]] | None = None, ) -> None: """Run the logs command.""" with suppress(KeyboardInterrupt): asyncio.run( - async_run_logs(config, addresses, subscribe_states=subscribe_states) + async_run_logs( + config, + addresses, + subscribe_states=subscribe_states, + mqtt_resolver=mqtt_resolver, + ) ) diff --git a/esphome/mqtt.py b/esphome/mqtt.py index 3198de9d21..62deafb09a 100644 --- a/esphome/mqtt.py +++ b/esphome/mqtt.py @@ -6,6 +6,7 @@ from pathlib import Path import ssl import tempfile import time +from typing import TYPE_CHECKING import paho.mqtt.client as mqtt @@ -31,6 +32,9 @@ from esphome.helpers import get_int_env, get_str_env from esphome.types import ConfigType from esphome.util import safe_print +if TYPE_CHECKING: + import threading + _LOGGER = logging.getLogger(__name__) @@ -164,6 +168,7 @@ def get_esphome_device_ip( password: str | None = None, client_id: str | None = None, timeout: float = 25, + stop_event: "threading.Event | None" = None, ) -> list[str]: if CONF_MQTT not in config: raise EsphomeError( @@ -182,55 +187,113 @@ def get_esphome_device_ip( dev_name = config[CONF_ESPHOME][CONF_NAME] dev_ip = None + failed = False topic = "esphome/discover/" + dev_name _LOGGER.info("Starting looking for IP in topic %s", topic) def on_message(client, userdata, msg): - nonlocal dev_ip + nonlocal dev_ip, failed time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") if len(payload) > 0: message = time_ + " " + payload _LOGGER.debug(message) - data = json.loads(payload) + try: + data = json.loads(payload) + except ValueError: + data = None + if not isinstance(data, dict): + # A raise in this handler would kill paho's network thread + _LOGGER.warning("Ignoring unparsable discovery payload") + return if "name" not in data or data["name"] != dev_name: _LOGGER.warning("Wrong device answer") return - dev_ip = [] + addresses = [] key = "ip" n = 0 while key in data: - dev_ip.append(data[key]) + value = data[key] + if ( + isinstance(value, str) + and (value := value.strip()) + and value.isprintable() + ): + addresses.append(value) + else: + # repr-escaped and truncated: must not forge log lines + _LOGGER.warning( + "Ignoring invalid address in discovery answer: %s", + repr(value)[:100], + ) n = n + 1 key = "ip" + str(n) - if dev_ip: - client.disconnect() + if not addresses: + _LOGGER.warning("Device answer did not include an IP address") + failed = True + return + + dev_ip = addresses + failed = False # a complete answer wins over an earlier empty one + client.disconnect() def on_connect(client, userdata, flags, return_code): topic = "esphome/ping/" + dev_name _LOGGER.info("Send discover via MQTT broker topic: %s", topic) client.publish(topic, None, retain=False) + if stop_event is not None and stop_event.is_set(): + # Teardown already started; don't open a broker connection at all + return [] + + def on_disconnect(client, userdata, result_code): + nonlocal failed + if result_code != 0: + _LOGGER.warning("Disconnected from MQTT broker (%s)", result_code) + failed = True + mqtt_client = prepare( config, [topic], on_message, on_connect, username, password, client_id ) + # Discovery is one-shot; prepare()'s reconnect-forever on_disconnect runs + # on the network thread and would make loop_stop() below join forever. + mqtt_client.on_disconnect = on_disconnect - mqtt_client.loop_start() - while timeout > 0: - if dev_ip is not None: - break - timeout -= 0.250 - time.sleep(0.250) - mqtt_client.loop_stop() + if stop_event is None: + import threading + + stop_event = threading.Event() # never set; wait() below is a plain sleep + stopped = stop_event.is_set() # teardown may have started during connect + try: + if not stopped: + mqtt_client.loop_start() + while timeout > 0: + if dev_ip is not None or failed: + break + if stop_event.wait(0.250): + stopped = True + break + timeout -= 0.250 + finally: + # A cleanup failure must not replace the discovery result or its + # EsphomeError; a second disconnect after on_message's is harmless. + try: + mqtt_client.disconnect() + except Exception: # pylint: disable=broad-except + _LOGGER.debug("Error disconnecting from MQTT broker", exc_info=True) + mqtt_client.loop_stop() # only signals and joins; does not raise if dev_ip is None: + if stopped: + # Aborted by the caller, not a failure; stay quiet + return [] raise EsphomeError("Failed to find IP via MQTT") - _LOGGER.info("Found IP: %s", dev_ip) + _LOGGER.info("Found IP via MQTT broker: %s", ", ".join(dev_ip)) return dev_ip diff --git a/tests/unit_tests/test_api_client.py b/tests/unit_tests/test_api_client.py index 19ed83abe1..405567d84f 100644 --- a/tests/unit_tests/test_api_client.py +++ b/tests/unit_tests/test_api_client.py @@ -56,7 +56,7 @@ async def test_async_run_logs_full_flow(caplog) -> None: with ( patch.object(api_client, "async_run", mock_run), - patch.object(api_client, "APIClient") as mock_client, + patch.object(api_client, "APIClient", autospec=True) as mock_client, patch.object(api_client, "safe_print", printed.append), ): task = asyncio.get_running_loop().create_task( @@ -163,3 +163,324 @@ async def test_async_run_logs_passes_deep_sleep( await api_client.async_run_logs(config, ["1.2.3.4"]) assert mock_run.call_args.kwargs["deep_sleep"] is expected_deep_sleep + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_feeds_addresses(caplog) -> None: + """Addresses discovered via MQTT are fed into the running client.""" + caplog.set_level("INFO", logger="esphome.api_client") + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + fed = asyncio.Event() + + def resolver(stop_event): + return ["10.0.0.9", "10.0.0.10"] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + mock_client.return_value.add_addresses.side_effect = lambda addrs: ( + fed.set() or True + ) + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + async with asyncio.timeout(1): + await fed.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_called_once_with( + ["10.0.0.9", "10.0.0.10"] + ) + assert "Discovered address(es) via MQTT" in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_no_addresses_keeps_running() -> None: + """A resolver returning nothing (failed lookup) leaves the session running.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + + def resolver(stop_event): + # The resolver owns failure handling; a failed lookup returns [] + resolver_ran.set() + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + await asyncio.sleep(0) + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_not_called() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_stopped_on_teardown() -> None: + """Teardown sets the resolver's stop event so the thread exits promptly.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + captured_event: threading.Event | None = None + resolver_started = threading.Event() + + def resolver(stop_event): + nonlocal captured_event + captured_event = stop_event + resolver_started.set() + # Simulate a slow broker lookup that only ends via the stop event. + stop_event.wait(timeout=5) + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_started.wait, 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert captured_event is not None + assert captured_event.is_set() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_crash_still_stops_cleanly(caplog) -> None: + """A resolver raising unexpectedly must not skip stop() at teardown.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + + def resolver(stop_event): + resolver_ran.set() + raise RuntimeError("resolver blew up") + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + await asyncio.sleep(0.05) + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert "MQTT address discovery failed" in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_connect_cancels_mqtt_discovery() -> None: + """A successful connection stops the in-flight broker lookup.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + captured_event: threading.Event | None = None + resolver_started = threading.Event() + + def resolver(stop_event): + nonlocal captured_event + captured_event = stop_event + resolver_started.set() + stop_event.wait(timeout=5) + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)) as mock_run, + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_started.wait, 1) + + # The runner reports a successful connection + on_connect = mock_run.call_args.kwargs["on_connect"] + on_connect() + await asyncio.sleep(0.05) + + assert captured_event is not None + assert captured_event.is_set() + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_not_called() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_connect_before_discovery_skips_lookup() -> None: + """A connection during async_run startup prevents the lookup from starting.""" + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver = Mock(name="resolver") + + async def fake_async_run(*args, **kwargs): + # Connection succeeds before async_run even returns + kwargs["on_connect"]() + return stop + + with ( + patch.object(api_client, "async_run", AsyncMock(side_effect=fake_async_run)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.sleep(0.05) + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + resolver.assert_not_called() + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_mqtt_resolver_duplicate_addresses_logged(caplog) -> None: + """A discovery the client rejects as already known leaves a debug trace.""" + import threading + + caplog.set_level("DEBUG", logger="esphome.api_client") + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + fed = threading.Event() + + def resolver(stop_event): + return ["1.2.3.4"] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True) as mock_client, + ): + mock_client.return_value.add_addresses.side_effect = lambda addrs: ( + fed.set() or False + ) + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(fed.wait, 1) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_client.return_value.add_addresses.assert_called_once_with(["1.2.3.4"]) + assert "MQTT-discovered address(es) already known: 1.2.3.4" in caplog.text + assert "Discovered address(es) via MQTT" not in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_base_exception_escape_logged_at_teardown(caplog) -> None: + """A BaseException escaping the worker is reported, and stop() still runs.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + + class WorkerEscape(BaseException): + """Not an Exception, so the task-level guard must not catch it.""" + + def resolver(stop_event): + resolver_ran.set() + raise WorkerEscape("worker bailed") + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert "MQTT address discovery failed" in caplog.text + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_stubborn_worker_cancelled_at_teardown() -> None: + """A worker that ignores the stop event is cancelled after the grace period.""" + import threading + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + resolver_ran = threading.Event() + release = threading.Event() + + def resolver(stop_event): + resolver_ran.set() + # Ignore stop_event entirely; only the test releases us + release.wait(timeout=10) + return [] + + with ( + patch.object(api_client, "async_run", AsyncMock(return_value=stop)), + patch.object(api_client, "APIClient", autospec=True), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"], mqtt_resolver=resolver) + ) + await asyncio.to_thread(resolver_ran.wait, 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + release.set() + + stop.assert_awaited_once() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 23bfdbcd69..a40341e194 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -25,6 +25,7 @@ from esphome.__main__ import ( _make_crystal_freq_callback, _redact_with_legacy_fallback, _resolve_network_devices, + _split_network_devices, _unresolved_default_error, _validate_bootloader_binary, _validate_partition_table_binary, @@ -2879,7 +2880,9 @@ def test_upload_program_ota_with_mqtt_resolution( assert exit_code == 0 assert host == "192.168.1.100" - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) expected_firmware = ( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) @@ -2926,7 +2929,9 @@ def test_upload_program_ota_with_mqtt_empty_broker( assert exit_code == 0 assert host == "192.168.1.50" # Verify MQTT was attempted but failed gracefully - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify we fell back to the IP address expected_firmware = ( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" @@ -3015,7 +3020,10 @@ def test_show_logs_api( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100", "192.168.1.101"], subscribe_states=True + CORE.config, + ["192.168.1.100", "192.168.1.101"], + subscribe_states=True, + mqtt_resolver=None, ) @@ -3042,7 +3050,7 @@ def test_show_logs_api_no_states( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=False + CORE.config, ["192.168.1.100"], subscribe_states=False, mqtt_resolver=None ) @@ -3069,7 +3077,7 @@ def test_show_logs_api_with_fqdn_mdns_disabled( assert result == 0 # Should use the FQDN directly, not try MQTT lookup mock_run_logs.assert_called_once_with( - CORE.config, ["device.example.com"], subscribe_states=True + CORE.config, ["device.example.com"], subscribe_states=True, mqtt_resolver=None ) @@ -3097,9 +3105,44 @@ def test_show_logs_api_with_mqtt_fallback( result = show_logs(CORE.config, args, devices) assert result == 0 - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None + ) mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.200"], subscribe_states=True + CORE.config, ["192.168.1.200"], subscribe_states=True, mqtt_resolver=None + ) + + +@patch("esphome.mqtt.show_logs") +def test_show_logs_api_mqtt_only_resolve_failure_falls_back_to_mqtt_logs( + mock_mqtt_show_logs: Mock, + mock_mqtt_get_ip: Mock, +) -> None: + """With no addresses at all after a failed MQTT lookup, MQTT logging is used.""" + setup_core( + config={ + "logger": {}, + CONF_API: {}, + CONF_MQTT: {CONF_BROKER: "mqtt.local"}, + }, + platform=PLATFORM_ESP32, + ) + mock_mqtt_show_logs.return_value = 0 + mock_mqtt_get_ip.side_effect = EsphomeError("Failed to find IP via MQTT") + + args = MockArgs( + topic="esphome/logs", username="user", password="pass", client_id="client" + ) + devices = ["MQTT", "MQTTIP"] + + result = show_logs(CORE.config, args, devices) + + assert result == 0 + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None + ) + mock_mqtt_show_logs.assert_called_once_with( + CORE.config, "esphome/logs", "user", "pass", "client" ) @@ -3466,7 +3509,9 @@ def test_mqtt_get_ip() -> None: result = mqtt_get_ip(config, "user", "pass", "client-id") assert result == ["192.168.1.100", "192.168.1.101"] - mock_get_ip.assert_called_once_with(config, "user", "pass", "client-id") + mock_get_ip.assert_called_once_with( + config, "user", "pass", "client-id", stop_event=None + ) def test_has_resolvable_address() -> None: @@ -3847,6 +3892,37 @@ def test_resolve_network_devices_keeps_uncached_hosts(tmp_path: Path) -> None: assert result == ["unknown.local", "192.168.1.50"] +def test_split_network_devices_direct_only(tmp_path: Path) -> None: + """Direct addresses pass through deduped, with no MQTT flag.""" + setup_core(tmp_path=tmp_path) + + assert _split_network_devices(["192.168.1.50", "device.local", "192.168.1.50"]) == ( + ["192.168.1.50", "device.local"], + False, + ) + + +def test_split_network_devices_mqtt_only(tmp_path: Path) -> None: + """MQTT magic strings produce no direct addresses, only the flag.""" + setup_core(tmp_path=tmp_path) + + assert _split_network_devices(["MQTTIP", "MQTT"]) == ([], True) + + +def test_split_network_devices_expands_cached_mdns_hosts(tmp_path: Path) -> None: + """Hostnames in ``CORE.address_cache`` are expanded like _resolve_network_devices.""" + setup_core(tmp_path=tmp_path) + CORE.address_cache = AddressCache( + mdns_cache={ + "device-abc123.local": ["10.0.0.1", "10.0.0.2"], + } + ) + + assert _split_network_devices( + ["device-abc123.local", "MQTTIP", "192.168.1.50", "device-abc123.local"] + ) == (["10.0.0.1", "10.0.0.2", "192.168.1.50"], True) + + def test_await_discovery_timeout_returns_empty( caplog: pytest.LogCaptureFixture, ) -> None: @@ -5022,7 +5098,9 @@ def test_upload_program_ota_static_ip_with_mqttip( assert host == "192.168.1.100" # Verify MQTT was resolved - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with both IPs expected_firmware = ( @@ -5069,7 +5147,9 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( assert host == "192.168.2.50" # Verify MQTT was only resolved once despite multiple MQTT magic strings - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with all unique IPs expected_firmware = ( @@ -5116,7 +5196,9 @@ def test_upload_program_ota_mqttip_deduplication( assert host == "192.168.1.100" # Verify MQTT was resolved - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with deduplicated IPs (only one instance of 192.168.1.100) # Note: Current implementation doesn't dedupe, so we'll get the IP twice @@ -5136,7 +5218,9 @@ def test_show_logs_api_static_ip_with_mqttip( This tests the scenario where a device has manual_ip (static IP) configured and MQTT is also configured. The devices list contains both the static IP - and "MQTTIP" magic string. + and "MQTTIP" magic string. The MQTT lookup must not block startup; it is + handed to run_logs as a deferred resolver instead (issue #18311), while + still being reachable as a fallback for a stale static IP. """ setup_core( config={ @@ -5157,12 +5241,19 @@ def test_show_logs_api_static_ip_with_mqttip( assert result == 0 - # Verify MQTT was resolved - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") + # The broker must not be contacted before run_logs starts + mock_mqtt_get_ip.assert_not_called() - # Verify run_logs was called with both IPs - mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100", "192.168.2.50"], subscribe_states=True + # run_logs gets the static IP immediately plus a deferred MQTT resolver + mock_run_logs.assert_called_once() + assert mock_run_logs.call_args.args == (CORE.config, ["192.168.1.100"]) + assert mock_run_logs.call_args.kwargs["subscribe_states"] is True + resolver = mock_run_logs.call_args.kwargs["mqtt_resolver"] + + # Invoking the resolver performs the MQTT lookup (the #11260 fallback) + assert resolver(None) == ["192.168.2.50"] + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None ) @@ -5171,7 +5262,7 @@ def test_show_logs_api_multiple_mqttip_resolves_once( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, ) -> None: - """Test that MQTT resolution only happens once for show_logs with multiple MQTT magic strings.""" + """Test that multiple MQTT magic strings collapse into one deferred resolver.""" setup_core( config={ "logger": {}, @@ -5191,16 +5282,16 @@ def test_show_logs_api_multiple_mqttip_resolves_once( assert result == 0 - # Verify MQTT was only resolved once despite multiple MQTT magic strings - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") + # Note: "MQTT" is a different magic string from "MQTTIP", but both defer + # to the same single resolver; the broker is not contacted eagerly + mock_mqtt_get_ip.assert_not_called() + mock_run_logs.assert_called_once() + assert mock_run_logs.call_args.args == (CORE.config, ["192.168.1.100"]) - # Verify run_logs was called with all unique IPs (MQTT strings replaced with IPs) - # Note: "MQTT" is a different magic string from "MQTTIP", but both trigger MQTT resolution - # The _resolve_network_devices helper filters out both after first resolution - mock_run_logs.assert_called_once_with( - CORE.config, - ["192.168.2.50", "192.168.2.51", "192.168.1.100"], - subscribe_states=True, + resolver = mock_run_logs.call_args.kwargs["mqtt_resolver"] + assert resolver(None) == ["192.168.2.50", "192.168.2.51"] + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None ) @@ -5238,7 +5329,9 @@ def test_upload_program_ota_mqtt_timeout_fallback( assert host == "192.168.1.100" # Verify MQTT was attempted - mock_mqtt_get_ip.assert_called_once_with(config, "user", "pass", "client") + mock_mqtt_get_ip.assert_called_once_with( + config, "user", "pass", "client", stop_event=None + ) # Verify espota2.run_ota was called with only the static IP (MQTT failed) expected_firmware = ( @@ -5254,7 +5347,7 @@ def test_show_logs_api_mqtt_timeout_fallback( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, ) -> None: - """Test show_logs falls back to other devices when MQTT times out.""" + """Test show_logs proceeds with the static IP when MQTT times out.""" setup_core( config={ "logger": {}, @@ -5273,15 +5366,17 @@ def test_show_logs_api_mqtt_timeout_fallback( result = show_logs(CORE.config, args, devices) - # Should succeed using the static IP even though MQTT failed + # Logs start on the static IP without waiting for the broker assert result == 0 + mock_run_logs.assert_called_once() + assert mock_run_logs.call_args.args == (CORE.config, ["192.168.1.100"]) - # Verify MQTT was attempted - mock_mqtt_get_ip.assert_called_once_with(CORE.config, "user", "pass", "client") - - # Verify run_logs was called with only the static IP (MQTT failed) - mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=True + # The deferred resolver owns the failure policy: it logs a warning and + # returns no addresses so the session keeps running on the known ones + resolver = mock_run_logs.call_args.kwargs["mqtt_resolver"] + assert resolver(None) == [] + mock_mqtt_get_ip.assert_called_once_with( + CORE.config, "user", "pass", "client", stop_event=None ) @@ -6764,7 +6859,7 @@ def test_command_run_passes_no_states_to_show_logs( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=False + CORE.config, ["192.168.1.100"], subscribe_states=False, mqtt_resolver=None ) @@ -6805,7 +6900,7 @@ def test_command_run_defaults_subscribe_states_true( assert result == 0 mock_run_logs.assert_called_once_with( - CORE.config, ["192.168.1.100"], subscribe_states=True + CORE.config, ["192.168.1.100"], subscribe_states=True, mqtt_resolver=None ) diff --git a/tests/unit_tests/test_mqtt.py b/tests/unit_tests/test_mqtt.py index 4c2c34dff1..1ae10d0eb5 100644 --- a/tests/unit_tests/test_mqtt.py +++ b/tests/unit_tests/test_mqtt.py @@ -2,6 +2,11 @@ from __future__ import annotations +import json +import threading +import time +from unittest.mock import MagicMock, patch + import pytest from esphome.const import CONF_BROKER, CONF_ESPHOME, CONF_MQTT, CONF_NAME @@ -89,3 +94,260 @@ def test_get_esphome_device_ip_missing_name() -> None: match="Cannot discover IP via MQTT as the config does not include the device name:", ): get_esphome_device_ip(config) + + +def _discovery_config() -> dict: + return { + CONF_MQTT: { + CONF_BROKER: "mqtt.local", + }, + CONF_ESPHOME: { + CONF_NAME: "test-device", + }, + } + + +def _deliver_on_loop_start(mock_prepare, client, payload: bytes) -> None: + """Deliver a discovery answer as soon as the network loop starts.""" + + def deliver(*args, **kwargs): + msg = MagicMock() + msg.payload = payload + mock_prepare.call_args.args[2](client, None, msg) + + client.loop_start.side_effect = deliver + + +def test_get_esphome_device_ip_success() -> None: + """A device answer on the discovery topic returns its IPs.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, + client, + json.dumps( + {"name": "test-device", "ip": "10.0.0.5", "ip1": "10.0.0.6"} + ).encode(), + ) + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5", "10.0.0.6"] + client.loop_stop.assert_called_once_with() + # Once from on_message on receiving the answer, once from the finally + assert client.disconnect.call_count == 2 + + +def test_get_esphome_device_ip_preset_stop_event_skips_lookup() -> None: + """A stop event set before the call returns [] without touching the broker.""" + stop_event = threading.Event() + stop_event.set() + + with patch("esphome.mqtt.prepare") as mock_prepare: + result = get_esphome_device_ip(_discovery_config(), stop_event=stop_event) + + assert result == [] + mock_prepare.assert_not_called() + + +def test_get_esphome_device_ip_stop_event_aborts_wait() -> None: + """A stop event set mid-wait exits quietly with no addresses.""" + stop_event = threading.Event() + client = MagicMock() + # Simulate teardown starting right after the network loop spins up + client.loop_start.side_effect = stop_event.set + + start = time.monotonic() + with patch("esphome.mqtt.prepare", return_value=client): + result = get_esphome_device_ip(_discovery_config(), stop_event=stop_event) + + # An abort is not a failure and must be nowhere near the 25s timeout + assert result == [] + assert time.monotonic() - start < 5 + client.disconnect.assert_called_once_with() + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_timeout_raises() -> None: + """No answer within the timeout raises EsphomeError (default stop event path).""" + client = MagicMock() + with ( + patch("esphome.mqtt.prepare", return_value=client), + pytest.raises(EsphomeError, match="Failed to find IP via MQTT"), + ): + get_esphome_device_ip(_discovery_config(), timeout=0.25) + + client.disconnect.assert_called_once_with() + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_stop_during_connect_skips_wait() -> None: + """A stop event set while the broker connect is in flight still cleans up.""" + stop_event = threading.Event() + client = MagicMock() + + def prepare_and_stop(*args): + stop_event.set() + return client + + with patch("esphome.mqtt.prepare", side_effect=prepare_and_stop): + result = get_esphome_device_ip(_discovery_config(), stop_event=stop_event) + + assert result == [] + client.loop_start.assert_not_called() + client.disconnect.assert_called_once_with() + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_replaces_reconnect_handler( + caplog: pytest.LogCaptureFixture, +) -> None: + """The one-shot discovery client must not inherit the reconnect-forever + handler, which would make loop_stop() join the network thread forever; + its replacement still reports a broker-initiated disconnect.""" + client = MagicMock() + prepare_handler = MagicMock() + client.on_disconnect = prepare_handler + + with ( + patch("esphome.mqtt.prepare", return_value=client), + pytest.raises(EsphomeError, match="Failed to find IP via MQTT"), + ): + get_esphome_device_ip(_discovery_config(), timeout=0.25) + + assert client.on_disconnect is not prepare_handler + client.on_disconnect(client, None, 0) + assert "Disconnected from MQTT broker" not in caplog.text + client.on_disconnect(client, None, 5) + assert "Disconnected from MQTT broker (5)" in caplog.text + + +def test_get_esphome_device_ip_answer_without_ip_fails_fast( + caplog: pytest.LogCaptureFixture, +) -> None: + """A device answer with no IP fields fails promptly, not at the timeout.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, client, json.dumps({"name": "test-device"}).encode() + ) + + start = time.monotonic() + with pytest.raises(EsphomeError, match="Failed to find IP via MQTT"): + get_esphome_device_ip(_discovery_config(), timeout=5) + + assert time.monotonic() - start < 1 + assert "Device answer did not include an IP address" in caplog.text + + +@pytest.mark.parametrize("payload", [b"not json {", b"123", b"null"]) +def test_get_esphome_device_ip_unparsable_payload_ignored( + caplog: pytest.LogCaptureFixture, + payload: bytes, +) -> None: + """Garbage on the discovery topic must not kill paho's network thread.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start(mock_prepare, client, payload) + + with pytest.raises(EsphomeError, match="Failed to find IP via MQTT"): + get_esphome_device_ip(_discovery_config(), timeout=0) + + assert "Ignoring unparsable discovery payload" in caplog.text + + +def test_get_esphome_device_ip_broker_disconnect_fails_fast( + caplog: pytest.LogCaptureFixture, +) -> None: + """A broker-initiated disconnect aborts the wait instead of timing out.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client): + + def drop_connection(*args, **kwargs): + client.on_disconnect(client, None, 5) + + client.loop_start.side_effect = drop_connection + + start = time.monotonic() + with pytest.raises(EsphomeError, match="Failed to find IP via MQTT"): + get_esphome_device_ip(_discovery_config(), timeout=5) + + assert time.monotonic() - start < 1 + assert "Disconnected from MQTT broker (5)" in caplog.text + + +def test_get_esphome_device_ip_sends_discovery_ping() -> None: + """Connecting publishes the discovery ping for the device.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + + def connect_then_answer(*args, **kwargs): + on_connect = mock_prepare.call_args.args[3] + on_connect(client, None, None, 0) + msg = MagicMock() + msg.payload = json.dumps({"name": "test-device", "ip": "10.0.0.5"}).encode() + mock_prepare.call_args.args[2](client, None, msg) + + client.loop_start.side_effect = connect_then_answer + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5"] + client.publish.assert_called_once_with( + "esphome/ping/test-device", None, retain=False + ) + + +def test_get_esphome_device_ip_disconnect_error_does_not_mask_result( + caplog: pytest.LogCaptureFixture, +) -> None: + """A cleanup failure must not replace the discovery result.""" + client = MagicMock() + # First disconnect (from on_message) succeeds; the finally's fails + client.disconnect.side_effect = [None, OSError("socket already closed")] + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, + client, + json.dumps({"name": "test-device", "ip": "10.0.0.5"}).encode(), + ) + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5"] + client.loop_stop.assert_called_once_with() + + +def test_get_esphome_device_ip_invalid_address_values_skipped( + caplog: pytest.LogCaptureFixture, +) -> None: + """Non-string or non-printable ip values are skipped, valid ones kept.""" + client = MagicMock() + + with patch("esphome.mqtt.prepare", return_value=client) as mock_prepare: + _deliver_on_loop_start( + mock_prepare, + client, + json.dumps( + { + "name": "test-device", + "ip": 1234, + "ip1": "x\n[00:00:00][I][forged] fake line", + "ip2": " 10.0.0.5 ", + } + ).encode(), + ) + + result = get_esphome_device_ip(_discovery_config()) + + assert result == ["10.0.0.5"] + assert caplog.text.count("Ignoring invalid address in discovery answer") == 2 + assert "forged" not in "".join( + r.getMessage() for r in caplog.records if "Found IP" in r.getMessage() + ) From bd58b5c8b31fb97618335e163170c12beb2e1c4c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 19:14:50 -0500 Subject: [PATCH 014/160] [core] Retry framework downloads on transient network errors (#18330) --- esphome/framework_helpers.py | 259 ++++++++++++++------- tests/unit_tests/test_framework_helpers.py | 201 +++++++++++++++- 2 files changed, 373 insertions(+), 87 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 6ed608b171..86d5e4eaea 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -25,9 +25,13 @@ _LOGGER = logging.getLogger(__name__) # Attempts per mirror URL before falling through to the next mirror; only # mid-stream drops retry (resuming when the server gave a validator), -# connect errors move on immediately. +# connect errors move on to the next mirror immediately. _MIRROR_ATTEMPTS = 3 +# Passes over the whole mirror list when a transient network error is in +# the mix; matches git.py's _NETWORK_MAX_ATTEMPTS (3 tries, 2s/4s backoff). +_MIRROR_SWEEP_ATTEMPTS = 3 + def get_project_link_flags() -> list[str]: """Return the sorted -Wl, linker flags from the current build.""" @@ -887,37 +891,51 @@ def _failure_reason(e: Exception) -> str: return str(e).split(" for url: ", maxsplit=1)[0] or repr(e) -def download_from_mirrors( - mirrors: list[str], - substitutions: dict[str, str], - target: io.RawIOBase | IO[bytes] | PathType, - timeout: int = 30, -) -> str: +def _spent_attempts_error(e: Exception, attempts: int) -> Exception: + """Wrap a failure whose mirror already consumed download attempts, so + the sweep classifies it as permanent.""" + from esphome.core import EsphomeError + + err = EsphomeError(f"failed after {attempts} attempts: {_failure_reason(e)}") + err.__cause__ = e + return err + + +def _is_transient_download_error(e: Exception) -> bool: + """Return True when a download failure is worth retrying. + + Connection-level failures and HTTP 429/5xx are transient. Other HTTP + errors, local errors, and exhausted-attempts EsphomeError wrappers + (their per-mirror retries are already spent) are permanent. """ - Download file from multiple mirrors with substitution support. + # Imported lazily: requests is a heavy import (~85ms) and is only + # needed when actually downloading, never during config validation. + import requests - Args: - mirrors: list of mirror URLs - substitutions: Dictionary of substitutions to apply to URLs - target: Target file path or file-like object - timeout: Download timeout in seconds + if isinstance(e, requests.exceptions.HTTPError): + resp = e.response + return resp is not None and (resp.status_code == 429 or resp.status_code >= 500) + return isinstance( + e, + ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, + ), + ) - Returns: - The source URL. - Mirror URL templates that reference a substitution not present in - ``substitutions`` are skipped, so callers can offer templates that only - apply to some downloads. +def _try_mirrors_once( + urls: list[str], + path_target: Path | None, + f: IO[bytes] | None, + timeout: int, + failures: list[tuple[str, Exception]], +) -> str | None: + """Single pass over the resolved mirror ``urls``, one try per URL. - A path target downloads through ``download_with_resume``, so an - interrupted download resumes on the next esphome run; a file-like target - only resumes mid-stream drops within this call. - - Raises: - ValueError: If mirrors list is empty. - EsphomeError: If all download attempts fail; the message lists every - attempted URL with its individual failure reason. Also raised if - no template matched the provided substitutions. + Returns the source URL on success, or None with each URL's exception + appended to ``failures``. """ # Imported lazily: requests is a heavy import (~85ms) and is only # needed when actually downloading, never during config validation. @@ -925,43 +943,7 @@ def download_from_mirrors( from esphome.core import EsphomeError - ensure_happy_eyeballs() - - # 1. Classify the target: filesystem path or open file object - path_target: Path | None = None - f: IO[bytes] | None = None - if isinstance(target, (str, os.PathLike)): - path_target = Path(target) - elif isinstance(target, (io.RawIOBase, io.IOBase)): - f = target - else: - raise TypeError( - f"target must be str, Path, or file-like object: {type(target)}" - ) - - # 2. Try each mirror in order - failures: list[tuple[str, Exception]] = [] - skipped: list[tuple[str, str]] = [] - - for mirror in mirrors: - # 3. Apply substitutions to URL - try: - url = mirror.format(**substitutions) - except KeyError as e: - # The template references a substitution not provided for - # this download (e.g. SHORT_VERSION only exists for x.y.0 - # versions) - expected, the template just doesn't apply. - _LOGGER.debug("Skipping mirror %s: %s not available", mirror, e) - skipped.append((mirror, f"not applicable ({e.args[0]} not available)")) - continue - except (IndexError, ValueError) as e: - # A malformed template (unbalanced braces, bad format spec) - # is an authoring error, not an expected fallthrough - warn - # even if a later mirror succeeds. - _LOGGER.warning("Skipping malformed mirror URL template %s: %r", mirror, e) - skipped.append((mirror, f"skipped ({e!r})")) - continue - + for url in urls: _LOGGER.debug("Trying to download from %s", url) # Path targets delegate to download_with_resume so a partial @@ -986,14 +968,14 @@ def download_from_mirrors( failures.append((url, e)) continue - # 4. Download; mid-stream failures retry the same mirror with - # resume (see download_with_resume) instead of starting over. - # There is no checksum to verify a resumed file against, so a - # stitch is only trusted when the server proves consistency: the - # If-Range validator guarantees 206 only for unchanged content, - # and the expected total length (when the first response carried - # one) guards against short or shifted bodies. Without a - # validator the retry restarts from zero. + # File-like targets download here; mid-stream failures retry the + # same mirror with resume (see download_with_resume) instead of + # starting over. There is no checksum to verify a resumed file + # against, so a stitch is only trusted when the server proves + # consistency: the If-Range validator guarantees 206 only for + # unchanged content, and the expected total length (when the first + # response carried one) guards against short or shifted bodies. + # Without a validator the retry restarts from zero. offset = 0 expected_total = 0 validator = None @@ -1001,9 +983,12 @@ def download_from_mirrors( try: resp, offset = _open_ranged(url, offset, timeout, validator) except (requests.RequestException, OSError) as e: - # Connect/HTTP error, no bytes flowed — next mirror. + # Connect/HTTP error, no bytes flowed — next mirror. Wrap + # when earlier attempts were already spent on this mirror. _LOGGER.debug("Failed to download %s: %s", url, str(e)) - failures.append((url, e)) + failures.append( + (url, _spent_attempts_error(e, attempt + 1) if attempt else e) + ) break try: @@ -1031,7 +1016,7 @@ def download_from_mirrors( _LOGGER.debug("Downloaded successfully from: %s", url) - # 5. Reset file pointer and return + # Reset file pointer and return f.seek(0) return url @@ -1054,16 +1039,124 @@ def download_from_mirrors( ) offset = 0 if attempt == _MIRROR_ATTEMPTS - 1: - failures.append((url, e)) + failures.append((url, _spent_attempts_error(e, _MIRROR_ATTEMPTS))) - # 6. Report every attempted URL if all mirrors failed. Falling back - # past an early mirror is normal (e.g. only one of the framework URL - # templates matches a given version's tag), so raising only the last - # error would hide the failure that actually matters. - if failures: - attempts = "".join( - f"\n {url}\n {_failure_reason(e)}" for url, e in failures + return None + + +def download_from_mirrors( + mirrors: list[str], + substitutions: dict[str, str], + target: io.RawIOBase | IO[bytes] | PathType, + timeout: int = 30, +) -> str: + """ + Download file from multiple mirrors with substitution support. + + Args: + mirrors: list of mirror URLs + substitutions: Dictionary of substitutions to apply to URLs + target: Target file path or file-like object + timeout: Download timeout in seconds + + Returns: + The source URL. + + Mirror URL templates that reference a substitution not present in + ``substitutions`` are skipped, so callers can offer templates that only + apply to some downloads. + + A path target downloads through ``download_with_resume``, so an + interrupted download resumes on the next esphome run; a file-like target + only resumes mid-stream drops within this call. + + When every mirror fails and at least one failure is transient (dropped + connection, timeout, HTTP 429/5xx), the whole list is retried with a + short backoff; permanent failures (e.g. 404) raise immediately. + + Raises: + ValueError: If mirrors list is empty. + EsphomeError: If all download attempts fail; the message lists every + attempted URL with its individual failure reason. Also raised if + no template matched the provided substitutions. + """ + from esphome.core import EsphomeError + + ensure_happy_eyeballs() + + # 1. Classify the target: filesystem path or open file object + path_target: Path | None = None + f: IO[bytes] | None = None + if isinstance(target, (str, os.PathLike)): + path_target = Path(target) + elif isinstance(target, (io.RawIOBase, io.IOBase)): + f = target + else: + raise TypeError( + f"target must be str, Path, or file-like object: {type(target)}" ) + + # 2. Resolve the mirror templates (invariant across retry sweeps) + urls: list[str] = [] + skipped: list[tuple[str, str]] = [] + for mirror in mirrors: + try: + urls.append(mirror.format(**substitutions)) + except KeyError as e: + # The template references a substitution not provided for + # this download (e.g. SHORT_VERSION only exists for x.y.0 + # versions) - expected, the template just doesn't apply. + _LOGGER.debug("Skipping mirror %s: %s not available", mirror, e) + skipped.append((mirror, f"not applicable ({e.args[0]} not available)")) + except (IndexError, ValueError) as e: + # A malformed template (unbalanced braces, bad format spec) + # is an authoring error, not an expected fallthrough - warn + # even if a later mirror succeeds. + _LOGGER.warning("Skipping malformed mirror URL template %s: %r", mirror, e) + skipped.append((mirror, f"skipped ({e!r})")) + + # 3. Sweep the mirror list, retrying transient failures with backoff: + # a single pass keeps mirror failover fast, re-sweeping keeps one + # network blip from failing the build when only one mirror applies. + failures: list[tuple[str, Exception]] = [] + for sweep in range(1, _MIRROR_SWEEP_ATTEMPTS + 1): + sweep_failures: list[tuple[str, Exception]] = [] + if ( + url := _try_mirrors_once(urls, path_target, f, timeout, sweep_failures) + ) is not None: + return url + failures.extend(sweep_failures) + # Permanent failures (404, verification mismatch) won't heal; + # only retry when a transient error is in the mix (as git.py does). + transient = next( + ((u, e) for u, e in sweep_failures if _is_transient_download_error(e)), + None, + ) + if transient is None: + break + if sweep < _MIRROR_SWEEP_ATTEMPTS: + delay = 2**sweep + _LOGGER.warning( + "Download of %s failed (%s); retrying in %d seconds (attempt %d/%d)", + transient[0], + _failure_reason(transient[1]), + delay, + sweep + 1, + _MIRROR_SWEEP_ATTEMPTS, + ) + time.sleep(delay) + + # 4. Report every attempted URL if all mirrors failed. failures spans + # all sweeps (deduplicated by URL and reason), so neither an early + # mirror's failure nor an earlier sweep's failure mode is hidden. + if failures: + seen: set[tuple[str, str]] = set() + attempts = "" + for url, e in failures: + reason = _failure_reason(e) + if (url, reason) not in seen: + seen.add((url, reason)) + attempts += f"\n {url}\n {reason}" attempts += "".join(f"\n {mirror}\n {reason}" for mirror, reason in skipped) raise EsphomeError( f"Failed to download from all mirrors:{attempts}" diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 08751879c2..7451ee9b39 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -12,7 +12,7 @@ from pathlib import Path import subprocess import sys import tarfile -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, call, patch import zipfile import pytest @@ -23,6 +23,7 @@ from esphome.core import EsphomeError from esphome.framework_helpers import ( _7z_extract_all, _detect_archive_root, + _is_transient_download_error, _rename_with_retry, _tar_extract_all, _zip_extract_all, @@ -515,16 +516,23 @@ class TestArchiveExtractAll: # --------------------------------------------------------------------------- -def _mock_response(content: bytes, ok: bool = True) -> MagicMock: +def _mock_response( + content: bytes, ok: bool = True, status: int | None = None +) -> MagicMock: + """A fake requests response. The HTTPError carries the response (as + ``raise_for_status`` on a real response) so the transient classifier + can see its ``status``; failures default to a permanent 404.""" + if status is None: + status = 200 if ok else 404 r = MagicMock() r.__enter__.return_value = r r.__exit__.return_value = False - r.status_code = 200 + r.status_code = status r.ok = ok if ok: r.raise_for_status.return_value = None else: - r.raise_for_status.side_effect = req.HTTPError("503") + r.raise_for_status.side_effect = req.HTTPError(str(status), response=r) r.headers = {"content-length": "0"} # suppress ProgressBar r.iter_content.return_value = [content] if content else [] return r @@ -1419,6 +1427,191 @@ class TestDownloadFromMirrors: assert target.exists() assert target.read_bytes() == b"" + @pytest.mark.parametrize("target_kind", ["path", "file-like"]) + def test_transient_failure_retries_mirror_sweep( + self, tmp_path: Path, target_kind: str + ) -> None: + """A transient connect error on the only applicable mirror retries the + whole mirror list with backoff instead of failing the build.""" + target = tmp_path / "idf.tar.xz" if target_kind == "path" else io.BytesIO() + with ( + patch( + "requests.get", + side_effect=[ + req.ConnectionError("Remote end closed connection"), + _mock_response(b"data"), + ], + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + ): + url = download_from_mirrors(["https://mirror1.com/f"], {}, target) + assert url == "https://mirror1.com/f" + data = target.read_bytes() if target_kind == "path" else target.getvalue() + assert data == b"data" + assert mock_get.call_count == 2 + mock_sleep.assert_called_once_with(2) + + def test_permanent_failure_does_not_retry_sweep(self, tmp_path: Path) -> None: + """An HTTP 404 will not heal on its own; fail after a single pass.""" + with ( + patch( + "requests.get", return_value=_mock_response(b"", ok=False, status=404) + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="all mirrors"), + ): + download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin") + assert mock_get.call_count == 1 + mock_sleep.assert_not_called() + + def test_transient_failure_exhausts_sweeps(self, tmp_path: Path) -> None: + """A persistent transient error gives up after the configured number + of passes, with 2s/4s backoff, and still lists the attempted URL.""" + with ( + patch("requests.get", side_effect=req.ConnectionError("down")) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="all mirrors") as ei, + ): + download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin") + assert mock_get.call_count == 3 + assert mock_sleep.call_args_list == [call(2), call(4)] + assert "https://mirror1.com/f" in str(ei.value) + + def test_mixed_permanent_and_transient_retries_sweep(self, tmp_path: Path) -> None: + """One mirror 404s permanently while another hits a transient error; + the transient failure makes the whole list worth another pass.""" + dest = tmp_path / "out.bin" + with ( + patch( + "requests.get", + side_effect=[ + _mock_response(b"", ok=False, status=404), + req.ConnectionError("down"), + _mock_response(b"", ok=False, status=404), + _mock_response(b"data"), + ], + ), + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + ): + url = download_from_mirrors( + ["https://mirror1.com/f", "https://mirror2.com/f"], {}, dest + ) + assert url == "https://mirror2.com/f" + assert dest.read_bytes() == b"data" + mock_sleep.assert_called_once_with(2) + + def test_http_5xx_retries_sweep(self, tmp_path: Path) -> None: + """A real 5xx (response attached to the HTTPError) is transient.""" + dest = tmp_path / "out.bin" + with ( + patch( + "requests.get", + side_effect=[ + _mock_response(b"", ok=False, status=503), + _mock_response(b"data"), + ], + ), + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + ): + url = download_from_mirrors(["https://mirror1.com/f"], {}, dest) + assert url == "https://mirror1.com/f" + assert dest.read_bytes() == b"data" + mock_sleep.assert_called_once_with(2) + + def test_error_reports_failure_modes_from_all_sweeps(self, tmp_path: Path) -> None: + """A failure mode that changes between sweeps stays in the final + error; the first failure (the one that started the retries) is + chained as the cause.""" + with ( + patch( + "requests.get", + side_effect=[ + req.ConnectionError("dropped by middlebox"), + _mock_response(b"", ok=False, status=404), + ], + ), + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="all mirrors") as ei, + ): + download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin") + assert "dropped by middlebox" in str(ei.value) + assert "404" in str(ei.value) + assert isinstance(ei.value.__cause__, req.ConnectionError) + mock_sleep.assert_called_once_with(2) + + def test_exhausted_mid_stream_attempts_not_swept(self) -> None: + """A file-like mirror that spent all its mid-stream attempts is not + retried again at the sweep level (unlike a path target, it has no + part file to resume from on a later sweep).""" + buf = io.BytesIO() + with ( + patch( + "requests.get", + side_effect=[_interrupted_response(b"1234") for _ in range(3)], + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="failed after 3 attempts"), + ): + download_from_mirrors(["https://mirror1.com/f"], {}, buf) + assert mock_get.call_count == 3 + mock_sleep.assert_not_called() + + def test_mid_stream_drop_then_connect_error_not_swept(self) -> None: + """A connect error on a later attempt (after a mid-stream drop spent + one) also counts as spent budget and does not re-arm the sweep.""" + buf = io.BytesIO() + with ( + patch( + "requests.get", + side_effect=[ + _interrupted_response(b"1234"), + req.ConnectionError("down"), + ], + ) as mock_get, + patch("esphome.framework_helpers.time.sleep") as mock_sleep, + pytest.raises(EsphomeError, match="failed after 2 attempts"), + ): + download_from_mirrors(["https://mirror1.com/f"], {}, buf) + assert mock_get.call_count == 2 + mock_sleep.assert_not_called() + + +def _http_error(status: int) -> req.HTTPError: + """An HTTPError carrying a response with the given status, as raised by + ``raise_for_status`` on a real response.""" + resp = MagicMock() + resp.status_code = status + return req.HTTPError(str(status), response=resp) + + +class TestIsTransientDownloadError: + def test_connection_errors_are_transient(self) -> None: + assert _is_transient_download_error(req.ConnectionError("reset")) + assert _is_transient_download_error(req.Timeout("timed out")) + assert _is_transient_download_error( + req.exceptions.ChunkedEncodingError("dropped") + ) + + def test_http_statuses(self) -> None: + assert not _is_transient_download_error(_http_error(404)) + assert not _is_transient_download_error(_http_error(403)) + assert _is_transient_download_error(_http_error(429)) + assert _is_transient_download_error(_http_error(503)) + + def test_http_error_without_response_is_permanent(self) -> None: + assert not _is_transient_download_error(req.HTTPError("boom")) + + def test_exhausted_resume_attempts_are_permanent(self) -> None: + """download_with_resume already spent its own resume attempts; its + EsphomeError wrapper is not retried again at the sweep level.""" + wrapped = EsphomeError("Failed to download after 3 attempts") + wrapped.__cause__ = req.ConnectionError("down") + assert not _is_transient_download_error(wrapped) + + def test_unrelated_errors_are_permanent(self) -> None: + assert not _is_transient_download_error(OSError("disk full")) + assert not _is_transient_download_error(EsphomeError("size mismatch")) + def test_importing_framework_helpers_does_not_import_requests() -> None: """Importing framework_helpers must not drag in requests. From d3e27054f6f621e712d85e1bba4172ea6d7444c5 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:30:47 +1200 Subject: [PATCH 015/160] Bump version to 2026.8.0b2 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 006f97acb7..a8c77f4bb8 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.0b1 +PROJECT_NUMBER = 2026.8.0b2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 623d9673bc..b6770d0001 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.0b1" +__version__ = "2026.8.0b2" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 9cc05b30d401daba19a061fe610a952590987961 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 21:09:01 -0500 Subject: [PATCH 016/160] [web_server_base] Stop deleting the web server on captive portal teardown (#18324) --- .../web_server/ota/ota_web_server.cpp | 2 +- .../web_server_base/web_server_base.h | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/esphome/components/web_server/ota/ota_web_server.cpp b/esphome/components/web_server/ota/ota_web_server.cpp index 9812714ec0..95763e2daf 100644 --- a/esphome/components/web_server/ota/ota_web_server.cpp +++ b/esphome/components/web_server/ota/ota_web_server.cpp @@ -249,7 +249,7 @@ void WebServerOTAComponent::setup() { return; } - // AsyncWebServer takes ownership of the handler and will delete it when the server is destroyed + // The handler lives for the life of the process; WebServerBase never destroys its server base->add_handler(new OTARequestHandler(this)); // NOLINT } diff --git a/esphome/components/web_server_base/web_server_base.h b/esphome/components/web_server_base/web_server_base.h index c647a13b50..94579de70f 100644 --- a/esphome/components/web_server_base/web_server_base.h +++ b/esphome/components/web_server_base/web_server_base.h @@ -112,9 +112,18 @@ class AuthMiddlewareHandler : public MiddlewareHandler { class WebServerBase final { public: + // The AsyncWebServer is created once and intentionally never deleted: on Arduino + // platforms ESPAsyncWebServer owns its registered handlers, so destroying it would + // also destroy live components (e.g. the captive portal) out from under us. + // init()/deinit() refcount users and start/stop the listener; handlers are + // registered once at creation and survive listener restarts. void init() { - if (this->initialized_) { - this->initialized_++; + this->initialized_++; + if (this->server_ != nullptr) { + if (this->initialized_ == 1) { + // Restart the listener after a previous deinit() + this->server_->begin(); + } return; } this->server_ = new AsyncWebServer(this->port_); @@ -126,14 +135,13 @@ class WebServerBase final { for (auto *handler : this->handlers_) this->server_->addHandler(handler); - - this->initialized_++; } void deinit() { + if (this->initialized_ == 0) + return; // unbalanced deinit() this->initialized_--; if (this->initialized_ == 0) { - delete this->server_; - this->server_ = nullptr; + this->server_->end(); } } AsyncWebServer *get_server() const { return this->server_; } From f337d0acff4dfae8e09021508f2bfdff070d2903 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2026 02:13:29 +0000 Subject: [PATCH 017/160] Bump pylint from 4.0.6 to 4.0.7 (#18320) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 0905fe6be1..1832ffd433 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,4 @@ -pylint==4.0.6 +pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.2 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating From f1c40867783064dc711be70c3412d0066a80c378 Mon Sep 17 00:00:00 2001 From: Oliver Kleinecke Date: Thu, 13 Aug 2026 13:36:11 +0200 Subject: [PATCH 018/160] [const] Move CONF_SLOT to components/const (#18350) Co-authored-by: Oliver Kleinecke --- esphome/components/const/__init__.py | 1 + esphome/components/esp32_hosted/__init__.py | 3 +-- esphome/components/sendspin/image/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 44878274d6..2d02c7d179 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -35,6 +35,7 @@ CONF_REQUEST_HEADERS = "request_headers" CONF_ROWS = "rows" CONF_SCAN_PARAMETERS = "scan_parameters" CONF_SHA256 = "sha256" +CONF_SLOT = "slot" CONF_STATE_SAVE_INTERVAL = "state_save_interval" CONF_STOP_BITS = "stop_bits" CONF_TARGET_COUNT = "target_count" diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index b15ae53711..c6a714aace 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -3,7 +3,7 @@ from pathlib import Path from esphome import pins from esphome.components import esp32 -from esphome.components.const import CONF_USE_PSRAM +from esphome.components.const import CONF_SLOT, CONF_USE_PSRAM import esphome.config_validation as cv from esphome.const import ( CONF_CLK_PIN, @@ -33,7 +33,6 @@ CONF_DATA_READY_PIN = "data_ready_pin" CONF_HANDSHAKE_ACTIVE_HIGH = "handshake_active_high" CONF_HANDSHAKE_PIN = "handshake_pin" CONF_SDIO_FREQUENCY = "sdio_frequency" -CONF_SLOT = "slot" CONF_SPI_MODE = "spi_mode" # Shared fields for both transport modes diff --git a/esphome/components/sendspin/image/__init__.py b/esphome/components/sendspin/image/__init__.py index 94d6e7cfca..3c6c82b009 100644 --- a/esphome/components/sendspin/image/__init__.py +++ b/esphome/components/sendspin/image/__init__.py @@ -3,6 +3,7 @@ from esphome import automation import esphome.codegen as cg from esphome.components import runtime_image +from esphome.components.const import CONF_SLOT from esphome.components.image import CONF_TRANSPARENCY, Image_, add_metadata import esphome.config_validation as cv from esphome.const import ( @@ -45,7 +46,6 @@ MAX_IMAGE_DIMENSION = 32767 MAX_DISPLAY_OFFSET = cv.TimePeriod(seconds=60) MIN_DISPLAY_OFFSET = cv.TimePeriod(seconds=-60) -CONF_SLOT = "slot" CONF_CURRENT_IMAGE = "current_image" CONF_TRANSITION_IMAGE = "transition_image" CONF_ON_IMAGE_DISPLAY = "on_image_display" From 87045ab9c020e95da9264c26391ffbf2bc5a4438 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:14:14 -0500 Subject: [PATCH 019/160] Bump aioesphomeapi from 45.10.1 to 45.10.2 (#18357) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 85a0f55263..683008400a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.10.1 +aioesphomeapi==45.10.2 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From c7940382a9a210226b2e2a7eee668dc6dc4c21a5 Mon Sep 17 00:00:00 2001 From: Oliver Kleinecke Date: Thu, 13 Aug 2026 20:18:22 +0200 Subject: [PATCH 020/160] [const] move CONF_LABEL to components/const (#18354) Co-authored-by: Oliver Kleinecke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/const/__init__.py | 1 + esphome/components/display_menu_base/__init__.py | 2 +- esphome/components/lvgl/widgets/label.py | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 2d02c7d179..3ba89d2838 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -22,6 +22,7 @@ CONF_GYROSCOPE_ODR = "gyroscope_odr" CONF_GYROSCOPE_RANGE = "gyroscope_range" CONF_IAQ = "iaq" CONF_IGNORE_NOT_FOUND = "ignore_not_found" +CONF_LABEL = "label" CONF_LIBRETINY = "libretiny" CONF_LOOP = "loop" CONF_NOX_INDEX = "nox_index" diff --git a/esphome/components/display_menu_base/__init__.py b/esphome/components/display_menu_base/__init__.py index 9125c43f0c..2120abe5f7 100644 --- a/esphome/components/display_menu_base/__init__.py +++ b/esphome/components/display_menu_base/__init__.py @@ -3,6 +3,7 @@ import re from esphome import automation, core from esphome.automation import maybe_simple_id import esphome.codegen as cg +from esphome.components.const import CONF_LABEL from esphome.components.number import Number from esphome.components.select import Select from esphome.components.switch import Switch @@ -30,7 +31,6 @@ display_menu_base_ns = cg.esphome_ns.namespace("display_menu_base") CONF_ROTARY = "rotary" CONF_JOYSTICK = "joystick" -CONF_LABEL = "label" CONF_MENU = "menu" CONF_BACK = "back" CONF_SELECT = "select" diff --git a/esphome/components/lvgl/widgets/label.py b/esphome/components/lvgl/widgets/label.py index 5ac92f2717..54c9819d2b 100644 --- a/esphome/components/lvgl/widgets/label.py +++ b/esphome/components/lvgl/widgets/label.py @@ -1,3 +1,4 @@ +from esphome.components.const import CONF_LABEL import esphome.config_validation as cv from esphome.const import CONF_TEXT @@ -14,8 +15,6 @@ from ..schemas import TEXT_SCHEMA from ..types import LvText from . import Widget, WidgetType -CONF_LABEL = "label" - class LabelType(WidgetType): def __init__(self): From db5173697a40c92e6f7a4dfc1d97c59580bc9271 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:22:48 -0500 Subject: [PATCH 021/160] [esp32_ble_tracker] Fix missed BLE advertisements with WiFi on ESP-IDF 5.5.5 (#18356) --- .../components/ble_device_base/__init__.py | 20 ++- .../components/esp32_ble_tracker/__init__.py | 71 +++++++++- .../test_scan_parameter_validation.py | 7 +- .../esp32_ble_tracker/__init__.py | 0 .../test_scan_window_default.py | 122 ++++++++++++++++++ 5 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 tests/component_tests/esp32_ble_tracker/__init__.py create mode 100644 tests/component_tests/esp32_ble_tracker/test_scan_window_default.py diff --git a/esphome/components/ble_device_base/__init__.py b/esphome/components/ble_device_base/__init__.py index 4da7d48882..15a8b08139 100644 --- a/esphome/components/ble_device_base/__init__.py +++ b/esphome/components/ble_device_base/__init__.py @@ -37,7 +37,7 @@ from esphome.const import ( CONF_INTERVAL, KEY_TARGET_PLATFORM, ) -from esphome.core import CORE, ID, KEY_CORE +from esphome.core import CORE, ID, KEY_CORE, TimePeriod from esphome.types import ConfigType CODEOWNERS = ["@Bl00d-B0b"] @@ -243,19 +243,27 @@ def validate_scan_parameters(config: ConfigType) -> ConfigType: return config +# The historical scan window default shared by the trackers that do not pin +# their own; also the fallback for esp32's conditional default. +DEFAULT_SCAN_WINDOW = "30ms" + + def scan_parameters_schema( interval_default: str, *, - window_default: str = "30ms", + window_default: str | Callable[[], TimePeriod] = DEFAULT_SCAN_WINDOW, ) -> cv.All: """Build the scan_parameters value schema shared by all BLE trackers. interval_default and window_default are per chip (e.g. esp32 320/30 ms, bk72xx/rp2 100/30 ms — the reference scan rates of the respective stacks; - LN882H's SDK recommends 100/50 ms). The `active` option (default on) is - unconditional: active scanning is part of the tracker contract — every - current proxy client assumes it, so a passive-only tracker must not share - this schema. + LN882H's SDK recommends 100/50 ms). window_default may also be a zero-arg + callable evaluated per validation when the user omits the key (esp32 uses + this to record that the window was defaulted, so a later validation step + can adjust it once sibling keys are resolved). The `active` option + (default on) is unconditional: active scanning is part of the tracker + contract — every current proxy client assumes it, so a passive-only + tracker must not share this schema. """ schema = { cv.Optional(CONF_DURATION, default="5min"): cv.positive_time_period_seconds, diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 634b8c3bef..28c8c7fcf1 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -1,5 +1,7 @@ from __future__ import annotations +import copy +from dataclasses import dataclass import logging from esphome import automation @@ -8,6 +10,7 @@ from esphome.components import ble_device_base, esp32_ble, ota from esphome.components.const import CONF_ON_SCAN_END, CONF_SCAN_PARAMETERS, CONF_WINDOW from esphome.components.esp32 import ( add_idf_sdkconfig_option, + idf_version, request_bluetooth, request_software_coexistence, ) @@ -35,10 +38,12 @@ from esphome.const import ( CONF_SERVICE_UUID, CONF_TRIGGER_ID, ) -from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.core import CORE, CoroPriority, TimePeriod, coroutine_with_priority from esphome.enum import StrEnum from esphome.types import ConfigType +DOMAIN = "esp32_ble_tracker" + AUTO_LOAD = ["ble_device_base", "esp32_ble"] DEPENDENCIES = ["esp32"] CODEOWNERS = ["@bdraco"] @@ -125,10 +130,71 @@ def validate_max_connections_deprecated(config: ConfigType) -> ConfigType: return config +# ESP-IDF 5.5.5 fixed a coexistence bug on the ESP32 where BLE scans ran far +# longer than the configured window (espressif/esp-idf#18931). Before the fix, +# the default 30 ms window in a 320 ms interval effectively scanned at a much +# higher duty cycle than requested; with the fix, that same default only +# listens 9.4 % of the time and misses most advertisements when wifi shares +# the radio. Espressif recommends setting the window equal to the interval in +# that case: the coexistence arbiter still shares the radio with wifi, and +# BLE uses the airtime wifi does not claim. +IDF_SCAN_WINDOW_FIX_VERSION = cv.Version(5, 5, 5) + + +@dataclass +class TrackerData: + """Per-run validation state, namespaced under DOMAIN in CORE.data.""" + + scan_window_defaulted: bool = False + + +def _get_data() -> TrackerData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = TrackerData() + return CORE.data[DOMAIN] + + +def _scan_window_default() -> TimePeriod: + """Schema default for the scan window. + + Records that the user did not set a window, so _raise_defaulted_scan_window + can tell a defaulted 30 ms from an explicit one; the raise itself must wait + for the outer schema because it depends on software_coexistence, a sibling + key not yet resolved here. + """ + _get_data().scan_window_defaulted = True + return cv.positive_time_period(ble_device_base.DEFAULT_SCAN_WINDOW) + + +def _raise_defaulted_scan_window(config: ConfigType) -> ConfigType: + """Raise a defaulted scan window to the interval where that is safe. + + Only when the coexistence arbiter is compiled in (software_coexistence, + present iff wifi is configured and not disabled by the user) and the IDF + honors the window strictly (>= 5.5.5); without the arbiter a full-duty + scan would starve wifi outright, and a user-set window is never touched. + Raising to the interval cannot invalidate the already-validated + parameters, so no re-validation is needed. + """ + if ( + _get_data().scan_window_defaulted + and config.get(CONF_SOFTWARE_COEXISTENCE) + and idf_version() >= IDF_SCAN_WINDOW_FIX_VERSION + ): + params = config[CONF_SCAN_PARAMETERS] + # Copy so the config dump shows a plain value instead of a YAML + # anchor/alias pair pointing at the interval. + params[CONF_WINDOW] = copy.copy(params[CONF_INTERVAL]) + return config + + # 320 ms is the ESP-IDF reference scan interval; the shared schema also # tightens validation to the controller's 2.5 ms .. 10240 ms range and rejects # window/interval pairs that collapse to the same 0.625 ms unit count. -SCAN_PARAMETERS_SCHEMA = ble_device_base.scan_parameters_schema("320ms") +# The window default is conditional (see _scan_window_default above). +SCAN_PARAMETERS_SCHEMA = ble_device_base.scan_parameters_schema( + "320ms", window_default=_scan_window_default +) # Codegen helpers are owned by ble_device_base; kept under the historical names # here for the components that import them from this module. @@ -183,6 +249,7 @@ CONFIG_SCHEMA = cv.All( } ).extend(cv.COMPONENT_SCHEMA), validate_max_connections_deprecated, + _raise_defaulted_scan_window, ) diff --git a/tests/component_tests/ble_device_base/test_scan_parameter_validation.py b/tests/component_tests/ble_device_base/test_scan_parameter_validation.py index 2549125a43..3774d990d3 100644 --- a/tests/component_tests/ble_device_base/test_scan_parameter_validation.py +++ b/tests/component_tests/ble_device_base/test_scan_parameter_validation.py @@ -57,7 +57,12 @@ def test_bk72xx_defaults_are_valid() -> None: def test_esp32_defaults_are_valid() -> None: - """esp32 pins the ESP-IDF reference rate and exposes active (default on).""" + """esp32 pins the ESP-IDF reference rate and exposes active (default on). + + Without wifi loaded, the conditional window default falls back to the + historical 30 ms; the wifi-aware resolution is covered by the + esp32_ble_tracker component tests. + """ config = ESP32_SCHEMA({}) assert to_ble_units(config["interval"]) == 512 assert to_ble_units(config["window"]) == 48 diff --git a/tests/component_tests/esp32_ble_tracker/__init__.py b/tests/component_tests/esp32_ble_tracker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py b/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py new file mode 100644 index 0000000000..8a25f488fa --- /dev/null +++ b/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py @@ -0,0 +1,122 @@ +"""Tests for the esp32_ble_tracker conditional scan window default. + +The scan window default depends on wifi coexistence and the IDF version: +IDF 5.5.5 fixed a coexistence bug where BLE scans ran far longer than the +configured window (espressif/esp-idf#18931), so on fixed versions the +historical 30 ms default would only listen 9.4 % of the time and miss most +advertisements. With the coexistence arbiter compiled in on a fixed IDF, the +window instead defaults to the interval, as Espressif recommends; without the +arbiter a full-duty scan would starve wifi, so the 30 ms default is kept. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest + +from esphome import config_validation as cv +from esphome.components.ble_device_base import to_ble_units +from esphome.components.const import CONF_SCAN_PARAMETERS, CONF_WINDOW +from esphome.components.esp32 import KEY_IDF_VERSION +from esphome.components.esp32_ble_tracker import ( + CONF_SOFTWARE_COEXISTENCE, + CONFIG_SCHEMA, +) +from esphome.const import CONF_INTERVAL, PlatformFramework +from esphome.core import CORE +from esphome.types import ConfigType + +from ..types import SetCoreConfigCallable + + +@pytest.fixture +def stage_esp32( + set_core_config: SetCoreConfigCallable, +) -> Callable[..., None]: + """Stage an esp32 build with a given IDF version and wifi presence.""" + + def stage(idf: str, *, wifi: bool) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_IDF_VERSION: cv.Version.parse(idf)}, + ) + if wifi: + # Makes cv.OnlyWith default software_coexistence to True, exactly + # as a real config with wifi: does. + CORE.loaded_integrations.add("wifi") + + return stage + + +def _scan_params(config: ConfigType) -> ConfigType: + return CONFIG_SCHEMA(config)[CONF_SCAN_PARAMETERS] + + +@pytest.mark.parametrize( + ("idf", "config", "expected_units"), + [ + ("5.5.5", {}, 512), # first fixed version, default 320 ms interval + ("6.0.1", {}, 512), # any newer version behaves the same + # Follows a user-set interval. + ("5.5.5", {"scan_parameters": {"interval": "1s"}}, 1600), + ], +) +def test_wifi_on_fixed_idf_defaults_window_to_interval( + stage_esp32: Callable[..., None], + idf: str, + config: ConfigType, + expected_units: int, +) -> None: + """With wifi coexistence on a fixed IDF, the window defaults to the interval.""" + stage_esp32(idf, wifi=True) + params = _scan_params(config) + assert params[CONF_WINDOW] == params[CONF_INTERVAL] + assert to_ble_units(params[CONF_WINDOW]) == expected_units + + +@pytest.mark.parametrize( + ("idf", "wifi", "config"), + [ + # Buggy IDF over-scans anyway; keep the 30 ms default. + ("5.5.4", True, {}), + # No wifi (e.g. ethernet) means no radio contention. + ("5.5.5", False, {}), + # Coexistence disabled: no arbiter, so a full-duty scan would starve + # wifi outright. + ("5.5.5", True, {CONF_SOFTWARE_COEXISTENCE: False}), + ], +) +def test_30ms_default_kept( + stage_esp32: Callable[..., None], + idf: str, + wifi: bool, + config: ConfigType, +) -> None: + stage_esp32(idf, wifi=wifi) + assert to_ble_units(_scan_params(config)[CONF_WINDOW]) == 48 + + +@pytest.mark.parametrize("window", ["60ms", "30ms"]) +def test_explicit_window_is_never_touched( + stage_esp32: Callable[..., None], window: str +) -> None: + """A user-set window wins over the conditional default. + + The explicit 30 ms case matters: it is indistinguishable from the + defaulted value by inspection, so the defaulted flag must separate them. + """ + stage_esp32("5.5.5", wifi=True) + params = _scan_params({"scan_parameters": {"window": window}}) + assert to_ble_units(params[CONF_WINDOW]) == to_ble_units( + cv.positive_time_period(window) + ) + + +def test_short_interval_without_window_still_rejected( + stage_esp32: Callable[..., None], +) -> None: + """The provisional 30 ms default validates against the interval as before.""" + stage_esp32("5.5.5", wifi=True) + with pytest.raises(cv.Invalid, match="needs to be smaller than scan interval"): + _scan_params({"scan_parameters": {"interval": "20ms"}}) From 137351fa8d85f27130b8ccfcbdfa9a42555a6635 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:27:01 -0500 Subject: [PATCH 022/160] [esp32_ble] Silence spurious warnings for local key GAP events (#18359) --- esphome/components/esp32_ble/ble.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 16501ef3b2..e2d79173ff 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -648,6 +648,8 @@ void ESP32BLE::gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_pa case ESP_GAP_BLE_SET_PKT_LENGTH_COMPLETE_EVT: case ESP_GAP_BLE_PHY_UPDATE_COMPLETE_EVT: // BLE 5.0 PHY update complete case ESP_GAP_BLE_CHANNEL_SELECT_ALGORITHM_EVT: // BLE 5.0 channel selection algorithm + case ESP_GAP_BLE_LOCAL_IR_EVT: // Local identity root key generated at security init + case ESP_GAP_BLE_LOCAL_ER_EVT: // Local encryption root key generated at security init return; default: From 191686c5b3e106581ec58ab0ee15e6b8af4e9527 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:30:53 -0500 Subject: [PATCH 023/160] [wifi] Fix ESP8266 crash in cnx_node_search when lwIP transmits after disconnect (#18333) --- .../wifi/wifi_component_esp8266.cpp | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 719a276bf9..acaa94b13c 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -136,10 +136,21 @@ bool WiFiComponent::wifi_apply_power_save_() { https://github.com/d-a-v/Arduino/blob/0e7d21e17144cfc5f53c016191daca8723e89ee8/libraries/ESP8266WiFi/src/ESP8266WiFiSTA.cpp#L251 */ #undef netif_set_addr // need to call lwIP-v1.4 netif_set_addr() +#undef netif_set_down // need to call lwIP-v1.4 netif_set_down() extern "C" { struct netif *eagle_lwip_getif(int netif_index); void netif_set_addr(struct netif *netif, const ip4_addr_t *ip, const ip4_addr_t *netmask, const ip4_addr_t *gw); +void netif_set_down(struct netif *netif); }; + +// The SDK can free its WiFi connection node before taking the STA netif down, letting lwIP +// timers (e.g. IGMP reports armed by mDNS) transmit into the dead driver and crash in +// cnx_node_search; taking the netif down first makes the glue drop such frames (#18308). +static void sta_netif_down() { + struct netif *iface = eagle_lwip_getif(STATION_IF); + if (iface != nullptr) + netif_set_down(iface); +} #endif bool WiFiComponent::wifi_sta_ip_config_(const optional &manual_ip) { @@ -523,6 +534,9 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { global_wifi_component->sta_state_ = static_cast(ESP8266WiFiSTAState::ERROR_FAILED); } global_wifi_component->error_from_callback_ = true; +#if LWIP_VERSION_MAJOR != 1 + sta_netif_down(); +#endif #ifdef USE_WIFI_CONNECT_STATE_LISTENERS global_wifi_component->pending_.disconnect = true; #endif @@ -536,6 +550,9 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { // https://lbsfilm.at/blog/wpa2-authenticationmode-downgrade-in-espressif-microprocessors if (it.old_mode != AUTH_OPEN && it.new_mode == AUTH_OPEN) { ESP_LOGW(TAG, "Potential Authmode downgrade detected, disconnecting"); +#if LWIP_VERSION_MAJOR != 1 + sta_netif_down(); +#endif wifi_station_disconnect(); global_wifi_component->error_from_callback_ = true; } @@ -719,8 +736,12 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { bool WiFiComponent::wifi_disconnect_() { bool ret = true; // Only call disconnect if interface is up - if (wifi_get_opmode() & WIFI_STA) + if (wifi_get_opmode() & WIFI_STA) { +#if LWIP_VERSION_MAJOR != 1 + sta_netif_down(); +#endif ret = wifi_station_disconnect(); + } station_config conf{}; memset(&conf, 0, sizeof(conf)); ETS_UART_INTR_DISABLE(); From 7420d238673fb3a593b9314bef93f8e8e1cd4546 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:31:06 -0500 Subject: [PATCH 024/160] [ota] Retry uploads that fail from network errors (#18332) --- esphome/espota2.py | 158 ++++++++++--- tests/unit_tests/test_espota2.py | 382 +++++++++++++++++++++++++++++-- 2 files changed, 493 insertions(+), 47 deletions(-) diff --git a/esphome/espota2.py b/esphome/espota2.py index fa15c1dda2..61e897f601 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +import contextlib import gzip import hashlib import io @@ -8,7 +9,6 @@ import logging from pathlib import Path import secrets import socket -import sys import time from typing import Any @@ -76,6 +76,14 @@ _SUPPORTED_OTA_TYPES: frozenset[int] = frozenset( UPLOAD_BLOCK_SIZE = 8192 UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 +# Flaky Wi-Fi links often drop the first OTA attempt, and the device may need time +# to clean up a half-open connection (its handshake watchdog runs at 20s) before it +# accepts a new one, so wait between attempts instead of failing the upload outright. +# Every resolved address is tried once, and this many extra attempts are shared +# across the addresses on top of that. +EXTRA_UPLOAD_ATTEMPTS = 2 +UPLOAD_RETRY_DELAY = 5.0 + _LOGGER = logging.getLogger(__name__) # Authentication method lookup table: response -> (hash_func, nonce_size, name) @@ -171,6 +179,23 @@ class OTAError(EsphomeError): pass +class OTANetworkError(OTAError): + """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" + + +def _committed_error(err: OTANetworkError) -> OTAError: + """Wrap a network failure that happened once the device had the full image. + + Past that point the device commits and reboots on its own, so the failure + must not be retried; a re-upload could flash a device that already updated. + """ + return OTAError( + f"{err} (the device may have already committed the update and " + f"be rebooting; check whether it comes back with the new " + f"firmware before uploading again)" + ) + + def recv_decode( sock: socket.socket, amount: int, decode: bool = True ) -> bytes | list[int]: @@ -209,19 +234,22 @@ def receive_exactly( try: data += recv_decode(sock, 1, decode=decode) # type: ignore[operator] except OSError as err: - raise OTAError(f"receiving {msg} response: {err}") from err + raise OTANetworkError(f"receiving {msg} response: {err}") from err try: check_error(data, expect) except OTAError as err: sock.close() - raise OTAError(f"receiving {msg}: {err}") from err + # type(err) preserves OTANetworkError vs OTAError so callers can tell + # retryable network failures from device-reported errors; subclasses + # must accept a single message argument + raise type(err)(f"receiving {msg}: {err}") from err while len(data) < amount: try: data += recv_decode(sock, amount - len(data), decode=decode) # type: ignore[operator] except OSError as err: - raise OTAError(f"receiving {msg}: {err}") from err + raise OTANetworkError(f"receiving {msg}: {err}") from err return data @@ -237,7 +265,7 @@ def check_error(data: list[int] | bytes, expect: int | list[int] | None) -> None # accept-any-response reads (e.g. feature negotiation, auth nonces) would be # silently passed through and surface later as cryptic decode/timeout failures. if not data: - raise OTAError( + raise OTANetworkError( "Device closed connection without responding. " "This may indicate the device ran out of memory, " "a network issue, or the connection was interrupted." @@ -274,7 +302,7 @@ def send_check( sock.sendall(data) except OSError as err: - raise OTAError(f"sending {msg}: {err}") from err + raise OTANetworkError(f"sending {msg}: {err}") from err def perform_ota( @@ -306,7 +334,7 @@ def perform_ota( send_check(sock, MAGIC_BYTES, "magic bytes") _, version = receive_exactly(sock, 2, "version", RESPONSE_OK) - _LOGGER.debug("Device support OTA version: %s", version) + _LOGGER.info("Connection established; device supports OTA version %s", version) supported_versions = (OTA_VERSION_1_0, OTA_VERSION_2_0) if version not in supported_versions: raise OTAError( @@ -417,6 +445,8 @@ def perform_ota( hash_func, nonce_size, hash_name = _AUTH_METHODS[auth] perform_auth(sock, password, hash_func, nonce_size, hash_name) + _LOGGER.info("Handshake complete") + # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures sock.settimeout(90.0) @@ -449,21 +479,43 @@ def perform_ota( offset = 0 progress = ProgressBar("Uploading") - while True: - chunk = upload_contents[offset : offset + UPLOAD_BLOCK_SIZE] - if not chunk: - break - offset += len(chunk) + try: + while True: + chunk = upload_contents[offset : offset + UPLOAD_BLOCK_SIZE] + if not chunk: + break + offset += len(chunk) + + try: + sock.sendall(chunk) + except OSError as err: + # A send failure can hide an error byte the device reported + # just before dropping the connection; surface that as the + # real, non-retryable cause when it is available + try: + sock.settimeout(1.0) + check_error(recv_decode(sock, 1), None) + except (OSError, OTANetworkError) as probe_err: + _LOGGER.debug( + "No device error behind the send failure: %s", probe_err + ) + raise OTANetworkError(f"sending data: {err}") from err - try: - sock.sendall(chunk) if version >= OTA_VERSION_2_0: - receive_exactly(sock, 1, "chunk result", RESPONSE_CHUNK_OK) - except OSError as err: - sys.stderr.write("\n") - raise OTAError(f"sending data: {err}") from err + try: + receive_exactly(sock, 1, "chunk result", RESPONSE_CHUNK_OK) + except OTANetworkError as err: + if offset < upload_size: + raise + # The device already had the complete image when this ack + # was lost, so it may be committing; do not retry + raise _committed_error(err) from err - progress.update(offset / upload_size) + progress.update(offset / upload_size) + except OTAError: + # Terminate the progress bar line before the error is logged + progress.done() + raise progress.done() # Enable nodelay for last checks @@ -472,11 +524,25 @@ def perform_ota( _LOGGER.info("Upload took %.2f seconds, waiting for result...", duration) - receive_exactly(sock, 1, "update receive result", RESPONSE_RECEIVE_OK) - receive_exactly(sock, 1, "update end result", RESPONSE_UPDATE_END_OK) - send_check(sock, RESPONSE_OK, "end acknowledgement") + # Once the device has the complete image it commits the update and + # reboots on its own; the exact commit point is not observable from + # here, so treat everything past the data phase as non-retryable. A + # re-upload could flash a device that already updated successfully. + try: + receive_exactly(sock, 1, "update receive result", RESPONSE_RECEIVE_OK) + receive_exactly(sock, 1, "update end result", RESPONSE_UPDATE_END_OK) + except OTANetworkError as err: + raise _committed_error(err) from err - _LOGGER.info("OTA successful") + try: + send_check(sock, RESPONSE_OK, "end acknowledgement") + except OTANetworkError as err: + # The device treats a missing end acknowledgement as non-fatal and is + # already rebooting into the new firmware, so the update succeeded + _LOGGER.warning("Failed sending end acknowledgement: %s", err) + _LOGGER.info("OTA successful (end acknowledgement not delivered)") + else: + _LOGGER.info("OTA successful") # Do not connect logs until it is fully on time.sleep(1) @@ -510,8 +576,33 @@ def run_ota_impl_( ) raise OTAError(err) from err - for r in res: - af, socktype, _, _, sa = r + if not res: + _LOGGER.error("No addresses to connect to for %s", remote_host) + return 1, None + + # Every address is tried at least once and EXTRA_UPLOAD_ATTEMPTS retries + # are shared across the addresses, cycling through them. Wait before an + # attempt when the previous one actually reached the device, or when + # revisiting an address, so a flaky link can recover and the device can + # clean up a half-open connection (its handshake watchdog runs at 20s); + # moving on to the next address family stays immediate. Known limitation: + # a silent mid-transfer drop with no reset can wedge the device until its + # 90s data timeout, which outlasts this budget; the retries target the + # common failures where the device resets or closes the link promptly. + total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS + last_error = "" + reached_device = False + for attempt in range(total_attempts): + af, socktype, _, _, sa = res[attempt % len(res)] + if reached_device or attempt >= len(res): + _LOGGER.info( + "Retrying in %.0f seconds (attempt %d of %d)...", + UPLOAD_RETRY_DELAY, + attempt + 1, + total_attempts, + ) + time.sleep(UPLOAD_RETRY_DELAY) + reached_device = False _LOGGER.info("Connecting to %s port %s...", sa[0], sa[1]) sock = socket.socket(af, socktype) sock.settimeout(20.0) @@ -519,23 +610,30 @@ def run_ota_impl_( sock.connect(sa) except OSError as err: sock.close() - _LOGGER.error("Connecting to %s port %s failed: %s", sa[0], sa[1], err) + _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) + last_error = f"connecting to {sa[0]} failed: {err}" continue _LOGGER.info("Connected to %s", sa[0]) - with Path(filename).open("rb") as file_handle: + reached_device = True + with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: perform_ota(sock, password, file_handle, filename, ota_type) + except OTANetworkError as err: + # Transient network failure; retry + last_error = str(err) + _LOGGER.warning("%s", last_error) + continue except OTAError as err: + # Device-reported error (wrong password, wrong flash size, ...); + # retrying cannot succeed, so fail immediately _LOGGER.error(str(err)) return 1, None - finally: - sock.close() # Successfully uploaded to sa[0] return 0, sa[0] - _LOGGER.error("Connection failed.") + _LOGGER.error("Upload failed after %d attempts: %s", total_attempts, last_error) return 1, None diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 9413fbcf29..db4a4b1117 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -44,13 +44,17 @@ def mock_file() -> io.BytesIO: @pytest.fixture -def mock_time() -> Generator[None]: +def mock_sleep() -> Generator[Mock]: + """Mock time.sleep so delays don't slow down tests.""" + with patch("time.sleep") as mock: + yield mock + + +@pytest.fixture +def mock_time(mock_sleep: Mock) -> Generator[None]: """Mock time-related functions for consistent testing.""" # Provide enough values for multiple calls (tests may call perform_ota multiple times) - with ( - patch("time.sleep"), - patch("time.perf_counter", side_effect=[0, 1, 0, 1, 0, 1]), - ): + with patch("time.perf_counter", side_effect=[0, 1, 0, 1, 0, 1]): yield @@ -79,6 +83,28 @@ def mock_resolve_ip() -> Generator[Mock]: yield mock +DUAL_STACK_SA6 = ("2001:db8::1", 3232, 0, 0) +DUAL_STACK_SA4 = ("192.168.1.100", 3232) + + +@pytest.fixture +def mock_resolve_ip_dual(mock_resolve_ip: Mock) -> Mock: + """Make resolve_ip_address return an IPv6 and an IPv4 address.""" + mock_resolve_ip.return_value = [ + (socket.AF_INET6, socket.SOCK_STREAM, 0, "", DUAL_STACK_SA6), + (socket.AF_INET, socket.SOCK_STREAM, 0, "", DUAL_STACK_SA4), + ] + return mock_resolve_ip + + +@pytest.fixture +def firmware_file(tmp_path: Path) -> Path: + """Create a firmware file on disk for run_ota_impl_ tests.""" + firmware = tmp_path / "firmware.bin" + firmware.write_bytes(b"firmware content") + return firmware + + @pytest.fixture def mock_perform_ota() -> Generator[Mock]: """Mock perform_ota function for testing.""" @@ -137,9 +163,11 @@ def test_receive_exactly_with_error_response(mock_socket: Mock) -> None: with pytest.raises( espota2.OTAError, match="receiving auth:.*Authentication invalid" - ): + ) as exc_info: espota2.receive_exactly(mock_socket, 1, "auth", [espota2.RESPONSE_OK]) + # Device-reported errors must stay plain OTAError, not the retryable kind + assert not isinstance(exc_info.value, espota2.OTANetworkError) mock_socket.close.assert_called_once() @@ -147,10 +175,30 @@ def test_receive_exactly_socket_error(mock_socket: Mock) -> None: """Test receive_exactly handles socket errors.""" mock_socket.recv.side_effect = OSError("Connection reset") - with pytest.raises(espota2.OTAError, match="receiving test response"): + with pytest.raises(espota2.OTANetworkError, match="receiving test response"): espota2.receive_exactly(mock_socket, 1, "test", espota2.RESPONSE_OK) +def test_receive_exactly_mid_read_socket_error(mock_socket: Mock) -> None: + """Test receive_exactly handles socket errors after the first byte.""" + mock_socket.recv.side_effect = [b"\x00", OSError("Connection reset")] + + with pytest.raises(espota2.OTANetworkError, match="receiving test:"): + espota2.receive_exactly(mock_socket, 3, "test", espota2.RESPONSE_OK) + + +def test_receive_exactly_closed_connection_is_network_error(mock_socket: Mock) -> None: + """Test receive_exactly raises OTANetworkError when the device closes the connection.""" + mock_socket.recv.return_value = b"" + + with pytest.raises( + espota2.OTANetworkError, match="Device closed connection without responding" + ): + espota2.receive_exactly(mock_socket, 1, "test", espota2.RESPONSE_OK) + + mock_socket.close.assert_called_once() + + @pytest.mark.parametrize( ("error_code", "expected_msg"), [ @@ -227,15 +275,15 @@ def test_check_error_unexpected_response() -> None: def test_check_error_empty_data() -> None: - """Test check_error raises error when device closes connection without responding.""" + """Test check_error raises the retryable OTANetworkError when the device closes the connection.""" with pytest.raises( - espota2.OTAError, match="Device closed connection without responding" + espota2.OTANetworkError, match="Device closed connection without responding" ): espota2.check_error([], [espota2.RESPONSE_OK]) # Also test with empty bytes with pytest.raises( - espota2.OTAError, match="Device closed connection without responding" + espota2.OTANetworkError, match="Device closed connection without responding" ): espota2.check_error(b"", [espota2.RESPONSE_OK]) @@ -530,6 +578,144 @@ def test_perform_ota_upload_error(mock_socket: Mock, mock_file: io.BytesIO) -> N espota2.perform_ota(mock_socket, None, mock_file, "test.bin") +def _no_auth_handshake(version: int) -> list[bytes]: + """Recv responses for a handshake without auth, up to the MD5 check.""" + return [ + bytes([espota2.RESPONSE_OK]), # First byte of version response + bytes([version]), # Version number + bytes([espota2.RESPONSE_HEADER_OK]), # Features response + bytes([espota2.RESPONSE_AUTH_OK]), # No auth required + bytes([espota2.RESPONSE_UPDATE_PREPARE_OK]), # Binary size OK + bytes([espota2.RESPONSE_BIN_MD5_OK]), # MD5 checksum OK + ] + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_chunk_send_error(mock_socket: Mock, mock_file: io.BytesIO) -> None: + """Test OTA raises the retryable OTANetworkError when sending a chunk fails.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_2_0), + OSError("Connection reset"), # Probe for a pending error byte fails too + ] + # Sends before the data phase: magic bytes, features, binary size, MD5; + # fail on the fifth sendall, the first firmware chunk + mock_socket.sendall.side_effect = [None] * 4 + [OSError("Broken pipe")] + + with pytest.raises(espota2.OTANetworkError, match="sending data:"): + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_chunk_send_error_surfaces_device_error( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a device error byte pending behind a send failure becomes the cause.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_ERROR_WRITING_FLASH]), # Reason the device closed + ] + mock_socket.sendall.side_effect = [None] * 4 + [OSError("Broken pipe")] + + with pytest.raises( + espota2.OTAError, match="Writing OTA data to flash memory failed" + ) as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # The device-reported error is not retryable + assert not isinstance(exc.value, espota2.OTANetworkError) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_final_chunk_ack_failure_not_retryable( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a lost ack for the final chunk is not retried.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_2_0), + OSError("Connection reset"), # Ack for the only (final) chunk is lost + ] + + with pytest.raises(espota2.OTAError, match="receiving chunk result") as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # The device already had the whole image, so it may be committing + assert not isinstance(exc.value, espota2.OTANetworkError) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_intermediate_chunk_ack_failure_retryable( + mock_socket: Mock, +) -> None: + """Test a lost ack for a non-final chunk stays retryable.""" + # Two chunks: the firmware is larger than one upload block + big_file = io.BytesIO(b"x" * (espota2.UPLOAD_BLOCK_SIZE + 1)) + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_2_0), + OSError("Connection reset"), # Ack for the first of two chunks is lost + ] + + with pytest.raises(espota2.OTANetworkError, match="receiving chunk result"): + espota2.perform_ota(mock_socket, None, big_file, "test.bin") + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_post_commit_failure_not_retryable( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a network failure after the device committed is a plain OTAError.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_RECEIVE_OK]), # Device received everything + OSError("Connection reset"), # Connection lost waiting for end result + ] + + with pytest.raises(espota2.OTAError, match="receiving update end result") as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # Must not be the retryable kind; the device is already rebooting + assert not isinstance(exc.value, espota2.OTANetworkError) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_md5_mismatch_not_marked_committed( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test an MD5 mismatch keeps its own message and stays non-retryable.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_RECEIVE_OK]), # Device received everything + bytes([espota2.RESPONSE_ERROR_MD5_MISMATCH]), # Device aborted the update + ] + + with pytest.raises(espota2.OTAError, match="MD5 code mismatch") as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # The device aborted without committing, so the message must not claim + # the update may have been installed, and the error must not be retried + assert not isinstance(exc.value, espota2.OTANetworkError) + assert "committed" not in str(exc.value) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_end_ack_send_failure_is_success( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a send failure on the final acknowledgement does not fail the OTA.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_RECEIVE_OK]), # Device received everything + bytes([espota2.RESPONSE_UPDATE_END_OK]), # Update committed + ] + # Sends: magic bytes, features, binary size, MD5, one firmware chunk; + # fail on the sixth sendall, the end acknowledgement + mock_socket.sendall.side_effect = [None] * 5 + [OSError("Broken pipe")] + + # Must not raise; the device treats a missing acknowledgement as non-fatal + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + assert mock_socket.sendall.call_count == 6 + + @pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") def test_run_ota_impl_successful( mock_socket: Mock, tmp_path: Path, mock_perform_ota: Mock @@ -564,21 +750,183 @@ def test_run_ota_impl_successful( @pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") -def test_run_ota_impl_connection_failed(mock_socket: Mock, tmp_path: Path) -> None: - """Test run_ota_impl_ when connection fails.""" +def test_run_ota_impl_connection_failed( + mock_socket: Mock, firmware_file: Path, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ retries when connection fails and eventually gives up.""" mock_socket.connect.side_effect = OSError("Connection refused") - # Create a real firmware file - firmware_file = tmp_path / "firmware.bin" - firmware_file.write_bytes(b"firmware content") - result_code, result_host = espota2.run_ota_impl_( "test.local", 3232, "password", str(firmware_file) ) assert result_code == 1 assert result_host is None - mock_socket.close.assert_called_once() + # A single address gets the whole attempt budget, with a delay before + # each revisit + assert mock_socket.connect.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + 1 + assert mock_socket.close.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + 1 + assert mock_sleep.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + mock_sleep.assert_called_with(espota2.UPLOAD_RETRY_DELAY) + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_connect_retry_succeeds( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ succeeds when a retry connects after a failed attempt.""" + mock_socket.connect.side_effect = [OSError("Connection timed out"), None] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + assert mock_socket.connect.call_count == 2 + mock_sleep.assert_called_once_with(espota2.UPLOAD_RETRY_DELAY) + mock_perform_ota.assert_called_once() + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_network_error_retry_succeeds( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ retries after a network error during the upload.""" + mock_perform_ota.side_effect = [ + espota2.OTANetworkError("receiving features: Device closed connection"), + None, + ] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + assert mock_perform_ota.call_count == 2 + mock_sleep.assert_called_once_with(espota2.UPLOAD_RETRY_DELAY) + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_network_error_exhausts_attempts( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ gives up after all attempts hit network errors.""" + mock_perform_ota.side_effect = espota2.OTANetworkError("sending data: broken pipe") + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + assert mock_perform_ota.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + 1 + assert mock_sleep.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip_dual") +def test_run_ota_impl_multiple_addresses_cycle( + mock_socket: Mock, firmware_file: Path, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ visits every address and cycles for the retries.""" + mock_socket.connect.side_effect = OSError("No route to host") + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + # Each address is visited once, then the EXTRA_UPLOAD_ATTEMPTS spare + # attempts cycle back through them; the budget is shared, not per address + assert mock_socket.connect.call_args_list == [ + call(DUAL_STACK_SA6), + call(DUAL_STACK_SA4), + call(DUAL_STACK_SA6), + call(DUAL_STACK_SA4), + ] + # No connect ever reached the device, so the delay only applies before + # the revisits + assert mock_sleep.call_count == 2 + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip_dual") +def test_run_ota_impl_second_address_succeeds_without_delay( + mock_socket: Mock, + firmware_file: Path, + mock_perform_ota: Mock, + mock_sleep: Mock, +) -> None: + """Test run_ota_impl_ falls through to the next address with no pause.""" + mock_socket.connect.side_effect = [OSError("No route to host"), None] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + mock_sleep.assert_not_called() + mock_perform_ota.assert_called_once() + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip_dual") +def test_run_ota_impl_pauses_after_reaching_device( + mock_socket: Mock, + firmware_file: Path, + mock_perform_ota: Mock, + mock_sleep: Mock, +) -> None: + """Test run_ota_impl_ pauses before the next address once the device was reached.""" + mock_perform_ota.side_effect = [ + espota2.OTANetworkError("sending data: connection reset"), + None, + ] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + # The first attempt reached the device, so the next one waits first even + # though it targets a fresh address + mock_sleep.assert_called_once_with(espota2.UPLOAD_RETRY_DELAY) + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_device_error_not_retried( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ fails immediately on a device-reported error.""" + mock_perform_ota.side_effect = espota2.OTAError( + "Authentication invalid. Is the password correct?" + ) + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + mock_perform_ota.assert_called_once() + mock_sleep.assert_not_called() + + +def test_run_ota_impl_no_addresses( + firmware_file: Path, mock_resolve_ip: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ fails cleanly when resolution yields no addresses.""" + mock_resolve_ip.return_value = [] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + mock_sleep.assert_not_called() def test_run_ota_impl_resolve_failed(tmp_path: Path, mock_resolve_ip: Mock) -> None: From 945c2458b3642964232503bf162bb9d1ab657d8a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:31:19 -0500 Subject: [PATCH 025/160] [core] Load component aliases from a generated registry (#18335) --- .github/workflows/ci.yml | 1 + esphome/component_aliases.py | 10 ++++++ esphome/loader.py | 61 +++++++++++++-------------------- script/build_alias_registry.py | 59 +++++++++++++++++++++++++++++++ tests/unit_tests/test_loader.py | 28 +++++++++++++++ 5 files changed, 122 insertions(+), 37 deletions(-) create mode 100644 esphome/component_aliases.py create mode 100755 script/build_alias_registry.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e695bb46b..b603e68ad7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,6 +179,7 @@ jobs: . venv/bin/activate script/ci-custom.py script/build_codeowners.py --check + script/build_alias_registry.py --check script/build_language_schema.py --check script/generate-esp32-boards.py --check script/generate-rp2-boards.py --check diff --git a/esphome/component_aliases.py b/esphome/component_aliases.py new file mode 100644 index 0000000000..e701bd98d4 --- /dev/null +++ b/esphome/component_aliases.py @@ -0,0 +1,10 @@ +"""Component alias registry. + +Generated by script/build_alias_registry.py - do not edit manually. +See the component-alias section of esphome/loader.py. +""" + +# alias -> (canonical component, removal version or None) +COMPONENT_ALIASES: dict[str, tuple[str, str | None]] = { + "rp2040": ("rp2", "2027.7.0"), +} diff --git a/esphome/loader.py b/esphome/loader.py index 7a659aa0a8..f994f0c5eb 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -269,10 +269,9 @@ def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None: # If `domain` is the legacy name of a renamed component, redirect to the # canonical module so the rest of the loader (and every caller of # `get_component(legacy)`) transparently sees the new component. - alias_map = _get_alias_map() - if domain in alias_map: - canonical = alias_map[domain] - manif = _lookup_module(canonical, exception) + alias_meta = get_alias_metadata().get(domain) + if alias_meta is not None: + manif = _lookup_module(alias_meta.canonical, exception) if manif is not None: _COMPONENT_CACHE[domain] = manif return manif @@ -329,8 +328,10 @@ def _replace_component_manifest(domain: str, manifest: ComponentManifest) -> Non # --------------------------------------------------------------------------- # # A component can declare ``ALIASES = ["legacy_name"]`` (and optionally -# ``ALIAS_REMOVAL_VERSION = "YYYY.M.0"``) in its ``__init__.py``. Two -# integrations are then wired up automatically: +# ``ALIAS_REMOVAL_VERSION = "YYYY.M.0"``) in its ``__init__.py``, then run +# ``script/build_alias_registry.py`` to regenerate +# ``esphome/component_aliases.py`` (CI and a unit test fail if the registry +# is stale). Two integrations are then wired up automatically: # # 1. **Python imports** — a ``sys.meta_path`` finder (``_AliasFinder``) # intercepts ``esphome.components.``/``....`` @@ -344,13 +345,13 @@ def _replace_component_manifest(domain: str, manifest: ComponentManifest) -> Non # dependency checks, schema validation and codegen all see only the # canonical name. # -# Both lookups are populated by ``_build_alias_map``, which **AST-parses** -# every component's ``__init__.py`` rather than importing it. That keeps the -# cost low: scanning ~400 components on disk takes ~5 ms instead of the -# multi-second cost of executing every component's import side-effects. +# Both lookups read the checked-in registry in ``esphome.component_aliases`` +# (generated by ``script/build_alias_registry.py``, verified in CI), so no +# component-directory scan happens at runtime. ``_build_alias_map`` below is +# the generator's scan implementation; it **AST-parses** each component's +# ``__init__.py`` rather than importing it. -_ALIAS_MAP_CACHE: dict[str, str] | None = None _ALIAS_META_CACHE: dict[str, "AliasMeta"] | None = None @@ -367,31 +368,17 @@ class AliasMeta: removal_version: str | None -def _ensure_alias_caches() -> None: - """Populate both alias caches from a single directory scan. - - ``_build_alias_map`` returns both maps together, so building them in one - shot avoids scanning every component's ``__init__.py`` twice when a run - needs both the canonical map (loader) and the metadata map (config - pre-pass). - """ - global _ALIAS_MAP_CACHE, _ALIAS_META_CACHE - if _ALIAS_MAP_CACHE is None or _ALIAS_META_CACHE is None: - _ALIAS_MAP_CACHE, _ALIAS_META_CACHE = _build_alias_map() - - -def _get_alias_map() -> dict[str, str]: - """Return the legacy-name → canonical-name map, building it lazily.""" - _ensure_alias_caches() - return _ALIAS_MAP_CACHE - - def get_alias_metadata() -> dict[str, AliasMeta]: - """Return the legacy-name → :class:`AliasMeta` map (cached). + """Return the legacy-name → :class:`AliasMeta` map, built lazily from + the generated registry.""" + global _ALIAS_META_CACHE # noqa: PLW0603 + if _ALIAS_META_CACHE is None: + from esphome.component_aliases import COMPONENT_ALIASES - Used by the YAML pre-pass to format a per-alias deprecation warning. - """ - _ensure_alias_caches() + _ALIAS_META_CACHE = { + alias: AliasMeta(canonical=canonical, removal_version=removal_version) + for alias, (canonical, removal_version) in COMPONENT_ALIASES.items() + } return _ALIAS_META_CACHE @@ -537,11 +524,11 @@ class _AliasFinder(importlib.abc.MetaPathFinder): # least three parts, so ``parts[2]`` (the domain) always exists. parts = fullname.split(".") domain = parts[2] - alias_map = _get_alias_map() - if domain not in alias_map: + alias_meta = get_alias_metadata().get(domain) + if alias_meta is None: return None - parts[2] = alias_map[domain] + parts[2] = alias_meta.canonical canonical_fullname = ".".join(parts) try: canonical_module = importlib.import_module(canonical_fullname) diff --git a/script/build_alias_registry.py b/script/build_alias_registry.py new file mode 100755 index 0000000000..e007c075eb --- /dev/null +++ b/script/build_alias_registry.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Generate esphome/component_aliases.py from component ALIASES declarations. + +Run without arguments to regenerate the registry; ``--check`` (run in CI) +verifies it is up to date. +""" + +import argparse +from pathlib import Path +import sys + +# The root directory of the repo +root = Path(__file__).parent.parent +# Make the repo's esphome package win over any installed copy +sys.path.insert(0, str(root)) + +from esphome.helpers import write_file_if_changed # noqa: E402 +from esphome.loader import _build_alias_map # noqa: E402 + +parser = argparse.ArgumentParser() +parser.add_argument( + "--check", + help="Check if the alias registry is up to date.", + action="store_true", +) +args = parser.parse_args() + +registry_file = root / "esphome" / "component_aliases.py" + +HEADER = '''"""Component alias registry. + +Generated by script/build_alias_registry.py - do not edit manually. +See the component-alias section of esphome/loader.py. +""" + +# alias -> (canonical component, removal version or None) +COMPONENT_ALIASES: dict[str, tuple[str, str | None]] = { +''' + +# _build_alias_map scans the real component tree and already rejects +# duplicate and shadowing aliases with an EsphomeError. +_, alias_meta = _build_alias_map() + +lines = [HEADER] +for alias, meta in sorted(alias_meta.items()): + removal = f'"{meta.removal_version}"' if meta.removal_version else "None" + lines.append(f' "{alias}": ("{meta.canonical}", {removal}),\n') +lines.append("}\n") +content = "".join(lines) + +if args.check: + if registry_file.read_text(encoding="utf-8") != content: + print("Component alias registry is not up to date.") + print("Please run `script/build_alias_registry.py`") + sys.exit(1) + print("Component alias registry is up to date") +else: + write_file_if_changed(registry_file, content) + print(f"Wrote {registry_file}") diff --git a/tests/unit_tests/test_loader.py b/tests/unit_tests/test_loader.py index 41dd462678..74515e9d4c 100644 --- a/tests/unit_tests/test_loader.py +++ b/tests/unit_tests/test_loader.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch import pytest +from esphome.component_aliases import COMPONENT_ALIASES from esphome.loader import ( AliasMeta, ComponentManifest, @@ -481,6 +482,33 @@ def test_real_alias_map_includes_rp2040() -> None: assert meta["rp2040"].removal_version == "2027.7.0" +def test_alias_registry_matches_component_tree() -> None: + """The checked-in registry must match a live scan of the component tree.""" + _, meta_map = _build_alias_map() + expected = { + alias: (meta.canonical, meta.removal_version) + for alias, meta in meta_map.items() + } + assert expected == COMPONENT_ALIASES, ( + "esphome/component_aliases.py is out of date; " + "run script/build_alias_registry.py" + ) + + +def test_alias_map_built_from_registry() -> None: + """The runtime alias map comes from the generated registry, not a scan.""" + with ( + patch( + "esphome.component_aliases.COMPONENT_ALIASES", + {"legacy": ("modern", "2099.1.0")}, + ), + patch("esphome.loader._ALIAS_META_CACHE", None), + ): + assert get_alias_metadata() == { + "legacy": AliasMeta(canonical="modern", removal_version="2099.1.0") + } + + def test_get_component_resolves_alias() -> None: """``get_component('rp2040')`` should return the rp2 manifest — every caller of the loader (dep checker, schema validator, codegen) hits From 37782f72069e10f0d6a32c79a46a4d58180cb240 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:54:37 -0500 Subject: [PATCH 026/160] Bump prek from 0.4.12 to 0.4.13 (#18362) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 1832ffd433..95ee97437d 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.2 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.4.12 # also change in .github/workflows/ci.yml when updating +prek==0.4.13 # also change in .github/workflows/ci.yml when updating # Unit tests pytest==9.1.1 From 45a056e33774333778e0264222d4434aac434a36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:54:51 -0500 Subject: [PATCH 027/160] Bump platformdirs from 4.11.1 to 4.11.2 (#18363) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 683008400a..6bc8bdf74a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==2.1.1 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.1 # native esp-idf toolchain global cache dir +platformdirs==4.11.2 # native esp-idf toolchain global cache dir filelock==3.32.2 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this From dd51624fbb909ccaa902e8d38480b91b782fb6ae Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 20:06:48 -0500 Subject: [PATCH 028/160] [core] Partially revert "Hash entity keys from the raw name to fix collisions" (#18361) --- esphome/components/api/api_connection.cpp | 6 +- esphome/components/infrared/infrared.cpp | 8 +- esphome/components/mqtt/__init__.py | 63 --- esphome/components/prometheus/__init__.py | 6 - .../radio_frequency/radio_frequency.cpp | 8 +- .../template/text/template_text.cpp | 20 +- .../components/template/text/template_text.h | 15 +- esphome/core/application.h | 8 +- esphome/core/entity_base.cpp | 52 +-- esphome/core/entity_base.h | 80 ++-- esphome/core/entity_helpers.py | 190 ++++---- esphome/core/helpers.h | 29 +- esphome/core/preference_backend.h | 14 +- esphome/core/preferences.cpp | 25 - esphome/core/preferences.h | 12 - esphome/helpers.py | 15 +- tests/integration/entity_utils.py | 27 +- .../fixtures/fnv1_hash_object_id.yaml | 32 -- .../fixtures/multi_device_preferences.yaml | 21 +- .../fixtures/preference_key_migration.yaml | 35 -- tests/integration/host_prefs.py | 24 +- tests/integration/test_fnv1_hash_object_id.py | 4 - .../test_object_id_api_verification.py | 10 +- ...t_object_id_friendly_name_no_mac_suffix.py | 4 +- .../test_object_id_no_friendly_name.py | 6 +- .../test_preference_key_migration.py | 165 ------- tests/unit_tests/components/mqtt/__init__.py | 0 .../mqtt/test_object_id_conflicts.py | 239 ---------- tests/unit_tests/core/test_entity_helpers.py | 432 ++++++++++-------- .../object_id_conflict_mqtt.yaml | 22 - .../object_id_conflict_no_mqtt.yaml | 15 - .../test_preference_hash_stability.py | 34 +- 32 files changed, 489 insertions(+), 1132 deletions(-) delete mode 100644 esphome/core/preferences.cpp delete mode 100644 tests/integration/fixtures/preference_key_migration.yaml delete mode 100644 tests/integration/test_preference_key_migration.py delete mode 100644 tests/unit_tests/components/mqtt/__init__.py delete mode 100644 tests/unit_tests/components/mqtt/test_object_id_conflicts.py delete mode 100644 tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml delete mode 100644 tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index d05f98d03b..73b4f3e5bd 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -448,7 +448,7 @@ void APIConnection::on_disconnect_response() { uint16_t APIConnection::fill_and_encode_entity_state(EntityBase *entity, StateResponseProtoMessage &msg, CalculateSizeFn size_fn, MessageEncodeFn encode_fn, APIConnection *conn, uint32_t remaining_size) { - msg.key = entity->get_entity_key(); + msg.key = entity->get_object_id_hash(); #ifdef USE_DEVICES msg.device_id = entity->get_device_id(); #endif @@ -459,7 +459,7 @@ uint16_t APIConnection::fill_and_encode_entity_info(EntityBase *entity, InfoResp CalculateSizeFn size_fn, MessageEncodeFn encode_fn, APIConnection *conn, uint32_t remaining_size) { // Set common fields that are shared by all entity types - msg.key = entity->get_entity_key(); + msg.key = entity->get_object_id_hash(); if (entity->has_own_name()) { msg.name = entity->get_name(); @@ -1149,7 +1149,7 @@ void APIConnection::try_send_camera_image_() { bool done = this->image_reader_->available() == to_send; CameraImageResponse msg; - msg.key = camera::Camera::instance()->get_entity_key(); + msg.key = camera::Camera::instance()->get_object_id_hash(); msg.set_data(this->image_reader_->peek_data_buffer(), to_send); msg.done = done; #ifdef USE_DEVICES diff --git a/esphome/components/infrared/infrared.cpp b/esphome/components/infrared/infrared.cpp index 288b1e5c40..9b97995a96 100644 --- a/esphome/components/infrared/infrared.cpp +++ b/esphome/components/infrared/infrared.cpp @@ -154,8 +154,12 @@ bool Infrared::on_receive(remote_base::RemoteReceiveData data) { // Forward received IR data to API server #if defined(USE_API) && defined(USE_IR_RF) if (api::global_api_server != nullptr) { - api::global_api_server->send_infrared_rf_receive_event(this->get_device_id_or_zero(), this->get_entity_key(), - &data.get_raw_data()); +#ifdef USE_DEVICES + uint32_t device_id = this->get_device_id(); +#else + uint32_t device_id = 0; +#endif + api::global_api_server->send_infrared_rf_receive_event(device_id, this->get_object_id_hash(), &data.get_raw_data()); } #endif return false; // Don't consume the event, allow other listeners to process it diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index 713969ab88..98ca23b60b 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -63,7 +63,6 @@ from esphome.const import ( PlatformFramework, ) from esphome.core import CORE, CoroPriority, coroutine_with_priority -from esphome.core.entity_helpers import ObjectIdEntity, validate_no_object_id_conflicts from esphome.types import ConfigType DEPENDENCIES = ["network"] @@ -333,68 +332,6 @@ CONFIG_SCHEMA = cv.All( ) -# Platforms whose MQTT components subscribe to an object_id-derived command topic. -# Keep in sync with the platforms extending cv.MQTT_COMMAND_COMPONENT_SCHEMA, plus -# text, whose MQTT component subscribes a command topic that cannot be overridden. -_COMMAND_TOPIC_PLATFORMS = frozenset( - { - "alarm_control_panel", - "button", - "climate", - "cover", - "datetime", - "fan", - "light", - "lock", - "number", - "select", - "switch", - "text", - "update", - "valve", - } -) - - -# Platforms whose MQTT components derive extra sub-topics (position/command, -# mode/command, speed/command, ...) from the object_id, each with its own config -# key; custom state and command topics cannot exempt them from conflicting. -_SUB_TOPIC_PLATFORMS = frozenset({"climate", "cover", "fan", "valve"}) - - -def _topics_conflict(entities: list[ObjectIdEntity], config: ConfigType) -> bool: - """Check whether more than one entity actually uses an object_id-derived topic. - - An empty topic_prefix disables default topics entirely, custom state and - command topics avoid the default topics, and disabling discovery (globally - or per entity) avoids the discovery config topic. - """ - if config[CONF_TOPIC_PREFIX]: - platform = entities[0].platform - if platform in _SUB_TOPIC_PLATFORMS: - return True - if sum(CONF_STATE_TOPIC not in entity.config for entity in entities) > 1: - return True - if ( - platform in _COMMAND_TOPIC_PLATFORMS - and sum(CONF_COMMAND_TOPIC not in entity.config for entity in entities) > 1 - ): - return True - if not config[CONF_DISCOVERY]: - return False - discovery_entities = sum( - entity.config.get(CONF_DISCOVERY, True) for entity in entities - ) - return discovery_entities > 1 - - -FINAL_VALIDATE_SCHEMA = validate_no_object_id_conflicts( - "mqtt builds default topics and discovery topics from the entity object_id, " - "which is the name converted to ASCII", - conflict_filter=_topics_conflict, -) - - def exp_mqtt_message(config): if config is None: return cg.optional(cg.TemplateArguments(MQTTMessage)) diff --git a/esphome/components/prometheus/__init__.py b/esphome/components/prometheus/__init__.py index 0a69160fc1..cc1541ce80 100644 --- a/esphome/components/prometheus/__init__.py +++ b/esphome/components/prometheus/__init__.py @@ -3,7 +3,6 @@ from esphome.components import web_server_base from esphome.components.web_server_base import CONF_WEB_SERVER_BASE_ID import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_INCLUDE_INTERNAL, CONF_NAME, CONF_RELABEL -from esphome.core.entity_helpers import validate_no_object_id_conflicts from esphome.cpp_types import EntityBase AUTO_LOAD = ["web_server_base"] @@ -36,11 +35,6 @@ CONFIG_SCHEMA = cv.Schema( }, ).extend(cv.COMPONENT_SCHEMA) -FINAL_VALIDATE_SCHEMA = validate_no_object_id_conflicts( - "prometheus builds metric labels from the entity object_id, " - "which is the name converted to ASCII" -) - async def to_code(config): paren = await cg.get_variable(config[CONF_WEB_SERVER_BASE_ID]) diff --git a/esphome/components/radio_frequency/radio_frequency.cpp b/esphome/components/radio_frequency/radio_frequency.cpp index fe6c6a9cb5..3e0a905737 100644 --- a/esphome/components/radio_frequency/radio_frequency.cpp +++ b/esphome/components/radio_frequency/radio_frequency.cpp @@ -99,8 +99,12 @@ bool RadioFrequency::on_receive(remote_base::RemoteReceiveData data) { // Forward received RF data to API server #if defined(USE_API) && defined(USE_RADIO_FREQUENCY) if (api::global_api_server != nullptr) { - api::global_api_server->send_infrared_rf_receive_event(this->get_device_id_or_zero(), this->get_entity_key(), - &data.get_raw_data()); +#ifdef USE_DEVICES + uint32_t device_id = this->get_device_id(); +#else + uint32_t device_id = 0; +#endif + api::global_api_server->send_infrared_rf_receive_event(device_id, this->get_object_id_hash(), &data.get_raw_data()); } #endif return false; // Don't consume the event, allow other listeners to process it diff --git a/esphome/components/template/text/template_text.cpp b/esphome/components/template/text/template_text.cpp index ffe11cf229..af134e6ed4 100644 --- a/esphome/components/template/text/template_text.cpp +++ b/esphome/components/template/text/template_text.cpp @@ -20,14 +20,18 @@ void TemplateText::setup() { // Need std::string for pref_->setup() to fill from flash std::string value{this->initial_value_ != nullptr ? this->initial_value_ : ""}; - uint32_t extra = 0; - extra += this->traits.get_min_length() << 2; - extra += this->traits.get_max_length() << 4; - extra += fnv1_hash(this->traits.get_pattern_c_str()) << 6; - // TextSaver::setup() picks the key for the platform and migrates old data once - uint32_t key = this->preference_key_base_() + extra; - uint32_t old_key = this->old_preference_key_base_() + extra; - this->pref_->setup(key, old_key, value); + // For future hash migration: use migrate_entity_preference_() with: + // old_key = get_preference_hash() + extra + // new_key = get_preference_hash_v2() + extra + // See: https://github.com/esphome/backlog/issues/85 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + uint32_t key = this->get_preference_hash(); +#pragma GCC diagnostic pop + key += this->traits.get_min_length() << 2; + key += this->traits.get_max_length() << 4; + key += fnv1_hash(this->traits.get_pattern_c_str()) << 6; + this->pref_->setup(key, value); if (!value.empty()) this->publish_state(value); } diff --git a/esphome/components/template/text/template_text.h b/esphome/components/template/text/template_text.h index beeea4396a..229a61d9b8 100644 --- a/esphome/components/template/text/template_text.h +++ b/esphome/components/template/text/template_text.h @@ -14,9 +14,7 @@ class TemplateTextSaverBase { public: virtual bool save(const std::string &value) { return true; } - /// old_id is the pre-2026.8.0 preference key; data stored under it is moved to id once. - /// See: https://github.com/esphome/backlog/issues/85 - virtual void setup(uint32_t id, uint32_t old_id, std::string &value) {} + virtual void setup(uint32_t id, std::string &value) {} protected: ESPPreferenceObject pref_; @@ -47,16 +45,11 @@ template class TextSaver : public TemplateTextSaverBase { // Make the preference object. Fill the provided location with the saved data // If it is available, else leave it alone - void setup(uint32_t id, uint32_t old_id, std::string &value) override { - char temp[SZ + 1]; -#ifdef USE_PREFERENCE_KEY_LOOKUP + void setup(uint32_t id, std::string &value) override { this->pref_ = global_preferences->make_preference(id); - bool hasdata = migrate_preference(this->pref_, reinterpret_cast(temp), SZ + 1, old_id, id); -#else - // Slot-based backends keep the old key; it is only a validity tag on a positional slot - this->pref_ = global_preferences->make_preference(old_id); + + char temp[SZ + 1]; bool hasdata = this->pref_.load(&temp); -#endif if (hasdata) { size_t len = static_cast(temp[0]); diff --git a/esphome/core/application.h b/esphome/core/application.h index a18a6b31c8..a12cdc4ac8 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -120,8 +120,8 @@ class Application { // NOLINTBEGIN(bugprone-macro-parentheses) #define ENTITY_TYPE_(type, singular, plural, count, upper) \ void register_##singular(type *obj) { this->plural##_.push_back(obj); } \ - void register_##singular(type *obj, const char *name, uint32_t entity_key, uint32_t entity_fields) { \ - obj->configure_entity_(name, entity_key, entity_fields); \ + void register_##singular(type *obj, const char *name, uint32_t object_id_hash, uint32_t entity_fields) { \ + obj->configure_entity_(name, object_id_hash, entity_fields); \ this->plural##_.push_back(obj); \ } #define ENTITY_CONTROLLER_TYPE_(type, singular, plural, count, upper, callback) \ @@ -329,7 +329,7 @@ class Application { #define GET_ENTITY_METHOD(entity_type, entity_name, entities_member) \ entity_type *get_##entity_name##_by_key(uint32_t key, uint32_t device_id, bool include_internal = false) { \ for (auto *obj : this->entities_member##_) { \ - if (obj->get_entity_key() == key && obj->get_device_id() == device_id && \ + if (obj->get_object_id_hash() == key && obj->get_device_id() == device_id && \ (include_internal || !obj->is_internal())) \ return obj; \ } \ @@ -340,7 +340,7 @@ class Application { #define GET_ENTITY_METHOD(entity_type, entity_name, entities_member) \ entity_type *get_##entity_name##_by_key(uint32_t key, bool include_internal = false) { \ for (auto *obj : this->entities_member##_) { \ - if (obj->get_entity_key() == key && (include_internal || !obj->is_internal())) \ + if (obj->get_object_id_hash() == key && (include_internal || !obj->is_internal())) \ return obj; \ } \ return nullptr; \ diff --git a/esphome/core/entity_base.cpp b/esphome/core/entity_base.cpp index 328de05302..fc6ac503b5 100644 --- a/esphome/core/entity_base.cpp +++ b/esphome/core/entity_base.cpp @@ -8,7 +8,7 @@ namespace esphome { static const char *const TAG = "entity_base"; -void EntityBase::configure_entity_(const char *name, uint32_t entity_key, uint32_t entity_fields) { +void EntityBase::configure_entity_(const char *name, uint32_t object_id_hash, uint32_t entity_fields) { this->name_ = StringRef(name); if (this->name_.empty()) { #ifdef USE_DEVICES @@ -30,15 +30,15 @@ void EntityBase::configure_entity_(const char *name, uint32_t entity_key, uint32 } } this->flags_.has_own_name = false; - // Dynamic name - must calculate key at runtime - this->calc_entity_key_(); + // Dynamic name - must calculate hash at runtime + this->calc_object_id_(); } else { this->flags_.has_own_name = true; - // Static name - use pre-computed key if provided - if (entity_key != 0) { - this->entity_key_ = entity_key; + // Static name - use pre-computed hash if provided + if (object_id_hash != 0) { + this->object_id_hash_ = object_id_hash; } else { - this->calc_entity_key_(); + this->calc_object_id_(); } } // Unpack entity string table indices and flags from entity_fields. @@ -147,15 +147,9 @@ std::string EntityBase::get_icon() const { } #endif // !USE_ESP8266 -// Calculate the entity key directly from the raw name (no transformations) -void EntityBase::calc_entity_key_() { this->entity_key_ = fnv1_hash_bytes(this->name_.c_str(), this->name_.size()); } - -// Reconstruct the OLD (pre-2026.8.0) object_id-based hash for preference key compatibility. -// Named entities historically used the hash pre-computed by Python code generation, which -// sanitized per UTF-8 code point; entities without their own name computed the hash at -// runtime per byte. See https://github.com/esphome/backlog/issues/85 -uint32_t EntityBase::calc_old_object_id_hash_() const { - return fnv1_hash_object_id(this->name_.c_str(), this->name_.size(), this->flags_.has_own_name); +// Calculate Object ID Hash directly from name using snake_case + sanitize +void EntityBase::calc_object_id_() { + this->object_id_hash_ = fnv1_hash_object_id(this->name_.c_str(), this->name_.size()); } size_t EntityBase::write_object_id_to(char *buf, size_t buf_size) const { @@ -173,22 +167,16 @@ StringRef EntityBase::get_object_id_to(std::span buf) c } ESPPreferenceObject EntityBase::make_entity_preference_(size_t size, uint32_t version) { - // The old key hashed the sanitized object_id, so multiple entity names could collide on - // one key and overwrite each other's stored preferences; the new key hashes the raw name. - // See: https://github.com/esphome/backlog/issues/85 - uint32_t old_key = this->old_preference_key_base_() ^ version; -#ifdef USE_PREFERENCE_KEY_LOOKUP - uint32_t new_key = this->preference_key_base_() ^ version; - auto pref = global_preferences->make_preference(size, new_key); - // All in-tree entity preferences fit the stack buffer, so migration never hits the heap - SmallBufferWithHeapFallback<64> buffer(size); - migrate_preference(pref, buffer.get(), size, old_key, new_key); - return pref; -#else - // Slot-based backends keep the old key: it is only a validity tag on a positional slot, - // so collisions cannot corrupt data there and keeping it preserves stored state. - return global_preferences->make_preference(size, old_key); -#endif + // The key hashes the sanitized object_id, so multiple entity names can collide on one + // key and overwrite each other's stored preferences ("Living Room" and "living_room", + // or two UTF-8 names that both sanitize to underscores). Keys hashed from the raw name + // fix this, but they change the entity key API clients track, which the Home Assistant + // esphome integration cannot handle yet. See: https://github.com/esphome/backlog/issues/85 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + uint32_t key = this->get_preference_hash() ^ version; +#pragma GCC diagnostic pop + return global_preferences->make_preference(size, key); } #ifdef USE_ENTITY_ICON diff --git a/esphome/core/entity_base.h b/esphome/core/entity_base.h index 7f8e5f2630..5f2e173d8d 100644 --- a/esphome/core/entity_base.h +++ b/esphome/core/entity_base.h @@ -73,17 +73,8 @@ class EntityBase { // Get whether this Entity has its own name or it should use the device friendly_name. bool has_own_name() const { return this->flags_.has_own_name; } - // Get the unique key of this Entity: FNV-1 hash of the raw entity name. - // This is the key sent to API clients and used to route entity state. - uint32_t get_entity_key() const { return this->entity_key_; } - - /// Returns the LEGACY object_id hash, unchanged from previous releases, so existing - /// callers keep getting stable values (for example preference keys). This is no longer - /// the key sent to API clients; that is get_entity_key(). - ESPDEPRECATED("Use get_entity_key() for the entity key sent to API clients, or " - "make_entity_preference() for preference storage. Will be removed in 2027.1.0.", - "2026.8.0") - uint32_t get_object_id_hash() const { return this->calc_old_object_id_hash_(); } + // Get the unique Object ID of this Entity + uint32_t get_object_id_hash() const { return this->object_id_hash_; } /// Get object_id with zero heap allocation /// For static case: returns StringRef to internal storage (buffer unused) @@ -190,23 +181,39 @@ class EntityBase { // Set has_state - for components that need to manually set this void set_has_state(bool state) { this->flags_.has_state = state; } - /// Get this entity's device id, or 0 when devices are not compiled in (main device). - uint32_t get_device_id_or_zero() const { -#ifdef USE_DEVICES - return this->get_device_id(); -#else - return 0; -#endif - } - - /// Get the LEGACY preference key: FNV-1 hash of the sanitized object_id, XOR device_id. - /// Intentionally keeps the old algorithm so external callers that store preferences under - /// this key keep stable keys; make_entity_preference() migrates to the new raw-name key, - /// this method never will. + /** + * @brief Get a unique hash for storing preferences/settings for this entity. + * + * This method returns a hash that uniquely identifies the entity for the purpose of + * storing preferences (such as calibration, state, etc.). Unlike get_object_id_hash(), + * this hash also incorporates the device_id (if devices are enabled), ensuring uniqueness + * across multiple devices that may have entities with the same object_id. + * + * Use this method when storing or retrieving preferences/settings that should be unique + * per device-entity pair. Use get_object_id_hash() when you need a hash that identifies + * the entity regardless of the device it belongs to. + * + * For backward compatibility, if device_id is 0 (the main device), the hash is unchanged + * from previous versions, so existing single-device configurations will continue to work. + * + * @return uint32_t The unique hash for preferences, including device_id if available. + * @deprecated Use make_entity_preference() instead, or preferences won't be migrated. + * See https://github.com/esphome/backlog/issues/85 + */ ESPDEPRECATED("Use make_entity_preference() instead, or preferences won't be migrated. " "See https://github.com/esphome/backlog/issues/85. Will be removed in 2027.1.0.", - "2026.8.0") - uint32_t get_preference_hash() { return this->old_preference_key_base_(); } + "2026.7.0") + uint32_t get_preference_hash() { +#ifdef USE_DEVICES + // Combine object_id_hash with device_id to ensure uniqueness across devices + // Note: device_id is 0 for the main device, so XORing with 0 preserves the original hash + // This ensures backward compatibility for existing single-device configurations + return this->get_object_id_hash() ^ this->get_device_id(); +#else + // Without devices, just use object_id_hash as before + return this->get_object_id_hash(); +#endif + } /// Create a preference object for storing this entity's state/settings. /// @tparam T The type of data to store (must be trivially copyable) @@ -223,9 +230,9 @@ class EntityBase { // before push_back, so codegen can emit a single combined call per entity. friend class Application; - /// Combined entity setup from codegen: set name, entity key, entity string indices, and flags. + /// Combined entity setup from codegen: set name, object_id hash, entity string indices, and flags. /// Bit layout of entity_fields is defined by the ENTITY_FIELD_*_SHIFT constants above. - void configure_entity_(const char *name, uint32_t entity_key, uint32_t entity_fields); + void configure_entity_(const char *name, uint32_t object_id_hash, uint32_t entity_fields); #ifdef USE_DEVICES // Codegen-only setter — only accessible from setup() via friend declaration. @@ -233,24 +240,13 @@ class EntityBase { #endif /// Non-template helper for make_entity_preference() to avoid code bloat. - /// Migrates preferences from the old sanitized-object_id key to the raw-name key - /// on key-lookup platforms. See: https://github.com/esphome/backlog/issues/85 + /// When the preference hash algorithm changes, migration logic goes here. ESPPreferenceObject make_entity_preference_(size_t size, uint32_t version); - void calc_entity_key_(); - - /// Reconstruct the OLD (pre-2026.8.0) sanitized-object_id hash for preference keys. - uint32_t calc_old_object_id_hash_() const; - - /// Preference key base for this entity: raw-name entity key XOR device_id. - uint32_t preference_key_base_() const { return this->entity_key_ ^ this->get_device_id_or_zero(); } - - /// Legacy preference key base: sanitized-object_id hash XOR device_id. - /// Note: device_id is 0 for the main device, so XORing with 0 preserves the original hash. - uint32_t old_preference_key_base_() const { return this->calc_old_object_id_hash_() ^ this->get_device_id_or_zero(); } + void calc_object_id_(); StringRef name_; - uint32_t entity_key_{}; + uint32_t object_id_hash_{}; #ifdef USE_DEVICES Device *device_{}; #endif diff --git a/esphome/core/entity_helpers.py b/esphome/core/entity_helpers.py index 5060e32a2d..54e2551cb4 100644 --- a/esphome/core/entity_helpers.py +++ b/esphome/core/entity_helpers.py @@ -25,86 +25,25 @@ from esphome.core.config import ( from esphome.cpp_generator import MockObj, RawStatement, add, get_variable from esphome.cpp_types import App import esphome.final_validate as fv -from esphome.helpers import cpp_string_escape, fnv1_hash_name, sanitize, snake_case +from esphome.helpers import ( + cpp_string_escape, + fnv1_hash, + fnv1_hash_object_id, + sanitize, + snake_case, +) from esphome.types import ConfigType, EntityMetadata _LOGGER = logging.getLogger(__name__) DOMAIN = "entity_string_pool" -_OBJECT_ID_DOMAIN = "entity_object_ids" - - -@dataclass -class ObjectIdEntity: - """An entity tracked by the sanitized object_id its name resolves to.""" - - name: str - platform: str - config: ConfigType - - -def _get_object_id_registry() -> dict[tuple[str, str, str], list[ObjectIdEntity]]: - """(device_id, platform, sanitized object_id) -> entities resolving to it.""" - return CORE.data.setdefault(_OBJECT_ID_DOMAIN, {}) - - -def validate_no_object_id_conflicts( - reason: str, - conflict_filter: Callable[[list[ObjectIdEntity], ConfigType], bool] | None = None, -) -> Callable[[ConfigType], ConfigType]: - """Create a final-validate step that rejects entities with colliding object_ids. - - Entity keys are hashed from the raw name, so names that only differ in characters - lost during sanitizing (for example two UTF-8 names) validate fine in general. - Components that still address entities by the sanitized object_id string must - reject those configs until they are migrated to raw names. - - Args: - reason: One sentence stating what the component builds from the object_id, - e.g. "mqtt builds default topics from the entity object_id" - conflict_filter: Optional predicate receiving the colliding entities and the - component config; return False when the component is not affected - - Returns: - A validator function for use as (or within) FINAL_VALIDATE_SCHEMA - """ - - def validator(config: ConfigType) -> ConfigType: - # Skip in testing_mode, which is used for grouped component testing - if CORE.testing_mode: - return config - conflicts = { - key: entities - for key, entities in _get_object_id_registry().items() - if len(entities) > 1 - and (conflict_filter is None or conflict_filter(entities, config)) - } - if not conflicts: - return config - lines = [f"{reason}, so these entities would conflict:"] - lines.extend( - f" - {platform} entities " - + ", ".join(f"'{e.name}'" for e in entities) - + (f" on device '{device_id}'" if device_id else "") - + f" share the object_id '{object_id}'" - for (device_id, platform, object_id), entities in conflicts.items() - ) - lines.append( - "To fix: Add unique ASCII characters (e.g., '1', '2', or 'A', 'B') " - "to distinguish the names" - ) - raise cv.Invalid("\n".join(lines)) - - return validator - - # Private config keys for storing registered string indices _KEY_DC_IDX = "_entity_dc_idx" _KEY_UOM_IDX = "_entity_uom_idx" _KEY_ICON_IDX = "_entity_icon_idx" _KEY_ENTITY_NAME = "_entity_name" -_KEY_ENTITY_KEY = "_entity_key" +_KEY_OBJECT_ID_HASH = "_entity_object_id_hash" # Bit layout for entity_fields in configure_entity_(). # Keep in sync with ENTITY_FIELD_*_SHIFT constants in esphome/core/entity_base.h @@ -367,7 +306,7 @@ def finalize_entity_strings(var: MockObj, config: ConfigType) -> None: standalone ``var->configure_entity_(name, hash, packed)``. """ entity_name = config[_KEY_ENTITY_NAME] - entity_key = config[_KEY_ENTITY_KEY] + object_id_hash = config[_KEY_OBJECT_ID_HASH] dc_idx = config.get(_KEY_DC_IDX, 0) uom_idx = config.get(_KEY_UOM_IDX, 0) icon_idx = config.get(_KEY_ICON_IDX, 0) @@ -387,30 +326,57 @@ def finalize_entity_strings(var: MockObj, config: ConfigType) -> None: register_method = config.get(_KEY_REGISTER_METHOD) if register_method is not None: expr = getattr(App, f"register_{register_method}")( - var, entity_name, entity_key, packed + var, entity_name, object_id_hash, packed ) else: - expr = var.configure_entity_(entity_name, entity_key, packed) + expr = var.configure_entity_(entity_name, object_id_hash, packed) if comment: add(RawStatement(f"{expr}; // {comment}")) else: add(expr) -def get_base_entity_name( +def get_base_entity_object_id( name: str, friendly_name: str | None, device_name: str | None = None ) -> str: - """Return the base name whose hash becomes this entity's key on the device. + """Calculate the base object ID for an entity that will be set via set_object_id(). - Follows the name selection in C++ EntityBase::configure_entity_() (entity_base.cpp): - entity name, then sub-device name, then friendly name, then the device name. + This function calculates what object_id_c_str_ should be set to in C++. - This is a config-time approximation for duplicate checking: when - name_add_mac_suffix is enabled the device appends the MAC suffix at runtime, - which is unknown here and identical for every entity on the device, so - ignoring it cannot change whether two entities collide with each other. + The C++ EntityBase::write_object_id_to() (entity_base.cpp) works as: + - If !has_own_name && is_name_add_mac_suffix_enabled(): + return str_sanitize(str_snake_case(App.get_friendly_name())) // Dynamic + - Else: + return object_id_c_str_ ?? "" // What we set via set_object_id() + + Since we're calculating what to pass to set_object_id(), we always need to + generate the object_id the same way, regardless of name_add_mac_suffix setting. + + Args: + name: The entity name (empty string if no name) + friendly_name: The friendly name from CORE.friendly_name + device_name: The device name if entity is on a sub-device + + Returns: + The base object ID to use for duplicate checking and to pass to set_object_id() """ - return name or device_name or friendly_name or CORE.name + + if name: + # Entity has its own name (has_own_name will be true) + base_str = name + elif device_name: + # Entity has empty name and is on a sub-device + # C++ EntityBase::set_name() uses device->get_name() when device is set + base_str = device_name + elif friendly_name: + # Entity has empty name (has_own_name will be false) + # C++ uses App.get_friendly_name() which returns friendly_name or device name + base_str = friendly_name + else: + # Fallback to device name + base_str = CORE.name + + return sanitize(snake_case(base_str)) def setup_entity(var_or_platform, config=None, platform=None): @@ -469,15 +435,15 @@ async def _setup_entity_impl(var: MockObj, config: ConfigType, platform: str) -> device: MockObj = await get_variable(device_id_obj) add(var.set_device_(device)) - # Pre-compute entity name and entity key for configure_entity_() + # Pre-compute entity name and object_id hash for configure_entity_() # which is emitted later by finalize_entity_strings(). - # For named entities: pre-compute the key from the raw entity name - # For empty-name entities: pass 0, C++ calculates the key at runtime from - # device name, friendly_name, or app name + # For named entities: pre-compute hash from entity name + # For empty-name entities: pass 0, C++ calculates hash at runtime from + # device name, friendly_name, or app name (bug-for-bug compatibility) entity_name = config[CONF_NAME] - entity_key = fnv1_hash_name(entity_name) if entity_name else 0 + object_id_hash = fnv1_hash_object_id(entity_name) if entity_name else 0 config[_KEY_ENTITY_NAME] = entity_name - config[_KEY_ENTITY_KEY] = entity_key + config[_KEY_OBJECT_ID_HASH] = object_id_hash # Store flags for packing into configure_entity_() config[_KEY_DISABLED_BY_DEFAULT] = int(config[CONF_DISABLED_BY_DEFAULT]) if CONF_INTERNAL in config: @@ -590,13 +556,16 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy # Use the device ID string directly for uniqueness device_id = device_id_obj.id - # Hash the same raw name the device hashes into the entity key at runtime. - # This handles empty names correctly by using device/friendly names. - base_name = get_base_entity_name(entity_name, CORE.friendly_name, device_name) - name_hash = fnv1_hash_name(base_name) + # Calculate what object_id will actually be used + # This handles empty names correctly by using device/friendly names + name_key = get_base_entity_object_id( + entity_name, CORE.friendly_name, device_name + ) - # Check for duplicates: two entities on the same device and platform must not - # share an entity key, since the key is what routes state to API clients + # Check for duplicates by the FNV-1 hash of the object_id, which is the entity + # key that routes state to API clients. This rejects names that sanitize to the + # same object_id, and also two different object_ids whose 32-bit hashes collide. + name_hash = fnv1_hash(name_key) unique_key = (device_id, platform, name_hash) if unique_key in CORE.unique_ids: # Get the existing entity metadata @@ -621,14 +590,26 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy if existing_component != "unknown": conflict_msg += f" from component '{existing_component}'" - # Different names can only clash here through a genuine hash collision + # Distinguish names that sanitize to the same object_id from a genuine + # 32-bit hash collision between two different object_ids collision_msg = "" if entity_name != existing_name: - collision_msg = ( - f"\n The names '{entity_name}' and '{existing_name}' produce the" - f"\n same entity key hash ({name_hash:#010x})." - "\n To fix: Rename one of the entities" + existing_object_id = get_base_entity_object_id( + existing_name, CORE.friendly_name, existing_device or None ) + if existing_object_id == name_key: + collision_msg = ( + f"\n Original names: '{entity_name}' and '{existing_name}'" + f"\n Both convert to ASCII ID: '{name_key}'" + "\n To fix: Add unique ASCII characters (e.g., '1', '2', or 'A', 'B')" + "\n to distinguish them" + ) + else: + collision_msg = ( + f"\n The object_ids '{name_key}' and '{existing_object_id}'" + f"\n produce the same entity key hash ({name_hash:#010x})." + "\n To fix: Rename one of the entities" + ) # Skip duplicate entity name validation when testing_mode is enabled # This flag is used for grouped component testing @@ -640,19 +621,6 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy f"{collision_msg}" ) - # Components that still address entities by the sanitized object_id reject - # colliding names in final validation via validate_no_object_id_conflicts(), - # so track every entity by the object_id its name resolves to. Scoped per - # device and platform to match the strictness configs had before entity keys - # moved to raw names: same-named entities on different sub-devices were - # already accepted then, internal entities were already skipped (above), and - # overlaps between platforms that share an MQTT component type (sensor and - # text_sensor both publish under "sensor") were already possible. - object_id = sanitize(snake_case(base_name)) - _get_object_id_registry().setdefault( - (device_id, platform, object_id), [] - ).append(ObjectIdEntity(base_name, platform, config)) - # Store metadata about this entity entity_metadata: EntityMetadata = { "name": entity_name, diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index d883ce146e..994fa2c26a 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -809,19 +809,6 @@ constexpr uint32_t FNV1_OFFSET_BASIS = 2166136261UL; /// FNV-1 32-bit prime constexpr uint32_t FNV1_PRIME = 16777619UL; -/// Calculate a FNV-1 hash over raw bytes with an explicit length. Unlike fnv1_hash(const char *), -/// each byte is hashed as an unsigned value, so results are platform-independent for bytes >= 0x80. -/// IMPORTANT: Must match Python fnv1_hash_name() in esphome/helpers.py, which hashes the UTF-8 -/// encoded bytes of the name. Used to compute entity keys from raw names. -inline uint32_t fnv1_hash_bytes(const char *str, size_t len) { - uint32_t hash = FNV1_OFFSET_BASIS; - for (size_t i = 0; i < len; i++) { - hash *= FNV1_PRIME; - hash ^= static_cast(str[i]); - } - return hash; -} - /// Extend a FNV-1 hash with an integer (hashes each byte). template constexpr uint32_t fnv1_hash_extend(uint32_t hash, T value) { using UnsignedT = std::make_unsigned_t; @@ -1026,20 +1013,12 @@ template inline char *str_sanitize_to(char (&buffer)[N], const char *s // str_sanitize moved to alloc_helpers.h - remove this comment before 2026.11.0 /// Calculate FNV-1 hash of a string while applying snake_case + sanitize transformations. -/// This is the LEGACY entity hash, kept only to reconstruct preference keys that existing -/// devices already have stored; see https://github.com/esphome/backlog/issues/85. -/// With per_code_point set, UTF-8 continuation bytes are skipped so each multi-byte character -/// contributes one underscore — this matches Python fnv1_hash_object_id() in esphome/helpers.py, -/// which produced the hash for named entities. The per-byte form (default) matches the old -/// runtime hash for entities without their own name. Do not change either behavior. -/// Known limitation: Python's lower() is Unicode aware, so the rare code points it maps to a -/// different number of characters or to ASCII (e.g. 'İ', the Kelvin sign) reconstruct wrong; -/// such names skip migration once and fall back to their defaults. -inline uint32_t fnv1_hash_object_id(const char *str, size_t len, bool per_code_point = false) { +/// This computes object_id hashes directly from names without creating an intermediate buffer. +/// IMPORTANT: Must match Python fnv1_hash_object_id() in esphome/helpers.py. +/// If you modify this function, update the Python version and tests in both places. +inline uint32_t fnv1_hash_object_id(const char *str, size_t len) { uint32_t hash = FNV1_OFFSET_BASIS; for (size_t i = 0; i < len; i++) { - if (per_code_point && (static_cast(str[i]) & 0xC0) == 0x80) - continue; // UTF-8 continuation byte, already counted via its lead byte hash *= FNV1_PRIME; // Apply snake_case (space->underscore, uppercase->lowercase) then sanitize hash ^= static_cast(to_sanitized_char(to_snake_case_char(str[i]))); diff --git a/esphome/core/preference_backend.h b/esphome/core/preference_backend.h index 0622376fca..5df0804bdd 100644 --- a/esphome/core/preference_backend.h +++ b/esphome/core/preference_backend.h @@ -24,9 +24,10 @@ #endif // Key-lookup preference backends find stored data by key; their platforms add the -// USE_PREFERENCE_KEY_LOOKUP define from Python codegen, which enables preference key -// migration. Slot-based backends (ESP8266, RP2040) instead allocate a storage slot for -// every make_preference() call and use the key only as a validity tag on that slot; +// USE_PREFERENCE_KEY_LOOKUP define from Python codegen, which enables one-shot reads +// of stored data by key (the primitive preference key migrations need). Slot-based +// backends (ESP8266, RP2040) instead allocate a storage slot for every +// make_preference() call and use the key only as a validity tag on that slot; // migration is not possible there, and key collisions cannot corrupt data. namespace esphome { @@ -104,10 +105,9 @@ concept PreferencesContract = requires(T prefs, size_t len, uint32_t type, bool }; // Key-lookup platforms additionally provide load_from_key(), a one-shot read -// of a stored preference by key that migrate_preference() relies on; see the -// key-lookup note at the top of this file. Not part of PreferencesContract, -// so it is asserted in preferences.h only where USE_PREFERENCE_KEY_LOOKUP -// is set. +// of a stored preference by key; see the key-lookup note at the top of this +// file. Not part of PreferencesContract, so it is asserted in preferences.h +// only where USE_PREFERENCE_KEY_LOOKUP is set. template concept PreferencesKeyLookupContract = requires(T prefs, uint32_t type, uint8_t *data, size_t len) { { prefs.load_from_key(type, data, len) } -> std::same_as; diff --git a/esphome/core/preferences.cpp b/esphome/core/preferences.cpp deleted file mode 100644 index 8508647255..0000000000 --- a/esphome/core/preferences.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include "esphome/core/preferences.h" -#include "esphome/core/log.h" -#include - -namespace esphome { - -#ifdef USE_PREFERENCE_KEY_LOOKUP -static const char *const TAG = "preferences"; - -bool migrate_preference(ESPPreferenceObject &new_pref, uint8_t *scratch, size_t size, uint32_t old_key, - uint32_t new_key) { - if (new_pref.load(scratch, size)) - return true; // Current data present - never overwrite newer data with the old copy - // One-shot read by key: no backend is allocated for the old key, so boots with - // nothing to migrate (for example fresh installs) cost no heap - if (old_key == new_key || !global_preferences->load_from_key(old_key, scratch, size)) - return false; // No data stored under the old key, nothing to migrate - if (!new_pref.save(scratch, size)) { - ESP_LOGW(TAG, "Pref migration %" PRIx32 " -> %" PRIx32 " failed", old_key, new_key); - } - return true; -} -#endif // USE_PREFERENCE_KEY_LOOKUP - -} // namespace esphome diff --git a/esphome/core/preferences.h b/esphome/core/preferences.h index cfeddebda7..ed23dfae56 100644 --- a/esphome/core/preferences.h +++ b/esphome/core/preferences.h @@ -56,17 +56,5 @@ namespace esphome { static_assert(PreferencesKeyLookupContract, "This platform emits USE_PREFERENCE_KEY_LOOKUP but its preferences manager does not provide " "load_from_key() (esphome/core/preference_backend.h)"); - -/// Copy preference data stored under old_key into new_pref (created for new_key) if the keys -/// differ and new_pref has no data yet. scratch must hold at least size bytes. -/// Returns true when scratch holds the entity's current data (loaded or just migrated). -/// The old entry is intentionally left in place so a firmware downgrade still finds its data. -/// If saving under the new key fails, callers that consume scratch (like TextSaver) still get -/// valid data for this boot, callers that reload from the preference fall back to their -/// defaults, and the migration simply runs again on the next boot. -/// Only available on key-lookup preference backends; slot-based backends keep their old -/// keys instead. See: https://github.com/esphome/backlog/issues/85 -bool migrate_preference(ESPPreferenceObject &new_pref, uint8_t *scratch, size_t size, uint32_t old_key, - uint32_t new_key); } // namespace esphome #endif // USE_PREFERENCE_KEY_LOOKUP diff --git a/esphome/helpers.py b/esphome/helpers.py index 2731109164..9b2a461ccd 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -91,13 +91,8 @@ def fnv1a_32bit_hash(string: str) -> int: def fnv1_hash_object_id(name: str) -> int: """Compute FNV-1 hash of name with snake_case + sanitize transformations. - IMPORTANT: Must produce same result as C++ fnv1_hash_object_id() in helpers.h - with per_code_point set. This is the OLD entity hash; it computes preference - keys that existing devices already have stored (see - https://github.com/esphome/backlog/issues/85) and is also still used for live - keys derived from config IDs (see the motion component's calibration key). - Note: lower() here is Unicode aware while the C++ reconstruction is not; see - the known limitation note on the C++ function. + IMPORTANT: Must produce same result as C++ fnv1_hash_object_id() in helpers.h. + If you modify this function, update the C++ version and tests in both places. """ return fnv1_hash(sanitize(snake_case(name))) @@ -105,9 +100,9 @@ def fnv1_hash_object_id(name: str) -> int: def fnv1_hash_name(name: str) -> int: """Compute FNV-1 hash of the raw entity name (UTF-8 bytes, no transformations). - IMPORTANT: Must produce same result as C++ fnv1_hash_bytes() in helpers.h, - which hashes the name bytes as stored on the device. - Used for pre-computing entity keys at code generation time. + 2026.8 beta firmware stored preferences under keys derived from this hash; + a future key migration must reconstruct those keys to recover that data + (see https://github.com/esphome/backlog/issues/85). """ return _fnv1_hash(name.encode("utf-8")) diff --git a/tests/integration/entity_utils.py b/tests/integration/entity_utils.py index 95f6a0321e..7596983ee2 100644 --- a/tests/integration/entity_utils.py +++ b/tests/integration/entity_utils.py @@ -8,7 +8,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from esphome.helpers import fnv1_hash_name, sanitize, snake_case +from esphome.helpers import fnv1_hash_object_id, sanitize, snake_case if TYPE_CHECKING: from aioesphomeapi import DeviceInfo, EntityInfo @@ -25,16 +25,15 @@ def infer_name_add_mac_suffix(device_info: DeviceInfo) -> bool: return device_info.name.endswith(f"-{mac_suffix}") -def _resolve_entity_name( +def _get_name_for_object_id( entity: EntityInfo, device_info: DeviceInfo, device_id_to_name: dict[int, str], ) -> str: - """Resolve the effective name for an entity. + """Get the name used for object_id computation. This is the algorithm that aioesphomeapi will use to determine which - name to use for computing object_id client-side from API data; the same - name is what the device hashes into the entity key. + name to use for computing object_id client-side from API data. Args: entity: The entity to get name for @@ -73,27 +72,27 @@ def compute_entity_object_id( Returns: The computed object_id string """ - name = _resolve_entity_name(entity, device_info, device_id_to_name) - return compute_object_id(name) + name_for_id = _get_name_for_object_id(entity, device_info, device_id_to_name) + return compute_object_id(name_for_id) -def compute_entity_key( +def compute_entity_hash( entity: EntityInfo, device_info: DeviceInfo, device_id_to_name: dict[int, str], ) -> int: - """Compute expected entity key for an entity. + """Compute expected object_id hash for an entity. Args: - entity: The entity to compute the key for + entity: The entity to compute hash for device_info: Device info from the API device_id_to_name: Mapping of device_id to device name for sub-devices Returns: - The computed FNV-1 hash of the raw name + The computed FNV-1 hash """ - name = _resolve_entity_name(entity, device_info, device_id_to_name) - return fnv1_hash_name(name) + name_for_id = _get_name_for_object_id(entity, device_info, device_id_to_name) + return fnv1_hash_object_id(name_for_id) def verify_entity_object_id( @@ -119,7 +118,7 @@ def verify_entity_object_id( f"expected '{expected_object_id}', got '{entity.object_id}'" ) - expected_hash = compute_entity_key(entity, device_info, device_id_to_name) + expected_hash = compute_entity_hash(entity, device_info, device_id_to_name) assert entity.key == expected_hash, ( f"hash mismatch for entity '{entity.name}': " f"expected {expected_hash:#x}, got {entity.key:#x}" diff --git a/tests/integration/fixtures/fnv1_hash_object_id.yaml b/tests/integration/fixtures/fnv1_hash_object_id.yaml index d4511bb8c6..2097b2fbf9 100644 --- a/tests/integration/fixtures/fnv1_hash_object_id.yaml +++ b/tests/integration/fixtures/fnv1_hash_object_id.yaml @@ -71,38 +71,6 @@ esphome: ESP_LOGE("FNV1_OID", "empty FAILED: 0x%08x != 0x811c9dc5", hash_empty); } - // Raw name hash: matches Python fnv1_hash_name("My Sensor Name") - uint32_t hash_raw = esphome::fnv1_hash_bytes("My Sensor Name", 14); - if (hash_raw == 0x8cec6fb0) { - ESP_LOGI("FNV1_OID", "raw PASSED"); - } else { - ESP_LOGE("FNV1_OID", "raw FAILED: 0x%08x != 0x8cec6fb0", hash_raw); - } - - // Raw name hash over UTF-8 bytes: matches Python fnv1_hash_name("Température") - uint32_t hash_raw_utf8 = esphome::fnv1_hash_bytes("Temp\xc3\xa9rature", 12); - if (hash_raw_utf8 == 0x531a74aa) { - ESP_LOGI("FNV1_OID", "raw_utf8 PASSED"); - } else { - ESP_LOGE("FNV1_OID", "raw_utf8 FAILED: 0x%08x != 0x531a74aa", hash_raw_utf8); - } - - // Old-key UTF-8 variant: matches Python fnv1_hash_object_id("Température") - uint32_t hash_old_utf8 = esphome::fnv1_hash_object_id("Temp\xc3\xa9rature", 12, true); - if (hash_old_utf8 == 0x965698f3) { - ESP_LOGI("FNV1_OID", "old_utf8 PASSED"); - } else { - ESP_LOGE("FNV1_OID", "old_utf8 FAILED: 0x%08x != 0x965698f3", hash_old_utf8); - } - - // Old-key UTF-8 variant with multi-byte only name: Python fnv1_hash_object_id("温度") - uint32_t hash_old_cjk = esphome::fnv1_hash_object_id("\xe6\xb8\xa9\xe5\xba\xa6", 6, true); - if (hash_old_cjk == 0x3276cb9f) { - ESP_LOGI("FNV1_OID", "old_cjk PASSED"); - } else { - ESP_LOGE("FNV1_OID", "old_cjk FAILED: 0x%08x != 0x3276cb9f", hash_old_cjk); - } - host: api: logger: diff --git a/tests/integration/fixtures/multi_device_preferences.yaml b/tests/integration/fixtures/multi_device_preferences.yaml index 582add90a8..01e4394559 100644 --- a/tests/integration/fixtures/multi_device_preferences.yaml +++ b/tests/integration/fixtures/multi_device_preferences.yaml @@ -156,17 +156,10 @@ button: ESP_LOGI("test", "Device A Mode: %s", id(mode_device_a).current_option().c_str()); ESP_LOGI("test", "Device B Mode: %s", id(mode_device_b).current_option().c_str()); ESP_LOGI("test", "Main Mode: %s", id(mode_main).current_option().c_str()); - // Log preference key bases for entities that actually store preferences. - // This is the key base make_entity_preference() uses: entity key XOR device id. - ESP_LOGI("test", "Device A Switch Pref Hash: %u", - id(light_device_a).get_entity_key() ^ id(light_device_a).get_device_id_or_zero()); - ESP_LOGI("test", "Device B Switch Pref Hash: %u", - id(light_device_b).get_entity_key() ^ id(light_device_b).get_device_id_or_zero()); - ESP_LOGI("test", "Main Switch Pref Hash: %u", - id(light_main).get_entity_key() ^ id(light_main).get_device_id_or_zero()); - ESP_LOGI("test", "Device A Number Pref Hash: %u", - id(setpoint_device_a).get_entity_key() ^ id(setpoint_device_a).get_device_id_or_zero()); - ESP_LOGI("test", "Device B Number Pref Hash: %u", - id(setpoint_device_b).get_entity_key() ^ id(setpoint_device_b).get_device_id_or_zero()); - ESP_LOGI("test", "Main Number Pref Hash: %u", - id(setpoint_main).get_entity_key() ^ id(setpoint_main).get_device_id_or_zero()); + // Log preference hashes for entities that actually store preferences + ESP_LOGI("test", "Device A Switch Pref Hash: %u", id(light_device_a).get_preference_hash()); + ESP_LOGI("test", "Device B Switch Pref Hash: %u", id(light_device_b).get_preference_hash()); + ESP_LOGI("test", "Main Switch Pref Hash: %u", id(light_main).get_preference_hash()); + ESP_LOGI("test", "Device A Number Pref Hash: %u", id(setpoint_device_a).get_preference_hash()); + ESP_LOGI("test", "Device B Number Pref Hash: %u", id(setpoint_device_b).get_preference_hash()); + ESP_LOGI("test", "Main Number Pref Hash: %u", id(setpoint_main).get_preference_hash()); diff --git a/tests/integration/fixtures/preference_key_migration.yaml b/tests/integration/fixtures/preference_key_migration.yaml deleted file mode 100644 index a9b01fc2d2..0000000000 --- a/tests/integration/fixtures/preference_key_migration.yaml +++ /dev/null @@ -1,35 +0,0 @@ -esphome: - name: host-pref-key-migration - -host: -api: -logger: - -switch: - - platform: template - id: test_switch_restore - name: Test Switch - optimistic: true - restore_mode: RESTORE_DEFAULT_OFF - -number: - - platform: template - id: test_number_restore - name: Test Number - optimistic: true - restore_value: true - initial_value: 1.0 - min_value: 0 - max_value: 100 - step: 0.5 - -text: - - platform: template - id: test_text_restore - name: Test Text - mode: text - optimistic: true - restore_value: true - initial_value: fallback - min_length: 0 - max_length: 20 diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index c7f21d8a01..f835bee3bc 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -25,25 +25,15 @@ def clear_host_prefs(device_name: str) -> None: host_prefs_path(device_name).unlink(missing_ok=True) -def write_host_prefs(device_name: str, entries: dict[int, bytes]) -> Path: - """Write preference entries, replacing the file's contents. - - Returns the path that was written. - """ - payload = b"" - for key, data in entries.items(): - if len(data) > 255: - raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") - payload += struct.pack(" Path: """Write a single preference entry, replacing the file's contents. Returns the path that was written. """ - return write_host_prefs(device_name, {key: data}) + if len(data) > 255: + raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") + path = host_prefs_path(device_name) + path.parent.mkdir(parents=True, exist_ok=True) + payload = struct.pack(" None: diff --git a/tests/integration/test_object_id_api_verification.py b/tests/integration/test_object_id_api_verification.py index 8dafb37c64..c8603e0682 100644 --- a/tests/integration/test_object_id_api_verification.py +++ b/tests/integration/test_object_id_api_verification.py @@ -2,8 +2,8 @@ This test verifies a three-way match between: 1. C++ object_id generation (get_object_id_to using to_sanitized_char/to_snake_case_char) -2. C++ entity key generation (fnv1_hash of the raw name in helpers.h) -3. Python computation (sanitize/snake_case and fnv1_hash_name in helpers.py) +2. C++ hash generation (fnv1_hash_object_id in helpers.h) +3. Python computation (sanitize/snake_case in helpers.py, fnv1_hash_object_id) The API response contains C++ computed values, so verifying API == Python implicitly verifies C++ == Python == API for both object_id and hash. @@ -25,7 +25,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_name +from esphome.helpers import fnv1_hash_object_id from .entity_utils import compute_object_id, verify_all_entities from .types import APIClientConnectedFactory, RunCompiledFunction @@ -123,7 +123,7 @@ async def test_object_id_api_verification( ) # Verify hash can be computed from the name - hash_from_name = fnv1_hash_name(entity_name) + hash_from_name = fnv1_hash_object_id(entity_name) assert hash_from_name == entity.key, ( f"Entity '{entity_name}': hash mismatch. " f"Python hash {hash_from_name:#x}, API key {entity.key:#x}" @@ -164,7 +164,7 @@ async def test_object_id_api_verification( ) # Verify hash matches - expected_hash = fnv1_hash_name(expected_name) + expected_hash = fnv1_hash_object_id(expected_name) assert entity.key == expected_hash, ( f"Empty-name entity (device_id={entity.device_id}): hash mismatch. " f"API key: {entity.key:#x}, expected: {expected_hash:#x}" diff --git a/tests/integration/test_object_id_friendly_name_no_mac_suffix.py b/tests/integration/test_object_id_friendly_name_no_mac_suffix.py index b58593f2ef..7199a2b371 100644 --- a/tests/integration/test_object_id_friendly_name_no_mac_suffix.py +++ b/tests/integration/test_object_id_friendly_name_no_mac_suffix.py @@ -11,7 +11,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_name +from esphome.helpers import fnv1_hash_object_id from .entity_utils import ( compute_object_id, @@ -62,7 +62,7 @@ async def test_object_id_friendly_name_no_mac_suffix( ) # Hash should match friendly_name - expected_hash = fnv1_hash_name("My Friendly Device") + expected_hash = fnv1_hash_object_id("My Friendly Device") assert entity.key == expected_hash, ( f"Expected hash {expected_hash:#x}, got {entity.key:#x}" ) diff --git a/tests/integration/test_object_id_no_friendly_name.py b/tests/integration/test_object_id_no_friendly_name.py index 45b5f730a6..b548f02fde 100644 --- a/tests/integration/test_object_id_no_friendly_name.py +++ b/tests/integration/test_object_id_no_friendly_name.py @@ -17,7 +17,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_name +from esphome.helpers import fnv1_hash_object_id from .entity_utils import compute_object_id, verify_all_entities from .types import APIClientConnectedFactory, RunCompiledFunction @@ -96,7 +96,7 @@ async def test_object_id_no_friendly_name_no_mac_suffix( OLD behavior: - is_object_id_dynamic_() returned false (mac suffix not enabled) - Used object_id_c_str_ which was pre-computed in Python - - Python used get_base_entity_name() with fallback to CORE.name + - Python used get_base_entity_object_id() with fallback to CORE.name Result: object_id = sanitize(snake_case(device_name)) """ @@ -126,7 +126,7 @@ async def test_object_id_no_friendly_name_no_mac_suffix( ) # Hash should match device name - expected_hash = fnv1_hash_name("test-device") + expected_hash = fnv1_hash_object_id("test-device") assert entity.key == expected_hash, ( f"Expected hash {expected_hash:#x}, got {entity.key:#x}" ) diff --git a/tests/integration/test_preference_key_migration.py b/tests/integration/test_preference_key_migration.py deleted file mode 100644 index e7f699bb12..0000000000 --- a/tests/integration/test_preference_key_migration.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Integration test for entity preference key migration. - -Entity keys are now the FNV-1 hash of the raw name instead of the sanitized -object_id (https://github.com/esphome/backlog/issues/85). On key-lookup -preference backends, make_entity_preference() must move data stored under the -old key to the new key, so devices keep their restored state after upgrading. - -This test seeds the host preferences file the way a pre-migration firmware -would have written it and verifies: -1. Data stored under the OLD key is restored (migration happened, no data loss) -2. Data already stored under the NEW key is never overwritten by old data -""" - -from __future__ import annotations - -import socket -import struct - -from aioesphomeapi import ( - NumberInfo, - NumberState, - SwitchInfo, - SwitchState, - TextInfo, - TextState, -) -import pytest - -from esphome.helpers import fnv1_hash, fnv1_hash_name, fnv1_hash_object_id - -from .conftest import run_binary_and_wait_for_port, wait_and_connect_api_client -from .host_prefs import clear_host_prefs, write_host_prefs -from .state_utils import InitialStateHelper, require_entity -from .types import CompileFunction, ConfigWriter - -DEVICE_NAME = "host-pref-key-migration" - -# The pre-migration preference key was the sanitized object_id hash; the new -# key is the raw-name hash. All entities are on the main device (device_id 0) -# and their preferences use no version salt, so the key is just the hash. -SWITCH_OLD_KEY = fnv1_hash_object_id("Test Switch") -SWITCH_NEW_KEY = fnv1_hash_name("Test Switch") -NUMBER_OLD_KEY = fnv1_hash_object_id("Test Number") -NUMBER_NEW_KEY = fnv1_hash_name("Test Number") - -# template_text salts its key with the length limits and pattern hash; this must -# match TemplateText::setup() in template_text.cpp (min_length 0, max_length 20, -# no pattern configured) -TEXT_KEY_EXTRA = (0 << 2) + (20 << 4) + (fnv1_hash("") << 6) -TEXT_OLD_KEY = (fnv1_hash_object_id("Test Text") + TEXT_KEY_EXTRA) & 0xFFFFFFFF -TEXT_NEW_KEY = (fnv1_hash_name("Test Text") + TEXT_KEY_EXTRA) & 0xFFFFFFFF - -# TextSaver<20> stores a length-prefixed buffer of max_length + 1 bytes -TEXT_MAX_LENGTH = 20 - - -def text_pref_payload(value: str) -> bytes: - """Build the length-prefixed buffer TextSaver stores for a value.""" - data = value.encode("utf-8") - assert len(data) <= TEXT_MAX_LENGTH - return bytes([len(data)]) + data + b"\x00" * (TEXT_MAX_LENGTH - len(data)) - - -@pytest.mark.asyncio -async def test_preference_key_migration( - yaml_config: str, - write_yaml_config: ConfigWriter, - compile_esphome: CompileFunction, - reserved_tcp_port: tuple[int, socket.socket], -) -> None: - """Test that preferences stored under the old key survive the upgrade.""" - port, port_socket = reserved_tcp_port - - assert SWITCH_OLD_KEY != SWITCH_NEW_KEY - assert NUMBER_OLD_KEY != NUMBER_NEW_KEY - assert TEXT_OLD_KEY != TEXT_NEW_KEY - - # Write and compile once - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - - # Release the reserved port so the binary can bind to it - port_socket.close() - - async def boot_and_get_initial_states() -> tuple[ - SwitchState, NumberState, TextState - ]: - """Boot the binary and return the restored entity states.""" - async with ( - run_binary_and_wait_for_port(binary_path, "127.0.0.1", port), - wait_and_connect_api_client(port=port) as client, - ): - device_info = await client.device_info() - assert device_info.name == DEVICE_NAME - - entities, _ = await client.list_entities_services() - switch_entity = require_entity( - entities, "test_switch", SwitchInfo, "Test Switch" - ) - number_entity = require_entity( - entities, "test_number", NumberInfo, "Test Number" - ) - text_entity = require_entity(entities, "test_text", TextInfo, "Test Text") - - initial_state_helper = InitialStateHelper(entities) - client.subscribe_states( - initial_state_helper.on_state_wrapper(lambda s: None) - ) - await initial_state_helper.wait_for_initial_states() - - switch_state = initial_state_helper.initial_states[switch_entity.key] - number_state = initial_state_helper.initial_states[number_entity.key] - text_state = initial_state_helper.initial_states[text_entity.key] - assert isinstance(switch_state, SwitchState) - assert isinstance(number_state, NumberState) - assert isinstance(text_state, TextState) - return switch_state, number_state, text_state - - try: - # --- Run 1: only OLD keys present, as written by pre-migration firmware. - # The restored states prove the data was migrated to the new keys. - write_host_prefs( - DEVICE_NAME, - { - SWITCH_OLD_KEY: b"\x01", # bool: switch was ON - NUMBER_OLD_KEY: struct.pack(" None: - """Verify _COMMAND_TOPIC_PLATFORMS matches the MQTT components that subscribe. - - Drift silently reintroduces shared subscribe topics, so this derives the set - from the C++ components that actually call subscribe(); that also catches - platforms like text that subscribe a command topic without exposing a - command_topic key in their schema. - """ - expected: set[str] = set() - for path in (COMPONENTS_DIR / "mqtt").glob("mqtt_*.cpp"): - if path.stem in _NON_ENTITY_MQTT_SOURCES: - continue - if "this->subscribe" not in path.read_text(encoding="utf-8"): - continue - stem = path.stem.removeprefix("mqtt_") - expected.add("datetime" if stem in _DATETIME_STEMS else stem) - assert expected == _COMMAND_TOPIC_PLATFORMS - - -def test_sub_topic_platforms_in_sync() -> None: - """Verify _SUB_TOPIC_PLATFORMS matches the MQTT components with sub-topics. - - Platforms whose MQTT headers use MQTT_COMPONENT_CUSTOM_TOPIC derive extra - topics such as position/command from the object_id. - """ - expected = { - path.stem.removeprefix("mqtt_") - for path in (COMPONENTS_DIR / "mqtt").glob("mqtt_*.h") - if path.stem != "mqtt_component" - and "MQTT_COMPONENT_CUSTOM_TOPIC" in path.read_text(encoding="utf-8") - } - assert expected == _SUB_TOPIC_PLATFORMS - - -def test_conflict_filter_exempts_custom_topics() -> None: - """Test that custom state topics with discovery off avoid the conflict.""" - validator = entity_duplicate_validator("sensor") - # Both entities have custom state topics and discovery disabled per entity, - # so no object_id-derived MQTT topic is used - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} - assert component_validator(config) is config - - # Without the filter the same conflicts are fatal - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - validate_no_object_id_conflicts(REASON)({}) - - -def test_conflict_on_default_command_topic() -> None: - """Test that commandable platforms conflict through their default command topic. - - Custom state topics with discovery off are not enough for platforms that also - subscribe to an object_id-derived command topic. - """ - validator = entity_duplicate_validator("switch") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - mqtt_config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} - # Both switches share the default command topic: rejected - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - component_validator(mqtt_config) - - # With custom command topics as well, nothing derives from the object_id - CORE.reset() - validator = entity_duplicate_validator("switch") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_COMMAND_TOPIC: "custom/cmd/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_COMMAND_TOPIC: "custom/cmd/b", - CONF_DISCOVERY: False, - } - ) - assert component_validator(mqtt_config) is mqtt_config - - -def test_conflict_on_sub_topic_platforms() -> None: - """Test that platforms with extra object_id sub-topics always conflict. - - Covers derive topics like position/command from the object_id through their - own config keys, so custom state and command topics cannot exempt them. - """ - validator = entity_duplicate_validator("cover") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_COMMAND_TOPIC: "custom/cmd/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_COMMAND_TOPIC: "custom/cmd/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - component_validator({CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"}) - - -def test_no_conflict_on_disjoint_default_topics() -> None: - """Test that entities whose default topics are disjoint do not conflict. - - One entity uses only the default command topic and the other only the default - state topic, so they never share a topic. - """ - validator = entity_duplicate_validator("switch") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_COMMAND_TOPIC: "custom/cmd/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} - assert component_validator(config) is config - - -def test_no_conflict_on_empty_topic_prefix() -> None: - """Test that an empty topic_prefix disables the default topic conflict. - - With topic_prefix set to null no default topics exist at runtime, so entities - without custom state topics cannot conflict; only discovery still matters. - """ - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Датчик открытия"}) - validator({CONF_NAME: "Датчик закрытия"}) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - # No default topics and no discovery: valid - config: dict = {CONF_DISCOVERY: False, CONF_TOPIC_PREFIX: ""} - assert component_validator(config) is config - - # Discovery still uses object_id-derived config topics: rejected - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - component_validator({CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: ""}) diff --git a/tests/unit_tests/core/test_entity_helpers.py b/tests/unit_tests/core/test_entity_helpers.py index 64400c4fd4..53035ad713 100644 --- a/tests/unit_tests/core/test_entity_helpers.py +++ b/tests/unit_tests/core/test_entity_helpers.py @@ -1,4 +1,4 @@ -"""Tests for entity helpers: name selection, entity key hashing, duplicate checks.""" +"""Test get_base_entity_object_id function matches C++ behavior.""" from collections.abc import Callable, Generator from pathlib import Path @@ -25,17 +25,16 @@ from esphome.core.entity_helpers import ( _setup_entity_impl, entity_duplicate_validator, finalize_entity_strings, - get_base_entity_name, + get_base_entity_object_id, register_device_class, register_icon, register_unit_of_measurement, setup_device_class, setup_entity, setup_unit_of_measurement, - validate_no_object_id_conflicts, ) from esphome.cpp_generator import MockObj -from esphome.helpers import fnv1_hash_name, sanitize, snake_case +from esphome.helpers import fnv1_hash, sanitize, snake_case from .common import load_config_from_fixture @@ -58,26 +57,206 @@ def restore_core_state() -> Generator[None, None, None]: CORE.friendly_name = original_friendly_name -def test_get_base_entity_name_priority_order() -> None: +def test_with_entity_name() -> None: + """Test when entity has its own name - should use entity name.""" + # Simple name + assert get_base_entity_object_id("Temperature Sensor", None) == "temperature_sensor" + assert ( + get_base_entity_object_id("Temperature Sensor", "Device Name") + == "temperature_sensor" + ) + # Even with device name, entity name takes precedence + assert ( + get_base_entity_object_id("Temperature Sensor", "Device Name", "Sub Device") + == "temperature_sensor" + ) + + # Name with special characters + assert ( + get_base_entity_object_id("Temp!@#$%^&*()Sensor", None) + == "temp__________sensor" + ) + assert get_base_entity_object_id("Temp-Sensor_123", None) == "temp-sensor_123" + + # Already snake_case + assert get_base_entity_object_id("temperature_sensor", None) == "temperature_sensor" + + # Mixed case + assert get_base_entity_object_id("TemperatureSensor", None) == "temperaturesensor" + assert get_base_entity_object_id("TEMPERATURE SENSOR", None) == "temperature_sensor" + + +def test_empty_name_with_device_name() -> None: + """Test when entity has empty name and is on a sub-device - should use device name.""" + # C++ behavior: when has_own_name is false and device is set, uses device->get_name() + assert ( + get_base_entity_object_id("", "Friendly Device", "Sub Device 1") + == "sub_device_1" + ) + assert ( + get_base_entity_object_id("", "Kitchen Controller", "controller_1") + == "controller_1" + ) + assert get_base_entity_object_id("", None, "Test-Device_123") == "test-device_123" + + +def test_empty_name_with_friendly_name() -> None: + """Test when entity has empty name and no device - should use friendly name.""" + # C++ behavior: when has_own_name is false, uses App.get_friendly_name() + assert get_base_entity_object_id("", "Friendly Device") == "friendly_device" + assert get_base_entity_object_id("", "Kitchen Controller") == "kitchen_controller" + assert get_base_entity_object_id("", "Test-Device_123") == "test-device_123" + + # Special characters in friendly name + assert get_base_entity_object_id("", "Device!@#$%") == "device_____" + + +def test_empty_name_no_friendly_name() -> None: + """Test when entity has empty name and no friendly name - should use device name.""" + # Test with CORE.name set + CORE.name = "device-name" + assert get_base_entity_object_id("", None) == "device-name" + + CORE.name = "Test Device" + assert get_base_entity_object_id("", None) == "test_device" + + +def test_edge_cases() -> None: + """Test edge cases.""" + # Only spaces + assert get_base_entity_object_id(" ", None) == "___" + + # Unicode characters (should be replaced) + assert get_base_entity_object_id("Température", None) == "temp_rature" + assert get_base_entity_object_id("测试", None) == "__" + + # Empty string with empty friendly name (empty friendly name is treated as None) + # Falls back to CORE.name + CORE.name = "device" + assert get_base_entity_object_id("", "") == "device" + + # Very long name (should work fine) + long_name = "a" * 100 + " " + "b" * 100 + expected = "a" * 100 + "_" + "b" * 100 + assert get_base_entity_object_id(long_name, None) == expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("Temperature Sensor", "temperature_sensor"), + ("Living Room Light", "living_room_light"), + ("Test-Device_123", "test-device_123"), + ("Special!@#Chars", "special___chars"), + ("UPPERCASE NAME", "uppercase_name"), + ("lowercase name", "lowercase_name"), + ("Mixed Case Name", "mixed_case_name"), + (" Spaces ", "___spaces___"), + ], +) +def test_matches_cpp_helpers(name: str, expected: str) -> None: + """Test that the logic matches using snake_case and sanitize directly.""" + # For non-empty names, verify our function produces same result as direct snake_case + sanitize + assert get_base_entity_object_id(name, None) == sanitize(snake_case(name)) + assert get_base_entity_object_id(name, None) == expected + + +def test_empty_name_fallback() -> None: + """Test empty name handling which falls back to friendly_name or CORE.name.""" + # Empty name is handled specially - it doesn't just use sanitize(snake_case("")) + # Instead it falls back to friendly_name or CORE.name + assert sanitize(snake_case("")) == "" # Direct conversion gives empty string + # But our function returns a fallback + CORE.name = "device" + assert get_base_entity_object_id("", None) == "device" # Uses device name + + +def test_name_add_mac_suffix_behavior() -> None: + """Test behavior related to name_add_mac_suffix. + + In C++, an entity's object_id is computed from its name_ via + write_object_id_to() (sanitized snake_case). When an entity has no name, + configure_entity_() sets name_ from the friendly name, with the MAC suffix + appended when name_add_mac_suffix is enabled. Our function always returns + the same result since we're calculating the base for duplicate tracking. + """ + # The function should always return the same result regardless of + # name_add_mac_suffix setting, as we're calculating the base object_id + assert get_base_entity_object_id("", "Test Device") == "test_device" + assert get_base_entity_object_id("Entity Name", "Test Device") == "entity_name" + + +def test_priority_order() -> None: """Test the priority order: entity name > device name > friendly name > CORE.name.""" CORE.name = "core-device" - # 1. Entity name has highest priority and is used as-is, no transformations + # 1. Entity name has highest priority assert ( - get_base_entity_name("Entity Name", "Friendly Name", "Device Name") - == "Entity Name" + get_base_entity_object_id("Entity Name", "Friendly Name", "Device Name") + == "entity_name" ) - assert get_base_entity_name("Température", None) == "Température" # 2. Device name is next priority (when entity name is empty) - assert get_base_entity_name("", "Friendly Name", "Device Name") == "Device Name" + assert ( + get_base_entity_object_id("", "Friendly Name", "Device Name") == "device_name" + ) # 3. Friendly name is next (when entity and device names are empty) - assert get_base_entity_name("", "Friendly Name", None) == "Friendly Name" + assert get_base_entity_object_id("", "Friendly Name", None) == "friendly_name" - # 4. CORE.name is last resort; an empty friendly name falls through to it - assert get_base_entity_name("", None, None) == "core-device" - assert get_base_entity_name("", "") == "core-device" + # 4. CORE.name is last resort + assert get_base_entity_object_id("", None, None) == "core-device" + + +@pytest.mark.parametrize( + ("name", "friendly_name", "device_name", "expected"), + [ + # name, friendly_name, device_name, expected + ("Living Room Light", None, None, "living_room_light"), + ("", "Kitchen Controller", None, "kitchen_controller"), + ( + "", + "ESP32 Device", + "controller_1", + "controller_1", + ), # Device name takes precedence + ("GPIO2 Button", None, None, "gpio2_button"), + ("WiFi Signal", "My Device", None, "wifi_signal"), + ("", None, "esp32_node", "esp32_node"), + ("Front Door Sensor", "Home Assistant", "door_controller", "front_door_sensor"), + ], +) +def test_real_world_examples( + name: str, friendly_name: str | None, device_name: str | None, expected: str +) -> None: + """Test real-world entity naming scenarios.""" + result = get_base_entity_object_id(name, friendly_name, device_name) + assert result == expected + + +def test_issue_6953_scenarios() -> None: + """Test specific scenarios from issue #6953.""" + # Scenario 1: Multiple empty names on main device with name_add_mac_suffix + # The Python code calculates the base, C++ might append MAC suffix dynamically + CORE.name = "device-name" + CORE.friendly_name = "Friendly Device" + + # All empty names should resolve to same base + assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" + assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" + assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" + + # Scenario 2: Empty names on sub-devices + assert ( + get_base_entity_object_id("", "Main Device", "controller_1") == "controller_1" + ) + assert ( + get_base_entity_object_id("", "Main Device", "controller_2") == "controller_2" + ) + + # Scenario 3: xyz duplicates + assert get_base_entity_object_id("xyz", None) == "xyz" + assert get_base_entity_object_id("xyz", "Device") == "xyz" # Tests for setup_entity function @@ -336,10 +515,9 @@ def test_entity_duplicate_validator() -> None: config1 = {CONF_NAME: "Temperature"} validated1 = validator(config1) assert validated1 == config1 - temperature_key = ("", "sensor", fnv1_hash_name("Temperature")) - assert temperature_key in CORE.unique_ids + assert ("", "sensor", fnv1_hash("temperature")) in CORE.unique_ids # Check metadata was stored - metadata = CORE.unique_ids[temperature_key] + metadata = CORE.unique_ids[("", "sensor", fnv1_hash("temperature"))] assert metadata["name"] == "Temperature" assert metadata["platform"] == "sensor" @@ -347,9 +525,8 @@ def test_entity_duplicate_validator() -> None: config2 = {CONF_NAME: "Humidity"} validated2 = validator(config2) assert validated2 == config2 - humidity_key = ("", "sensor", fnv1_hash_name("Humidity")) - assert humidity_key in CORE.unique_ids - metadata2 = CORE.unique_ids[humidity_key] + assert ("", "sensor", fnv1_hash("humidity")) in CORE.unique_ids + metadata2 = CORE.unique_ids[("", "sensor", fnv1_hash("humidity"))] assert metadata2["name"] == "Humidity" # Duplicate entity should fail @@ -360,6 +537,34 @@ def test_entity_duplicate_validator() -> None: validator(config3) +def test_entity_duplicate_validator_hash_collision() -> None: + """Test that two different object_ids with the same FNV-1 hash are rejected.""" + # Brute-forced FNV-1 32-bit collision pair; both object_ids hash to 0xe95747e4 + name_a = "Sensor aooxzi" + name_b = "Sensor baraia" + object_id_a = sanitize(snake_case(name_a)) + object_id_b = sanitize(snake_case(name_b)) + assert object_id_a != object_id_b + assert fnv1_hash(object_id_a) == fnv1_hash(object_id_b) + + validator = entity_duplicate_validator("sensor") + + config1 = {CONF_NAME: name_a} + validated1 = validator(config1) + assert validated1 == config1 + + config2 = {CONF_NAME: name_b} + with pytest.raises( + Invalid, + match=re.compile( + r"Duplicate sensor entity with name 'Sensor baraia' found.*" + r"produce the same entity key hash \(0xe95747e4\)", + re.DOTALL, + ), + ): + validator(config2) + + def test_entity_duplicate_validator_with_devices() -> None: """Test entity_duplicate_validator with devices.""" # Create validator for sensor platform @@ -370,19 +575,18 @@ def test_entity_duplicate_validator_with_devices() -> None: device2 = ID("device2", type="Device") # Same name on different devices should pass - name_hash = fnv1_hash_name("Temperature") config1 = {CONF_NAME: "Temperature", CONF_DEVICE_ID: device1} validated1 = validator(config1) assert validated1 == config1 - assert ("device1", "sensor", name_hash) in CORE.unique_ids - metadata1 = CORE.unique_ids[("device1", "sensor", name_hash)] + assert ("device1", "sensor", fnv1_hash("temperature")) in CORE.unique_ids + metadata1 = CORE.unique_ids[("device1", "sensor", fnv1_hash("temperature"))] assert metadata1["device_id"] == "device1" config2 = {CONF_NAME: "Temperature", CONF_DEVICE_ID: device2} validated2 = validator(config2) assert validated2 == config2 - assert ("device2", "sensor", name_hash) in CORE.unique_ids - metadata2 = CORE.unique_ids[("device2", "sensor", name_hash)] + assert ("device2", "sensor", fnv1_hash("temperature")) in CORE.unique_ids + metadata2 = CORE.unique_ids[("device2", "sensor", fnv1_hash("temperature"))] assert metadata2["device_id"] == "device2" # Duplicate on same device should fail @@ -434,33 +638,6 @@ def test_entity_different_platforms_yaml_validation( assert result is not None -def test_object_id_conflict_mqtt_yaml_validation( - yaml_file: Callable[[str], str], capsys: pytest.CaptureFixture[str] -) -> None: - """Test that names sanitizing to the same object_id fail when mqtt is configured.""" - result = load_config_from_fixture( - yaml_file, "object_id_conflict_mqtt.yaml", FIXTURES_DIR - ) - assert result is None - - captured = capsys.readouterr() - assert ( - "mqtt builds default topics and discovery topics from the entity object_id" - in captured.out - ) - - -def test_object_id_conflict_without_mqtt_yaml_validation( - yaml_file: Callable[[str], str], -) -> None: - """Test that names sanitizing to the same object_id pass without mqtt/prometheus.""" - result = load_config_from_fixture( - yaml_file, "object_id_conflict_no_mqtt.yaml", FIXTURES_DIR - ) - # This should succeed - assert result is not None - - def test_entity_duplicate_validator_error_message() -> None: """Test that duplicate entity error messages include helpful metadata.""" # Create validator for sensor platform @@ -519,8 +696,7 @@ def test_entity_duplicate_validator_internal_entities() -> None: validated1 = validator(config1) assert validated1 == config1 # New format includes device_id (empty string for main device) - temperature_key = ("", "sensor", fnv1_hash_name("Temperature")) - assert temperature_key in CORE.unique_ids + assert ("", "sensor", fnv1_hash("temperature")) in CORE.unique_ids # Internal entity with same name should pass (not added to unique_ids) config2 = {CONF_NAME: "Temperature", CONF_INTERNAL: True} @@ -528,7 +704,9 @@ def test_entity_duplicate_validator_internal_entities() -> None: assert validated2 == config2 # Internal entity should not be added to unique_ids # Count how many times the key appears (should still be 1) - count = sum(1 for k in CORE.unique_ids if k == temperature_key) + count = sum( + 1 for k in CORE.unique_ids if k == ("", "sensor", fnv1_hash("temperature")) + ) assert count == 1 # Another internal entity with same name should also pass @@ -536,7 +714,9 @@ def test_entity_duplicate_validator_internal_entities() -> None: validated3 = validator(config3) assert validated3 == config3 # Still only one entry in unique_ids (from the non-internal entity) - count = sum(1 for k in CORE.unique_ids if k == temperature_key) + count = sum( + 1 for k in CORE.unique_ids if k == ("", "sensor", fnv1_hash("temperature")) + ) assert count == 1 # Non-internal entity with same name should fail @@ -564,148 +744,30 @@ def test_empty_or_null_device_id_on_entity() -> None: def test_entity_duplicate_validator_non_ascii_names() -> None: - """Test that distinct non-ASCII names no longer collide. - - These names used to be rejected because both sanitize to only underscores; - the entity key now hashes the raw name so they stay distinct. - """ + """Test that non-ASCII names show helpful error messages.""" # Create validator for binary_sensor platform validator = entity_duplicate_validator("binary_sensor") - # Both Russian sensors should pass even though they sanitize identically + # First Russian sensor should pass config1 = {CONF_NAME: "Датчик открытия основного крана"} validated1 = validator(config1) assert validated1 == config1 + # Second Russian sensor with different text but same ASCII conversion should fail config2 = {CONF_NAME: "Датчик закрытия основного крана"} - validated2 = validator(config2) - assert validated2 == config2 - - # An exact duplicate still fails - config3 = {CONF_NAME: "Датчик открытия основного крана"} - with pytest.raises( - Invalid, - match=r"Duplicate binary_sensor entity with name 'Датчик открытия основного крана' found", - ): - validator(config3) - - -def test_entity_duplicate_validator_hash_collision() -> None: - """Test that two different names with the same FNV-1 hash are rejected.""" - # Brute-forced FNV-1 32-bit collision pair; both hash to 0x0ee5ff7b - name_a = "Sensor m2CZ" - name_b = "Sensor qCaa" - assert name_a != name_b - assert fnv1_hash_name(name_a) == fnv1_hash_name(name_b) - - validator = entity_duplicate_validator("sensor") - - config1 = {CONF_NAME: name_a} - validated1 = validator(config1) - assert validated1 == config1 - - config2 = {CONF_NAME: name_b} with pytest.raises( Invalid, match=re.compile( - rf"Duplicate sensor entity with name '{name_b}' found.*" - rf"The names '{name_b}' and '{name_a}' produce the.*" - r"same entity key hash \(0x0ee5ff7b\).*" - r"To fix: Rename one of the entities", + r"Duplicate binary_sensor entity with name 'Датчик закрытия основного крана' found.*" + r"Original names: 'Датчик закрытия основного крана' and 'Датчик открытия основного крана'.*" + r"Both convert to ASCII ID: '_______________________________'.*" + r"To fix: Add unique ASCII characters \(e\.g\., '1', '2', or 'A', 'B'\)", re.DOTALL, ), ): validator(config2) -def test_object_id_conflicts_rejected_by_component_validator() -> None: - """Test that object_id conflicts pass entity validation but fail for mqtt/prometheus.""" - validator = entity_duplicate_validator("sensor") - - # Both names validate fine in general (distinct raw names, distinct keys) - validator({CONF_NAME: "Датчик открытия"}) - validator({CONF_NAME: "Датчик закрытия"}) - - # A component that addresses entities by object_id must reject the config - component_validator = validate_no_object_id_conflicts( - "mqtt builds default topics from the entity object_id" - ) - with pytest.raises( - Invalid, - match=re.compile( - r"mqtt builds default topics from the entity object_id.*" - r"sensor entities 'Датчик открытия', 'Датчик закрытия' " - r"share the object_id '_______________'.*" - r"To fix: Add unique ASCII characters", - re.DOTALL, - ), - ): - component_validator({}) - - -def test_object_id_conflicts_skipped_in_testing_mode() -> None: - """Test that testing_mode skips the conflict check, as used for grouped testing.""" - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Датчик открытия"}) - validator({CONF_NAME: "Датчик закрытия"}) - - component_validator = validate_no_object_id_conflicts( - "mqtt builds default topics from the entity object_id" - ) - CORE.testing_mode = True - try: - config: dict = {} - assert component_validator(config) is config - finally: - CORE.testing_mode = False - - -def test_object_id_conflicts_none_recorded() -> None: - """Test that distinct object_ids produce no conflicts.""" - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Temperature"}) - validator({CONF_NAME: "Humidity"}) - - component_validator = validate_no_object_id_conflicts( - "mqtt builds default topics from the entity object_id" - ) - config: dict = {} - assert component_validator(config) is config - - -def test_object_id_conflicts_device_scoped() -> None: - """Test that the object_id conflict check is scoped per device. - - Same-named entities on different sub-devices were accepted before entity keys - moved to raw names, so the check keeps that scope; conflicts within one device - are still reported with the device named in the message. - """ - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Temperature", CONF_DEVICE_ID: ID("device1", type="Device")}) - validator({CONF_NAME: "Temperature", CONF_DEVICE_ID: ID("device2", type="Device")}) - - component_validator = validate_no_object_id_conflicts( - "prometheus builds metric labels from the entity object_id" - ) - config: dict = {} - assert component_validator(config) is config - - # Two names sanitizing identically on the same sub-device still conflict - validator( - {CONF_NAME: "Датчик открытия", CONF_DEVICE_ID: ID("device1", type="Device")} - ) - validator( - {CONF_NAME: "Датчик закрытия", CONF_DEVICE_ID: ID("device1", type="Device")} - ) - with pytest.raises( - Invalid, - match=re.compile( - r"prometheus builds metric labels.*on device 'device1'", re.DOTALL - ), - ): - component_validator({}) - - def test_entity_duplicate_validator_same_name_no_enhanced_message() -> None: """Test that identical names don't show the enhanced message.""" # Create validator for sensor platform @@ -763,7 +825,7 @@ async def test_setup_entity_empty_name_with_device( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 @pytest.mark.asyncio @@ -792,7 +854,7 @@ async def test_setup_entity_empty_name_with_mac_suffix( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 @pytest.mark.asyncio @@ -822,7 +884,7 @@ async def test_setup_entity_empty_name_with_mac_suffix_no_friendly_name( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 @pytest.mark.asyncio @@ -853,7 +915,7 @@ async def test_setup_entity_empty_name_no_mac_suffix_no_friendly_name( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 def test_register_string_overflow() -> None: diff --git a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml deleted file mode 100644 index 4a6f56f473..0000000000 --- a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml +++ /dev/null @@ -1,22 +0,0 @@ -esphome: - name: test-object-id-conflict - -esp32: - board: esp32dev - -wifi: - ssid: MySSID - password: password1 - -mqtt: - broker: test.mosquitto.org - -sensor: - # Distinct raw names are fine in general, but both sanitize to the same - # object_id, which MQTT still uses to build default topics - should fail - - platform: template - name: "Датчик открытия" - lambda: return 21.0; - - platform: template - name: "Датчик закрытия" - lambda: return 22.0; diff --git a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml deleted file mode 100644 index c0fbd5cbba..0000000000 --- a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml +++ /dev/null @@ -1,15 +0,0 @@ -esphome: - name: test-object-id-ok - -esp32: - board: esp32dev - -sensor: - # Distinct raw names that sanitize to the same object_id are allowed when no - # component addresses entities by object_id (no mqtt or prometheus configured) - - platform: template - name: "Датчик открытия" - lambda: return 21.0; - - platform: template - name: "Датчик закрытия" - lambda: return 22.0; diff --git a/tests/unit_tests/test_preference_hash_stability.py b/tests/unit_tests/test_preference_hash_stability.py index d3e5fac36a..d8506afae7 100644 --- a/tests/unit_tests/test_preference_hash_stability.py +++ b/tests/unit_tests/test_preference_hash_stability.py @@ -5,11 +5,11 @@ users to lose stored preferences (calibration values, restore states, etc.) on firmware upgrades, or break entity state routing to API clients. Two algorithms are locked here (see https://github.com/esphome/backlog/issues/85): -1. `fnv1_hash_object_id(name)` - the LEGACY hash (snake_case + sanitize, then FNV-1). - Existing devices have preferences stored under keys derived from it; slot-based - backends (ESP8266, RP2040) keep using it, and key-lookup backends migrate FROM it. -2. `fnv1_hash_name(name)` - the entity key (FNV-1 over the raw UTF-8 name bytes). - Sent to API clients and used as the preference key base on key-lookup backends. +1. `fnv1_hash_object_id(name)` - the object_id hash (snake_case + sanitize, then FNV-1). + The entity key sent to API clients and the base of every stored preference key. +2. `fnv1_hash_name(name)` - FNV-1 over the raw UTF-8 name bytes. 2026.8 beta + firmware stored preferences under keys derived from it; a future key migration + must reconstruct those keys to recover that data. DO NOT CHANGE THE EXPECTED VALUES - if tests fail after modifying a hash algorithm, the change breaks backward compatibility and will cause data loss. @@ -124,8 +124,9 @@ def test_entity_object_id_hash_stability( """Verify fnv1_hash_object_id produces stable hashes for entity names. CRITICAL: These expected values MUST NOT CHANGE. Existing devices have - preferences stored under keys derived from this legacy hash; changing it - breaks the old-to-new key migration and loses stored preferences. + preferences stored under keys derived from this hash, and it is the entity + key sent to API clients; changing it loses stored preferences and breaks + entity state routing. """ actual = fnv1_hash_object_id(entity_name) assert actual == expected_object_id_hash, ( @@ -144,9 +145,8 @@ def compute_legacy_preference_key( ) -> int: """Compute the legacy preference key: (object_id_hash ^ device_id) ^ version. - This is the key existing devices have data stored under. Slot-based backends - (ESP8266, RP2040) still use it directly; key-lookup backends compute it as the - migration source in EntityBase::make_entity_preference_() (entity_base.cpp). + This is the key EntityBase::make_entity_preference_() (entity_base.cpp) + stores every entity preference under. """ object_id_hash = fnv1_hash_object_id(entity_name) preference_hash = object_id_hash ^ device_id @@ -179,8 +179,8 @@ def test_legacy_preference_key_computation( ) -> None: """Verify legacy preference key computation matches expected values. - This test ensures the formula doesn't change, which would break both slot-based - preference storage and the migration source keys on key-lookup backends. + This test ensures the formula doesn't change, which would lose stored + preferences on every platform. """ actual_key = compute_legacy_preference_key(entity_name, version, device_id) @@ -215,12 +215,12 @@ def test_legacy_preference_key_computation( ], ) def test_entity_key_hash_stability(entity_name: str, expected_key: int) -> None: - """Verify fnv1_hash_name produces stable entity keys. + """Verify fnv1_hash_name produces stable raw-name hashes. - CRITICAL: These expected values MUST NOT CHANGE. The entity key is sent to - API clients and is the new preference key base; changing the algorithm - would break state routing and lose stored preferences. - Must match C++ fnv1_hash_bytes() in esphome/core/helpers.h. + CRITICAL: These expected values MUST NOT CHANGE. 2026.8 beta firmware stored + preferences under keys derived from this hash; a future key migration must + reconstruct those keys, and changing the algorithm would strand that data. + Matched C++ fnv1_hash_bytes() (2026.8 beta), which the unrevert restores. """ actual = fnv1_hash_name(entity_name) assert actual == expected_key, ( From 990fc402fdf12fd71e0d327edc3045591617aa53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:24:02 -0500 Subject: [PATCH 029/160] Bump bleak from 2.1.1 to 3.0.2 (#16246) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6bc8bdf74a..876b13793c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ pillow==12.3.0 resvg-py==0.3.4 freetype-py==2.5.1 jinja2==3.1.6 -bleak==2.1.1 +bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 From b05465145fb261eb3d142982fda2d4741fa49c92 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 21:14:30 -0500 Subject: [PATCH 030/160] [core] Add preference key stability integration test (#18364) --- .../fixtures/preference_key_stability.yaml | 35 ++++ tests/integration/host_prefs.py | 24 ++- .../test_preference_key_stability.py | 168 ++++++++++++++++++ 3 files changed, 220 insertions(+), 7 deletions(-) create mode 100644 tests/integration/fixtures/preference_key_stability.yaml create mode 100644 tests/integration/test_preference_key_stability.py diff --git a/tests/integration/fixtures/preference_key_stability.yaml b/tests/integration/fixtures/preference_key_stability.yaml new file mode 100644 index 0000000000..a74bb2c7f7 --- /dev/null +++ b/tests/integration/fixtures/preference_key_stability.yaml @@ -0,0 +1,35 @@ +esphome: + name: host-pref-key-stability + +host: +api: +logger: + +switch: + - platform: template + id: test_switch_restore + name: Test Switch + optimistic: true + restore_mode: RESTORE_DEFAULT_OFF + +number: + - platform: template + id: test_number_restore + name: Test Number + optimistic: true + restore_value: true + initial_value: 1.0 + min_value: 0 + max_value: 100 + step: 0.5 + +text: + - platform: template + id: test_text_restore + name: Test Text + mode: text + optimistic: true + restore_value: true + initial_value: fallback + min_length: 0 + max_length: 20 diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index f835bee3bc..c7f21d8a01 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -25,15 +25,25 @@ def clear_host_prefs(device_name: str) -> None: host_prefs_path(device_name).unlink(missing_ok=True) +def write_host_prefs(device_name: str, entries: dict[int, bytes]) -> Path: + """Write preference entries, replacing the file's contents. + + Returns the path that was written. + """ + payload = b"" + for key, data in entries.items(): + if len(data) > 255: + raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") + payload += struct.pack(" Path: """Write a single preference entry, replacing the file's contents. Returns the path that was written. """ - if len(data) > 255: - raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") - path = host_prefs_path(device_name) - path.parent.mkdir(parents=True, exist_ok=True) - payload = struct.pack(" stores a length-prefixed buffer of max_length + 1 bytes +TEXT_MAX_LENGTH = 20 + + +def text_pref_payload(value: str) -> bytes: + """Build the length-prefixed buffer TextSaver stores for a value.""" + data = value.encode("utf-8") + assert len(data) <= TEXT_MAX_LENGTH + return bytes([len(data)]) + data + b"\x00" * (TEXT_MAX_LENGTH - len(data)) + + +@pytest.mark.asyncio +async def test_preference_key_stability( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> None: + """Test that preferences stored by earlier firmware are restored.""" + port, port_socket = reserved_tcp_port + + assert SWITCH_KEY != SWITCH_BETA_KEY + assert NUMBER_KEY != NUMBER_BETA_KEY + assert TEXT_KEY != TEXT_BETA_KEY + + # Write and compile once + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + + # Release the reserved port so the binary can bind to it + port_socket.close() + + async def boot_and_get_initial_states() -> tuple[ + SwitchState, NumberState, TextState + ]: + """Boot the binary and return the restored entity states.""" + async with ( + run_binary_and_wait_for_port(binary_path, "127.0.0.1", port), + wait_and_connect_api_client(port=port) as client, + ): + device_info = await client.device_info() + assert device_info.name == DEVICE_NAME + + entities, _ = await client.list_entities_services() + switch_entity = require_entity( + entities, "test_switch", SwitchInfo, "Test Switch" + ) + number_entity = require_entity( + entities, "test_number", NumberInfo, "Test Number" + ) + text_entity = require_entity(entities, "test_text", TextInfo, "Test Text") + + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda s: None) + ) + await initial_state_helper.wait_for_initial_states() + + switch_state = initial_state_helper.initial_states[switch_entity.key] + number_state = initial_state_helper.initial_states[number_entity.key] + text_state = initial_state_helper.initial_states[text_entity.key] + assert isinstance(switch_state, SwitchState) + assert isinstance(number_state, NumberState) + assert isinstance(text_state, TextState) + return switch_state, number_state, text_state + + try: + # --- Run 1: entries under the object_id-hash keys, exactly as any + # earlier firmware wrote them. The restored states prove the key + # scheme has not drifted. + write_host_prefs( + DEVICE_NAME, + { + SWITCH_KEY: b"\x01", # bool: switch was ON + NUMBER_KEY: struct.pack(" Date: Fri, 14 Aug 2026 00:22:22 -0500 Subject: [PATCH 031/160] [core] Restore cv.parse_esphome_version as a deprecated helper (#18366) --- esphome/config_validation.py | 4 ++++ esphome/util.py | 14 ++++++++++++++ tests/unit_tests/test_config_validation.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 0eebf12e66..f455c7b8bf 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -99,6 +99,10 @@ from esphome.schema_extractors import ( schema_extractor_registry, schema_extractor_typed, ) + +# Deprecated re-export for external components; remove before 2027.2.0 +# pylint: disable-next=unused-import +from esphome.util import parse_esphome_version # noqa: F401 from esphome.voluptuous_schema import _Schema from esphome.yaml_util import SensitiveStr, make_data_base diff --git a/esphome/util.py b/esphome/util.py index 2fc34f3a69..b8ffa048ca 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -390,6 +390,20 @@ def is_dev_esphome_version(): return "dev" in const.__version__ +# Remove before 2027.2.0 +def parse_esphome_version() -> tuple[int, int, int]: + """Deprecated: use esphome.config_validation.require_esphome_version instead.""" + from esphome.core import Version + + _LOGGER.warning( + "parse_esphome_version() is deprecated. Use " + "cv.require_esphome_version to gate on a minimum version. " + "Removed in 2027.2.0" + ) + version = Version.parse(const.__version__) + return version.major, version.minor, version.patch + + # Custom OrderedDict with nicer repr method for debugging class OrderedDict(collections.OrderedDict): def __repr__(self): diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 7627ef9273..971c4e462d 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -2967,6 +2967,23 @@ def test_require_esphome_version_older_prerelease_fails() -> None: cv.require_esphome_version(2026, 8, 0)("test") +def test_parse_esphome_version_deprecated_shim( + caplog: pytest.LogCaptureFixture, +) -> None: + """The removed helper still works for external components and warns.""" + from esphome import const, util + + with ( + patch.object(const, "__version__", "2026.9.0-dev"), + caplog.at_level(logging.WARNING), + ): + assert cv.parse_esphome_version() == (2026, 9, 0) + assert cv.parse_esphome_version() < (9999, 0, 0) + assert "parse_esphome_version() is deprecated" in caplog.text + # Both historical import paths resolve to the same function + assert cv.parse_esphome_version is util.parse_esphome_version + + # --------------------------------------------------------------------------- # suppress_invalid / validate_source_shorthand / rename_key # --------------------------------------------------------------------------- From 617e2ec1e051f181ca7892966c2be34c46e2e806 Mon Sep 17 00:00:00 2001 From: Karl Beecken Date: Fri, 14 Aug 2026 07:23:34 +0200 Subject: [PATCH 032/160] [core] fix PYTHONPATH leak (#18360) --- esphome/espidf/toolchain.py | 2 ++ esphome/framework_helpers.py | 2 ++ tests/unit_tests/test_espidf_toolchain.py | 15 +++++++++++++++ tests/unit_tests/test_framework_helpers.py | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index e1688f4170..bb6452acf2 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -109,6 +109,8 @@ def _get_idf_env(version: str | None = None) -> dict[str, str]: env_cache = _cache().env if version not in env_cache: env_cache[version] = os.environ.copy() + # Do not leak PYTHONPATH into child env + env_cache[version].pop("PYTHONPATH", None) # Use provided IDF framework if available if "IDF_PATH" not in os.environ: diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 86d5e4eaea..b8a43220ff 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -155,6 +155,8 @@ def run_command( _LOGGER.debug("%s - running ...", cmd_str) run_env = os.environ.copy() + # Do not leak PYTHONPATH + run_env.pop("PYTHONPATH", None) if env: run_env.update(env) diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 56f358a24c..26d812af8b 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -265,6 +265,21 @@ def test_get_idf_env_sets_git_ceiling_directories(setup_core: Path) -> None: assert str(CORE.config_dir) in env["GIT_CEILING_DIRECTORIES"].split(os.pathsep) +def test_get_idf_env_pops_inherited_pythonpath(setup_core: Path) -> None: + """A PYTHONPATH from the parent environment must not reach idf.py. + + It would override the IDF venv's isolation, shadowing its pinned + packages and failing idf.py's dependency check. + """ + toolchain._cache().env.clear() + with patch.dict( + os.environ, + {"IDF_PATH": str(setup_core), "PYTHONPATH": "/outside/site-packages"}, + ): + env = toolchain._get_idf_env(version="5.5.4") + assert "PYTHONPATH" not in env + + def test_get_cmake_output_without_build_dir(setup_core: Path) -> None: """A build dir that was never created raises EsphomeError. diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 7451ee9b39..2022c15bfe 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -188,6 +188,24 @@ def test_run_command_passes_env(mock_subprocess_run: Mock) -> None: assert mock_subprocess_run.call_args[1]["env"]["MY_VAR"] == "42" +def test_run_command_pops_inherited_pythonpath(mock_subprocess_run: Mock) -> None: + """A PYTHONPATH from the parent environment must not leak into subprocesses.""" + mock_subprocess_run.return_value = Mock(returncode=0, stdout="", stderr="") + with patch.dict(os.environ, {"PYTHONPATH": "/outside/site-packages"}): + run_command(["cmd"]) + assert "PYTHONPATH" not in mock_subprocess_run.call_args[1]["env"] + + +def test_run_command_env_pythonpath_preferred_over_pop( + mock_subprocess_run: Mock, +) -> None: + """A PYTHONPATH set explicitly via ``env`` is passed through.""" + mock_subprocess_run.return_value = Mock(returncode=0, stdout="", stderr="") + with patch.dict(os.environ, {"PYTHONPATH": "/outside/site-packages"}): + run_command(["cmd"], env={"PYTHONPATH": "/idf/tools"}) + assert mock_subprocess_run.call_args[1]["env"]["PYTHONPATH"] == "/idf/tools" + + def test_run_command_passes_cwd(mock_subprocess_run: Mock, tmp_path: Path) -> None: mock_subprocess_run.return_value = Mock(returncode=0, stdout="", stderr="") run_command(["cmd"], cwd=str(tmp_path)) From d72bab79d7363c81d849078556f1de71953cf60c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 12 Aug 2026 21:09:01 -0500 Subject: [PATCH 033/160] [web_server_base] Stop deleting the web server on captive portal teardown (#18324) --- .../web_server/ota/ota_web_server.cpp | 2 +- .../web_server_base/web_server_base.h | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/esphome/components/web_server/ota/ota_web_server.cpp b/esphome/components/web_server/ota/ota_web_server.cpp index 9812714ec0..95763e2daf 100644 --- a/esphome/components/web_server/ota/ota_web_server.cpp +++ b/esphome/components/web_server/ota/ota_web_server.cpp @@ -249,7 +249,7 @@ void WebServerOTAComponent::setup() { return; } - // AsyncWebServer takes ownership of the handler and will delete it when the server is destroyed + // The handler lives for the life of the process; WebServerBase never destroys its server base->add_handler(new OTARequestHandler(this)); // NOLINT } diff --git a/esphome/components/web_server_base/web_server_base.h b/esphome/components/web_server_base/web_server_base.h index c647a13b50..94579de70f 100644 --- a/esphome/components/web_server_base/web_server_base.h +++ b/esphome/components/web_server_base/web_server_base.h @@ -112,9 +112,18 @@ class AuthMiddlewareHandler : public MiddlewareHandler { class WebServerBase final { public: + // The AsyncWebServer is created once and intentionally never deleted: on Arduino + // platforms ESPAsyncWebServer owns its registered handlers, so destroying it would + // also destroy live components (e.g. the captive portal) out from under us. + // init()/deinit() refcount users and start/stop the listener; handlers are + // registered once at creation and survive listener restarts. void init() { - if (this->initialized_) { - this->initialized_++; + this->initialized_++; + if (this->server_ != nullptr) { + if (this->initialized_ == 1) { + // Restart the listener after a previous deinit() + this->server_->begin(); + } return; } this->server_ = new AsyncWebServer(this->port_); @@ -126,14 +135,13 @@ class WebServerBase final { for (auto *handler : this->handlers_) this->server_->addHandler(handler); - - this->initialized_++; } void deinit() { + if (this->initialized_ == 0) + return; // unbalanced deinit() this->initialized_--; if (this->initialized_ == 0) { - delete this->server_; - this->server_ = nullptr; + this->server_->end(); } } AsyncWebServer *get_server() const { return this->server_; } From 7c07fb48c5cbf1c5ca7c8ba04e033d2d0eb14568 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:22:48 -0500 Subject: [PATCH 034/160] [esp32_ble_tracker] Fix missed BLE advertisements with WiFi on ESP-IDF 5.5.5 (#18356) --- .../components/ble_device_base/__init__.py | 20 ++- .../components/esp32_ble_tracker/__init__.py | 71 +++++++++- .../test_scan_parameter_validation.py | 7 +- .../esp32_ble_tracker/__init__.py | 0 .../test_scan_window_default.py | 122 ++++++++++++++++++ 5 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 tests/component_tests/esp32_ble_tracker/__init__.py create mode 100644 tests/component_tests/esp32_ble_tracker/test_scan_window_default.py diff --git a/esphome/components/ble_device_base/__init__.py b/esphome/components/ble_device_base/__init__.py index 4da7d48882..15a8b08139 100644 --- a/esphome/components/ble_device_base/__init__.py +++ b/esphome/components/ble_device_base/__init__.py @@ -37,7 +37,7 @@ from esphome.const import ( CONF_INTERVAL, KEY_TARGET_PLATFORM, ) -from esphome.core import CORE, ID, KEY_CORE +from esphome.core import CORE, ID, KEY_CORE, TimePeriod from esphome.types import ConfigType CODEOWNERS = ["@Bl00d-B0b"] @@ -243,19 +243,27 @@ def validate_scan_parameters(config: ConfigType) -> ConfigType: return config +# The historical scan window default shared by the trackers that do not pin +# their own; also the fallback for esp32's conditional default. +DEFAULT_SCAN_WINDOW = "30ms" + + def scan_parameters_schema( interval_default: str, *, - window_default: str = "30ms", + window_default: str | Callable[[], TimePeriod] = DEFAULT_SCAN_WINDOW, ) -> cv.All: """Build the scan_parameters value schema shared by all BLE trackers. interval_default and window_default are per chip (e.g. esp32 320/30 ms, bk72xx/rp2 100/30 ms — the reference scan rates of the respective stacks; - LN882H's SDK recommends 100/50 ms). The `active` option (default on) is - unconditional: active scanning is part of the tracker contract — every - current proxy client assumes it, so a passive-only tracker must not share - this schema. + LN882H's SDK recommends 100/50 ms). window_default may also be a zero-arg + callable evaluated per validation when the user omits the key (esp32 uses + this to record that the window was defaulted, so a later validation step + can adjust it once sibling keys are resolved). The `active` option + (default on) is unconditional: active scanning is part of the tracker + contract — every current proxy client assumes it, so a passive-only + tracker must not share this schema. """ schema = { cv.Optional(CONF_DURATION, default="5min"): cv.positive_time_period_seconds, diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 634b8c3bef..28c8c7fcf1 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -1,5 +1,7 @@ from __future__ import annotations +import copy +from dataclasses import dataclass import logging from esphome import automation @@ -8,6 +10,7 @@ from esphome.components import ble_device_base, esp32_ble, ota from esphome.components.const import CONF_ON_SCAN_END, CONF_SCAN_PARAMETERS, CONF_WINDOW from esphome.components.esp32 import ( add_idf_sdkconfig_option, + idf_version, request_bluetooth, request_software_coexistence, ) @@ -35,10 +38,12 @@ from esphome.const import ( CONF_SERVICE_UUID, CONF_TRIGGER_ID, ) -from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.core import CORE, CoroPriority, TimePeriod, coroutine_with_priority from esphome.enum import StrEnum from esphome.types import ConfigType +DOMAIN = "esp32_ble_tracker" + AUTO_LOAD = ["ble_device_base", "esp32_ble"] DEPENDENCIES = ["esp32"] CODEOWNERS = ["@bdraco"] @@ -125,10 +130,71 @@ def validate_max_connections_deprecated(config: ConfigType) -> ConfigType: return config +# ESP-IDF 5.5.5 fixed a coexistence bug on the ESP32 where BLE scans ran far +# longer than the configured window (espressif/esp-idf#18931). Before the fix, +# the default 30 ms window in a 320 ms interval effectively scanned at a much +# higher duty cycle than requested; with the fix, that same default only +# listens 9.4 % of the time and misses most advertisements when wifi shares +# the radio. Espressif recommends setting the window equal to the interval in +# that case: the coexistence arbiter still shares the radio with wifi, and +# BLE uses the airtime wifi does not claim. +IDF_SCAN_WINDOW_FIX_VERSION = cv.Version(5, 5, 5) + + +@dataclass +class TrackerData: + """Per-run validation state, namespaced under DOMAIN in CORE.data.""" + + scan_window_defaulted: bool = False + + +def _get_data() -> TrackerData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = TrackerData() + return CORE.data[DOMAIN] + + +def _scan_window_default() -> TimePeriod: + """Schema default for the scan window. + + Records that the user did not set a window, so _raise_defaulted_scan_window + can tell a defaulted 30 ms from an explicit one; the raise itself must wait + for the outer schema because it depends on software_coexistence, a sibling + key not yet resolved here. + """ + _get_data().scan_window_defaulted = True + return cv.positive_time_period(ble_device_base.DEFAULT_SCAN_WINDOW) + + +def _raise_defaulted_scan_window(config: ConfigType) -> ConfigType: + """Raise a defaulted scan window to the interval where that is safe. + + Only when the coexistence arbiter is compiled in (software_coexistence, + present iff wifi is configured and not disabled by the user) and the IDF + honors the window strictly (>= 5.5.5); without the arbiter a full-duty + scan would starve wifi outright, and a user-set window is never touched. + Raising to the interval cannot invalidate the already-validated + parameters, so no re-validation is needed. + """ + if ( + _get_data().scan_window_defaulted + and config.get(CONF_SOFTWARE_COEXISTENCE) + and idf_version() >= IDF_SCAN_WINDOW_FIX_VERSION + ): + params = config[CONF_SCAN_PARAMETERS] + # Copy so the config dump shows a plain value instead of a YAML + # anchor/alias pair pointing at the interval. + params[CONF_WINDOW] = copy.copy(params[CONF_INTERVAL]) + return config + + # 320 ms is the ESP-IDF reference scan interval; the shared schema also # tightens validation to the controller's 2.5 ms .. 10240 ms range and rejects # window/interval pairs that collapse to the same 0.625 ms unit count. -SCAN_PARAMETERS_SCHEMA = ble_device_base.scan_parameters_schema("320ms") +# The window default is conditional (see _scan_window_default above). +SCAN_PARAMETERS_SCHEMA = ble_device_base.scan_parameters_schema( + "320ms", window_default=_scan_window_default +) # Codegen helpers are owned by ble_device_base; kept under the historical names # here for the components that import them from this module. @@ -183,6 +249,7 @@ CONFIG_SCHEMA = cv.All( } ).extend(cv.COMPONENT_SCHEMA), validate_max_connections_deprecated, + _raise_defaulted_scan_window, ) diff --git a/tests/component_tests/ble_device_base/test_scan_parameter_validation.py b/tests/component_tests/ble_device_base/test_scan_parameter_validation.py index 2549125a43..3774d990d3 100644 --- a/tests/component_tests/ble_device_base/test_scan_parameter_validation.py +++ b/tests/component_tests/ble_device_base/test_scan_parameter_validation.py @@ -57,7 +57,12 @@ def test_bk72xx_defaults_are_valid() -> None: def test_esp32_defaults_are_valid() -> None: - """esp32 pins the ESP-IDF reference rate and exposes active (default on).""" + """esp32 pins the ESP-IDF reference rate and exposes active (default on). + + Without wifi loaded, the conditional window default falls back to the + historical 30 ms; the wifi-aware resolution is covered by the + esp32_ble_tracker component tests. + """ config = ESP32_SCHEMA({}) assert to_ble_units(config["interval"]) == 512 assert to_ble_units(config["window"]) == 48 diff --git a/tests/component_tests/esp32_ble_tracker/__init__.py b/tests/component_tests/esp32_ble_tracker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py b/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py new file mode 100644 index 0000000000..8a25f488fa --- /dev/null +++ b/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py @@ -0,0 +1,122 @@ +"""Tests for the esp32_ble_tracker conditional scan window default. + +The scan window default depends on wifi coexistence and the IDF version: +IDF 5.5.5 fixed a coexistence bug where BLE scans ran far longer than the +configured window (espressif/esp-idf#18931), so on fixed versions the +historical 30 ms default would only listen 9.4 % of the time and miss most +advertisements. With the coexistence arbiter compiled in on a fixed IDF, the +window instead defaults to the interval, as Espressif recommends; without the +arbiter a full-duty scan would starve wifi, so the 30 ms default is kept. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest + +from esphome import config_validation as cv +from esphome.components.ble_device_base import to_ble_units +from esphome.components.const import CONF_SCAN_PARAMETERS, CONF_WINDOW +from esphome.components.esp32 import KEY_IDF_VERSION +from esphome.components.esp32_ble_tracker import ( + CONF_SOFTWARE_COEXISTENCE, + CONFIG_SCHEMA, +) +from esphome.const import CONF_INTERVAL, PlatformFramework +from esphome.core import CORE +from esphome.types import ConfigType + +from ..types import SetCoreConfigCallable + + +@pytest.fixture +def stage_esp32( + set_core_config: SetCoreConfigCallable, +) -> Callable[..., None]: + """Stage an esp32 build with a given IDF version and wifi presence.""" + + def stage(idf: str, *, wifi: bool) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_IDF_VERSION: cv.Version.parse(idf)}, + ) + if wifi: + # Makes cv.OnlyWith default software_coexistence to True, exactly + # as a real config with wifi: does. + CORE.loaded_integrations.add("wifi") + + return stage + + +def _scan_params(config: ConfigType) -> ConfigType: + return CONFIG_SCHEMA(config)[CONF_SCAN_PARAMETERS] + + +@pytest.mark.parametrize( + ("idf", "config", "expected_units"), + [ + ("5.5.5", {}, 512), # first fixed version, default 320 ms interval + ("6.0.1", {}, 512), # any newer version behaves the same + # Follows a user-set interval. + ("5.5.5", {"scan_parameters": {"interval": "1s"}}, 1600), + ], +) +def test_wifi_on_fixed_idf_defaults_window_to_interval( + stage_esp32: Callable[..., None], + idf: str, + config: ConfigType, + expected_units: int, +) -> None: + """With wifi coexistence on a fixed IDF, the window defaults to the interval.""" + stage_esp32(idf, wifi=True) + params = _scan_params(config) + assert params[CONF_WINDOW] == params[CONF_INTERVAL] + assert to_ble_units(params[CONF_WINDOW]) == expected_units + + +@pytest.mark.parametrize( + ("idf", "wifi", "config"), + [ + # Buggy IDF over-scans anyway; keep the 30 ms default. + ("5.5.4", True, {}), + # No wifi (e.g. ethernet) means no radio contention. + ("5.5.5", False, {}), + # Coexistence disabled: no arbiter, so a full-duty scan would starve + # wifi outright. + ("5.5.5", True, {CONF_SOFTWARE_COEXISTENCE: False}), + ], +) +def test_30ms_default_kept( + stage_esp32: Callable[..., None], + idf: str, + wifi: bool, + config: ConfigType, +) -> None: + stage_esp32(idf, wifi=wifi) + assert to_ble_units(_scan_params(config)[CONF_WINDOW]) == 48 + + +@pytest.mark.parametrize("window", ["60ms", "30ms"]) +def test_explicit_window_is_never_touched( + stage_esp32: Callable[..., None], window: str +) -> None: + """A user-set window wins over the conditional default. + + The explicit 30 ms case matters: it is indistinguishable from the + defaulted value by inspection, so the defaulted flag must separate them. + """ + stage_esp32("5.5.5", wifi=True) + params = _scan_params({"scan_parameters": {"window": window}}) + assert to_ble_units(params[CONF_WINDOW]) == to_ble_units( + cv.positive_time_period(window) + ) + + +def test_short_interval_without_window_still_rejected( + stage_esp32: Callable[..., None], +) -> None: + """The provisional 30 ms default validates against the interval as before.""" + stage_esp32("5.5.5", wifi=True) + with pytest.raises(cv.Invalid, match="needs to be smaller than scan interval"): + _scan_params({"scan_parameters": {"interval": "20ms"}}) From 236ff33a09e4865c4ee88a0aae711d71955640f8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:27:01 -0500 Subject: [PATCH 035/160] [esp32_ble] Silence spurious warnings for local key GAP events (#18359) --- esphome/components/esp32_ble/ble.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 16501ef3b2..e2d79173ff 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -648,6 +648,8 @@ void ESP32BLE::gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_pa case ESP_GAP_BLE_SET_PKT_LENGTH_COMPLETE_EVT: case ESP_GAP_BLE_PHY_UPDATE_COMPLETE_EVT: // BLE 5.0 PHY update complete case ESP_GAP_BLE_CHANNEL_SELECT_ALGORITHM_EVT: // BLE 5.0 channel selection algorithm + case ESP_GAP_BLE_LOCAL_IR_EVT: // Local identity root key generated at security init + case ESP_GAP_BLE_LOCAL_ER_EVT: // Local encryption root key generated at security init return; default: From 1c3a67b5e815617abf419721525c182d302931aa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:30:53 -0500 Subject: [PATCH 036/160] [wifi] Fix ESP8266 crash in cnx_node_search when lwIP transmits after disconnect (#18333) --- .../wifi/wifi_component_esp8266.cpp | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 719a276bf9..acaa94b13c 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -136,10 +136,21 @@ bool WiFiComponent::wifi_apply_power_save_() { https://github.com/d-a-v/Arduino/blob/0e7d21e17144cfc5f53c016191daca8723e89ee8/libraries/ESP8266WiFi/src/ESP8266WiFiSTA.cpp#L251 */ #undef netif_set_addr // need to call lwIP-v1.4 netif_set_addr() +#undef netif_set_down // need to call lwIP-v1.4 netif_set_down() extern "C" { struct netif *eagle_lwip_getif(int netif_index); void netif_set_addr(struct netif *netif, const ip4_addr_t *ip, const ip4_addr_t *netmask, const ip4_addr_t *gw); +void netif_set_down(struct netif *netif); }; + +// The SDK can free its WiFi connection node before taking the STA netif down, letting lwIP +// timers (e.g. IGMP reports armed by mDNS) transmit into the dead driver and crash in +// cnx_node_search; taking the netif down first makes the glue drop such frames (#18308). +static void sta_netif_down() { + struct netif *iface = eagle_lwip_getif(STATION_IF); + if (iface != nullptr) + netif_set_down(iface); +} #endif bool WiFiComponent::wifi_sta_ip_config_(const optional &manual_ip) { @@ -523,6 +534,9 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { global_wifi_component->sta_state_ = static_cast(ESP8266WiFiSTAState::ERROR_FAILED); } global_wifi_component->error_from_callback_ = true; +#if LWIP_VERSION_MAJOR != 1 + sta_netif_down(); +#endif #ifdef USE_WIFI_CONNECT_STATE_LISTENERS global_wifi_component->pending_.disconnect = true; #endif @@ -536,6 +550,9 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { // https://lbsfilm.at/blog/wpa2-authenticationmode-downgrade-in-espressif-microprocessors if (it.old_mode != AUTH_OPEN && it.new_mode == AUTH_OPEN) { ESP_LOGW(TAG, "Potential Authmode downgrade detected, disconnecting"); +#if LWIP_VERSION_MAJOR != 1 + sta_netif_down(); +#endif wifi_station_disconnect(); global_wifi_component->error_from_callback_ = true; } @@ -719,8 +736,12 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { bool WiFiComponent::wifi_disconnect_() { bool ret = true; // Only call disconnect if interface is up - if (wifi_get_opmode() & WIFI_STA) + if (wifi_get_opmode() & WIFI_STA) { +#if LWIP_VERSION_MAJOR != 1 + sta_netif_down(); +#endif ret = wifi_station_disconnect(); + } station_config conf{}; memset(&conf, 0, sizeof(conf)); ETS_UART_INTR_DISABLE(); From 4f3153375a7acb307de0ce6deef708975fff58cf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:31:06 -0500 Subject: [PATCH 037/160] [ota] Retry uploads that fail from network errors (#18332) --- esphome/espota2.py | 158 ++++++++++--- tests/unit_tests/test_espota2.py | 382 +++++++++++++++++++++++++++++-- 2 files changed, 493 insertions(+), 47 deletions(-) diff --git a/esphome/espota2.py b/esphome/espota2.py index fa15c1dda2..61e897f601 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +import contextlib import gzip import hashlib import io @@ -8,7 +9,6 @@ import logging from pathlib import Path import secrets import socket -import sys import time from typing import Any @@ -76,6 +76,14 @@ _SUPPORTED_OTA_TYPES: frozenset[int] = frozenset( UPLOAD_BLOCK_SIZE = 8192 UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 +# Flaky Wi-Fi links often drop the first OTA attempt, and the device may need time +# to clean up a half-open connection (its handshake watchdog runs at 20s) before it +# accepts a new one, so wait between attempts instead of failing the upload outright. +# Every resolved address is tried once, and this many extra attempts are shared +# across the addresses on top of that. +EXTRA_UPLOAD_ATTEMPTS = 2 +UPLOAD_RETRY_DELAY = 5.0 + _LOGGER = logging.getLogger(__name__) # Authentication method lookup table: response -> (hash_func, nonce_size, name) @@ -171,6 +179,23 @@ class OTAError(EsphomeError): pass +class OTANetworkError(OTAError): + """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" + + +def _committed_error(err: OTANetworkError) -> OTAError: + """Wrap a network failure that happened once the device had the full image. + + Past that point the device commits and reboots on its own, so the failure + must not be retried; a re-upload could flash a device that already updated. + """ + return OTAError( + f"{err} (the device may have already committed the update and " + f"be rebooting; check whether it comes back with the new " + f"firmware before uploading again)" + ) + + def recv_decode( sock: socket.socket, amount: int, decode: bool = True ) -> bytes | list[int]: @@ -209,19 +234,22 @@ def receive_exactly( try: data += recv_decode(sock, 1, decode=decode) # type: ignore[operator] except OSError as err: - raise OTAError(f"receiving {msg} response: {err}") from err + raise OTANetworkError(f"receiving {msg} response: {err}") from err try: check_error(data, expect) except OTAError as err: sock.close() - raise OTAError(f"receiving {msg}: {err}") from err + # type(err) preserves OTANetworkError vs OTAError so callers can tell + # retryable network failures from device-reported errors; subclasses + # must accept a single message argument + raise type(err)(f"receiving {msg}: {err}") from err while len(data) < amount: try: data += recv_decode(sock, amount - len(data), decode=decode) # type: ignore[operator] except OSError as err: - raise OTAError(f"receiving {msg}: {err}") from err + raise OTANetworkError(f"receiving {msg}: {err}") from err return data @@ -237,7 +265,7 @@ def check_error(data: list[int] | bytes, expect: int | list[int] | None) -> None # accept-any-response reads (e.g. feature negotiation, auth nonces) would be # silently passed through and surface later as cryptic decode/timeout failures. if not data: - raise OTAError( + raise OTANetworkError( "Device closed connection without responding. " "This may indicate the device ran out of memory, " "a network issue, or the connection was interrupted." @@ -274,7 +302,7 @@ def send_check( sock.sendall(data) except OSError as err: - raise OTAError(f"sending {msg}: {err}") from err + raise OTANetworkError(f"sending {msg}: {err}") from err def perform_ota( @@ -306,7 +334,7 @@ def perform_ota( send_check(sock, MAGIC_BYTES, "magic bytes") _, version = receive_exactly(sock, 2, "version", RESPONSE_OK) - _LOGGER.debug("Device support OTA version: %s", version) + _LOGGER.info("Connection established; device supports OTA version %s", version) supported_versions = (OTA_VERSION_1_0, OTA_VERSION_2_0) if version not in supported_versions: raise OTAError( @@ -417,6 +445,8 @@ def perform_ota( hash_func, nonce_size, hash_name = _AUTH_METHODS[auth] perform_auth(sock, password, hash_func, nonce_size, hash_name) + _LOGGER.info("Handshake complete") + # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures sock.settimeout(90.0) @@ -449,21 +479,43 @@ def perform_ota( offset = 0 progress = ProgressBar("Uploading") - while True: - chunk = upload_contents[offset : offset + UPLOAD_BLOCK_SIZE] - if not chunk: - break - offset += len(chunk) + try: + while True: + chunk = upload_contents[offset : offset + UPLOAD_BLOCK_SIZE] + if not chunk: + break + offset += len(chunk) + + try: + sock.sendall(chunk) + except OSError as err: + # A send failure can hide an error byte the device reported + # just before dropping the connection; surface that as the + # real, non-retryable cause when it is available + try: + sock.settimeout(1.0) + check_error(recv_decode(sock, 1), None) + except (OSError, OTANetworkError) as probe_err: + _LOGGER.debug( + "No device error behind the send failure: %s", probe_err + ) + raise OTANetworkError(f"sending data: {err}") from err - try: - sock.sendall(chunk) if version >= OTA_VERSION_2_0: - receive_exactly(sock, 1, "chunk result", RESPONSE_CHUNK_OK) - except OSError as err: - sys.stderr.write("\n") - raise OTAError(f"sending data: {err}") from err + try: + receive_exactly(sock, 1, "chunk result", RESPONSE_CHUNK_OK) + except OTANetworkError as err: + if offset < upload_size: + raise + # The device already had the complete image when this ack + # was lost, so it may be committing; do not retry + raise _committed_error(err) from err - progress.update(offset / upload_size) + progress.update(offset / upload_size) + except OTAError: + # Terminate the progress bar line before the error is logged + progress.done() + raise progress.done() # Enable nodelay for last checks @@ -472,11 +524,25 @@ def perform_ota( _LOGGER.info("Upload took %.2f seconds, waiting for result...", duration) - receive_exactly(sock, 1, "update receive result", RESPONSE_RECEIVE_OK) - receive_exactly(sock, 1, "update end result", RESPONSE_UPDATE_END_OK) - send_check(sock, RESPONSE_OK, "end acknowledgement") + # Once the device has the complete image it commits the update and + # reboots on its own; the exact commit point is not observable from + # here, so treat everything past the data phase as non-retryable. A + # re-upload could flash a device that already updated successfully. + try: + receive_exactly(sock, 1, "update receive result", RESPONSE_RECEIVE_OK) + receive_exactly(sock, 1, "update end result", RESPONSE_UPDATE_END_OK) + except OTANetworkError as err: + raise _committed_error(err) from err - _LOGGER.info("OTA successful") + try: + send_check(sock, RESPONSE_OK, "end acknowledgement") + except OTANetworkError as err: + # The device treats a missing end acknowledgement as non-fatal and is + # already rebooting into the new firmware, so the update succeeded + _LOGGER.warning("Failed sending end acknowledgement: %s", err) + _LOGGER.info("OTA successful (end acknowledgement not delivered)") + else: + _LOGGER.info("OTA successful") # Do not connect logs until it is fully on time.sleep(1) @@ -510,8 +576,33 @@ def run_ota_impl_( ) raise OTAError(err) from err - for r in res: - af, socktype, _, _, sa = r + if not res: + _LOGGER.error("No addresses to connect to for %s", remote_host) + return 1, None + + # Every address is tried at least once and EXTRA_UPLOAD_ATTEMPTS retries + # are shared across the addresses, cycling through them. Wait before an + # attempt when the previous one actually reached the device, or when + # revisiting an address, so a flaky link can recover and the device can + # clean up a half-open connection (its handshake watchdog runs at 20s); + # moving on to the next address family stays immediate. Known limitation: + # a silent mid-transfer drop with no reset can wedge the device until its + # 90s data timeout, which outlasts this budget; the retries target the + # common failures where the device resets or closes the link promptly. + total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS + last_error = "" + reached_device = False + for attempt in range(total_attempts): + af, socktype, _, _, sa = res[attempt % len(res)] + if reached_device or attempt >= len(res): + _LOGGER.info( + "Retrying in %.0f seconds (attempt %d of %d)...", + UPLOAD_RETRY_DELAY, + attempt + 1, + total_attempts, + ) + time.sleep(UPLOAD_RETRY_DELAY) + reached_device = False _LOGGER.info("Connecting to %s port %s...", sa[0], sa[1]) sock = socket.socket(af, socktype) sock.settimeout(20.0) @@ -519,23 +610,30 @@ def run_ota_impl_( sock.connect(sa) except OSError as err: sock.close() - _LOGGER.error("Connecting to %s port %s failed: %s", sa[0], sa[1], err) + _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) + last_error = f"connecting to {sa[0]} failed: {err}" continue _LOGGER.info("Connected to %s", sa[0]) - with Path(filename).open("rb") as file_handle: + reached_device = True + with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: perform_ota(sock, password, file_handle, filename, ota_type) + except OTANetworkError as err: + # Transient network failure; retry + last_error = str(err) + _LOGGER.warning("%s", last_error) + continue except OTAError as err: + # Device-reported error (wrong password, wrong flash size, ...); + # retrying cannot succeed, so fail immediately _LOGGER.error(str(err)) return 1, None - finally: - sock.close() # Successfully uploaded to sa[0] return 0, sa[0] - _LOGGER.error("Connection failed.") + _LOGGER.error("Upload failed after %d attempts: %s", total_attempts, last_error) return 1, None diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 9413fbcf29..db4a4b1117 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -44,13 +44,17 @@ def mock_file() -> io.BytesIO: @pytest.fixture -def mock_time() -> Generator[None]: +def mock_sleep() -> Generator[Mock]: + """Mock time.sleep so delays don't slow down tests.""" + with patch("time.sleep") as mock: + yield mock + + +@pytest.fixture +def mock_time(mock_sleep: Mock) -> Generator[None]: """Mock time-related functions for consistent testing.""" # Provide enough values for multiple calls (tests may call perform_ota multiple times) - with ( - patch("time.sleep"), - patch("time.perf_counter", side_effect=[0, 1, 0, 1, 0, 1]), - ): + with patch("time.perf_counter", side_effect=[0, 1, 0, 1, 0, 1]): yield @@ -79,6 +83,28 @@ def mock_resolve_ip() -> Generator[Mock]: yield mock +DUAL_STACK_SA6 = ("2001:db8::1", 3232, 0, 0) +DUAL_STACK_SA4 = ("192.168.1.100", 3232) + + +@pytest.fixture +def mock_resolve_ip_dual(mock_resolve_ip: Mock) -> Mock: + """Make resolve_ip_address return an IPv6 and an IPv4 address.""" + mock_resolve_ip.return_value = [ + (socket.AF_INET6, socket.SOCK_STREAM, 0, "", DUAL_STACK_SA6), + (socket.AF_INET, socket.SOCK_STREAM, 0, "", DUAL_STACK_SA4), + ] + return mock_resolve_ip + + +@pytest.fixture +def firmware_file(tmp_path: Path) -> Path: + """Create a firmware file on disk for run_ota_impl_ tests.""" + firmware = tmp_path / "firmware.bin" + firmware.write_bytes(b"firmware content") + return firmware + + @pytest.fixture def mock_perform_ota() -> Generator[Mock]: """Mock perform_ota function for testing.""" @@ -137,9 +163,11 @@ def test_receive_exactly_with_error_response(mock_socket: Mock) -> None: with pytest.raises( espota2.OTAError, match="receiving auth:.*Authentication invalid" - ): + ) as exc_info: espota2.receive_exactly(mock_socket, 1, "auth", [espota2.RESPONSE_OK]) + # Device-reported errors must stay plain OTAError, not the retryable kind + assert not isinstance(exc_info.value, espota2.OTANetworkError) mock_socket.close.assert_called_once() @@ -147,10 +175,30 @@ def test_receive_exactly_socket_error(mock_socket: Mock) -> None: """Test receive_exactly handles socket errors.""" mock_socket.recv.side_effect = OSError("Connection reset") - with pytest.raises(espota2.OTAError, match="receiving test response"): + with pytest.raises(espota2.OTANetworkError, match="receiving test response"): espota2.receive_exactly(mock_socket, 1, "test", espota2.RESPONSE_OK) +def test_receive_exactly_mid_read_socket_error(mock_socket: Mock) -> None: + """Test receive_exactly handles socket errors after the first byte.""" + mock_socket.recv.side_effect = [b"\x00", OSError("Connection reset")] + + with pytest.raises(espota2.OTANetworkError, match="receiving test:"): + espota2.receive_exactly(mock_socket, 3, "test", espota2.RESPONSE_OK) + + +def test_receive_exactly_closed_connection_is_network_error(mock_socket: Mock) -> None: + """Test receive_exactly raises OTANetworkError when the device closes the connection.""" + mock_socket.recv.return_value = b"" + + with pytest.raises( + espota2.OTANetworkError, match="Device closed connection without responding" + ): + espota2.receive_exactly(mock_socket, 1, "test", espota2.RESPONSE_OK) + + mock_socket.close.assert_called_once() + + @pytest.mark.parametrize( ("error_code", "expected_msg"), [ @@ -227,15 +275,15 @@ def test_check_error_unexpected_response() -> None: def test_check_error_empty_data() -> None: - """Test check_error raises error when device closes connection without responding.""" + """Test check_error raises the retryable OTANetworkError when the device closes the connection.""" with pytest.raises( - espota2.OTAError, match="Device closed connection without responding" + espota2.OTANetworkError, match="Device closed connection without responding" ): espota2.check_error([], [espota2.RESPONSE_OK]) # Also test with empty bytes with pytest.raises( - espota2.OTAError, match="Device closed connection without responding" + espota2.OTANetworkError, match="Device closed connection without responding" ): espota2.check_error(b"", [espota2.RESPONSE_OK]) @@ -530,6 +578,144 @@ def test_perform_ota_upload_error(mock_socket: Mock, mock_file: io.BytesIO) -> N espota2.perform_ota(mock_socket, None, mock_file, "test.bin") +def _no_auth_handshake(version: int) -> list[bytes]: + """Recv responses for a handshake without auth, up to the MD5 check.""" + return [ + bytes([espota2.RESPONSE_OK]), # First byte of version response + bytes([version]), # Version number + bytes([espota2.RESPONSE_HEADER_OK]), # Features response + bytes([espota2.RESPONSE_AUTH_OK]), # No auth required + bytes([espota2.RESPONSE_UPDATE_PREPARE_OK]), # Binary size OK + bytes([espota2.RESPONSE_BIN_MD5_OK]), # MD5 checksum OK + ] + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_chunk_send_error(mock_socket: Mock, mock_file: io.BytesIO) -> None: + """Test OTA raises the retryable OTANetworkError when sending a chunk fails.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_2_0), + OSError("Connection reset"), # Probe for a pending error byte fails too + ] + # Sends before the data phase: magic bytes, features, binary size, MD5; + # fail on the fifth sendall, the first firmware chunk + mock_socket.sendall.side_effect = [None] * 4 + [OSError("Broken pipe")] + + with pytest.raises(espota2.OTANetworkError, match="sending data:"): + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_chunk_send_error_surfaces_device_error( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a device error byte pending behind a send failure becomes the cause.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_ERROR_WRITING_FLASH]), # Reason the device closed + ] + mock_socket.sendall.side_effect = [None] * 4 + [OSError("Broken pipe")] + + with pytest.raises( + espota2.OTAError, match="Writing OTA data to flash memory failed" + ) as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # The device-reported error is not retryable + assert not isinstance(exc.value, espota2.OTANetworkError) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_final_chunk_ack_failure_not_retryable( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a lost ack for the final chunk is not retried.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_2_0), + OSError("Connection reset"), # Ack for the only (final) chunk is lost + ] + + with pytest.raises(espota2.OTAError, match="receiving chunk result") as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # The device already had the whole image, so it may be committing + assert not isinstance(exc.value, espota2.OTANetworkError) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_intermediate_chunk_ack_failure_retryable( + mock_socket: Mock, +) -> None: + """Test a lost ack for a non-final chunk stays retryable.""" + # Two chunks: the firmware is larger than one upload block + big_file = io.BytesIO(b"x" * (espota2.UPLOAD_BLOCK_SIZE + 1)) + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_2_0), + OSError("Connection reset"), # Ack for the first of two chunks is lost + ] + + with pytest.raises(espota2.OTANetworkError, match="receiving chunk result"): + espota2.perform_ota(mock_socket, None, big_file, "test.bin") + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_post_commit_failure_not_retryable( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a network failure after the device committed is a plain OTAError.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_RECEIVE_OK]), # Device received everything + OSError("Connection reset"), # Connection lost waiting for end result + ] + + with pytest.raises(espota2.OTAError, match="receiving update end result") as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # Must not be the retryable kind; the device is already rebooting + assert not isinstance(exc.value, espota2.OTANetworkError) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_md5_mismatch_not_marked_committed( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test an MD5 mismatch keeps its own message and stays non-retryable.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_RECEIVE_OK]), # Device received everything + bytes([espota2.RESPONSE_ERROR_MD5_MISMATCH]), # Device aborted the update + ] + + with pytest.raises(espota2.OTAError, match="MD5 code mismatch") as exc: + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + # The device aborted without committing, so the message must not claim + # the update may have been installed, and the error must not be retried + assert not isinstance(exc.value, espota2.OTANetworkError) + assert "committed" not in str(exc.value) + + +@pytest.mark.usefixtures("mock_time") +def test_perform_ota_end_ack_send_failure_is_success( + mock_socket: Mock, mock_file: io.BytesIO +) -> None: + """Test a send failure on the final acknowledgement does not fail the OTA.""" + mock_socket.recv.side_effect = [ + *_no_auth_handshake(espota2.OTA_VERSION_1_0), + bytes([espota2.RESPONSE_RECEIVE_OK]), # Device received everything + bytes([espota2.RESPONSE_UPDATE_END_OK]), # Update committed + ] + # Sends: magic bytes, features, binary size, MD5, one firmware chunk; + # fail on the sixth sendall, the end acknowledgement + mock_socket.sendall.side_effect = [None] * 5 + [OSError("Broken pipe")] + + # Must not raise; the device treats a missing acknowledgement as non-fatal + espota2.perform_ota(mock_socket, None, mock_file, "test.bin") + + assert mock_socket.sendall.call_count == 6 + + @pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") def test_run_ota_impl_successful( mock_socket: Mock, tmp_path: Path, mock_perform_ota: Mock @@ -564,21 +750,183 @@ def test_run_ota_impl_successful( @pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") -def test_run_ota_impl_connection_failed(mock_socket: Mock, tmp_path: Path) -> None: - """Test run_ota_impl_ when connection fails.""" +def test_run_ota_impl_connection_failed( + mock_socket: Mock, firmware_file: Path, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ retries when connection fails and eventually gives up.""" mock_socket.connect.side_effect = OSError("Connection refused") - # Create a real firmware file - firmware_file = tmp_path / "firmware.bin" - firmware_file.write_bytes(b"firmware content") - result_code, result_host = espota2.run_ota_impl_( "test.local", 3232, "password", str(firmware_file) ) assert result_code == 1 assert result_host is None - mock_socket.close.assert_called_once() + # A single address gets the whole attempt budget, with a delay before + # each revisit + assert mock_socket.connect.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + 1 + assert mock_socket.close.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + 1 + assert mock_sleep.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + mock_sleep.assert_called_with(espota2.UPLOAD_RETRY_DELAY) + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_connect_retry_succeeds( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ succeeds when a retry connects after a failed attempt.""" + mock_socket.connect.side_effect = [OSError("Connection timed out"), None] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + assert mock_socket.connect.call_count == 2 + mock_sleep.assert_called_once_with(espota2.UPLOAD_RETRY_DELAY) + mock_perform_ota.assert_called_once() + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_network_error_retry_succeeds( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ retries after a network error during the upload.""" + mock_perform_ota.side_effect = [ + espota2.OTANetworkError("receiving features: Device closed connection"), + None, + ] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + assert mock_perform_ota.call_count == 2 + mock_sleep.assert_called_once_with(espota2.UPLOAD_RETRY_DELAY) + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_network_error_exhausts_attempts( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ gives up after all attempts hit network errors.""" + mock_perform_ota.side_effect = espota2.OTANetworkError("sending data: broken pipe") + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + assert mock_perform_ota.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + 1 + assert mock_sleep.call_count == espota2.EXTRA_UPLOAD_ATTEMPTS + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip_dual") +def test_run_ota_impl_multiple_addresses_cycle( + mock_socket: Mock, firmware_file: Path, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ visits every address and cycles for the retries.""" + mock_socket.connect.side_effect = OSError("No route to host") + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + # Each address is visited once, then the EXTRA_UPLOAD_ATTEMPTS spare + # attempts cycle back through them; the budget is shared, not per address + assert mock_socket.connect.call_args_list == [ + call(DUAL_STACK_SA6), + call(DUAL_STACK_SA4), + call(DUAL_STACK_SA6), + call(DUAL_STACK_SA4), + ] + # No connect ever reached the device, so the delay only applies before + # the revisits + assert mock_sleep.call_count == 2 + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip_dual") +def test_run_ota_impl_second_address_succeeds_without_delay( + mock_socket: Mock, + firmware_file: Path, + mock_perform_ota: Mock, + mock_sleep: Mock, +) -> None: + """Test run_ota_impl_ falls through to the next address with no pause.""" + mock_socket.connect.side_effect = [OSError("No route to host"), None] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + mock_sleep.assert_not_called() + mock_perform_ota.assert_called_once() + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip_dual") +def test_run_ota_impl_pauses_after_reaching_device( + mock_socket: Mock, + firmware_file: Path, + mock_perform_ota: Mock, + mock_sleep: Mock, +) -> None: + """Test run_ota_impl_ pauses before the next address once the device was reached.""" + mock_perform_ota.side_effect = [ + espota2.OTANetworkError("sending data: connection reset"), + None, + ] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 0 + assert result_host == "192.168.1.100" + # The first attempt reached the device, so the next one waits first even + # though it targets a fresh address + mock_sleep.assert_called_once_with(espota2.UPLOAD_RETRY_DELAY) + + +@pytest.mark.usefixtures("mock_socket_constructor", "mock_resolve_ip") +def test_run_ota_impl_device_error_not_retried( + mock_socket: Mock, firmware_file: Path, mock_perform_ota: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ fails immediately on a device-reported error.""" + mock_perform_ota.side_effect = espota2.OTAError( + "Authentication invalid. Is the password correct?" + ) + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + mock_perform_ota.assert_called_once() + mock_sleep.assert_not_called() + + +def test_run_ota_impl_no_addresses( + firmware_file: Path, mock_resolve_ip: Mock, mock_sleep: Mock +) -> None: + """Test run_ota_impl_ fails cleanly when resolution yields no addresses.""" + mock_resolve_ip.return_value = [] + + result_code, result_host = espota2.run_ota_impl_( + "test.local", 3232, "password", str(firmware_file) + ) + + assert result_code == 1 + assert result_host is None + mock_sleep.assert_not_called() def test_run_ota_impl_resolve_failed(tmp_path: Path, mock_resolve_ip: Mock) -> None: From 02c1810c3ad388b37025fd65307fd86dbf3e66a8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 13:31:19 -0500 Subject: [PATCH 038/160] [core] Load component aliases from a generated registry (#18335) --- .github/workflows/ci.yml | 1 + esphome/component_aliases.py | 10 ++++++ esphome/loader.py | 61 +++++++++++++-------------------- script/build_alias_registry.py | 59 +++++++++++++++++++++++++++++++ tests/unit_tests/test_loader.py | 28 +++++++++++++++ 5 files changed, 122 insertions(+), 37 deletions(-) create mode 100644 esphome/component_aliases.py create mode 100755 script/build_alias_registry.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e695bb46b..b603e68ad7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,6 +179,7 @@ jobs: . venv/bin/activate script/ci-custom.py script/build_codeowners.py --check + script/build_alias_registry.py --check script/build_language_schema.py --check script/generate-esp32-boards.py --check script/generate-rp2-boards.py --check diff --git a/esphome/component_aliases.py b/esphome/component_aliases.py new file mode 100644 index 0000000000..e701bd98d4 --- /dev/null +++ b/esphome/component_aliases.py @@ -0,0 +1,10 @@ +"""Component alias registry. + +Generated by script/build_alias_registry.py - do not edit manually. +See the component-alias section of esphome/loader.py. +""" + +# alias -> (canonical component, removal version or None) +COMPONENT_ALIASES: dict[str, tuple[str, str | None]] = { + "rp2040": ("rp2", "2027.7.0"), +} diff --git a/esphome/loader.py b/esphome/loader.py index 7a659aa0a8..f994f0c5eb 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -269,10 +269,9 @@ def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None: # If `domain` is the legacy name of a renamed component, redirect to the # canonical module so the rest of the loader (and every caller of # `get_component(legacy)`) transparently sees the new component. - alias_map = _get_alias_map() - if domain in alias_map: - canonical = alias_map[domain] - manif = _lookup_module(canonical, exception) + alias_meta = get_alias_metadata().get(domain) + if alias_meta is not None: + manif = _lookup_module(alias_meta.canonical, exception) if manif is not None: _COMPONENT_CACHE[domain] = manif return manif @@ -329,8 +328,10 @@ def _replace_component_manifest(domain: str, manifest: ComponentManifest) -> Non # --------------------------------------------------------------------------- # # A component can declare ``ALIASES = ["legacy_name"]`` (and optionally -# ``ALIAS_REMOVAL_VERSION = "YYYY.M.0"``) in its ``__init__.py``. Two -# integrations are then wired up automatically: +# ``ALIAS_REMOVAL_VERSION = "YYYY.M.0"``) in its ``__init__.py``, then run +# ``script/build_alias_registry.py`` to regenerate +# ``esphome/component_aliases.py`` (CI and a unit test fail if the registry +# is stale). Two integrations are then wired up automatically: # # 1. **Python imports** — a ``sys.meta_path`` finder (``_AliasFinder``) # intercepts ``esphome.components.``/``....`` @@ -344,13 +345,13 @@ def _replace_component_manifest(domain: str, manifest: ComponentManifest) -> Non # dependency checks, schema validation and codegen all see only the # canonical name. # -# Both lookups are populated by ``_build_alias_map``, which **AST-parses** -# every component's ``__init__.py`` rather than importing it. That keeps the -# cost low: scanning ~400 components on disk takes ~5 ms instead of the -# multi-second cost of executing every component's import side-effects. +# Both lookups read the checked-in registry in ``esphome.component_aliases`` +# (generated by ``script/build_alias_registry.py``, verified in CI), so no +# component-directory scan happens at runtime. ``_build_alias_map`` below is +# the generator's scan implementation; it **AST-parses** each component's +# ``__init__.py`` rather than importing it. -_ALIAS_MAP_CACHE: dict[str, str] | None = None _ALIAS_META_CACHE: dict[str, "AliasMeta"] | None = None @@ -367,31 +368,17 @@ class AliasMeta: removal_version: str | None -def _ensure_alias_caches() -> None: - """Populate both alias caches from a single directory scan. - - ``_build_alias_map`` returns both maps together, so building them in one - shot avoids scanning every component's ``__init__.py`` twice when a run - needs both the canonical map (loader) and the metadata map (config - pre-pass). - """ - global _ALIAS_MAP_CACHE, _ALIAS_META_CACHE - if _ALIAS_MAP_CACHE is None or _ALIAS_META_CACHE is None: - _ALIAS_MAP_CACHE, _ALIAS_META_CACHE = _build_alias_map() - - -def _get_alias_map() -> dict[str, str]: - """Return the legacy-name → canonical-name map, building it lazily.""" - _ensure_alias_caches() - return _ALIAS_MAP_CACHE - - def get_alias_metadata() -> dict[str, AliasMeta]: - """Return the legacy-name → :class:`AliasMeta` map (cached). + """Return the legacy-name → :class:`AliasMeta` map, built lazily from + the generated registry.""" + global _ALIAS_META_CACHE # noqa: PLW0603 + if _ALIAS_META_CACHE is None: + from esphome.component_aliases import COMPONENT_ALIASES - Used by the YAML pre-pass to format a per-alias deprecation warning. - """ - _ensure_alias_caches() + _ALIAS_META_CACHE = { + alias: AliasMeta(canonical=canonical, removal_version=removal_version) + for alias, (canonical, removal_version) in COMPONENT_ALIASES.items() + } return _ALIAS_META_CACHE @@ -537,11 +524,11 @@ class _AliasFinder(importlib.abc.MetaPathFinder): # least three parts, so ``parts[2]`` (the domain) always exists. parts = fullname.split(".") domain = parts[2] - alias_map = _get_alias_map() - if domain not in alias_map: + alias_meta = get_alias_metadata().get(domain) + if alias_meta is None: return None - parts[2] = alias_map[domain] + parts[2] = alias_meta.canonical canonical_fullname = ".".join(parts) try: canonical_module = importlib.import_module(canonical_fullname) diff --git a/script/build_alias_registry.py b/script/build_alias_registry.py new file mode 100755 index 0000000000..e007c075eb --- /dev/null +++ b/script/build_alias_registry.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Generate esphome/component_aliases.py from component ALIASES declarations. + +Run without arguments to regenerate the registry; ``--check`` (run in CI) +verifies it is up to date. +""" + +import argparse +from pathlib import Path +import sys + +# The root directory of the repo +root = Path(__file__).parent.parent +# Make the repo's esphome package win over any installed copy +sys.path.insert(0, str(root)) + +from esphome.helpers import write_file_if_changed # noqa: E402 +from esphome.loader import _build_alias_map # noqa: E402 + +parser = argparse.ArgumentParser() +parser.add_argument( + "--check", + help="Check if the alias registry is up to date.", + action="store_true", +) +args = parser.parse_args() + +registry_file = root / "esphome" / "component_aliases.py" + +HEADER = '''"""Component alias registry. + +Generated by script/build_alias_registry.py - do not edit manually. +See the component-alias section of esphome/loader.py. +""" + +# alias -> (canonical component, removal version or None) +COMPONENT_ALIASES: dict[str, tuple[str, str | None]] = { +''' + +# _build_alias_map scans the real component tree and already rejects +# duplicate and shadowing aliases with an EsphomeError. +_, alias_meta = _build_alias_map() + +lines = [HEADER] +for alias, meta in sorted(alias_meta.items()): + removal = f'"{meta.removal_version}"' if meta.removal_version else "None" + lines.append(f' "{alias}": ("{meta.canonical}", {removal}),\n') +lines.append("}\n") +content = "".join(lines) + +if args.check: + if registry_file.read_text(encoding="utf-8") != content: + print("Component alias registry is not up to date.") + print("Please run `script/build_alias_registry.py`") + sys.exit(1) + print("Component alias registry is up to date") +else: + write_file_if_changed(registry_file, content) + print(f"Wrote {registry_file}") diff --git a/tests/unit_tests/test_loader.py b/tests/unit_tests/test_loader.py index 41dd462678..74515e9d4c 100644 --- a/tests/unit_tests/test_loader.py +++ b/tests/unit_tests/test_loader.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch import pytest +from esphome.component_aliases import COMPONENT_ALIASES from esphome.loader import ( AliasMeta, ComponentManifest, @@ -481,6 +482,33 @@ def test_real_alias_map_includes_rp2040() -> None: assert meta["rp2040"].removal_version == "2027.7.0" +def test_alias_registry_matches_component_tree() -> None: + """The checked-in registry must match a live scan of the component tree.""" + _, meta_map = _build_alias_map() + expected = { + alias: (meta.canonical, meta.removal_version) + for alias, meta in meta_map.items() + } + assert expected == COMPONENT_ALIASES, ( + "esphome/component_aliases.py is out of date; " + "run script/build_alias_registry.py" + ) + + +def test_alias_map_built_from_registry() -> None: + """The runtime alias map comes from the generated registry, not a scan.""" + with ( + patch( + "esphome.component_aliases.COMPONENT_ALIASES", + {"legacy": ("modern", "2099.1.0")}, + ), + patch("esphome.loader._ALIAS_META_CACHE", None), + ): + assert get_alias_metadata() == { + "legacy": AliasMeta(canonical="modern", removal_version="2099.1.0") + } + + def test_get_component_resolves_alias() -> None: """``get_component('rp2040')`` should return the rp2 manifest — every caller of the loader (dep checker, schema validator, codegen) hits From add18d4e351f582757781aa92f68eaa534ea8748 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Aug 2026 20:06:48 -0500 Subject: [PATCH 039/160] [core] Partially revert "Hash entity keys from the raw name to fix collisions" (#18361) --- esphome/components/api/api_connection.cpp | 6 +- esphome/components/infrared/infrared.cpp | 8 +- esphome/components/mqtt/__init__.py | 63 --- esphome/components/prometheus/__init__.py | 6 - .../radio_frequency/radio_frequency.cpp | 8 +- .../template/text/template_text.cpp | 20 +- .../components/template/text/template_text.h | 15 +- esphome/core/application.h | 8 +- esphome/core/entity_base.cpp | 52 +-- esphome/core/entity_base.h | 80 ++-- esphome/core/entity_helpers.py | 190 ++++---- esphome/core/helpers.h | 29 +- esphome/core/preference_backend.h | 14 +- esphome/core/preferences.cpp | 25 - esphome/core/preferences.h | 12 - esphome/helpers.py | 15 +- tests/integration/entity_utils.py | 27 +- .../fixtures/fnv1_hash_object_id.yaml | 32 -- .../fixtures/multi_device_preferences.yaml | 21 +- .../fixtures/preference_key_migration.yaml | 35 -- tests/integration/host_prefs.py | 24 +- tests/integration/test_fnv1_hash_object_id.py | 4 - .../test_object_id_api_verification.py | 10 +- ...t_object_id_friendly_name_no_mac_suffix.py | 4 +- .../test_object_id_no_friendly_name.py | 6 +- .../test_preference_key_migration.py | 165 ------- tests/unit_tests/components/mqtt/__init__.py | 0 .../mqtt/test_object_id_conflicts.py | 239 ---------- tests/unit_tests/core/test_entity_helpers.py | 432 ++++++++++-------- .../object_id_conflict_mqtt.yaml | 22 - .../object_id_conflict_no_mqtt.yaml | 15 - .../test_preference_hash_stability.py | 34 +- 32 files changed, 489 insertions(+), 1132 deletions(-) delete mode 100644 esphome/core/preferences.cpp delete mode 100644 tests/integration/fixtures/preference_key_migration.yaml delete mode 100644 tests/integration/test_preference_key_migration.py delete mode 100644 tests/unit_tests/components/mqtt/__init__.py delete mode 100644 tests/unit_tests/components/mqtt/test_object_id_conflicts.py delete mode 100644 tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml delete mode 100644 tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index d05f98d03b..73b4f3e5bd 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -448,7 +448,7 @@ void APIConnection::on_disconnect_response() { uint16_t APIConnection::fill_and_encode_entity_state(EntityBase *entity, StateResponseProtoMessage &msg, CalculateSizeFn size_fn, MessageEncodeFn encode_fn, APIConnection *conn, uint32_t remaining_size) { - msg.key = entity->get_entity_key(); + msg.key = entity->get_object_id_hash(); #ifdef USE_DEVICES msg.device_id = entity->get_device_id(); #endif @@ -459,7 +459,7 @@ uint16_t APIConnection::fill_and_encode_entity_info(EntityBase *entity, InfoResp CalculateSizeFn size_fn, MessageEncodeFn encode_fn, APIConnection *conn, uint32_t remaining_size) { // Set common fields that are shared by all entity types - msg.key = entity->get_entity_key(); + msg.key = entity->get_object_id_hash(); if (entity->has_own_name()) { msg.name = entity->get_name(); @@ -1149,7 +1149,7 @@ void APIConnection::try_send_camera_image_() { bool done = this->image_reader_->available() == to_send; CameraImageResponse msg; - msg.key = camera::Camera::instance()->get_entity_key(); + msg.key = camera::Camera::instance()->get_object_id_hash(); msg.set_data(this->image_reader_->peek_data_buffer(), to_send); msg.done = done; #ifdef USE_DEVICES diff --git a/esphome/components/infrared/infrared.cpp b/esphome/components/infrared/infrared.cpp index 288b1e5c40..9b97995a96 100644 --- a/esphome/components/infrared/infrared.cpp +++ b/esphome/components/infrared/infrared.cpp @@ -154,8 +154,12 @@ bool Infrared::on_receive(remote_base::RemoteReceiveData data) { // Forward received IR data to API server #if defined(USE_API) && defined(USE_IR_RF) if (api::global_api_server != nullptr) { - api::global_api_server->send_infrared_rf_receive_event(this->get_device_id_or_zero(), this->get_entity_key(), - &data.get_raw_data()); +#ifdef USE_DEVICES + uint32_t device_id = this->get_device_id(); +#else + uint32_t device_id = 0; +#endif + api::global_api_server->send_infrared_rf_receive_event(device_id, this->get_object_id_hash(), &data.get_raw_data()); } #endif return false; // Don't consume the event, allow other listeners to process it diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index 713969ab88..98ca23b60b 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -63,7 +63,6 @@ from esphome.const import ( PlatformFramework, ) from esphome.core import CORE, CoroPriority, coroutine_with_priority -from esphome.core.entity_helpers import ObjectIdEntity, validate_no_object_id_conflicts from esphome.types import ConfigType DEPENDENCIES = ["network"] @@ -333,68 +332,6 @@ CONFIG_SCHEMA = cv.All( ) -# Platforms whose MQTT components subscribe to an object_id-derived command topic. -# Keep in sync with the platforms extending cv.MQTT_COMMAND_COMPONENT_SCHEMA, plus -# text, whose MQTT component subscribes a command topic that cannot be overridden. -_COMMAND_TOPIC_PLATFORMS = frozenset( - { - "alarm_control_panel", - "button", - "climate", - "cover", - "datetime", - "fan", - "light", - "lock", - "number", - "select", - "switch", - "text", - "update", - "valve", - } -) - - -# Platforms whose MQTT components derive extra sub-topics (position/command, -# mode/command, speed/command, ...) from the object_id, each with its own config -# key; custom state and command topics cannot exempt them from conflicting. -_SUB_TOPIC_PLATFORMS = frozenset({"climate", "cover", "fan", "valve"}) - - -def _topics_conflict(entities: list[ObjectIdEntity], config: ConfigType) -> bool: - """Check whether more than one entity actually uses an object_id-derived topic. - - An empty topic_prefix disables default topics entirely, custom state and - command topics avoid the default topics, and disabling discovery (globally - or per entity) avoids the discovery config topic. - """ - if config[CONF_TOPIC_PREFIX]: - platform = entities[0].platform - if platform in _SUB_TOPIC_PLATFORMS: - return True - if sum(CONF_STATE_TOPIC not in entity.config for entity in entities) > 1: - return True - if ( - platform in _COMMAND_TOPIC_PLATFORMS - and sum(CONF_COMMAND_TOPIC not in entity.config for entity in entities) > 1 - ): - return True - if not config[CONF_DISCOVERY]: - return False - discovery_entities = sum( - entity.config.get(CONF_DISCOVERY, True) for entity in entities - ) - return discovery_entities > 1 - - -FINAL_VALIDATE_SCHEMA = validate_no_object_id_conflicts( - "mqtt builds default topics and discovery topics from the entity object_id, " - "which is the name converted to ASCII", - conflict_filter=_topics_conflict, -) - - def exp_mqtt_message(config): if config is None: return cg.optional(cg.TemplateArguments(MQTTMessage)) diff --git a/esphome/components/prometheus/__init__.py b/esphome/components/prometheus/__init__.py index 0a69160fc1..cc1541ce80 100644 --- a/esphome/components/prometheus/__init__.py +++ b/esphome/components/prometheus/__init__.py @@ -3,7 +3,6 @@ from esphome.components import web_server_base from esphome.components.web_server_base import CONF_WEB_SERVER_BASE_ID import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_INCLUDE_INTERNAL, CONF_NAME, CONF_RELABEL -from esphome.core.entity_helpers import validate_no_object_id_conflicts from esphome.cpp_types import EntityBase AUTO_LOAD = ["web_server_base"] @@ -36,11 +35,6 @@ CONFIG_SCHEMA = cv.Schema( }, ).extend(cv.COMPONENT_SCHEMA) -FINAL_VALIDATE_SCHEMA = validate_no_object_id_conflicts( - "prometheus builds metric labels from the entity object_id, " - "which is the name converted to ASCII" -) - async def to_code(config): paren = await cg.get_variable(config[CONF_WEB_SERVER_BASE_ID]) diff --git a/esphome/components/radio_frequency/radio_frequency.cpp b/esphome/components/radio_frequency/radio_frequency.cpp index fe6c6a9cb5..3e0a905737 100644 --- a/esphome/components/radio_frequency/radio_frequency.cpp +++ b/esphome/components/radio_frequency/radio_frequency.cpp @@ -99,8 +99,12 @@ bool RadioFrequency::on_receive(remote_base::RemoteReceiveData data) { // Forward received RF data to API server #if defined(USE_API) && defined(USE_RADIO_FREQUENCY) if (api::global_api_server != nullptr) { - api::global_api_server->send_infrared_rf_receive_event(this->get_device_id_or_zero(), this->get_entity_key(), - &data.get_raw_data()); +#ifdef USE_DEVICES + uint32_t device_id = this->get_device_id(); +#else + uint32_t device_id = 0; +#endif + api::global_api_server->send_infrared_rf_receive_event(device_id, this->get_object_id_hash(), &data.get_raw_data()); } #endif return false; // Don't consume the event, allow other listeners to process it diff --git a/esphome/components/template/text/template_text.cpp b/esphome/components/template/text/template_text.cpp index ffe11cf229..af134e6ed4 100644 --- a/esphome/components/template/text/template_text.cpp +++ b/esphome/components/template/text/template_text.cpp @@ -20,14 +20,18 @@ void TemplateText::setup() { // Need std::string for pref_->setup() to fill from flash std::string value{this->initial_value_ != nullptr ? this->initial_value_ : ""}; - uint32_t extra = 0; - extra += this->traits.get_min_length() << 2; - extra += this->traits.get_max_length() << 4; - extra += fnv1_hash(this->traits.get_pattern_c_str()) << 6; - // TextSaver::setup() picks the key for the platform and migrates old data once - uint32_t key = this->preference_key_base_() + extra; - uint32_t old_key = this->old_preference_key_base_() + extra; - this->pref_->setup(key, old_key, value); + // For future hash migration: use migrate_entity_preference_() with: + // old_key = get_preference_hash() + extra + // new_key = get_preference_hash_v2() + extra + // See: https://github.com/esphome/backlog/issues/85 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + uint32_t key = this->get_preference_hash(); +#pragma GCC diagnostic pop + key += this->traits.get_min_length() << 2; + key += this->traits.get_max_length() << 4; + key += fnv1_hash(this->traits.get_pattern_c_str()) << 6; + this->pref_->setup(key, value); if (!value.empty()) this->publish_state(value); } diff --git a/esphome/components/template/text/template_text.h b/esphome/components/template/text/template_text.h index beeea4396a..229a61d9b8 100644 --- a/esphome/components/template/text/template_text.h +++ b/esphome/components/template/text/template_text.h @@ -14,9 +14,7 @@ class TemplateTextSaverBase { public: virtual bool save(const std::string &value) { return true; } - /// old_id is the pre-2026.8.0 preference key; data stored under it is moved to id once. - /// See: https://github.com/esphome/backlog/issues/85 - virtual void setup(uint32_t id, uint32_t old_id, std::string &value) {} + virtual void setup(uint32_t id, std::string &value) {} protected: ESPPreferenceObject pref_; @@ -47,16 +45,11 @@ template class TextSaver : public TemplateTextSaverBase { // Make the preference object. Fill the provided location with the saved data // If it is available, else leave it alone - void setup(uint32_t id, uint32_t old_id, std::string &value) override { - char temp[SZ + 1]; -#ifdef USE_PREFERENCE_KEY_LOOKUP + void setup(uint32_t id, std::string &value) override { this->pref_ = global_preferences->make_preference(id); - bool hasdata = migrate_preference(this->pref_, reinterpret_cast(temp), SZ + 1, old_id, id); -#else - // Slot-based backends keep the old key; it is only a validity tag on a positional slot - this->pref_ = global_preferences->make_preference(old_id); + + char temp[SZ + 1]; bool hasdata = this->pref_.load(&temp); -#endif if (hasdata) { size_t len = static_cast(temp[0]); diff --git a/esphome/core/application.h b/esphome/core/application.h index a18a6b31c8..a12cdc4ac8 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -120,8 +120,8 @@ class Application { // NOLINTBEGIN(bugprone-macro-parentheses) #define ENTITY_TYPE_(type, singular, plural, count, upper) \ void register_##singular(type *obj) { this->plural##_.push_back(obj); } \ - void register_##singular(type *obj, const char *name, uint32_t entity_key, uint32_t entity_fields) { \ - obj->configure_entity_(name, entity_key, entity_fields); \ + void register_##singular(type *obj, const char *name, uint32_t object_id_hash, uint32_t entity_fields) { \ + obj->configure_entity_(name, object_id_hash, entity_fields); \ this->plural##_.push_back(obj); \ } #define ENTITY_CONTROLLER_TYPE_(type, singular, plural, count, upper, callback) \ @@ -329,7 +329,7 @@ class Application { #define GET_ENTITY_METHOD(entity_type, entity_name, entities_member) \ entity_type *get_##entity_name##_by_key(uint32_t key, uint32_t device_id, bool include_internal = false) { \ for (auto *obj : this->entities_member##_) { \ - if (obj->get_entity_key() == key && obj->get_device_id() == device_id && \ + if (obj->get_object_id_hash() == key && obj->get_device_id() == device_id && \ (include_internal || !obj->is_internal())) \ return obj; \ } \ @@ -340,7 +340,7 @@ class Application { #define GET_ENTITY_METHOD(entity_type, entity_name, entities_member) \ entity_type *get_##entity_name##_by_key(uint32_t key, bool include_internal = false) { \ for (auto *obj : this->entities_member##_) { \ - if (obj->get_entity_key() == key && (include_internal || !obj->is_internal())) \ + if (obj->get_object_id_hash() == key && (include_internal || !obj->is_internal())) \ return obj; \ } \ return nullptr; \ diff --git a/esphome/core/entity_base.cpp b/esphome/core/entity_base.cpp index 328de05302..fc6ac503b5 100644 --- a/esphome/core/entity_base.cpp +++ b/esphome/core/entity_base.cpp @@ -8,7 +8,7 @@ namespace esphome { static const char *const TAG = "entity_base"; -void EntityBase::configure_entity_(const char *name, uint32_t entity_key, uint32_t entity_fields) { +void EntityBase::configure_entity_(const char *name, uint32_t object_id_hash, uint32_t entity_fields) { this->name_ = StringRef(name); if (this->name_.empty()) { #ifdef USE_DEVICES @@ -30,15 +30,15 @@ void EntityBase::configure_entity_(const char *name, uint32_t entity_key, uint32 } } this->flags_.has_own_name = false; - // Dynamic name - must calculate key at runtime - this->calc_entity_key_(); + // Dynamic name - must calculate hash at runtime + this->calc_object_id_(); } else { this->flags_.has_own_name = true; - // Static name - use pre-computed key if provided - if (entity_key != 0) { - this->entity_key_ = entity_key; + // Static name - use pre-computed hash if provided + if (object_id_hash != 0) { + this->object_id_hash_ = object_id_hash; } else { - this->calc_entity_key_(); + this->calc_object_id_(); } } // Unpack entity string table indices and flags from entity_fields. @@ -147,15 +147,9 @@ std::string EntityBase::get_icon() const { } #endif // !USE_ESP8266 -// Calculate the entity key directly from the raw name (no transformations) -void EntityBase::calc_entity_key_() { this->entity_key_ = fnv1_hash_bytes(this->name_.c_str(), this->name_.size()); } - -// Reconstruct the OLD (pre-2026.8.0) object_id-based hash for preference key compatibility. -// Named entities historically used the hash pre-computed by Python code generation, which -// sanitized per UTF-8 code point; entities without their own name computed the hash at -// runtime per byte. See https://github.com/esphome/backlog/issues/85 -uint32_t EntityBase::calc_old_object_id_hash_() const { - return fnv1_hash_object_id(this->name_.c_str(), this->name_.size(), this->flags_.has_own_name); +// Calculate Object ID Hash directly from name using snake_case + sanitize +void EntityBase::calc_object_id_() { + this->object_id_hash_ = fnv1_hash_object_id(this->name_.c_str(), this->name_.size()); } size_t EntityBase::write_object_id_to(char *buf, size_t buf_size) const { @@ -173,22 +167,16 @@ StringRef EntityBase::get_object_id_to(std::span buf) c } ESPPreferenceObject EntityBase::make_entity_preference_(size_t size, uint32_t version) { - // The old key hashed the sanitized object_id, so multiple entity names could collide on - // one key and overwrite each other's stored preferences; the new key hashes the raw name. - // See: https://github.com/esphome/backlog/issues/85 - uint32_t old_key = this->old_preference_key_base_() ^ version; -#ifdef USE_PREFERENCE_KEY_LOOKUP - uint32_t new_key = this->preference_key_base_() ^ version; - auto pref = global_preferences->make_preference(size, new_key); - // All in-tree entity preferences fit the stack buffer, so migration never hits the heap - SmallBufferWithHeapFallback<64> buffer(size); - migrate_preference(pref, buffer.get(), size, old_key, new_key); - return pref; -#else - // Slot-based backends keep the old key: it is only a validity tag on a positional slot, - // so collisions cannot corrupt data there and keeping it preserves stored state. - return global_preferences->make_preference(size, old_key); -#endif + // The key hashes the sanitized object_id, so multiple entity names can collide on one + // key and overwrite each other's stored preferences ("Living Room" and "living_room", + // or two UTF-8 names that both sanitize to underscores). Keys hashed from the raw name + // fix this, but they change the entity key API clients track, which the Home Assistant + // esphome integration cannot handle yet. See: https://github.com/esphome/backlog/issues/85 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + uint32_t key = this->get_preference_hash() ^ version; +#pragma GCC diagnostic pop + return global_preferences->make_preference(size, key); } #ifdef USE_ENTITY_ICON diff --git a/esphome/core/entity_base.h b/esphome/core/entity_base.h index 7f8e5f2630..5f2e173d8d 100644 --- a/esphome/core/entity_base.h +++ b/esphome/core/entity_base.h @@ -73,17 +73,8 @@ class EntityBase { // Get whether this Entity has its own name or it should use the device friendly_name. bool has_own_name() const { return this->flags_.has_own_name; } - // Get the unique key of this Entity: FNV-1 hash of the raw entity name. - // This is the key sent to API clients and used to route entity state. - uint32_t get_entity_key() const { return this->entity_key_; } - - /// Returns the LEGACY object_id hash, unchanged from previous releases, so existing - /// callers keep getting stable values (for example preference keys). This is no longer - /// the key sent to API clients; that is get_entity_key(). - ESPDEPRECATED("Use get_entity_key() for the entity key sent to API clients, or " - "make_entity_preference() for preference storage. Will be removed in 2027.1.0.", - "2026.8.0") - uint32_t get_object_id_hash() const { return this->calc_old_object_id_hash_(); } + // Get the unique Object ID of this Entity + uint32_t get_object_id_hash() const { return this->object_id_hash_; } /// Get object_id with zero heap allocation /// For static case: returns StringRef to internal storage (buffer unused) @@ -190,23 +181,39 @@ class EntityBase { // Set has_state - for components that need to manually set this void set_has_state(bool state) { this->flags_.has_state = state; } - /// Get this entity's device id, or 0 when devices are not compiled in (main device). - uint32_t get_device_id_or_zero() const { -#ifdef USE_DEVICES - return this->get_device_id(); -#else - return 0; -#endif - } - - /// Get the LEGACY preference key: FNV-1 hash of the sanitized object_id, XOR device_id. - /// Intentionally keeps the old algorithm so external callers that store preferences under - /// this key keep stable keys; make_entity_preference() migrates to the new raw-name key, - /// this method never will. + /** + * @brief Get a unique hash for storing preferences/settings for this entity. + * + * This method returns a hash that uniquely identifies the entity for the purpose of + * storing preferences (such as calibration, state, etc.). Unlike get_object_id_hash(), + * this hash also incorporates the device_id (if devices are enabled), ensuring uniqueness + * across multiple devices that may have entities with the same object_id. + * + * Use this method when storing or retrieving preferences/settings that should be unique + * per device-entity pair. Use get_object_id_hash() when you need a hash that identifies + * the entity regardless of the device it belongs to. + * + * For backward compatibility, if device_id is 0 (the main device), the hash is unchanged + * from previous versions, so existing single-device configurations will continue to work. + * + * @return uint32_t The unique hash for preferences, including device_id if available. + * @deprecated Use make_entity_preference() instead, or preferences won't be migrated. + * See https://github.com/esphome/backlog/issues/85 + */ ESPDEPRECATED("Use make_entity_preference() instead, or preferences won't be migrated. " "See https://github.com/esphome/backlog/issues/85. Will be removed in 2027.1.0.", - "2026.8.0") - uint32_t get_preference_hash() { return this->old_preference_key_base_(); } + "2026.7.0") + uint32_t get_preference_hash() { +#ifdef USE_DEVICES + // Combine object_id_hash with device_id to ensure uniqueness across devices + // Note: device_id is 0 for the main device, so XORing with 0 preserves the original hash + // This ensures backward compatibility for existing single-device configurations + return this->get_object_id_hash() ^ this->get_device_id(); +#else + // Without devices, just use object_id_hash as before + return this->get_object_id_hash(); +#endif + } /// Create a preference object for storing this entity's state/settings. /// @tparam T The type of data to store (must be trivially copyable) @@ -223,9 +230,9 @@ class EntityBase { // before push_back, so codegen can emit a single combined call per entity. friend class Application; - /// Combined entity setup from codegen: set name, entity key, entity string indices, and flags. + /// Combined entity setup from codegen: set name, object_id hash, entity string indices, and flags. /// Bit layout of entity_fields is defined by the ENTITY_FIELD_*_SHIFT constants above. - void configure_entity_(const char *name, uint32_t entity_key, uint32_t entity_fields); + void configure_entity_(const char *name, uint32_t object_id_hash, uint32_t entity_fields); #ifdef USE_DEVICES // Codegen-only setter — only accessible from setup() via friend declaration. @@ -233,24 +240,13 @@ class EntityBase { #endif /// Non-template helper for make_entity_preference() to avoid code bloat. - /// Migrates preferences from the old sanitized-object_id key to the raw-name key - /// on key-lookup platforms. See: https://github.com/esphome/backlog/issues/85 + /// When the preference hash algorithm changes, migration logic goes here. ESPPreferenceObject make_entity_preference_(size_t size, uint32_t version); - void calc_entity_key_(); - - /// Reconstruct the OLD (pre-2026.8.0) sanitized-object_id hash for preference keys. - uint32_t calc_old_object_id_hash_() const; - - /// Preference key base for this entity: raw-name entity key XOR device_id. - uint32_t preference_key_base_() const { return this->entity_key_ ^ this->get_device_id_or_zero(); } - - /// Legacy preference key base: sanitized-object_id hash XOR device_id. - /// Note: device_id is 0 for the main device, so XORing with 0 preserves the original hash. - uint32_t old_preference_key_base_() const { return this->calc_old_object_id_hash_() ^ this->get_device_id_or_zero(); } + void calc_object_id_(); StringRef name_; - uint32_t entity_key_{}; + uint32_t object_id_hash_{}; #ifdef USE_DEVICES Device *device_{}; #endif diff --git a/esphome/core/entity_helpers.py b/esphome/core/entity_helpers.py index 5060e32a2d..54e2551cb4 100644 --- a/esphome/core/entity_helpers.py +++ b/esphome/core/entity_helpers.py @@ -25,86 +25,25 @@ from esphome.core.config import ( from esphome.cpp_generator import MockObj, RawStatement, add, get_variable from esphome.cpp_types import App import esphome.final_validate as fv -from esphome.helpers import cpp_string_escape, fnv1_hash_name, sanitize, snake_case +from esphome.helpers import ( + cpp_string_escape, + fnv1_hash, + fnv1_hash_object_id, + sanitize, + snake_case, +) from esphome.types import ConfigType, EntityMetadata _LOGGER = logging.getLogger(__name__) DOMAIN = "entity_string_pool" -_OBJECT_ID_DOMAIN = "entity_object_ids" - - -@dataclass -class ObjectIdEntity: - """An entity tracked by the sanitized object_id its name resolves to.""" - - name: str - platform: str - config: ConfigType - - -def _get_object_id_registry() -> dict[tuple[str, str, str], list[ObjectIdEntity]]: - """(device_id, platform, sanitized object_id) -> entities resolving to it.""" - return CORE.data.setdefault(_OBJECT_ID_DOMAIN, {}) - - -def validate_no_object_id_conflicts( - reason: str, - conflict_filter: Callable[[list[ObjectIdEntity], ConfigType], bool] | None = None, -) -> Callable[[ConfigType], ConfigType]: - """Create a final-validate step that rejects entities with colliding object_ids. - - Entity keys are hashed from the raw name, so names that only differ in characters - lost during sanitizing (for example two UTF-8 names) validate fine in general. - Components that still address entities by the sanitized object_id string must - reject those configs until they are migrated to raw names. - - Args: - reason: One sentence stating what the component builds from the object_id, - e.g. "mqtt builds default topics from the entity object_id" - conflict_filter: Optional predicate receiving the colliding entities and the - component config; return False when the component is not affected - - Returns: - A validator function for use as (or within) FINAL_VALIDATE_SCHEMA - """ - - def validator(config: ConfigType) -> ConfigType: - # Skip in testing_mode, which is used for grouped component testing - if CORE.testing_mode: - return config - conflicts = { - key: entities - for key, entities in _get_object_id_registry().items() - if len(entities) > 1 - and (conflict_filter is None or conflict_filter(entities, config)) - } - if not conflicts: - return config - lines = [f"{reason}, so these entities would conflict:"] - lines.extend( - f" - {platform} entities " - + ", ".join(f"'{e.name}'" for e in entities) - + (f" on device '{device_id}'" if device_id else "") - + f" share the object_id '{object_id}'" - for (device_id, platform, object_id), entities in conflicts.items() - ) - lines.append( - "To fix: Add unique ASCII characters (e.g., '1', '2', or 'A', 'B') " - "to distinguish the names" - ) - raise cv.Invalid("\n".join(lines)) - - return validator - - # Private config keys for storing registered string indices _KEY_DC_IDX = "_entity_dc_idx" _KEY_UOM_IDX = "_entity_uom_idx" _KEY_ICON_IDX = "_entity_icon_idx" _KEY_ENTITY_NAME = "_entity_name" -_KEY_ENTITY_KEY = "_entity_key" +_KEY_OBJECT_ID_HASH = "_entity_object_id_hash" # Bit layout for entity_fields in configure_entity_(). # Keep in sync with ENTITY_FIELD_*_SHIFT constants in esphome/core/entity_base.h @@ -367,7 +306,7 @@ def finalize_entity_strings(var: MockObj, config: ConfigType) -> None: standalone ``var->configure_entity_(name, hash, packed)``. """ entity_name = config[_KEY_ENTITY_NAME] - entity_key = config[_KEY_ENTITY_KEY] + object_id_hash = config[_KEY_OBJECT_ID_HASH] dc_idx = config.get(_KEY_DC_IDX, 0) uom_idx = config.get(_KEY_UOM_IDX, 0) icon_idx = config.get(_KEY_ICON_IDX, 0) @@ -387,30 +326,57 @@ def finalize_entity_strings(var: MockObj, config: ConfigType) -> None: register_method = config.get(_KEY_REGISTER_METHOD) if register_method is not None: expr = getattr(App, f"register_{register_method}")( - var, entity_name, entity_key, packed + var, entity_name, object_id_hash, packed ) else: - expr = var.configure_entity_(entity_name, entity_key, packed) + expr = var.configure_entity_(entity_name, object_id_hash, packed) if comment: add(RawStatement(f"{expr}; // {comment}")) else: add(expr) -def get_base_entity_name( +def get_base_entity_object_id( name: str, friendly_name: str | None, device_name: str | None = None ) -> str: - """Return the base name whose hash becomes this entity's key on the device. + """Calculate the base object ID for an entity that will be set via set_object_id(). - Follows the name selection in C++ EntityBase::configure_entity_() (entity_base.cpp): - entity name, then sub-device name, then friendly name, then the device name. + This function calculates what object_id_c_str_ should be set to in C++. - This is a config-time approximation for duplicate checking: when - name_add_mac_suffix is enabled the device appends the MAC suffix at runtime, - which is unknown here and identical for every entity on the device, so - ignoring it cannot change whether two entities collide with each other. + The C++ EntityBase::write_object_id_to() (entity_base.cpp) works as: + - If !has_own_name && is_name_add_mac_suffix_enabled(): + return str_sanitize(str_snake_case(App.get_friendly_name())) // Dynamic + - Else: + return object_id_c_str_ ?? "" // What we set via set_object_id() + + Since we're calculating what to pass to set_object_id(), we always need to + generate the object_id the same way, regardless of name_add_mac_suffix setting. + + Args: + name: The entity name (empty string if no name) + friendly_name: The friendly name from CORE.friendly_name + device_name: The device name if entity is on a sub-device + + Returns: + The base object ID to use for duplicate checking and to pass to set_object_id() """ - return name or device_name or friendly_name or CORE.name + + if name: + # Entity has its own name (has_own_name will be true) + base_str = name + elif device_name: + # Entity has empty name and is on a sub-device + # C++ EntityBase::set_name() uses device->get_name() when device is set + base_str = device_name + elif friendly_name: + # Entity has empty name (has_own_name will be false) + # C++ uses App.get_friendly_name() which returns friendly_name or device name + base_str = friendly_name + else: + # Fallback to device name + base_str = CORE.name + + return sanitize(snake_case(base_str)) def setup_entity(var_or_platform, config=None, platform=None): @@ -469,15 +435,15 @@ async def _setup_entity_impl(var: MockObj, config: ConfigType, platform: str) -> device: MockObj = await get_variable(device_id_obj) add(var.set_device_(device)) - # Pre-compute entity name and entity key for configure_entity_() + # Pre-compute entity name and object_id hash for configure_entity_() # which is emitted later by finalize_entity_strings(). - # For named entities: pre-compute the key from the raw entity name - # For empty-name entities: pass 0, C++ calculates the key at runtime from - # device name, friendly_name, or app name + # For named entities: pre-compute hash from entity name + # For empty-name entities: pass 0, C++ calculates hash at runtime from + # device name, friendly_name, or app name (bug-for-bug compatibility) entity_name = config[CONF_NAME] - entity_key = fnv1_hash_name(entity_name) if entity_name else 0 + object_id_hash = fnv1_hash_object_id(entity_name) if entity_name else 0 config[_KEY_ENTITY_NAME] = entity_name - config[_KEY_ENTITY_KEY] = entity_key + config[_KEY_OBJECT_ID_HASH] = object_id_hash # Store flags for packing into configure_entity_() config[_KEY_DISABLED_BY_DEFAULT] = int(config[CONF_DISABLED_BY_DEFAULT]) if CONF_INTERNAL in config: @@ -590,13 +556,16 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy # Use the device ID string directly for uniqueness device_id = device_id_obj.id - # Hash the same raw name the device hashes into the entity key at runtime. - # This handles empty names correctly by using device/friendly names. - base_name = get_base_entity_name(entity_name, CORE.friendly_name, device_name) - name_hash = fnv1_hash_name(base_name) + # Calculate what object_id will actually be used + # This handles empty names correctly by using device/friendly names + name_key = get_base_entity_object_id( + entity_name, CORE.friendly_name, device_name + ) - # Check for duplicates: two entities on the same device and platform must not - # share an entity key, since the key is what routes state to API clients + # Check for duplicates by the FNV-1 hash of the object_id, which is the entity + # key that routes state to API clients. This rejects names that sanitize to the + # same object_id, and also two different object_ids whose 32-bit hashes collide. + name_hash = fnv1_hash(name_key) unique_key = (device_id, platform, name_hash) if unique_key in CORE.unique_ids: # Get the existing entity metadata @@ -621,14 +590,26 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy if existing_component != "unknown": conflict_msg += f" from component '{existing_component}'" - # Different names can only clash here through a genuine hash collision + # Distinguish names that sanitize to the same object_id from a genuine + # 32-bit hash collision between two different object_ids collision_msg = "" if entity_name != existing_name: - collision_msg = ( - f"\n The names '{entity_name}' and '{existing_name}' produce the" - f"\n same entity key hash ({name_hash:#010x})." - "\n To fix: Rename one of the entities" + existing_object_id = get_base_entity_object_id( + existing_name, CORE.friendly_name, existing_device or None ) + if existing_object_id == name_key: + collision_msg = ( + f"\n Original names: '{entity_name}' and '{existing_name}'" + f"\n Both convert to ASCII ID: '{name_key}'" + "\n To fix: Add unique ASCII characters (e.g., '1', '2', or 'A', 'B')" + "\n to distinguish them" + ) + else: + collision_msg = ( + f"\n The object_ids '{name_key}' and '{existing_object_id}'" + f"\n produce the same entity key hash ({name_hash:#010x})." + "\n To fix: Rename one of the entities" + ) # Skip duplicate entity name validation when testing_mode is enabled # This flag is used for grouped component testing @@ -640,19 +621,6 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy f"{collision_msg}" ) - # Components that still address entities by the sanitized object_id reject - # colliding names in final validation via validate_no_object_id_conflicts(), - # so track every entity by the object_id its name resolves to. Scoped per - # device and platform to match the strictness configs had before entity keys - # moved to raw names: same-named entities on different sub-devices were - # already accepted then, internal entities were already skipped (above), and - # overlaps between platforms that share an MQTT component type (sensor and - # text_sensor both publish under "sensor") were already possible. - object_id = sanitize(snake_case(base_name)) - _get_object_id_registry().setdefault( - (device_id, platform, object_id), [] - ).append(ObjectIdEntity(base_name, platform, config)) - # Store metadata about this entity entity_metadata: EntityMetadata = { "name": entity_name, diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index d883ce146e..994fa2c26a 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -809,19 +809,6 @@ constexpr uint32_t FNV1_OFFSET_BASIS = 2166136261UL; /// FNV-1 32-bit prime constexpr uint32_t FNV1_PRIME = 16777619UL; -/// Calculate a FNV-1 hash over raw bytes with an explicit length. Unlike fnv1_hash(const char *), -/// each byte is hashed as an unsigned value, so results are platform-independent for bytes >= 0x80. -/// IMPORTANT: Must match Python fnv1_hash_name() in esphome/helpers.py, which hashes the UTF-8 -/// encoded bytes of the name. Used to compute entity keys from raw names. -inline uint32_t fnv1_hash_bytes(const char *str, size_t len) { - uint32_t hash = FNV1_OFFSET_BASIS; - for (size_t i = 0; i < len; i++) { - hash *= FNV1_PRIME; - hash ^= static_cast(str[i]); - } - return hash; -} - /// Extend a FNV-1 hash with an integer (hashes each byte). template constexpr uint32_t fnv1_hash_extend(uint32_t hash, T value) { using UnsignedT = std::make_unsigned_t; @@ -1026,20 +1013,12 @@ template inline char *str_sanitize_to(char (&buffer)[N], const char *s // str_sanitize moved to alloc_helpers.h - remove this comment before 2026.11.0 /// Calculate FNV-1 hash of a string while applying snake_case + sanitize transformations. -/// This is the LEGACY entity hash, kept only to reconstruct preference keys that existing -/// devices already have stored; see https://github.com/esphome/backlog/issues/85. -/// With per_code_point set, UTF-8 continuation bytes are skipped so each multi-byte character -/// contributes one underscore — this matches Python fnv1_hash_object_id() in esphome/helpers.py, -/// which produced the hash for named entities. The per-byte form (default) matches the old -/// runtime hash for entities without their own name. Do not change either behavior. -/// Known limitation: Python's lower() is Unicode aware, so the rare code points it maps to a -/// different number of characters or to ASCII (e.g. 'İ', the Kelvin sign) reconstruct wrong; -/// such names skip migration once and fall back to their defaults. -inline uint32_t fnv1_hash_object_id(const char *str, size_t len, bool per_code_point = false) { +/// This computes object_id hashes directly from names without creating an intermediate buffer. +/// IMPORTANT: Must match Python fnv1_hash_object_id() in esphome/helpers.py. +/// If you modify this function, update the Python version and tests in both places. +inline uint32_t fnv1_hash_object_id(const char *str, size_t len) { uint32_t hash = FNV1_OFFSET_BASIS; for (size_t i = 0; i < len; i++) { - if (per_code_point && (static_cast(str[i]) & 0xC0) == 0x80) - continue; // UTF-8 continuation byte, already counted via its lead byte hash *= FNV1_PRIME; // Apply snake_case (space->underscore, uppercase->lowercase) then sanitize hash ^= static_cast(to_sanitized_char(to_snake_case_char(str[i]))); diff --git a/esphome/core/preference_backend.h b/esphome/core/preference_backend.h index 0622376fca..5df0804bdd 100644 --- a/esphome/core/preference_backend.h +++ b/esphome/core/preference_backend.h @@ -24,9 +24,10 @@ #endif // Key-lookup preference backends find stored data by key; their platforms add the -// USE_PREFERENCE_KEY_LOOKUP define from Python codegen, which enables preference key -// migration. Slot-based backends (ESP8266, RP2040) instead allocate a storage slot for -// every make_preference() call and use the key only as a validity tag on that slot; +// USE_PREFERENCE_KEY_LOOKUP define from Python codegen, which enables one-shot reads +// of stored data by key (the primitive preference key migrations need). Slot-based +// backends (ESP8266, RP2040) instead allocate a storage slot for every +// make_preference() call and use the key only as a validity tag on that slot; // migration is not possible there, and key collisions cannot corrupt data. namespace esphome { @@ -104,10 +105,9 @@ concept PreferencesContract = requires(T prefs, size_t len, uint32_t type, bool }; // Key-lookup platforms additionally provide load_from_key(), a one-shot read -// of a stored preference by key that migrate_preference() relies on; see the -// key-lookup note at the top of this file. Not part of PreferencesContract, -// so it is asserted in preferences.h only where USE_PREFERENCE_KEY_LOOKUP -// is set. +// of a stored preference by key; see the key-lookup note at the top of this +// file. Not part of PreferencesContract, so it is asserted in preferences.h +// only where USE_PREFERENCE_KEY_LOOKUP is set. template concept PreferencesKeyLookupContract = requires(T prefs, uint32_t type, uint8_t *data, size_t len) { { prefs.load_from_key(type, data, len) } -> std::same_as; diff --git a/esphome/core/preferences.cpp b/esphome/core/preferences.cpp deleted file mode 100644 index 8508647255..0000000000 --- a/esphome/core/preferences.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include "esphome/core/preferences.h" -#include "esphome/core/log.h" -#include - -namespace esphome { - -#ifdef USE_PREFERENCE_KEY_LOOKUP -static const char *const TAG = "preferences"; - -bool migrate_preference(ESPPreferenceObject &new_pref, uint8_t *scratch, size_t size, uint32_t old_key, - uint32_t new_key) { - if (new_pref.load(scratch, size)) - return true; // Current data present - never overwrite newer data with the old copy - // One-shot read by key: no backend is allocated for the old key, so boots with - // nothing to migrate (for example fresh installs) cost no heap - if (old_key == new_key || !global_preferences->load_from_key(old_key, scratch, size)) - return false; // No data stored under the old key, nothing to migrate - if (!new_pref.save(scratch, size)) { - ESP_LOGW(TAG, "Pref migration %" PRIx32 " -> %" PRIx32 " failed", old_key, new_key); - } - return true; -} -#endif // USE_PREFERENCE_KEY_LOOKUP - -} // namespace esphome diff --git a/esphome/core/preferences.h b/esphome/core/preferences.h index cfeddebda7..ed23dfae56 100644 --- a/esphome/core/preferences.h +++ b/esphome/core/preferences.h @@ -56,17 +56,5 @@ namespace esphome { static_assert(PreferencesKeyLookupContract, "This platform emits USE_PREFERENCE_KEY_LOOKUP but its preferences manager does not provide " "load_from_key() (esphome/core/preference_backend.h)"); - -/// Copy preference data stored under old_key into new_pref (created for new_key) if the keys -/// differ and new_pref has no data yet. scratch must hold at least size bytes. -/// Returns true when scratch holds the entity's current data (loaded or just migrated). -/// The old entry is intentionally left in place so a firmware downgrade still finds its data. -/// If saving under the new key fails, callers that consume scratch (like TextSaver) still get -/// valid data for this boot, callers that reload from the preference fall back to their -/// defaults, and the migration simply runs again on the next boot. -/// Only available on key-lookup preference backends; slot-based backends keep their old -/// keys instead. See: https://github.com/esphome/backlog/issues/85 -bool migrate_preference(ESPPreferenceObject &new_pref, uint8_t *scratch, size_t size, uint32_t old_key, - uint32_t new_key); } // namespace esphome #endif // USE_PREFERENCE_KEY_LOOKUP diff --git a/esphome/helpers.py b/esphome/helpers.py index 2731109164..9b2a461ccd 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -91,13 +91,8 @@ def fnv1a_32bit_hash(string: str) -> int: def fnv1_hash_object_id(name: str) -> int: """Compute FNV-1 hash of name with snake_case + sanitize transformations. - IMPORTANT: Must produce same result as C++ fnv1_hash_object_id() in helpers.h - with per_code_point set. This is the OLD entity hash; it computes preference - keys that existing devices already have stored (see - https://github.com/esphome/backlog/issues/85) and is also still used for live - keys derived from config IDs (see the motion component's calibration key). - Note: lower() here is Unicode aware while the C++ reconstruction is not; see - the known limitation note on the C++ function. + IMPORTANT: Must produce same result as C++ fnv1_hash_object_id() in helpers.h. + If you modify this function, update the C++ version and tests in both places. """ return fnv1_hash(sanitize(snake_case(name))) @@ -105,9 +100,9 @@ def fnv1_hash_object_id(name: str) -> int: def fnv1_hash_name(name: str) -> int: """Compute FNV-1 hash of the raw entity name (UTF-8 bytes, no transformations). - IMPORTANT: Must produce same result as C++ fnv1_hash_bytes() in helpers.h, - which hashes the name bytes as stored on the device. - Used for pre-computing entity keys at code generation time. + 2026.8 beta firmware stored preferences under keys derived from this hash; + a future key migration must reconstruct those keys to recover that data + (see https://github.com/esphome/backlog/issues/85). """ return _fnv1_hash(name.encode("utf-8")) diff --git a/tests/integration/entity_utils.py b/tests/integration/entity_utils.py index 95f6a0321e..7596983ee2 100644 --- a/tests/integration/entity_utils.py +++ b/tests/integration/entity_utils.py @@ -8,7 +8,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from esphome.helpers import fnv1_hash_name, sanitize, snake_case +from esphome.helpers import fnv1_hash_object_id, sanitize, snake_case if TYPE_CHECKING: from aioesphomeapi import DeviceInfo, EntityInfo @@ -25,16 +25,15 @@ def infer_name_add_mac_suffix(device_info: DeviceInfo) -> bool: return device_info.name.endswith(f"-{mac_suffix}") -def _resolve_entity_name( +def _get_name_for_object_id( entity: EntityInfo, device_info: DeviceInfo, device_id_to_name: dict[int, str], ) -> str: - """Resolve the effective name for an entity. + """Get the name used for object_id computation. This is the algorithm that aioesphomeapi will use to determine which - name to use for computing object_id client-side from API data; the same - name is what the device hashes into the entity key. + name to use for computing object_id client-side from API data. Args: entity: The entity to get name for @@ -73,27 +72,27 @@ def compute_entity_object_id( Returns: The computed object_id string """ - name = _resolve_entity_name(entity, device_info, device_id_to_name) - return compute_object_id(name) + name_for_id = _get_name_for_object_id(entity, device_info, device_id_to_name) + return compute_object_id(name_for_id) -def compute_entity_key( +def compute_entity_hash( entity: EntityInfo, device_info: DeviceInfo, device_id_to_name: dict[int, str], ) -> int: - """Compute expected entity key for an entity. + """Compute expected object_id hash for an entity. Args: - entity: The entity to compute the key for + entity: The entity to compute hash for device_info: Device info from the API device_id_to_name: Mapping of device_id to device name for sub-devices Returns: - The computed FNV-1 hash of the raw name + The computed FNV-1 hash """ - name = _resolve_entity_name(entity, device_info, device_id_to_name) - return fnv1_hash_name(name) + name_for_id = _get_name_for_object_id(entity, device_info, device_id_to_name) + return fnv1_hash_object_id(name_for_id) def verify_entity_object_id( @@ -119,7 +118,7 @@ def verify_entity_object_id( f"expected '{expected_object_id}', got '{entity.object_id}'" ) - expected_hash = compute_entity_key(entity, device_info, device_id_to_name) + expected_hash = compute_entity_hash(entity, device_info, device_id_to_name) assert entity.key == expected_hash, ( f"hash mismatch for entity '{entity.name}': " f"expected {expected_hash:#x}, got {entity.key:#x}" diff --git a/tests/integration/fixtures/fnv1_hash_object_id.yaml b/tests/integration/fixtures/fnv1_hash_object_id.yaml index d4511bb8c6..2097b2fbf9 100644 --- a/tests/integration/fixtures/fnv1_hash_object_id.yaml +++ b/tests/integration/fixtures/fnv1_hash_object_id.yaml @@ -71,38 +71,6 @@ esphome: ESP_LOGE("FNV1_OID", "empty FAILED: 0x%08x != 0x811c9dc5", hash_empty); } - // Raw name hash: matches Python fnv1_hash_name("My Sensor Name") - uint32_t hash_raw = esphome::fnv1_hash_bytes("My Sensor Name", 14); - if (hash_raw == 0x8cec6fb0) { - ESP_LOGI("FNV1_OID", "raw PASSED"); - } else { - ESP_LOGE("FNV1_OID", "raw FAILED: 0x%08x != 0x8cec6fb0", hash_raw); - } - - // Raw name hash over UTF-8 bytes: matches Python fnv1_hash_name("Température") - uint32_t hash_raw_utf8 = esphome::fnv1_hash_bytes("Temp\xc3\xa9rature", 12); - if (hash_raw_utf8 == 0x531a74aa) { - ESP_LOGI("FNV1_OID", "raw_utf8 PASSED"); - } else { - ESP_LOGE("FNV1_OID", "raw_utf8 FAILED: 0x%08x != 0x531a74aa", hash_raw_utf8); - } - - // Old-key UTF-8 variant: matches Python fnv1_hash_object_id("Température") - uint32_t hash_old_utf8 = esphome::fnv1_hash_object_id("Temp\xc3\xa9rature", 12, true); - if (hash_old_utf8 == 0x965698f3) { - ESP_LOGI("FNV1_OID", "old_utf8 PASSED"); - } else { - ESP_LOGE("FNV1_OID", "old_utf8 FAILED: 0x%08x != 0x965698f3", hash_old_utf8); - } - - // Old-key UTF-8 variant with multi-byte only name: Python fnv1_hash_object_id("温度") - uint32_t hash_old_cjk = esphome::fnv1_hash_object_id("\xe6\xb8\xa9\xe5\xba\xa6", 6, true); - if (hash_old_cjk == 0x3276cb9f) { - ESP_LOGI("FNV1_OID", "old_cjk PASSED"); - } else { - ESP_LOGE("FNV1_OID", "old_cjk FAILED: 0x%08x != 0x3276cb9f", hash_old_cjk); - } - host: api: logger: diff --git a/tests/integration/fixtures/multi_device_preferences.yaml b/tests/integration/fixtures/multi_device_preferences.yaml index 582add90a8..01e4394559 100644 --- a/tests/integration/fixtures/multi_device_preferences.yaml +++ b/tests/integration/fixtures/multi_device_preferences.yaml @@ -156,17 +156,10 @@ button: ESP_LOGI("test", "Device A Mode: %s", id(mode_device_a).current_option().c_str()); ESP_LOGI("test", "Device B Mode: %s", id(mode_device_b).current_option().c_str()); ESP_LOGI("test", "Main Mode: %s", id(mode_main).current_option().c_str()); - // Log preference key bases for entities that actually store preferences. - // This is the key base make_entity_preference() uses: entity key XOR device id. - ESP_LOGI("test", "Device A Switch Pref Hash: %u", - id(light_device_a).get_entity_key() ^ id(light_device_a).get_device_id_or_zero()); - ESP_LOGI("test", "Device B Switch Pref Hash: %u", - id(light_device_b).get_entity_key() ^ id(light_device_b).get_device_id_or_zero()); - ESP_LOGI("test", "Main Switch Pref Hash: %u", - id(light_main).get_entity_key() ^ id(light_main).get_device_id_or_zero()); - ESP_LOGI("test", "Device A Number Pref Hash: %u", - id(setpoint_device_a).get_entity_key() ^ id(setpoint_device_a).get_device_id_or_zero()); - ESP_LOGI("test", "Device B Number Pref Hash: %u", - id(setpoint_device_b).get_entity_key() ^ id(setpoint_device_b).get_device_id_or_zero()); - ESP_LOGI("test", "Main Number Pref Hash: %u", - id(setpoint_main).get_entity_key() ^ id(setpoint_main).get_device_id_or_zero()); + // Log preference hashes for entities that actually store preferences + ESP_LOGI("test", "Device A Switch Pref Hash: %u", id(light_device_a).get_preference_hash()); + ESP_LOGI("test", "Device B Switch Pref Hash: %u", id(light_device_b).get_preference_hash()); + ESP_LOGI("test", "Main Switch Pref Hash: %u", id(light_main).get_preference_hash()); + ESP_LOGI("test", "Device A Number Pref Hash: %u", id(setpoint_device_a).get_preference_hash()); + ESP_LOGI("test", "Device B Number Pref Hash: %u", id(setpoint_device_b).get_preference_hash()); + ESP_LOGI("test", "Main Number Pref Hash: %u", id(setpoint_main).get_preference_hash()); diff --git a/tests/integration/fixtures/preference_key_migration.yaml b/tests/integration/fixtures/preference_key_migration.yaml deleted file mode 100644 index a9b01fc2d2..0000000000 --- a/tests/integration/fixtures/preference_key_migration.yaml +++ /dev/null @@ -1,35 +0,0 @@ -esphome: - name: host-pref-key-migration - -host: -api: -logger: - -switch: - - platform: template - id: test_switch_restore - name: Test Switch - optimistic: true - restore_mode: RESTORE_DEFAULT_OFF - -number: - - platform: template - id: test_number_restore - name: Test Number - optimistic: true - restore_value: true - initial_value: 1.0 - min_value: 0 - max_value: 100 - step: 0.5 - -text: - - platform: template - id: test_text_restore - name: Test Text - mode: text - optimistic: true - restore_value: true - initial_value: fallback - min_length: 0 - max_length: 20 diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index c7f21d8a01..f835bee3bc 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -25,25 +25,15 @@ def clear_host_prefs(device_name: str) -> None: host_prefs_path(device_name).unlink(missing_ok=True) -def write_host_prefs(device_name: str, entries: dict[int, bytes]) -> Path: - """Write preference entries, replacing the file's contents. - - Returns the path that was written. - """ - payload = b"" - for key, data in entries.items(): - if len(data) > 255: - raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") - payload += struct.pack(" Path: """Write a single preference entry, replacing the file's contents. Returns the path that was written. """ - return write_host_prefs(device_name, {key: data}) + if len(data) > 255: + raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") + path = host_prefs_path(device_name) + path.parent.mkdir(parents=True, exist_ok=True) + payload = struct.pack(" None: diff --git a/tests/integration/test_object_id_api_verification.py b/tests/integration/test_object_id_api_verification.py index 8dafb37c64..c8603e0682 100644 --- a/tests/integration/test_object_id_api_verification.py +++ b/tests/integration/test_object_id_api_verification.py @@ -2,8 +2,8 @@ This test verifies a three-way match between: 1. C++ object_id generation (get_object_id_to using to_sanitized_char/to_snake_case_char) -2. C++ entity key generation (fnv1_hash of the raw name in helpers.h) -3. Python computation (sanitize/snake_case and fnv1_hash_name in helpers.py) +2. C++ hash generation (fnv1_hash_object_id in helpers.h) +3. Python computation (sanitize/snake_case in helpers.py, fnv1_hash_object_id) The API response contains C++ computed values, so verifying API == Python implicitly verifies C++ == Python == API for both object_id and hash. @@ -25,7 +25,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_name +from esphome.helpers import fnv1_hash_object_id from .entity_utils import compute_object_id, verify_all_entities from .types import APIClientConnectedFactory, RunCompiledFunction @@ -123,7 +123,7 @@ async def test_object_id_api_verification( ) # Verify hash can be computed from the name - hash_from_name = fnv1_hash_name(entity_name) + hash_from_name = fnv1_hash_object_id(entity_name) assert hash_from_name == entity.key, ( f"Entity '{entity_name}': hash mismatch. " f"Python hash {hash_from_name:#x}, API key {entity.key:#x}" @@ -164,7 +164,7 @@ async def test_object_id_api_verification( ) # Verify hash matches - expected_hash = fnv1_hash_name(expected_name) + expected_hash = fnv1_hash_object_id(expected_name) assert entity.key == expected_hash, ( f"Empty-name entity (device_id={entity.device_id}): hash mismatch. " f"API key: {entity.key:#x}, expected: {expected_hash:#x}" diff --git a/tests/integration/test_object_id_friendly_name_no_mac_suffix.py b/tests/integration/test_object_id_friendly_name_no_mac_suffix.py index b58593f2ef..7199a2b371 100644 --- a/tests/integration/test_object_id_friendly_name_no_mac_suffix.py +++ b/tests/integration/test_object_id_friendly_name_no_mac_suffix.py @@ -11,7 +11,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_name +from esphome.helpers import fnv1_hash_object_id from .entity_utils import ( compute_object_id, @@ -62,7 +62,7 @@ async def test_object_id_friendly_name_no_mac_suffix( ) # Hash should match friendly_name - expected_hash = fnv1_hash_name("My Friendly Device") + expected_hash = fnv1_hash_object_id("My Friendly Device") assert entity.key == expected_hash, ( f"Expected hash {expected_hash:#x}, got {entity.key:#x}" ) diff --git a/tests/integration/test_object_id_no_friendly_name.py b/tests/integration/test_object_id_no_friendly_name.py index 45b5f730a6..b548f02fde 100644 --- a/tests/integration/test_object_id_no_friendly_name.py +++ b/tests/integration/test_object_id_no_friendly_name.py @@ -17,7 +17,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_name +from esphome.helpers import fnv1_hash_object_id from .entity_utils import compute_object_id, verify_all_entities from .types import APIClientConnectedFactory, RunCompiledFunction @@ -96,7 +96,7 @@ async def test_object_id_no_friendly_name_no_mac_suffix( OLD behavior: - is_object_id_dynamic_() returned false (mac suffix not enabled) - Used object_id_c_str_ which was pre-computed in Python - - Python used get_base_entity_name() with fallback to CORE.name + - Python used get_base_entity_object_id() with fallback to CORE.name Result: object_id = sanitize(snake_case(device_name)) """ @@ -126,7 +126,7 @@ async def test_object_id_no_friendly_name_no_mac_suffix( ) # Hash should match device name - expected_hash = fnv1_hash_name("test-device") + expected_hash = fnv1_hash_object_id("test-device") assert entity.key == expected_hash, ( f"Expected hash {expected_hash:#x}, got {entity.key:#x}" ) diff --git a/tests/integration/test_preference_key_migration.py b/tests/integration/test_preference_key_migration.py deleted file mode 100644 index e7f699bb12..0000000000 --- a/tests/integration/test_preference_key_migration.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Integration test for entity preference key migration. - -Entity keys are now the FNV-1 hash of the raw name instead of the sanitized -object_id (https://github.com/esphome/backlog/issues/85). On key-lookup -preference backends, make_entity_preference() must move data stored under the -old key to the new key, so devices keep their restored state after upgrading. - -This test seeds the host preferences file the way a pre-migration firmware -would have written it and verifies: -1. Data stored under the OLD key is restored (migration happened, no data loss) -2. Data already stored under the NEW key is never overwritten by old data -""" - -from __future__ import annotations - -import socket -import struct - -from aioesphomeapi import ( - NumberInfo, - NumberState, - SwitchInfo, - SwitchState, - TextInfo, - TextState, -) -import pytest - -from esphome.helpers import fnv1_hash, fnv1_hash_name, fnv1_hash_object_id - -from .conftest import run_binary_and_wait_for_port, wait_and_connect_api_client -from .host_prefs import clear_host_prefs, write_host_prefs -from .state_utils import InitialStateHelper, require_entity -from .types import CompileFunction, ConfigWriter - -DEVICE_NAME = "host-pref-key-migration" - -# The pre-migration preference key was the sanitized object_id hash; the new -# key is the raw-name hash. All entities are on the main device (device_id 0) -# and their preferences use no version salt, so the key is just the hash. -SWITCH_OLD_KEY = fnv1_hash_object_id("Test Switch") -SWITCH_NEW_KEY = fnv1_hash_name("Test Switch") -NUMBER_OLD_KEY = fnv1_hash_object_id("Test Number") -NUMBER_NEW_KEY = fnv1_hash_name("Test Number") - -# template_text salts its key with the length limits and pattern hash; this must -# match TemplateText::setup() in template_text.cpp (min_length 0, max_length 20, -# no pattern configured) -TEXT_KEY_EXTRA = (0 << 2) + (20 << 4) + (fnv1_hash("") << 6) -TEXT_OLD_KEY = (fnv1_hash_object_id("Test Text") + TEXT_KEY_EXTRA) & 0xFFFFFFFF -TEXT_NEW_KEY = (fnv1_hash_name("Test Text") + TEXT_KEY_EXTRA) & 0xFFFFFFFF - -# TextSaver<20> stores a length-prefixed buffer of max_length + 1 bytes -TEXT_MAX_LENGTH = 20 - - -def text_pref_payload(value: str) -> bytes: - """Build the length-prefixed buffer TextSaver stores for a value.""" - data = value.encode("utf-8") - assert len(data) <= TEXT_MAX_LENGTH - return bytes([len(data)]) + data + b"\x00" * (TEXT_MAX_LENGTH - len(data)) - - -@pytest.mark.asyncio -async def test_preference_key_migration( - yaml_config: str, - write_yaml_config: ConfigWriter, - compile_esphome: CompileFunction, - reserved_tcp_port: tuple[int, socket.socket], -) -> None: - """Test that preferences stored under the old key survive the upgrade.""" - port, port_socket = reserved_tcp_port - - assert SWITCH_OLD_KEY != SWITCH_NEW_KEY - assert NUMBER_OLD_KEY != NUMBER_NEW_KEY - assert TEXT_OLD_KEY != TEXT_NEW_KEY - - # Write and compile once - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - - # Release the reserved port so the binary can bind to it - port_socket.close() - - async def boot_and_get_initial_states() -> tuple[ - SwitchState, NumberState, TextState - ]: - """Boot the binary and return the restored entity states.""" - async with ( - run_binary_and_wait_for_port(binary_path, "127.0.0.1", port), - wait_and_connect_api_client(port=port) as client, - ): - device_info = await client.device_info() - assert device_info.name == DEVICE_NAME - - entities, _ = await client.list_entities_services() - switch_entity = require_entity( - entities, "test_switch", SwitchInfo, "Test Switch" - ) - number_entity = require_entity( - entities, "test_number", NumberInfo, "Test Number" - ) - text_entity = require_entity(entities, "test_text", TextInfo, "Test Text") - - initial_state_helper = InitialStateHelper(entities) - client.subscribe_states( - initial_state_helper.on_state_wrapper(lambda s: None) - ) - await initial_state_helper.wait_for_initial_states() - - switch_state = initial_state_helper.initial_states[switch_entity.key] - number_state = initial_state_helper.initial_states[number_entity.key] - text_state = initial_state_helper.initial_states[text_entity.key] - assert isinstance(switch_state, SwitchState) - assert isinstance(number_state, NumberState) - assert isinstance(text_state, TextState) - return switch_state, number_state, text_state - - try: - # --- Run 1: only OLD keys present, as written by pre-migration firmware. - # The restored states prove the data was migrated to the new keys. - write_host_prefs( - DEVICE_NAME, - { - SWITCH_OLD_KEY: b"\x01", # bool: switch was ON - NUMBER_OLD_KEY: struct.pack(" None: - """Verify _COMMAND_TOPIC_PLATFORMS matches the MQTT components that subscribe. - - Drift silently reintroduces shared subscribe topics, so this derives the set - from the C++ components that actually call subscribe(); that also catches - platforms like text that subscribe a command topic without exposing a - command_topic key in their schema. - """ - expected: set[str] = set() - for path in (COMPONENTS_DIR / "mqtt").glob("mqtt_*.cpp"): - if path.stem in _NON_ENTITY_MQTT_SOURCES: - continue - if "this->subscribe" not in path.read_text(encoding="utf-8"): - continue - stem = path.stem.removeprefix("mqtt_") - expected.add("datetime" if stem in _DATETIME_STEMS else stem) - assert expected == _COMMAND_TOPIC_PLATFORMS - - -def test_sub_topic_platforms_in_sync() -> None: - """Verify _SUB_TOPIC_PLATFORMS matches the MQTT components with sub-topics. - - Platforms whose MQTT headers use MQTT_COMPONENT_CUSTOM_TOPIC derive extra - topics such as position/command from the object_id. - """ - expected = { - path.stem.removeprefix("mqtt_") - for path in (COMPONENTS_DIR / "mqtt").glob("mqtt_*.h") - if path.stem != "mqtt_component" - and "MQTT_COMPONENT_CUSTOM_TOPIC" in path.read_text(encoding="utf-8") - } - assert expected == _SUB_TOPIC_PLATFORMS - - -def test_conflict_filter_exempts_custom_topics() -> None: - """Test that custom state topics with discovery off avoid the conflict.""" - validator = entity_duplicate_validator("sensor") - # Both entities have custom state topics and discovery disabled per entity, - # so no object_id-derived MQTT topic is used - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} - assert component_validator(config) is config - - # Without the filter the same conflicts are fatal - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - validate_no_object_id_conflicts(REASON)({}) - - -def test_conflict_on_default_command_topic() -> None: - """Test that commandable platforms conflict through their default command topic. - - Custom state topics with discovery off are not enough for platforms that also - subscribe to an object_id-derived command topic. - """ - validator = entity_duplicate_validator("switch") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - mqtt_config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} - # Both switches share the default command topic: rejected - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - component_validator(mqtt_config) - - # With custom command topics as well, nothing derives from the object_id - CORE.reset() - validator = entity_duplicate_validator("switch") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_COMMAND_TOPIC: "custom/cmd/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_COMMAND_TOPIC: "custom/cmd/b", - CONF_DISCOVERY: False, - } - ) - assert component_validator(mqtt_config) is mqtt_config - - -def test_conflict_on_sub_topic_platforms() -> None: - """Test that platforms with extra object_id sub-topics always conflict. - - Covers derive topics like position/command from the object_id through their - own config keys, so custom state and command topics cannot exempt them. - """ - validator = entity_duplicate_validator("cover") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_COMMAND_TOPIC: "custom/cmd/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_STATE_TOPIC: "custom/topic/b", - CONF_COMMAND_TOPIC: "custom/cmd/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - component_validator({CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"}) - - -def test_no_conflict_on_disjoint_default_topics() -> None: - """Test that entities whose default topics are disjoint do not conflict. - - One entity uses only the default command topic and the other only the default - state topic, so they never share a topic. - """ - validator = entity_duplicate_validator("switch") - validator( - { - CONF_NAME: "Датчик открытия", - CONF_STATE_TOPIC: "custom/topic/a", - CONF_DISCOVERY: False, - } - ) - validator( - { - CONF_NAME: "Датчик закрытия", - CONF_COMMAND_TOPIC: "custom/cmd/b", - CONF_DISCOVERY: False, - } - ) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} - assert component_validator(config) is config - - -def test_no_conflict_on_empty_topic_prefix() -> None: - """Test that an empty topic_prefix disables the default topic conflict. - - With topic_prefix set to null no default topics exist at runtime, so entities - without custom state topics cannot conflict; only discovery still matters. - """ - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Датчик открытия"}) - validator({CONF_NAME: "Датчик закрытия"}) - - component_validator = validate_no_object_id_conflicts( - REASON, conflict_filter=_topics_conflict - ) - # No default topics and no discovery: valid - config: dict = {CONF_DISCOVERY: False, CONF_TOPIC_PREFIX: ""} - assert component_validator(config) is config - - # Discovery still uses object_id-derived config topics: rejected - with pytest.raises(Invalid, match=r"mqtt builds default topics"): - component_validator({CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: ""}) diff --git a/tests/unit_tests/core/test_entity_helpers.py b/tests/unit_tests/core/test_entity_helpers.py index 64400c4fd4..53035ad713 100644 --- a/tests/unit_tests/core/test_entity_helpers.py +++ b/tests/unit_tests/core/test_entity_helpers.py @@ -1,4 +1,4 @@ -"""Tests for entity helpers: name selection, entity key hashing, duplicate checks.""" +"""Test get_base_entity_object_id function matches C++ behavior.""" from collections.abc import Callable, Generator from pathlib import Path @@ -25,17 +25,16 @@ from esphome.core.entity_helpers import ( _setup_entity_impl, entity_duplicate_validator, finalize_entity_strings, - get_base_entity_name, + get_base_entity_object_id, register_device_class, register_icon, register_unit_of_measurement, setup_device_class, setup_entity, setup_unit_of_measurement, - validate_no_object_id_conflicts, ) from esphome.cpp_generator import MockObj -from esphome.helpers import fnv1_hash_name, sanitize, snake_case +from esphome.helpers import fnv1_hash, sanitize, snake_case from .common import load_config_from_fixture @@ -58,26 +57,206 @@ def restore_core_state() -> Generator[None, None, None]: CORE.friendly_name = original_friendly_name -def test_get_base_entity_name_priority_order() -> None: +def test_with_entity_name() -> None: + """Test when entity has its own name - should use entity name.""" + # Simple name + assert get_base_entity_object_id("Temperature Sensor", None) == "temperature_sensor" + assert ( + get_base_entity_object_id("Temperature Sensor", "Device Name") + == "temperature_sensor" + ) + # Even with device name, entity name takes precedence + assert ( + get_base_entity_object_id("Temperature Sensor", "Device Name", "Sub Device") + == "temperature_sensor" + ) + + # Name with special characters + assert ( + get_base_entity_object_id("Temp!@#$%^&*()Sensor", None) + == "temp__________sensor" + ) + assert get_base_entity_object_id("Temp-Sensor_123", None) == "temp-sensor_123" + + # Already snake_case + assert get_base_entity_object_id("temperature_sensor", None) == "temperature_sensor" + + # Mixed case + assert get_base_entity_object_id("TemperatureSensor", None) == "temperaturesensor" + assert get_base_entity_object_id("TEMPERATURE SENSOR", None) == "temperature_sensor" + + +def test_empty_name_with_device_name() -> None: + """Test when entity has empty name and is on a sub-device - should use device name.""" + # C++ behavior: when has_own_name is false and device is set, uses device->get_name() + assert ( + get_base_entity_object_id("", "Friendly Device", "Sub Device 1") + == "sub_device_1" + ) + assert ( + get_base_entity_object_id("", "Kitchen Controller", "controller_1") + == "controller_1" + ) + assert get_base_entity_object_id("", None, "Test-Device_123") == "test-device_123" + + +def test_empty_name_with_friendly_name() -> None: + """Test when entity has empty name and no device - should use friendly name.""" + # C++ behavior: when has_own_name is false, uses App.get_friendly_name() + assert get_base_entity_object_id("", "Friendly Device") == "friendly_device" + assert get_base_entity_object_id("", "Kitchen Controller") == "kitchen_controller" + assert get_base_entity_object_id("", "Test-Device_123") == "test-device_123" + + # Special characters in friendly name + assert get_base_entity_object_id("", "Device!@#$%") == "device_____" + + +def test_empty_name_no_friendly_name() -> None: + """Test when entity has empty name and no friendly name - should use device name.""" + # Test with CORE.name set + CORE.name = "device-name" + assert get_base_entity_object_id("", None) == "device-name" + + CORE.name = "Test Device" + assert get_base_entity_object_id("", None) == "test_device" + + +def test_edge_cases() -> None: + """Test edge cases.""" + # Only spaces + assert get_base_entity_object_id(" ", None) == "___" + + # Unicode characters (should be replaced) + assert get_base_entity_object_id("Température", None) == "temp_rature" + assert get_base_entity_object_id("测试", None) == "__" + + # Empty string with empty friendly name (empty friendly name is treated as None) + # Falls back to CORE.name + CORE.name = "device" + assert get_base_entity_object_id("", "") == "device" + + # Very long name (should work fine) + long_name = "a" * 100 + " " + "b" * 100 + expected = "a" * 100 + "_" + "b" * 100 + assert get_base_entity_object_id(long_name, None) == expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("Temperature Sensor", "temperature_sensor"), + ("Living Room Light", "living_room_light"), + ("Test-Device_123", "test-device_123"), + ("Special!@#Chars", "special___chars"), + ("UPPERCASE NAME", "uppercase_name"), + ("lowercase name", "lowercase_name"), + ("Mixed Case Name", "mixed_case_name"), + (" Spaces ", "___spaces___"), + ], +) +def test_matches_cpp_helpers(name: str, expected: str) -> None: + """Test that the logic matches using snake_case and sanitize directly.""" + # For non-empty names, verify our function produces same result as direct snake_case + sanitize + assert get_base_entity_object_id(name, None) == sanitize(snake_case(name)) + assert get_base_entity_object_id(name, None) == expected + + +def test_empty_name_fallback() -> None: + """Test empty name handling which falls back to friendly_name or CORE.name.""" + # Empty name is handled specially - it doesn't just use sanitize(snake_case("")) + # Instead it falls back to friendly_name or CORE.name + assert sanitize(snake_case("")) == "" # Direct conversion gives empty string + # But our function returns a fallback + CORE.name = "device" + assert get_base_entity_object_id("", None) == "device" # Uses device name + + +def test_name_add_mac_suffix_behavior() -> None: + """Test behavior related to name_add_mac_suffix. + + In C++, an entity's object_id is computed from its name_ via + write_object_id_to() (sanitized snake_case). When an entity has no name, + configure_entity_() sets name_ from the friendly name, with the MAC suffix + appended when name_add_mac_suffix is enabled. Our function always returns + the same result since we're calculating the base for duplicate tracking. + """ + # The function should always return the same result regardless of + # name_add_mac_suffix setting, as we're calculating the base object_id + assert get_base_entity_object_id("", "Test Device") == "test_device" + assert get_base_entity_object_id("Entity Name", "Test Device") == "entity_name" + + +def test_priority_order() -> None: """Test the priority order: entity name > device name > friendly name > CORE.name.""" CORE.name = "core-device" - # 1. Entity name has highest priority and is used as-is, no transformations + # 1. Entity name has highest priority assert ( - get_base_entity_name("Entity Name", "Friendly Name", "Device Name") - == "Entity Name" + get_base_entity_object_id("Entity Name", "Friendly Name", "Device Name") + == "entity_name" ) - assert get_base_entity_name("Température", None) == "Température" # 2. Device name is next priority (when entity name is empty) - assert get_base_entity_name("", "Friendly Name", "Device Name") == "Device Name" + assert ( + get_base_entity_object_id("", "Friendly Name", "Device Name") == "device_name" + ) # 3. Friendly name is next (when entity and device names are empty) - assert get_base_entity_name("", "Friendly Name", None) == "Friendly Name" + assert get_base_entity_object_id("", "Friendly Name", None) == "friendly_name" - # 4. CORE.name is last resort; an empty friendly name falls through to it - assert get_base_entity_name("", None, None) == "core-device" - assert get_base_entity_name("", "") == "core-device" + # 4. CORE.name is last resort + assert get_base_entity_object_id("", None, None) == "core-device" + + +@pytest.mark.parametrize( + ("name", "friendly_name", "device_name", "expected"), + [ + # name, friendly_name, device_name, expected + ("Living Room Light", None, None, "living_room_light"), + ("", "Kitchen Controller", None, "kitchen_controller"), + ( + "", + "ESP32 Device", + "controller_1", + "controller_1", + ), # Device name takes precedence + ("GPIO2 Button", None, None, "gpio2_button"), + ("WiFi Signal", "My Device", None, "wifi_signal"), + ("", None, "esp32_node", "esp32_node"), + ("Front Door Sensor", "Home Assistant", "door_controller", "front_door_sensor"), + ], +) +def test_real_world_examples( + name: str, friendly_name: str | None, device_name: str | None, expected: str +) -> None: + """Test real-world entity naming scenarios.""" + result = get_base_entity_object_id(name, friendly_name, device_name) + assert result == expected + + +def test_issue_6953_scenarios() -> None: + """Test specific scenarios from issue #6953.""" + # Scenario 1: Multiple empty names on main device with name_add_mac_suffix + # The Python code calculates the base, C++ might append MAC suffix dynamically + CORE.name = "device-name" + CORE.friendly_name = "Friendly Device" + + # All empty names should resolve to same base + assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" + assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" + assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" + + # Scenario 2: Empty names on sub-devices + assert ( + get_base_entity_object_id("", "Main Device", "controller_1") == "controller_1" + ) + assert ( + get_base_entity_object_id("", "Main Device", "controller_2") == "controller_2" + ) + + # Scenario 3: xyz duplicates + assert get_base_entity_object_id("xyz", None) == "xyz" + assert get_base_entity_object_id("xyz", "Device") == "xyz" # Tests for setup_entity function @@ -336,10 +515,9 @@ def test_entity_duplicate_validator() -> None: config1 = {CONF_NAME: "Temperature"} validated1 = validator(config1) assert validated1 == config1 - temperature_key = ("", "sensor", fnv1_hash_name("Temperature")) - assert temperature_key in CORE.unique_ids + assert ("", "sensor", fnv1_hash("temperature")) in CORE.unique_ids # Check metadata was stored - metadata = CORE.unique_ids[temperature_key] + metadata = CORE.unique_ids[("", "sensor", fnv1_hash("temperature"))] assert metadata["name"] == "Temperature" assert metadata["platform"] == "sensor" @@ -347,9 +525,8 @@ def test_entity_duplicate_validator() -> None: config2 = {CONF_NAME: "Humidity"} validated2 = validator(config2) assert validated2 == config2 - humidity_key = ("", "sensor", fnv1_hash_name("Humidity")) - assert humidity_key in CORE.unique_ids - metadata2 = CORE.unique_ids[humidity_key] + assert ("", "sensor", fnv1_hash("humidity")) in CORE.unique_ids + metadata2 = CORE.unique_ids[("", "sensor", fnv1_hash("humidity"))] assert metadata2["name"] == "Humidity" # Duplicate entity should fail @@ -360,6 +537,34 @@ def test_entity_duplicate_validator() -> None: validator(config3) +def test_entity_duplicate_validator_hash_collision() -> None: + """Test that two different object_ids with the same FNV-1 hash are rejected.""" + # Brute-forced FNV-1 32-bit collision pair; both object_ids hash to 0xe95747e4 + name_a = "Sensor aooxzi" + name_b = "Sensor baraia" + object_id_a = sanitize(snake_case(name_a)) + object_id_b = sanitize(snake_case(name_b)) + assert object_id_a != object_id_b + assert fnv1_hash(object_id_a) == fnv1_hash(object_id_b) + + validator = entity_duplicate_validator("sensor") + + config1 = {CONF_NAME: name_a} + validated1 = validator(config1) + assert validated1 == config1 + + config2 = {CONF_NAME: name_b} + with pytest.raises( + Invalid, + match=re.compile( + r"Duplicate sensor entity with name 'Sensor baraia' found.*" + r"produce the same entity key hash \(0xe95747e4\)", + re.DOTALL, + ), + ): + validator(config2) + + def test_entity_duplicate_validator_with_devices() -> None: """Test entity_duplicate_validator with devices.""" # Create validator for sensor platform @@ -370,19 +575,18 @@ def test_entity_duplicate_validator_with_devices() -> None: device2 = ID("device2", type="Device") # Same name on different devices should pass - name_hash = fnv1_hash_name("Temperature") config1 = {CONF_NAME: "Temperature", CONF_DEVICE_ID: device1} validated1 = validator(config1) assert validated1 == config1 - assert ("device1", "sensor", name_hash) in CORE.unique_ids - metadata1 = CORE.unique_ids[("device1", "sensor", name_hash)] + assert ("device1", "sensor", fnv1_hash("temperature")) in CORE.unique_ids + metadata1 = CORE.unique_ids[("device1", "sensor", fnv1_hash("temperature"))] assert metadata1["device_id"] == "device1" config2 = {CONF_NAME: "Temperature", CONF_DEVICE_ID: device2} validated2 = validator(config2) assert validated2 == config2 - assert ("device2", "sensor", name_hash) in CORE.unique_ids - metadata2 = CORE.unique_ids[("device2", "sensor", name_hash)] + assert ("device2", "sensor", fnv1_hash("temperature")) in CORE.unique_ids + metadata2 = CORE.unique_ids[("device2", "sensor", fnv1_hash("temperature"))] assert metadata2["device_id"] == "device2" # Duplicate on same device should fail @@ -434,33 +638,6 @@ def test_entity_different_platforms_yaml_validation( assert result is not None -def test_object_id_conflict_mqtt_yaml_validation( - yaml_file: Callable[[str], str], capsys: pytest.CaptureFixture[str] -) -> None: - """Test that names sanitizing to the same object_id fail when mqtt is configured.""" - result = load_config_from_fixture( - yaml_file, "object_id_conflict_mqtt.yaml", FIXTURES_DIR - ) - assert result is None - - captured = capsys.readouterr() - assert ( - "mqtt builds default topics and discovery topics from the entity object_id" - in captured.out - ) - - -def test_object_id_conflict_without_mqtt_yaml_validation( - yaml_file: Callable[[str], str], -) -> None: - """Test that names sanitizing to the same object_id pass without mqtt/prometheus.""" - result = load_config_from_fixture( - yaml_file, "object_id_conflict_no_mqtt.yaml", FIXTURES_DIR - ) - # This should succeed - assert result is not None - - def test_entity_duplicate_validator_error_message() -> None: """Test that duplicate entity error messages include helpful metadata.""" # Create validator for sensor platform @@ -519,8 +696,7 @@ def test_entity_duplicate_validator_internal_entities() -> None: validated1 = validator(config1) assert validated1 == config1 # New format includes device_id (empty string for main device) - temperature_key = ("", "sensor", fnv1_hash_name("Temperature")) - assert temperature_key in CORE.unique_ids + assert ("", "sensor", fnv1_hash("temperature")) in CORE.unique_ids # Internal entity with same name should pass (not added to unique_ids) config2 = {CONF_NAME: "Temperature", CONF_INTERNAL: True} @@ -528,7 +704,9 @@ def test_entity_duplicate_validator_internal_entities() -> None: assert validated2 == config2 # Internal entity should not be added to unique_ids # Count how many times the key appears (should still be 1) - count = sum(1 for k in CORE.unique_ids if k == temperature_key) + count = sum( + 1 for k in CORE.unique_ids if k == ("", "sensor", fnv1_hash("temperature")) + ) assert count == 1 # Another internal entity with same name should also pass @@ -536,7 +714,9 @@ def test_entity_duplicate_validator_internal_entities() -> None: validated3 = validator(config3) assert validated3 == config3 # Still only one entry in unique_ids (from the non-internal entity) - count = sum(1 for k in CORE.unique_ids if k == temperature_key) + count = sum( + 1 for k in CORE.unique_ids if k == ("", "sensor", fnv1_hash("temperature")) + ) assert count == 1 # Non-internal entity with same name should fail @@ -564,148 +744,30 @@ def test_empty_or_null_device_id_on_entity() -> None: def test_entity_duplicate_validator_non_ascii_names() -> None: - """Test that distinct non-ASCII names no longer collide. - - These names used to be rejected because both sanitize to only underscores; - the entity key now hashes the raw name so they stay distinct. - """ + """Test that non-ASCII names show helpful error messages.""" # Create validator for binary_sensor platform validator = entity_duplicate_validator("binary_sensor") - # Both Russian sensors should pass even though they sanitize identically + # First Russian sensor should pass config1 = {CONF_NAME: "Датчик открытия основного крана"} validated1 = validator(config1) assert validated1 == config1 + # Second Russian sensor with different text but same ASCII conversion should fail config2 = {CONF_NAME: "Датчик закрытия основного крана"} - validated2 = validator(config2) - assert validated2 == config2 - - # An exact duplicate still fails - config3 = {CONF_NAME: "Датчик открытия основного крана"} - with pytest.raises( - Invalid, - match=r"Duplicate binary_sensor entity with name 'Датчик открытия основного крана' found", - ): - validator(config3) - - -def test_entity_duplicate_validator_hash_collision() -> None: - """Test that two different names with the same FNV-1 hash are rejected.""" - # Brute-forced FNV-1 32-bit collision pair; both hash to 0x0ee5ff7b - name_a = "Sensor m2CZ" - name_b = "Sensor qCaa" - assert name_a != name_b - assert fnv1_hash_name(name_a) == fnv1_hash_name(name_b) - - validator = entity_duplicate_validator("sensor") - - config1 = {CONF_NAME: name_a} - validated1 = validator(config1) - assert validated1 == config1 - - config2 = {CONF_NAME: name_b} with pytest.raises( Invalid, match=re.compile( - rf"Duplicate sensor entity with name '{name_b}' found.*" - rf"The names '{name_b}' and '{name_a}' produce the.*" - r"same entity key hash \(0x0ee5ff7b\).*" - r"To fix: Rename one of the entities", + r"Duplicate binary_sensor entity with name 'Датчик закрытия основного крана' found.*" + r"Original names: 'Датчик закрытия основного крана' and 'Датчик открытия основного крана'.*" + r"Both convert to ASCII ID: '_______________________________'.*" + r"To fix: Add unique ASCII characters \(e\.g\., '1', '2', or 'A', 'B'\)", re.DOTALL, ), ): validator(config2) -def test_object_id_conflicts_rejected_by_component_validator() -> None: - """Test that object_id conflicts pass entity validation but fail for mqtt/prometheus.""" - validator = entity_duplicate_validator("sensor") - - # Both names validate fine in general (distinct raw names, distinct keys) - validator({CONF_NAME: "Датчик открытия"}) - validator({CONF_NAME: "Датчик закрытия"}) - - # A component that addresses entities by object_id must reject the config - component_validator = validate_no_object_id_conflicts( - "mqtt builds default topics from the entity object_id" - ) - with pytest.raises( - Invalid, - match=re.compile( - r"mqtt builds default topics from the entity object_id.*" - r"sensor entities 'Датчик открытия', 'Датчик закрытия' " - r"share the object_id '_______________'.*" - r"To fix: Add unique ASCII characters", - re.DOTALL, - ), - ): - component_validator({}) - - -def test_object_id_conflicts_skipped_in_testing_mode() -> None: - """Test that testing_mode skips the conflict check, as used for grouped testing.""" - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Датчик открытия"}) - validator({CONF_NAME: "Датчик закрытия"}) - - component_validator = validate_no_object_id_conflicts( - "mqtt builds default topics from the entity object_id" - ) - CORE.testing_mode = True - try: - config: dict = {} - assert component_validator(config) is config - finally: - CORE.testing_mode = False - - -def test_object_id_conflicts_none_recorded() -> None: - """Test that distinct object_ids produce no conflicts.""" - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Temperature"}) - validator({CONF_NAME: "Humidity"}) - - component_validator = validate_no_object_id_conflicts( - "mqtt builds default topics from the entity object_id" - ) - config: dict = {} - assert component_validator(config) is config - - -def test_object_id_conflicts_device_scoped() -> None: - """Test that the object_id conflict check is scoped per device. - - Same-named entities on different sub-devices were accepted before entity keys - moved to raw names, so the check keeps that scope; conflicts within one device - are still reported with the device named in the message. - """ - validator = entity_duplicate_validator("sensor") - validator({CONF_NAME: "Temperature", CONF_DEVICE_ID: ID("device1", type="Device")}) - validator({CONF_NAME: "Temperature", CONF_DEVICE_ID: ID("device2", type="Device")}) - - component_validator = validate_no_object_id_conflicts( - "prometheus builds metric labels from the entity object_id" - ) - config: dict = {} - assert component_validator(config) is config - - # Two names sanitizing identically on the same sub-device still conflict - validator( - {CONF_NAME: "Датчик открытия", CONF_DEVICE_ID: ID("device1", type="Device")} - ) - validator( - {CONF_NAME: "Датчик закрытия", CONF_DEVICE_ID: ID("device1", type="Device")} - ) - with pytest.raises( - Invalid, - match=re.compile( - r"prometheus builds metric labels.*on device 'device1'", re.DOTALL - ), - ): - component_validator({}) - - def test_entity_duplicate_validator_same_name_no_enhanced_message() -> None: """Test that identical names don't show the enhanced message.""" # Create validator for sensor platform @@ -763,7 +825,7 @@ async def test_setup_entity_empty_name_with_device( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 @pytest.mark.asyncio @@ -792,7 +854,7 @@ async def test_setup_entity_empty_name_with_mac_suffix( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 @pytest.mark.asyncio @@ -822,7 +884,7 @@ async def test_setup_entity_empty_name_with_mac_suffix_no_friendly_name( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 @pytest.mark.asyncio @@ -853,7 +915,7 @@ async def test_setup_entity_empty_name_no_mac_suffix_no_friendly_name( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_key") == 0 + assert config.get("_entity_object_id_hash") == 0 def test_register_string_overflow() -> None: diff --git a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml deleted file mode 100644 index 4a6f56f473..0000000000 --- a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml +++ /dev/null @@ -1,22 +0,0 @@ -esphome: - name: test-object-id-conflict - -esp32: - board: esp32dev - -wifi: - ssid: MySSID - password: password1 - -mqtt: - broker: test.mosquitto.org - -sensor: - # Distinct raw names are fine in general, but both sanitize to the same - # object_id, which MQTT still uses to build default topics - should fail - - platform: template - name: "Датчик открытия" - lambda: return 21.0; - - platform: template - name: "Датчик закрытия" - lambda: return 22.0; diff --git a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml deleted file mode 100644 index c0fbd5cbba..0000000000 --- a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml +++ /dev/null @@ -1,15 +0,0 @@ -esphome: - name: test-object-id-ok - -esp32: - board: esp32dev - -sensor: - # Distinct raw names that sanitize to the same object_id are allowed when no - # component addresses entities by object_id (no mqtt or prometheus configured) - - platform: template - name: "Датчик открытия" - lambda: return 21.0; - - platform: template - name: "Датчик закрытия" - lambda: return 22.0; diff --git a/tests/unit_tests/test_preference_hash_stability.py b/tests/unit_tests/test_preference_hash_stability.py index d3e5fac36a..d8506afae7 100644 --- a/tests/unit_tests/test_preference_hash_stability.py +++ b/tests/unit_tests/test_preference_hash_stability.py @@ -5,11 +5,11 @@ users to lose stored preferences (calibration values, restore states, etc.) on firmware upgrades, or break entity state routing to API clients. Two algorithms are locked here (see https://github.com/esphome/backlog/issues/85): -1. `fnv1_hash_object_id(name)` - the LEGACY hash (snake_case + sanitize, then FNV-1). - Existing devices have preferences stored under keys derived from it; slot-based - backends (ESP8266, RP2040) keep using it, and key-lookup backends migrate FROM it. -2. `fnv1_hash_name(name)` - the entity key (FNV-1 over the raw UTF-8 name bytes). - Sent to API clients and used as the preference key base on key-lookup backends. +1. `fnv1_hash_object_id(name)` - the object_id hash (snake_case + sanitize, then FNV-1). + The entity key sent to API clients and the base of every stored preference key. +2. `fnv1_hash_name(name)` - FNV-1 over the raw UTF-8 name bytes. 2026.8 beta + firmware stored preferences under keys derived from it; a future key migration + must reconstruct those keys to recover that data. DO NOT CHANGE THE EXPECTED VALUES - if tests fail after modifying a hash algorithm, the change breaks backward compatibility and will cause data loss. @@ -124,8 +124,9 @@ def test_entity_object_id_hash_stability( """Verify fnv1_hash_object_id produces stable hashes for entity names. CRITICAL: These expected values MUST NOT CHANGE. Existing devices have - preferences stored under keys derived from this legacy hash; changing it - breaks the old-to-new key migration and loses stored preferences. + preferences stored under keys derived from this hash, and it is the entity + key sent to API clients; changing it loses stored preferences and breaks + entity state routing. """ actual = fnv1_hash_object_id(entity_name) assert actual == expected_object_id_hash, ( @@ -144,9 +145,8 @@ def compute_legacy_preference_key( ) -> int: """Compute the legacy preference key: (object_id_hash ^ device_id) ^ version. - This is the key existing devices have data stored under. Slot-based backends - (ESP8266, RP2040) still use it directly; key-lookup backends compute it as the - migration source in EntityBase::make_entity_preference_() (entity_base.cpp). + This is the key EntityBase::make_entity_preference_() (entity_base.cpp) + stores every entity preference under. """ object_id_hash = fnv1_hash_object_id(entity_name) preference_hash = object_id_hash ^ device_id @@ -179,8 +179,8 @@ def test_legacy_preference_key_computation( ) -> None: """Verify legacy preference key computation matches expected values. - This test ensures the formula doesn't change, which would break both slot-based - preference storage and the migration source keys on key-lookup backends. + This test ensures the formula doesn't change, which would lose stored + preferences on every platform. """ actual_key = compute_legacy_preference_key(entity_name, version, device_id) @@ -215,12 +215,12 @@ def test_legacy_preference_key_computation( ], ) def test_entity_key_hash_stability(entity_name: str, expected_key: int) -> None: - """Verify fnv1_hash_name produces stable entity keys. + """Verify fnv1_hash_name produces stable raw-name hashes. - CRITICAL: These expected values MUST NOT CHANGE. The entity key is sent to - API clients and is the new preference key base; changing the algorithm - would break state routing and lose stored preferences. - Must match C++ fnv1_hash_bytes() in esphome/core/helpers.h. + CRITICAL: These expected values MUST NOT CHANGE. 2026.8 beta firmware stored + preferences under keys derived from this hash; a future key migration must + reconstruct those keys, and changing the algorithm would strand that data. + Matched C++ fnv1_hash_bytes() (2026.8 beta), which the unrevert restores. """ actual = fnv1_hash_name(entity_name) assert actual == expected_key, ( From b794b7b1d19d7491df12d417bca5ad19187ac767 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 14 Aug 2026 00:22:22 -0500 Subject: [PATCH 040/160] [core] Restore cv.parse_esphome_version as a deprecated helper (#18366) --- esphome/config_validation.py | 4 ++++ esphome/util.py | 14 ++++++++++++++ tests/unit_tests/test_config_validation.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 0eebf12e66..f455c7b8bf 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -99,6 +99,10 @@ from esphome.schema_extractors import ( schema_extractor_registry, schema_extractor_typed, ) + +# Deprecated re-export for external components; remove before 2027.2.0 +# pylint: disable-next=unused-import +from esphome.util import parse_esphome_version # noqa: F401 from esphome.voluptuous_schema import _Schema from esphome.yaml_util import SensitiveStr, make_data_base diff --git a/esphome/util.py b/esphome/util.py index 2fc34f3a69..b8ffa048ca 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -390,6 +390,20 @@ def is_dev_esphome_version(): return "dev" in const.__version__ +# Remove before 2027.2.0 +def parse_esphome_version() -> tuple[int, int, int]: + """Deprecated: use esphome.config_validation.require_esphome_version instead.""" + from esphome.core import Version + + _LOGGER.warning( + "parse_esphome_version() is deprecated. Use " + "cv.require_esphome_version to gate on a minimum version. " + "Removed in 2027.2.0" + ) + version = Version.parse(const.__version__) + return version.major, version.minor, version.patch + + # Custom OrderedDict with nicer repr method for debugging class OrderedDict(collections.OrderedDict): def __repr__(self): diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 7627ef9273..971c4e462d 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -2967,6 +2967,23 @@ def test_require_esphome_version_older_prerelease_fails() -> None: cv.require_esphome_version(2026, 8, 0)("test") +def test_parse_esphome_version_deprecated_shim( + caplog: pytest.LogCaptureFixture, +) -> None: + """The removed helper still works for external components and warns.""" + from esphome import const, util + + with ( + patch.object(const, "__version__", "2026.9.0-dev"), + caplog.at_level(logging.WARNING), + ): + assert cv.parse_esphome_version() == (2026, 9, 0) + assert cv.parse_esphome_version() < (9999, 0, 0) + assert "parse_esphome_version() is deprecated" in caplog.text + # Both historical import paths resolve to the same function + assert cv.parse_esphome_version is util.parse_esphome_version + + # --------------------------------------------------------------------------- # suppress_invalid / validate_source_shorthand / rename_key # --------------------------------------------------------------------------- From c8de63276479cc80db40c03079b90b7c97a11f4f Mon Sep 17 00:00:00 2001 From: Karl Beecken Date: Fri, 14 Aug 2026 07:23:34 +0200 Subject: [PATCH 041/160] [core] fix PYTHONPATH leak (#18360) --- esphome/espidf/toolchain.py | 2 ++ esphome/framework_helpers.py | 2 ++ tests/unit_tests/test_espidf_toolchain.py | 15 +++++++++++++++ tests/unit_tests/test_framework_helpers.py | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index e1688f4170..bb6452acf2 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -109,6 +109,8 @@ def _get_idf_env(version: str | None = None) -> dict[str, str]: env_cache = _cache().env if version not in env_cache: env_cache[version] = os.environ.copy() + # Do not leak PYTHONPATH into child env + env_cache[version].pop("PYTHONPATH", None) # Use provided IDF framework if available if "IDF_PATH" not in os.environ: diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 86d5e4eaea..b8a43220ff 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -155,6 +155,8 @@ def run_command( _LOGGER.debug("%s - running ...", cmd_str) run_env = os.environ.copy() + # Do not leak PYTHONPATH + run_env.pop("PYTHONPATH", None) if env: run_env.update(env) diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 56f358a24c..26d812af8b 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -265,6 +265,21 @@ def test_get_idf_env_sets_git_ceiling_directories(setup_core: Path) -> None: assert str(CORE.config_dir) in env["GIT_CEILING_DIRECTORIES"].split(os.pathsep) +def test_get_idf_env_pops_inherited_pythonpath(setup_core: Path) -> None: + """A PYTHONPATH from the parent environment must not reach idf.py. + + It would override the IDF venv's isolation, shadowing its pinned + packages and failing idf.py's dependency check. + """ + toolchain._cache().env.clear() + with patch.dict( + os.environ, + {"IDF_PATH": str(setup_core), "PYTHONPATH": "/outside/site-packages"}, + ): + env = toolchain._get_idf_env(version="5.5.4") + assert "PYTHONPATH" not in env + + def test_get_cmake_output_without_build_dir(setup_core: Path) -> None: """A build dir that was never created raises EsphomeError. diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 7451ee9b39..2022c15bfe 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -188,6 +188,24 @@ def test_run_command_passes_env(mock_subprocess_run: Mock) -> None: assert mock_subprocess_run.call_args[1]["env"]["MY_VAR"] == "42" +def test_run_command_pops_inherited_pythonpath(mock_subprocess_run: Mock) -> None: + """A PYTHONPATH from the parent environment must not leak into subprocesses.""" + mock_subprocess_run.return_value = Mock(returncode=0, stdout="", stderr="") + with patch.dict(os.environ, {"PYTHONPATH": "/outside/site-packages"}): + run_command(["cmd"]) + assert "PYTHONPATH" not in mock_subprocess_run.call_args[1]["env"] + + +def test_run_command_env_pythonpath_preferred_over_pop( + mock_subprocess_run: Mock, +) -> None: + """A PYTHONPATH set explicitly via ``env`` is passed through.""" + mock_subprocess_run.return_value = Mock(returncode=0, stdout="", stderr="") + with patch.dict(os.environ, {"PYTHONPATH": "/outside/site-packages"}): + run_command(["cmd"], env={"PYTHONPATH": "/idf/tools"}) + assert mock_subprocess_run.call_args[1]["env"]["PYTHONPATH"] == "/idf/tools" + + def test_run_command_passes_cwd(mock_subprocess_run: Mock, tmp_path: Path) -> None: mock_subprocess_run.return_value = Mock(returncode=0, stdout="", stderr="") run_command(["cmd"], cwd=str(tmp_path)) From 4db47de556375f0554001d03c6c80fdea005db97 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:36:56 +1200 Subject: [PATCH 042/160] Bump version to 2026.8.0b3 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index a8c77f4bb8..d9421273af 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.0b2 +PROJECT_NUMBER = 2026.8.0b3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index b6770d0001..1a8be98c03 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.0b2" +__version__ = "2026.8.0b3" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From be66e8b99c3aaee6b2ed8b3ab75434ecdfdde612 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:33:47 -0700 Subject: [PATCH 043/160] [ci] Disable CodSpeed benchmarks job outside esphome/esphome (#18372) --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b603e68ad7..026c2ba27a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -445,8 +445,12 @@ jobs: - common - determine-jobs if: >- - (github.event_name == 'push' && github.ref_name == 'dev') || - (github.event_name == 'pull_request' && needs.determine-jobs.outputs.benchmarks == 'true') + github.repository == 'esphome/esphome' && ( + (github.event_name == 'push' && github.ref_name == 'dev') || + (github.event_name == 'pull_request' && needs.determine-jobs.outputs.benchmarks == 'true') + ) + # CodSpeed benchmarks require a CodSpeed account linked to the repository to run + # (https://codspeed.io) -- disabled on forks that aren't esphome/esphome itself. steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From e5224e22ae1fd07a284794690db68544f76f1b0c Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:18:22 -0700 Subject: [PATCH 044/160] [ci] Compare merge-branch base ref against the default branch (#18385) --- .github/scripts/auto-label-pr/detectors.js | 3 ++- .../auto-label-pr/tests/detectors.test.js | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/scripts/auto-label-pr/detectors.js b/.github/scripts/auto-label-pr/detectors.js index bb85ccd681..1d76c18be8 100644 --- a/.github/scripts/auto-label-pr/detectors.js +++ b/.github/scripts/auto-label-pr/detectors.js @@ -70,6 +70,7 @@ async function isStackedPr(github, context) { async function detectMergeBranch(github, context) { const labels = new Set(); const baseRef = context.payload.pull_request.base.ref; + const defaultBranch = context.payload.repository.default_branch; if (baseRef === 'release') { labels.add('merging-to-release'); @@ -78,7 +79,7 @@ async function detectMergeBranch(github, context) { } else if (await isStackedPr(github, context)) { // GitHub manages the merge order for a stack, so these are not blocked. labels.add('stacked-pr'); - } else if (baseRef !== 'dev') { + } else if (baseRef !== defaultBranch) { // A chain built by hand: it must not merge until its base branch does. labels.add('chained-pr'); } diff --git a/.github/scripts/auto-label-pr/tests/detectors.test.js b/.github/scripts/auto-label-pr/tests/detectors.test.js index f30ceff8c1..be239e2f1b 100644 --- a/.github/scripts/auto-label-pr/tests/detectors.test.js +++ b/.github/scripts/auto-label-pr/tests/detectors.test.js @@ -43,14 +43,14 @@ const WITHOUT_SCHEMA = 'CODEOWNERS = ["@esphome/core"]'; // Builds a fresh context for detectMergeBranch tests instead of mutating the // shared CONTEXT fixture above (which other describe blocks rely on). -function makeMergeContext(baseRef, { stack } = {}) { +function makeMergeContext(baseRef, { stack, defaultBranch = 'dev' } = {}) { const pull_request = { number: 1, base: { ref: baseRef } }; if (stack !== undefined) { pull_request.stack = stack; } return { repo: { owner: 'esphome', repo: 'esphome' }, - payload: { pull_request } + payload: { pull_request, repository: { default_branch: defaultBranch } } }; } @@ -136,6 +136,21 @@ describe('detectMergeBranch', () => { assert.deepEqual(Array.from(labels).sort(), ['chained-pr']); assert.equal(state.calls, 1); }); + + it('base ref matches default branch adds no labels', async () => { + const { github } = makeStackGithub({ stack: null }); + const context = makeMergeContext('other', { defaultBranch: 'other' }); + const labels = await detectMergeBranch(github, context); + assert.deepEqual(Array.from(labels).sort(), []); + }); + + it('base ref dev when the default branch is main adds chained-pr', async () => { + const { github } = makeStackGithub({ stack: null }); + const context = makeMergeContext('dev', { defaultBranch: 'main' }); + const labels = await detectMergeBranch(github, context); + assert.deepEqual(Array.from(labels).sort(), ['chained-pr']); + }); + }); // --------------------------------------------------------------------------- From b178f74e5d6b229293b28bfd2cc78ffb79f5be77 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 14 Aug 2026 18:45:23 -0700 Subject: [PATCH 045/160] [core] Save the validated config cache on the first upload or logs run (#18367) --- esphome/__main__.py | 28 +- esphome/compiled_config.py | 77 ++++- esphome/components/esp32/__init__.py | 3 + esphome/components/esp8266/__init__.py | 3 + esphome/components/libretiny/__init__.py | 3 + esphome/components/nrf52/__init__.py | 3 + esphome/components/rp2/__init__.py | 3 + esphome/storage_json.py | 56 +++- .../fixtures/lazy_imports/_storage.py | 11 +- tests/unit_tests/test_compiled_config.py | 299 ++++++++++++++++-- tests/unit_tests/test_download_types.py | 52 +++ tests/unit_tests/test_storage_json.py | 99 ++++++ 12 files changed, 573 insertions(+), 64 deletions(-) create mode 100644 tests/unit_tests/test_download_types.py diff --git a/esphome/__main__.py b/esphome/__main__.py index 1262a4525e..c1e05d2ea7 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2732,7 +2732,8 @@ def run_esphome(argv): conf_path.name, ) - if config is None: + cache_missed = config is None + if cache_missed: from esphome.config import read_config config = read_config( @@ -2741,26 +2742,25 @@ def run_esphome(argv): # Snapshot only needed by `esphome config --no-defaults`. snapshot_user_config=getattr(args, "no_defaults", False), ) - # Refresh the cache so the next upload/logs hits the fast path - # instead of re-running read_config. Skip when the storage - # sidecar is absent (no compile has run): the cache would - # never be loaded back, so writing secrets to disk is wasted. - if cache_eligible and config is not None: - from esphome.compiled_config import save_compiled_config - from esphome.storage_json import ext_storage_path - - if ext_storage_path(conf_path.name).exists(): - save_compiled_config(config) - if config is None: - return 2 + if config is None: + return 2 CORE.config = config # Fallback for platforms whose validators didn't set the toolchain # (only the esp32 component reads esp32.framework.toolchain). All - # other platforms only support PlatformIO today. + # other platforms only support PlatformIO today. Must run before the + # cache refresh below so its sidecar records the same toolchain a + # compile would. if CORE.toolchain is None: CORE.toolchain = Toolchain.PLATFORMIO + # Refresh the cache so the next upload/logs hits the fast path + # instead of re-running read_config. + if cache_eligible and cache_missed: + from esphome.compiled_config import save_compiled_config_and_sidecar + + save_compiled_config_and_sidecar(config) + if args.command not in POST_CONFIG_ACTIONS: safe_print(f"Unknown command {args.command}") return 1 diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py index 303af99e66..be03eea965 100644 --- a/esphome/compiled_config.py +++ b/esphome/compiled_config.py @@ -18,9 +18,9 @@ from pathlib import Path from typing import Any from esphome.const import __version__ as ESPHOME_VERSION -from esphome.core import CORE, Lambda +from esphome.core import CORE, EsphomeError, Lambda from esphome.helpers import write_file -from esphome.storage_json import StorageJSON, ext_storage_path +from esphome.storage_json import StorageJSON, ext_storage_path, storage_path from esphome.types import ConfigType _LOGGER = logging.getLogger(__name__) @@ -65,7 +65,71 @@ def save_compiled_config(config: ConfigType) -> None: # non-basic dict key), so every upload/logs pays the slow path. _LOGGER.warning("Cannot cache the validated config: %s", err) except Exception as err: # noqa: BLE001 # pylint: disable=broad-except - _LOGGER.debug("Skipping compiled config cache write: %s", err) + # Likely persistent (permissions, full disk): every upload/logs + # pays the slow path until it clears, so surface it. + _LOGGER.warning("Skipping compiled config cache write: %s", err) + + +def save_compiled_config_and_sidecar(config: ConfigType) -> None: + """Refresh the cache from the upload/logs fallback (CORE.config must be set). + + The cache is only written when a complete sidecar is on disk: + load_compiled_config can't use it otherwise, and it holds resolved + secrets. + """ + if _refresh_sidecar(): + save_compiled_config(config) + + +def _refresh_sidecar() -> bool: + """Ensure a complete sidecar is on disk; True when one is. + + Writes one (without claiming a build) when missing or wizard-only. + Failures are non-fatal; the next upload/logs pays the slow path again. + """ + try: + path = storage_path() + try: + old = StorageJSON.load_strict(path) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + # Present but unreadable: it may hold a real build's metadata, + # and a fresh rewrite would also stop the next compile from + # cleaning a possibly incoherent build tree. + _LOGGER.warning( + "Not caching: storage sidecar %s is unreadable (%s)", path, err + ) + return False + if old is not None and old.can_apply_to_core(): + # Compile-written; nothing to refresh. + return True + if CORE.build_path is not None and CORE.build_path.exists(): + # An unvalidated build tree: its absent or mismatched sidecar + # is what makes the next compile wipe it, so don't vouch for + # a build this run never saw. + _LOGGER.warning( + "Not caching: build tree %s has no matching sidecar; " + "'esphome compile' will settle it", + CORE.build_path, + ) + return False + new = StorageJSON.from_esphome_core(CORE, old, claim_build=False) + if not new.can_apply_to_core(): + _LOGGER.warning("Not caching: rebuilt storage sidecar is still incomplete") + return False + new.save(path) + return True + except (OSError, EsphomeError) as err: + # write_file wraps OSError into EsphomeError. Persistent + # (unwritable storage dir), so surface that every upload/logs + # pays the slow path. + _LOGGER.warning("Could not refresh the storage sidecar: %s", err) + except Exception: # noqa: BLE001 # pylint: disable=broad-except + # A structural bug; keep the traceback so it isn't mistaken + # for the I/O failure above. + _LOGGER.warning( + "Unexpected error refreshing the storage sidecar", exc_info=True + ) + return False def load_compiled_config(conf_path: Path) -> ConfigType | None: @@ -98,11 +162,8 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None: return None storage = StorageJSON.load(ext_storage_path(conf_path.name)) - if storage is None: - return None - # apply_to_core assumes a real compile wrote the sidecar; wizard-only - # sidecars leave both of these unset and can't drive upload/logs. - if not storage.core_platform and not storage.target_platform: + if storage is None or not storage.can_apply_to_core(): + _LOGGER.debug("Ignoring compiled config cache: sidecar missing or incomplete") return None storage.apply_to_core() return config diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index ada6d25db5..7263571d69 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -570,6 +570,9 @@ def get_download_types(storage_json): the shape stable so the download panel doesn't have to special-case per-platform schemas. """ + # No recorded firmware path means nothing was built; no downloads. + if storage_json.firmware_bin_path is None: + return [] return [ { "title": "Factory format (Previously Modern)", diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 1f7159919d..2161a902cb 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -113,6 +113,9 @@ def get_download_types(storage_json): the shape stable so the download panel doesn't have to special-case per-platform schemas. """ + # No recorded firmware path means nothing was built; no downloads. + if storage_json.firmware_bin_path is None: + return [] return [ { "title": "Standard format", diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index c51af373b3..c56cc48055 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -182,6 +182,9 @@ def get_download_types(storage_json: StorageJSON = None): the shape stable so the download panel doesn't have to special-case per-platform schemas. """ + # No recorded firmware path means nothing was built; no downloads. + if storage_json.firmware_bin_path is None: + return [] types = [ { "title": "UF2 package (recommended)", diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 386fed5412..2d25558254 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -473,6 +473,9 @@ def copy_files() -> None: def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]: """Get the download types for the firmware.""" + # No recorded firmware path means nothing was built; no downloads. + if storage_json.firmware_bin_path is None: + return [] types = [] UF2_PATH = "zephyr/zephyr.uf2" DFU_PATH = "firmware.zip" diff --git a/esphome/components/rp2/__init__.py b/esphome/components/rp2/__init__.py index 87e78003ed..60fcd4f8b0 100644 --- a/esphome/components/rp2/__init__.py +++ b/esphome/components/rp2/__init__.py @@ -156,6 +156,9 @@ def get_download_types(storage_json): the shape stable so the download panel doesn't have to special-case per-platform schemas. """ + # No recorded firmware path means nothing was built; no downloads. + if storage_json.firmware_bin_path is None: + return [] return [ { "title": "UF2 factory format", diff --git a/esphome/storage_json.py b/esphome/storage_json.py index a90a36b848..9219914529 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -71,8 +71,11 @@ def archive_storage_path() -> Path: def _to_path_if_not_none(value: str | None) -> Path | None: - """Convert a string to Path if it's not None.""" - return Path(value) if value is not None else None + """Convert a string to Path; None and the legacy "None" both map to None. + + Sidecars written before as_dict skipped unset paths hold str(None). + """ + return Path(value) if value is not None and value != "None" else None def _parse_framework_version(framework_version: str) -> Version: @@ -170,8 +173,10 @@ class StorageJSON: "address": self.address, "web_port": self.web_port, "esp_platform": self.target_platform, - "build_path": str(self.build_path), - "firmware_bin_path": str(self.firmware_bin_path), + "build_path": str(self.build_path) if self.build_path else None, + "firmware_bin_path": ( + str(self.firmware_bin_path) if self.firmware_bin_path else None + ), "loaded_integrations": sorted(self.loaded_integrations), "loaded_platforms": sorted(self.loaded_platforms), "no_mdns": self.no_mdns, @@ -189,7 +194,18 @@ class StorageJSON: write_file_if_changed(path, self.to_json()) @staticmethod - def from_esphome_core(esph: CoreType, old: StorageJSON | None) -> StorageJSON: + def from_esphome_core( + esph: CoreType, old: StorageJSON | None, *, claim_build: bool = True + ) -> StorageJSON: + """Build a sidecar from post-validation CORE state. + + claim_build=False (the upload/logs fallback, which runs no build) + carries the build-artifact fields (esphome_version, + firmware_bin_path) from *old* instead of asserting this run built + firmware. Validation-derived fields (platform, framework_version, + toolchain, build_path) always stamp; storage_should_clean compares + them against the next compile. + """ hardware = esph.target_platform.upper() framework_version: str | None = None if esph.is_esp32: @@ -204,13 +220,21 @@ class StorageJSON: name=esph.name, friendly_name=esph.friendly_name, comment=esph.comment, - esphome_version=const.__version__, + esphome_version=( + const.__version__ + if claim_build + else (old.esphome_version if old else None) + ), src_version=1, address=esph.address, web_port=esph.web_port, target_platform=hardware, build_path=esph.build_path, - firmware_bin_path=esph.firmware_bin, + firmware_bin_path=( + esph.firmware_bin + if claim_build + else (old.firmware_bin_path if old else None) + ), loaded_integrations=esph.loaded_integrations, loaded_platforms=esph.loaded_platforms, no_mdns=( @@ -302,11 +326,27 @@ class StorageJSON: except Exception: # noqa: BLE001 # pylint: disable=broad-except return None + @staticmethod + def load_strict(path: Path) -> StorageJSON | None: + """Like load, but None only means missing; an unreadable file raises.""" + if not path.is_file(): + return None + return StorageJSON._load_impl(path) + + def can_apply_to_core(self) -> bool: + """True when the sidecar carries everything apply_to_core hands CORE. + + Wizard-written sidecars leave build_path unset (older wizards also + the platform fields) and can't drive upload/logs. + """ + return bool((self.core_platform or self.target_platform) and self.build_path) + def apply_to_core(self) -> None: """Populate CORE with the metadata upload/logs read. Inverse of :meth:`from_esphome_core`. Keep paired -- a new - attribute upload/logs needs has to be captured there too. + attribute upload/logs needs has to be captured there too and + reflected in :meth:`can_apply_to_core`. Validator-only fields (loaded_integrations/platforms, friendly_name) are skipped; the fast path doesn't run validation and CORE.__init__ defaults them. diff --git a/tests/unit_tests/fixtures/lazy_imports/_storage.py b/tests/unit_tests/fixtures/lazy_imports/_storage.py index 969528304b..94acd2e93a 100644 --- a/tests/unit_tests/fixtures/lazy_imports/_storage.py +++ b/tests/unit_tests/fixtures/lazy_imports/_storage.py @@ -1,10 +1,15 @@ """Shared storage-sidecar factory for the lazy-import fixture scripts.""" +from pathlib import Path + from esphome.storage_json import StorageJSON def make_storage() -> StorageJSON: - """A minimal post-compile esp32 sidecar the upload/logs fast path accepts.""" + """A minimal post-compile esp32 sidecar the upload/logs fast path accepts. + + build_path must be set: the fast path rejects sidecars without one. + """ return StorageJSON( storage_version=1, name="test", @@ -15,8 +20,8 @@ def make_storage() -> StorageJSON: address="1.2.3.4", web_port=None, target_platform="ESP32S3", - build_path=None, - firmware_bin_path=None, + build_path=Path("/build/test"), + firmware_bin_path=Path("/build/test/firmware.bin"), loaded_integrations=set(), loaded_platforms=set(), no_mdns=False, diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index b3c2170c3f..77690a6897 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import contextmanager from ipaddress import IPv4Address, IPv4Network import json import os @@ -19,6 +20,7 @@ from esphome.compiled_config import ( compiled_config_path, load_compiled_config, save_compiled_config, + save_compiled_config_and_sidecar, ) from esphome.const import ( CONF_API, @@ -31,7 +33,16 @@ from esphome.const import ( KEY_VARIANT, Toolchain, ) -from esphome.core import CORE, ID, HexInt, Lambda, MACAddress, TimePeriodMilliseconds +from esphome.core import ( + CORE, + ID, + EsphomeError, + HexInt, + Lambda, + MACAddress, + TimePeriodMilliseconds, +) +from esphome.storage_json import StorageJSON from esphome.util import OrderedDict _VALIDATED_CONFIG = { @@ -54,8 +65,9 @@ def _cache_body(config: dict | None = None) -> str: def _write_storage( storage_path: Path, *, - esp_platform: str = "ESP32", + esp_platform: str | None = "ESP32", core_platform: str | None = "esp32", + build_path: str | None = "/build/lite_test", ) -> None: """Write a vanilla StorageJSON sidecar for the cache tests.""" storage_path.parent.mkdir(parents=True, exist_ok=True) @@ -69,7 +81,7 @@ def _write_storage( "address": "192.168.1.42", "web_port": None, "esp_platform": esp_platform, - "build_path": "/build/lite_test", + "build_path": build_path, "firmware_bin_path": "/build/lite_test/firmware.bin", "loaded_integrations": ["api", "logger", "ota", "wifi"], "loaded_platforms": [], @@ -359,31 +371,262 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache( mock_read.assert_called_once() -def test_run_esphome_upload_does_not_refresh_cache_without_sidecar( - tmp_path: Path, -) -> None: - """Without a StorageJSON sidecar (no compile has run), the fallback - skips the cache write -- load_compiled_config requires the sidecar, - so writing the rendered (secret-resolved) config would be inert and - leak secrets to disk for nothing.""" +def _storage_fixture(tmp_path: Path) -> StorageJSON: + """A loaded StorageJSON instance matching _write_storage's contents.""" + fixture = tmp_path / "fixture_storage.json" + _write_storage(fixture) + return StorageJSON.load(fixture) + + +def _bare_yaml(tmp_path: Path) -> Path: + """A minimal YAML with CORE.config_path pointed at it.""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path + return yaml_path + +@contextmanager +def _fallback_run(command: str = "upload", **from_core_kwargs) -> Any: + """Patch the fallback path's collaborators for a run_esphome call. + + Without kwargs, from_esphome_core stays real (yielded mock is None). + """ with ( patch( "esphome.config.read_config", return_value={"esphome": {"name": "lite_test"}}, - ), - patch("esphome.compiled_config.save_compiled_config") as mock_save, + ) as mock_read, patch.dict( "esphome.__main__.POST_CONFIG_ACTIONS", - {"upload": lambda args, config: 0}, + {command: lambda args, config: 0}, ), ): - run_esphome(["esphome", "upload", str(yaml_path)]) + if not from_core_kwargs: + yield mock_read, None + return + with patch.object( + StorageJSON, "from_esphome_core", **from_core_kwargs + ) as mock_from_core: + yield mock_read, mock_from_core + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_fallback_writes_sidecar_and_cache_without_sidecar( + tmp_path: Path, command: str +) -> None: + """A never-compiled config caches on its first upload/logs run: the + fallback writes the StorageJSON sidecar itself (load_compiled_config + needs it), so the second run hits the fast path.""" + yaml_path = _bare_yaml(tmp_path) + storage_dir = tmp_path / ".esphome" / "storage" + + with _fallback_run(command, return_value=_storage_fixture(tmp_path)) as ( + mock_read, + mock_from_core, + ): + assert run_esphome(["esphome", command, str(yaml_path)]) == 0 + mock_from_core.assert_called_once() + assert (storage_dir / "lite_test.yaml.validated.json").exists() + storage = StorageJSON.load(storage_dir / "lite_test.yaml.json") + assert storage is not None + # No compile happened, so the sidecar must not claim one. + assert mock_from_core.call_args.kwargs == {"claim_build": False} + + # The second run loads the cache instead of re-validating. + assert run_esphome(["esphome", command, str(yaml_path)]) == 0 + mock_read.assert_called_once() + + +# as_dict serialized unset paths as str(None) until 2026.9; files +# written by those wizards are still on disk. +_WIZARD_SIDECAR_CASES = pytest.mark.parametrize( + "wizard_kwargs", + [ + {"esp_platform": None, "core_platform": None, "build_path": None}, + {"build_path": None}, + {"build_path": "None"}, + ], + ids=["legacy_wizard", "modern_wizard", "none_string_wizard"], +) + + +def _prime_core(tmp_path: Path) -> None: + """Set the post-validation CORE state from_esphome_core reads.""" + CORE.name = "lite_test" + CORE.build_path = tmp_path / "build" / "lite_test" + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp8266", + KEY_TARGET_FRAMEWORK: "arduino", + } + + +@_WIZARD_SIDECAR_CASES +def test_run_esphome_fallback_completes_wizard_sidecar( + tmp_path: Path, wizard_kwargs: dict[str, Any] +) -> None: + """A wizard-written sidecar can't drive the fast path (no build_path; + older wizards also no platform fields); the fallback rewrites it from + CORE so the cache loads on the next run.""" + yaml_path = _bare_yaml(tmp_path) + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json", **wizard_kwargs) + + with _fallback_run(return_value=_storage_fixture(tmp_path)) as (_, mock_from_core): + assert run_esphome(["esphome", "upload", str(yaml_path)]) == 0 + + mock_from_core.assert_called_once() + storage = StorageJSON.load(storage_dir / "lite_test.yaml.json") + assert storage is not None and storage.core_platform == "esp32" + # What the wizard recorded about a build (nothing, or a real one) + # carries through instead of being stamped with this run's values. + assert storage.esphome_version == "2026.1.0" + assert load_compiled_config(yaml_path) is not None + + +def test_run_esphome_fallback_skips_cache_when_sidecar_write_fails( + tmp_path: Path, +) -> None: + """A failed sidecar write is non-fatal and skips the cache save too: + without the sidecar the cache could never be loaded back, so writing + it would only leave resolved secrets on disk.""" + yaml_path = _bare_yaml(tmp_path) + + with ( + _fallback_run(side_effect=RuntimeError("boom")), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + ): + assert run_esphome(["esphome", "upload", str(yaml_path)]) == 0 mock_save.assert_not_called() + assert not (tmp_path / ".esphome" / "storage" / "lite_test.yaml.json").exists() + + +def test_run_esphome_fallback_write_failure_takes_io_branch( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """StorageJSON.save raises EsphomeError (write_file wraps OSError into + it), which must land in the plain I/O warning, not the traceback + branch for structural bugs.""" + yaml_path = _bare_yaml(tmp_path) + + with ( + _fallback_run(return_value=_storage_fixture(tmp_path)), + patch.object(StorageJSON, "save", side_effect=EsphomeError("boom")), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + caplog.at_level("WARNING", logger="esphome.compiled_config"), + ): + assert run_esphome(["esphome", "upload", str(yaml_path)]) == 0 + + mock_save.assert_not_called() + assert "Could not refresh the storage sidecar" in caplog.text + assert "Unexpected error" not in caplog.text + + +def test_run_esphome_fallback_leaves_unreadable_sidecar_alone(tmp_path: Path) -> None: + """A present-but-corrupt sidecar is not overwritten: it may hold a real + build's metadata, and replacing it would suppress the next compile's + clean of a possibly incoherent build tree. The cache save is skipped.""" + yaml_path = _bare_yaml(tmp_path) + storage_dir = tmp_path / ".esphome" / "storage" + sidecar = storage_dir / "lite_test.yaml.json" + sidecar.parent.mkdir(parents=True, exist_ok=True) + sidecar.write_text("{truncated", encoding="utf-8") + + with _fallback_run(return_value=None) as (_, mock_from_core): + assert run_esphome(["esphome", "upload", str(yaml_path)]) == 0 + + mock_from_core.assert_not_called() + assert sidecar.read_text(encoding="utf-8") == "{truncated" + assert not (storage_dir / "lite_test.yaml.validated.json").exists() + + +def test_run_esphome_fallback_skips_cache_when_rebuilt_sidecar_incomplete( + tmp_path: Path, +) -> None: + """If the rebuilt sidecar would still be incomplete, nothing is written: + the cache could never be loaded back, so saving it would only rewrite + resolved secrets on every run.""" + yaml_path = _bare_yaml(tmp_path) + storage_dir = tmp_path / ".esphome" / "storage" + + incomplete = tmp_path / "incomplete_storage.json" + _write_storage(incomplete, build_path=None) + + with _fallback_run(return_value=StorageJSON.load(incomplete)): + assert run_esphome(["esphome", "upload", str(yaml_path)]) == 0 + + assert not (storage_dir / "lite_test.yaml.json").exists() + assert not (storage_dir / "lite_test.yaml.validated.json").exists() + + +def test_run_esphome_fallback_sidecar_records_platformio_toolchain( + tmp_path: Path, +) -> None: + """The toolchain fallback runs before the sidecar write, so platforms + whose validators leave CORE.toolchain unset record the same + "platformio" a compile writes, not null.""" + yaml_path = _bare_yaml(tmp_path) + _prime_core(tmp_path) + assert CORE.toolchain is None + + with _fallback_run(): + assert run_esphome(["esphome", "upload", str(yaml_path)]) == 0 + + storage = StorageJSON.load( + tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" + ) + assert storage is not None + assert storage.toolchain == "platformio" + + +@pytest.mark.parametrize("existing_sidecar", [None, "wizard"]) +def test_run_esphome_fallback_skips_sidecar_when_build_tree_exists( + tmp_path: Path, existing_sidecar: str | None +) -> None: + """An existing build tree with a missing or wizard-only sidecar keeps + it that way: the mismatch is what makes the next compile wipe the + unknown tree, so the fallback writes nothing and skips the cache.""" + yaml_path = _bare_yaml(tmp_path) + _prime_core(tmp_path) + CORE.build_path.mkdir(parents=True) + storage_dir = tmp_path / ".esphome" / "storage" + if existing_sidecar == "wizard": + _write_storage(storage_dir / "lite_test.yaml.json", build_path=None) + wizard_body = (storage_dir / "lite_test.yaml.json").read_text(encoding="utf-8") + + with _fallback_run(return_value=_storage_fixture(tmp_path)) as (_, mock_from_core): + assert run_esphome(["esphome", "upload", str(yaml_path)]) == 0 + + mock_from_core.assert_not_called() + assert not (storage_dir / "lite_test.yaml.validated.json").exists() + if existing_sidecar == "wizard": + sidecar_body = (storage_dir / "lite_test.yaml.json").read_text(encoding="utf-8") + assert sidecar_body == wizard_body + else: + assert not (storage_dir / "lite_test.yaml.json").exists() + + +def test_save_compiled_config_and_sidecar_builds_real_sidecar(tmp_path: Path) -> None: + """Drive the real from_esphome_core on the fallback path: the + post-validation CORE state yields a complete, loadable sidecar.""" + yaml_path = _bare_yaml(tmp_path) + _prime_core(tmp_path) + CORE.config = {CONF_ESPHOME: {CONF_NAME: "lite_test"}} + CORE.toolchain = Toolchain.PLATFORMIO + + save_compiled_config_and_sidecar(CORE.config) + + storage = StorageJSON.load( + tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" + ) + assert storage is not None + assert storage.core_platform == "esp8266" + assert storage.build_path is not None + # No compile happened, so the sidecar must not claim one. + assert storage.esphome_version is None + assert storage.firmware_bin_path is None + assert load_compiled_config(yaml_path) is not None @pytest.mark.parametrize("command", ["upload", "logs"]) @@ -409,6 +652,7 @@ def test_run_esphome_upload_and_logs_refresh_cache_on_fallback( patch( "esphome.compiled_config.save_compiled_config", wraps=save_compiled_config ) as mock_save, + patch.object(StorageJSON, "from_esphome_core") as mock_from_core, patch.dict( "esphome.__main__.POST_CONFIG_ACTIONS", {command: lambda args, config: 0}, @@ -417,6 +661,8 @@ def test_run_esphome_upload_and_logs_refresh_cache_on_fallback( assert run_esphome(["esphome", command, str(yaml_path)]) == 0 mock_save.assert_called_once_with(fresh_config) + # The compile-written sidecar is complete; the fallback leaves it alone. + mock_from_core.assert_not_called() # mtime is now newer than the source YAML, so a follow-up call hits # the fast path instead of repeating read_config. assert cache.stat().st_mtime >= yaml_path.stat().st_mtime @@ -647,24 +893,15 @@ def test_int_keys_coerce_to_strings(primed_storage: Path) -> None: assert config["table"] == {"1": "a", "2": "b"} -def test_load_compiled_config_rejects_wizard_only_sidecar(tmp_path: Path) -> None: - """A wizard-only sidecar (no compile -- no core_platform / target_platform) - can't drive upload/logs, so the fast path falls back.""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - CORE.config_path = yaml_path - +@_WIZARD_SIDECAR_CASES +def test_load_compiled_config_rejects_wizard_only_sidecar( + tmp_path: Path, wizard_kwargs: dict[str, Any] +) -> None: + """A wizard-written sidecar (no build_path; older wizards also no + platform fields) can't drive upload/logs, so the fast path falls back.""" + yaml_path = _bare_yaml(tmp_path) storage_dir = tmp_path / ".esphome" / "storage" - storage_dir.mkdir(parents=True, exist_ok=True) - # StorageJSON with both core_platform and target_platform unset. - (storage_dir / "lite_test.yaml.json").write_text( - '{"storage_version": 1, "name": "lite_test", "friendly_name": null, ' - '"comment": null, "esphome_version": null, "src_version": 1, ' - '"address": null, "web_port": null, "esp_platform": null, ' - '"build_path": null, "firmware_bin_path": null, ' - '"loaded_integrations": [], "loaded_platforms": [], "no_mdns": false, ' - '"framework": null, "core_platform": null}' - ) + _write_storage(storage_dir / "lite_test.yaml.json", **wizard_kwargs) cache_path = _write_cache(storage_dir / "lite_test.yaml.validated.json") _set_cache_mtime(cache_path, yaml_path, offset=5) diff --git a/tests/unit_tests/test_download_types.py b/tests/unit_tests/test_download_types.py new file mode 100644 index 0000000000..2ccf53f7e3 --- /dev/null +++ b/tests/unit_tests/test_download_types.py @@ -0,0 +1,52 @@ +"""Platform get_download_types contract for never-built configs. + +Wizard-written and upload/logs-fallback sidecars record no +firmware_bin_path; the download panel must get an empty list for them, +not entries pointing at files that were never built. +""" + +from __future__ import annotations + +from importlib import import_module +from pathlib import Path +from typing import Any + +import pytest + +from esphome.storage_json import StorageJSON + +PLATFORMS = ["esp32", "esp8266", "rp2", "libretiny", "nrf52"] + + +def _download_types(platform: str, storage: StorageJSON) -> list[dict[str, Any]]: + return import_module(f"esphome.components.{platform}").get_download_types(storage) + + +def _wizard_storage() -> StorageJSON: + return StorageJSON.from_wizard( + name="test_device", + friendly_name="Test Device", + address="test_device.local", + platform="ESP32", + ) + + +@pytest.mark.parametrize("platform", PLATFORMS) +def test_no_firmware_path_yields_no_downloads(platform: str) -> None: + """No recorded firmware path means nothing was built; no downloads.""" + assert _download_types(platform, _wizard_storage()) == [] + + +@pytest.mark.parametrize("platform", PLATFORMS) +def test_recorded_firmware_path_yields_downloads(platform: str, tmp_path: Path) -> None: + """With a firmware path recorded, every platform offers entries in + the documented title/description/file/download shape.""" + storage = _wizard_storage() + storage.firmware_bin_path = tmp_path / "firmware.bin" + + types = _download_types(platform, storage) + + assert types + assert all( + {"title", "description", "file", "download"} <= entry.keys() for entry in types + ) diff --git a/tests/unit_tests/test_storage_json.py b/tests/unit_tests/test_storage_json.py index 01683507c1..857795d02f 100644 --- a/tests/unit_tests/test_storage_json.py +++ b/tests/unit_tests/test_storage_json.py @@ -915,3 +915,102 @@ def test_storage_json_load_area(tmp_path: Path) -> None: legacy = storage_json.StorageJSON.load(legacy_path) assert legacy is not None assert legacy.area is None + + +def test_from_esphome_core_without_claiming_a_build(setup_core: Path) -> None: + """claim_build=False carries the build artifact fields from the old + sidecar while validation-derived fields still stamp from CORE.""" + mock_core = MagicMock() + mock_core.name = "my_device" + mock_core.friendly_name = "My Device" + mock_core.comment = None + mock_core.address = "my_device.local" + mock_core.web_port = None + mock_core.target_platform = "esp8266" + mock_core.is_esp32 = False + mock_core.is_nrf52 = False + mock_core.build_path = "/build/my_device" + mock_core.loaded_integrations = set() + mock_core.loaded_platforms = set() + mock_core.config = {} + mock_core.target_framework = "arduino" + mock_core.toolchain = Toolchain.PLATFORMIO + mock_core.area = None + + old = storage_json.StorageJSON.from_wizard( + name="my_device", + friendly_name="My Device", + address="my_device.local", + platform="ESP8266", + ) + old.esphome_version = "2025.1.0" + old.firmware_bin_path = Path("/old/firmware.bin") + + result = storage_json.StorageJSON.from_esphome_core( + mock_core, old, claim_build=False + ) + + # Build artifact fields carry from the old sidecar, not this run. + assert result.esphome_version == "2025.1.0" + assert result.firmware_bin_path == Path("/old/firmware.bin") + # Validation-derived fields stamp from CORE. + assert result.build_path == "/build/my_device" + assert result.toolchain == "platformio" + assert result.core_platform == "esp8266" + + # With no old sidecar, no build is claimed at all. + bare = storage_json.StorageJSON.from_esphome_core( + mock_core, None, claim_build=False + ) + assert bare.esphome_version is None + assert bare.firmware_bin_path is None + + +def test_load_strict_distinguishes_missing_from_unreadable(tmp_path: Path) -> None: + """load_strict returns None only for a missing file; corrupt raises.""" + assert storage_json.StorageJSON.load_strict(tmp_path / "missing.json") is None + + corrupt = tmp_path / "corrupt.json" + corrupt.write_text("{truncated") + with pytest.raises(ValueError): + storage_json.StorageJSON.load_strict(corrupt) + + +def test_as_dict_serializes_unset_paths_as_null(setup_core: Path) -> None: + """Unset build/firmware paths serialize as JSON null, not str(None).""" + storage = storage_json.StorageJSON.from_wizard( + name="wiz", + friendly_name="Wiz", + address="wiz.local", + platform="ESP32", + ) + + result = storage.as_dict() + + assert result["build_path"] is None + assert result["firmware_bin_path"] is None + + +def test_load_treats_legacy_none_string_paths_as_unset(tmp_path: Path) -> None: + """Sidecars written before as_dict emitted null hold str(None); those + must load as unset, not as Path("None").""" + file_path = tmp_path / "legacy_none.json" + file_path.write_text( + json.dumps( + { + "storage_version": 1, + "name": "wiz", + "friendly_name": "Wiz", + "esp_platform": "ESP32", + "core_platform": "esp32", + "build_path": "None", + "firmware_bin_path": "None", + } + ) + ) + + result = storage_json.StorageJSON.load(file_path) + + assert result is not None + assert result.build_path is None + assert result.firmware_bin_path is None From 039b897e7b83267ffe2cee749138b29cf1a5b2cc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 14 Aug 2026 18:45:36 -0700 Subject: [PATCH 046/160] [ethernet] Defer clk_mode removal to 2026.11.0 (#18380) --- esphome/components/ethernet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/ethernet/__init__.py b/esphome/components/ethernet/__init__.py index 8bdd536ffb..f3c77baaae 100644 --- a/esphome/components/ethernet/__init__.py +++ b/esphome/components/ethernet/__init__.py @@ -355,7 +355,7 @@ def _validate(config): " clk:\n" " mode: %s\n" " pin: %s\n" - "Removal scheduled for 2026.9.0.", + "Removal scheduled for 2026.11.0.", config[CONF_CLK_MODE], mode, pin, From 7cceddb8a34b891681b150a8e45af49d80898228 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:48:27 +0000 Subject: [PATCH 047/160] Bump bundled esphome-device-builder to 1.10.0 (#18389) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d7ae2cd4ec..a62eb59a58 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.9.6 +RUN uv pip install --no-cache-dir esphome-device-builder==1.10.0 RUN \ platformio settings set enable_telemetry No \ From 6ed676fe32a35a82f9857fdb2319c18102d1f8cd Mon Sep 17 00:00:00 2001 From: Joppy Furr Date: Sat, 15 Aug 2026 18:14:53 +1200 Subject: [PATCH 048/160] [lvgl] Restore long_press_repeat_time functionality (#18393) --- esphome/components/lvgl/lvgl_esphome.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index b66a904437..acd5a9bdef 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -444,7 +444,7 @@ LVTouchListener::LVTouchListener(uint16_t long_press_time, uint16_t long_press_r lv_indev_set_type(this->drv_, LV_INDEV_TYPE_POINTER); lv_indev_set_disp(this->drv_, parent->get_disp()); lv_indev_set_long_press_time(this->drv_, long_press_time); - // long press repeat time TBD + lv_indev_set_long_press_repeat_time(this->drv_, long_press_repeat_time); lv_indev_set_user_data(this->drv_, this); lv_indev_set_read_cb(this->drv_, [](lv_indev_t *d, lv_indev_data_t *data) { auto *l = static_cast(lv_indev_get_user_data(d)); From 5a000cf5e43acbbdd3f9a82e84302094cd9b2e0f Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Sat, 15 Aug 2026 11:16:04 -0700 Subject: [PATCH 049/160] [rotary_encoder] account for min and max value when resetting (#18197) --- esphome/components/rotary_encoder/rotary_encoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/rotary_encoder/rotary_encoder.cpp b/esphome/components/rotary_encoder/rotary_encoder.cpp index 0831822d86..0734ca87d3 100644 --- a/esphome/components/rotary_encoder/rotary_encoder.cpp +++ b/esphome/components/rotary_encoder/rotary_encoder.cpp @@ -220,7 +220,7 @@ void RotaryEncoderSensor::loop() { } if (this->pin_i_ != nullptr && this->pin_i_->digital_read()) { - this->store_.counter = 0; + this->store_.counter = std::clamp(0, this->store_.min_value, this->store_.max_value); } int counter = this->store_.counter; if (this->store_.last_read != counter || this->publish_initial_value_) { From 1add72689222010acbd521d2437260183fb3c731 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:36:52 -0700 Subject: [PATCH 050/160] Bump bundled esphome-device-builder to 1.11.0 (#18403) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a62eb59a58..2f23b2f690 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.10.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.0 RUN \ platformio settings set enable_telemetry No \ From de3e657d8bcae1ec1c9298ff869390d77d2e25d2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 08:42:47 -0700 Subject: [PATCH 051/160] [platformio] Skip ccache when the binary on PATH fails to run (#18407) --- esphome/platformio/toolchain.py | 32 ++++++++++++- tests/unit_tests/test_platformio_toolchain.py | 48 ++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 0e7ffce939..08a4fcff78 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -5,6 +5,7 @@ import os from pathlib import Path import re import shutil +import subprocess import sys from typing import TYPE_CHECKING, Any @@ -234,6 +235,35 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) +def _ccache_usable() -> bool: + """Return True when the ``ccache`` on PATH actually runs. + + ``shutil.which`` proves existence, not runnability: on Windows it also + matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose + target is gone. Wrapping compiles around such a find fails every compile + step with an opaque OS error, so probe once and fall back to compiling + without ccache when the probe fails. + """ + ccache = shutil.which("ccache") + if ccache is None: + return False + try: + subprocess.run( + [ccache, "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=15, + ) + except (OSError, subprocess.SubprocessError): + _LOGGER.warning( + "Ignoring ccache at %s because it failed to run; compiling without ccache", + ccache, + ) + return False + return True + + def _ccache_env() -> dict[str, str]: """Return ccache settings for PlatformIO builds. @@ -266,7 +296,7 @@ def _ccache_env() -> dict[str, str]: if "ESPHOME_CCACHE_ENABLE" in os.environ: enabled = get_bool_env("ESPHOME_CCACHE_ENABLE") else: - enabled = shutil.which("ccache") is not None + enabled = _ccache_usable() env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"} if not enabled: return env diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 02c11b4e45..eebb0b8cd7 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -9,6 +9,7 @@ import json import os from pathlib import Path import shutil +import subprocess import sys import threading from types import SimpleNamespace @@ -431,6 +432,7 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: with ( patch.dict(os.environ, {}, clear=True), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), ): env = toolchain._ccache_env() @@ -457,6 +459,44 @@ def test_ccache_env_disabled_without_binary(setup_core: Path) -> None: assert env == {"ESPHOME_CCACHE_ENABLE": "0"} +@pytest.mark.parametrize( + "probe_error", + [ + pytest.param(OSError("not runnable"), id="oserror"), + pytest.param(subprocess.CalledProcessError(1, "ccache"), id="nonzero-exit"), + pytest.param(subprocess.TimeoutExpired("ccache", 15), id="timeout"), + ], +) +def test_ccache_env_disabled_when_probe_fails( + setup_core: Path, probe_error: Exception +) -> None: + """A ccache that resolves on PATH but fails to run stays disabled.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run", side_effect=probe_error), + ): + env = toolchain._ccache_env() + + assert env == {"ESPHOME_CCACHE_ENABLE": "0"} + + +def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None: + """An explicit ESPHOME_CCACHE_ENABLE=1 does not probe the binary.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), + patch.object(toolchain.subprocess, "run") as mock_probe, + ): + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + mock_probe.assert_not_called() + + def test_ccache_env_opt_out(setup_core: Path) -> None: """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present.""" CORE.build_path = setup_core / "build" / "test" @@ -496,6 +536,7 @@ def test_ccache_env_respects_user_values_and_refreshes_basedir( with ( patch.dict(os.environ, user_env, clear=True), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), ): env = toolchain._ccache_env() @@ -514,6 +555,7 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only( with ( patch.dict(os.environ, {}, clear=False), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), ): os.environ.pop("ESPHOME_CCACHE_ENABLE", None) mock_run_external_process.return_value = 0 @@ -533,6 +575,7 @@ def test_ccache_env_requires_build_path(setup_core: Path) -> None: with ( patch.dict(os.environ, {}, clear=True), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), pytest.raises(ValueError, match="CORE.build_path must be set"), ): toolchain._ccache_env() @@ -544,7 +587,10 @@ def test_run_platformio_cli_merges_caller_env( """A caller-supplied env is the base and gains the ccache settings.""" CORE.build_path = str(setup_core / "build" / "test") - with patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"): + with ( + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), + ): mock_run_external_process.return_value = 0 toolchain.run_platformio_cli( "test", env={"CUSTOM_VAR": "1", "ESPHOME_CCACHE_ENABLE": "0"} From 646501b0eff760267fd12de74c5fb5d283779eaa Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:36:51 -0400 Subject: [PATCH 052/160] [sensor] Pass NaN through the delta filter again (#18400) --- esphome/components/sensor/filter.cpp | 10 +++-- .../fixtures/sensor_filters_delta.yaml | 36 ++++++++++++++++++ .../integration/test_sensor_filters_delta.py | 38 +++++++++++++++++-- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 5f7f19769a..0105580d26 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -283,8 +283,11 @@ DeltaFilter::DeltaFilter(float min_a0, float min_a1, float max_a0, float max_a1) void DeltaFilter::set_baseline(float (*fn)(float)) { this->baseline_ = fn; } optional DeltaFilter::new_value(float value) { - // Always yield the first value. - if (std::isnan(this->last_value_)) { + const bool no_value = std::isnan(value); + const bool no_reference = std::isnan(this->last_value_); + if (no_value && no_reference) + return {}; + if (no_value || no_reference) { this->last_value_ = value; return value; } @@ -293,8 +296,7 @@ optional DeltaFilter::new_value(float value) { float min = fabsf(this->min_a0_ + ref * this->min_a1_); float max = fabsf(this->max_a0_ + ref * this->max_a1_); float delta = fabsf(value - ref); - // if there is no reference, e.g. for the first value, just accept this one, - // otherwise accept only if within range. + // accept only if within range if (delta > min && delta <= max) { this->last_value_ = value; return value; diff --git a/tests/integration/fixtures/sensor_filters_delta.yaml b/tests/integration/fixtures/sensor_filters_delta.yaml index 2494a430da..b01c8e452b 100644 --- a/tests/integration/fixtures/sensor_filters_delta.yaml +++ b/tests/integration/fixtures/sensor_filters_delta.yaml @@ -33,6 +33,11 @@ sensor: id: source_sensor_5 accuracy_decimals: 1 + - platform: template + name: "Source Sensor 6" + id: source_sensor_6 + accuracy_decimals: 1 + - platform: copy source_id: source_sensor_1 name: "Filter Min" @@ -81,6 +86,13 @@ sensor: filters: - delta: 50% + - platform: copy + source_id: source_sensor_6 + name: "Filter NaN" + id: filter_nan + filters: + - delta: 0 + script: - id: test_filter_min then: @@ -188,6 +200,24 @@ script: id: source_sensor_5 state: 250.0 # Passes (delta=90 > 80) + - id: test_filter_nan + then: + - sensor.template.publish: + id: source_sensor_6 + state: 1.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor_6 + state: !lambda "return NAN;" + - delay: 20ms + - sensor.template.publish: + id: source_sensor_6 + state: !lambda "return NAN;" # Filtered out + - delay: 20ms + - sensor.template.publish: + id: source_sensor_6 + state: 2.0 + button: - platform: template name: "Test Filter Min" @@ -218,3 +248,9 @@ button: id: btn_filter_percentage on_press: - script.execute: test_filter_percentage + + - platform: template + name: "Test Filter NaN" + id: btn_filter_nan + on_press: + - script.execute: test_filter_nan diff --git a/tests/integration/test_sensor_filters_delta.py b/tests/integration/test_sensor_filters_delta.py index 9d0114e0c4..af8f314f49 100644 --- a/tests/integration/test_sensor_filters_delta.py +++ b/tests/integration/test_sensor_filters_delta.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import math from aioesphomeapi import ButtonInfo, EntityState, SensorState import pytest @@ -25,6 +26,7 @@ async def test_sensor_filters_delta( "filter_baseline_max": [], "filter_zero_delta": [], "filter_percentage": [], + "filter_nan": [], } filter_min_done = loop.create_future() @@ -32,16 +34,23 @@ async def test_sensor_filters_delta( filter_baseline_max_done = loop.create_future() filter_zero_delta_done = loop.create_future() filter_percentage_done = loop.create_future() + filter_nan_done = loop.create_future() def on_state(state: EntityState) -> None: - if not isinstance(state, SensorState) or state.missing_state: + if not isinstance(state, SensorState): return sensor_name = key_to_sensor.get(state.key) if sensor_name not in sensor_values: return - sensor_values[sensor_name].append(state.state) + if state.missing_state: + # Only the NaN test is interested in unavailable states + if sensor_name != "filter_nan": + return + sensor_values[sensor_name].append(math.nan) + else: + sensor_values[sensor_name].append(state.state) # Check completion conditions if ( @@ -74,6 +83,12 @@ async def test_sensor_filters_delta( and not filter_percentage_done.done() ): filter_percentage_done.set_result(True) + elif ( + sensor_name == "filter_nan" + and len(sensor_values[sensor_name]) == 3 + and not filter_nan_done.done() + ): + filter_nan_done.set_result(True) async with ( run_compiled(yaml_config), @@ -89,6 +104,7 @@ async def test_sensor_filters_delta( "filter_baseline_max": "Filter Baseline Max", "filter_zero_delta": "Filter Zero Delta", "filter_percentage": "Filter Percentage", + "filter_nan": "Filter NaN", }, ) @@ -108,13 +124,14 @@ async def test_sensor_filters_delta( "Test Filter Baseline Max": "filter_baseline_max", "Test Filter Zero Delta": "filter_zero_delta", "Test Filter Percentage": "filter_percentage", + "Test Filter NaN": "filter_nan", } buttons = {} for entity in entities: if isinstance(entity, ButtonInfo) and entity.name in button_name_map: buttons[button_name_map[entity.name]] = entity.key - assert len(buttons) == 5, f"Expected 5 buttons, found {len(buttons)}" + assert len(buttons) == 6, f"Expected 6 buttons, found {len(buttons)}" # Test 1: Min sensor_values["filter_min"].clear() @@ -186,3 +203,18 @@ async def test_sensor_filters_delta( assert sensor_values["filter_percentage"] == pytest.approx(expected), ( f"Test 5 failed: expected {expected}, got {sensor_values['filter_percentage']}" ) + + # Test 6: NaN passes through once, then is suppressed + sensor_values["filter_nan"].clear() + client.button_command(buttons["filter_nan"]) + try: + await asyncio.wait_for(filter_nan_done, timeout=2.0) + except TimeoutError: + pytest.fail(f"Test 6 timed out. Values: {sensor_values['filter_nan']}") + + values = sensor_values["filter_nan"] + assert values[0] == pytest.approx(1.0), f"Test 6 failed: got {values}" + assert math.isnan(values[1]), ( + f"Test 6 failed: NaN not passed through, got {values}" + ) + assert values[2] == pytest.approx(2.0), f"Test 6 failed: got {values}" From 2bc4681fd6d54d5959b93e6e5873b35ece42196d Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Sun, 16 Aug 2026 19:33:25 +0200 Subject: [PATCH 053/160] [zigbee] bump esp-zigbee-sdk to 2.0.4 (#18415) --- esphome/components/zigbee/zigbee_esp32.cpp | 5 +++++ esphome/components/zigbee/zigbee_esp32.py | 2 +- esphome/idf_component.yml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index 482995e2c5..cd094306f4 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -307,6 +307,11 @@ void ZigbeeComponent::setup() { return; } #endif + +#ifdef CONFIG_ZB_ZCZR + ezb_bdb_set_router_rejoin_required(true); +#endif + ezb_aps_secur_enable_distributed_security(false); ezb_nwk_set_min_join_lqi(32); if (ezb_app_signal_add_handler(ZigbeeComponent::app_signal_handler) != ESP_OK) { diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index 8e63c09e67..ade45e8cc3 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -285,7 +285,7 @@ async def attributes_to_code( async def esp32_to_code(config: ConfigType) -> "MockObj": add_idf_component( name="espressif/esp-zigbee-lib", - ref="2.0.3", + ref="2.0.4", ) # add sdkconfigs later so they can overwrite esp32 defaults diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index aff1a6819f..62fd597845 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -48,7 +48,7 @@ dependencies: rules: - if: "target in [esp32, esp32p4]" espressif/esp-zigbee-lib: - version: 2.0.3 + version: 2.0.4 rules: - if: "target in [esp32h2, esp32c5, esp32c6]" espressif/lan87xx: From c664f5fc951a8ae55eef64f14a382cdfc9e0b3dc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 11:06:22 -0700 Subject: [PATCH 054/160] [bk72xx_ble] Fail early with a clear error on non BLE 5.x SoCs (#18406) --- esphome/components/bk72xx_ble/__init__.py | 44 ++++++++++++++++--- esphome/components/bk72xx_ble/bdk_scan.cpp | 4 +- esphome/components/bk72xx_ble/bk72xx_ble.cpp | 22 ++++++---- tests/component_tests/bk72xx_ble/__init__.py | 0 .../bk72xx_ble/config/test_bk7231n.yaml | 7 +++ .../bk72xx_ble/config/test_bk7231q.yaml | 7 +++ .../bk72xx_ble/config/test_bk7231t.yaml | 7 +++ .../bk72xx_ble/config/test_bk7252.yaml | 7 +++ .../bk72xx_ble/test_family_gate.py | 40 +++++++++++++++++ .../config/bk72xx_controller_only.yaml | 2 +- .../config/bk72xx_tracker.yaml | 2 +- 11 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 tests/component_tests/bk72xx_ble/__init__.py create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7252.yaml create mode 100644 tests/component_tests/bk72xx_ble/test_family_gate.py diff --git a/esphome/components/bk72xx_ble/__init__.py b/esphome/components/bk72xx_ble/__init__.py index 23f3d06184..b58464a1f6 100644 --- a/esphome/components/bk72xx_ble/__init__.py +++ b/esphome/components/bk72xx_ble/__init__.py @@ -5,11 +5,11 @@ bring-up and the controller BLE address. Consumers (bk72xx_ble_tracker) build on this component and contain no SDK calls of their own. Supported SoCs (BLE 5.x): BK7231N/BK7236 (BLE 5.1), BK7238/BK7252N/BK7253 -(BLE 5.2), and any future BLE-5.x SoC. Capability is detected at compile time, -not by a chip list: the C++ guards on `__has_include("ble_api.h")` — the Beken -BLE 5.x public API header, which the LibreTiny beken-72xx builder ships only -for BLE-5.x SoCs. BK7231T/BK7251/BK7271 (BLE 4.2) and BK7231Q (no BLE) fail -with a clear #error. +(BLE 5.2), and any future BLE-5.x SoC. Known non-5.x families are rejected in +to_code; unknown families are capability-checked at compile time via +`__has_include("app_ble.h")`, a header only on the BLE 5.x include path +(ble_api.h ships for every SoC, so it cannot be the probe). A non-5.x build +fails with a clear #error. No framework patch is needed: the LibreTiny beken-72xx builder already compiles and links the BLE 5.x stack (CFG_SUPPORT_BLE=1 + CFG_BLE_VERSION=BLE_VERSION_5_x; @@ -21,9 +21,16 @@ import logging import esphome.codegen as cg from esphome.components import libretiny -from esphome.components.libretiny.const import FAMILY_BK7231N, FAMILY_BK7238 +from esphome.components.libretiny.const import ( + FAMILY_BK7231N, + FAMILY_BK7231Q, + FAMILY_BK7231T, + FAMILY_BK7238, + FAMILY_BK7251, +) import esphome.config_validation as cv from esphome.const import CONF_ENABLE_ON_BOOT, CONF_ID +from esphome.core import EsphomeError from esphome.types import ConfigType DEPENDENCIES = ["bk72xx"] @@ -50,7 +57,32 @@ CONFIG_SCHEMA = cv.Schema( request_scan_listener_slot = cg.slot_counter("BK72XX_BLE_SCAN_LISTENER_COUNT") +def _unsupported_family_message(family: str) -> str | None: + if family in (FAMILY_BK7231T, FAMILY_BK7251): + return ( + f"bk72xx_ble does not support {family}: this SoC has the Beken BLE 4.2 " + "stack; a BLE 5.x SoC such as BK7231N or BK7238 is required" + ) + if family == FAMILY_BK7231Q: + return "bk72xx_ble does not support BK7231Q: this SoC has no BLE" + return None + + +def _final_validate(config: ConfigType) -> ConfigType: + # Warn only: a hard error here would break the validate-only CI fixtures, + # which run on a BLE 4.2 board. The hard error is raised at codegen. + if msg := _unsupported_family_message(libretiny.get_libretiny_family()): + _LOGGER.warning("%s (this configuration cannot compile)", msg) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + async def to_code(config: ConfigType) -> None: + if msg := _unsupported_family_message(libretiny.get_libretiny_family()): + raise EsphomeError(msg) + var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/esphome/components/bk72xx_ble/bdk_scan.cpp b/esphome/components/bk72xx_ble/bdk_scan.cpp index bd4e51d9b7..f17f21c06b 100644 --- a/esphome/components/bk72xx_ble/bdk_scan.cpp +++ b/esphome/components/bk72xx_ble/bdk_scan.cpp @@ -10,7 +10,7 @@ #ifdef USE_BK72XX_BLE // Same SDK gate as bk72xx_ble.cpp (which carries the explanatory #error). -#if !defined(CLANG_TIDY) && __has_include("ble_api.h") +#if !defined(CLANG_TIDY) && __has_include("ble_api.h") && __has_include("app_ble.h") extern "C" { #include "app_ble.h" // app_ble_env, app_ble_run, app_ble_reset, actv_state_t, @@ -115,5 +115,5 @@ BdkOpResult bdk_scan_release(uint8_t activity_idx, bool created, int *err_out) { } // namespace esphome::bk72xx_ble -#endif // !CLANG_TIDY && ble_api.h +#endif // !CLANG_TIDY && ble_api.h && app_ble.h #endif // USE_BK72XX_BLE diff --git a/esphome/components/bk72xx_ble/bk72xx_ble.cpp b/esphome/components/bk72xx_ble/bk72xx_ble.cpp index d40f08d111..52401114e6 100644 --- a/esphome/components/bk72xx_ble/bk72xx_ble.cpp +++ b/esphome/components/bk72xx_ble/bk72xx_ble.cpp @@ -34,22 +34,26 @@ // --------------------------------------------------------------------------- // SDK-capability gate (not a chip allowlist). -// This component drives the Beken BLE *5.x* controller via its public API, -// `ble_api.h`, which the LibreTiny beken-72xx builder ships only for the -// BLE-5.x SoCs (it selects the `ble_pub` 5.x stack from CFG_BLE_VERSION; the -// 4.2 SoCs build a different, older API with no ble_api.h). Gate on the header -// itself so any BLE-5.x Beken chip — present or future — is supported without a -// hard-coded list, and a non-5.x build fails here with a clear message instead -// of a cryptic "ble_api.h: No such file or directory". +// This component drives the Beken BLE *5.x* controller. `ble_api.h` cannot be +// the probe: it ships for every SoC (driver/include) and merely switches on +// CFG_BLE_VERSION internally. `app_ble.h` is on the include path only when the +// LibreTiny beken-72xx builder selects a 5.x stack, so gating on it supports +// any BLE-5.x chip — present or future — without a hard-coded list, and a +// non-5.x build fails here with a clear message instead of a cryptic +// "app_ble.h: No such file or directory". // --------------------------------------------------------------------------- #if defined(CLANG_TIDY) // The clang-tidy environment does not carry the full Beken BDK BLE 5.x API // (its ble_api.h variant lacks parts of the 5.x surface), so there is nothing // accurate to analyze the SDK calls against — skip the file under analysis. #define BK72XX_BLE_NO_SDK -#elif !__has_include("ble_api.h") +#elif !__has_include("ble_api.h") || !__has_include("app_ble.h") +// Also skip the SDK body: #error does not stop the preprocessor, and on a 4.2 +// SoC ble_api.h exists, so without the guard the 5.x symbols would fail one by +// one and bury this message. +#define BK72XX_BLE_NO_SDK #error \ - "bk72xx_ble requires a BLE 5.x Beken SDK (ble_api.h). Supported SoCs: BK7231N/BK7236 (BLE 5.1) and BK7238/BK7252N/BK7253 (BLE 5.2). BK7231T/BK7251/BK7271 (BLE 4.2) and BK7231Q (no BLE) are not supported." + "bk72xx_ble requires a BLE 5.x Beken SDK (app_ble.h). Supported SoCs: BK7231N/BK7236 (BLE 5.1) and BK7238/BK7252N/BK7253 (BLE 5.2). BK7231T/BK7251/BK7271 (BLE 4.2) and BK7231Q (no BLE) are not supported." #endif #ifndef BK72XX_BLE_NO_SDK diff --git a/tests/component_tests/bk72xx_ble/__init__.py b/tests/component_tests/bk72xx_ble/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml new file mode 100644 index 0000000000..772ab93c79 --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-n + +bk72xx: + board: cb2s + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml new file mode 100644 index 0000000000..17fd15b1b4 --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-q + +bk72xx: + board: wa2 + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml new file mode 100644 index 0000000000..fec21a6aae --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-t + +bk72xx: + board: generic-bk7231t-qfn32-tuya + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7252.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7252.yaml new file mode 100644 index 0000000000..a3290ab50a --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7252.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-7252 + +bk72xx: + board: generic-bk7252 + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/test_family_gate.py b/tests/component_tests/bk72xx_ble/test_family_gate.py new file mode 100644 index 0000000000..da67749bb3 --- /dev/null +++ b/tests/component_tests/bk72xx_ble/test_family_gate.py @@ -0,0 +1,40 @@ +"""The non-5.x family rejection lives in to_code (config validation must stay +family-agnostic for the validate-only CI fixtures), so codegen is the only +place it can be pinned.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.core import EsphomeError + + +@pytest.mark.parametrize( + ("config_file", "match"), + [ + ("test_bk7231t.yaml", "BK7231T.*BLE 4.2"), + ("test_bk7252.yaml", "BK7251.*BLE 4.2"), + ("test_bk7231q.yaml", "BK7231Q.*no BLE"), + ], +) +def test_unsupported_family_rejected( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + match: str, + caplog: pytest.LogCaptureFixture, +) -> None: + with pytest.raises(EsphomeError, match=match): + generate_main(component_config_path(config_file)) + # Validation itself must not fail (CI validate fixtures run on a BLE 4.2 + # board), but it warns before codegen raises. + assert "cannot compile" in caplog.text + + +def test_ble5_family_generates( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("test_bk7231n.yaml")) + assert "bk72xx_ble::BK72xxBLE" in main_cpp diff --git a/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml b/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml index 4d4dab0198..7912fceed6 100644 --- a/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml +++ b/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml @@ -2,6 +2,6 @@ esphome: name: slotcount-controller bk72xx: - board: generic-bk7252 + board: cb2s bk72xx_ble: diff --git a/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml b/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml index 79e9644006..b813e2702e 100644 --- a/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml +++ b/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml @@ -2,6 +2,6 @@ esphome: name: slotcount-tracker bk72xx: - board: generic-bk7252 + board: cb2s bk72xx_ble_tracker: From 32c76ae8289326cb2f17d9712db190fc2d599028 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 12:43:27 -0700 Subject: [PATCH 055/160] [core] Skip redundant ESP8266 main loop wake posts from ISR context (#18416) --- esphome/core/wake/wake_esp8266.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/esphome/core/wake/wake_esp8266.h b/esphome/core/wake/wake_esp8266.h index 7eaaae5293..73b7a38a35 100644 --- a/esphome/core/wake/wake_esp8266.h +++ b/esphome/core/wake/wake_esp8266.h @@ -15,6 +15,13 @@ inline void ESPHOME_ALWAYS_INLINE wake_loop_impl() { // Set the wake-requested flag BEFORE esp_schedule so the consumer is // guaranteed to see it on its next gate check. wake_request_set(); + // Skip the post when a wake was already signalled and not yet consumed by + // wakeable_delay(): esp_schedule() -> ets_post() can enter SDK WiFi pm code, + // which must not be poked per-byte from the software serial RX ISR (see + // esphome#18409). The flag can stay latched while the loop is awake, which + // is intentional; posts are only needed to cut a suspend short. + if (g_main_loop_woke) + return; g_main_loop_woke = true; esp_schedule(); } From 801a1817b58909e5bc243b493c53e3e2265ada07 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 12:44:00 -0700 Subject: [PATCH 056/160] [esp32] Split crash handler addr2line hint per core (#18418) --- esphome/components/esp32/crash_handler.cpp | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/esphome/components/esp32/crash_handler.cpp b/esphome/components/esp32/crash_handler.cpp index 1b054dcc49..b61dad7386 100644 --- a/esphome/components/esp32/crash_handler.cpp +++ b/esphome/components/esp32/crash_handler.cpp @@ -360,17 +360,6 @@ static bool has_fault_addr() { return s_raw_crash_data.exception == PANIC_EXCEPTION_FAULT && !s_raw_crash_data.pseudo_excause; } -// Append both cores' backtrace addresses to buf; returns the new position. -static int append_all_backtraces(char *buf, int size, int pos) { - pos = append_addrs_to_hint(buf, size, pos, s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, - s_raw_crash_data.reg_frame_count); -#if SOC_CPU_CORES_NUM > 1 - pos = append_addrs_to_hint(buf, size, pos, s_raw_crash_data.other_backtrace, s_raw_crash_data.other_backtrace_count, - s_raw_crash_data.other_reg_frame_count); -#endif - return pos; -} - // The record was captured by a different firmware build (it survives soft // resets, including the OTA reboot), so symbolizing its addresses against the // current ELF would produce misleading symbols. Print them with lowercase @@ -443,11 +432,23 @@ void crash_handler_log() { } #endif - // Build addr2line hint with all captured addresses for easy copy-paste + // Build addr2line hints for easy copy-paste. One line per core: the two + // backtraces are separate stacks, and a combined list decodes as one + // impossible call chain (and can overflow the buffer, dropping addresses). + static const char *const ADDR2LINE_CMD = "addr2line -pfiaC -e firmware.elf"; char hint[256]; - int pos = snprintf(hint, sizeof(hint), "Use: addr2line -pfiaC -e firmware.elf 0x%08" PRIX32, s_raw_crash_data.pc); - append_all_backtraces(hint, sizeof(hint), pos); + int pos = snprintf(hint, sizeof(hint), "Use: %s 0x%08" PRIX32, ADDR2LINE_CMD, s_raw_crash_data.pc); + append_addrs_to_hint(hint, sizeof(hint), pos, s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, + s_raw_crash_data.reg_frame_count); ESP_LOGE(TAG, "%s", hint); +#if SOC_CPU_CORES_NUM > 1 + if (s_raw_crash_data.other_backtrace_count > 0) { + pos = snprintf(hint, sizeof(hint), "Other core: %s", ADDR2LINE_CMD); + append_addrs_to_hint(hint, sizeof(hint), pos, s_raw_crash_data.other_backtrace, + s_raw_crash_data.other_backtrace_count, s_raw_crash_data.other_reg_frame_count); + ESP_LOGE(TAG, "%s", hint); + } +#endif } } // namespace esphome::esp32 From 3f01f9895f0c98179d8301dbed46aa02801d7f77 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 12:44:49 -0700 Subject: [PATCH 057/160] [esp32_hosted] Require ESP-IDF 5.3 or newer (#18417) --- esphome/components/esp32_hosted/__init__.py | 34 ++++++++++++------ .../component_tests/esp32_hosted/__init__.py | 0 .../component_tests/esp32_hosted/test_init.py | 35 +++++++++++++++++++ 3 files changed, 59 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/esp32_hosted/__init__.py create mode 100644 tests/component_tests/esp32_hosted/test_init.py diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index c6a714aace..d3432fb461 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -16,8 +16,10 @@ from esphome.const import ( CONF_VARIANT, ) from esphome.cpp_generator import add_define +from esphome.types import ConfigType CODEOWNERS = ["@swoboda1337"] +DEPENDENCIES = ["esp32"] # esp32_ble raises the task watchdog around the remote BT controller bring-up AUTO_LOAD = ["watchdog"] @@ -124,6 +126,22 @@ CONFIG_SCHEMA = cv.typed_schema( ) +def _final_validate(config: ConfigType) -> ConfigType: + # The esp_hosted releases compatible with older ESP-IDF versions crash at + # boot with a heap double free in the SDIO RX path (fixed in esp_hosted + # 2.11.0, which requires ESP-IDF 5.3), so reject them at validation time. + if (idf_ver := esp32.idf_version()) < cv.Version(5, 3, 0): + raise cv.Invalid( + f"esp32_hosted requires ESP-IDF 5.3 or newer, got {idf_ver}. " + "Remove the framework version from your configuration to use the " + "recommended version, or pin a version at or above 5.3." + ) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + def _configure_sdio(config): slot = config[CONF_SLOT] esp32.add_idf_sdkconfig_option( @@ -251,18 +269,14 @@ async def to_code(config): if config[CONF_USE_PSRAM]: esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_MEMPOOL_PREFER_SPIRAM", True) - # Library versions + # Library versions; this component set requires ESP-IDF 5.3 or newer, + # which is enforced at validation time. idf_ver = esp32.idf_version() os.environ["ESP_IDF_VERSION"] = f"{idf_ver.major}.{idf_ver.minor}" - if idf_ver >= cv.Version(5, 5, 0): - esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.6.3") - esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.3") - esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5") - esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.12") - else: - esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="0.13.0") - esp32.add_idf_component(name="espressif/eppp_link", ref="0.2.0") - esp32.add_idf_component(name="espressif/esp_hosted", ref="2.0.11") + esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.6.3") + esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.3") + esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5") + esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.12") esp32.add_extra_script( "post", "esp32_hosted.py", diff --git a/tests/component_tests/esp32_hosted/__init__.py b/tests/component_tests/esp32_hosted/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/esp32_hosted/test_init.py b/tests/component_tests/esp32_hosted/test_init.py new file mode 100644 index 0000000000..cec81e4e83 --- /dev/null +++ b/tests/component_tests/esp32_hosted/test_init.py @@ -0,0 +1,35 @@ +"""Tests for the esp32_hosted ESP-IDF version gate.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.esp32 import KEY_IDF_VERSION +from esphome.components.esp32_hosted import _final_validate +from esphome.const import PlatformFramework + +from ..types import SetCoreConfigCallable + + +@pytest.mark.parametrize("idf", ["5.3.0", "5.4.2", "5.5.5"]) +def test_final_validate_accepts_supported_idf( + set_core_config: SetCoreConfigCallable, idf: str +) -> None: + """ESP-IDF 5.3 and newer passes validation unchanged.""" + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_IDF_VERSION: cv.Version.parse(idf)}, + ) + assert _final_validate({}) == {} + + +@pytest.mark.parametrize("idf", ["5.0.0", "5.2.2"]) +def test_final_validate_rejects_old_idf( + set_core_config: SetCoreConfigCallable, idf: str +) -> None: + """ESP-IDF older than 5.3 is rejected with a clear error.""" + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_IDF_VERSION: cv.Version.parse(idf)}, + ) + with pytest.raises(cv.Invalid, match="requires ESP-IDF 5.3 or newer"): + _final_validate({}) From 9161f74bb1e58b29f76f92bd5c298adbcbdf728b Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:48:27 +0000 Subject: [PATCH 058/160] Bump bundled esphome-device-builder to 1.10.0 (#18389) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d7ae2cd4ec..a62eb59a58 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.9.6 +RUN uv pip install --no-cache-dir esphome-device-builder==1.10.0 RUN \ platformio settings set enable_telemetry No \ From 46a5665a66873f990398a477dab767c8620e66a1 Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Sat, 15 Aug 2026 11:16:04 -0700 Subject: [PATCH 059/160] [rotary_encoder] account for min and max value when resetting (#18197) --- esphome/components/rotary_encoder/rotary_encoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/rotary_encoder/rotary_encoder.cpp b/esphome/components/rotary_encoder/rotary_encoder.cpp index 0831822d86..0734ca87d3 100644 --- a/esphome/components/rotary_encoder/rotary_encoder.cpp +++ b/esphome/components/rotary_encoder/rotary_encoder.cpp @@ -220,7 +220,7 @@ void RotaryEncoderSensor::loop() { } if (this->pin_i_ != nullptr && this->pin_i_->digital_read()) { - this->store_.counter = 0; + this->store_.counter = std::clamp(0, this->store_.min_value, this->store_.max_value); } int counter = this->store_.counter; if (this->store_.last_read != counter || this->publish_initial_value_) { From dda4566b9e32fd2fab3faa5b7a7335c0bda2fda3 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:36:52 -0700 Subject: [PATCH 060/160] Bump bundled esphome-device-builder to 1.11.0 (#18403) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a62eb59a58..2f23b2f690 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.10.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.0 RUN \ platformio settings set enable_telemetry No \ From ce09504c923a171935d4cb80e598aeaf1cdea1fe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 08:42:47 -0700 Subject: [PATCH 061/160] [platformio] Skip ccache when the binary on PATH fails to run (#18407) --- esphome/platformio/toolchain.py | 32 ++++++++++++- tests/unit_tests/test_platformio_toolchain.py | 48 ++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 0e7ffce939..08a4fcff78 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -5,6 +5,7 @@ import os from pathlib import Path import re import shutil +import subprocess import sys from typing import TYPE_CHECKING, Any @@ -234,6 +235,35 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) +def _ccache_usable() -> bool: + """Return True when the ``ccache`` on PATH actually runs. + + ``shutil.which`` proves existence, not runnability: on Windows it also + matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose + target is gone. Wrapping compiles around such a find fails every compile + step with an opaque OS error, so probe once and fall back to compiling + without ccache when the probe fails. + """ + ccache = shutil.which("ccache") + if ccache is None: + return False + try: + subprocess.run( + [ccache, "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=15, + ) + except (OSError, subprocess.SubprocessError): + _LOGGER.warning( + "Ignoring ccache at %s because it failed to run; compiling without ccache", + ccache, + ) + return False + return True + + def _ccache_env() -> dict[str, str]: """Return ccache settings for PlatformIO builds. @@ -266,7 +296,7 @@ def _ccache_env() -> dict[str, str]: if "ESPHOME_CCACHE_ENABLE" in os.environ: enabled = get_bool_env("ESPHOME_CCACHE_ENABLE") else: - enabled = shutil.which("ccache") is not None + enabled = _ccache_usable() env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"} if not enabled: return env diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 02c11b4e45..eebb0b8cd7 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -9,6 +9,7 @@ import json import os from pathlib import Path import shutil +import subprocess import sys import threading from types import SimpleNamespace @@ -431,6 +432,7 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: with ( patch.dict(os.environ, {}, clear=True), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), ): env = toolchain._ccache_env() @@ -457,6 +459,44 @@ def test_ccache_env_disabled_without_binary(setup_core: Path) -> None: assert env == {"ESPHOME_CCACHE_ENABLE": "0"} +@pytest.mark.parametrize( + "probe_error", + [ + pytest.param(OSError("not runnable"), id="oserror"), + pytest.param(subprocess.CalledProcessError(1, "ccache"), id="nonzero-exit"), + pytest.param(subprocess.TimeoutExpired("ccache", 15), id="timeout"), + ], +) +def test_ccache_env_disabled_when_probe_fails( + setup_core: Path, probe_error: Exception +) -> None: + """A ccache that resolves on PATH but fails to run stays disabled.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run", side_effect=probe_error), + ): + env = toolchain._ccache_env() + + assert env == {"ESPHOME_CCACHE_ENABLE": "0"} + + +def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None: + """An explicit ESPHOME_CCACHE_ENABLE=1 does not probe the binary.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), + patch.object(toolchain.subprocess, "run") as mock_probe, + ): + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + mock_probe.assert_not_called() + + def test_ccache_env_opt_out(setup_core: Path) -> None: """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present.""" CORE.build_path = setup_core / "build" / "test" @@ -496,6 +536,7 @@ def test_ccache_env_respects_user_values_and_refreshes_basedir( with ( patch.dict(os.environ, user_env, clear=True), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), ): env = toolchain._ccache_env() @@ -514,6 +555,7 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only( with ( patch.dict(os.environ, {}, clear=False), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), ): os.environ.pop("ESPHOME_CCACHE_ENABLE", None) mock_run_external_process.return_value = 0 @@ -533,6 +575,7 @@ def test_ccache_env_requires_build_path(setup_core: Path) -> None: with ( patch.dict(os.environ, {}, clear=True), patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), pytest.raises(ValueError, match="CORE.build_path must be set"), ): toolchain._ccache_env() @@ -544,7 +587,10 @@ def test_run_platformio_cli_merges_caller_env( """A caller-supplied env is the base and gains the ccache settings.""" CORE.build_path = str(setup_core / "build" / "test") - with patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"): + with ( + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch.object(toolchain.subprocess, "run"), + ): mock_run_external_process.return_value = 0 toolchain.run_platformio_cli( "test", env={"CUSTOM_VAR": "1", "ESPHOME_CCACHE_ENABLE": "0"} From bca72e9b6d7d6a4bebff6da0a946e952aef081e5 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:36:51 -0400 Subject: [PATCH 062/160] [sensor] Pass NaN through the delta filter again (#18400) --- esphome/components/sensor/filter.cpp | 10 +++-- .../fixtures/sensor_filters_delta.yaml | 36 ++++++++++++++++++ .../integration/test_sensor_filters_delta.py | 38 +++++++++++++++++-- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 5f7f19769a..0105580d26 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -283,8 +283,11 @@ DeltaFilter::DeltaFilter(float min_a0, float min_a1, float max_a0, float max_a1) void DeltaFilter::set_baseline(float (*fn)(float)) { this->baseline_ = fn; } optional DeltaFilter::new_value(float value) { - // Always yield the first value. - if (std::isnan(this->last_value_)) { + const bool no_value = std::isnan(value); + const bool no_reference = std::isnan(this->last_value_); + if (no_value && no_reference) + return {}; + if (no_value || no_reference) { this->last_value_ = value; return value; } @@ -293,8 +296,7 @@ optional DeltaFilter::new_value(float value) { float min = fabsf(this->min_a0_ + ref * this->min_a1_); float max = fabsf(this->max_a0_ + ref * this->max_a1_); float delta = fabsf(value - ref); - // if there is no reference, e.g. for the first value, just accept this one, - // otherwise accept only if within range. + // accept only if within range if (delta > min && delta <= max) { this->last_value_ = value; return value; diff --git a/tests/integration/fixtures/sensor_filters_delta.yaml b/tests/integration/fixtures/sensor_filters_delta.yaml index 2494a430da..b01c8e452b 100644 --- a/tests/integration/fixtures/sensor_filters_delta.yaml +++ b/tests/integration/fixtures/sensor_filters_delta.yaml @@ -33,6 +33,11 @@ sensor: id: source_sensor_5 accuracy_decimals: 1 + - platform: template + name: "Source Sensor 6" + id: source_sensor_6 + accuracy_decimals: 1 + - platform: copy source_id: source_sensor_1 name: "Filter Min" @@ -81,6 +86,13 @@ sensor: filters: - delta: 50% + - platform: copy + source_id: source_sensor_6 + name: "Filter NaN" + id: filter_nan + filters: + - delta: 0 + script: - id: test_filter_min then: @@ -188,6 +200,24 @@ script: id: source_sensor_5 state: 250.0 # Passes (delta=90 > 80) + - id: test_filter_nan + then: + - sensor.template.publish: + id: source_sensor_6 + state: 1.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor_6 + state: !lambda "return NAN;" + - delay: 20ms + - sensor.template.publish: + id: source_sensor_6 + state: !lambda "return NAN;" # Filtered out + - delay: 20ms + - sensor.template.publish: + id: source_sensor_6 + state: 2.0 + button: - platform: template name: "Test Filter Min" @@ -218,3 +248,9 @@ button: id: btn_filter_percentage on_press: - script.execute: test_filter_percentage + + - platform: template + name: "Test Filter NaN" + id: btn_filter_nan + on_press: + - script.execute: test_filter_nan diff --git a/tests/integration/test_sensor_filters_delta.py b/tests/integration/test_sensor_filters_delta.py index 9d0114e0c4..af8f314f49 100644 --- a/tests/integration/test_sensor_filters_delta.py +++ b/tests/integration/test_sensor_filters_delta.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import math from aioesphomeapi import ButtonInfo, EntityState, SensorState import pytest @@ -25,6 +26,7 @@ async def test_sensor_filters_delta( "filter_baseline_max": [], "filter_zero_delta": [], "filter_percentage": [], + "filter_nan": [], } filter_min_done = loop.create_future() @@ -32,16 +34,23 @@ async def test_sensor_filters_delta( filter_baseline_max_done = loop.create_future() filter_zero_delta_done = loop.create_future() filter_percentage_done = loop.create_future() + filter_nan_done = loop.create_future() def on_state(state: EntityState) -> None: - if not isinstance(state, SensorState) or state.missing_state: + if not isinstance(state, SensorState): return sensor_name = key_to_sensor.get(state.key) if sensor_name not in sensor_values: return - sensor_values[sensor_name].append(state.state) + if state.missing_state: + # Only the NaN test is interested in unavailable states + if sensor_name != "filter_nan": + return + sensor_values[sensor_name].append(math.nan) + else: + sensor_values[sensor_name].append(state.state) # Check completion conditions if ( @@ -74,6 +83,12 @@ async def test_sensor_filters_delta( and not filter_percentage_done.done() ): filter_percentage_done.set_result(True) + elif ( + sensor_name == "filter_nan" + and len(sensor_values[sensor_name]) == 3 + and not filter_nan_done.done() + ): + filter_nan_done.set_result(True) async with ( run_compiled(yaml_config), @@ -89,6 +104,7 @@ async def test_sensor_filters_delta( "filter_baseline_max": "Filter Baseline Max", "filter_zero_delta": "Filter Zero Delta", "filter_percentage": "Filter Percentage", + "filter_nan": "Filter NaN", }, ) @@ -108,13 +124,14 @@ async def test_sensor_filters_delta( "Test Filter Baseline Max": "filter_baseline_max", "Test Filter Zero Delta": "filter_zero_delta", "Test Filter Percentage": "filter_percentage", + "Test Filter NaN": "filter_nan", } buttons = {} for entity in entities: if isinstance(entity, ButtonInfo) and entity.name in button_name_map: buttons[button_name_map[entity.name]] = entity.key - assert len(buttons) == 5, f"Expected 5 buttons, found {len(buttons)}" + assert len(buttons) == 6, f"Expected 6 buttons, found {len(buttons)}" # Test 1: Min sensor_values["filter_min"].clear() @@ -186,3 +203,18 @@ async def test_sensor_filters_delta( assert sensor_values["filter_percentage"] == pytest.approx(expected), ( f"Test 5 failed: expected {expected}, got {sensor_values['filter_percentage']}" ) + + # Test 6: NaN passes through once, then is suppressed + sensor_values["filter_nan"].clear() + client.button_command(buttons["filter_nan"]) + try: + await asyncio.wait_for(filter_nan_done, timeout=2.0) + except TimeoutError: + pytest.fail(f"Test 6 timed out. Values: {sensor_values['filter_nan']}") + + values = sensor_values["filter_nan"] + assert values[0] == pytest.approx(1.0), f"Test 6 failed: got {values}" + assert math.isnan(values[1]), ( + f"Test 6 failed: NaN not passed through, got {values}" + ) + assert values[2] == pytest.approx(2.0), f"Test 6 failed: got {values}" From 594c12b3d961a20576b2425e75d4d05f18fc1993 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Sun, 16 Aug 2026 19:33:25 +0200 Subject: [PATCH 063/160] [zigbee] bump esp-zigbee-sdk to 2.0.4 (#18415) --- esphome/components/zigbee/zigbee_esp32.cpp | 5 +++++ esphome/components/zigbee/zigbee_esp32.py | 2 +- esphome/idf_component.yml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index 482995e2c5..cd094306f4 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -307,6 +307,11 @@ void ZigbeeComponent::setup() { return; } #endif + +#ifdef CONFIG_ZB_ZCZR + ezb_bdb_set_router_rejoin_required(true); +#endif + ezb_aps_secur_enable_distributed_security(false); ezb_nwk_set_min_join_lqi(32); if (ezb_app_signal_add_handler(ZigbeeComponent::app_signal_handler) != ESP_OK) { diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index 8e63c09e67..ade45e8cc3 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -285,7 +285,7 @@ async def attributes_to_code( async def esp32_to_code(config: ConfigType) -> "MockObj": add_idf_component( name="espressif/esp-zigbee-lib", - ref="2.0.3", + ref="2.0.4", ) # add sdkconfigs later so they can overwrite esp32 defaults diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index aff1a6819f..62fd597845 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -48,7 +48,7 @@ dependencies: rules: - if: "target in [esp32, esp32p4]" espressif/esp-zigbee-lib: - version: 2.0.3 + version: 2.0.4 rules: - if: "target in [esp32h2, esp32c5, esp32c6]" espressif/lan87xx: From 0bc2d7137078ccb28aa3a8fc8ddbb4ae100a3c52 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 11:06:22 -0700 Subject: [PATCH 064/160] [bk72xx_ble] Fail early with a clear error on non BLE 5.x SoCs (#18406) --- esphome/components/bk72xx_ble/__init__.py | 44 ++++++++++++++++--- esphome/components/bk72xx_ble/bdk_scan.cpp | 4 +- esphome/components/bk72xx_ble/bk72xx_ble.cpp | 22 ++++++---- tests/component_tests/bk72xx_ble/__init__.py | 0 .../bk72xx_ble/config/test_bk7231n.yaml | 7 +++ .../bk72xx_ble/config/test_bk7231q.yaml | 7 +++ .../bk72xx_ble/config/test_bk7231t.yaml | 7 +++ .../bk72xx_ble/config/test_bk7252.yaml | 7 +++ .../bk72xx_ble/test_family_gate.py | 40 +++++++++++++++++ .../config/bk72xx_controller_only.yaml | 2 +- .../config/bk72xx_tracker.yaml | 2 +- 11 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 tests/component_tests/bk72xx_ble/__init__.py create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml create mode 100644 tests/component_tests/bk72xx_ble/config/test_bk7252.yaml create mode 100644 tests/component_tests/bk72xx_ble/test_family_gate.py diff --git a/esphome/components/bk72xx_ble/__init__.py b/esphome/components/bk72xx_ble/__init__.py index 23f3d06184..b58464a1f6 100644 --- a/esphome/components/bk72xx_ble/__init__.py +++ b/esphome/components/bk72xx_ble/__init__.py @@ -5,11 +5,11 @@ bring-up and the controller BLE address. Consumers (bk72xx_ble_tracker) build on this component and contain no SDK calls of their own. Supported SoCs (BLE 5.x): BK7231N/BK7236 (BLE 5.1), BK7238/BK7252N/BK7253 -(BLE 5.2), and any future BLE-5.x SoC. Capability is detected at compile time, -not by a chip list: the C++ guards on `__has_include("ble_api.h")` — the Beken -BLE 5.x public API header, which the LibreTiny beken-72xx builder ships only -for BLE-5.x SoCs. BK7231T/BK7251/BK7271 (BLE 4.2) and BK7231Q (no BLE) fail -with a clear #error. +(BLE 5.2), and any future BLE-5.x SoC. Known non-5.x families are rejected in +to_code; unknown families are capability-checked at compile time via +`__has_include("app_ble.h")`, a header only on the BLE 5.x include path +(ble_api.h ships for every SoC, so it cannot be the probe). A non-5.x build +fails with a clear #error. No framework patch is needed: the LibreTiny beken-72xx builder already compiles and links the BLE 5.x stack (CFG_SUPPORT_BLE=1 + CFG_BLE_VERSION=BLE_VERSION_5_x; @@ -21,9 +21,16 @@ import logging import esphome.codegen as cg from esphome.components import libretiny -from esphome.components.libretiny.const import FAMILY_BK7231N, FAMILY_BK7238 +from esphome.components.libretiny.const import ( + FAMILY_BK7231N, + FAMILY_BK7231Q, + FAMILY_BK7231T, + FAMILY_BK7238, + FAMILY_BK7251, +) import esphome.config_validation as cv from esphome.const import CONF_ENABLE_ON_BOOT, CONF_ID +from esphome.core import EsphomeError from esphome.types import ConfigType DEPENDENCIES = ["bk72xx"] @@ -50,7 +57,32 @@ CONFIG_SCHEMA = cv.Schema( request_scan_listener_slot = cg.slot_counter("BK72XX_BLE_SCAN_LISTENER_COUNT") +def _unsupported_family_message(family: str) -> str | None: + if family in (FAMILY_BK7231T, FAMILY_BK7251): + return ( + f"bk72xx_ble does not support {family}: this SoC has the Beken BLE 4.2 " + "stack; a BLE 5.x SoC such as BK7231N or BK7238 is required" + ) + if family == FAMILY_BK7231Q: + return "bk72xx_ble does not support BK7231Q: this SoC has no BLE" + return None + + +def _final_validate(config: ConfigType) -> ConfigType: + # Warn only: a hard error here would break the validate-only CI fixtures, + # which run on a BLE 4.2 board. The hard error is raised at codegen. + if msg := _unsupported_family_message(libretiny.get_libretiny_family()): + _LOGGER.warning("%s (this configuration cannot compile)", msg) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + async def to_code(config: ConfigType) -> None: + if msg := _unsupported_family_message(libretiny.get_libretiny_family()): + raise EsphomeError(msg) + var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/esphome/components/bk72xx_ble/bdk_scan.cpp b/esphome/components/bk72xx_ble/bdk_scan.cpp index bd4e51d9b7..f17f21c06b 100644 --- a/esphome/components/bk72xx_ble/bdk_scan.cpp +++ b/esphome/components/bk72xx_ble/bdk_scan.cpp @@ -10,7 +10,7 @@ #ifdef USE_BK72XX_BLE // Same SDK gate as bk72xx_ble.cpp (which carries the explanatory #error). -#if !defined(CLANG_TIDY) && __has_include("ble_api.h") +#if !defined(CLANG_TIDY) && __has_include("ble_api.h") && __has_include("app_ble.h") extern "C" { #include "app_ble.h" // app_ble_env, app_ble_run, app_ble_reset, actv_state_t, @@ -115,5 +115,5 @@ BdkOpResult bdk_scan_release(uint8_t activity_idx, bool created, int *err_out) { } // namespace esphome::bk72xx_ble -#endif // !CLANG_TIDY && ble_api.h +#endif // !CLANG_TIDY && ble_api.h && app_ble.h #endif // USE_BK72XX_BLE diff --git a/esphome/components/bk72xx_ble/bk72xx_ble.cpp b/esphome/components/bk72xx_ble/bk72xx_ble.cpp index d40f08d111..52401114e6 100644 --- a/esphome/components/bk72xx_ble/bk72xx_ble.cpp +++ b/esphome/components/bk72xx_ble/bk72xx_ble.cpp @@ -34,22 +34,26 @@ // --------------------------------------------------------------------------- // SDK-capability gate (not a chip allowlist). -// This component drives the Beken BLE *5.x* controller via its public API, -// `ble_api.h`, which the LibreTiny beken-72xx builder ships only for the -// BLE-5.x SoCs (it selects the `ble_pub` 5.x stack from CFG_BLE_VERSION; the -// 4.2 SoCs build a different, older API with no ble_api.h). Gate on the header -// itself so any BLE-5.x Beken chip — present or future — is supported without a -// hard-coded list, and a non-5.x build fails here with a clear message instead -// of a cryptic "ble_api.h: No such file or directory". +// This component drives the Beken BLE *5.x* controller. `ble_api.h` cannot be +// the probe: it ships for every SoC (driver/include) and merely switches on +// CFG_BLE_VERSION internally. `app_ble.h` is on the include path only when the +// LibreTiny beken-72xx builder selects a 5.x stack, so gating on it supports +// any BLE-5.x chip — present or future — without a hard-coded list, and a +// non-5.x build fails here with a clear message instead of a cryptic +// "app_ble.h: No such file or directory". // --------------------------------------------------------------------------- #if defined(CLANG_TIDY) // The clang-tidy environment does not carry the full Beken BDK BLE 5.x API // (its ble_api.h variant lacks parts of the 5.x surface), so there is nothing // accurate to analyze the SDK calls against — skip the file under analysis. #define BK72XX_BLE_NO_SDK -#elif !__has_include("ble_api.h") +#elif !__has_include("ble_api.h") || !__has_include("app_ble.h") +// Also skip the SDK body: #error does not stop the preprocessor, and on a 4.2 +// SoC ble_api.h exists, so without the guard the 5.x symbols would fail one by +// one and bury this message. +#define BK72XX_BLE_NO_SDK #error \ - "bk72xx_ble requires a BLE 5.x Beken SDK (ble_api.h). Supported SoCs: BK7231N/BK7236 (BLE 5.1) and BK7238/BK7252N/BK7253 (BLE 5.2). BK7231T/BK7251/BK7271 (BLE 4.2) and BK7231Q (no BLE) are not supported." + "bk72xx_ble requires a BLE 5.x Beken SDK (app_ble.h). Supported SoCs: BK7231N/BK7236 (BLE 5.1) and BK7238/BK7252N/BK7253 (BLE 5.2). BK7231T/BK7251/BK7271 (BLE 4.2) and BK7231Q (no BLE) are not supported." #endif #ifndef BK72XX_BLE_NO_SDK diff --git a/tests/component_tests/bk72xx_ble/__init__.py b/tests/component_tests/bk72xx_ble/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml new file mode 100644 index 0000000000..772ab93c79 --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7231n.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-n + +bk72xx: + board: cb2s + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml new file mode 100644 index 0000000000..17fd15b1b4 --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7231q.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-q + +bk72xx: + board: wa2 + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml new file mode 100644 index 0000000000..fec21a6aae --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7231t.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-t + +bk72xx: + board: generic-bk7231t-qfn32-tuya + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/config/test_bk7252.yaml b/tests/component_tests/bk72xx_ble/config/test_bk7252.yaml new file mode 100644 index 0000000000..a3290ab50a --- /dev/null +++ b/tests/component_tests/bk72xx_ble/config/test_bk7252.yaml @@ -0,0 +1,7 @@ +esphome: + name: bk-family-gate-7252 + +bk72xx: + board: generic-bk7252 + +bk72xx_ble: diff --git a/tests/component_tests/bk72xx_ble/test_family_gate.py b/tests/component_tests/bk72xx_ble/test_family_gate.py new file mode 100644 index 0000000000..da67749bb3 --- /dev/null +++ b/tests/component_tests/bk72xx_ble/test_family_gate.py @@ -0,0 +1,40 @@ +"""The non-5.x family rejection lives in to_code (config validation must stay +family-agnostic for the validate-only CI fixtures), so codegen is the only +place it can be pinned.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.core import EsphomeError + + +@pytest.mark.parametrize( + ("config_file", "match"), + [ + ("test_bk7231t.yaml", "BK7231T.*BLE 4.2"), + ("test_bk7252.yaml", "BK7251.*BLE 4.2"), + ("test_bk7231q.yaml", "BK7231Q.*no BLE"), + ], +) +def test_unsupported_family_rejected( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + match: str, + caplog: pytest.LogCaptureFixture, +) -> None: + with pytest.raises(EsphomeError, match=match): + generate_main(component_config_path(config_file)) + # Validation itself must not fail (CI validate fixtures run on a BLE 4.2 + # board), but it warns before codegen raises. + assert "cannot compile" in caplog.text + + +def test_ble5_family_generates( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("test_bk7231n.yaml")) + assert "bk72xx_ble::BK72xxBLE" in main_cpp diff --git a/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml b/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml index 4d4dab0198..7912fceed6 100644 --- a/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml +++ b/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml @@ -2,6 +2,6 @@ esphome: name: slotcount-controller bk72xx: - board: generic-bk7252 + board: cb2s bk72xx_ble: diff --git a/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml b/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml index 79e9644006..b813e2702e 100644 --- a/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml +++ b/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml @@ -2,6 +2,6 @@ esphome: name: slotcount-tracker bk72xx: - board: generic-bk7252 + board: cb2s bk72xx_ble_tracker: From f42fe9af297c8a19c63fdaa2ae06aac43748186b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 12:43:27 -0700 Subject: [PATCH 065/160] [core] Skip redundant ESP8266 main loop wake posts from ISR context (#18416) --- esphome/core/wake/wake_esp8266.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/esphome/core/wake/wake_esp8266.h b/esphome/core/wake/wake_esp8266.h index 7eaaae5293..73b7a38a35 100644 --- a/esphome/core/wake/wake_esp8266.h +++ b/esphome/core/wake/wake_esp8266.h @@ -15,6 +15,13 @@ inline void ESPHOME_ALWAYS_INLINE wake_loop_impl() { // Set the wake-requested flag BEFORE esp_schedule so the consumer is // guaranteed to see it on its next gate check. wake_request_set(); + // Skip the post when a wake was already signalled and not yet consumed by + // wakeable_delay(): esp_schedule() -> ets_post() can enter SDK WiFi pm code, + // which must not be poked per-byte from the software serial RX ISR (see + // esphome#18409). The flag can stay latched while the loop is awake, which + // is intentional; posts are only needed to cut a suspend short. + if (g_main_loop_woke) + return; g_main_loop_woke = true; esp_schedule(); } From bb7d4c3630bf085c45c6991c8d5964baeb2da832 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 12:44:00 -0700 Subject: [PATCH 066/160] [esp32] Split crash handler addr2line hint per core (#18418) --- esphome/components/esp32/crash_handler.cpp | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/esphome/components/esp32/crash_handler.cpp b/esphome/components/esp32/crash_handler.cpp index 1b054dcc49..b61dad7386 100644 --- a/esphome/components/esp32/crash_handler.cpp +++ b/esphome/components/esp32/crash_handler.cpp @@ -360,17 +360,6 @@ static bool has_fault_addr() { return s_raw_crash_data.exception == PANIC_EXCEPTION_FAULT && !s_raw_crash_data.pseudo_excause; } -// Append both cores' backtrace addresses to buf; returns the new position. -static int append_all_backtraces(char *buf, int size, int pos) { - pos = append_addrs_to_hint(buf, size, pos, s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, - s_raw_crash_data.reg_frame_count); -#if SOC_CPU_CORES_NUM > 1 - pos = append_addrs_to_hint(buf, size, pos, s_raw_crash_data.other_backtrace, s_raw_crash_data.other_backtrace_count, - s_raw_crash_data.other_reg_frame_count); -#endif - return pos; -} - // The record was captured by a different firmware build (it survives soft // resets, including the OTA reboot), so symbolizing its addresses against the // current ELF would produce misleading symbols. Print them with lowercase @@ -443,11 +432,23 @@ void crash_handler_log() { } #endif - // Build addr2line hint with all captured addresses for easy copy-paste + // Build addr2line hints for easy copy-paste. One line per core: the two + // backtraces are separate stacks, and a combined list decodes as one + // impossible call chain (and can overflow the buffer, dropping addresses). + static const char *const ADDR2LINE_CMD = "addr2line -pfiaC -e firmware.elf"; char hint[256]; - int pos = snprintf(hint, sizeof(hint), "Use: addr2line -pfiaC -e firmware.elf 0x%08" PRIX32, s_raw_crash_data.pc); - append_all_backtraces(hint, sizeof(hint), pos); + int pos = snprintf(hint, sizeof(hint), "Use: %s 0x%08" PRIX32, ADDR2LINE_CMD, s_raw_crash_data.pc); + append_addrs_to_hint(hint, sizeof(hint), pos, s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, + s_raw_crash_data.reg_frame_count); ESP_LOGE(TAG, "%s", hint); +#if SOC_CPU_CORES_NUM > 1 + if (s_raw_crash_data.other_backtrace_count > 0) { + pos = snprintf(hint, sizeof(hint), "Other core: %s", ADDR2LINE_CMD); + append_addrs_to_hint(hint, sizeof(hint), pos, s_raw_crash_data.other_backtrace, + s_raw_crash_data.other_backtrace_count, s_raw_crash_data.other_reg_frame_count); + ESP_LOGE(TAG, "%s", hint); + } +#endif } } // namespace esphome::esp32 From 1ec21a22450393cfe777fc6f3923adaa085ff890 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:22:41 +1200 Subject: [PATCH 067/160] Bump version to 2026.8.0b4 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index d9421273af..2df6d3ded0 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.0b3 +PROJECT_NUMBER = 2026.8.0b4 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 1a8be98c03..73155e06ee 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.0b3" +__version__ = "2026.8.0b4" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 58d549ed4c53ddc72408a8e18f81c12a3648d30a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 19:56:37 -0700 Subject: [PATCH 068/160] [api] Move NoiseProtocolId off the connection object (#18420) --- .../components/api/api_frame_helper_noise.cpp | 25 +++++++++++-------- .../components/api/api_frame_helper_noise.h | 3 --- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 225bac51a6..09e3ca2b9e 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -591,18 +591,21 @@ APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) { */ APIError APINoiseFrameHelper::init_handshake_() { int err; - memset(&nid_, 0, sizeof(nid_)); - // const char *proto = "Noise_NNpsk0_25519_ChaChaPoly_SHA256"; - // err = noise_protocol_name_to_id(&nid_, proto, strlen(proto)); - nid_.pattern_id = NOISE_PATTERN_NN; - nid_.cipher_id = NOISE_CIPHER_CHACHAPOLY; - nid_.dh_id = NOISE_DH_CURVE25519; - nid_.prefix_id = NOISE_PREFIX_STANDARD; - nid_.hybrid_id = NOISE_DH_NONE; - nid_.hash_id = NOISE_HASH_SHA256; - nid_.modifier_ids[0] = NOISE_MODIFIER_PSK0; + // Noise_NNpsk0_25519_ChaChaPoly_SHA256, built on the stack: + // noise_handshakestate_new_by_id copies it, so a member would waste + // 104 bytes per connection, and a static const would sit in RAM on + // ESP8266 (.rodata is DRAM there). + const NoiseProtocolId nid = { + .prefix_id = NOISE_PREFIX_STANDARD, + .pattern_id = NOISE_PATTERN_NN, + .modifier_ids = {NOISE_MODIFIER_PSK0}, + .dh_id = NOISE_DH_CURVE25519, + .cipher_id = NOISE_CIPHER_CHACHAPOLY, + .hash_id = NOISE_HASH_SHA256, + .hybrid_id = NOISE_DH_NONE, + }; - err = noise_handshakestate_new_by_id(&handshake_, &nid_, NOISE_ROLE_RESPONDER); + err = noise_handshakestate_new_by_id(&handshake_, &nid, NOISE_ROLE_RESPONDER); APIError aerr = handle_noise_error_(err, LOG_STR("noise_handshakestate_new_by_id"), APIError::HANDSHAKESTATE_SETUP_FAILED); if (aerr != APIError::OK) diff --git a/esphome/components/api/api_frame_helper_noise.h b/esphome/components/api/api_frame_helper_noise.h index b0ba9fd01c..46bd366672 100644 --- a/esphome/components/api/api_frame_helper_noise.h +++ b/esphome/components/api/api_frame_helper_noise.h @@ -63,9 +63,6 @@ class APINoiseFrameHelper final : public APIFrameHelper { // Buffer for noise handshake prologue (released after handshake) APIBuffer prologue_; - // NoiseProtocolId (size depends on implementation) - NoiseProtocolId nid_; - // Group small types together // Fixed-size header buffer for noise protocol: // 1 byte for indicator + 2 bytes for message size (16-bit value, not varint) From cf764740cf8c186907edb09354df0e4d95750f1c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 19:56:58 -0700 Subject: [PATCH 069/160] [api] Create the camera image reader lazily (#18421) --- esphome/components/api/api_connection.cpp | 26 +++--- esphome/components/camera/camera.h | 3 +- tests/integration/fixtures/camera_mock.yaml | 19 +++++ .../mock_camera/__init__.py | 28 +++++++ .../mock_camera/mock_camera.cpp | 30 +++++++ .../mock_camera/mock_camera.h | 80 +++++++++++++++++++ tests/integration/test_camera_mock.py | 73 +++++++++++++++++ 7 files changed, 245 insertions(+), 14 deletions(-) create mode 100644 tests/integration/fixtures/camera_mock.yaml create mode 100644 tests/integration/fixtures/external_components/mock_camera/__init__.py create mode 100644 tests/integration/fixtures/external_components/mock_camera/mock_camera.cpp create mode 100644 tests/integration/fixtures/external_components/mock_camera/mock_camera.h create mode 100644 tests/integration/test_camera_mock.py diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 73b4f3e5bd..2eb8c21c73 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -160,11 +160,6 @@ APIConnection::APIConnection(std::unique_ptr sock, APIServer *pa #else #error "No frame helper defined" #endif -#ifdef USE_CAMERA - if (camera::Camera::instance() != nullptr) { - this->image_reader_ = std::unique_ptr{camera::Camera::instance()->create_image_reader()}; - } -#endif } void APIConnection::start() { @@ -1140,6 +1135,7 @@ void APIConnection::try_send_camera_image_() { if (!this->image_reader_) return; + const auto *cam = camera::Camera::instance(); // Send as many chunks as possible without blocking while (this->image_reader_->available()) { if (!this->helper_->can_write_without_blocking()) @@ -1149,11 +1145,11 @@ void APIConnection::try_send_camera_image_() { bool done = this->image_reader_->available() == to_send; CameraImageResponse msg; - msg.key = camera::Camera::instance()->get_object_id_hash(); + msg.key = cam->get_object_id_hash(); msg.set_data(this->image_reader_->peek_data_buffer(), to_send); msg.done = done; #ifdef USE_DEVICES - msg.device_id = camera::Camera::instance()->get_device_id(); + msg.device_id = cam->get_device_id(); #endif if (!this->send_message(msg)) { @@ -1169,15 +1165,19 @@ void APIConnection::try_send_camera_image_() { void APIConnection::set_camera_state(std::shared_ptr image) { if (!this->flags_.state_subscription) return; - if (!this->image_reader_) + if (this->image_reader_ && this->image_reader_->available()) return; - if (this->image_reader_->available()) + if (!image->was_requested_by(esphome::camera::API_REQUESTER) && !image->was_requested_by(esphome::camera::IDLE)) return; - if (image->was_requested_by(esphome::camera::API_REQUESTER) || image->was_requested_by(esphome::camera::IDLE)) { - this->image_reader_->set_image(std::move(image)); - // Try to send immediately to reduce latency - this->try_send_camera_image_(); + if (!this->image_reader_) { + // Created on the first image this connection will send, so connections + // that never receive one never pay for a reader. Only a registered + // camera's listener can reach this, so instance() is non-null here. + this->image_reader_ = std::unique_ptr{camera::Camera::instance()->create_image_reader()}; } + this->image_reader_->set_image(std::move(image)); + // Try to send immediately to reduce latency + this->try_send_camera_image_(); } uint16_t APIConnection::try_send_camera_info(EntityBase *entity, APIConnection *conn, uint32_t remaining_size) { auto *camera = static_cast(entity); diff --git a/esphome/components/camera/camera.h b/esphome/components/camera/camera.h index bf80b42e54..433361d298 100644 --- a/esphome/components/camera/camera.h +++ b/esphome/components/camera/camera.h @@ -103,7 +103,8 @@ struct CameraImageSpec { /** Abstract camera base class. Collaborates with API. * 1) API server starts and registers as a listener (add_listener) * to receive new images from the camera. - * 2) New API client connects and creates a new image reader (create_image_reader). + * 2) API connection creates an image reader (create_image_reader) when it receives + * the first image it will send. * 3) API connection receives protobuf CameraImageRequest and calls request_image. * 3.a) API connection receives protobuf CameraImageRequest and calls start_stream. * 4) Camera implementation provides JPEG data in the CameraImage and notifies listeners. diff --git a/tests/integration/fixtures/camera_mock.yaml b/tests/integration/fixtures/camera_mock.yaml new file mode 100644 index 0000000000..fa354d341f --- /dev/null +++ b/tests/integration/fixtures/camera_mock.yaml @@ -0,0 +1,19 @@ +esphome: + name: camera-mock-test + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +mock_camera: + name: Mock Camera + # Larger than MAX_BATCH_PACKET_SIZE (1390) so the image is split across + # multiple CameraImageResponse chunks and the client must reassemble. + # Must match IMAGE_SIZE in test_camera_mock.py. + image_size: 4096 diff --git a/tests/integration/fixtures/external_components/mock_camera/__init__.py b/tests/integration/fixtures/external_components/mock_camera/__init__.py new file mode 100644 index 0000000000..57aaf07ab9 --- /dev/null +++ b/tests/integration/fixtures/external_components/mock_camera/__init__.py @@ -0,0 +1,28 @@ +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_ID +from esphome.core.entity_helpers import setup_entity +from esphome.types import ConfigType + +CODEOWNERS = ["@esphome/tests"] +AUTO_LOAD = ["camera"] + +CONF_IMAGE_SIZE = "image_size" + +mock_camera_ns = cg.esphome_ns.namespace("mock_camera") +MockCamera = mock_camera_ns.class_("MockCamera", cg.Component, cg.EntityBase) + +CONFIG_SCHEMA = cv.ENTITY_BASE_SCHEMA.extend( + { + cv.GenerateID(): cv.declare_id(MockCamera), + cv.Optional(CONF_IMAGE_SIZE, default=1024): cv.positive_not_null_int, + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config: ConfigType) -> None: + cg.add_define("USE_CAMERA") + var = cg.new_Pvariable(config[CONF_ID]) + await setup_entity(var, config, "camera") + await cg.register_component(var, config) + cg.add(var.set_image_size(config[CONF_IMAGE_SIZE])) diff --git a/tests/integration/fixtures/external_components/mock_camera/mock_camera.cpp b/tests/integration/fixtures/external_components/mock_camera/mock_camera.cpp new file mode 100644 index 0000000000..64ed6bfe5c --- /dev/null +++ b/tests/integration/fixtures/external_components/mock_camera/mock_camera.cpp @@ -0,0 +1,30 @@ +#include "mock_camera.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +namespace esphome::mock_camera { + +static const char *const TAG = "mock_camera"; + +void MockCamera::loop() { + uint8_t requesters = this->single_requesters_ | this->stream_requesters_; + if (requesters == 0) + return; + uint32_t now = App.get_loop_component_start_time(); + if (now - this->last_frame_ms_ < FRAME_INTERVAL_MS) + return; + this->last_frame_ms_ = now; + this->single_requesters_ = 0; + + auto image = std::make_shared(this->image_size_, this->frame_counter_, requesters); + ESP_LOGV(TAG, "Producing frame %u (%u bytes, requesters 0x%02X)", this->frame_counter_, this->image_size_, + requesters); + this->frame_counter_++; + for (auto *listener : this->listeners_) { + listener->on_camera_image(image); + } +} + +void MockCamera::dump_config() { ESP_LOGCONFIG(TAG, "Mock Camera (%u byte frames)", this->image_size_); } + +} // namespace esphome::mock_camera diff --git a/tests/integration/fixtures/external_components/mock_camera/mock_camera.h b/tests/integration/fixtures/external_components/mock_camera/mock_camera.h new file mode 100644 index 0000000000..bcf40bba67 --- /dev/null +++ b/tests/integration/fixtures/external_components/mock_camera/mock_camera.h @@ -0,0 +1,80 @@ +#pragma once + +#include "esphome/components/camera/camera.h" +#include "esphome/core/component.h" + +#include +#include + +namespace esphome::mock_camera { + +/** Deterministic in-memory camera image. + * Byte i of frame N is (N + i) & 0xFF so tests can validate + * reassembled data from just the first byte. + */ +class MockCameraImage : public camera::CameraImage { + public: + MockCameraImage(size_t size, uint8_t frame_counter, uint8_t requesters) + : data_(new uint8_t[size]), size_(size), requesters_(requesters) { + for (size_t i = 0; i < size; i++) { + this->data_[i] = static_cast(frame_counter + i); + } + } + uint8_t *get_data_buffer() override { return this->data_.get(); } + size_t get_data_length() override { return this->size_; } + bool was_requested_by(camera::CameraRequester requester) const override { + return (this->requesters_ & (1 << requester)) != 0; + } + + protected: + std::unique_ptr data_; + size_t size_; + uint8_t requesters_; +}; + +class MockCameraImageReader : public camera::CameraImageReader { + public: + void set_image(std::shared_ptr image) override { + this->image_ = std::move(image); + this->offset_ = 0; + } + size_t available() const override { return this->image_ ? this->image_->get_data_length() - this->offset_ : 0; } + uint8_t *peek_data_buffer() override { return this->image_->get_data_buffer() + this->offset_; } + void consume_data(size_t consumed) override { this->offset_ += consumed; } + void return_image() override { + this->image_.reset(); + this->offset_ = 0; + } + + protected: + std::shared_ptr image_; + size_t offset_{0}; +}; + +/** Virtual camera producing deterministic frames on request or stream. */ +class MockCamera : public camera::Camera { + public: + void loop() override; + void dump_config() override; + + void add_listener(camera::CameraListener *listener) override { this->listeners_.push_back(listener); } + camera::CameraImageReader *create_image_reader() override { return new MockCameraImageReader(); } + void request_image(camera::CameraRequester requester) override { this->single_requesters_ |= (1 << requester); } + void start_stream(camera::CameraRequester requester) override { this->stream_requesters_ |= (1 << requester); } + void stop_stream(camera::CameraRequester requester) override { this->stream_requesters_ &= ~(1 << requester); } + + void set_image_size(uint32_t size) { this->image_size_ = size; } + + protected: + static constexpr uint32_t FRAME_INTERVAL_MS = 50; + + // Members ordered largest to smallest to minimize padding + std::vector listeners_; + uint32_t image_size_{1024}; + uint32_t last_frame_ms_{0}; + uint8_t frame_counter_{0}; + uint8_t single_requesters_{0}; + uint8_t stream_requesters_{0}; +}; + +} // namespace esphome::mock_camera diff --git a/tests/integration/test_camera_mock.py b/tests/integration/test_camera_mock.py new file mode 100644 index 0000000000..6819d7a6d4 --- /dev/null +++ b/tests/integration/test_camera_mock.py @@ -0,0 +1,73 @@ +"""Integration test for the camera API flow using a mock camera platform.""" + +from __future__ import annotations + +import asyncio + +from aioesphomeapi import CameraInfo, CameraState, EntityState +import pytest + +from .state_utils import require_entity +from .types import APIClientConnectedFactory, RunCompiledFunction + +# Must match image_size in fixtures/camera_mock.yaml +IMAGE_SIZE = 4096 +STREAM_FRAMES = 3 + + +def _verify_frame(data: bytes) -> int: + """Verify the deterministic frame pattern and return the frame counter.""" + assert len(data) == IMAGE_SIZE, f"expected {IMAGE_SIZE} bytes, got {len(data)}" + counter = data[0] + assert data == bytes((counter + i) & 0xFF for i in range(IMAGE_SIZE)), ( + "frame pattern mismatch" + ) + return counter + + +@pytest.mark.asyncio +async def test_camera_mock( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Single-image and stream requests deliver reassembled deterministic frames.""" + async with run_compiled(yaml_config), api_client_connected() as client: + entities, _ = await client.list_entities_services() + camera = require_entity(entities, "mock_camera", CameraInfo) + + loop = asyncio.get_running_loop() + images: list[bytes] = [] + single_image: asyncio.Future[None] = loop.create_future() + stream_done: asyncio.Future[None] = loop.create_future() + + def on_state(state: EntityState) -> None: + if not (isinstance(state, CameraState) and state.key == camera.key): + return + images.append(bytes(state.data)) + if not single_image.done(): + single_image.set_result(None) + elif len(images) >= STREAM_FRAMES and not stream_done.done(): + stream_done.set_result(None) + + client.subscribe_states(on_state) + + # Single image request: one complete frame arrives, reassembled + # from multiple chunks (4096 > 1390 byte packets) + client.request_single_image() + await asyncio.wait_for(single_image, timeout=10) + first_counter = _verify_frame(images[0]) + + # Stream request: multiple consecutive frames arrive + images.clear() + client.request_image_stream() + await asyncio.wait_for(stream_done, timeout=10) + + # Frames are distinct, ordered, and fresh per the mock's counter. + # Not exactly consecutive: the API drops frames by design while the + # previous image is still being sent, so allow small gaps. + counters = [_verify_frame(img) for img in images[:STREAM_FRAMES]] + for prev, cur in zip(counters, counters[1:], strict=False): + assert cur != prev, f"duplicate frames: {counters}" + assert ((cur - prev) & 0xFF) < 16, f"frames out of order: {counters}" + assert counters[0] != first_counter, "stream should produce new frames" From e1c279718fafe3884101efdbff3a9d1a1d5ed529 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 19:57:12 -0700 Subject: [PATCH 070/160] [ld2420] Drop the setup priority override so setup runs after the UART bus (#18428) --- esphome/components/ld2420/ld2420.cpp | 2 -- esphome/components/ld2420/ld2420.h | 1 - 2 files changed, 3 deletions(-) diff --git a/esphome/components/ld2420/ld2420.cpp b/esphome/components/ld2420/ld2420.cpp index f71bec7e5f..4aa00f8fd4 100644 --- a/esphome/components/ld2420/ld2420.cpp +++ b/esphome/components/ld2420/ld2420.cpp @@ -184,8 +184,6 @@ static int32_t get_firmware_int(const char *version_string) { return result; } -float LD2420Component::get_setup_priority() const { return setup_priority::BUS; } - void LD2420Component::dump_config() { ESP_LOGCONFIG(TAG, "LD2420:\n" diff --git a/esphome/components/ld2420/ld2420.h b/esphome/components/ld2420/ld2420.h index 977ee2eccc..e13d0271e1 100644 --- a/esphome/components/ld2420/ld2420.h +++ b/esphome/components/ld2420/ld2420.h @@ -105,7 +105,6 @@ class LD2420Component final : public Component, public uart::UARTDevice { void apply_config_action(); void factory_reset_action(); void revert_config_action(); - float get_setup_priority() const override; int send_cmd_from_array(CmdFrameT cmd_frame); void report_gate_data(); void handle_cmd_error(uint16_t error); From ebb0923362601879742a870d39e114b2279258cc Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:34:00 -0500 Subject: [PATCH 071/160] Bump aioesphomeapi from 45.10.2 to 45.10.3 (#18433) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 876b13793c..61011f2fbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.10.2 +aioesphomeapi==45.10.3 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 07e8b303b9a4f588285795b841c8ae7061d31712 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:52:56 -0700 Subject: [PATCH 072/160] [ota] Shorten platform backend TAG strings (#18438) --- esphome/components/ota/ota_backend_arduino_libretiny.cpp | 2 +- esphome/components/ota/ota_backend_arduino_rp2.cpp | 2 +- esphome/components/ota/ota_backend_esp8266.cpp | 2 +- esphome/components/ota/ota_backend_esp_idf.cpp | 2 +- esphome/components/ota/ota_backend_host.cpp | 2 +- esphome/components/ota/ota_bootloader_esp_idf.cpp | 2 +- esphome/components/ota/ota_partitions_esp_idf.cpp | 2 +- esphome/components/ota/ota_signature_esp_idf.cpp | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/esphome/components/ota/ota_backend_arduino_libretiny.cpp b/esphome/components/ota/ota_backend_arduino_libretiny.cpp index 4cc99202a7..231c4d2dd2 100644 --- a/esphome/components/ota/ota_backend_arduino_libretiny.cpp +++ b/esphome/components/ota/ota_backend_arduino_libretiny.cpp @@ -9,7 +9,7 @@ namespace esphome::ota { -static const char *const TAG = "ota.arduino_libretiny"; +static const char *const TAG = "ota"; std::unique_ptr make_ota_backend() { return make_unique(); } diff --git a/esphome/components/ota/ota_backend_arduino_rp2.cpp b/esphome/components/ota/ota_backend_arduino_rp2.cpp index b35eb38c12..48725b1265 100644 --- a/esphome/components/ota/ota_backend_arduino_rp2.cpp +++ b/esphome/components/ota/ota_backend_arduino_rp2.cpp @@ -11,7 +11,7 @@ namespace esphome::ota { -static const char *const TAG = "ota.arduino_rp2"; +static const char *const TAG = "ota"; std::unique_ptr make_ota_backend() { return make_unique(); } diff --git a/esphome/components/ota/ota_backend_esp8266.cpp b/esphome/components/ota/ota_backend_esp8266.cpp index 6a678fb419..2a6a9e08b1 100644 --- a/esphome/components/ota/ota_backend_esp8266.cpp +++ b/esphome/components/ota/ota_backend_esp8266.cpp @@ -46,7 +46,7 @@ static constexpr size_t MIN_BUFFER_SIZE = 256; namespace esphome::ota { -static const char *const TAG = "ota.esp8266"; +static const char *const TAG = "ota"; std::unique_ptr make_ota_backend() { return make_unique(); } diff --git a/esphome/components/ota/ota_backend_esp_idf.cpp b/esphome/components/ota/ota_backend_esp_idf.cpp index 108605e4c9..eb23ad82dd 100644 --- a/esphome/components/ota/ota_backend_esp_idf.cpp +++ b/esphome/components/ota/ota_backend_esp_idf.cpp @@ -15,7 +15,7 @@ namespace esphome::ota { -static const char *const TAG = "ota.idf"; +static const char *const TAG = "ota"; std::unique_ptr make_ota_backend() { return make_unique(); } diff --git a/esphome/components/ota/ota_backend_host.cpp b/esphome/components/ota/ota_backend_host.cpp index ee503a49e1..89e3f99e1e 100644 --- a/esphome/components/ota/ota_backend_host.cpp +++ b/esphome/components/ota/ota_backend_host.cpp @@ -27,7 +27,7 @@ namespace esphome::ota { namespace { -const char *const TAG = "ota.host"; +const char *const TAG = "ota"; constexpr size_t MAX_OTA_SIZE = 256u * 1024u * 1024u; // 256 MiB constexpr size_t HEADER_PEEK_SIZE = 64; diff --git a/esphome/components/ota/ota_bootloader_esp_idf.cpp b/esphome/components/ota/ota_bootloader_esp_idf.cpp index 264218a3df..57b5529350 100644 --- a/esphome/components/ota/ota_bootloader_esp_idf.cpp +++ b/esphome/components/ota/ota_bootloader_esp_idf.cpp @@ -11,7 +11,7 @@ namespace esphome::ota { -static const char *const TAG = "ota.idf"; +static const char *const TAG = "ota"; OTAResponseTypes IDFOTABackend::register_and_validate_bootloader_part_() { // Register the bootloader partition diff --git a/esphome/components/ota/ota_partitions_esp_idf.cpp b/esphome/components/ota/ota_partitions_esp_idf.cpp index a7fc709313..d2b1196de6 100644 --- a/esphome/components/ota/ota_partitions_esp_idf.cpp +++ b/esphome/components/ota/ota_partitions_esp_idf.cpp @@ -16,7 +16,7 @@ namespace esphome::ota { -static const char *const TAG = "ota.idf"; +static const char *const TAG = "ota"; static inline bool check_overlap(uint32_t a_offset, size_t a_size, uint32_t b_offset, size_t b_size) { return (a_offset + a_size > b_offset && b_offset + b_size > a_offset); diff --git a/esphome/components/ota/ota_signature_esp_idf.cpp b/esphome/components/ota/ota_signature_esp_idf.cpp index b327988d2d..71dcc0eb83 100644 --- a/esphome/components/ota/ota_signature_esp_idf.cpp +++ b/esphome/components/ota/ota_signature_esp_idf.cpp @@ -31,7 +31,7 @@ namespace esphome::ota { -static const char *const TAG = "ota.idf"; +static const char *const TAG = "ota"; // Route the "Signature check: " prefix (and its per-block form) through one // shared format string each, so the prefix is pooled once by the linker instead From c01f24553c129327ef591cb9e7198369918663b5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:53:29 -0700 Subject: [PATCH 073/160] [uart] Shorten platform backend TAG strings (#18439) --- esphome/components/uart/uart_component_esp8266.cpp | 2 +- esphome/components/uart/uart_component_esp_idf.cpp | 2 +- esphome/components/uart/uart_component_host.cpp | 2 +- esphome/components/uart/uart_component_libretiny.cpp | 2 +- esphome/components/uart/uart_component_rp2.cpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/esphome/components/uart/uart_component_esp8266.cpp b/esphome/components/uart/uart_component_esp8266.cpp index fc1509f737..2f8b4dbd11 100644 --- a/esphome/components/uart/uart_component_esp8266.cpp +++ b/esphome/components/uart/uart_component_esp8266.cpp @@ -14,7 +14,7 @@ namespace esphome::uart { -static const char *const TAG = "uart.arduino_esp8266"; +static const char *const TAG = "uart"; bool ESP8266UartComponent::serial0_in_use = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) uint32_t ESP8266UartComponent::get_config() { diff --git a/esphome/components/uart/uart_component_esp_idf.cpp b/esphome/components/uart/uart_component_esp_idf.cpp index 93e43e0372..a61339feb4 100644 --- a/esphome/components/uart/uart_component_esp_idf.cpp +++ b/esphome/components/uart/uart_component_esp_idf.cpp @@ -21,7 +21,7 @@ namespace esphome::uart { -static const char *const TAG = "uart.idf"; +static const char *const TAG = "uart"; /// Check if a pin number matches one of the default UART0 GPIO pins. /// These pins may have residual IOMUX state from the ROM bootloader that diff --git a/esphome/components/uart/uart_component_host.cpp b/esphome/components/uart/uart_component_host.cpp index 5bb7a49726..63b5631564 100644 --- a/esphome/components/uart/uart_component_host.cpp +++ b/esphome/components/uart/uart_component_host.cpp @@ -98,7 +98,7 @@ speed_t get_baud(int baud) { namespace esphome::uart { -static const char *const TAG = "uart.host"; +static const char *const TAG = "uart"; HostUartComponent::~HostUartComponent() { if (this->file_descriptor_ != -1) { diff --git a/esphome/components/uart/uart_component_libretiny.cpp b/esphome/components/uart/uart_component_libretiny.cpp index fbf0c20ded..4eacd980db 100644 --- a/esphome/components/uart/uart_component_libretiny.cpp +++ b/esphome/components/uart/uart_component_libretiny.cpp @@ -16,7 +16,7 @@ namespace esphome::uart { -static const char *const TAG = "uart.lt"; +static const char *const TAG = "uart"; static const char *const UART_TYPE[] = { "hardware", diff --git a/esphome/components/uart/uart_component_rp2.cpp b/esphome/components/uart/uart_component_rp2.cpp index 9cc3009a22..ffb9bc0f2d 100644 --- a/esphome/components/uart/uart_component_rp2.cpp +++ b/esphome/components/uart/uart_component_rp2.cpp @@ -13,7 +13,7 @@ namespace esphome::uart { -static const char *const TAG = "uart.arduino_rp2"; +static const char *const TAG = "uart"; uint16_t RP2UartComponent::get_config() { uint16_t config = 0; From d1a7b8df8b616cd49affa5a70298fbdfce07d6be Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:53:42 -0700 Subject: [PATCH 074/160] [adc] Shorten platform TAG strings (#18441) --- esphome/components/adc/adc_sensor_common.cpp | 2 +- esphome/components/adc/adc_sensor_esp32.cpp | 2 +- esphome/components/adc/adc_sensor_esp8266.cpp | 2 +- esphome/components/adc/adc_sensor_libretiny.cpp | 2 +- esphome/components/adc/adc_sensor_rp2.cpp | 2 +- esphome/components/adc/adc_sensor_zephyr.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/components/adc/adc_sensor_common.cpp b/esphome/components/adc/adc_sensor_common.cpp index 16c86aee18..5ca58df10e 100644 --- a/esphome/components/adc/adc_sensor_common.cpp +++ b/esphome/components/adc/adc_sensor_common.cpp @@ -3,7 +3,7 @@ namespace esphome::adc { -static const char *const TAG = "adc.common"; +static const char *const TAG = "adc"; const LogString *sampling_mode_to_str(SamplingMode mode) { switch (mode) { diff --git a/esphome/components/adc/adc_sensor_esp32.cpp b/esphome/components/adc/adc_sensor_esp32.cpp index a761b37749..a0f7a1ed08 100644 --- a/esphome/components/adc/adc_sensor_esp32.cpp +++ b/esphome/components/adc/adc_sensor_esp32.cpp @@ -6,7 +6,7 @@ namespace esphome::adc { -static const char *const TAG = "adc.esp32"; +static const char *const TAG = "adc"; adc_oneshot_unit_handle_t ADCSensor::shared_adc_handles[2] = {nullptr, nullptr}; diff --git a/esphome/components/adc/adc_sensor_esp8266.cpp b/esphome/components/adc/adc_sensor_esp8266.cpp index e4f2f82f08..77a192e025 100644 --- a/esphome/components/adc/adc_sensor_esp8266.cpp +++ b/esphome/components/adc/adc_sensor_esp8266.cpp @@ -13,7 +13,7 @@ ADC_MODE(ADC_VCC) namespace esphome::adc { -static const char *const TAG = "adc.esp8266"; +static const char *const TAG = "adc"; void ADCSensor::setup() { #ifndef USE_ADC_SENSOR_VCC diff --git a/esphome/components/adc/adc_sensor_libretiny.cpp b/esphome/components/adc/adc_sensor_libretiny.cpp index d9b9f50be1..dfa545b395 100644 --- a/esphome/components/adc/adc_sensor_libretiny.cpp +++ b/esphome/components/adc/adc_sensor_libretiny.cpp @@ -5,7 +5,7 @@ namespace esphome::adc { -static const char *const TAG = "adc.libretiny"; +static const char *const TAG = "adc"; void ADCSensor::setup() { #ifndef USE_ADC_SENSOR_VCC diff --git a/esphome/components/adc/adc_sensor_rp2.cpp b/esphome/components/adc/adc_sensor_rp2.cpp index 8652a46029..ce665e8501 100644 --- a/esphome/components/adc/adc_sensor_rp2.cpp +++ b/esphome/components/adc/adc_sensor_rp2.cpp @@ -17,7 +17,7 @@ namespace esphome::adc { -static const char *const TAG = "adc.rp2"; +static const char *const TAG = "adc"; // The on-die temperature sensor sits on the last ADC channel: input 4 on RP2040 // and RP2350A, but input 8 on RP2350B, which has eight external channels rather diff --git a/esphome/components/adc/adc_sensor_zephyr.cpp b/esphome/components/adc/adc_sensor_zephyr.cpp index c3632b00e2..bf45059740 100644 --- a/esphome/components/adc/adc_sensor_zephyr.cpp +++ b/esphome/components/adc/adc_sensor_zephyr.cpp @@ -7,7 +7,7 @@ namespace esphome::adc { -static const char *const TAG = "adc.zephyr"; +static const char *const TAG = "adc"; void ADCSensor::setup() { if (!adc_is_ready_dt(this->channel_)) { From 37bea1c1538c830691e95bcce398e816069f7556 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:53:48 -0700 Subject: [PATCH 075/160] [spi] Shorten platform backend TAG strings (#18442) --- esphome/components/spi/spi_arduino.cpp | 2 +- esphome/components/spi/spi_esp_idf.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/spi/spi_arduino.cpp b/esphome/components/spi/spi_arduino.cpp index a3e09d2800..14428bed62 100644 --- a/esphome/components/spi/spi_arduino.cpp +++ b/esphome/components/spi/spi_arduino.cpp @@ -4,7 +4,7 @@ namespace esphome::spi { #if defined(USE_ARDUINO) && !defined(USE_ESP32) -static const char *const TAG = "spi-esp-arduino"; +static const char *const TAG = "spi"; class SPIDelegateHw : public SPIDelegate { public: SPIDelegateHw(SPIInterface channel, uint32_t data_rate, SPIBitOrder bit_order, SPIMode mode, GPIOPin *cs_pin) diff --git a/esphome/components/spi/spi_esp_idf.cpp b/esphome/components/spi/spi_esp_idf.cpp index 0731078eec..d5d5053117 100644 --- a/esphome/components/spi/spi_esp_idf.cpp +++ b/esphome/components/spi/spi_esp_idf.cpp @@ -4,7 +4,7 @@ namespace esphome::spi { #ifdef USE_ESP32 -static const char *const TAG = "spi-esp-idf"; +static const char *const TAG = "spi"; static const size_t MAX_TRANSFER_SIZE = 4092; // dictated by ESP-IDF API. class SPIDelegateHw : public SPIDelegate { From 47a58dd7991affd47b61df0cd491076d77ceff18 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:53:56 -0700 Subject: [PATCH 076/160] [internal_temperature] Shorten platform TAG strings (#18443) --- .../internal_temperature/internal_temperature_bk72xx.cpp | 2 +- .../internal_temperature/internal_temperature_esp32.cpp | 2 +- .../internal_temperature/internal_temperature_rp2.cpp | 2 +- .../internal_temperature/internal_temperature_zephyr.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/internal_temperature/internal_temperature_bk72xx.cpp b/esphome/components/internal_temperature/internal_temperature_bk72xx.cpp index b7332ee81f..91f47d831f 100644 --- a/esphome/components/internal_temperature/internal_temperature_bk72xx.cpp +++ b/esphome/components/internal_temperature/internal_temperature_bk72xx.cpp @@ -9,7 +9,7 @@ uint32_t temp_single_get_current_temperature(uint32_t *temp_value); namespace esphome::internal_temperature { -static const char *const TAG = "internal_temperature.bk72xx"; +static const char *const TAG = "internal_temperature"; void InternalTemperatureSensor::update() { float temperature = NAN; diff --git a/esphome/components/internal_temperature/internal_temperature_esp32.cpp b/esphome/components/internal_temperature/internal_temperature_esp32.cpp index 64fe3707b1..2c6fda2af4 100644 --- a/esphome/components/internal_temperature/internal_temperature_esp32.cpp +++ b/esphome/components/internal_temperature/internal_temperature_esp32.cpp @@ -16,7 +16,7 @@ uint8_t temprature_sens_read(); namespace esphome::internal_temperature { -static const char *const TAG = "internal_temperature.esp32"; +static const char *const TAG = "internal_temperature"; void InternalTemperatureSensor::update() { float temperature = NAN; diff --git a/esphome/components/internal_temperature/internal_temperature_rp2.cpp b/esphome/components/internal_temperature/internal_temperature_rp2.cpp index 2e408b3b01..c4ab33b0a5 100644 --- a/esphome/components/internal_temperature/internal_temperature_rp2.cpp +++ b/esphome/components/internal_temperature/internal_temperature_rp2.cpp @@ -16,7 +16,7 @@ namespace esphome::internal_temperature { -static const char *const TAG = "internal_temperature.rp2"; +static const char *const TAG = "internal_temperature"; // The on-die temperature sensor sits on the last ADC channel: input 4 on RP2040 // and RP2350A, but input 8 on RP2350B, which has eight external channels rather diff --git a/esphome/components/internal_temperature/internal_temperature_zephyr.cpp b/esphome/components/internal_temperature/internal_temperature_zephyr.cpp index be72ab6f51..50c597f6f1 100644 --- a/esphome/components/internal_temperature/internal_temperature_zephyr.cpp +++ b/esphome/components/internal_temperature/internal_temperature_zephyr.cpp @@ -8,7 +8,7 @@ namespace esphome::internal_temperature { -static const char *const TAG = "internal_temperature.zephyr"; +static const char *const TAG = "internal_temperature"; static const struct device *const DIE_TEMPERATURE_SENSOR = DEVICE_DT_GET_ONE(nordic_nrf_temp); From e0d28d7f5c9128ca98436ec7d4a25cc5ebe91914 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:54:03 -0700 Subject: [PATCH 077/160] [http_request] Shorten platform backend TAG strings (#18444) --- esphome/components/http_request/http_request_arduino.cpp | 2 +- esphome/components/http_request/http_request_host.cpp | 2 +- esphome/components/http_request/http_request_idf.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/http_request/http_request_arduino.cpp b/esphome/components/http_request/http_request_arduino.cpp index 84333e7169..43ab2e5b53 100644 --- a/esphome/components/http_request/http_request_arduino.cpp +++ b/esphome/components/http_request/http_request_arduino.cpp @@ -16,7 +16,7 @@ namespace esphome::http_request { -static const char *const TAG = "http_request.arduino"; +static const char *const TAG = "http_request"; #ifdef USE_ESP8266 // ESP8266 Arduino core (WiFiClientSecureBearSSL.cpp) returns -1000 on OOM static constexpr int ESP8266_SSL_ERR_OOM = -1000; diff --git a/esphome/components/http_request/http_request_host.cpp b/esphome/components/http_request/http_request_host.cpp index 85c6e8b3c7..cf231e20bd 100644 --- a/esphome/components/http_request/http_request_host.cpp +++ b/esphome/components/http_request/http_request_host.cpp @@ -14,7 +14,7 @@ namespace esphome::http_request { -static const char *const TAG = "http_request.host"; +static const char *const TAG = "http_request"; std::shared_ptr HttpRequestHost::perform(const std::string &url, const std::string &method, const std::string &body, diff --git a/esphome/components/http_request/http_request_idf.cpp b/esphome/components/http_request/http_request_idf.cpp index a437540241..ddff954950 100644 --- a/esphome/components/http_request/http_request_idf.cpp +++ b/esphome/components/http_request/http_request_idf.cpp @@ -16,7 +16,7 @@ namespace esphome::http_request { -static const char *const TAG = "http_request.idf"; +static const char *const TAG = "http_request"; static constexpr uint32_t ERROR_DURATION_MS = 1000; void HttpRequestIDF::dump_config() { From 1f000ba66899be59be0aaa655692757111c8f435 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:54:17 -0700 Subject: [PATCH 078/160] [mqtt] Shorten esp32 backend TAG string (#18446) --- esphome/components/mqtt/mqtt_backend_esp32.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/mqtt/mqtt_backend_esp32.cpp b/esphome/components/mqtt/mqtt_backend_esp32.cpp index 499a330730..09eb5f97dc 100644 --- a/esphome/components/mqtt/mqtt_backend_esp32.cpp +++ b/esphome/components/mqtt/mqtt_backend_esp32.cpp @@ -10,7 +10,7 @@ namespace esphome::mqtt { -static const char *const TAG = "mqtt.idf"; +static const char *const TAG = "mqtt"; bool MQTTBackendESP32::initialize_() { mqtt_cfg_.broker.address.hostname = this->host_.c_str(); From 1f4fcead38d9897e37c6c3ec059c86988408e5b5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:54:29 -0700 Subject: [PATCH 079/160] [nextion] Shorten upload TAG strings (#18448) --- esphome/components/nextion/nextion_upload_arduino.cpp | 2 +- esphome/components/nextion/nextion_upload_esp32.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/nextion/nextion_upload_arduino.cpp b/esphome/components/nextion/nextion_upload_arduino.cpp index 2f3377d950..f02f32d5ca 100644 --- a/esphome/components/nextion/nextion_upload_arduino.cpp +++ b/esphome/components/nextion/nextion_upload_arduino.cpp @@ -13,7 +13,7 @@ namespace esphome::nextion { -static const char *const TAG = "nextion.upload.arduino"; +static const char *const TAG = "nextion.upload"; static constexpr size_t NEXTION_MAX_RESPONSE_LOG_BYTES = 16; // Timeout for display acknowledgment during TFT upload (ms). diff --git a/esphome/components/nextion/nextion_upload_esp32.cpp b/esphome/components/nextion/nextion_upload_esp32.cpp index e2d5ae8ad7..c4dc74b5d3 100644 --- a/esphome/components/nextion/nextion_upload_esp32.cpp +++ b/esphome/components/nextion/nextion_upload_esp32.cpp @@ -16,7 +16,7 @@ namespace esphome::nextion { -static const char *const TAG = "nextion.upload.esp32"; +static const char *const TAG = "nextion.upload"; static constexpr size_t NEXTION_MAX_RESPONSE_LOG_BYTES = 16; // Timeout for display acknowledgment during TFT upload (ms). From ebe93e2c684c46ea960262c05a0914b5f6bda61e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:54:35 -0700 Subject: [PATCH 080/160] [bluetooth_connection] Shorten platform TAG strings (#18449) --- .../bluetooth_connection/bluetooth_connection_bluedroid.cpp | 2 +- .../bluetooth_connection/bluetooth_connection_rp2.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp index 076c77b18e..15f854239d 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp @@ -20,7 +20,7 @@ namespace esphome::bluetooth_connection { -static const char *const TAG = "bluetooth_connection.bluedroid"; +static const char *const TAG = "bluetooth_connection"; using ble_device_base::FAST_CONN_TIMEOUT; using ble_device_base::FAST_MAX_CONN_INTERVAL; diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_rp2.cpp b/esphome/components/bluetooth_connection/bluetooth_connection_rp2.cpp index 855c895196..16a89dcfdd 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_rp2.cpp +++ b/esphome/components/bluetooth_connection/bluetooth_connection_rp2.cpp @@ -15,7 +15,7 @@ namespace esphome::bluetooth_connection { -static const char *const TAG = "bluetooth_connection.rp2"; +static const char *const TAG = "bluetooth_connection"; using ble_device_base::ESPBTUUID; using ble_device_base::GATT_ERR_NOT_CONNECTED; From 3d9fecb56229ddefc7eaf246e23d4ed656f28f03 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:54:52 -0700 Subject: [PATCH 081/160] [remote_receiver] Shorten esp32 TAG string (#18447) --- esphome/components/remote_receiver/remote_receiver_rmt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/remote_receiver/remote_receiver_rmt.cpp b/esphome/components/remote_receiver/remote_receiver_rmt.cpp index 596608a4d0..632ca9763a 100644 --- a/esphome/components/remote_receiver/remote_receiver_rmt.cpp +++ b/esphome/components/remote_receiver/remote_receiver_rmt.cpp @@ -9,7 +9,7 @@ namespace esphome::remote_receiver { -static const char *const TAG = "remote_receiver.esp32"; +static const char *const TAG = "remote_receiver"; static bool IRAM_ATTR HOT rmt_callback(rmt_channel_handle_t channel, const rmt_rx_done_event_data_t *event, void *arg) { RemoteReceiverComponentStore *store = (RemoteReceiverComponentStore *) arg; From f6c7434b2abb0cba04ce08e669c14b380b9622bf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:54:55 -0700 Subject: [PATCH 082/160] [i2c] Shorten platform backend TAG strings (#18440) --- esphome/components/i2c/i2c_bus_arduino.cpp | 2 +- esphome/components/i2c/i2c_bus_esp_idf.cpp | 2 +- esphome/components/i2c/i2c_bus_host.cpp | 2 +- esphome/components/i2c/i2c_bus_zephyr.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/i2c/i2c_bus_arduino.cpp b/esphome/components/i2c/i2c_bus_arduino.cpp index cc036b12c3..39a6aec774 100644 --- a/esphome/components/i2c/i2c_bus_arduino.cpp +++ b/esphome/components/i2c/i2c_bus_arduino.cpp @@ -9,7 +9,7 @@ namespace esphome::i2c { -static const char *const TAG = "i2c.arduino"; +static const char *const TAG = "i2c"; // Maximum bytes to log in hex format (truncates larger transfers) static constexpr size_t I2C_MAX_LOG_BYTES = 32; diff --git a/esphome/components/i2c/i2c_bus_esp_idf.cpp b/esphome/components/i2c/i2c_bus_esp_idf.cpp index 4aca4f0fae..7ca9537e2d 100644 --- a/esphome/components/i2c/i2c_bus_esp_idf.cpp +++ b/esphome/components/i2c/i2c_bus_esp_idf.cpp @@ -12,7 +12,7 @@ namespace esphome::i2c { -static const char *const TAG = "i2c.idf"; +static const char *const TAG = "i2c"; // Maximum bytes to log in hex format (truncates larger transfers) static constexpr size_t I2C_MAX_LOG_BYTES = 32; diff --git a/esphome/components/i2c/i2c_bus_host.cpp b/esphome/components/i2c/i2c_bus_host.cpp index 17279fda50..303944636b 100644 --- a/esphome/components/i2c/i2c_bus_host.cpp +++ b/esphome/components/i2c/i2c_bus_host.cpp @@ -16,7 +16,7 @@ namespace esphome::i2c { -static const char *const TAG = "i2c.host"; +static const char *const TAG = "i2c"; HostI2CBus::~HostI2CBus() { if (this->file_descriptor_ != -1) { diff --git a/esphome/components/i2c/i2c_bus_zephyr.cpp b/esphome/components/i2c/i2c_bus_zephyr.cpp index 1eb9944dcb..ffdd2ba8bb 100644 --- a/esphome/components/i2c/i2c_bus_zephyr.cpp +++ b/esphome/components/i2c/i2c_bus_zephyr.cpp @@ -6,7 +6,7 @@ namespace esphome::i2c { -static const char *const TAG = "i2c.zephyr"; +static const char *const TAG = "i2c"; static const char *get_speed(uint32_t dev_config) { switch (I2C_SPEED_GET(dev_config)) { From 031a038b49318018ca1aeee08cc111c2aa5e9b9c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 09:56:37 -0700 Subject: [PATCH 083/160] [deep_sleep] Shorten bk72xx TAG string (#18445) --- esphome/components/deep_sleep/deep_sleep_bk72xx.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp index 73e0331c76..2c97dc3211 100644 --- a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp +++ b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp @@ -5,7 +5,7 @@ namespace esphome::deep_sleep { -static const char *const TAG = "deep_sleep.bk72xx"; +static const char *const TAG = "deep_sleep"; #ifdef USE_DEEP_SLEEP_ON_WAKE WakeupCause get_wakeup_cause() { From 6d20ebc66b309df4d413d316f369ffc48742f4cc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 10:18:43 -0700 Subject: [PATCH 084/160] [socket] Shorten lwip TAG string (#18450) --- esphome/components/socket/lwip_raw_tcp_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index 4fcec553fa..b80a394eec 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -43,7 +43,7 @@ namespace esphome::socket { // (Ethernet). On ESP8266, it's a no-op. #define LWIP_LOCK() esphome::LwIPLock lwip_lock_guard // NOLINT -static const char *const TAG = "socket.lwip"; +static const char *const TAG = "socket"; // set to 1 to enable verbose lwip logging #if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if) From 27483a4101e2098cefff3a5c7c56d0b1594b2506 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 11:15:31 -0700 Subject: [PATCH 085/160] [core] Retry gh CLI calls on transient network errors in CI scripts (#18292) --- script/ci_memory_impact_comment.py | 24 +++--- script/helpers.py | 91 +++++++++++++++++++++- tests/script/test_helpers.py | 121 +++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 15 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 0908b99595..33ca84d76c 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -20,17 +20,21 @@ from jinja2 import Environment, FileSystemLoader sys.path.insert(0, str(Path(__file__).parent.parent)) # pylint: disable=wrong-import-position +from helpers import run_gh_command # noqa: E402 # Comment marker to identify our memory impact comments COMMENT_MARKER = "" -def run_gh_command(args: list[str], operation: str) -> subprocess.CompletedProcess: - """Run a gh CLI command with error handling. +def run_gh_command_logged( + args: list[str], operation: str, *, retry: bool = True +) -> subprocess.CompletedProcess: + """Run a gh CLI command with retries and error reporting. Args: args: Command arguments (including 'gh') operation: Description of the operation for error messages + retry: Pass False for non-idempotent commands (see run_gh_command) Returns: CompletedProcess result @@ -39,12 +43,7 @@ def run_gh_command(args: list[str], operation: str) -> subprocess.CompletedProce subprocess.CalledProcessError: If command fails (with detailed error output) """ try: - return subprocess.run( - args, - check=True, - capture_output=True, - text=True, - ) + return run_gh_command(args, retry=retry) except subprocess.CalledProcessError as e: print( f"ERROR: {operation} failed with exit code {e.returncode}", file=sys.stderr @@ -472,7 +471,7 @@ def find_existing_comment(pr_number: str) -> str | None: print(f"DEBUG: Looking for existing comment on PR #{pr_number}", file=sys.stderr) # Use gh api to get comments directly - this returns the numeric id field - result = run_gh_command( + result = run_gh_command_logged( [ "gh", "api", @@ -535,7 +534,7 @@ def update_existing_comment(comment_id: str, comment_body: str) -> None: """ print(f"DEBUG: Updating existing comment {comment_id}", file=sys.stderr) print(f"DEBUG: Comment body length: {len(comment_body)} bytes", file=sys.stderr) - result = run_gh_command( + result = run_gh_command_logged( [ "gh", "api", @@ -562,9 +561,12 @@ def create_new_comment(pr_number: str, comment_body: str) -> None: """ print(f"DEBUG: Posting new comment on PR #{pr_number}", file=sys.stderr) print(f"DEBUG: Comment body length: {len(comment_body)} bytes", file=sys.stderr) - result = run_gh_command( + # Creating a comment is not idempotent: a retry after a dropped response + # could post the same comment twice, so fail on the first error instead. + result = run_gh_command_logged( ["gh", "pr", "comment", pr_number, "--body", comment_body], operation="Create PR comment", + retry=False, ) print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr) diff --git a/script/helpers.py b/script/helpers.py index 7cc001d92f..11549808ff 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -469,6 +469,77 @@ def get_target_branch() -> str | None: return None +# Substrings (matched case-insensitively against gh's stderr) that identify +# transient failures worth retrying: server errors (HTTP 5xx) and dropped or +# failed connections. Permanent failures (bad auth, missing PR, the 300-file +# diff limit) never match so callers see them immediately. Phrases are +# anchored so gh's GraphQL "Could not resolve to a PullRequest" (a missing +# PR) never classifies as a DNS failure. +_TRANSIENT_GH_ERROR_RE = re.compile( + r"http 5\d\d" + r"|timed out|timeout" + r"|connection (?:reset|refused|closed)" + r"|no such host|could not resolve host" + # gh intercepts DNS errors and prints its own "error connecting to + # " text; the Go phrases above are kept as a hedge in case a + # future gh stops swallowing the underlying error + r"|error connecting to" + r"|failed to verify certificate" + # Go reports a server-closed connection as 'Post "": EOF'; the + # quote-and-colon anchor keeps a URL or message body containing the + # letters from matching + r"|unexpected eof" + r'|": eof' + r"|network is unreachable" + r"|temporary failure" +) + +# Same retry policy as git network commands in esphome/git.py: 3 attempts +# with 2s/4s backoff. +_GH_MAX_ATTEMPTS = 3 + + +def run_gh_command( + args: list[str], *, retry: bool = True +) -> subprocess.CompletedProcess[str]: + """Run a gh CLI command, retrying transient network and server failures. + + Args: + args: Full command line, including the leading "gh". + retry: Pass False for commands that are not idempotent (e.g. posting + a comment), where a retry after a dropped response could repeat + a write that already succeeded server-side. + + Returns: + CompletedProcess with captured text output. + + Raises: + subprocess.CalledProcessError: If the command fails with a permanent + error, or is still failing after the retries are exhausted. + """ + attempts = _GH_MAX_ATTEMPTS if retry else 1 + attempt = 0 + while True: + try: + return subprocess.run( + args, check=True, capture_output=True, text=True, close_fds=False + ) + except subprocess.CalledProcessError as err: + attempt += 1 + stderr = err.stderr or "" + if attempt >= attempts or not _TRANSIENT_GH_ERROR_RE.search(stderr.lower()): + raise + delay = 2**attempt + # Only the leading arguments: comment-update calls carry the + # whole multi-KB comment body in the argument list + print( + f"WARNING: {' '.join(args[:3])} failed: {stderr.strip()}; " + f"retrying in {delay}s (attempt {attempt}/{attempts})", + file=sys.stderr, + ) + time.sleep(delay) + + @cache def _get_changed_files_github_actions() -> list[str] | None: """Get changed files in GitHub Actions environment. @@ -542,10 +613,22 @@ def changed_files(branch: str | None = None) -> list[str]: def _get_changed_files_from_command(command: list[str]) -> list[str]: - """Run a git command to get changed files and return them as a list.""" - proc = subprocess.run(command, capture_output=True, text=True, check=False) - if proc.returncode != 0: - raise Exception(f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}") + """Run a git or gh command to get changed files and return them as a list.""" + if command[0] == "gh": + try: + proc = run_gh_command(command) + except subprocess.CalledProcessError as e: + raise Exception( + f"Command failed: {' '.join(command)}\nstderr: {e.stderr}" + ) from e + else: + proc = subprocess.run( + command, capture_output=True, text=True, check=False, close_fds=False + ) + if proc.returncode != 0: + raise Exception( + f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}" + ) changed_files = splitlines_no_ends(proc.stdout) cwd = Path.cwd() diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 077b6ef23e..a07e56cea5 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -20,6 +20,7 @@ changed_files = helpers.changed_files filter_changed = helpers.filter_changed get_changed_components = helpers.get_changed_components _get_changed_files_from_command = helpers._get_changed_files_from_command +run_gh_command = helpers.run_gh_command _get_pr_number_from_github_env = helpers._get_pr_number_from_github_env _get_changed_files_github_actions = helpers._get_changed_files_github_actions _filter_changed_ci = helpers._filter_changed_ci @@ -1872,3 +1873,123 @@ def test_is_validate_only_file(filename: str, expected: bool, tmp_path: Path) -> def test_base_python_changed(files: list[str], expected: bool) -> None: """Only Python modules directly in esphome/ count as base Python changes.""" assert helpers.base_python_changed(files) is expected + + +def _gh_error(stderr: str) -> subprocess.CalledProcessError: + return subprocess.CalledProcessError(1, ["gh"], output="", stderr=stderr) + + +def _gh_success(stdout: str = "ok\n") -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(["gh"], 0, stdout=stdout, stderr="") + + +def test_run_gh_command_success() -> None: + """A successful command returns without retrying.""" + with patch("helpers.subprocess.run", return_value=_gh_success()) as mock_run: + result = run_gh_command(["gh", "pr", "diff", "123", "--name-only"]) + + assert result.stdout == "ok\n" + mock_run.assert_called_once() + + +@pytest.mark.parametrize( + "second_error", + [ + ( + 'Post "https://api.github.com/graphql": tls: failed to verify' + " certificate: x509: certificate is not valid for any names," + " but wanted to match api.github.com" + ), + 'Post "https://api.github.com/graphql": EOF', + ( + "error connecting to api.github.com\n" + "check your internet connection or https://githubstatus.com" + ), + ], +) +def test_run_gh_command_retries_transient_error(second_error: str) -> None: + """Transient server errors are retried with 2s/4s backoff.""" + with ( + patch( + "helpers.subprocess.run", + side_effect=[ + _gh_error("HTTP 502: 502 Bad Gateway (https://api.github.com/graphql)"), + _gh_error(second_error), + _gh_success(), + ], + ) as mock_run, + patch("helpers.time.sleep") as mock_sleep, + ): + result = run_gh_command(["gh", "pr", "diff", "123", "--name-only"]) + + assert result.stdout == "ok\n" + assert mock_run.call_count == 3 + assert [call.args[0] for call in mock_sleep.call_args_list] == [2, 4] + + +def test_run_gh_command_gives_up_after_max_attempts() -> None: + """A persistent transient error raises after the third attempt.""" + with ( + patch( + "helpers.subprocess.run", + side_effect=_gh_error("HTTP 503: Service Unavailable"), + ) as mock_run, + patch("helpers.time.sleep") as mock_sleep, + pytest.raises(subprocess.CalledProcessError), + ): + run_gh_command(["gh", "pr", "diff", "123", "--name-only"]) + + assert mock_run.call_count == 3 + assert mock_sleep.call_count == 2 + + +@pytest.mark.parametrize( + "stderr", + [ + "HTTP 404: Not Found (https://api.github.com/repos/x)", + "HTTP 401: Bad credentials", + "HTTP 403: API rate limit exceeded for installation ID 123.", + "diff exceeded the maximum number of changed files (300)", + ( + "GraphQL: Could not resolve to a PullRequest with the number of 999999." + " (repository.pullRequest)" + ), + ], +) +def test_run_gh_command_permanent_error_not_retried(stderr: str) -> None: + """Permanent failures raise immediately without any retry.""" + with ( + patch("helpers.subprocess.run", side_effect=_gh_error(stderr)) as mock_run, + patch("helpers.time.sleep") as mock_sleep, + pytest.raises(subprocess.CalledProcessError), + ): + run_gh_command(["gh", "pr", "diff", "123", "--name-only"]) + + mock_run.assert_called_once() + mock_sleep.assert_not_called() + + +def test_run_gh_command_no_retry_for_non_idempotent_commands() -> None: + """retry=False fails on the first error even when it looks transient.""" + with ( + patch( + "helpers.subprocess.run", + side_effect=_gh_error("HTTP 502: 502 Bad Gateway"), + ) as mock_run, + patch("helpers.time.sleep") as mock_sleep, + pytest.raises(subprocess.CalledProcessError), + ): + run_gh_command(["gh", "pr", "comment", "123", "--body", "x"], retry=False) + + mock_run.assert_called_once() + mock_sleep.assert_not_called() + + +def test_get_changed_files_from_command_gh_failure_keeps_stderr() -> None: + """Failures from gh surface stderr so callers can detect the 300-file limit.""" + stderr = "diff exceeded the maximum number of changed files (300)" + with ( + patch("helpers.subprocess.run", side_effect=_gh_error(stderr)), + pytest.raises(Exception, match="maximum number of changed files"), + ): + _get_changed_files_from_command(["gh", "pr", "diff", "123", "--name-only"]) From a3af82867b3cebfbf6af9a3be9c54731f0e8d544 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 12:00:39 -0700 Subject: [PATCH 086/160] [api] Bump noise-c to 0.1.18 (#18451) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 8ec94df1db..5ca9484336 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.11") + cg.add_library("esphome/noise-c", "0.1.18") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index bf3b0685f8..2c22523be5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.11 ; api + esphome/noise-c@0.1.18 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.11 ; api + esphome/noise-c@0.1.18 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.11 ; used by api + esphome/noise-c@0.1.18 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 6d20943a9c17a994338ba176c2709de80897d1df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:38:06 -0400 Subject: [PATCH 087/160] Bump esphome/workflows/.github/workflows/stale.yml from 61fd37a044cad4e9aa4303027b2a61b6a34da855 to a1c1485ab46ef41a84a6a9d8abd7fa4b7628fd70 (#18464) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3c471b6efb..aa31094f81 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: # No GITHUB_TOKEN permissions: the reusable workflow mints an ESPHome # GitHub App token so the labels, comments and closures come from # esphome[bot] instead of github-actions[bot]. - uses: esphome/workflows/.github/workflows/stale.yml@61fd37a044cad4e9aa4303027b2a61b6a34da855 # main + uses: esphome/workflows/.github/workflows/stale.yml@a1c1485ab46ef41a84a6a9d8abd7fa4b7628fd70 # main secrets: ESPHOME_GITHUB_APP_PRIVATE_KEY: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} with: From 9bc72529a6a4dab3bcfb6d182b1a5aa2f0b66b9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:42:08 -0500 Subject: [PATCH 088/160] Bump astral-sh/setup-uv from 9.0.0 to 10.0.1 in /.github/actions/restore-python (#18462) Signed-off-by: dependabot[bot] --- .github/actions/restore-python/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/restore-python/action.yml b/.github/actions/restore-python/action.yml index daf041819c..6279a26dc4 100644 --- a/.github/actions/restore-python/action.yml +++ b/.github/actions/restore-python/action.yml @@ -32,7 +32,7 @@ runs: # detects the activated venv via ``VIRTUAL_ENV`` so the venv layout # downstream jobs rely on is preserved. if: steps.cache-venv.outputs.cache-hit != 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true # Pull request saves land in per-PR scopes nothing else can From 4712c15c75d0457aea5d75ff717a0f2bc7171d00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:42:24 -0500 Subject: [PATCH 089/160] Bump github/codeql-action/init from 4.37.6 to 4.37.7 (#18466) Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4e164cd9f6..f01441cdfd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} From 3d5f6f692f4916fd2923c490d28a5df3287c087c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:45:41 -0500 Subject: [PATCH 090/160] Bump filelock from 3.32.2 to 3.32.3 (#18461) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 61011f2fbd..4b1708637d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.2 # native esp-idf toolchain global cache dir -filelock==3.32.2 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==3.32.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.2 From 95180067245bf49d5154889082323c91450145c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:46:04 -0500 Subject: [PATCH 091/160] Bump ruff from 0.16.2 to 0.16.3 (#18458) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 95ee97437d..cedc107b17 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.16.2 # also change in .pre-commit-config.yaml when updating +ruff==0.16.3 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating prek==0.4.13 # also change in .github/workflows/ci.yml when updating From 346ba7e831d21d5b005276ca1085fcc570ce3ce8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:47:18 -0500 Subject: [PATCH 092/160] Bump github/codeql-action/analyze from 4.37.6 to 4.37.7 (#18467) Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f01441cdfd..103cecc1f9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: category: "/language:${{matrix.language}}" From 55726120db6bfaf8ed8fc699a92580f59b7b0e46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:47:28 -0500 Subject: [PATCH 093/160] Bump esphome/workflows/.github/workflows/lock.yml from 2026.7.0 to 2026.8.1 (#18465) Signed-off-by: dependabot[bot] --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index ec736a2002..e09e9bf2d1 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -14,4 +14,4 @@ jobs: permissions: issues: write # issues.lock on closed issues pull-requests: write # issues.lock on closed pull requests - uses: esphome/workflows/.github/workflows/lock.yml@9f6577fd37b5cf773ab1b9be929714a0dcd15661 # 2026.7.0 + uses: esphome/workflows/.github/workflows/lock.yml@0fdd5e311b7e744069166696072a1a9cbc5fbeb6 # 2026.8.1 From 199e368fe2dfca47d8c59796bb54d1d292934396 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:57:50 -0500 Subject: [PATCH 094/160] Bump astral-sh/setup-uv from 9.0.0 to 10.0.1 (#18463) Signed-off-by: dependabot[bot] --- .github/workflows/ci-api-proto.yml | 2 +- .github/workflows/ci.yml | 6 +++--- .github/workflows/sync-device-classes.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-api-proto.yml b/.github/workflows/ci-api-proto.yml index 820081cc46..1ccff96f24 100644 --- a/.github/workflows/ci-api-proto.yml +++ b/.github/workflows/ci-api-proto.yml @@ -29,7 +29,7 @@ jobs: - name: Set up uv # ``--system`` (below) installs into the setup-python interpreter; # no venv is created or restored by this workflow. - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true # Pull-request-only workflow: a save could never be shared and diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 026c2ba27a..cd1a382c21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: # detects the activated venv via ``VIRTUAL_ENV`` so downstream jobs # that ``. venv/bin/activate`` see an identical layout. if: steps.cache-venv.outputs.cache-hit != 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true # Pull request saves land in per-PR scopes nothing else can @@ -367,7 +367,7 @@ jobs: - name: Set up uv # Only needed on cache miss to populate the venv. if: steps.cache-venv.outputs.cache-hit != 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true # Pull request saves land in per-PR scopes nothing else can @@ -1095,7 +1095,7 @@ jobs: # install step (order-of-magnitude faster on cold boots, # with its own wheel cache). actions/setup-python still # provides the interpreter. - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true # Pull request saves land in per-PR scopes nothing else can diff --git a/.github/workflows/sync-device-classes.yml b/.github/workflows/sync-device-classes.yml index a299e76584..9100064176 100644 --- a/.github/workflows/sync-device-classes.yml +++ b/.github/workflows/sync-device-classes.yml @@ -47,7 +47,7 @@ jobs: # setup-python interpreter so subsequent ``prek`` / # ``script/run-in-env.py`` steps find the deps without a # ``uv run`` prefix. - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true # Pin uv version so the action does not have to fetch the From e32329cc11a38d9b0a70bbd401b1e0ea6a062423 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 15:00:37 -0500 Subject: [PATCH 095/160] [core] Sync pre-commit ruff hook with requirements (0.16.3) (#18469) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99a4f40201..0ea799aa4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.0 + rev: v0.16.3 hooks: # Run the linter. - id: ruff From e45b4e493886e089880cef1b77a4ae367ca0aea9 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:26:24 +1200 Subject: [PATCH 096/160] [core] Make FINAL_VALIDATE_SCHEMA functions return None (#18457) --- esphome/components/bk72xx_ble/__init__.py | 3 +-- esphome/components/captive_portal/__init__.py | 4 +--- esphome/components/dsmr/__init__.py | 4 +--- esphome/components/emontx/__init__.py | 4 ++-- esphome/components/epaper_spi/display.py | 3 +-- esphome/components/esp32/__init__.py | 4 +--- esphome/components/esp32_ble/__init__.py | 4 +--- esphome/components/esp32_ble_server/__init__.py | 3 +-- esphome/components/esp32_hosted/__init__.py | 3 +-- esphome/components/ethernet/__init__.py | 3 +-- esphome/components/factory_reset/__init__.py | 3 +-- esphome/components/file/image.py | 3 +-- .../components/gpio/binary_sensor/__init__.py | 10 ++++------ esphome/components/growatt_solar/sensor.py | 4 ++-- esphome/components/haier/climate.py | 3 +-- esphome/components/haier/switch/__init__.py | 3 +-- esphome/components/havells_solar/sensor.py | 4 ++-- esphome/components/hub75/display.py | 4 +--- esphome/components/improv_serial/__init__.py | 3 +-- esphome/components/inkplate/display.py | 3 +-- esphome/components/it8951/display.py | 3 +-- esphome/components/kuntze/sensor.py | 4 ++-- esphome/components/ld6002b/button/__init__.py | 4 +--- esphome/components/ld6002b/number/__init__.py | 6 ++---- esphome/components/light/__init__.py | 6 ++---- esphome/components/mcp4461/output/__init__.py | 5 ++--- esphome/components/mdns/__init__.py | 5 ++--- esphome/components/mipi_dsi/display.py | 3 +-- esphome/components/mipi_rgb/display.py | 3 +-- esphome/components/mitsubishi_cn105/climate.py | 6 +++--- .../components/modbus_controller/__init__.py | 6 ++---- esphome/components/modbus_server/__init__.py | 4 ++-- .../packet_transport/binary_sensor.py | 6 +++--- esphome/components/provisioning/__init__.py | 3 +-- esphome/components/pzemac/sensor.py | 4 ++-- esphome/components/pzemdc/sensor.py | 4 ++-- esphome/components/router/speaker/__init__.py | 3 +-- esphome/components/rp2040_ble/__init__.py | 3 +-- esphome/components/sdm_meter/sensor.py | 4 ++-- esphome/components/sds011/sensor.py | 3 +-- esphome/components/selec_meter/sensor.py | 4 ++-- esphome/components/tinyusb/__init__.py | 3 +-- esphome/components/web_server/__init__.py | 3 +-- esphome/components/zephyr_pwm/output.py | 3 +-- esphome/components/zwave_proxy/__init__.py | 4 +--- tests/component_tests/esp32_hosted/test_init.py | 2 +- tests/component_tests/image/test_init.py | 17 +++++++++-------- .../provisioning/test_provisioning.py | 6 +++--- 48 files changed, 79 insertions(+), 123 deletions(-) diff --git a/esphome/components/bk72xx_ble/__init__.py b/esphome/components/bk72xx_ble/__init__.py index b58464a1f6..81073c9b02 100644 --- a/esphome/components/bk72xx_ble/__init__.py +++ b/esphome/components/bk72xx_ble/__init__.py @@ -68,12 +68,11 @@ def _unsupported_family_message(family: str) -> str | None: return None -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: # Warn only: a hard error here would break the validate-only CI fixtures, # which run on a BLE 4.2 board. The hard error is raised at codegen. if msg := _unsupported_family_message(libretiny.get_libretiny_family()): _LOGGER.warning("%s (this configuration cannot compile)", msg) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/captive_portal/__init__.py b/esphome/components/captive_portal/__init__.py index d62c718097..8e5274f58f 100644 --- a/esphome/components/captive_portal/__init__.py +++ b/esphome/components/captive_portal/__init__.py @@ -61,7 +61,7 @@ CONFIG_SCHEMA = cv.All( ) -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: full_config = fv.full_config.get() wifi_conf = full_config.get("wifi") @@ -88,8 +88,6 @@ def _final_validate(config: ConfigType) -> ConfigType: socket.consume_sockets(3, "captive_portal")(config) socket.consume_sockets(1, "captive_portal", socket.SocketType.UDP)(config) - return config - FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/dsmr/__init__.py b/esphome/components/dsmr/__init__.py index 34f37ace35..eaf36d34fa 100644 --- a/esphome/components/dsmr/__init__.py +++ b/esphome/components/dsmr/__init__.py @@ -88,7 +88,7 @@ async def to_code(config): cg.add_library("esphome/dsmr_parser", "1.9.0") -def final_validate(config: ConfigType) -> ConfigType: +def final_validate(config: ConfigType) -> None: full_config = fv.full_config.get() for uart_conf in full_config["uart"]: @@ -102,7 +102,5 @@ def final_validate(config: ConfigType) -> ConfigType: ) break - return config - FINAL_VALIDATE_SCHEMA = final_validate diff --git a/esphome/components/emontx/__init__.py b/esphome/components/emontx/__init__.py index a2d4349698..3f83578926 100644 --- a/esphome/components/emontx/__init__.py +++ b/esphome/components/emontx/__init__.py @@ -59,7 +59,7 @@ CONFIG_SCHEMA = ( ) -def final_validate(config: ConfigType) -> ConfigType: +def final_validate(config: ConfigType) -> None: full_config = fv.full_config.get() # Count sensors registered to this hub (IDs are resolved at final_validate stage) @@ -95,7 +95,7 @@ def final_validate(config: ConfigType) -> ConfigType: parity="NONE", stop_bits=1, ) - return schema(config) + schema(config) FINAL_VALIDATE_SCHEMA = final_validate diff --git a/esphome/components/epaper_spi/display.py b/esphome/components/epaper_spi/display.py index 0b82850f1e..e9da924de5 100644 --- a/esphome/components/epaper_spi/display.py +++ b/esphome/components/epaper_spi/display.py @@ -153,7 +153,7 @@ def customise_schema(config): CONFIG_SCHEMA = customise_schema -def _final_validate(config): +def _final_validate(config) -> None: spi.final_validate_device_schema( "epaper_spi", require_miso=False, require_mosi=True )(config) @@ -170,7 +170,6 @@ def _final_validate(config): config[CONF_SHOW_TEST_CARD] = True elif CONF_UPDATE_INTERVAL not in config: config[CONF_UPDATE_INTERVAL] = update_interval("1min") - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 7263571d69..7d43c3ac07 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1368,7 +1368,7 @@ def _validate_signed_ota_keys(config: ConfigType) -> ConfigType: return config -def final_validate(config): +def final_validate(config) -> None: # Imported locally to avoid circular import issues from esphome.components.psram import DOMAIN as PSRAM_DOMAIN @@ -1629,8 +1629,6 @@ def final_validate(config): if errs: raise cv.MultipleInvalid(errs) - return config - CONF_SDKCONFIG_OPTIONS = "sdkconfig_options" CONF_ENABLE_LWIP_DHCP_SERVER = "enable_lwip_dhcp_server" diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index 935d8b1b7e..f099c68e57 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -443,7 +443,7 @@ def validate_connection_slots(max_connections: int) -> None: ) -def final_validation(config): +def final_validation(config) -> None: validate_variant(config) if (name := config.get(CONF_NAME)) is not None: full_config = fv.full_config.get() @@ -514,8 +514,6 @@ def final_validation(config): # For newer chips (C3/S3/etc), different configs are used automatically add_idf_sdkconfig_option("CONFIG_BTDM_CTRL_BLE_MAX_CONN", max_connections) - return config - FINAL_VALIDATE_SCHEMA = final_validation diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index ea2a9667d7..855a3be29b 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -307,7 +307,7 @@ def create_device_information_service(config): return config -def final_validate_config(config): +def final_validate_config(config) -> None: # Validate max_clients does not exceed esp32_ble max_connections max_clients = config[CONF_MAX_CLIENTS] if max_clients > 1: @@ -355,7 +355,6 @@ def final_validate_config(config): raise cv.Invalid( f"Characteristic {char_config[CONF_UUID]} has both a set_value action and a templated value" ) - return config def validate_value_type(value_config): diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index d3432fb461..7dc61ce382 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -126,7 +126,7 @@ CONFIG_SCHEMA = cv.typed_schema( ) -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: # The esp_hosted releases compatible with older ESP-IDF versions crash at # boot with a heap double free in the SDIO RX path (fixed in esp_hosted # 2.11.0, which requires ESP-IDF 5.3), so reject them at validation time. @@ -136,7 +136,6 @@ def _final_validate(config: ConfigType) -> ConfigType: "Remove the framework version from your configuration to use the " "recommended version, or pin a version at or above 5.3." ) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/ethernet/__init__.py b/esphome/components/ethernet/__init__.py index f3c77baaae..5eda0fc12c 100644 --- a/esphome/components/ethernet/__init__.py +++ b/esphome/components/ethernet/__init__.py @@ -767,7 +767,7 @@ def _final_validate_rmii_pins(config: ConfigType) -> None: raise cv.Invalid(error_msg, path=pin_path) -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: """Final validation for Ethernet component.""" # Allow ethernet + wifi coexistence only when both are declared in network: priority:. if "wifi" in fv.full_config.get(): @@ -787,7 +787,6 @@ def _final_validate(config: ConfigType) -> ConfigType: _final_validate_spi(config) _final_validate_rmii_pins(config) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/factory_reset/__init__.py b/esphome/components/factory_reset/__init__.py index 818a53c0ed..d5d5d2ecb5 100644 --- a/esphome/components/factory_reset/__init__.py +++ b/esphome/components/factory_reset/__init__.py @@ -60,14 +60,13 @@ CONFIG_SCHEMA = cv.All( ) -def _final_validate(config): +def _final_validate(config) -> None: if CORE.is_esp8266 and CONF_RESETS_REQUIRED in config: fconfig = full_config.get() if not fconfig.get_config_for_path([KEY_ESP8266, CONF_RESTORE_FROM_FLASH]): raise cv.Invalid( "'resets_required' needs 'restore_from_flash' to be enabled in the 'esp8266' configuration" ) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/file/image.py b/esphome/components/file/image.py index b54c3f2adf..d340d21490 100644 --- a/esphome/components/file/image.py +++ b/esphome/components/file/image.py @@ -225,7 +225,7 @@ def image_schema(class_: MockObjClass = Image_) -> cv.Schema: ) -def validate_image_final(config: ConfigType) -> ConfigType: +def validate_image_final(config: ConfigType) -> None: """Per-entry final validation, shared by file-backed image platforms. For LVGL 9 the default byte order for RGB565 images is little-endian, so @@ -240,7 +240,6 @@ def validate_image_final(config: ConfigType) -> ConfigType: ) else: config[CONF_BYTE_ORDER] = "LITTLE_ENDIAN" - return config async def new_image(config: ConfigType) -> MockObj: diff --git a/esphome/components/gpio/binary_sensor/__init__.py b/esphome/components/gpio/binary_sensor/__init__.py index 43358baedb..703806670c 100644 --- a/esphome/components/gpio/binary_sensor/__init__.py +++ b/esphome/components/gpio/binary_sensor/__init__.py @@ -68,10 +68,10 @@ def _pin_shared_only_with_deep_sleep(pin_num: int) -> bool: return any(path and path[0] == "deep_sleep" for path, _, _ in pin_users) -def _final_validate(config): +def _final_validate(config) -> None: use_interrupt = config[CONF_USE_INTERRUPT] if not use_interrupt: - return config + return # Expander pins (e.g. PCF8574, MCP23017) don't support direct interrupt # attachment — only internal/native GPIO pins do. @@ -82,7 +82,7 @@ def _final_validate(config): config.get(CONF_NAME, config[CONF_ID]), ) config[CONF_USE_INTERRUPT] = False - return config + return pin_num = config[CONF_PIN][CONF_NUMBER] @@ -96,7 +96,7 @@ def _final_validate(config): config.get(CONF_NAME, config[CONF_ID]), ) config[CONF_USE_INTERRUPT] = False - return config + return # When a pin is shared, interrupts can interfere with other components # (e.g., duty_cycle sensor) that need to monitor the pin's state changes. @@ -120,8 +120,6 @@ def _final_validate(config): pin_num, ) - return config - FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/growatt_solar/sensor.py b/esphome/components/growatt_solar/sensor.py index d1f0069341..d62486f5ec 100644 --- a/esphome/components/growatt_solar/sensor.py +++ b/esphome/components/growatt_solar/sensor.py @@ -163,8 +163,8 @@ CONFIG_SCHEMA = ( ) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("growatt_solar", role="client")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("growatt_solar", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/haier/climate.py b/esphome/components/haier/climate.py index 424ef46392..70ae36f528 100644 --- a/esphome/components/haier/climate.py +++ b/esphome/components/haier/climate.py @@ -424,7 +424,7 @@ async def power_action_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg, paren) -def _final_validate(config): +def _final_validate(config) -> None: full_config = fv.full_config.get() if CONF_LOGGER in full_config: _level = "NONE" @@ -448,7 +448,6 @@ def _final_validate(config): raise cv.Invalid( f"No WiFi configured, if you want to use haier climate without WiFi add {CONF_WIFI_SIGNAL}: false to climate configuration" ) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/haier/switch/__init__.py b/esphome/components/haier/switch/__init__.py index acff0cf265..99ffcb37af 100644 --- a/esphome/components/haier/switch/__init__.py +++ b/esphome/components/haier/switch/__init__.py @@ -60,7 +60,7 @@ CONFIG_SCHEMA = cv.Schema( ) -def _final_validate(config): +def _final_validate(config) -> None: full_config = fv.full_config.get() for switch_type in [CONF_BEEPER, CONF_QUIET_MODE]: # Check switches that are only supported for HonClimate @@ -72,7 +72,6 @@ def _final_validate(config): raise cv.Invalid( f"{switch_type} switch is only supported for hon climate" ) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/havells_solar/sensor.py b/esphome/components/havells_solar/sensor.py index d18ae0d9af..8eafe1d9d6 100644 --- a/esphome/components/havells_solar/sensor.py +++ b/esphome/components/havells_solar/sensor.py @@ -217,8 +217,8 @@ CONFIG_SCHEMA = ( ) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("havells_solar", role="client")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("havells_solar", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/hub75/display.py b/esphome/components/hub75/display.py index a404fbbade..24b8197073 100644 --- a/esphome/components/hub75/display.py +++ b/esphome/components/hub75/display.py @@ -315,7 +315,7 @@ def _validate_config(config: ConfigType) -> ConfigType: return config -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: """Validate requirements when using HUB75 display.""" # Local imports to avoid circular dependencies from esphome.components.esp32 import get_esp32_variant @@ -381,8 +381,6 @@ def _final_validate(config: ConfigType) -> ConfigType: if errs: raise cv.MultipleInvalid(errs) - return config - FINAL_VALIDATE_SCHEMA = cv.Schema(_final_validate) diff --git a/esphome/components/improv_serial/__init__.py b/esphome/components/improv_serial/__init__.py index 4266f5b78b..3e2a6db1bc 100644 --- a/esphome/components/improv_serial/__init__.py +++ b/esphome/components/improv_serial/__init__.py @@ -22,7 +22,7 @@ CONFIG_SCHEMA = ( ) -def validate_logger(config): +def validate_logger(config) -> None: logger_conf = fv.full_config.get()[CONF_LOGGER] if logger_conf[CONF_BAUD_RATE] == 0: raise cv.Invalid("improv_serial requires the logger baud_rate to be not 0") @@ -33,7 +33,6 @@ def validate_logger(config): raise cv.Invalid( "improv_serial does not support the selected logger hardware_uart" ) - return config FINAL_VALIDATE_SCHEMA = validate_logger diff --git a/esphome/components/inkplate/display.py b/esphome/components/inkplate/display.py index 47c8c898e5..a0c0d5dc18 100644 --- a/esphome/components/inkplate/display.py +++ b/esphome/components/inkplate/display.py @@ -146,13 +146,12 @@ CONFIG_SCHEMA = cv.All( ) -def _validate_cpu_frequency(config): +def _validate_cpu_frequency(config) -> None: esp32_config = fv.full_config.get()[PLATFORM_ESP32] if esp32_config[CONF_CPU_FREQUENCY] != "240MHZ": raise cv.Invalid( "Inkplate requires 240MHz CPU frequency (set in esp32 component)" ) - return config FINAL_VALIDATE_SCHEMA = _validate_cpu_frequency diff --git a/esphome/components/it8951/display.py b/esphome/components/it8951/display.py index 51c5fc6118..bdc68b5257 100644 --- a/esphome/components/it8951/display.py +++ b/esphome/components/it8951/display.py @@ -336,7 +336,7 @@ def _customise_schema(config): CONFIG_SCHEMA = _customise_schema -def _final_validate(config): +def _final_validate(config) -> None: # IT8951 reads from SPI (DevInfo, VCOM, register reads) so MISO is required. spi.final_validate_device_schema("it8951", require_miso=True, require_mosi=True)( config @@ -351,7 +351,6 @@ def _final_validate(config): config[CONF_UPDATE_INTERVAL] = update_interval("never") else: config[CONF_SHOW_TEST_CARD] = True - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/kuntze/sensor.py b/esphome/components/kuntze/sensor.py index c11ede9db6..2b53e70756 100644 --- a/esphome/components/kuntze/sensor.py +++ b/esphome/components/kuntze/sensor.py @@ -89,8 +89,8 @@ CONFIG_SCHEMA = ( ) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("kuntze", role="client")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("kuntze", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/ld6002b/button/__init__.py b/esphome/components/ld6002b/button/__init__.py index c327c331c6..508d5c2bc6 100644 --- a/esphome/components/ld6002b/button/__init__.py +++ b/esphome/components/ld6002b/button/__init__.py @@ -84,7 +84,7 @@ CONFIG_SCHEMA = cv.Schema( ) -def final_validate(config: ConfigType) -> ConfigType: +def final_validate(config: ConfigType) -> None: full_config = fv.full_config.get() hub_id = config[CONF_LD6002B_ID] @@ -108,8 +108,6 @@ def final_validate(config: ConfigType) -> ConfigType: path=[CONF_WAKE], ) - return config - FINAL_VALIDATE_SCHEMA = final_validate diff --git a/esphome/components/ld6002b/number/__init__.py b/esphome/components/ld6002b/number/__init__.py index 7e0be66c64..452e38d6e3 100644 --- a/esphome/components/ld6002b/number/__init__.py +++ b/esphome/components/ld6002b/number/__init__.py @@ -105,9 +105,9 @@ CONFIG_SCHEMA = cv.Schema( ) -def final_validate(config: ConfigType) -> ConfigType: +def final_validate(config: ConfigType) -> None: if config.get(CONF_AREA_CONFIG) is None: - return config + return full_config = fv.full_config.get() hub_id = config[CONF_LD6002B_ID] @@ -132,8 +132,6 @@ def final_validate(config: ConfigType) -> ConfigType: path=[CONF_AREA_CONFIG], ) - return config - FINAL_VALIDATE_SCHEMA = final_validate diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index 7c4d7ed431..b5b3d7c905 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -165,7 +165,7 @@ def available_effects_str(effects: list) -> str: return ", ".join(f"'{name}'" for name in available) if available else "none" -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: """Validate all recorded effect name references against their target lights. This runs once per light platform instance. If no light platform is configured, @@ -173,7 +173,7 @@ def _final_validate(config: ConfigType) -> ConfigType: """ data = _get_data() if not data.effect_refs and not data.effect_cycle_refs: - return config + return # Drain the lists so we only validate once even though # FINAL_VALIDATE_SCHEMA runs for each light platform instance. @@ -217,8 +217,6 @@ def _final_validate(config: ConfigType) -> ConfigType: path=[cv.ROOT_CONFIG_PATH] + ref.component_path, ) - return config - FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/mcp4461/output/__init__.py b/esphome/components/mcp4461/output/__init__.py index 1642f6149a..99d4988c90 100644 --- a/esphome/components/mcp4461/output/__init__.py +++ b/esphome/components/mcp4461/output/__init__.py @@ -34,7 +34,7 @@ CONF_NONVOLATILE_WRITE_DELAY = "nonvolatile_write_delay" VOLATILE_CHANNELS = ("A", "B", "C", "D") -def _validate_nonvolatile(config): +def _validate_nonvolatile(config) -> None: channel = str(config[CONF_CHANNEL]) # Channels E-H address the nonvolatile registers directly — the mirroring options only @@ -49,7 +49,7 @@ def _validate_nonvolatile(config): f"enabling '{CONF_NONVOLATILE}' or setting '{CONF_NONVOLATILE_WRITE_DELAY}' is only valid for the " f"volatile channels A-D; channels E-H are the nonvolatile registers themselves" ) - return config + return config.setdefault(CONF_NONVOLATILE, True) if config[CONF_NONVOLATILE]: @@ -62,7 +62,6 @@ def _validate_nonvolatile(config): raise cv.Invalid( f"'{CONF_NONVOLATILE_WRITE_DELAY}' requires '{CONF_NONVOLATILE}: true'" ) - return config CONFIG_SCHEMA = output.FLOAT_OUTPUT_SCHEMA.extend( diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index 2d4f6085e5..24bce0cc3c 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -62,7 +62,7 @@ def _consume_mdns_sockets(config: ConfigType) -> ConfigType: return config -def _require_network_interface(config: ConfigType) -> ConfigType: +def _require_network_interface(config: ConfigType) -> None: """Require a network interface for mDNS on Arduino/LEAmDNS platforms. On ESP8266 and RP2040 the C++ implementation needs at least one IP state @@ -71,7 +71,7 @@ def _require_network_interface(config: ConfigType) -> ConfigType: that never initializes. """ if config.get(CONF_DISABLED) or not (CORE.is_esp8266 or CORE.is_rp2): - return config + return full_config = fv.full_config.get() has_wifi = "wifi" in full_config has_ethernet = CORE.is_rp2 and "ethernet" in full_config @@ -81,7 +81,6 @@ def _require_network_interface(config: ConfigType) -> ConfigType: "mdns on this platform requires a network interface — " f"add a {options} component to your configuration." ) - return config CONFIG_SCHEMA = cv.All( diff --git a/esphome/components/mipi_dsi/display.py b/esphome/components/mipi_dsi/display.py index e5bb3d413d..8c125a9606 100644 --- a/esphome/components/mipi_dsi/display.py +++ b/esphome/components/mipi_dsi/display.py @@ -175,7 +175,7 @@ def _config_schema(config): return config -def _final_validate(config): +def _final_validate(config) -> None: global_config = full_config.get() from esphome.components.lvgl import DOMAIN as LVGL_DOMAIN @@ -183,7 +183,6 @@ def _final_validate(config): if not requires_buffer(config) and LVGL_DOMAIN not in global_config: # If no drawing methods are configured, and LVGL is not enabled, show a test card config[CONF_SHOW_TEST_CARD] = True - return config CONFIG_SCHEMA = _config_schema diff --git a/esphome/components/mipi_rgb/display.py b/esphome/components/mipi_rgb/display.py index ebe930d37a..897088a257 100644 --- a/esphome/components/mipi_rgb/display.py +++ b/esphome/components/mipi_rgb/display.py @@ -248,7 +248,7 @@ def _config_schema(config): CONFIG_SCHEMA = _config_schema -def _final_validate(config): +def _final_validate(config) -> None: global_config = full_config.get() from esphome.components.lvgl import DOMAIN as LVGL_DOMAIN @@ -260,7 +260,6 @@ def _final_validate(config): config = spi.final_validate_device_schema( "mipi_rgb", require_miso=False, require_mosi=True )(config) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/mitsubishi_cn105/climate.py b/esphome/components/mitsubishi_cn105/climate.py index 64475d0e32..05a29b3665 100644 --- a/esphome/components/mitsubishi_cn105/climate.py +++ b/esphome/components/mitsubishi_cn105/climate.py @@ -143,11 +143,11 @@ def CONFIG_SCHEMA(config: ConfigType) -> ConfigType: # Legacy climate-owned hub compatibility. Remove in 2027.2.0. -def _legacy_final_validate(config: ConfigType) -> ConfigType: +def _legacy_final_validate(config: ConfigType) -> None: if CONF_MITSUBISHI_CN105_ID in config: - return config + return - return uart.final_validate_device_schema( + uart.final_validate_device_schema( DOMAIN, require_rx=True, require_tx=True, diff --git a/esphome/components/modbus_controller/__init__.py b/esphome/components/modbus_controller/__init__.py index 1ce1e38d16..f3cd28d138 100644 --- a/esphome/components/modbus_controller/__init__.py +++ b/esphome/components/modbus_controller/__init__.py @@ -135,10 +135,8 @@ def validate_modbus_register(config): return config -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("modbus_controller", role="client")( - config - ) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("modbus_controller", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/modbus_server/__init__.py b/esphome/components/modbus_server/__init__.py index 16b956d7b5..249454b6b0 100644 --- a/esphome/components/modbus_server/__init__.py +++ b/esphome/components/modbus_server/__init__.py @@ -144,8 +144,8 @@ CONFIG_SCHEMA = cv.All( ) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("modbus_server", role="server")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("modbus_server", role="server")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/packet_transport/binary_sensor.py b/esphome/components/packet_transport/binary_sensor.py index 09bbf91c99..3291ff2c59 100644 --- a/esphome/components/packet_transport/binary_sensor.py +++ b/esphome/components/packet_transport/binary_sensor.py @@ -44,10 +44,10 @@ CONFIG_SCHEMA = cv.typed_schema( ) -def _final_validate(config): +def _final_validate(config) -> None: if config[CONF_TYPE] != CONF_STATUS: # Only run this validation if a status sensor is being configured - return config + return full_config = fv.full_config.get() transport_path = full_config.get_path_for_id(config[CONF_TRANSPORT_ID])[:-1] transport_config = full_config.get_config_for_path(transport_path) @@ -56,7 +56,7 @@ def _final_validate(config): for p in transport_config[CONF_PROVIDERS] if p[CONF_NAME] == config[CONF_PROVIDER] ): - return config + return raise cv.Invalid( "Status sensor requires ping-pong to be enabled and the nominated provider to use encryption." ) diff --git a/esphome/components/provisioning/__init__.py b/esphome/components/provisioning/__init__.py index 36fa69357a..9462bbb3b7 100644 --- a/esphome/components/provisioning/__init__.py +++ b/esphome/components/provisioning/__init__.py @@ -67,7 +67,7 @@ CONFIG_SCHEMA = cv.Schema( ).extend(cv.COMPONENT_SCHEMA) -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: """Validate the provisioning setup once every component has been processed. Sources register during their own config validation, so by final validation @@ -89,7 +89,6 @@ def _final_validate(config: ConfigType) -> ConfigType: "hardcoding them makes the window pointless.", ", ".join(sorted(data.hardcoded_credentials)), ) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/pzemac/sensor.py b/esphome/components/pzemac/sensor.py index 4e228f6aa3..5bb734cb2d 100644 --- a/esphome/components/pzemac/sensor.py +++ b/esphome/components/pzemac/sensor.py @@ -98,8 +98,8 @@ async def reset_energy_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg, paren) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("pzemac", role="client")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("pzemac", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/pzemdc/sensor.py b/esphome/components/pzemdc/sensor.py index 40cfe7b08a..b2c7c3a29d 100644 --- a/esphome/components/pzemdc/sensor.py +++ b/esphome/components/pzemdc/sensor.py @@ -80,8 +80,8 @@ async def reset_energy_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg, paren) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("pzemdc", role="client")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("pzemdc", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/router/speaker/__init__.py b/esphome/components/router/speaker/__init__.py index 2b2dc56433..18311416c3 100644 --- a/esphome/components/router/speaker/__init__.py +++ b/esphome/components/router/speaker/__init__.py @@ -63,7 +63,7 @@ CONFIG_SCHEMA = cv.All( ) -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: # Validate every configured output speaker can accept the router's format. # Switching to an output that can't reproduce the format the producer is # already sending would otherwise fail silently at runtime. @@ -76,7 +76,6 @@ def _final_validate(config: ConfigType) -> ConfigType: channels=config[CONF_NUM_CHANNELS], sample_rate=config[CONF_SAMPLE_RATE], )(proxy) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/rp2040_ble/__init__.py b/esphome/components/rp2040_ble/__init__.py index 332ea73a61..d2a08e9fc0 100644 --- a/esphome/components/rp2040_ble/__init__.py +++ b/esphome/components/rp2040_ble/__init__.py @@ -71,10 +71,9 @@ def validate_connection_slots() -> None: ) -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: _validate_board(config) validate_connection_slots() - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/sdm_meter/sensor.py b/esphome/components/sdm_meter/sensor.py index 46f5025080..125240e891 100644 --- a/esphome/components/sdm_meter/sensor.py +++ b/esphome/components/sdm_meter/sensor.py @@ -148,8 +148,8 @@ CONFIG_SCHEMA = ( ) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("sdm_meter", role="client")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("sdm_meter", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/sds011/sensor.py b/esphome/components/sds011/sensor.py index 2d7b6b07e5..59ee6667a1 100644 --- a/esphome/components/sds011/sensor.py +++ b/esphome/components/sds011/sensor.py @@ -63,7 +63,7 @@ CONFIG_SCHEMA = cv.All( ) -def _final_validate(config): +def _final_validate(config) -> None: # In the default mode setup() writes config commands, so tx is required; # rx_only mode never writes, so tx is optional. uart.final_validate_device_schema( @@ -75,7 +75,6 @@ def _final_validate(config): parity="NONE", stop_bits=1, )(config) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/selec_meter/sensor.py b/esphome/components/selec_meter/sensor.py index ef4929c375..120b997605 100644 --- a/esphome/components/selec_meter/sensor.py +++ b/esphome/components/selec_meter/sensor.py @@ -164,8 +164,8 @@ CONFIG_SCHEMA = ( ) -def _final_validate(config: ConfigType) -> ConfigType: - return modbus.final_validate_modbus_device("selec_meter", role="client")(config) +def _final_validate(config: ConfigType) -> None: + modbus.final_validate_modbus_device("selec_meter", role="client")(config) FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/tinyusb/__init__.py b/esphome/components/tinyusb/__init__.py index 9e1ad3afc4..4c6f4db85b 100644 --- a/esphome/components/tinyusb/__init__.py +++ b/esphome/components/tinyusb/__init__.py @@ -57,7 +57,7 @@ CONFIG_SCHEMA = cv.All( ) -def _final_validate(config): +def _final_validate(config) -> None: full_config = fv.full_config.get() if not any(name in full_config for name in _USB_CLASS_COMPONENTS): raise cv.Invalid( @@ -75,7 +75,6 @@ def _final_validate(config): "USB_SERIAL_JTAG on variants that support it " "(ESP32-S3, ESP32-S31, ESP32-P4, ESP32-H4)" ) - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index c1887cc3fc..b2c0ea14ad 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -193,7 +193,7 @@ def _validate_no_sorting_component( ) -def _final_validate_sorting(config: ConfigType) -> ConfigType: +def _final_validate_sorting(config: ConfigType) -> None: if (webserver_version := config.get(CONF_VERSION)) != 3: _validate_no_sorting_component( CONF_SORTING_WEIGHT, webserver_version, fv.full_config.get() @@ -201,7 +201,6 @@ def _final_validate_sorting(config: ConfigType) -> ConfigType: _validate_no_sorting_component( CONF_SORTING_GROUP_ID, webserver_version, fv.full_config.get() ) - return config FINAL_VALIDATE_SCHEMA = _final_validate_sorting diff --git a/esphome/components/zephyr_pwm/output.py b/esphome/components/zephyr_pwm/output.py index 54c04473e3..b7ee27f63c 100644 --- a/esphome/components/zephyr_pwm/output.py +++ b/esphome/components/zephyr_pwm/output.py @@ -102,9 +102,8 @@ def _allocate_blocks() -> None: _get_data().pwm_blocks = pwm_blocks -def _final_validate(config: ConfigType) -> ConfigType: +def _final_validate(config: ConfigType) -> None: _allocate_blocks() - return config FINAL_VALIDATE_SCHEMA = _final_validate diff --git a/esphome/components/zwave_proxy/__init__.py b/esphome/components/zwave_proxy/__init__.py index d88f9f7041..14b8474045 100644 --- a/esphome/components/zwave_proxy/__init__.py +++ b/esphome/components/zwave_proxy/__init__.py @@ -11,7 +11,7 @@ zwave_proxy_ns = cg.esphome_ns.namespace("zwave_proxy") ZWaveProxy = zwave_proxy_ns.class_("ZWaveProxy", cg.Component, uart.UARTDevice) -def final_validate(config): +def final_validate(config) -> None: full_config = fv.full_config.get() if (wifi_conf := full_config.get(CONF_WIFI)) and ( wifi_conf.get(CONF_POWER_SAVE_MODE).lower() != "none" @@ -20,8 +20,6 @@ def final_validate(config): f"{CONF_WIFI} {CONF_POWER_SAVE_MODE} must be set to 'none' when using Z-Wave proxy" ) - return config - CONFIG_SCHEMA = ( cv.Schema( diff --git a/tests/component_tests/esp32_hosted/test_init.py b/tests/component_tests/esp32_hosted/test_init.py index cec81e4e83..5cc3f928cc 100644 --- a/tests/component_tests/esp32_hosted/test_init.py +++ b/tests/component_tests/esp32_hosted/test_init.py @@ -19,7 +19,7 @@ def test_final_validate_accepts_supported_idf( PlatformFramework.ESP32_IDF, platform_data={KEY_IDF_VERSION: cv.Version.parse(idf)}, ) - assert _final_validate({}) == {} + _final_validate({}) @pytest.mark.parametrize("idf", ["5.0.0", "5.2.2"]) diff --git a/tests/component_tests/image/test_init.py b/tests/component_tests/image/test_init.py index 78462463b1..f52c477c85 100644 --- a/tests/component_tests/image/test_init.py +++ b/tests/component_tests/image/test_init.py @@ -371,27 +371,28 @@ def test_migrate_returns_none_for_invalid_legacy_shapes( def test_validate_image_final_defaults_to_little_endian() -> None: - out = validate_image_final({CONF_FILE: "x.png"}) - assert out[CONF_BYTE_ORDER] == "LITTLE_ENDIAN" + config = {CONF_FILE: "x.png"} + validate_image_final(config) + assert config[CONF_BYTE_ORDER] == "LITTLE_ENDIAN" def test_validate_image_final_keeps_little_endian( caplog: pytest.LogCaptureFixture, ) -> None: + config = {CONF_FILE: "x.png", CONF_BYTE_ORDER: "LITTLE_ENDIAN"} with caplog.at_level(logging.WARNING): - out = validate_image_final( - {CONF_FILE: "x.png", CONF_BYTE_ORDER: "LITTLE_ENDIAN"} - ) - assert out[CONF_BYTE_ORDER] == "LITTLE_ENDIAN" + validate_image_final(config) + assert config[CONF_BYTE_ORDER] == "LITTLE_ENDIAN" assert "big-endian" not in caplog.text def test_validate_image_final_warns_on_big_endian( caplog: pytest.LogCaptureFixture, ) -> None: + config = {CONF_FILE: "x.png", CONF_BYTE_ORDER: "BIG_ENDIAN"} with caplog.at_level(logging.WARNING): - out = validate_image_final({CONF_FILE: "x.png", CONF_BYTE_ORDER: "BIG_ENDIAN"}) - assert out[CONF_BYTE_ORDER] == "BIG_ENDIAN" + validate_image_final(config) + assert config[CONF_BYTE_ORDER] == "BIG_ENDIAN" assert "big-endian" in caplog.text diff --git a/tests/component_tests/provisioning/test_provisioning.py b/tests/component_tests/provisioning/test_provisioning.py index 07f5065241..d3a3771bbc 100644 --- a/tests/component_tests/provisioning/test_provisioning.py +++ b/tests/component_tests/provisioning/test_provisioning.py @@ -37,7 +37,7 @@ def test_provisioning_accepts_a_registered_source( set_core_config(PlatformFramework.ESP32_IDF) register_source("network") # Should not raise. - assert FINAL_VALIDATE_SCHEMA({}) == {} + FINAL_VALIDATE_SCHEMA({}) def test_provisioning_warns_on_hardcoded_credentials( @@ -49,7 +49,7 @@ def test_provisioning_warns_on_hardcoded_credentials( register_source("network") report_hardcoded_credentials("wifi") with caplog.at_level(logging.WARNING): - assert FINAL_VALIDATE_SCHEMA({}) == {} + FINAL_VALIDATE_SCHEMA({}) assert "wifi" in caplog.text assert "credentials" in caplog.text @@ -62,7 +62,7 @@ def test_provisioning_no_warning_without_hardcoded_credentials( set_core_config(PlatformFramework.ESP32_IDF) register_source("network") with caplog.at_level(logging.WARNING): - assert FINAL_VALIDATE_SCHEMA({}) == {} + FINAL_VALIDATE_SCHEMA({}) assert "credentials" not in caplog.text From 7362c01c6744e0be6eae307a31486d59f8b50f49 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 16:14:06 -0500 Subject: [PATCH 097/160] [core] Replace base64 lookup tables with arithmetic mapping (#18454) --- esphome/core/alloc_helpers.cpp | 16 ++++-- esphome/core/helpers.cpp | 24 ++++----- tests/components/core/test_helpers.cpp | 67 ++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/esphome/core/alloc_helpers.cpp b/esphome/core/alloc_helpers.cpp index d9cfad70b9..f6130b7b78 100644 --- a/esphome/core/alloc_helpers.cpp +++ b/esphome/core/alloc_helpers.cpp @@ -88,9 +88,17 @@ std::string str_sprintf(const char *fmt, ...) { // --- Base64 helpers --- -static constexpr const char *BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; +// Map a 6-bit value (0-63) to its base64 character arithmetically. +// No lookup table: a table would occupy RAM on ESP8266 (.rodata lives in DRAM there). +static inline char base64_char(uint8_t index) { + if (index < 26) + return 'A' + index; + if (index < 52) + return 'a' + (index - 26); + if (index < 62) + return '0' + (index - 52); + return index == 62 ? '+' : '/'; +} // Encode 3 input bytes to 4 base64 characters, append 'count' to ret. static inline void base64_encode_triple(const char *char_array_3, int count, std::string &ret) { @@ -101,7 +109,7 @@ static inline void base64_encode_triple(const char *char_array_3, int count, std char_array_4[3] = char_array_3[2] & 0x3f; for (int j = 0; j < count; j++) - ret += BASE64_CHARS[static_cast(char_array_4[j])]; + ret += base64_char(static_cast(char_array_4[j])); } std::string base64_encode(const std::vector &buf) { return base64_encode(buf.data(), buf.size()); } diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index 8c4442f1b2..bd08d3b63e 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -579,13 +579,8 @@ int8_t step_to_accuracy_decimals(float step) { return str.length() - dot_pos - 1; } -// Use C-style string constant to store in ROM instead of RAM (saves 24 bytes) -static constexpr const char *BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; - -// Helper function to find the index of a base64/base64url character in the lookup table. -// Returns the character's position (0-63) if found, or 0 if not found. +// Map a base64/base64url character to its 6-bit value (0-63) arithmetically. +// No lookup table: a table would occupy RAM on ESP8266 (.rodata lives in DRAM there). // Supports both standard base64 (+/) and base64url (-_) alphabets. // NOTE: This returns 0 for both 'A' (valid base64 char at index 0) and invalid characters. // This is safe because is_base64() is ALWAYS checked before calling this function, @@ -593,13 +588,18 @@ static constexpr const char *BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" // stops processing at the first invalid character due to the is_base64() check in its // while loop condition, making this edge case harmless in practice. static inline uint8_t base64_find_char(char c) { - // Handle base64url variants: '-' maps to '+' (index 62), '_' maps to '/' (index 63) - if (c == '-') + if (c >= 'A' && c <= 'Z') + return c - 'A'; + if (c >= 'a' && c <= 'z') + return c - 'a' + 26; + if (c >= '0' && c <= '9') + return c - '0' + 52; + // base64url variants: '-' maps to '+' (index 62), '_' maps to '/' (index 63) + if (c == '+' || c == '-') return 62; - if (c == '_') + if (c == '/' || c == '_') return 63; - const char *pos = strchr(BASE64_CHARS, c); - return pos ? (pos - BASE64_CHARS) : 0; + return 0; } // Check if character is valid base64 or base64url diff --git a/tests/components/core/test_helpers.cpp b/tests/components/core/test_helpers.cpp index 5fb77ef753..3767b24d86 100644 --- a/tests/components/core/test_helpers.cpp +++ b/tests/components/core/test_helpers.cpp @@ -1,6 +1,7 @@ #include #include +#include "esphome/core/alloc_helpers.h" #include "esphome/core/helpers.h" namespace esphome::core::testing { @@ -213,4 +214,70 @@ TEST(BufAppendSepStr, Truncation) { EXPECT_EQ(end - buf, 7); } +// --- base64 encode/decode --- + +static const char BASE64_ALPHABET[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +// Pack 6-bit indices 0..63 into 48 bytes so encoding yields the full alphabet in order +TEST(Base64, EncodeProducesCanonicalAlphabet) { + uint8_t bytes[48]; + size_t n = 0; + for (uint8_t i = 0; i < 64; i += 4) { + bytes[n++] = (i << 2) | ((i + 1) >> 4); + bytes[n++] = ((i + 1) & 0x0F) << 4 | ((i + 2) >> 2); + bytes[n++] = ((i + 2) & 0x03) << 6 | (i + 3); + } + std::string encoded = base64_encode(bytes, sizeof(bytes)); // NOLINT(esphome-heap-allocation) - host test + EXPECT_EQ(encoded, BASE64_ALPHABET); +} + +// Decode the alphabet then re-encode: locks the encode and decode mappings together +TEST(Base64, DecodeCanonicalAlphabetRoundTrip) { + uint8_t buf[48]; + size_t len = base64_decode(std::string(BASE64_ALPHABET), buf, sizeof(buf)); + EXPECT_EQ(len, 48u); + std::string reencoded = base64_encode(buf, len); // NOLINT(esphome-heap-allocation) - host test + EXPECT_EQ(reencoded, BASE64_ALPHABET); +} + +TEST(Base64, DecodeBase64UrlMatchesStandard) { + std::string url = BASE64_ALPHABET; + for (char &c : url) { + if (c == '+') + c = '-'; + if (c == '/') + c = '_'; + } + uint8_t standard[48], urlsafe[48]; + size_t len_standard = base64_decode(std::string(BASE64_ALPHABET), standard, sizeof(standard)); + size_t len_url = base64_decode(url, urlsafe, sizeof(urlsafe)); + EXPECT_EQ(len_standard, len_url); + EXPECT_EQ(memcmp(standard, urlsafe, len_standard), 0); +} + +// RFC 4648 vectors cover both padding cases (len % 3 == 1 and len % 3 == 2) +TEST(Base64, Rfc4648Vectors) { + const struct { + const char *plain; + const char *encoded; + } vectors[] = { + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, + }; + for (const auto &v : vectors) { + const auto *plain = reinterpret_cast(v.plain); + std::string encoded = base64_encode(plain, strlen(v.plain)); // NOLINT(esphome-heap-allocation) - host test + EXPECT_EQ(encoded, v.encoded); + uint8_t buf[8]; + size_t len = base64_decode(reinterpret_cast(v.encoded), strlen(v.encoded), buf, sizeof(buf)); + EXPECT_EQ(len, strlen(v.plain)); + EXPECT_EQ(memcmp(buf, v.plain, len), 0); + } +} + } // namespace esphome::core::testing From 96e26c6a5f4d7eed5cae1a46eaa1d23b348b5c36 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:54:23 -0500 Subject: [PATCH 098/160] Bump bundled esphome-device-builder to 1.11.1 (#18475) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2f23b2f690..b78d183e02 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.1 RUN \ platformio settings set enable_telemetry No \ From 7be4566b411d96f7a103879c80f38c9248ec97a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:33:13 -0500 Subject: [PATCH 099/160] Bump platformdirs from 4.11.2 to 4.11.3 (#18468) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4b1708637d..a986646230 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.2 # native esp-idf toolchain global cache dir +platformdirs==4.11.3 # native esp-idf toolchain global cache dir filelock==3.32.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this From a347a2e8793243dbaa5ffcc4b07fe3481abbe252 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 18:58:11 -0500 Subject: [PATCH 100/160] [api] Bump noise-c to 0.1.19 (#18473) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 5ca9484336..cdc0d97c49 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.18") + cg.add_library("esphome/noise-c", "0.1.19") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 2c22523be5..39600d622a 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.18 ; api + esphome/noise-c@0.1.19 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.18 ; api + esphome/noise-c@0.1.19 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.18 ; used by api + esphome/noise-c@0.1.19 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 15a626bcf34cc6905bb5c972dcf74475a86af691 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:16:38 -0500 Subject: [PATCH 101/160] Bump bundled esphome-device-builder to 1.11.2 (#18477) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b78d183e02..4a8daeaaf6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.2 RUN \ platformio settings set enable_telemetry No \ From 4416aacebb4b991a3c8916efffa21a62fa42cf70 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 20:09:29 -0500 Subject: [PATCH 102/160] [socket] Fix multi-second TCP stalls on ESP8266 by yielding to the SYS context (#18455) --- .../components/socket/lwip_raw_tcp_impl.cpp | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index b80a394eec..8d00dbede2 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -45,6 +45,11 @@ namespace esphome::socket { static const char *const TAG = "socket"; +#ifdef USE_ESP8266 +// optimistic_yield() rate limit in microseconds of CONT time; cheap when hot. +static constexpr uint32_t ESP8266_YIELD_INTERVAL_US = 1000; +#endif + // set to 1 to enable verbose lwip logging #if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if) #define LWIP_LOG(msg, ...) ESP_LOGVV(TAG, "socket %p: " msg, this, ##__VA_ARGS__) @@ -535,6 +540,14 @@ ssize_t LWIPRawImpl::read_locked_(void *buf, size_t len) { } ssize_t LWIPRawImpl::read(void *buf, size_t len) { +#ifdef USE_ESP8266 + // Would block: yield to SYS so queued WiFi RX reaches lwip and this read + // may succeed. Without this, inbound segments can sit unprocessed for + // seconds while the main loop polls (CONT/SYS are cooperative on ESP8266). + if (this->waiting_for_data_()) { + optimistic_yield(ESP8266_YIELD_INTERVAL_US); + } +#endif // See waiting_for_data_() for safety of unlocked reads. if (this->recv_timeout_cs_ > 0 && this->waiting_for_data_()) { this->wait_for_data_(); @@ -545,6 +558,8 @@ ssize_t LWIPRawImpl::read(void *buf, size_t len) { } ssize_t LWIPRawImpl::readv(const struct iovec *iov, int iovcnt) { + // No ESP8266 SYS yield here: only read() needs it today. If a consumer + // switches to scatter-gather reads, mirror the yield from read(). // See waiting_for_data_() for safety of unlocked reads. if (this->recv_timeout_cs_ > 0 && this->waiting_for_data_()) { this->wait_for_data_(); @@ -609,19 +624,24 @@ int LWIPRawImpl::internal_output_() { } LWIP_LOG("tcp_output(%p)", this->pcb_); err_t err = tcp_output(this->pcb_); - if (err == ERR_ABRT) { - // sometimes lwip returns ERR_ABRT for no apparent reason - // the connection works fine afterwards, and back with ESPAsyncTCP we - // indirectly also ignored this error - // FIXME: figure out where this is returned and what it means in this context - LWIP_LOG(" -> err ERR_ABRT"); - return 0; - } if (err != ERR_OK) { LWIP_LOG(" -> err %d", err); - errno = ECONNRESET; - return -1; + // ERR_ABRT: sometimes lwip returns it for no apparent reason; the + // connection works fine afterwards, and back with ESPAsyncTCP we + // indirectly also ignored this error, so treat it as success for + // flush purposes too. + // FIXME: figure out where this is returned and what it means in this context + if (err != ERR_ABRT) { + errno = ECONNRESET; + return -1; + } } +#ifdef USE_ESP8266 + // Flushed: yield to SYS so the queued segments reach the WiFi driver + // instead of waiting seconds for an unrelated SYS slot. Callers only get + // here after a successful tcp_write, so idle paths never yield. + optimistic_yield(ESP8266_YIELD_INTERVAL_US); +#endif return 0; } From 463e3833dae23329ad484c1a549dab13c2de7541 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:19:48 +1200 Subject: [PATCH 103/160] [light] Replace rgb_order/is_rgbw/is_wrgb with channel_colors (#18474) --- .../beken_spi_led_strip/led_strip.cpp | 75 ++------- .../beken_spi_led_strip/led_strip.h | 23 +-- .../components/beken_spi_led_strip/light.py | 41 +++-- esphome/components/const/__init__.py | 2 + .../esp32_rmt_led_strip/led_strip.cpp | 86 ++--------- .../esp32_rmt_led_strip/led_strip.h | 29 +--- .../components/esp32_rmt_led_strip/light.py | 65 ++------ esphome/components/light/__init__.py | 106 +++++++++++++ esphome/components/light/channel_colors.h | 41 +++++ esphome/components/light/types.py | 3 + .../rp2040_pio_led_strip/led_strip.cpp | 59 ++----- .../rp2040_pio_led_strip/led_strip.h | 42 +---- .../components/rp2040_pio_led_strip/light.py | 33 ++-- .../common-ard-esp32_rmt_led_strip.yaml | 2 +- .../common-idf-esp32_rmt_led_strip.yaml | 2 +- .../beken_spi_led_strip/test.bk72xx-ard.yaml | 2 +- .../validate-legacy.bk72xx-ard.yaml | 10 ++ tests/components/e131/common-ard.yaml | 2 +- tests/components/e131/common-idf.yaml | 2 +- tests/components/e131/test.rp2040-ard.yaml | 2 +- .../esp32_rmt_led_strip/common.yaml | 4 +- .../test.esp32-s3-idf.yaml | 4 +- .../validate-legacy.esp32-idf.yaml | 23 +++ tests/components/partition/common-ard.yaml | 2 +- tests/components/partition/common-idf.yaml | 2 +- .../rp2040_pio_led_strip/common.yaml | 4 +- .../validate-legacy.rp2040-ard.yaml | 18 +++ tests/components/wled/test.esp32-ard.yaml | 2 +- .../components/light/test_channel_colors.py | 144 ++++++++++++++++++ .../components/test_esp32_rmt_led_strip.py | 57 ------- 30 files changed, 454 insertions(+), 433 deletions(-) create mode 100644 esphome/components/light/channel_colors.h create mode 100644 tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml create mode 100644 tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml create mode 100644 tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml create mode 100644 tests/unit_tests/components/light/test_channel_colors.py delete mode 100644 tests/unit_tests/components/test_esp32_rmt_led_strip.py diff --git a/esphome/components/beken_spi_led_strip/led_strip.cpp b/esphome/components/beken_spi_led_strip/led_strip.cpp index 9e14615d7a..0cf970b3cc 100644 --- a/esphome/components/beken_spi_led_strip/led_strip.cpp +++ b/esphome/components/beken_spi_led_strip/led_strip.cpp @@ -300,46 +300,12 @@ void BekenSPILEDStripLightOutput::write_state(light::LightState *state) { } light::ESPColorView BekenSPILEDStripLightOutput::get_view_internal(int32_t index) const { - int32_t r = 0, g = 0, b = 0; - switch (this->rgb_order_) { - case ORDER_RGB: - r = 0; - g = 1; - b = 2; - break; - case ORDER_RBG: - r = 0; - g = 2; - b = 1; - break; - case ORDER_GRB: - r = 1; - g = 0; - b = 2; - break; - case ORDER_GBR: - r = 2; - g = 0; - b = 1; - break; - case ORDER_BGR: - r = 2; - g = 1; - b = 0; - break; - case ORDER_BRG: - r = 1; - g = 2; - b = 0; - break; - } - uint8_t multiplier = this->is_rgbw_ || this->is_wrgb_ ? 4 : 3; - uint8_t white = this->is_wrgb_ ? 0 : 3; - - return {this->buf_ + (index * multiplier) + r + this->is_wrgb_, - this->buf_ + (index * multiplier) + g + this->is_wrgb_, - this->buf_ + (index * multiplier) + b + this->is_wrgb_, - this->is_rgbw_ || this->is_wrgb_ ? this->buf_ + (index * multiplier) + white : nullptr, + const light::ChannelColors &colors = this->channel_colors_; + uint8_t *led = this->buf_ + (index * colors.bytes_per_led()); + return {led + colors.r, + led + colors.g, + led + colors.b, + colors.has_white() ? led + colors.w : nullptr, &this->effect_data_[index], &this->correction_}; } @@ -349,35 +315,12 @@ void BekenSPILEDStripLightOutput::dump_config() { "Beken SPI LED Strip:\n" " Pin: %u", this->pin_); - const char *rgb_order; - switch (this->rgb_order_) { - case ORDER_RGB: - rgb_order = "RGB"; - break; - case ORDER_RBG: - rgb_order = "RBG"; - break; - case ORDER_GRB: - rgb_order = "GRB"; - break; - case ORDER_GBR: - rgb_order = "GBR"; - break; - case ORDER_BGR: - rgb_order = "BGR"; - break; - case ORDER_BRG: - rgb_order = "BRG"; - break; - default: - rgb_order = "UNKNOWN"; - break; - } + char channel_colors[5]; ESP_LOGCONFIG(TAG, - " RGB Order: %s\n" + " Channel colors: %s\n" " Max refresh rate: %" PRIu32 "\n" " Number of LEDs: %u", - rgb_order, this->max_refresh_rate_.value_or(0), this->num_leds_); + this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_.value_or(0), this->num_leds_); } float BekenSPILEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; } diff --git a/esphome/components/beken_spi_led_strip/led_strip.h b/esphome/components/beken_spi_led_strip/led_strip.h index 909634e266..1496e65d4d 100644 --- a/esphome/components/beken_spi_led_strip/led_strip.h +++ b/esphome/components/beken_spi_led_strip/led_strip.h @@ -3,6 +3,7 @@ #ifdef USE_BK72XX #include "esphome/components/light/addressable_light.h" +#include "esphome/components/light/channel_colors.h" #include "esphome/components/light/light_output.h" #include "esphome/core/color.h" #include "esphome/core/component.h" @@ -10,15 +11,6 @@ namespace esphome::beken_spi_led_strip { -enum RGBOrder : uint8_t { - ORDER_RGB, - ORDER_RBG, - ORDER_GRB, - ORDER_GBR, - ORDER_BGR, - ORDER_BRG, -}; - class BekenSPILEDStripLightOutput final : public light::AddressableLight { public: void setup() override; @@ -28,7 +20,7 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { int32_t size() const override { return this->num_leds_; } light::LightTraits get_traits() override { auto traits = light::LightTraits(); - if (this->is_rgbw_ || this->is_wrgb_) { + if (this->channel_colors_.has_white()) { traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}); } else { traits.set_supported_color_modes({light::ColorMode::RGB}); @@ -38,16 +30,13 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { void set_pin(uint8_t pin) { this->pin_ = pin; } void set_num_leds(uint16_t num_leds) { this->num_leds_ = num_leds; } - void set_is_rgbw(bool is_rgbw) { this->is_rgbw_ = is_rgbw; } - void set_is_wrgb(bool is_wrgb) { this->is_wrgb_ = is_wrgb; } + void set_channel_colors(light::ChannelColors channel_colors) { this->channel_colors_ = channel_colors; } /// Set a maximum refresh rate in µs as some lights do not like being updated too often. void set_max_refresh_rate(uint32_t interval_us) { this->max_refresh_rate_ = interval_us; } void set_led_params(uint8_t bit0, uint8_t bit1, uint32_t spi_frequency); - void set_rgb_order(RGBOrder rgb_order) { this->rgb_order_ = rgb_order; } - void clear_effect_data() override { for (int i = 0; i < this->size(); i++) this->effect_data_[i] = 0; @@ -58,7 +47,7 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { protected: light::ESPColorView get_view_internal(int32_t index) const override; - size_t get_buffer_size_() const { return this->num_leds_ * (this->is_rgbw_ || this->is_wrgb_ ? 4 : 3); } + size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); } uint8_t *buf_{nullptr}; uint8_t *effect_data_{nullptr}; @@ -66,13 +55,11 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { uint8_t pin_; uint16_t num_leds_; - bool is_rgbw_; - bool is_wrgb_; uint32_t spi_frequency_{6666666}; uint8_t bit0_{0xE0}; uint8_t bit1_{0xFC}; - RGBOrder rgb_order_; + light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE}; uint32_t last_refresh_{0}; optional max_refresh_rate_{}; diff --git a/esphome/components/beken_spi_led_strip/light.py b/esphome/components/beken_spi_led_strip/light.py index 9093b08b62..2be5842818 100644 --- a/esphome/components/beken_spi_led_strip/light.py +++ b/esphome/components/beken_spi_led_strip/light.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from esphome import pins import esphome.codegen as cg from esphome.components import libretiny, light +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB import esphome.config_validation as cv from esphome.const import ( CONF_CHIPSET, @@ -13,6 +14,7 @@ from esphome.const import ( CONF_PIN, CONF_RGB_ORDER, ) +from esphome.types import ConfigType CODEOWNERS = ["@Mat931"] DEPENDENCIES = ["libretiny"] @@ -22,17 +24,6 @@ BekenSPILEDStripLightOutput = beken_spi_led_strip_ns.class_( "BekenSPILEDStripLightOutput", light.AddressableLight ) -RGBOrder = beken_spi_led_strip_ns.enum("RGBOrder") - -RGB_ORDERS = { - "RGB": RGBOrder.ORDER_RGB, - "RBG": RGBOrder.ORDER_RBG, - "GRB": RGBOrder.ORDER_GRB, - "GBR": RGBOrder.ORDER_GBR, - "BGR": RGBOrder.ORDER_BGR, - "BRG": RGBOrder.ORDER_BRG, -} - @dataclass class LEDStripTimings: @@ -57,8 +48,6 @@ CHIPSETS = { } -CONF_IS_WRGB = "is_wrgb" - SUPPORTED_PINS = { libretiny.const.FAMILY_BK7231N: [16], libretiny.const.FAMILY_BK7231T: [16], @@ -79,10 +68,9 @@ def _validate_pin(value): return value -def _validate_num_leds(value): - max_num_leds = 165 # 170 - if value[CONF_IS_RGBW] or value[CONF_IS_WRGB]: - max_num_leds = 123 # 127 +def _validate_num_leds(value: ConfigType) -> ConfigType: + # A white channel makes each LED one byte wider, so fewer of them fit in the DMA buffer. + max_num_leds = 123 if "W" in value[CONF_CHANNEL_COLORS] else 165 # 127 / 170 if value[CONF_NUM_LEDS] > max_num_leds: raise cv.Invalid( f"The maximum number of LEDs for this configuration is {max_num_leds}.", @@ -99,18 +87,23 @@ CONFIG_SCHEMA = cv.All( pins.internal_gpio_output_pin_number, _validate_pin ), cv.Required(CONF_NUM_LEDS): cv.positive_not_null_int, - cv.Required(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), + cv.Optional(CONF_CHANNEL_COLORS): light.validate_channel_colors, + # Deprecated in favour of CONF_CHANNEL_COLORS, remove in 2027.3.0 + cv.Optional(CONF_RGB_ORDER): cv.one_of(*light.RGB_ORDERS, upper=True), + cv.Optional(CONF_IS_RGBW): cv.boolean, + cv.Optional(CONF_IS_WRGB): cv.boolean, cv.Optional(CONF_MAX_REFRESH_RATE): cv.positive_time_period_microseconds, cv.Required(CONF_CHIPSET): cv.one_of(*CHIPSETS, upper=True), - cv.Optional(CONF_IS_RGBW, default=False): cv.boolean, - cv.Optional(CONF_IS_WRGB, default=False): cv.boolean, } ), + light.migrate_channel_colors( + removed_in="2027.3.0", component="beken_spi_led_strip" + ), _validate_num_leds, ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_OUTPUT_ID]) await light.register_light(var, config) await cg.register_component(var, config) @@ -130,6 +123,6 @@ async def to_code(config): ) ) - cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) - cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) - cg.add(var.set_is_wrgb(config[CONF_IS_WRGB])) + cg.add( + var.set_channel_colors(light.channel_colors_struct(config[CONF_CHANNEL_COLORS])) + ) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 3ba89d2838..10710c8d29 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -10,6 +10,7 @@ CONF_ACCELEROMETER_RANGE = "accelerometer_range" CONF_B_CONSTANT = "b_constant" CONF_BREATH_VOC_EQUIVALENT = "breath_voc_equivalent" CONF_BYTE_ORDER = "byte_order" +CONF_CHANNEL_COLORS = "channel_colors" CONF_CLIMATE_ID = "climate_id" CONF_CO2_EQUIVALENT = "co2_equivalent" CONF_COLOR_DEPTH = "color_depth" @@ -22,6 +23,7 @@ CONF_GYROSCOPE_ODR = "gyroscope_odr" CONF_GYROSCOPE_RANGE = "gyroscope_range" CONF_IAQ = "iaq" CONF_IGNORE_NOT_FOUND = "ignore_not_found" +CONF_IS_WRGB = "is_wrgb" CONF_LABEL = "label" CONF_LIBRETINY = "libretiny" CONF_LOOP = "loop" diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.cpp b/esphome/components/esp32_rmt_led_strip/led_strip.cpp index 95391ef100..7cac1dfb41 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.cpp +++ b/esphome/components/esp32_rmt_led_strip/led_strip.cpp @@ -221,46 +221,12 @@ void ESP32RMTLEDStripLightOutput::write_state(light::LightState *state) { } light::ESPColorView ESP32RMTLEDStripLightOutput::get_view_internal(int32_t index) const { - int32_t r = 0, g = 0, b = 0; - switch (this->rgb_order_) { - case ORDER_RGB: - r = 0; - g = 1; - b = 2; - break; - case ORDER_RBG: - r = 0; - g = 2; - b = 1; - break; - case ORDER_GRB: - r = 1; - g = 0; - b = 2; - break; - case ORDER_GBR: - r = 2; - g = 0; - b = 1; - break; - case ORDER_BGR: - r = 2; - g = 1; - b = 0; - break; - case ORDER_BRG: - r = 1; - g = 2; - b = 0; - break; - } - uint8_t multiplier = this->is_rgbw_ || this->is_wrgb_ ? 4 : 3; - uint8_t white = this->is_wrgb_ ? 0 : this->white_index_; - - return {this->buf_ + (index * multiplier) + r + (white <= r), - this->buf_ + (index * multiplier) + g + (white <= g), - this->buf_ + (index * multiplier) + b + (white <= b), - this->is_rgbw_ || this->is_wrgb_ ? this->buf_ + (index * multiplier) + white : nullptr, + const light::ChannelColors &colors = this->channel_colors_; + uint8_t *led = this->buf_ + (index * colors.bytes_per_led()); + return {led + colors.r, + led + colors.g, + led + colors.b, + colors.has_white() ? led + colors.w : nullptr, &this->effect_data_[index], &this->correction_}; } @@ -271,46 +237,12 @@ void ESP32RMTLEDStripLightOutput::dump_config() { " Pin: %u", this->pin_); ESP_LOGCONFIG(TAG, " RMT Symbols: %" PRIu32, this->rmt_symbols_); - const char *rgb_order; - switch (this->rgb_order_) { - case ORDER_RGB: - rgb_order = "RGB"; - break; - case ORDER_RBG: - rgb_order = "RBG"; - break; - case ORDER_GRB: - rgb_order = "GRB"; - break; - case ORDER_GBR: - rgb_order = "GBR"; - break; - case ORDER_BGR: - rgb_order = "BGR"; - break; - case ORDER_BRG: - rgb_order = "BRG"; - break; - default: - rgb_order = "UNKNOWN"; - break; - } - if (this->is_rgbw_ || this->is_wrgb_) { - char rgbw_order[5]; - uint8_t white = this->is_wrgb_ ? 0 : this->white_index_; - uint8_t rgb_index = 0; - for (uint8_t i = 0; i < 4; i++) { - rgbw_order[i] = i == white ? 'W' : rgb_order[rgb_index++]; - } - rgbw_order[4] = '\0'; - ESP_LOGCONFIG(TAG, " RGBW Order: %s", rgbw_order); - } else { - ESP_LOGCONFIG(TAG, " RGB Order: %s", rgb_order); - } + char channel_colors[5]; ESP_LOGCONFIG(TAG, + " Channel colors: %s\n" " Max refresh rate: %" PRIu32 "\n" " Number of LEDs: %u", - this->max_refresh_rate_.value_or(0), this->num_leds_); + this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_.value_or(0), this->num_leds_); } float ESP32RMTLEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; } diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.h b/esphome/components/esp32_rmt_led_strip/led_strip.h index 3e31309bff..61aac06d76 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.h +++ b/esphome/components/esp32_rmt_led_strip/led_strip.h @@ -3,6 +3,7 @@ #ifdef USE_ESP32 #include "esphome/components/light/addressable_light.h" +#include "esphome/components/light/channel_colors.h" #include "esphome/components/light/light_output.h" #include "esphome/core/color.h" #include "esphome/core/component.h" @@ -15,15 +16,6 @@ namespace esphome::esp32_rmt_led_strip { -enum RGBOrder : uint8_t { - ORDER_RGB, - ORDER_RBG, - ORDER_GRB, - ORDER_GBR, - ORDER_BGR, - ORDER_BRG, -}; - struct LedParams { rmt_symbol_word_t bit0; rmt_symbol_word_t bit1; @@ -39,7 +31,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { int32_t size() const override { return this->num_leds_; } light::LightTraits get_traits() override { auto traits = light::LightTraits(); - if (this->is_rgbw_ || this->is_wrgb_) { + if (this->channel_colors_.has_white()) { traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}); } else { traits.set_supported_color_modes({light::ColorMode::RGB}); @@ -50,13 +42,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { void set_pin(uint8_t pin) { this->pin_ = pin; } void set_inverted(bool inverted) { this->invert_out_ = inverted; } void set_num_leds(uint16_t num_leds) { this->num_leds_ = num_leds; } - void set_is_rgbw(bool is_rgbw) { this->is_rgbw_ = is_rgbw; } - void set_is_wrgb(bool is_wrgb) { this->is_wrgb_ = is_wrgb; } - void set_rgbw_order(uint8_t white_index) { - this->is_rgbw_ = true; - this->is_wrgb_ = false; - this->white_index_ = white_index; - } + void set_channel_colors(light::ChannelColors channel_colors) { this->channel_colors_ = channel_colors; } void set_use_dma(bool use_dma) { this->use_dma_ = use_dma; } void set_use_psram(bool use_psram) { this->use_psram_ = use_psram; } @@ -66,7 +52,6 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { void set_led_params(uint32_t bit0_high, uint32_t bit0_low, uint32_t bit1_high, uint32_t bit1_low, uint32_t reset_time_high, uint32_t reset_time_low); - void set_rgb_order(RGBOrder rgb_order) { this->rgb_order_ = rgb_order; } void set_rmt_symbols(uint32_t rmt_symbols) { this->rmt_symbols_ = rmt_symbols; } void clear_effect_data() override { @@ -79,7 +64,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { protected: light::ESPColorView get_view_internal(int32_t index) const override; - size_t get_buffer_size_() const { return this->num_leds_ * (this->is_rgbw_ || this->is_wrgb_ ? 4 : 3); } + size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); } uint8_t *buf_{nullptr}; uint8_t *effect_data_{nullptr}; @@ -94,15 +79,11 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { uint32_t rmt_symbols_{48}; uint8_t pin_; uint16_t num_leds_; - bool is_rgbw_{false}; - bool is_wrgb_{false}; - // An index after the RGB channels makes offset adjustment a no-op for three-channel strips. - uint8_t white_index_{3}; bool use_dma_{false}; bool use_psram_{false}; bool invert_out_{false}; - RGBOrder rgb_order_{ORDER_RGB}; + light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE}; uint32_t last_refresh_{0}; optional max_refresh_rate_{}; diff --git a/esphome/components/esp32_rmt_led_strip/light.py b/esphome/components/esp32_rmt_led_strip/light.py index 2722a9b656..571b7d93b8 100644 --- a/esphome/components/esp32_rmt_led_strip/light.py +++ b/esphome/components/esp32_rmt_led_strip/light.py @@ -1,10 +1,9 @@ from dataclasses import dataclass -import logging from esphome import pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, light -from esphome.components.const import CONF_USE_PSRAM +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB, CONF_USE_PSRAM from esphome.components.esp32 import include_builtin_idf_component import esphome.config_validation as cv from esphome.const import ( @@ -22,8 +21,6 @@ from esphome.const import ( ) from esphome.types import ConfigType -_LOGGER = logging.getLogger(__name__) - CODEOWNERS = ["@jesserockz"] DEPENDENCIES = ["esp32"] @@ -32,17 +29,6 @@ ESP32RMTLEDStripLightOutput = esp32_rmt_led_strip_ns.class_( "ESP32RMTLEDStripLightOutput", light.AddressableLight ) -RGBOrder = esp32_rmt_led_strip_ns.enum("RGBOrder") - -RGB_ORDERS = { - "RGB": RGBOrder.ORDER_RGB, - "RBG": RGBOrder.ORDER_RBG, - "GRB": RGBOrder.ORDER_GRB, - "GBR": RGBOrder.ORDER_GBR, - "BGR": RGBOrder.ORDER_BGR, - "BRG": RGBOrder.ORDER_BRG, -} - @dataclass class LEDStripTimings: @@ -62,8 +48,6 @@ CHIPSETS = { "SM16703": LEDStripTimings(300, 900, 900, 300, 0, 0), } -CONF_IS_WRGB = "is_wrgb" -CONF_RGBW_ORDER = "rgbw_order" CONF_BIT0_HIGH = "bit0_high" CONF_BIT0_LOW = "bit0_low" CONF_BIT1_HIGH = "bit1_high" @@ -72,26 +56,6 @@ CONF_RESET_HIGH = "reset_high" CONF_RESET_LOW = "reset_low" -def _validate_rgbw_order(value: str) -> str: - value = cv.string(value).upper() - if len(value) != 4 or set(value) != set("RGBW"): - raise cv.Invalid("RGBW order must be a permutation of RGBW") - return value - - -def _split_rgbw_order(rgbw_order: str) -> tuple[str, int]: - return rgbw_order.replace("W", ""), rgbw_order.index("W") - - -def _validate_rgbw_order_exclusivity(config: ConfigType) -> ConfigType: - if CONF_RGBW_ORDER in config and (config[CONF_IS_RGBW] or config[CONF_IS_WRGB]): - raise cv.Invalid( - f"'{CONF_RGBW_ORDER}' cannot be used with '{CONF_IS_RGBW}' or " - f"'{CONF_IS_WRGB}'" - ) - return config - - CONFIG_SCHEMA = cv.All( esp32.only_on_variant( unsupported=list(esp32_rmt.VARIANTS_NO_RMT), @@ -102,8 +66,11 @@ CONFIG_SCHEMA = cv.All( cv.GenerateID(CONF_OUTPUT_ID): cv.declare_id(ESP32RMTLEDStripLightOutput), cv.Required(CONF_PIN): pins.internal_gpio_output_pin_schema, cv.Required(CONF_NUM_LEDS): cv.positive_not_null_int, - cv.Optional(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), - cv.Optional(CONF_RGBW_ORDER): _validate_rgbw_order, + cv.Optional(CONF_CHANNEL_COLORS): light.validate_channel_colors, + # Deprecated in favour of CONF_CHANNEL_COLORS, remove in 2027.3.0 + cv.Optional(CONF_RGB_ORDER): cv.one_of(*light.RGB_ORDERS, upper=True), + cv.Optional(CONF_IS_RGBW): cv.boolean, + cv.Optional(CONF_IS_WRGB): cv.boolean, cv.SplitDefault( CONF_RMT_SYMBOLS, esp32=192, @@ -117,8 +84,6 @@ CONFIG_SCHEMA = cv.All( ): cv.int_range(min=2), cv.Optional(CONF_MAX_REFRESH_RATE): cv.positive_time_period_microseconds, cv.Optional(CONF_CHIPSET): cv.one_of(*CHIPSETS, upper=True), - cv.Optional(CONF_IS_RGBW, default=False): cv.boolean, - cv.Optional(CONF_IS_WRGB, default=False): cv.boolean, cv.Optional(CONF_USE_DMA): cv.All( esp32.only_on_variant( supported=[esp32.VARIANT_ESP32P4, esp32.VARIANT_ESP32S3] @@ -153,12 +118,13 @@ CONFIG_SCHEMA = cv.All( } ).extend(cv.COMPONENT_SCHEMA), cv.has_exactly_one_key(CONF_CHIPSET, CONF_BIT0_HIGH), - cv.has_exactly_one_key(CONF_RGB_ORDER, CONF_RGBW_ORDER), - _validate_rgbw_order_exclusivity, + light.migrate_channel_colors( + removed_in="2027.3.0", component="esp32_rmt_led_strip" + ), ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: # Re-enable ESP-IDF's RMT driver (excluded by default to save compile time) include_builtin_idf_component("esp_driver_rmt") @@ -198,14 +164,9 @@ async def to_code(config): ) ) - if (rgbw_order := config.get(CONF_RGBW_ORDER)) is not None: - rgb_order, white_index = _split_rgbw_order(rgbw_order) - cg.add(var.set_rgb_order(RGB_ORDERS[rgb_order])) - cg.add(var.set_rgbw_order(white_index)) - else: - cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) - cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) - cg.add(var.set_is_wrgb(config[CONF_IS_WRGB])) + cg.add( + var.set_channel_colors(light.channel_colors_struct(config[CONF_CHANNEL_COLORS])) + ) cg.add(var.set_use_psram(config[CONF_USE_PSRAM])) cg.add(var.set_rmt_symbols(config[CONF_RMT_SYMBOLS])) if CONF_USE_DMA in config: diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index b5b3d7c905..175f5b43cf 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -1,9 +1,12 @@ +from collections.abc import Callable from dataclasses import dataclass, field import enum +import logging import esphome.automation as auto import esphome.codegen as cg from esphome.components import mqtt, power_supply, web_server +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB import esphome.config_validation as cv from esphome.const import ( CONF_BLUE, @@ -23,6 +26,7 @@ from esphome.const import ( CONF_ICON, CONF_ID, CONF_INITIAL_STATE, + CONF_IS_RGBW, CONF_MQTT_ID, CONF_NAME, CONF_ON_STATE, @@ -32,6 +36,7 @@ from esphome.const import ( CONF_POWER_SUPPLY, CONF_RED, CONF_RESTORE_MODE, + CONF_RGB_ORDER, CONF_STATE, CONF_TRIGGER_ID, CONF_WARM_WHITE, @@ -61,6 +66,7 @@ from .effects import ( from .types import ( # noqa: F401 AddressableLight, AddressableLightState, + ChannelColors, ColorMode, LightOutput, LightState, @@ -71,6 +77,8 @@ from .types import ( # noqa: F401 light_ns, ) +_LOGGER = logging.getLogger(__name__) + CODEOWNERS = ["@esphome/core"] IS_PLATFORM_COMPONENT = True @@ -165,6 +173,104 @@ def available_effects_str(effects: list) -> str: return ", ".join(f"'{name}'" for name in available) if available else "none" +# Accepted values of the deprecated `rgb_order` key. +RGB_ORDERS = ("RGB", "RBG", "GRB", "GBR", "BGR", "BRG") + +_RGB_CHANNELS = frozenset("RGB") +_RGBW_CHANNELS = frozenset("RGBW") + + +def validate_channel_colors(value: str) -> str: + """Validate the channel order of an addressable strip, e.g. "GRB" or "WRGB".""" + value = cv.string_strict(value).upper() + channels = frozenset(value) + if len(channels) != len(value) or channels not in (_RGB_CHANNELS, _RGBW_CHANNELS): + raise cv.Invalid( + f"'{value}' is not a valid channel order. List each of R, G and B exactly " + "once, optionally with a single W, in the order the strip expects them " + "(for example GRB, GRBW or WRGB)" + ) + return value + + +def channel_colors_struct(value: str) -> cg.StructInitializer: + """Build the C++ `light::ChannelColors` for a validated channel order string.""" + return cg.StructInitializer( + ChannelColors, + ("r", value.index("R")), + ("g", value.index("G")), + ("b", value.index("B")), + ( + "w", + value.index("W") + if "W" in value + else cg.RawExpression(f"{ChannelColors}::NO_WHITE"), + ), + ) + + +def _quote_and_join(keys: list[str]) -> str: + """Quote each key and join them into a readable list, e.g. "'a', 'b' and 'c'".""" + quoted = [f"'{key}'" for key in keys] + if len(quoted) == 1: + return quoted[0] + return f"{', '.join(quoted[:-1])} and {quoted[-1]}" + + +def migrate_channel_colors( + *, removed_in: str, component: str +) -> Callable[[ConfigType], ConfigType]: + """Fold the deprecated `rgb_order`, `is_rgbw` and `is_wrgb` keys into `channel_colors`. + + This also enforces that `channel_colors` is set, which the schema cannot do on its + own while the deprecated keys are still accepted. After this runs, `to_code` only + ever sees `channel_colors`. + """ + + def validator(config: ConfigType) -> ConfigType: + config = config.copy() + deprecated = [ + key for key in (CONF_RGB_ORDER, CONF_IS_RGBW, CONF_IS_WRGB) if key in config + ] + if CONF_CHANNEL_COLORS in config: + if deprecated: + raise cv.Invalid( + f"'{CONF_CHANNEL_COLORS}' cannot be combined with " + f"{_quote_and_join(deprecated)}" + ) + return config + if CONF_RGB_ORDER not in config: + raise cv.Invalid( + f"'{CONF_CHANNEL_COLORS}' is required", path=[CONF_CHANNEL_COLORS] + ) + rgb_order = config.pop(CONF_RGB_ORDER) + is_rgbw = config.pop(CONF_IS_RGBW, False) + is_wrgb = config.pop(CONF_IS_WRGB, False) + if is_rgbw and is_wrgb: + raise cv.Invalid( + f"'{CONF_IS_RGBW}' and '{CONF_IS_WRGB}' cannot both be enabled" + ) + if is_wrgb: + channel_colors = f"W{rgb_order}" + elif is_rgbw: + channel_colors = f"{rgb_order}W" + else: + channel_colors = rgb_order + _LOGGER.warning( + "[%s] %s %s deprecated, use '%s: %s'. Will be removed in %s", + component, + _quote_and_join(deprecated), + "are" if len(deprecated) > 1 else "is", + CONF_CHANNEL_COLORS, + channel_colors, + removed_in, + ) + config[CONF_CHANNEL_COLORS] = channel_colors + return config + + return validator + + def _final_validate(config: ConfigType) -> None: """Validate all recorded effect name references against their target lights. diff --git a/esphome/components/light/channel_colors.h b/esphome/components/light/channel_colors.h new file mode 100644 index 0000000000..9d8f46d575 --- /dev/null +++ b/esphome/components/light/channel_colors.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +namespace esphome::light { + +/// Which byte of an addressable LED's data carries each colour. +/// +/// Built from a configuration string such as "GRB" or "WRGB": every field holds the +/// position that colour occupies in the bytes the strip expects. `w` is NO_WHITE when +/// the strip has no separate white channel. +struct ChannelColors { + /// Value of `w` for a strip that only has red, green and blue channels. + static constexpr uint8_t NO_WHITE = 0xFF; + + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t w; + + bool has_white() const { return this->w != NO_WHITE; } + + uint8_t bytes_per_led() const { return this->has_white() ? 4 : 3; } + + /// Write the order back out as text, e.g. "GRBW". + /// + /// `buf` must have room for at least 5 characters. Returns `buf` so the result can be + /// passed straight to a log call. + const char *to_string(char *buf) const { + buf[this->r] = 'R'; + buf[this->g] = 'G'; + buf[this->b] = 'B'; + if (this->has_white()) { + buf[this->w] = 'W'; + } + buf[this->bytes_per_led()] = '\0'; + return buf; + } +}; + +} // namespace esphome::light diff --git a/esphome/components/light/types.py b/esphome/components/light/types.py index 9c1c7331d1..1778aa8410 100644 --- a/esphome/components/light/types.py +++ b/esphome/components/light/types.py @@ -16,6 +16,9 @@ LightColorValues = light_ns.class_("LightColorValues") LightStateRTCState = light_ns.struct("LightStateRTCState") LightCall = light_ns.class_("LightCall") +# Addressable strips +ChannelColors = light_ns.struct("ChannelColors") + # Color modes ColorMode = light_ns.enum("ColorMode", is_class=True) COLOR_MODES = { diff --git a/esphome/components/rp2040_pio_led_strip/led_strip.cpp b/esphome/components/rp2040_pio_led_strip/led_strip.cpp index cf7041931e..1f4bea9ecd 100644 --- a/esphome/components/rp2040_pio_led_strip/led_strip.cpp +++ b/esphome/components/rp2040_pio_led_strip/led_strip.cpp @@ -107,10 +107,10 @@ void RP2040PIOLEDStripLightOutput::setup() { pio_get_dreq(this->pio_, this->sm_, true)); // set the DREQ to the state machine's TX FIFO dma_channel_configure(this->dma_chan_, &this->dma_config_, - &this->pio_->txf[this->sm_], // write to the state machine's TX FIFO - this->buf_, // read from memory - this->is_rgbw_ ? num_leds_ * 4 : num_leds_ * 3, // number of bytes to transfer - false // don't start yet + &this->pio_->txf[this->sm_], // write to the state machine's TX FIFO + this->buf_, // read from memory + this->get_buffer_size_(), // number of bytes to transfer + false // don't start yet ); // Initialize the semaphore for this DMA channel @@ -142,58 +142,25 @@ void RP2040PIOLEDStripLightOutput::write_state(light::LightState *state) { } light::ESPColorView RP2040PIOLEDStripLightOutput::get_view_internal(int32_t index) const { - int32_t r = 0, g = 0, b = 0; - switch (this->rgb_order_) { - case ORDER_RGB: - r = 0; - g = 1; - b = 2; - break; - case ORDER_RBG: - r = 0; - g = 2; - b = 1; - break; - case ORDER_GRB: - r = 1; - g = 0; - b = 2; - break; - case ORDER_GBR: - r = 2; - g = 0; - b = 1; - break; - case ORDER_BGR: - r = 2; - g = 1; - b = 0; - break; - case ORDER_BRG: - r = 1; - g = 2; - b = 0; - break; - } - uint8_t multiplier = this->is_rgbw_ ? 4 : 3; - return {this->buf_ + (index * multiplier) + r, - this->buf_ + (index * multiplier) + g, - this->buf_ + (index * multiplier) + b, - this->is_rgbw_ ? this->buf_ + (index * multiplier) + 3 : nullptr, + const light::ChannelColors &colors = this->channel_colors_; + uint8_t *led = this->buf_ + (index * colors.bytes_per_led()); + return {led + colors.r, + led + colors.g, + led + colors.b, + colors.has_white() ? led + colors.w : nullptr, &this->effect_data_[index], &this->correction_}; } void RP2040PIOLEDStripLightOutput::dump_config() { + char channel_colors[5]; ESP_LOGCONFIG(TAG, "RP2040 PIO LED Strip Light Output:\n" " Pin: GPIO%d\n" " Number of LEDs: %d\n" - " RGBW: %s\n" - " RGB Order: %s\n" + " Channel colors: %s\n" " Max Refresh Rate: %f Hz", - this->pin_, this->num_leds_, YESNO(this->is_rgbw_), rgb_order_to_string(this->rgb_order_), - this->max_refresh_rate_); + this->pin_, this->num_leds_, this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_); } float RP2040PIOLEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; } diff --git a/esphome/components/rp2040_pio_led_strip/led_strip.h b/esphome/components/rp2040_pio_led_strip/led_strip.h index c499f0a7ca..b2162f641d 100644 --- a/esphome/components/rp2040_pio_led_strip/led_strip.h +++ b/esphome/components/rp2040_pio_led_strip/led_strip.h @@ -7,6 +7,7 @@ #include "esphome/core/helpers.h" #include "esphome/components/light/addressable_light.h" +#include "esphome/components/light/channel_colors.h" #include "esphome/components/light/light_output.h" #include @@ -18,15 +19,6 @@ namespace esphome::rp2040_pio_led_strip { -enum RGBOrder : uint8_t { - ORDER_RGB, - ORDER_RBG, - ORDER_GRB, - ORDER_GBR, - ORDER_BGR, - ORDER_BRG, -}; - enum Chipset : uint8_t { CHIPSET_WS2812, CHIPSET_WS2812B, @@ -36,25 +28,6 @@ enum Chipset : uint8_t { CHIPSET_CUSTOM = 0xFF, }; -inline const char *rgb_order_to_string(RGBOrder order) { - switch (order) { - case ORDER_RGB: - return "RGB"; - case ORDER_RBG: - return "RBG"; - case ORDER_GRB: - return "GRB"; - case ORDER_GBR: - return "GBR"; - case ORDER_BGR: - return "BGR"; - case ORDER_BRG: - return "BRG"; - default: - return "UNKNOWN"; - } -} - using init_fn = void (*)(PIO pio, uint sm, uint offset, uint pin, float freq); class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { @@ -66,13 +39,14 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { int32_t size() const override { return this->num_leds_; } light::LightTraits get_traits() override { auto traits = light::LightTraits(); - this->is_rgbw_ ? traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}) - : traits.set_supported_color_modes({light::ColorMode::RGB}); + this->channel_colors_.has_white() + ? traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}) + : traits.set_supported_color_modes({light::ColorMode::RGB}); return traits; } void set_pin(uint8_t pin) { this->pin_ = pin; } void set_num_leds(uint32_t num_leds) { this->num_leds_ = num_leds; } - void set_is_rgbw(bool is_rgbw) { this->is_rgbw_ = is_rgbw; } + void set_channel_colors(light::ChannelColors channel_colors) { this->channel_colors_ = channel_colors; } void set_max_refresh_rate(float interval_us) { this->max_refresh_rate_ = interval_us; } @@ -81,7 +55,6 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { void set_init_function(init_fn init) { this->init_ = init; } void set_chipset(Chipset chipset) { this->chipset_ = chipset; }; - void set_rgb_order(RGBOrder rgb_order) { this->rgb_order_ = rgb_order; } void clear_effect_data() override { for (int i = 0; i < this->size(); i++) { this->effect_data_[i] = 0; @@ -93,7 +66,7 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { protected: light::ESPColorView get_view_internal(int32_t index) const override; - size_t get_buffer_size_() const { return this->num_leds_ * (3 + this->is_rgbw_); } + size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); } static void dma_write_complete_handler(); @@ -102,14 +75,13 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { uint8_t pin_; uint32_t num_leds_; - bool is_rgbw_; pio_hw_t *pio_; uint sm_; uint dma_chan_; dma_channel_config dma_config_; - RGBOrder rgb_order_{ORDER_RGB}; + light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE}; Chipset chipset_{CHIPSET_CUSTOM}; uint32_t last_refresh_{0}; diff --git a/esphome/components/rp2040_pio_led_strip/light.py b/esphome/components/rp2040_pio_led_strip/light.py index b3f816102a..9f7479edd0 100644 --- a/esphome/components/rp2040_pio_led_strip/light.py +++ b/esphome/components/rp2040_pio_led_strip/light.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from esphome import pins import esphome.codegen as cg from esphome.components import light, rp2 +from esphome.components.const import CONF_CHANNEL_COLORS import esphome.config_validation as cv from esphome.const import ( CONF_CHIPSET, @@ -13,6 +14,7 @@ from esphome.const import ( CONF_PIN, CONF_RGB_ORDER, ) +from esphome.types import ConfigType from esphome.util import _LOGGER @@ -37,7 +39,7 @@ def get_nops(timing): return nops -def generate_assembly_code(id, rgbw, t0h, t0l, t1h, t1l): +def generate_assembly_code(id, t0h, t0l, t1h, t1l): """ Generate assembly code with the given timing values. """ @@ -139,8 +141,6 @@ RP2040PIOLEDStripLightOutput = rp2040_pio_led_strip_ns.class_( "RP2040PIOLEDStripLightOutput", light.AddressableLight ) -RGBOrder = rp2040_pio_led_strip_ns.enum("RGBOrder") - Chipset = rp2040_pio_led_strip_ns.enum("Chipset") CHIPSETS = { @@ -159,15 +159,6 @@ class LEDStripTimings: T1L: int -RGB_ORDERS = { - "RGB": RGBOrder.ORDER_RGB, - "RBG": RGBOrder.ORDER_RBG, - "GRB": RGBOrder.ORDER_GRB, - "GBR": RGBOrder.ORDER_GBR, - "BGR": RGBOrder.ORDER_BGR, - "BRG": RGBOrder.ORDER_BRG, -} - CHIPSET_TIMINGS = { "WS2812": LEDStripTimings(20, 40, 46, 34), "WS2812B": LEDStripTimings(23, 49, 46, 26), @@ -199,10 +190,12 @@ CONFIG_SCHEMA = cv.All( cv.GenerateID(CONF_OUTPUT_ID): cv.declare_id(RP2040PIOLEDStripLightOutput), cv.Required(CONF_PIN): pins.internal_gpio_output_pin_number, cv.Required(CONF_NUM_LEDS): cv.positive_not_null_int, - cv.Required(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), + cv.Optional(CONF_CHANNEL_COLORS): light.validate_channel_colors, + # Deprecated in favour of CONF_CHANNEL_COLORS, remove in 2027.3.0 + cv.Optional(CONF_RGB_ORDER): cv.one_of(*light.RGB_ORDERS, upper=True), + cv.Optional(CONF_IS_RGBW): cv.boolean, cv.Required(CONF_PIO): cv.one_of(0, 1, int=True), cv.Optional(CONF_CHIPSET): cv.enum(CHIPSETS, upper=True), - cv.Optional(CONF_IS_RGBW, default=False): cv.boolean, cv.Inclusive( CONF_BIT0_HIGH, "custom", @@ -222,10 +215,13 @@ CONFIG_SCHEMA = cv.All( } ), cv.has_exactly_one_key(CONF_CHIPSET, CONF_BIT0_HIGH), + light.migrate_channel_colors( + removed_in="2027.3.0", component="rp2040_pio_led_strip" + ), ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_OUTPUT_ID]) id = config[CONF_ID].id await light.register_light(var, config) @@ -234,8 +230,9 @@ async def to_code(config): cg.add(var.set_num_leds(config[CONF_NUM_LEDS])) cg.add(var.set_pin(config[CONF_PIN])) - cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) - cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) + cg.add( + var.set_channel_colors(light.channel_colors_struct(config[CONF_CHANNEL_COLORS])) + ) cg.add(var.set_pio(config[CONF_PIO])) cg.add(var.set_program(cg.RawExpression(f"&rp2040_pio_led_strip_{id}_program"))) @@ -255,7 +252,6 @@ async def to_code(config): key, generate_assembly_code( id, - config[CONF_IS_RGBW], CHIPSET_TIMINGS[chipset].T0H, CHIPSET_TIMINGS[chipset].T0L, CHIPSET_TIMINGS[chipset].T1H, @@ -270,7 +266,6 @@ async def to_code(config): key, generate_assembly_code( id, - config[CONF_IS_RGBW], time_to_cycles(config[CONF_BIT0_HIGH]), time_to_cycles(config[CONF_BIT0_LOW]), time_to_cycles(config[CONF_BIT1_HIGH]), diff --git a/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml b/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml index a071f9df91..d21c4b61b9 100644 --- a/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml +++ b/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml @@ -3,7 +3,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} diff --git a/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml b/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml index a071f9df91..d21c4b61b9 100644 --- a/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml +++ b/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml @@ -3,7 +3,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} diff --git a/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml b/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml index 15409caeaf..2bb831848c 100644 --- a/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml +++ b/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml @@ -1,6 +1,6 @@ light: - platform: beken_spi_led_strip - rgb_order: GRB + channel_colors: GRB pin: P16 num_leds: 30 chipset: ws2812 diff --git a/tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml b/tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml new file mode 100644 index 0000000000..3ca78398c3 --- /dev/null +++ b/tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml @@ -0,0 +1,10 @@ +# The deprecated rgb_order / is_rgbw / is_wrgb keys, kept working until 2027.3.0. +# Config-only, and only one strip because P16 is the sole supported pin. +light: + - platform: beken_spi_led_strip + name: Legacy RGBW + pin: P16 + num_leds: 30 + chipset: sk6812 + rgb_order: GRB + is_rgbw: true # -> GRBW diff --git a/tests/components/e131/common-ard.yaml b/tests/components/e131/common-ard.yaml index 8300dbb01b..48ccafc2d2 100644 --- a/tests/components/e131/common-ard.yaml +++ b/tests/components/e131/common-ard.yaml @@ -5,7 +5,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} effects: diff --git a/tests/components/e131/common-idf.yaml b/tests/components/e131/common-idf.yaml index 8300dbb01b..48ccafc2d2 100644 --- a/tests/components/e131/common-idf.yaml +++ b/tests/components/e131/common-idf.yaml @@ -5,7 +5,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} effects: diff --git a/tests/components/e131/test.rp2040-ard.yaml b/tests/components/e131/test.rp2040-ard.yaml index 4593784ef9..89255e2d87 100644 --- a/tests/components/e131/test.rp2040-ard.yaml +++ b/tests/components/e131/test.rp2040-ard.yaml @@ -6,7 +6,7 @@ light: pin: 2 pio: 0 num_leds: 256 - rgb_order: GRB + channel_colors: GRB chipset: WS2812 effects: - e131: diff --git a/tests/components/esp32_rmt_led_strip/common.yaml b/tests/components/esp32_rmt_led_strip/common.yaml index 701e513ebd..7f52d32229 100644 --- a/tests/components/esp32_rmt_led_strip/common.yaml +++ b/tests/components/esp32_rmt_led_strip/common.yaml @@ -3,13 +3,13 @@ light: id: led_strip1 pin: ${pin1} num_leds: 60 - rgb_order: GRB + channel_colors: GRB chipset: ws2812 - platform: esp32_rmt_led_strip id: led_strip2 pin: ${pin2} num_leds: 60 - rgbw_order: RWGB + channel_colors: RWGB bit0_high: 100us bit0_low: 100us bit1_high: 100us diff --git a/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml b/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml index 6bf0639a52..132966eddf 100644 --- a/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml +++ b/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml @@ -8,14 +8,14 @@ light: id: led_strip1 pin: ${pin1} num_leds: 60 - rgb_order: GRB + channel_colors: GRB chipset: ws2812 use_dma: "true" - platform: esp32_rmt_led_strip id: led_strip2 pin: ${pin2} num_leds: 60 - rgb_order: RGB + channel_colors: RGB bit0_high: 100us bit0_low: 100us bit1_high: 100us diff --git a/tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml b/tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml new file mode 100644 index 0000000000..6dd1bcdad3 --- /dev/null +++ b/tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml @@ -0,0 +1,23 @@ +# The deprecated rgb_order / is_rgbw / is_wrgb keys, kept working until 2027.3.0. +# Config-only: each strip below must migrate to the channel_colors shown in the comment. +light: + - platform: esp32_rmt_led_strip + id: legacy_rgb + pin: GPIO13 + num_leds: 60 + chipset: ws2812 + rgb_order: GRB # -> GRB + - platform: esp32_rmt_led_strip + id: legacy_rgbw + pin: GPIO14 + num_leds: 60 + chipset: sk6812 + rgb_order: GRB + is_rgbw: true # -> GRBW + - platform: esp32_rmt_led_strip + id: legacy_wrgb + pin: GPIO15 + num_leds: 60 + chipset: sk6812 + rgb_order: GRB + is_wrgb: true # -> WGRB diff --git a/tests/components/partition/common-ard.yaml b/tests/components/partition/common-ard.yaml index b2ceadd6f7..8d39670e32 100644 --- a/tests/components/partition/common-ard.yaml +++ b/tests/components/partition/common-ard.yaml @@ -4,7 +4,7 @@ light: default_transition_length: 500ms chipset: ws2812 num_leds: 256 - rgb_order: GRB + channel_colors: GRB pin: ${pin} - platform: partition name: Partition Light diff --git a/tests/components/partition/common-idf.yaml b/tests/components/partition/common-idf.yaml index b2ceadd6f7..8d39670e32 100644 --- a/tests/components/partition/common-idf.yaml +++ b/tests/components/partition/common-idf.yaml @@ -4,7 +4,7 @@ light: default_transition_length: 500ms chipset: ws2812 num_leds: 256 - rgb_order: GRB + channel_colors: GRB pin: ${pin} - platform: partition name: Partition Light diff --git a/tests/components/rp2040_pio_led_strip/common.yaml b/tests/components/rp2040_pio_led_strip/common.yaml index 254ac0e13d..1cb5fe0737 100644 --- a/tests/components/rp2040_pio_led_strip/common.yaml +++ b/tests/components/rp2040_pio_led_strip/common.yaml @@ -4,14 +4,14 @@ light: pin: 4 num_leds: 60 pio: 0 - rgb_order: GRB + channel_colors: GRB chipset: WS2812 - platform: rp2040_pio_led_strip id: led_strip_custom_timings pin: 5 num_leds: 60 pio: 1 - rgb_order: GRB + channel_colors: GRB bit0_high: .1us bit0_low: 1.2us bit1_high: .69us diff --git a/tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml b/tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml new file mode 100644 index 0000000000..2ab124393b --- /dev/null +++ b/tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml @@ -0,0 +1,18 @@ +# The deprecated rgb_order / is_rgbw keys, kept working until 2027.3.0. +# Config-only: each strip below must migrate to the channel_colors shown in the comment. +light: + - platform: rp2040_pio_led_strip + id: legacy_rgb + pin: 4 + num_leds: 60 + pio: 0 + chipset: WS2812 + rgb_order: GRB # -> GRB + - platform: rp2040_pio_led_strip + id: legacy_rgbw + pin: 5 + num_leds: 60 + pio: 1 + chipset: SK6812 + rgb_order: GRB + is_rgbw: true # -> GRBW diff --git a/tests/components/wled/test.esp32-ard.yaml b/tests/components/wled/test.esp32-ard.yaml index 156b31181e..ecab767812 100644 --- a/tests/components/wled/test.esp32-ard.yaml +++ b/tests/components/wled/test.esp32-ard.yaml @@ -9,7 +9,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: 2 effects: diff --git a/tests/unit_tests/components/light/test_channel_colors.py b/tests/unit_tests/components/light/test_channel_colors.py new file mode 100644 index 0000000000..0c129a8bb2 --- /dev/null +++ b/tests/unit_tests/components/light/test_channel_colors.py @@ -0,0 +1,144 @@ +"""Tests for the shared addressable-strip channel order helpers.""" + +import logging + +import pytest + +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB +from esphome.components.light import ( + channel_colors_struct, + migrate_channel_colors, + validate_channel_colors, +) +import esphome.config_validation as cv +from esphome.const import CONF_IS_RGBW, CONF_RGB_ORDER +from esphome.types import ConfigType + +NO_WHITE = "light::ChannelColors::NO_WHITE" + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("RGB", "RGB"), + ("grb", "GRB"), + ("BRG", "BRG"), + ("rgbw", "RGBW"), + ("WRGB", "WRGB"), + ("GWRB", "GWRB"), + ], +) +def test_validate_channel_colors(value: str, expected: str) -> None: + assert validate_channel_colors(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + "RG", # missing a channel + "RGBB", # duplicate channel + "RRGB", # duplicate channel, correct length + "RGBWW", # two white channels + "RGBX", # unknown channel + "RGBWX", # unknown channel, correct length + "", + ], +) +def test_validate_channel_colors_rejects_invalid(value: str) -> None: + with pytest.raises(cv.Invalid, match="is not a valid channel order"): + validate_channel_colors(value) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("RGB", (0, 1, 2, NO_WHITE)), + ("GRB", (1, 0, 2, NO_WHITE)), + ("BRG", (1, 2, 0, NO_WHITE)), + ("RGBW", (0, 1, 2, 3)), + ("GRBW", (1, 0, 2, 3)), + ("WRGB", (1, 2, 3, 0)), + ("GWRB", (2, 0, 3, 1)), + ], +) +def test_channel_colors_struct(value: str, expected: tuple[int, int, int, int]) -> None: + struct = channel_colors_struct(value) + assert str(struct.base) == "light::ChannelColors" + assert tuple(str(arg) for arg in struct.args.values()) == tuple( + str(field) for field in expected + ) + + +def _migrate(config: ConfigType) -> ConfigType: + return migrate_channel_colors(removed_in="2027.3.0", component="test_strip")(config) + + +def test_migrate_passes_through_channel_colors() -> None: + config = {CONF_CHANNEL_COLORS: "GRBW"} + assert _migrate(config) == {CONF_CHANNEL_COLORS: "GRBW"} + + +@pytest.mark.parametrize( + ("deprecated", "expected", "named"), + [ + ({}, "GRB", "'rgb_order' is"), + ( + {CONF_IS_RGBW: False, CONF_IS_WRGB: False}, + "GRB", + "'rgb_order', 'is_rgbw' and 'is_wrgb' are", + ), + ({CONF_IS_RGBW: True}, "GRBW", "'rgb_order' and 'is_rgbw' are"), + ({CONF_IS_WRGB: True}, "WGRB", "'rgb_order' and 'is_wrgb' are"), + ], +) +def test_migrate_folds_deprecated_keys( + deprecated: ConfigType, + expected: str, + named: str, + caplog: pytest.LogCaptureFixture, +) -> None: + config = {CONF_RGB_ORDER: "GRB", "num_leds": 1, **deprecated} + with caplog.at_level(logging.WARNING): + result = _migrate(config) + + assert result == {CONF_CHANNEL_COLORS: expected, "num_leds": 1} + assert f"[test_strip] {named} deprecated" in caplog.text + assert f"'{CONF_CHANNEL_COLORS}: {expected}'" in caplog.text + assert "2027.3.0" in caplog.text + + +def test_migrate_does_not_mutate_input() -> None: + config = {CONF_RGB_ORDER: "GRB", CONF_IS_RGBW: True} + _migrate(config) + assert config == {CONF_RGB_ORDER: "GRB", CONF_IS_RGBW: True} + + +@pytest.mark.parametrize("deprecated", [CONF_RGB_ORDER, CONF_IS_RGBW, CONF_IS_WRGB]) +def test_migrate_rejects_mixing_old_and_new(deprecated: str) -> None: + config = {CONF_CHANNEL_COLORS: "GRBW", deprecated: "GRB"} + with pytest.raises(cv.Invalid, match=f"cannot be combined with '{deprecated}'"): + _migrate(config) + + +def test_migrate_reports_every_conflicting_key() -> None: + config = { + CONF_CHANNEL_COLORS: "GRBW", + CONF_RGB_ORDER: "GRB", + CONF_IS_RGBW: True, + CONF_IS_WRGB: False, + } + with pytest.raises( + cv.Invalid, match="cannot be combined with 'rgb_order', 'is_rgbw' and 'is_wrgb'" + ): + _migrate(config) + + +def test_migrate_requires_channel_colors() -> None: + with pytest.raises(cv.Invalid, match=f"'{CONF_CHANNEL_COLORS}' is required"): + _migrate({"num_leds": 1}) + + +def test_migrate_rejects_is_rgbw_with_is_wrgb() -> None: + config = {CONF_RGB_ORDER: "GRB", CONF_IS_RGBW: True, CONF_IS_WRGB: True} + with pytest.raises(cv.Invalid, match="cannot both be enabled"): + _migrate(config) diff --git a/tests/unit_tests/components/test_esp32_rmt_led_strip.py b/tests/unit_tests/components/test_esp32_rmt_led_strip.py deleted file mode 100644 index e2cb513e3b..0000000000 --- a/tests/unit_tests/components/test_esp32_rmt_led_strip.py +++ /dev/null @@ -1,57 +0,0 @@ -import pytest - -from esphome.components.esp32_rmt_led_strip.light import ( - CONF_IS_WRGB, - CONF_RGBW_ORDER, - _split_rgbw_order, - _validate_rgbw_order, - _validate_rgbw_order_exclusivity, -) -import esphome.config_validation as cv -from esphome.const import CONF_IS_RGBW - - -def test_validate_rgbw_order() -> None: - assert _validate_rgbw_order("rwgb") == "RWGB" - - -@pytest.mark.parametrize("rgbw_order", ["RGB", "RRGB", "RGBWW"]) -def test_validate_rgbw_order_rejects_invalid_order(rgbw_order: str) -> None: - with pytest.raises(cv.Invalid, match="permutation of RGBW"): - _validate_rgbw_order(rgbw_order) - - -@pytest.mark.parametrize( - ("rgbw_order", "expected"), - [ - ("WRGB", ("RGB", 0)), - ("RWGB", ("RGB", 1)), - ("GWRB", ("GRB", 1)), - ("RGBW", ("RGB", 3)), - ], -) -def test_split_rgbw_order(rgbw_order: str, expected: tuple[str, int]) -> None: - assert _split_rgbw_order(rgbw_order) == expected - - -@pytest.mark.parametrize("conflict", [CONF_IS_RGBW, CONF_IS_WRGB]) -def test_rgbw_order_is_mutually_exclusive(conflict: str) -> None: - with pytest.raises(cv.Invalid, match="cannot be used with"): - _validate_rgbw_order_exclusivity( - { - CONF_RGBW_ORDER: "RGBW", - CONF_IS_RGBW: conflict == CONF_IS_RGBW, - CONF_IS_WRGB: conflict == CONF_IS_WRGB, - } - ) - - -@pytest.mark.parametrize("legacy_option", [CONF_IS_RGBW, CONF_IS_WRGB]) -def test_rgbw_order_allows_disabled_legacy_options(legacy_option: str) -> None: - config = { - CONF_RGBW_ORDER: "RGBW", - CONF_IS_RGBW: False, - CONF_IS_WRGB: False, - } - config[legacy_option] = False - assert _validate_rgbw_order_exclusivity(config) is config From fffa902a1a78ac0daa6c759de0f20611e4846fbb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 20:32:03 -0500 Subject: [PATCH 104/160] [gpio_expander][pcf8574][pca9554][tca9555][pca6416a][pi4ioe5v6408][mcp23016][mcp23xxx_base] Reject unsupported interrupt_pin options (inverted, allow_other_uses) (#18472) --- esphome/components/gpio_expander/__init__.py | 22 +++++++ esphome/components/mcp23016/__init__.py | 4 +- esphome/components/mcp23xxx_base/__init__.py | 22 +------ esphome/components/pca6416a/__init__.py | 4 +- esphome/components/pca9554/__init__.py | 4 +- esphome/components/pcf8574/__init__.py | 4 +- esphome/components/pi4ioe5v6408/__init__.py | 4 +- esphome/components/tca9555/__init__.py | 4 +- script/build_language_schema.py | 10 +++ .../component_tests/gpio_expander/__init__.py | 0 .../gpio_expander/test_init.py | 61 +++++++++++++++++++ 11 files changed, 107 insertions(+), 32 deletions(-) create mode 100644 tests/component_tests/gpio_expander/__init__.py create mode 100644 tests/component_tests/gpio_expander/test_init.py diff --git a/esphome/components/gpio_expander/__init__.py b/esphome/components/gpio_expander/__init__.py index e69de29bb2..0c7199b6df 100644 --- a/esphome/components/gpio_expander/__init__.py +++ b/esphome/components/gpio_expander/__init__.py @@ -0,0 +1,22 @@ +from esphome import pins +import esphome.config_validation as cv +from esphome.const import CONF_ALLOW_OTHER_USES, CONF_INTERRUPT_PIN, CONF_INVERTED +from esphome.types import ConfigType + + +def validate_interrupt_pin(value: ConfigType) -> ConfigType: + # The expander components own INT polarity (active-low, hardcoded falling-edge ISR) + # and install a single ISR per GPIO, so neither inversion nor sharing is supported. + value = pins.internal_gpio_input_pin_schema(value) + if value.get(CONF_INVERTED): + raise cv.Invalid( + f"'{CONF_INVERTED}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " + "the expander INT line is fixed active-low" + ) + if value.get(CONF_ALLOW_OTHER_USES): + raise cv.Invalid( + f"'{CONF_ALLOW_OTHER_USES}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " + "sharing the interrupt pin between multiple components is not implemented. " + f"Remove the '{CONF_INTERRUPT_PIN}' to fall back to polling." + ) + return value diff --git a/esphome/components/mcp23016/__init__.py b/esphome/components/mcp23016/__init__.py index b71d57498a..37c5205fe8 100644 --- a/esphome/components/mcp23016/__init__.py +++ b/esphome/components/mcp23016/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -25,7 +25,7 @@ CONFIG_SCHEMA = ( cv.Schema( { cv.Required(CONF_ID): cv.declare_id(MCP23016), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/mcp23xxx_base/__init__.py b/esphome/components/mcp23xxx_base/__init__.py index 76a3aabe3f..d53499a78f 100644 --- a/esphome/components/mcp23xxx_base/__init__.py +++ b/esphome/components/mcp23xxx_base/__init__.py @@ -1,8 +1,8 @@ from esphome import pins import esphome.codegen as cg +from esphome.components import gpio_expander import esphome.config_validation as cv from esphome.const import ( - CONF_ALLOW_OTHER_USES, CONF_ID, CONF_INPUT, CONF_INTERRUPT, @@ -32,28 +32,10 @@ MCP23XXX_INTERRUPT_MODES = { } -def _validate_interrupt_pin(value): - # The MCP component owns INT polarity (active-low, hardcoded falling-edge ISR) - # and installs a single ISR per GPIO, so neither inversion nor sharing is supported. - value = pins.internal_gpio_input_pin_schema(value) - if value.get(CONF_INVERTED): - raise cv.Invalid( - f"'{CONF_INVERTED}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " - "the MCP23xxx INT line is fixed active-low" - ) - if value.get(CONF_ALLOW_OTHER_USES): - raise cv.Invalid( - f"'{CONF_ALLOW_OTHER_USES}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " - "sharing the interrupt pin between multiple MCP23xxx (or other components) " - "is not implemented. Remove the interrupt_pin to fall back to polling." - ) - return value - - MCP23XXX_CONFIG_SCHEMA = cv.Schema( { cv.Optional(CONF_OPEN_DRAIN_INTERRUPT, default=False): cv.boolean, - cv.Optional(CONF_INTERRUPT_PIN): _validate_interrupt_pin, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ).extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pca6416a/__init__.py b/esphome/components/pca6416a/__init__.py index 813bb35c48..1df22a8ff5 100644 --- a/esphome/components/pca6416a/__init__.py +++ b/esphome/components/pca6416a/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -29,7 +29,7 @@ CONFIG_SCHEMA = ( cv.Schema( { cv.Required(CONF_ID): cv.declare_id(PCA6416AComponent), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pca9554/__init__.py b/esphome/components/pca9554/__init__.py index 99b812b33b..f49a68bc3f 100644 --- a/esphome/components/pca9554/__init__.py +++ b/esphome/components/pca9554/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -30,7 +30,7 @@ CONFIG_SCHEMA = ( { cv.Required(CONF_ID): cv.declare_id(PCA9554Component), cv.Optional(CONF_PIN_COUNT, default=8): cv.one_of(4, 8, 16), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pcf8574/__init__.py b/esphome/components/pcf8574/__init__.py index d8a1e20db6..559fe1d76d 100644 --- a/esphome/components/pcf8574/__init__.py +++ b/esphome/components/pcf8574/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -28,7 +28,7 @@ CONFIG_SCHEMA = ( { cv.Required(CONF_ID): cv.declare_id(PCF8574Component), cv.Optional(CONF_PCF8575, default=False): cv.boolean, - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pi4ioe5v6408/__init__.py b/esphome/components/pi4ioe5v6408/__init__.py index d5b19dab1c..ee270138e1 100644 --- a/esphome/components/pi4ioe5v6408/__init__.py +++ b/esphome/components/pi4ioe5v6408/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -34,7 +34,7 @@ CONFIG_SCHEMA = ( { cv.Required(CONF_ID): cv.declare_id(PI4IOE5V6408Component), cv.Optional(CONF_RESET, default=True): cv.boolean, - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/tca9555/__init__.py b/esphome/components/tca9555/__init__.py index 5f571fcea6..1c643fe1c9 100644 --- a/esphome/components/tca9555/__init__.py +++ b/esphome/components/tca9555/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -28,7 +28,7 @@ CONFIG_SCHEMA = ( cv.Schema( { cv.Required(CONF_ID): cv.declare_id(TCA9555Component), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/script/build_language_schema.py b/script/build_language_schema.py index 2b64cb0256..91c1de00cd 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -250,6 +250,16 @@ def add_pin_validators(): "modes": ["input"], } + from esphome.components import gpio_expander + + # Wraps pins.internal_gpio_input_pin_schema, so the editor schema must keep + # treating the config var as a pin + pin_validators[repr(gpio_expander.validate_interrupt_pin)] = { + "schema": True, + "internal": True, + "modes": ["input"], + } + def add_module_registries(domain, module): for attr_name in dir(module): diff --git a/tests/component_tests/gpio_expander/__init__.py b/tests/component_tests/gpio_expander/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/gpio_expander/test_init.py b/tests/component_tests/gpio_expander/test_init.py new file mode 100644 index 0000000000..806b1775d2 --- /dev/null +++ b/tests/component_tests/gpio_expander/test_init.py @@ -0,0 +1,61 @@ +"""Tests for the shared io expander interrupt_pin validator.""" + +from __future__ import annotations + +import importlib + +import pytest + +from esphome import config_validation as cv +from esphome.components.esp32 import KEY_BOARD, KEY_VARIANT, VARIANT_ESP32 +from esphome.components.gpio_expander import validate_interrupt_pin +from esphome.const import PlatformFramework +from tests.component_tests.types import SetCoreConfigCallable + + +@pytest.fixture +def stage_esp32(set_core_config: SetCoreConfigCallable) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + + +def test_plain_pin_accepted(stage_esp32: None) -> None: + value = validate_interrupt_pin( + {"number": 16, "mode": {"input": True, "pullup": True}} + ) + assert value["number"] == 16 + + +def test_inverted_rejected(stage_esp32: None) -> None: + with pytest.raises(cv.Invalid, match="'inverted: true' is not supported"): + validate_interrupt_pin({"number": 16, "inverted": True}) + + +def test_allow_other_uses_rejected(stage_esp32: None) -> None: + with pytest.raises(cv.Invalid, match="'allow_other_uses: true' is not supported"): + validate_interrupt_pin({"number": 16, "allow_other_uses": True}) + + +# mcp23017 covers the shared mcp23xxx_base schema +@pytest.mark.parametrize( + "component", + [ + "pcf8574", + "pca9554", + "tca9555", + "pca6416a", + "pi4ioe5v6408", + "mcp23016", + "mcp23017", + ], +) +def test_component_schemas_route_through_validator( + stage_esp32: None, component: str +) -> None: + module = importlib.import_module(f"esphome.components.{component}") + with pytest.raises(cv.Invalid, match="'inverted: true' is not supported"): + module.CONFIG_SCHEMA( + {"id": "expander_hub", "interrupt_pin": {"number": 16, "inverted": True}} + ) From 096e71bd678ff5707ddbd013fe59c012e3abc8f9 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:14:14 -0500 Subject: [PATCH 105/160] Bump aioesphomeapi from 45.10.1 to 45.10.2 (#18357) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 85a0f55263..683008400a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.10.1 +aioesphomeapi==45.10.2 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 3a403c40d5f7d02dcf6bfb8eca6d614471fa3b91 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 16 Aug 2026 19:57:12 -0700 Subject: [PATCH 106/160] [ld2420] Drop the setup priority override so setup runs after the UART bus (#18428) --- esphome/components/ld2420/ld2420.cpp | 2 -- esphome/components/ld2420/ld2420.h | 1 - 2 files changed, 3 deletions(-) diff --git a/esphome/components/ld2420/ld2420.cpp b/esphome/components/ld2420/ld2420.cpp index f71bec7e5f..4aa00f8fd4 100644 --- a/esphome/components/ld2420/ld2420.cpp +++ b/esphome/components/ld2420/ld2420.cpp @@ -184,8 +184,6 @@ static int32_t get_firmware_int(const char *version_string) { return result; } -float LD2420Component::get_setup_priority() const { return setup_priority::BUS; } - void LD2420Component::dump_config() { ESP_LOGCONFIG(TAG, "LD2420:\n" diff --git a/esphome/components/ld2420/ld2420.h b/esphome/components/ld2420/ld2420.h index 977ee2eccc..e13d0271e1 100644 --- a/esphome/components/ld2420/ld2420.h +++ b/esphome/components/ld2420/ld2420.h @@ -105,7 +105,6 @@ class LD2420Component final : public Component, public uart::UARTDevice { void apply_config_action(); void factory_reset_action(); void revert_config_action(); - float get_setup_priority() const override; int send_cmd_from_array(CmdFrameT cmd_frame); void report_gate_data(); void handle_cmd_error(uint16_t error); From b9041566eaee70079271526dc3104360a78d067c Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:34:00 -0500 Subject: [PATCH 107/160] Bump aioesphomeapi from 45.10.2 to 45.10.3 (#18433) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 683008400a..080a437147 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.10.2 +aioesphomeapi==45.10.3 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 014cc199021325153f0572c17f85386760e5ae09 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 12:00:39 -0700 Subject: [PATCH 108/160] [api] Bump noise-c to 0.1.18 (#18451) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 8ec94df1db..5ca9484336 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.11") + cg.add_library("esphome/noise-c", "0.1.18") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index bf3b0685f8..2c22523be5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.11 ; api + esphome/noise-c@0.1.18 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.11 ; api + esphome/noise-c@0.1.18 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.11 ; used by api + esphome/noise-c@0.1.18 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 4ce6d59484be6fb55b1bf7cd4d0bdea4785ee1d1 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:54:23 -0500 Subject: [PATCH 109/160] Bump bundled esphome-device-builder to 1.11.1 (#18475) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2f23b2f690..b78d183e02 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.1 RUN \ platformio settings set enable_telemetry No \ From 1fd63372545525bfa8cc48e781fb96101b37e3f0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 18:58:11 -0500 Subject: [PATCH 110/160] [api] Bump noise-c to 0.1.19 (#18473) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 5ca9484336..cdc0d97c49 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.18") + cg.add_library("esphome/noise-c", "0.1.19") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 2c22523be5..39600d622a 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.18 ; api + esphome/noise-c@0.1.19 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.18 ; api + esphome/noise-c@0.1.19 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.18 ; used by api + esphome/noise-c@0.1.19 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 4dea147386d4d309e84ec3eee96e6169a224cf40 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:16:38 -0500 Subject: [PATCH 111/160] Bump bundled esphome-device-builder to 1.11.2 (#18477) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b78d183e02..4a8daeaaf6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.2 RUN \ platformio settings set enable_telemetry No \ From 482869fbbe04a8ff28746f066e90ee7d57bbe81e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 20:09:29 -0500 Subject: [PATCH 112/160] [socket] Fix multi-second TCP stalls on ESP8266 by yielding to the SYS context (#18455) --- .../components/socket/lwip_raw_tcp_impl.cpp | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index 4fcec553fa..098056d499 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -45,6 +45,11 @@ namespace esphome::socket { static const char *const TAG = "socket.lwip"; +#ifdef USE_ESP8266 +// optimistic_yield() rate limit in microseconds of CONT time; cheap when hot. +static constexpr uint32_t ESP8266_YIELD_INTERVAL_US = 1000; +#endif + // set to 1 to enable verbose lwip logging #if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if) #define LWIP_LOG(msg, ...) ESP_LOGVV(TAG, "socket %p: " msg, this, ##__VA_ARGS__) @@ -535,6 +540,14 @@ ssize_t LWIPRawImpl::read_locked_(void *buf, size_t len) { } ssize_t LWIPRawImpl::read(void *buf, size_t len) { +#ifdef USE_ESP8266 + // Would block: yield to SYS so queued WiFi RX reaches lwip and this read + // may succeed. Without this, inbound segments can sit unprocessed for + // seconds while the main loop polls (CONT/SYS are cooperative on ESP8266). + if (this->waiting_for_data_()) { + optimistic_yield(ESP8266_YIELD_INTERVAL_US); + } +#endif // See waiting_for_data_() for safety of unlocked reads. if (this->recv_timeout_cs_ > 0 && this->waiting_for_data_()) { this->wait_for_data_(); @@ -545,6 +558,8 @@ ssize_t LWIPRawImpl::read(void *buf, size_t len) { } ssize_t LWIPRawImpl::readv(const struct iovec *iov, int iovcnt) { + // No ESP8266 SYS yield here: only read() needs it today. If a consumer + // switches to scatter-gather reads, mirror the yield from read(). // See waiting_for_data_() for safety of unlocked reads. if (this->recv_timeout_cs_ > 0 && this->waiting_for_data_()) { this->wait_for_data_(); @@ -609,19 +624,24 @@ int LWIPRawImpl::internal_output_() { } LWIP_LOG("tcp_output(%p)", this->pcb_); err_t err = tcp_output(this->pcb_); - if (err == ERR_ABRT) { - // sometimes lwip returns ERR_ABRT for no apparent reason - // the connection works fine afterwards, and back with ESPAsyncTCP we - // indirectly also ignored this error - // FIXME: figure out where this is returned and what it means in this context - LWIP_LOG(" -> err ERR_ABRT"); - return 0; - } if (err != ERR_OK) { LWIP_LOG(" -> err %d", err); - errno = ECONNRESET; - return -1; + // ERR_ABRT: sometimes lwip returns it for no apparent reason; the + // connection works fine afterwards, and back with ESPAsyncTCP we + // indirectly also ignored this error, so treat it as success for + // flush purposes too. + // FIXME: figure out where this is returned and what it means in this context + if (err != ERR_ABRT) { + errno = ECONNRESET; + return -1; + } } +#ifdef USE_ESP8266 + // Flushed: yield to SYS so the queued segments reach the WiFi driver + // instead of waiting seconds for an unrelated SYS slot. Callers only get + // here after a successful tcp_write, so idle paths never yield. + optimistic_yield(ESP8266_YIELD_INTERVAL_US); +#endif return 0; } From 6a247dfe912477e516f8da6f13e4ab002544a44e Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:19:48 +1200 Subject: [PATCH 113/160] [light] Replace rgb_order/is_rgbw/is_wrgb with channel_colors (#18474) --- .../beken_spi_led_strip/led_strip.cpp | 75 ++------- .../beken_spi_led_strip/led_strip.h | 23 +-- .../components/beken_spi_led_strip/light.py | 41 +++-- esphome/components/const/__init__.py | 2 + .../esp32_rmt_led_strip/led_strip.cpp | 86 ++--------- .../esp32_rmt_led_strip/led_strip.h | 29 +--- .../components/esp32_rmt_led_strip/light.py | 65 ++------ esphome/components/light/__init__.py | 108 ++++++++++++- esphome/components/light/channel_colors.h | 41 +++++ esphome/components/light/types.py | 3 + .../rp2040_pio_led_strip/led_strip.cpp | 59 ++----- .../rp2040_pio_led_strip/led_strip.h | 42 +---- .../components/rp2040_pio_led_strip/light.py | 33 ++-- .../common-ard-esp32_rmt_led_strip.yaml | 2 +- .../common-idf-esp32_rmt_led_strip.yaml | 2 +- .../beken_spi_led_strip/test.bk72xx-ard.yaml | 2 +- .../validate-legacy.bk72xx-ard.yaml | 10 ++ tests/components/e131/common-ard.yaml | 2 +- tests/components/e131/common-idf.yaml | 2 +- tests/components/e131/test.rp2040-ard.yaml | 2 +- .../esp32_rmt_led_strip/common.yaml | 4 +- .../test.esp32-s3-idf.yaml | 4 +- .../validate-legacy.esp32-idf.yaml | 23 +++ tests/components/partition/common-ard.yaml | 2 +- tests/components/partition/common-idf.yaml | 2 +- .../rp2040_pio_led_strip/common.yaml | 4 +- .../validate-legacy.rp2040-ard.yaml | 18 +++ tests/components/wled/test.esp32-ard.yaml | 2 +- .../components/light/test_channel_colors.py | 144 ++++++++++++++++++ .../components/test_esp32_rmt_led_strip.py | 57 ------- 30 files changed, 455 insertions(+), 434 deletions(-) create mode 100644 esphome/components/light/channel_colors.h create mode 100644 tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml create mode 100644 tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml create mode 100644 tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml create mode 100644 tests/unit_tests/components/light/test_channel_colors.py delete mode 100644 tests/unit_tests/components/test_esp32_rmt_led_strip.py diff --git a/esphome/components/beken_spi_led_strip/led_strip.cpp b/esphome/components/beken_spi_led_strip/led_strip.cpp index 9e14615d7a..0cf970b3cc 100644 --- a/esphome/components/beken_spi_led_strip/led_strip.cpp +++ b/esphome/components/beken_spi_led_strip/led_strip.cpp @@ -300,46 +300,12 @@ void BekenSPILEDStripLightOutput::write_state(light::LightState *state) { } light::ESPColorView BekenSPILEDStripLightOutput::get_view_internal(int32_t index) const { - int32_t r = 0, g = 0, b = 0; - switch (this->rgb_order_) { - case ORDER_RGB: - r = 0; - g = 1; - b = 2; - break; - case ORDER_RBG: - r = 0; - g = 2; - b = 1; - break; - case ORDER_GRB: - r = 1; - g = 0; - b = 2; - break; - case ORDER_GBR: - r = 2; - g = 0; - b = 1; - break; - case ORDER_BGR: - r = 2; - g = 1; - b = 0; - break; - case ORDER_BRG: - r = 1; - g = 2; - b = 0; - break; - } - uint8_t multiplier = this->is_rgbw_ || this->is_wrgb_ ? 4 : 3; - uint8_t white = this->is_wrgb_ ? 0 : 3; - - return {this->buf_ + (index * multiplier) + r + this->is_wrgb_, - this->buf_ + (index * multiplier) + g + this->is_wrgb_, - this->buf_ + (index * multiplier) + b + this->is_wrgb_, - this->is_rgbw_ || this->is_wrgb_ ? this->buf_ + (index * multiplier) + white : nullptr, + const light::ChannelColors &colors = this->channel_colors_; + uint8_t *led = this->buf_ + (index * colors.bytes_per_led()); + return {led + colors.r, + led + colors.g, + led + colors.b, + colors.has_white() ? led + colors.w : nullptr, &this->effect_data_[index], &this->correction_}; } @@ -349,35 +315,12 @@ void BekenSPILEDStripLightOutput::dump_config() { "Beken SPI LED Strip:\n" " Pin: %u", this->pin_); - const char *rgb_order; - switch (this->rgb_order_) { - case ORDER_RGB: - rgb_order = "RGB"; - break; - case ORDER_RBG: - rgb_order = "RBG"; - break; - case ORDER_GRB: - rgb_order = "GRB"; - break; - case ORDER_GBR: - rgb_order = "GBR"; - break; - case ORDER_BGR: - rgb_order = "BGR"; - break; - case ORDER_BRG: - rgb_order = "BRG"; - break; - default: - rgb_order = "UNKNOWN"; - break; - } + char channel_colors[5]; ESP_LOGCONFIG(TAG, - " RGB Order: %s\n" + " Channel colors: %s\n" " Max refresh rate: %" PRIu32 "\n" " Number of LEDs: %u", - rgb_order, this->max_refresh_rate_.value_or(0), this->num_leds_); + this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_.value_or(0), this->num_leds_); } float BekenSPILEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; } diff --git a/esphome/components/beken_spi_led_strip/led_strip.h b/esphome/components/beken_spi_led_strip/led_strip.h index 909634e266..1496e65d4d 100644 --- a/esphome/components/beken_spi_led_strip/led_strip.h +++ b/esphome/components/beken_spi_led_strip/led_strip.h @@ -3,6 +3,7 @@ #ifdef USE_BK72XX #include "esphome/components/light/addressable_light.h" +#include "esphome/components/light/channel_colors.h" #include "esphome/components/light/light_output.h" #include "esphome/core/color.h" #include "esphome/core/component.h" @@ -10,15 +11,6 @@ namespace esphome::beken_spi_led_strip { -enum RGBOrder : uint8_t { - ORDER_RGB, - ORDER_RBG, - ORDER_GRB, - ORDER_GBR, - ORDER_BGR, - ORDER_BRG, -}; - class BekenSPILEDStripLightOutput final : public light::AddressableLight { public: void setup() override; @@ -28,7 +20,7 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { int32_t size() const override { return this->num_leds_; } light::LightTraits get_traits() override { auto traits = light::LightTraits(); - if (this->is_rgbw_ || this->is_wrgb_) { + if (this->channel_colors_.has_white()) { traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}); } else { traits.set_supported_color_modes({light::ColorMode::RGB}); @@ -38,16 +30,13 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { void set_pin(uint8_t pin) { this->pin_ = pin; } void set_num_leds(uint16_t num_leds) { this->num_leds_ = num_leds; } - void set_is_rgbw(bool is_rgbw) { this->is_rgbw_ = is_rgbw; } - void set_is_wrgb(bool is_wrgb) { this->is_wrgb_ = is_wrgb; } + void set_channel_colors(light::ChannelColors channel_colors) { this->channel_colors_ = channel_colors; } /// Set a maximum refresh rate in µs as some lights do not like being updated too often. void set_max_refresh_rate(uint32_t interval_us) { this->max_refresh_rate_ = interval_us; } void set_led_params(uint8_t bit0, uint8_t bit1, uint32_t spi_frequency); - void set_rgb_order(RGBOrder rgb_order) { this->rgb_order_ = rgb_order; } - void clear_effect_data() override { for (int i = 0; i < this->size(); i++) this->effect_data_[i] = 0; @@ -58,7 +47,7 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { protected: light::ESPColorView get_view_internal(int32_t index) const override; - size_t get_buffer_size_() const { return this->num_leds_ * (this->is_rgbw_ || this->is_wrgb_ ? 4 : 3); } + size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); } uint8_t *buf_{nullptr}; uint8_t *effect_data_{nullptr}; @@ -66,13 +55,11 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight { uint8_t pin_; uint16_t num_leds_; - bool is_rgbw_; - bool is_wrgb_; uint32_t spi_frequency_{6666666}; uint8_t bit0_{0xE0}; uint8_t bit1_{0xFC}; - RGBOrder rgb_order_; + light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE}; uint32_t last_refresh_{0}; optional max_refresh_rate_{}; diff --git a/esphome/components/beken_spi_led_strip/light.py b/esphome/components/beken_spi_led_strip/light.py index 9093b08b62..2be5842818 100644 --- a/esphome/components/beken_spi_led_strip/light.py +++ b/esphome/components/beken_spi_led_strip/light.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from esphome import pins import esphome.codegen as cg from esphome.components import libretiny, light +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB import esphome.config_validation as cv from esphome.const import ( CONF_CHIPSET, @@ -13,6 +14,7 @@ from esphome.const import ( CONF_PIN, CONF_RGB_ORDER, ) +from esphome.types import ConfigType CODEOWNERS = ["@Mat931"] DEPENDENCIES = ["libretiny"] @@ -22,17 +24,6 @@ BekenSPILEDStripLightOutput = beken_spi_led_strip_ns.class_( "BekenSPILEDStripLightOutput", light.AddressableLight ) -RGBOrder = beken_spi_led_strip_ns.enum("RGBOrder") - -RGB_ORDERS = { - "RGB": RGBOrder.ORDER_RGB, - "RBG": RGBOrder.ORDER_RBG, - "GRB": RGBOrder.ORDER_GRB, - "GBR": RGBOrder.ORDER_GBR, - "BGR": RGBOrder.ORDER_BGR, - "BRG": RGBOrder.ORDER_BRG, -} - @dataclass class LEDStripTimings: @@ -57,8 +48,6 @@ CHIPSETS = { } -CONF_IS_WRGB = "is_wrgb" - SUPPORTED_PINS = { libretiny.const.FAMILY_BK7231N: [16], libretiny.const.FAMILY_BK7231T: [16], @@ -79,10 +68,9 @@ def _validate_pin(value): return value -def _validate_num_leds(value): - max_num_leds = 165 # 170 - if value[CONF_IS_RGBW] or value[CONF_IS_WRGB]: - max_num_leds = 123 # 127 +def _validate_num_leds(value: ConfigType) -> ConfigType: + # A white channel makes each LED one byte wider, so fewer of them fit in the DMA buffer. + max_num_leds = 123 if "W" in value[CONF_CHANNEL_COLORS] else 165 # 127 / 170 if value[CONF_NUM_LEDS] > max_num_leds: raise cv.Invalid( f"The maximum number of LEDs for this configuration is {max_num_leds}.", @@ -99,18 +87,23 @@ CONFIG_SCHEMA = cv.All( pins.internal_gpio_output_pin_number, _validate_pin ), cv.Required(CONF_NUM_LEDS): cv.positive_not_null_int, - cv.Required(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), + cv.Optional(CONF_CHANNEL_COLORS): light.validate_channel_colors, + # Deprecated in favour of CONF_CHANNEL_COLORS, remove in 2027.3.0 + cv.Optional(CONF_RGB_ORDER): cv.one_of(*light.RGB_ORDERS, upper=True), + cv.Optional(CONF_IS_RGBW): cv.boolean, + cv.Optional(CONF_IS_WRGB): cv.boolean, cv.Optional(CONF_MAX_REFRESH_RATE): cv.positive_time_period_microseconds, cv.Required(CONF_CHIPSET): cv.one_of(*CHIPSETS, upper=True), - cv.Optional(CONF_IS_RGBW, default=False): cv.boolean, - cv.Optional(CONF_IS_WRGB, default=False): cv.boolean, } ), + light.migrate_channel_colors( + removed_in="2027.3.0", component="beken_spi_led_strip" + ), _validate_num_leds, ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_OUTPUT_ID]) await light.register_light(var, config) await cg.register_component(var, config) @@ -130,6 +123,6 @@ async def to_code(config): ) ) - cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) - cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) - cg.add(var.set_is_wrgb(config[CONF_IS_WRGB])) + cg.add( + var.set_channel_colors(light.channel_colors_struct(config[CONF_CHANNEL_COLORS])) + ) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 44878274d6..956a5490e3 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -10,6 +10,7 @@ CONF_ACCELEROMETER_RANGE = "accelerometer_range" CONF_B_CONSTANT = "b_constant" CONF_BREATH_VOC_EQUIVALENT = "breath_voc_equivalent" CONF_BYTE_ORDER = "byte_order" +CONF_CHANNEL_COLORS = "channel_colors" CONF_CLIMATE_ID = "climate_id" CONF_CO2_EQUIVALENT = "co2_equivalent" CONF_COLOR_DEPTH = "color_depth" @@ -22,6 +23,7 @@ CONF_GYROSCOPE_ODR = "gyroscope_odr" CONF_GYROSCOPE_RANGE = "gyroscope_range" CONF_IAQ = "iaq" CONF_IGNORE_NOT_FOUND = "ignore_not_found" +CONF_IS_WRGB = "is_wrgb" CONF_LIBRETINY = "libretiny" CONF_LOOP = "loop" CONF_NOX_INDEX = "nox_index" diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.cpp b/esphome/components/esp32_rmt_led_strip/led_strip.cpp index 95391ef100..7cac1dfb41 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.cpp +++ b/esphome/components/esp32_rmt_led_strip/led_strip.cpp @@ -221,46 +221,12 @@ void ESP32RMTLEDStripLightOutput::write_state(light::LightState *state) { } light::ESPColorView ESP32RMTLEDStripLightOutput::get_view_internal(int32_t index) const { - int32_t r = 0, g = 0, b = 0; - switch (this->rgb_order_) { - case ORDER_RGB: - r = 0; - g = 1; - b = 2; - break; - case ORDER_RBG: - r = 0; - g = 2; - b = 1; - break; - case ORDER_GRB: - r = 1; - g = 0; - b = 2; - break; - case ORDER_GBR: - r = 2; - g = 0; - b = 1; - break; - case ORDER_BGR: - r = 2; - g = 1; - b = 0; - break; - case ORDER_BRG: - r = 1; - g = 2; - b = 0; - break; - } - uint8_t multiplier = this->is_rgbw_ || this->is_wrgb_ ? 4 : 3; - uint8_t white = this->is_wrgb_ ? 0 : this->white_index_; - - return {this->buf_ + (index * multiplier) + r + (white <= r), - this->buf_ + (index * multiplier) + g + (white <= g), - this->buf_ + (index * multiplier) + b + (white <= b), - this->is_rgbw_ || this->is_wrgb_ ? this->buf_ + (index * multiplier) + white : nullptr, + const light::ChannelColors &colors = this->channel_colors_; + uint8_t *led = this->buf_ + (index * colors.bytes_per_led()); + return {led + colors.r, + led + colors.g, + led + colors.b, + colors.has_white() ? led + colors.w : nullptr, &this->effect_data_[index], &this->correction_}; } @@ -271,46 +237,12 @@ void ESP32RMTLEDStripLightOutput::dump_config() { " Pin: %u", this->pin_); ESP_LOGCONFIG(TAG, " RMT Symbols: %" PRIu32, this->rmt_symbols_); - const char *rgb_order; - switch (this->rgb_order_) { - case ORDER_RGB: - rgb_order = "RGB"; - break; - case ORDER_RBG: - rgb_order = "RBG"; - break; - case ORDER_GRB: - rgb_order = "GRB"; - break; - case ORDER_GBR: - rgb_order = "GBR"; - break; - case ORDER_BGR: - rgb_order = "BGR"; - break; - case ORDER_BRG: - rgb_order = "BRG"; - break; - default: - rgb_order = "UNKNOWN"; - break; - } - if (this->is_rgbw_ || this->is_wrgb_) { - char rgbw_order[5]; - uint8_t white = this->is_wrgb_ ? 0 : this->white_index_; - uint8_t rgb_index = 0; - for (uint8_t i = 0; i < 4; i++) { - rgbw_order[i] = i == white ? 'W' : rgb_order[rgb_index++]; - } - rgbw_order[4] = '\0'; - ESP_LOGCONFIG(TAG, " RGBW Order: %s", rgbw_order); - } else { - ESP_LOGCONFIG(TAG, " RGB Order: %s", rgb_order); - } + char channel_colors[5]; ESP_LOGCONFIG(TAG, + " Channel colors: %s\n" " Max refresh rate: %" PRIu32 "\n" " Number of LEDs: %u", - this->max_refresh_rate_.value_or(0), this->num_leds_); + this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_.value_or(0), this->num_leds_); } float ESP32RMTLEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; } diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.h b/esphome/components/esp32_rmt_led_strip/led_strip.h index 3e31309bff..61aac06d76 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.h +++ b/esphome/components/esp32_rmt_led_strip/led_strip.h @@ -3,6 +3,7 @@ #ifdef USE_ESP32 #include "esphome/components/light/addressable_light.h" +#include "esphome/components/light/channel_colors.h" #include "esphome/components/light/light_output.h" #include "esphome/core/color.h" #include "esphome/core/component.h" @@ -15,15 +16,6 @@ namespace esphome::esp32_rmt_led_strip { -enum RGBOrder : uint8_t { - ORDER_RGB, - ORDER_RBG, - ORDER_GRB, - ORDER_GBR, - ORDER_BGR, - ORDER_BRG, -}; - struct LedParams { rmt_symbol_word_t bit0; rmt_symbol_word_t bit1; @@ -39,7 +31,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { int32_t size() const override { return this->num_leds_; } light::LightTraits get_traits() override { auto traits = light::LightTraits(); - if (this->is_rgbw_ || this->is_wrgb_) { + if (this->channel_colors_.has_white()) { traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}); } else { traits.set_supported_color_modes({light::ColorMode::RGB}); @@ -50,13 +42,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { void set_pin(uint8_t pin) { this->pin_ = pin; } void set_inverted(bool inverted) { this->invert_out_ = inverted; } void set_num_leds(uint16_t num_leds) { this->num_leds_ = num_leds; } - void set_is_rgbw(bool is_rgbw) { this->is_rgbw_ = is_rgbw; } - void set_is_wrgb(bool is_wrgb) { this->is_wrgb_ = is_wrgb; } - void set_rgbw_order(uint8_t white_index) { - this->is_rgbw_ = true; - this->is_wrgb_ = false; - this->white_index_ = white_index; - } + void set_channel_colors(light::ChannelColors channel_colors) { this->channel_colors_ = channel_colors; } void set_use_dma(bool use_dma) { this->use_dma_ = use_dma; } void set_use_psram(bool use_psram) { this->use_psram_ = use_psram; } @@ -66,7 +52,6 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { void set_led_params(uint32_t bit0_high, uint32_t bit0_low, uint32_t bit1_high, uint32_t bit1_low, uint32_t reset_time_high, uint32_t reset_time_low); - void set_rgb_order(RGBOrder rgb_order) { this->rgb_order_ = rgb_order; } void set_rmt_symbols(uint32_t rmt_symbols) { this->rmt_symbols_ = rmt_symbols; } void clear_effect_data() override { @@ -79,7 +64,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { protected: light::ESPColorView get_view_internal(int32_t index) const override; - size_t get_buffer_size_() const { return this->num_leds_ * (this->is_rgbw_ || this->is_wrgb_ ? 4 : 3); } + size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); } uint8_t *buf_{nullptr}; uint8_t *effect_data_{nullptr}; @@ -94,15 +79,11 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { uint32_t rmt_symbols_{48}; uint8_t pin_; uint16_t num_leds_; - bool is_rgbw_{false}; - bool is_wrgb_{false}; - // An index after the RGB channels makes offset adjustment a no-op for three-channel strips. - uint8_t white_index_{3}; bool use_dma_{false}; bool use_psram_{false}; bool invert_out_{false}; - RGBOrder rgb_order_{ORDER_RGB}; + light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE}; uint32_t last_refresh_{0}; optional max_refresh_rate_{}; diff --git a/esphome/components/esp32_rmt_led_strip/light.py b/esphome/components/esp32_rmt_led_strip/light.py index 2722a9b656..571b7d93b8 100644 --- a/esphome/components/esp32_rmt_led_strip/light.py +++ b/esphome/components/esp32_rmt_led_strip/light.py @@ -1,10 +1,9 @@ from dataclasses import dataclass -import logging from esphome import pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, light -from esphome.components.const import CONF_USE_PSRAM +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB, CONF_USE_PSRAM from esphome.components.esp32 import include_builtin_idf_component import esphome.config_validation as cv from esphome.const import ( @@ -22,8 +21,6 @@ from esphome.const import ( ) from esphome.types import ConfigType -_LOGGER = logging.getLogger(__name__) - CODEOWNERS = ["@jesserockz"] DEPENDENCIES = ["esp32"] @@ -32,17 +29,6 @@ ESP32RMTLEDStripLightOutput = esp32_rmt_led_strip_ns.class_( "ESP32RMTLEDStripLightOutput", light.AddressableLight ) -RGBOrder = esp32_rmt_led_strip_ns.enum("RGBOrder") - -RGB_ORDERS = { - "RGB": RGBOrder.ORDER_RGB, - "RBG": RGBOrder.ORDER_RBG, - "GRB": RGBOrder.ORDER_GRB, - "GBR": RGBOrder.ORDER_GBR, - "BGR": RGBOrder.ORDER_BGR, - "BRG": RGBOrder.ORDER_BRG, -} - @dataclass class LEDStripTimings: @@ -62,8 +48,6 @@ CHIPSETS = { "SM16703": LEDStripTimings(300, 900, 900, 300, 0, 0), } -CONF_IS_WRGB = "is_wrgb" -CONF_RGBW_ORDER = "rgbw_order" CONF_BIT0_HIGH = "bit0_high" CONF_BIT0_LOW = "bit0_low" CONF_BIT1_HIGH = "bit1_high" @@ -72,26 +56,6 @@ CONF_RESET_HIGH = "reset_high" CONF_RESET_LOW = "reset_low" -def _validate_rgbw_order(value: str) -> str: - value = cv.string(value).upper() - if len(value) != 4 or set(value) != set("RGBW"): - raise cv.Invalid("RGBW order must be a permutation of RGBW") - return value - - -def _split_rgbw_order(rgbw_order: str) -> tuple[str, int]: - return rgbw_order.replace("W", ""), rgbw_order.index("W") - - -def _validate_rgbw_order_exclusivity(config: ConfigType) -> ConfigType: - if CONF_RGBW_ORDER in config and (config[CONF_IS_RGBW] or config[CONF_IS_WRGB]): - raise cv.Invalid( - f"'{CONF_RGBW_ORDER}' cannot be used with '{CONF_IS_RGBW}' or " - f"'{CONF_IS_WRGB}'" - ) - return config - - CONFIG_SCHEMA = cv.All( esp32.only_on_variant( unsupported=list(esp32_rmt.VARIANTS_NO_RMT), @@ -102,8 +66,11 @@ CONFIG_SCHEMA = cv.All( cv.GenerateID(CONF_OUTPUT_ID): cv.declare_id(ESP32RMTLEDStripLightOutput), cv.Required(CONF_PIN): pins.internal_gpio_output_pin_schema, cv.Required(CONF_NUM_LEDS): cv.positive_not_null_int, - cv.Optional(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), - cv.Optional(CONF_RGBW_ORDER): _validate_rgbw_order, + cv.Optional(CONF_CHANNEL_COLORS): light.validate_channel_colors, + # Deprecated in favour of CONF_CHANNEL_COLORS, remove in 2027.3.0 + cv.Optional(CONF_RGB_ORDER): cv.one_of(*light.RGB_ORDERS, upper=True), + cv.Optional(CONF_IS_RGBW): cv.boolean, + cv.Optional(CONF_IS_WRGB): cv.boolean, cv.SplitDefault( CONF_RMT_SYMBOLS, esp32=192, @@ -117,8 +84,6 @@ CONFIG_SCHEMA = cv.All( ): cv.int_range(min=2), cv.Optional(CONF_MAX_REFRESH_RATE): cv.positive_time_period_microseconds, cv.Optional(CONF_CHIPSET): cv.one_of(*CHIPSETS, upper=True), - cv.Optional(CONF_IS_RGBW, default=False): cv.boolean, - cv.Optional(CONF_IS_WRGB, default=False): cv.boolean, cv.Optional(CONF_USE_DMA): cv.All( esp32.only_on_variant( supported=[esp32.VARIANT_ESP32P4, esp32.VARIANT_ESP32S3] @@ -153,12 +118,13 @@ CONFIG_SCHEMA = cv.All( } ).extend(cv.COMPONENT_SCHEMA), cv.has_exactly_one_key(CONF_CHIPSET, CONF_BIT0_HIGH), - cv.has_exactly_one_key(CONF_RGB_ORDER, CONF_RGBW_ORDER), - _validate_rgbw_order_exclusivity, + light.migrate_channel_colors( + removed_in="2027.3.0", component="esp32_rmt_led_strip" + ), ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: # Re-enable ESP-IDF's RMT driver (excluded by default to save compile time) include_builtin_idf_component("esp_driver_rmt") @@ -198,14 +164,9 @@ async def to_code(config): ) ) - if (rgbw_order := config.get(CONF_RGBW_ORDER)) is not None: - rgb_order, white_index = _split_rgbw_order(rgbw_order) - cg.add(var.set_rgb_order(RGB_ORDERS[rgb_order])) - cg.add(var.set_rgbw_order(white_index)) - else: - cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) - cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) - cg.add(var.set_is_wrgb(config[CONF_IS_WRGB])) + cg.add( + var.set_channel_colors(light.channel_colors_struct(config[CONF_CHANNEL_COLORS])) + ) cg.add(var.set_use_psram(config[CONF_USE_PSRAM])) cg.add(var.set_rmt_symbols(config[CONF_RMT_SYMBOLS])) if CONF_USE_DMA in config: diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index 7c4d7ed431..f3a859e38c 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -1,9 +1,12 @@ +from collections.abc import Callable from dataclasses import dataclass, field import enum +import logging import esphome.automation as auto import esphome.codegen as cg from esphome.components import mqtt, power_supply, web_server +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB import esphome.config_validation as cv from esphome.const import ( CONF_BLUE, @@ -23,6 +26,7 @@ from esphome.const import ( CONF_ICON, CONF_ID, CONF_INITIAL_STATE, + CONF_IS_RGBW, CONF_MQTT_ID, CONF_NAME, CONF_ON_STATE, @@ -32,6 +36,7 @@ from esphome.const import ( CONF_POWER_SUPPLY, CONF_RED, CONF_RESTORE_MODE, + CONF_RGB_ORDER, CONF_STATE, CONF_TRIGGER_ID, CONF_WARM_WHITE, @@ -61,6 +66,7 @@ from .effects import ( from .types import ( # noqa: F401 AddressableLight, AddressableLightState, + ChannelColors, ColorMode, LightOutput, LightState, @@ -71,6 +77,8 @@ from .types import ( # noqa: F401 light_ns, ) +_LOGGER = logging.getLogger(__name__) + CODEOWNERS = ["@esphome/core"] IS_PLATFORM_COMPONENT = True @@ -165,7 +173,105 @@ def available_effects_str(effects: list) -> str: return ", ".join(f"'{name}'" for name in available) if available else "none" -def _final_validate(config: ConfigType) -> ConfigType: +# Accepted values of the deprecated `rgb_order` key. +RGB_ORDERS = ("RGB", "RBG", "GRB", "GBR", "BGR", "BRG") + +_RGB_CHANNELS = frozenset("RGB") +_RGBW_CHANNELS = frozenset("RGBW") + + +def validate_channel_colors(value: str) -> str: + """Validate the channel order of an addressable strip, e.g. "GRB" or "WRGB".""" + value = cv.string_strict(value).upper() + channels = frozenset(value) + if len(channels) != len(value) or channels not in (_RGB_CHANNELS, _RGBW_CHANNELS): + raise cv.Invalid( + f"'{value}' is not a valid channel order. List each of R, G and B exactly " + "once, optionally with a single W, in the order the strip expects them " + "(for example GRB, GRBW or WRGB)" + ) + return value + + +def channel_colors_struct(value: str) -> cg.StructInitializer: + """Build the C++ `light::ChannelColors` for a validated channel order string.""" + return cg.StructInitializer( + ChannelColors, + ("r", value.index("R")), + ("g", value.index("G")), + ("b", value.index("B")), + ( + "w", + value.index("W") + if "W" in value + else cg.RawExpression(f"{ChannelColors}::NO_WHITE"), + ), + ) + + +def _quote_and_join(keys: list[str]) -> str: + """Quote each key and join them into a readable list, e.g. "'a', 'b' and 'c'".""" + quoted = [f"'{key}'" for key in keys] + if len(quoted) == 1: + return quoted[0] + return f"{', '.join(quoted[:-1])} and {quoted[-1]}" + + +def migrate_channel_colors( + *, removed_in: str, component: str +) -> Callable[[ConfigType], ConfigType]: + """Fold the deprecated `rgb_order`, `is_rgbw` and `is_wrgb` keys into `channel_colors`. + + This also enforces that `channel_colors` is set, which the schema cannot do on its + own while the deprecated keys are still accepted. After this runs, `to_code` only + ever sees `channel_colors`. + """ + + def validator(config: ConfigType) -> ConfigType: + config = config.copy() + deprecated = [ + key for key in (CONF_RGB_ORDER, CONF_IS_RGBW, CONF_IS_WRGB) if key in config + ] + if CONF_CHANNEL_COLORS in config: + if deprecated: + raise cv.Invalid( + f"'{CONF_CHANNEL_COLORS}' cannot be combined with " + f"{_quote_and_join(deprecated)}" + ) + return config + if CONF_RGB_ORDER not in config: + raise cv.Invalid( + f"'{CONF_CHANNEL_COLORS}' is required", path=[CONF_CHANNEL_COLORS] + ) + rgb_order = config.pop(CONF_RGB_ORDER) + is_rgbw = config.pop(CONF_IS_RGBW, False) + is_wrgb = config.pop(CONF_IS_WRGB, False) + if is_rgbw and is_wrgb: + raise cv.Invalid( + f"'{CONF_IS_RGBW}' and '{CONF_IS_WRGB}' cannot both be enabled" + ) + if is_wrgb: + channel_colors = f"W{rgb_order}" + elif is_rgbw: + channel_colors = f"{rgb_order}W" + else: + channel_colors = rgb_order + _LOGGER.warning( + "[%s] %s %s deprecated, use '%s: %s'. Will be removed in %s", + component, + _quote_and_join(deprecated), + "are" if len(deprecated) > 1 else "is", + CONF_CHANNEL_COLORS, + channel_colors, + removed_in, + ) + config[CONF_CHANNEL_COLORS] = channel_colors + return config + + return validator + + +def _final_validate(config: ConfigType) -> None: """Validate all recorded effect name references against their target lights. This runs once per light platform instance. If no light platform is configured, diff --git a/esphome/components/light/channel_colors.h b/esphome/components/light/channel_colors.h new file mode 100644 index 0000000000..9d8f46d575 --- /dev/null +++ b/esphome/components/light/channel_colors.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +namespace esphome::light { + +/// Which byte of an addressable LED's data carries each colour. +/// +/// Built from a configuration string such as "GRB" or "WRGB": every field holds the +/// position that colour occupies in the bytes the strip expects. `w` is NO_WHITE when +/// the strip has no separate white channel. +struct ChannelColors { + /// Value of `w` for a strip that only has red, green and blue channels. + static constexpr uint8_t NO_WHITE = 0xFF; + + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t w; + + bool has_white() const { return this->w != NO_WHITE; } + + uint8_t bytes_per_led() const { return this->has_white() ? 4 : 3; } + + /// Write the order back out as text, e.g. "GRBW". + /// + /// `buf` must have room for at least 5 characters. Returns `buf` so the result can be + /// passed straight to a log call. + const char *to_string(char *buf) const { + buf[this->r] = 'R'; + buf[this->g] = 'G'; + buf[this->b] = 'B'; + if (this->has_white()) { + buf[this->w] = 'W'; + } + buf[this->bytes_per_led()] = '\0'; + return buf; + } +}; + +} // namespace esphome::light diff --git a/esphome/components/light/types.py b/esphome/components/light/types.py index 9c1c7331d1..1778aa8410 100644 --- a/esphome/components/light/types.py +++ b/esphome/components/light/types.py @@ -16,6 +16,9 @@ LightColorValues = light_ns.class_("LightColorValues") LightStateRTCState = light_ns.struct("LightStateRTCState") LightCall = light_ns.class_("LightCall") +# Addressable strips +ChannelColors = light_ns.struct("ChannelColors") + # Color modes ColorMode = light_ns.enum("ColorMode", is_class=True) COLOR_MODES = { diff --git a/esphome/components/rp2040_pio_led_strip/led_strip.cpp b/esphome/components/rp2040_pio_led_strip/led_strip.cpp index cf7041931e..1f4bea9ecd 100644 --- a/esphome/components/rp2040_pio_led_strip/led_strip.cpp +++ b/esphome/components/rp2040_pio_led_strip/led_strip.cpp @@ -107,10 +107,10 @@ void RP2040PIOLEDStripLightOutput::setup() { pio_get_dreq(this->pio_, this->sm_, true)); // set the DREQ to the state machine's TX FIFO dma_channel_configure(this->dma_chan_, &this->dma_config_, - &this->pio_->txf[this->sm_], // write to the state machine's TX FIFO - this->buf_, // read from memory - this->is_rgbw_ ? num_leds_ * 4 : num_leds_ * 3, // number of bytes to transfer - false // don't start yet + &this->pio_->txf[this->sm_], // write to the state machine's TX FIFO + this->buf_, // read from memory + this->get_buffer_size_(), // number of bytes to transfer + false // don't start yet ); // Initialize the semaphore for this DMA channel @@ -142,58 +142,25 @@ void RP2040PIOLEDStripLightOutput::write_state(light::LightState *state) { } light::ESPColorView RP2040PIOLEDStripLightOutput::get_view_internal(int32_t index) const { - int32_t r = 0, g = 0, b = 0; - switch (this->rgb_order_) { - case ORDER_RGB: - r = 0; - g = 1; - b = 2; - break; - case ORDER_RBG: - r = 0; - g = 2; - b = 1; - break; - case ORDER_GRB: - r = 1; - g = 0; - b = 2; - break; - case ORDER_GBR: - r = 2; - g = 0; - b = 1; - break; - case ORDER_BGR: - r = 2; - g = 1; - b = 0; - break; - case ORDER_BRG: - r = 1; - g = 2; - b = 0; - break; - } - uint8_t multiplier = this->is_rgbw_ ? 4 : 3; - return {this->buf_ + (index * multiplier) + r, - this->buf_ + (index * multiplier) + g, - this->buf_ + (index * multiplier) + b, - this->is_rgbw_ ? this->buf_ + (index * multiplier) + 3 : nullptr, + const light::ChannelColors &colors = this->channel_colors_; + uint8_t *led = this->buf_ + (index * colors.bytes_per_led()); + return {led + colors.r, + led + colors.g, + led + colors.b, + colors.has_white() ? led + colors.w : nullptr, &this->effect_data_[index], &this->correction_}; } void RP2040PIOLEDStripLightOutput::dump_config() { + char channel_colors[5]; ESP_LOGCONFIG(TAG, "RP2040 PIO LED Strip Light Output:\n" " Pin: GPIO%d\n" " Number of LEDs: %d\n" - " RGBW: %s\n" - " RGB Order: %s\n" + " Channel colors: %s\n" " Max Refresh Rate: %f Hz", - this->pin_, this->num_leds_, YESNO(this->is_rgbw_), rgb_order_to_string(this->rgb_order_), - this->max_refresh_rate_); + this->pin_, this->num_leds_, this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_); } float RP2040PIOLEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; } diff --git a/esphome/components/rp2040_pio_led_strip/led_strip.h b/esphome/components/rp2040_pio_led_strip/led_strip.h index c499f0a7ca..b2162f641d 100644 --- a/esphome/components/rp2040_pio_led_strip/led_strip.h +++ b/esphome/components/rp2040_pio_led_strip/led_strip.h @@ -7,6 +7,7 @@ #include "esphome/core/helpers.h" #include "esphome/components/light/addressable_light.h" +#include "esphome/components/light/channel_colors.h" #include "esphome/components/light/light_output.h" #include @@ -18,15 +19,6 @@ namespace esphome::rp2040_pio_led_strip { -enum RGBOrder : uint8_t { - ORDER_RGB, - ORDER_RBG, - ORDER_GRB, - ORDER_GBR, - ORDER_BGR, - ORDER_BRG, -}; - enum Chipset : uint8_t { CHIPSET_WS2812, CHIPSET_WS2812B, @@ -36,25 +28,6 @@ enum Chipset : uint8_t { CHIPSET_CUSTOM = 0xFF, }; -inline const char *rgb_order_to_string(RGBOrder order) { - switch (order) { - case ORDER_RGB: - return "RGB"; - case ORDER_RBG: - return "RBG"; - case ORDER_GRB: - return "GRB"; - case ORDER_GBR: - return "GBR"; - case ORDER_BGR: - return "BGR"; - case ORDER_BRG: - return "BRG"; - default: - return "UNKNOWN"; - } -} - using init_fn = void (*)(PIO pio, uint sm, uint offset, uint pin, float freq); class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { @@ -66,13 +39,14 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { int32_t size() const override { return this->num_leds_; } light::LightTraits get_traits() override { auto traits = light::LightTraits(); - this->is_rgbw_ ? traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}) - : traits.set_supported_color_modes({light::ColorMode::RGB}); + this->channel_colors_.has_white() + ? traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE}) + : traits.set_supported_color_modes({light::ColorMode::RGB}); return traits; } void set_pin(uint8_t pin) { this->pin_ = pin; } void set_num_leds(uint32_t num_leds) { this->num_leds_ = num_leds; } - void set_is_rgbw(bool is_rgbw) { this->is_rgbw_ = is_rgbw; } + void set_channel_colors(light::ChannelColors channel_colors) { this->channel_colors_ = channel_colors; } void set_max_refresh_rate(float interval_us) { this->max_refresh_rate_ = interval_us; } @@ -81,7 +55,6 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { void set_init_function(init_fn init) { this->init_ = init; } void set_chipset(Chipset chipset) { this->chipset_ = chipset; }; - void set_rgb_order(RGBOrder rgb_order) { this->rgb_order_ = rgb_order; } void clear_effect_data() override { for (int i = 0; i < this->size(); i++) { this->effect_data_[i] = 0; @@ -93,7 +66,7 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { protected: light::ESPColorView get_view_internal(int32_t index) const override; - size_t get_buffer_size_() const { return this->num_leds_ * (3 + this->is_rgbw_); } + size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); } static void dma_write_complete_handler(); @@ -102,14 +75,13 @@ class RP2040PIOLEDStripLightOutput final : public light::AddressableLight { uint8_t pin_; uint32_t num_leds_; - bool is_rgbw_; pio_hw_t *pio_; uint sm_; uint dma_chan_; dma_channel_config dma_config_; - RGBOrder rgb_order_{ORDER_RGB}; + light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE}; Chipset chipset_{CHIPSET_CUSTOM}; uint32_t last_refresh_{0}; diff --git a/esphome/components/rp2040_pio_led_strip/light.py b/esphome/components/rp2040_pio_led_strip/light.py index b3f816102a..9f7479edd0 100644 --- a/esphome/components/rp2040_pio_led_strip/light.py +++ b/esphome/components/rp2040_pio_led_strip/light.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from esphome import pins import esphome.codegen as cg from esphome.components import light, rp2 +from esphome.components.const import CONF_CHANNEL_COLORS import esphome.config_validation as cv from esphome.const import ( CONF_CHIPSET, @@ -13,6 +14,7 @@ from esphome.const import ( CONF_PIN, CONF_RGB_ORDER, ) +from esphome.types import ConfigType from esphome.util import _LOGGER @@ -37,7 +39,7 @@ def get_nops(timing): return nops -def generate_assembly_code(id, rgbw, t0h, t0l, t1h, t1l): +def generate_assembly_code(id, t0h, t0l, t1h, t1l): """ Generate assembly code with the given timing values. """ @@ -139,8 +141,6 @@ RP2040PIOLEDStripLightOutput = rp2040_pio_led_strip_ns.class_( "RP2040PIOLEDStripLightOutput", light.AddressableLight ) -RGBOrder = rp2040_pio_led_strip_ns.enum("RGBOrder") - Chipset = rp2040_pio_led_strip_ns.enum("Chipset") CHIPSETS = { @@ -159,15 +159,6 @@ class LEDStripTimings: T1L: int -RGB_ORDERS = { - "RGB": RGBOrder.ORDER_RGB, - "RBG": RGBOrder.ORDER_RBG, - "GRB": RGBOrder.ORDER_GRB, - "GBR": RGBOrder.ORDER_GBR, - "BGR": RGBOrder.ORDER_BGR, - "BRG": RGBOrder.ORDER_BRG, -} - CHIPSET_TIMINGS = { "WS2812": LEDStripTimings(20, 40, 46, 34), "WS2812B": LEDStripTimings(23, 49, 46, 26), @@ -199,10 +190,12 @@ CONFIG_SCHEMA = cv.All( cv.GenerateID(CONF_OUTPUT_ID): cv.declare_id(RP2040PIOLEDStripLightOutput), cv.Required(CONF_PIN): pins.internal_gpio_output_pin_number, cv.Required(CONF_NUM_LEDS): cv.positive_not_null_int, - cv.Required(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), + cv.Optional(CONF_CHANNEL_COLORS): light.validate_channel_colors, + # Deprecated in favour of CONF_CHANNEL_COLORS, remove in 2027.3.0 + cv.Optional(CONF_RGB_ORDER): cv.one_of(*light.RGB_ORDERS, upper=True), + cv.Optional(CONF_IS_RGBW): cv.boolean, cv.Required(CONF_PIO): cv.one_of(0, 1, int=True), cv.Optional(CONF_CHIPSET): cv.enum(CHIPSETS, upper=True), - cv.Optional(CONF_IS_RGBW, default=False): cv.boolean, cv.Inclusive( CONF_BIT0_HIGH, "custom", @@ -222,10 +215,13 @@ CONFIG_SCHEMA = cv.All( } ), cv.has_exactly_one_key(CONF_CHIPSET, CONF_BIT0_HIGH), + light.migrate_channel_colors( + removed_in="2027.3.0", component="rp2040_pio_led_strip" + ), ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_OUTPUT_ID]) id = config[CONF_ID].id await light.register_light(var, config) @@ -234,8 +230,9 @@ async def to_code(config): cg.add(var.set_num_leds(config[CONF_NUM_LEDS])) cg.add(var.set_pin(config[CONF_PIN])) - cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) - cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) + cg.add( + var.set_channel_colors(light.channel_colors_struct(config[CONF_CHANNEL_COLORS])) + ) cg.add(var.set_pio(config[CONF_PIO])) cg.add(var.set_program(cg.RawExpression(f"&rp2040_pio_led_strip_{id}_program"))) @@ -255,7 +252,6 @@ async def to_code(config): key, generate_assembly_code( id, - config[CONF_IS_RGBW], CHIPSET_TIMINGS[chipset].T0H, CHIPSET_TIMINGS[chipset].T0L, CHIPSET_TIMINGS[chipset].T1H, @@ -270,7 +266,6 @@ async def to_code(config): key, generate_assembly_code( id, - config[CONF_IS_RGBW], time_to_cycles(config[CONF_BIT0_HIGH]), time_to_cycles(config[CONF_BIT0_LOW]), time_to_cycles(config[CONF_BIT1_HIGH]), diff --git a/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml b/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml index a071f9df91..d21c4b61b9 100644 --- a/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml +++ b/tests/components/addressable_light/common-ard-esp32_rmt_led_strip.yaml @@ -3,7 +3,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} diff --git a/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml b/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml index a071f9df91..d21c4b61b9 100644 --- a/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml +++ b/tests/components/addressable_light/common-idf-esp32_rmt_led_strip.yaml @@ -3,7 +3,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} diff --git a/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml b/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml index 15409caeaf..2bb831848c 100644 --- a/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml +++ b/tests/components/beken_spi_led_strip/test.bk72xx-ard.yaml @@ -1,6 +1,6 @@ light: - platform: beken_spi_led_strip - rgb_order: GRB + channel_colors: GRB pin: P16 num_leds: 30 chipset: ws2812 diff --git a/tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml b/tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml new file mode 100644 index 0000000000..3ca78398c3 --- /dev/null +++ b/tests/components/beken_spi_led_strip/validate-legacy.bk72xx-ard.yaml @@ -0,0 +1,10 @@ +# The deprecated rgb_order / is_rgbw / is_wrgb keys, kept working until 2027.3.0. +# Config-only, and only one strip because P16 is the sole supported pin. +light: + - platform: beken_spi_led_strip + name: Legacy RGBW + pin: P16 + num_leds: 30 + chipset: sk6812 + rgb_order: GRB + is_rgbw: true # -> GRBW diff --git a/tests/components/e131/common-ard.yaml b/tests/components/e131/common-ard.yaml index 8300dbb01b..48ccafc2d2 100644 --- a/tests/components/e131/common-ard.yaml +++ b/tests/components/e131/common-ard.yaml @@ -5,7 +5,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} effects: diff --git a/tests/components/e131/common-idf.yaml b/tests/components/e131/common-idf.yaml index 8300dbb01b..48ccafc2d2 100644 --- a/tests/components/e131/common-idf.yaml +++ b/tests/components/e131/common-idf.yaml @@ -5,7 +5,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: ${pin} effects: diff --git a/tests/components/e131/test.rp2040-ard.yaml b/tests/components/e131/test.rp2040-ard.yaml index 4593784ef9..89255e2d87 100644 --- a/tests/components/e131/test.rp2040-ard.yaml +++ b/tests/components/e131/test.rp2040-ard.yaml @@ -6,7 +6,7 @@ light: pin: 2 pio: 0 num_leds: 256 - rgb_order: GRB + channel_colors: GRB chipset: WS2812 effects: - e131: diff --git a/tests/components/esp32_rmt_led_strip/common.yaml b/tests/components/esp32_rmt_led_strip/common.yaml index 701e513ebd..7f52d32229 100644 --- a/tests/components/esp32_rmt_led_strip/common.yaml +++ b/tests/components/esp32_rmt_led_strip/common.yaml @@ -3,13 +3,13 @@ light: id: led_strip1 pin: ${pin1} num_leds: 60 - rgb_order: GRB + channel_colors: GRB chipset: ws2812 - platform: esp32_rmt_led_strip id: led_strip2 pin: ${pin2} num_leds: 60 - rgbw_order: RWGB + channel_colors: RWGB bit0_high: 100us bit0_low: 100us bit1_high: 100us diff --git a/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml b/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml index 6bf0639a52..132966eddf 100644 --- a/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml +++ b/tests/components/esp32_rmt_led_strip/test.esp32-s3-idf.yaml @@ -8,14 +8,14 @@ light: id: led_strip1 pin: ${pin1} num_leds: 60 - rgb_order: GRB + channel_colors: GRB chipset: ws2812 use_dma: "true" - platform: esp32_rmt_led_strip id: led_strip2 pin: ${pin2} num_leds: 60 - rgb_order: RGB + channel_colors: RGB bit0_high: 100us bit0_low: 100us bit1_high: 100us diff --git a/tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml b/tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml new file mode 100644 index 0000000000..6dd1bcdad3 --- /dev/null +++ b/tests/components/esp32_rmt_led_strip/validate-legacy.esp32-idf.yaml @@ -0,0 +1,23 @@ +# The deprecated rgb_order / is_rgbw / is_wrgb keys, kept working until 2027.3.0. +# Config-only: each strip below must migrate to the channel_colors shown in the comment. +light: + - platform: esp32_rmt_led_strip + id: legacy_rgb + pin: GPIO13 + num_leds: 60 + chipset: ws2812 + rgb_order: GRB # -> GRB + - platform: esp32_rmt_led_strip + id: legacy_rgbw + pin: GPIO14 + num_leds: 60 + chipset: sk6812 + rgb_order: GRB + is_rgbw: true # -> GRBW + - platform: esp32_rmt_led_strip + id: legacy_wrgb + pin: GPIO15 + num_leds: 60 + chipset: sk6812 + rgb_order: GRB + is_wrgb: true # -> WGRB diff --git a/tests/components/partition/common-ard.yaml b/tests/components/partition/common-ard.yaml index b2ceadd6f7..8d39670e32 100644 --- a/tests/components/partition/common-ard.yaml +++ b/tests/components/partition/common-ard.yaml @@ -4,7 +4,7 @@ light: default_transition_length: 500ms chipset: ws2812 num_leds: 256 - rgb_order: GRB + channel_colors: GRB pin: ${pin} - platform: partition name: Partition Light diff --git a/tests/components/partition/common-idf.yaml b/tests/components/partition/common-idf.yaml index b2ceadd6f7..8d39670e32 100644 --- a/tests/components/partition/common-idf.yaml +++ b/tests/components/partition/common-idf.yaml @@ -4,7 +4,7 @@ light: default_transition_length: 500ms chipset: ws2812 num_leds: 256 - rgb_order: GRB + channel_colors: GRB pin: ${pin} - platform: partition name: Partition Light diff --git a/tests/components/rp2040_pio_led_strip/common.yaml b/tests/components/rp2040_pio_led_strip/common.yaml index 254ac0e13d..1cb5fe0737 100644 --- a/tests/components/rp2040_pio_led_strip/common.yaml +++ b/tests/components/rp2040_pio_led_strip/common.yaml @@ -4,14 +4,14 @@ light: pin: 4 num_leds: 60 pio: 0 - rgb_order: GRB + channel_colors: GRB chipset: WS2812 - platform: rp2040_pio_led_strip id: led_strip_custom_timings pin: 5 num_leds: 60 pio: 1 - rgb_order: GRB + channel_colors: GRB bit0_high: .1us bit0_low: 1.2us bit1_high: .69us diff --git a/tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml b/tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml new file mode 100644 index 0000000000..2ab124393b --- /dev/null +++ b/tests/components/rp2040_pio_led_strip/validate-legacy.rp2040-ard.yaml @@ -0,0 +1,18 @@ +# The deprecated rgb_order / is_rgbw keys, kept working until 2027.3.0. +# Config-only: each strip below must migrate to the channel_colors shown in the comment. +light: + - platform: rp2040_pio_led_strip + id: legacy_rgb + pin: 4 + num_leds: 60 + pio: 0 + chipset: WS2812 + rgb_order: GRB # -> GRB + - platform: rp2040_pio_led_strip + id: legacy_rgbw + pin: 5 + num_leds: 60 + pio: 1 + chipset: SK6812 + rgb_order: GRB + is_rgbw: true # -> GRBW diff --git a/tests/components/wled/test.esp32-ard.yaml b/tests/components/wled/test.esp32-ard.yaml index 156b31181e..ecab767812 100644 --- a/tests/components/wled/test.esp32-ard.yaml +++ b/tests/components/wled/test.esp32-ard.yaml @@ -9,7 +9,7 @@ light: id: led_matrix_32x8 default_transition_length: 500ms chipset: ws2812 - rgb_order: GRB + channel_colors: GRB num_leds: 256 pin: 2 effects: diff --git a/tests/unit_tests/components/light/test_channel_colors.py b/tests/unit_tests/components/light/test_channel_colors.py new file mode 100644 index 0000000000..0c129a8bb2 --- /dev/null +++ b/tests/unit_tests/components/light/test_channel_colors.py @@ -0,0 +1,144 @@ +"""Tests for the shared addressable-strip channel order helpers.""" + +import logging + +import pytest + +from esphome.components.const import CONF_CHANNEL_COLORS, CONF_IS_WRGB +from esphome.components.light import ( + channel_colors_struct, + migrate_channel_colors, + validate_channel_colors, +) +import esphome.config_validation as cv +from esphome.const import CONF_IS_RGBW, CONF_RGB_ORDER +from esphome.types import ConfigType + +NO_WHITE = "light::ChannelColors::NO_WHITE" + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("RGB", "RGB"), + ("grb", "GRB"), + ("BRG", "BRG"), + ("rgbw", "RGBW"), + ("WRGB", "WRGB"), + ("GWRB", "GWRB"), + ], +) +def test_validate_channel_colors(value: str, expected: str) -> None: + assert validate_channel_colors(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + "RG", # missing a channel + "RGBB", # duplicate channel + "RRGB", # duplicate channel, correct length + "RGBWW", # two white channels + "RGBX", # unknown channel + "RGBWX", # unknown channel, correct length + "", + ], +) +def test_validate_channel_colors_rejects_invalid(value: str) -> None: + with pytest.raises(cv.Invalid, match="is not a valid channel order"): + validate_channel_colors(value) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("RGB", (0, 1, 2, NO_WHITE)), + ("GRB", (1, 0, 2, NO_WHITE)), + ("BRG", (1, 2, 0, NO_WHITE)), + ("RGBW", (0, 1, 2, 3)), + ("GRBW", (1, 0, 2, 3)), + ("WRGB", (1, 2, 3, 0)), + ("GWRB", (2, 0, 3, 1)), + ], +) +def test_channel_colors_struct(value: str, expected: tuple[int, int, int, int]) -> None: + struct = channel_colors_struct(value) + assert str(struct.base) == "light::ChannelColors" + assert tuple(str(arg) for arg in struct.args.values()) == tuple( + str(field) for field in expected + ) + + +def _migrate(config: ConfigType) -> ConfigType: + return migrate_channel_colors(removed_in="2027.3.0", component="test_strip")(config) + + +def test_migrate_passes_through_channel_colors() -> None: + config = {CONF_CHANNEL_COLORS: "GRBW"} + assert _migrate(config) == {CONF_CHANNEL_COLORS: "GRBW"} + + +@pytest.mark.parametrize( + ("deprecated", "expected", "named"), + [ + ({}, "GRB", "'rgb_order' is"), + ( + {CONF_IS_RGBW: False, CONF_IS_WRGB: False}, + "GRB", + "'rgb_order', 'is_rgbw' and 'is_wrgb' are", + ), + ({CONF_IS_RGBW: True}, "GRBW", "'rgb_order' and 'is_rgbw' are"), + ({CONF_IS_WRGB: True}, "WGRB", "'rgb_order' and 'is_wrgb' are"), + ], +) +def test_migrate_folds_deprecated_keys( + deprecated: ConfigType, + expected: str, + named: str, + caplog: pytest.LogCaptureFixture, +) -> None: + config = {CONF_RGB_ORDER: "GRB", "num_leds": 1, **deprecated} + with caplog.at_level(logging.WARNING): + result = _migrate(config) + + assert result == {CONF_CHANNEL_COLORS: expected, "num_leds": 1} + assert f"[test_strip] {named} deprecated" in caplog.text + assert f"'{CONF_CHANNEL_COLORS}: {expected}'" in caplog.text + assert "2027.3.0" in caplog.text + + +def test_migrate_does_not_mutate_input() -> None: + config = {CONF_RGB_ORDER: "GRB", CONF_IS_RGBW: True} + _migrate(config) + assert config == {CONF_RGB_ORDER: "GRB", CONF_IS_RGBW: True} + + +@pytest.mark.parametrize("deprecated", [CONF_RGB_ORDER, CONF_IS_RGBW, CONF_IS_WRGB]) +def test_migrate_rejects_mixing_old_and_new(deprecated: str) -> None: + config = {CONF_CHANNEL_COLORS: "GRBW", deprecated: "GRB"} + with pytest.raises(cv.Invalid, match=f"cannot be combined with '{deprecated}'"): + _migrate(config) + + +def test_migrate_reports_every_conflicting_key() -> None: + config = { + CONF_CHANNEL_COLORS: "GRBW", + CONF_RGB_ORDER: "GRB", + CONF_IS_RGBW: True, + CONF_IS_WRGB: False, + } + with pytest.raises( + cv.Invalid, match="cannot be combined with 'rgb_order', 'is_rgbw' and 'is_wrgb'" + ): + _migrate(config) + + +def test_migrate_requires_channel_colors() -> None: + with pytest.raises(cv.Invalid, match=f"'{CONF_CHANNEL_COLORS}' is required"): + _migrate({"num_leds": 1}) + + +def test_migrate_rejects_is_rgbw_with_is_wrgb() -> None: + config = {CONF_RGB_ORDER: "GRB", CONF_IS_RGBW: True, CONF_IS_WRGB: True} + with pytest.raises(cv.Invalid, match="cannot both be enabled"): + _migrate(config) diff --git a/tests/unit_tests/components/test_esp32_rmt_led_strip.py b/tests/unit_tests/components/test_esp32_rmt_led_strip.py deleted file mode 100644 index e2cb513e3b..0000000000 --- a/tests/unit_tests/components/test_esp32_rmt_led_strip.py +++ /dev/null @@ -1,57 +0,0 @@ -import pytest - -from esphome.components.esp32_rmt_led_strip.light import ( - CONF_IS_WRGB, - CONF_RGBW_ORDER, - _split_rgbw_order, - _validate_rgbw_order, - _validate_rgbw_order_exclusivity, -) -import esphome.config_validation as cv -from esphome.const import CONF_IS_RGBW - - -def test_validate_rgbw_order() -> None: - assert _validate_rgbw_order("rwgb") == "RWGB" - - -@pytest.mark.parametrize("rgbw_order", ["RGB", "RRGB", "RGBWW"]) -def test_validate_rgbw_order_rejects_invalid_order(rgbw_order: str) -> None: - with pytest.raises(cv.Invalid, match="permutation of RGBW"): - _validate_rgbw_order(rgbw_order) - - -@pytest.mark.parametrize( - ("rgbw_order", "expected"), - [ - ("WRGB", ("RGB", 0)), - ("RWGB", ("RGB", 1)), - ("GWRB", ("GRB", 1)), - ("RGBW", ("RGB", 3)), - ], -) -def test_split_rgbw_order(rgbw_order: str, expected: tuple[str, int]) -> None: - assert _split_rgbw_order(rgbw_order) == expected - - -@pytest.mark.parametrize("conflict", [CONF_IS_RGBW, CONF_IS_WRGB]) -def test_rgbw_order_is_mutually_exclusive(conflict: str) -> None: - with pytest.raises(cv.Invalid, match="cannot be used with"): - _validate_rgbw_order_exclusivity( - { - CONF_RGBW_ORDER: "RGBW", - CONF_IS_RGBW: conflict == CONF_IS_RGBW, - CONF_IS_WRGB: conflict == CONF_IS_WRGB, - } - ) - - -@pytest.mark.parametrize("legacy_option", [CONF_IS_RGBW, CONF_IS_WRGB]) -def test_rgbw_order_allows_disabled_legacy_options(legacy_option: str) -> None: - config = { - CONF_RGBW_ORDER: "RGBW", - CONF_IS_RGBW: False, - CONF_IS_WRGB: False, - } - config[legacy_option] = False - assert _validate_rgbw_order_exclusivity(config) is config From 8b888f31e0bf4d2dedd34fa25d7f1aa593a0c581 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 17 Aug 2026 20:32:03 -0500 Subject: [PATCH 114/160] [gpio_expander][pcf8574][pca9554][tca9555][pca6416a][pi4ioe5v6408][mcp23016][mcp23xxx_base] Reject unsupported interrupt_pin options (inverted, allow_other_uses) (#18472) --- esphome/components/gpio_expander/__init__.py | 22 +++++++ esphome/components/mcp23016/__init__.py | 4 +- esphome/components/mcp23xxx_base/__init__.py | 22 +------ esphome/components/pca6416a/__init__.py | 4 +- esphome/components/pca9554/__init__.py | 4 +- esphome/components/pcf8574/__init__.py | 4 +- esphome/components/pi4ioe5v6408/__init__.py | 4 +- esphome/components/tca9555/__init__.py | 4 +- script/build_language_schema.py | 10 +++ .../component_tests/gpio_expander/__init__.py | 0 .../gpio_expander/test_init.py | 61 +++++++++++++++++++ 11 files changed, 107 insertions(+), 32 deletions(-) create mode 100644 tests/component_tests/gpio_expander/__init__.py create mode 100644 tests/component_tests/gpio_expander/test_init.py diff --git a/esphome/components/gpio_expander/__init__.py b/esphome/components/gpio_expander/__init__.py index e69de29bb2..0c7199b6df 100644 --- a/esphome/components/gpio_expander/__init__.py +++ b/esphome/components/gpio_expander/__init__.py @@ -0,0 +1,22 @@ +from esphome import pins +import esphome.config_validation as cv +from esphome.const import CONF_ALLOW_OTHER_USES, CONF_INTERRUPT_PIN, CONF_INVERTED +from esphome.types import ConfigType + + +def validate_interrupt_pin(value: ConfigType) -> ConfigType: + # The expander components own INT polarity (active-low, hardcoded falling-edge ISR) + # and install a single ISR per GPIO, so neither inversion nor sharing is supported. + value = pins.internal_gpio_input_pin_schema(value) + if value.get(CONF_INVERTED): + raise cv.Invalid( + f"'{CONF_INVERTED}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " + "the expander INT line is fixed active-low" + ) + if value.get(CONF_ALLOW_OTHER_USES): + raise cv.Invalid( + f"'{CONF_ALLOW_OTHER_USES}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " + "sharing the interrupt pin between multiple components is not implemented. " + f"Remove the '{CONF_INTERRUPT_PIN}' to fall back to polling." + ) + return value diff --git a/esphome/components/mcp23016/__init__.py b/esphome/components/mcp23016/__init__.py index b71d57498a..37c5205fe8 100644 --- a/esphome/components/mcp23016/__init__.py +++ b/esphome/components/mcp23016/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -25,7 +25,7 @@ CONFIG_SCHEMA = ( cv.Schema( { cv.Required(CONF_ID): cv.declare_id(MCP23016), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/mcp23xxx_base/__init__.py b/esphome/components/mcp23xxx_base/__init__.py index 76a3aabe3f..d53499a78f 100644 --- a/esphome/components/mcp23xxx_base/__init__.py +++ b/esphome/components/mcp23xxx_base/__init__.py @@ -1,8 +1,8 @@ from esphome import pins import esphome.codegen as cg +from esphome.components import gpio_expander import esphome.config_validation as cv from esphome.const import ( - CONF_ALLOW_OTHER_USES, CONF_ID, CONF_INPUT, CONF_INTERRUPT, @@ -32,28 +32,10 @@ MCP23XXX_INTERRUPT_MODES = { } -def _validate_interrupt_pin(value): - # The MCP component owns INT polarity (active-low, hardcoded falling-edge ISR) - # and installs a single ISR per GPIO, so neither inversion nor sharing is supported. - value = pins.internal_gpio_input_pin_schema(value) - if value.get(CONF_INVERTED): - raise cv.Invalid( - f"'{CONF_INVERTED}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " - "the MCP23xxx INT line is fixed active-low" - ) - if value.get(CONF_ALLOW_OTHER_USES): - raise cv.Invalid( - f"'{CONF_ALLOW_OTHER_USES}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " - "sharing the interrupt pin between multiple MCP23xxx (or other components) " - "is not implemented. Remove the interrupt_pin to fall back to polling." - ) - return value - - MCP23XXX_CONFIG_SCHEMA = cv.Schema( { cv.Optional(CONF_OPEN_DRAIN_INTERRUPT, default=False): cv.boolean, - cv.Optional(CONF_INTERRUPT_PIN): _validate_interrupt_pin, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ).extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pca6416a/__init__.py b/esphome/components/pca6416a/__init__.py index 813bb35c48..1df22a8ff5 100644 --- a/esphome/components/pca6416a/__init__.py +++ b/esphome/components/pca6416a/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -29,7 +29,7 @@ CONFIG_SCHEMA = ( cv.Schema( { cv.Required(CONF_ID): cv.declare_id(PCA6416AComponent), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pca9554/__init__.py b/esphome/components/pca9554/__init__.py index 99b812b33b..f49a68bc3f 100644 --- a/esphome/components/pca9554/__init__.py +++ b/esphome/components/pca9554/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -30,7 +30,7 @@ CONFIG_SCHEMA = ( { cv.Required(CONF_ID): cv.declare_id(PCA9554Component), cv.Optional(CONF_PIN_COUNT, default=8): cv.one_of(4, 8, 16), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pcf8574/__init__.py b/esphome/components/pcf8574/__init__.py index d8a1e20db6..559fe1d76d 100644 --- a/esphome/components/pcf8574/__init__.py +++ b/esphome/components/pcf8574/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -28,7 +28,7 @@ CONFIG_SCHEMA = ( { cv.Required(CONF_ID): cv.declare_id(PCF8574Component), cv.Optional(CONF_PCF8575, default=False): cv.boolean, - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/pi4ioe5v6408/__init__.py b/esphome/components/pi4ioe5v6408/__init__.py index d5b19dab1c..ee270138e1 100644 --- a/esphome/components/pi4ioe5v6408/__init__.py +++ b/esphome/components/pi4ioe5v6408/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -34,7 +34,7 @@ CONFIG_SCHEMA = ( { cv.Required(CONF_ID): cv.declare_id(PI4IOE5V6408Component), cv.Optional(CONF_RESET, default=True): cv.boolean, - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/esphome/components/tca9555/__init__.py b/esphome/components/tca9555/__init__.py index 5f571fcea6..1c643fe1c9 100644 --- a/esphome/components/tca9555/__init__.py +++ b/esphome/components/tca9555/__init__.py @@ -1,6 +1,6 @@ from esphome import pins import esphome.codegen as cg -from esphome.components import i2c +from esphome.components import gpio_expander, i2c import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -28,7 +28,7 @@ CONFIG_SCHEMA = ( cv.Schema( { cv.Required(CONF_ID): cv.declare_id(TCA9555Component), - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/script/build_language_schema.py b/script/build_language_schema.py index 2b64cb0256..91c1de00cd 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -250,6 +250,16 @@ def add_pin_validators(): "modes": ["input"], } + from esphome.components import gpio_expander + + # Wraps pins.internal_gpio_input_pin_schema, so the editor schema must keep + # treating the config var as a pin + pin_validators[repr(gpio_expander.validate_interrupt_pin)] = { + "schema": True, + "internal": True, + "modes": ["input"], + } + def add_module_registries(domain, module): for attr_name in dir(module): diff --git a/tests/component_tests/gpio_expander/__init__.py b/tests/component_tests/gpio_expander/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/gpio_expander/test_init.py b/tests/component_tests/gpio_expander/test_init.py new file mode 100644 index 0000000000..806b1775d2 --- /dev/null +++ b/tests/component_tests/gpio_expander/test_init.py @@ -0,0 +1,61 @@ +"""Tests for the shared io expander interrupt_pin validator.""" + +from __future__ import annotations + +import importlib + +import pytest + +from esphome import config_validation as cv +from esphome.components.esp32 import KEY_BOARD, KEY_VARIANT, VARIANT_ESP32 +from esphome.components.gpio_expander import validate_interrupt_pin +from esphome.const import PlatformFramework +from tests.component_tests.types import SetCoreConfigCallable + + +@pytest.fixture +def stage_esp32(set_core_config: SetCoreConfigCallable) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + + +def test_plain_pin_accepted(stage_esp32: None) -> None: + value = validate_interrupt_pin( + {"number": 16, "mode": {"input": True, "pullup": True}} + ) + assert value["number"] == 16 + + +def test_inverted_rejected(stage_esp32: None) -> None: + with pytest.raises(cv.Invalid, match="'inverted: true' is not supported"): + validate_interrupt_pin({"number": 16, "inverted": True}) + + +def test_allow_other_uses_rejected(stage_esp32: None) -> None: + with pytest.raises(cv.Invalid, match="'allow_other_uses: true' is not supported"): + validate_interrupt_pin({"number": 16, "allow_other_uses": True}) + + +# mcp23017 covers the shared mcp23xxx_base schema +@pytest.mark.parametrize( + "component", + [ + "pcf8574", + "pca9554", + "tca9555", + "pca6416a", + "pi4ioe5v6408", + "mcp23016", + "mcp23017", + ], +) +def test_component_schemas_route_through_validator( + stage_esp32: None, component: str +) -> None: + module = importlib.import_module(f"esphome.components.{component}") + with pytest.raises(cv.Invalid, match="'inverted: true' is not supported"): + module.CONFIG_SCHEMA( + {"id": "expander_hub", "interrupt_pin": {"number": 16, "inverted": True}} + ) From d1391c2b10a2f473b11d2a69c0ea8e3eecd260c2 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:04:49 +1200 Subject: [PATCH 115/160] Bump version to 2026.8.0b5 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 2df6d3ded0..3dad4629be 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.0b4 +PROJECT_NUMBER = 2026.8.0b5 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 73155e06ee..e86465f9a0 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.0b4" +__version__ = "2026.8.0b5" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 9823205ef3080e5e7fd9c4004f3cefc1d68a0e37 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:03:14 -0500 Subject: [PATCH 116/160] [api] Bump noise-c to 0.1.20 (#18482) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index cdc0d97c49..3e69d5842c 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.19") + cg.add_library("esphome/noise-c", "0.1.20") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 39600d622a..13bb5a556f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.19 ; api + esphome/noise-c@0.1.20 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.19 ; api + esphome/noise-c@0.1.20 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.19 ; used by api + esphome/noise-c@0.1.20 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From ae730d6357e6a82eed82f89a3e396793bf499baf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:03:33 -0500 Subject: [PATCH 117/160] [ci] Fail the benchmark job when the C++ benchmark build fails (#18480) --- .github/workflows/ci.yml | 17 ++++++++++++++--- tests/benchmarks/components/api/__init__.py | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd1a382c21..0c81c783b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -464,10 +464,21 @@ jobs: - name: Build benchmarks id: build run: | + # pipefail: without it a failed build is masked by the grep/cut + # pipeline below, leaving BINARY empty and silently dropping every + # C++ benchmark from the run while the job still reports success. + set -o pipefail . venv/bin/activate - export BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) - # --build-only prints BUILD_BINARY= to stdout - BINARY=$(script/cpp_benchmark.py --all --build-only | grep '^BUILD_BINARY=' | tail -1 | cut -d= -f2-) + BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) + export BENCHMARK_LIB_CONFIG + # --build-only prints BUILD_BINARY= to stdout; the grep is + # non-fatal so a missing marker reaches the check below instead of + # tripping errexit at this assignment + BINARY=$(script/cpp_benchmark.py --all --build-only | { grep '^BUILD_BINARY=' || true; } | tail -1 | cut -d= -f2-) + if [ -z "$BINARY" ]; then + echo "::error::Benchmark build did not report a binary path" + exit 1 + fi echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/tests/benchmarks/components/api/__init__.py b/tests/benchmarks/components/api/__init__.py index 0d02e0b054..0565bc5330 100644 --- a/tests/benchmarks/components/api/__init__.py +++ b/tests/benchmarks/components/api/__init__.py @@ -15,6 +15,7 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # components have hardware dependencies (BLE/UART/RMT); lightweight # stub headers in tests/benchmarks/stubs/ satisfy the includes. cg.add_define("USE_BLUETOOTH_PROXY") + cg.add_define("USE_BLUETOOTH_PROXY_CONNECTIONS") cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", 3) cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) cg.add_define("USE_ZWAVE_PROXY") From 476d540065ecd352a5aa4ff52179966d1f732163 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:22:28 -0400 Subject: [PATCH 118/160] [ci] Fall back to files API when PR diff exceeds GitHub line limit (#18486) --- script/helpers.py | 5 +++-- tests/script/test_helpers.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/script/helpers.py b/script/helpers.py index 11549808ff..8132ee49e5 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -558,8 +558,9 @@ def _get_changed_files_github_actions() -> list[str] | None: try: return _get_changed_files_from_command(cmd) except Exception as e: - # If it fails due to the 300 file limit, use the API method - if "maximum" in str(e) and "files" in str(e): + # If it fails due to a diff limit (300 files or 20000 lines), + # use the API method which only returns filenames + if "diff exceeded the maximum" in str(e): cmd = [ "gh", "api", diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index a07e56cea5..2c3ae95655 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -244,6 +244,44 @@ def test_get_changed_files_github_actions_pull_request_large_pr( assert result == expected_files +def test_get_changed_files_github_actions_pull_request_large_diff( + monkeypatch: MonkeyPatch, +) -> None: + """Test _get_changed_files_github_actions fallback for PRs with >20000 diff lines.""" + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + expected_files = ["file1.py", "file2.cpp"] + + with ( + patch("helpers._get_pr_number_from_github_env", return_value="17909"), + patch("helpers._get_changed_files_from_command") as mock_get, + ): + # First call fails with too many diff lines error, second succeeds with API method + mock_get.side_effect = [ + Exception( + "could not find pull request diff: HTTP 406: Sorry, " + "the diff exceeded the maximum number of lines (20000)" + ), + expected_files, + ] + + result = _get_changed_files_github_actions() + + assert mock_get.call_count == 2 + mock_get.assert_any_call(["gh", "pr", "diff", "17909", "--name-only"]) + mock_get.assert_any_call( + [ + "gh", + "api", + "repos/esphome/esphome/pulls/17909/files", + "--paginate", + "--jq", + ".[].filename", + ] + ) + assert result == expected_files + + def test_get_changed_files_github_actions_pull_request_other_error( monkeypatch: MonkeyPatch, ) -> None: From 92f55f721f35d36b4a883811c6cebb6b1027cb7b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:32:37 -0500 Subject: [PATCH 119/160] [api] Bump noise-c to 0.1.21 (#18484) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3e69d5842c..912d580a0f 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.20") + cg.add_library("esphome/noise-c", "0.1.21") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 13bb5a556f..4c372cc0bb 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.20 ; api + esphome/noise-c@0.1.21 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.20 ; api + esphome/noise-c@0.1.21 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.20 ; used by api + esphome/noise-c@0.1.21 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 285a508e09effe69510fbde25f92b6eb7dd21c03 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Tue, 18 Aug 2026 09:09:12 -0700 Subject: [PATCH 120/160] [modbus] CRC scan all unknown function codes (#18483) Co-authored-by: Claude Opus 4.8 (1M context) --- esphome/components/modbus/modbus.cpp | 33 ++-- esphome/components/modbus/modbus.h | 2 +- esphome/components/modbus/modbus_helpers.h | 32 ++++ tests/components/modbus/common.h | 36 +++++ .../modbus/modbus_unknown_function_test.cpp | 141 ++++++++++++++++++ 5 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 tests/components/modbus/modbus_unknown_function_test.cpp diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index 5305f6313f..e4bd51ad5a 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -219,14 +219,25 @@ void ModbusServerHub::parse_modbus_frames() { this->clear_rx_buffer_(LOG_STR("timeout after partial response"), true); } -uint16_t Modbus::find_custom_frame_end_(uint16_t min_length) const { - // Custom functions could be any length - we have to rely on the CRC to determine completeness. +uint16_t Modbus::find_frame_end_by_crc_(uint16_t min_length) const { + // Unknown-length functions (user-defined codes, unimplemented management codes, unassigned values) + // could be any length - we have to rely on the CRC to determine completeness. // If a CRC match is never found, the buffer will eventually overflow and be cleared. const uint8_t *raw = &this->rx_buffer_[0]; const size_t size = this->rx_buffer_.size(); - for (uint16_t len = min_length; len <= std::min(size, size_t(MAX_FRAME_SIZE)); len++) { - if (crc16(raw, len) == 0) - return len; + const auto max_len = static_cast(std::min(size, size_t(MAX_FRAME_SIZE))); + if (min_length > max_len) + return 0; + // The Modbus CRC (poly 0xa001, refin/refout false) keeps its running state in the returned value, + // so we seed once over the first min_length bytes and extend one byte at a time instead of + // recomputing the whole prefix for every candidate length. + uint16_t crc = crc16(raw, min_length); + if (crc == 0) + return min_length; + for (uint16_t len = min_length; len < max_len; len++) { + crc = crc16(&raw[len], 1, crc); + if (crc == 0) + return len + 1; } return 0; } @@ -241,11 +252,11 @@ bool Modbus::parse_modbus_server_frame_() { uint8_t address = this->rx_buffer_[0]; uint8_t function_code = this->rx_buffer_[1]; - if (helpers::is_function_code_custom(function_code)) { - frame_length = this->find_custom_frame_end_(frame_length); + if (helpers::is_function_code_unknown_length(function_code)) { + frame_length = this->find_frame_end_by_crc_(frame_length); if (frame_length == 0) return size < MAX_FRAME_SIZE; // Continue to parse until we hit max size - ESP_LOGD(TAG, "User-defined function %02X found", function_code); + ESP_LOGD(TAG, "Unknown-length function %02X found", function_code); } else { if (crc16(&this->rx_buffer_[0], frame_length) != 0) return false; @@ -272,11 +283,11 @@ bool ModbusServerHub::parse_modbus_client_frame_() { uint8_t address = this->rx_buffer_[0]; uint8_t function_code = this->rx_buffer_[1]; - if (helpers::is_function_code_custom(function_code)) { - frame_length = this->find_custom_frame_end_(frame_length); + if (helpers::is_function_code_unknown_length(function_code)) { + frame_length = this->find_frame_end_by_crc_(frame_length); if (frame_length == 0) return size < MAX_FRAME_SIZE; // Continue to parse until we hit max size - ESP_LOGD(TAG, "User-defined function %02X found", function_code); + ESP_LOGD(TAG, "Unknown-length function %02X found", function_code); } else { if (crc16(&this->rx_buffer_[0], frame_length) != 0) return false; diff --git a/esphome/components/modbus/modbus.h b/esphome/components/modbus/modbus.h index dfe4a4872d..bb303c43a8 100644 --- a/esphome/components/modbus/modbus.h +++ b/esphome/components/modbus/modbus.h @@ -82,7 +82,7 @@ class Modbus : public uart::UARTDevice, public Component { bool send_frame_(const ModbusFrame &frame); // Scans forward from min_length to find a frame boundary by CRC match for custom function codes. // Returns the matched frame length, or 0 if no valid CRC was found within MAX_FRAME_SIZE. - uint16_t find_custom_frame_end_(uint16_t min_length) const; + uint16_t find_frame_end_by_crc_(uint16_t min_length) const; uint32_t last_modbus_byte_{0}; uint32_t last_receive_check_{0}; diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index c737e206c0..b2454e6f14 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -55,6 +55,38 @@ inline bool is_function_code_custom(uint8_t function_code) { masked_function_code <= FUNCTION_CODE_USER_DEFINED_SPACE_2_END); } +/// True for any function code whose frame length the parsers cannot predict - everything the +/// server_pdu_length()/client_pdu_length() switches fall through to `default` on (keep the case list +/// in step with those switches). Deliberately wider than is_function_code_custom(): the user-defined +/// ranges are unknown to the parser too, but so are the assigned-but-unimplemented codes +/// (READ_EXCEPTION_STATUS, DIAGNOSTICS, GET_COMM_EVENT_*, REPORT_SERVER_ID) and every unassigned value. +/// The 0x80 exception flag is masked off first, so a frame with it set classifies by its base code - +/// even though a spec exception reply has a known 2-byte PDU. That is deliberate, matching what +/// is_function_code_custom() has always done: some vendors use codes with the 0x80 bit set as ordinary +/// codes with longer payloads, so the response parser CRC-scans these rather than assuming the spec +/// length. For an intact spec exception the scan matches at its first candidate, so only a corrupt one +/// pays (recovery by timeout instead of an immediate CRC failure). +inline bool is_function_code_unknown_length(uint8_t function_code) { + switch (static_cast(function_code & FUNCTION_CODE_MASK)) { + case FunctionCode::READ_COILS: + case FunctionCode::READ_DISCRETE_INPUTS: + case FunctionCode::READ_HOLDING_REGISTERS: + case FunctionCode::READ_INPUT_REGISTERS: + case FunctionCode::WRITE_SINGLE_COIL: + case FunctionCode::WRITE_SINGLE_REGISTER: + case FunctionCode::WRITE_MULTIPLE_COILS: + case FunctionCode::WRITE_MULTIPLE_REGISTERS: + case FunctionCode::READ_FILE_RECORD: + case FunctionCode::WRITE_FILE_RECORD: + case FunctionCode::MASK_WRITE_REGISTER: + case FunctionCode::READ_WRITE_MULTIPLE_REGISTERS: + case FunctionCode::READ_FIFO_QUEUE: + return false; + default: + return true; + } +} + // Returns the expected length of a server response PDU based on the function code. // If too few bytes have arrived to determine the length, returns the minimum length. `size` is the // number of bytes available so far, which may exceed the eventual PDU (e.g. include the frame's CRC diff --git a/tests/components/modbus/common.h b/tests/components/modbus/common.h index d03ccf8ec3..e6c37b0e6d 100644 --- a/tests/components/modbus/common.h +++ b/tests/components/modbus/common.h @@ -1,7 +1,10 @@ #pragma once #include +#include +#include #include #include "esphome/components/uart/uart_component.h" +#include "esphome/core/helpers.h" namespace esphome::modbus::testing { @@ -30,4 +33,37 @@ class RecordingUART : public NullUART { std::vector written; }; +// A UART the test can inject received bytes into, so frames travel the full receive path +// (receive_modbus_frames -> parse -> dispatch) through hub.loop(). Writes are recorded. +class InjectableUART : public RecordingUART { + public: + bool peek_byte(uint8_t *data) override { + if (this->rx_.empty()) + return false; + *data = this->rx_.front(); + return true; + } + bool read_array(uint8_t *data, size_t len) override { + if (len > this->rx_.size()) + return false; + memcpy(data, this->rx_.data(), len); + this->rx_.erase(this->rx_.begin(), this->rx_.begin() + len); + return true; + } + size_t available() override { return this->rx_.size(); } + + // Queues a complete wire frame: address + PDU + CRC16 (low byte first). + void inject_frame(uint8_t address, std::span pdu) { + size_t start = this->rx_.size(); + this->rx_.push_back(address); + this->rx_.insert(this->rx_.end(), pdu.begin(), pdu.end()); + uint16_t crc = crc16(this->rx_.data() + start, this->rx_.size() - start); + this->rx_.push_back(crc & 0xFF); + this->rx_.push_back(crc >> 8); + } + + private: + std::vector rx_; +}; + } // namespace esphome::modbus::testing diff --git a/tests/components/modbus/modbus_unknown_function_test.cpp b/tests/components/modbus/modbus_unknown_function_test.cpp new file mode 100644 index 0000000000..8b91d088b8 --- /dev/null +++ b/tests/components/modbus/modbus_unknown_function_test.cpp @@ -0,0 +1,141 @@ +#include + +#include +#include +#include + +#include "common.h" +#include "esphome/components/modbus/modbus.h" + +namespace esphome::modbus::testing { + +namespace { + +// Records custom-response dispatches so tests can assert an unknown-length frame reached the device. +class CustomRecordingDevice : public ModbusClientDevice { + public: + using ModbusClientDevice::ModbusClientDevice; + void on_custom_response(std::span request_pdu, std::span response_pdu, + ResponseStatus status) override { + this->requests.emplace_back(request_pdu.begin(), request_pdu.end()); + this->responses.emplace_back(response_pdu.begin(), response_pdu.end()); + this->statuses.push_back(status); + } + std::vector> requests; + std::vector> responses; + std::vector statuses; +}; + +// Every handler keeps its ILLEGAL_FUNCTION default; the hub's dispatch is what is under test. +class SilentServerDevice : public ModbusServerDevice {}; + +// Drives full client frames through the server hub's receive path (same shape as the broadcast tests). +class TestServerHub : public ModbusServerHub { + public: + bool tx_blocked() override { return false; } + + // Builds a complete client frame (address + FC + data + CRC) and runs the full receive-side parser. + // Returns true once the buffer has fully drained. + bool run_receive_parser_for_test(uint8_t address, uint8_t function_code, std::span data) { + this->rx_buffer_.clear(); + this->rx_buffer_.reserve(data.size() + 4); + this->rx_buffer_.push_back(address); + this->rx_buffer_.push_back(function_code); + this->rx_buffer_.insert(this->rx_buffer_.end(), data.begin(), data.end()); + uint16_t crc = crc16(this->rx_buffer_.data(), this->rx_buffer_.size()); + this->rx_buffer_.push_back(crc & 0xFF); + this->rx_buffer_.push_back(crc >> 8); + this->parse_modbus_frames(); + return this->rx_buffer_.empty(); + } +}; + +} // namespace + +// The frame-length parsers have explicit cases for exactly these 13 codes; every other value - the +// assigned-but-unimplemented management codes, both user-defined ranges, and all unassigned codes - +// must classify as unknown length. The exception flag masks off first. +TEST(ModbusUnknownFunction, HelperMatchesParserCoverage) { + for (uint8_t fc : {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x0F, 0x10, 0x14, 0x15, 0x16, 0x17, 0x18}) { + EXPECT_FALSE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); + } + for (uint8_t fc : {0x07, 0x08, 0x0B, 0x0C, 0x11, 0x2A, 0x41, 0x48, 0x49, 0x64, 0x6E, 0x00, 0x7F}) { + EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); + } + // Exception replies classify by their base code. + EXPECT_FALSE(helpers::is_function_code_unknown_length(0x83)); + EXPECT_TRUE(helpers::is_function_code_unknown_length(0x87)); + // Strictly wider than the user-defined ranges: every custom code is unknown-length, but not vice versa. + for (int fc = 0; fc <= 0xFF; fc++) { + if (helpers::is_function_code_custom(fc)) + EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << fc; + } + EXPECT_FALSE(helpers::is_function_code_custom(0x49)); + + // Derived contract check: the helper must say "unknown" exactly when both length parsers fall + // through to default. With a zero-filled max-size PDU every explicit case returns at least 2 + // (file records bottom out at 2, FIFO at 3) and only default returns MIN_PDU_SIZE, so comparing + // against MIN_PDU_SIZE detects a case added to either switch without updating the helper. The + // loop stops at 0x7F: above it the helper masks the exception flag off while client_pdu_length() + // switches on the unmasked byte and server_pdu_length() early-returns the exception length. + for (int fc = 0; fc <= 0x7F; fc++) { + const uint8_t pdu[MAX_PDU_SIZE] = {static_cast(fc)}; // zero header fields + EXPECT_EQ(helpers::is_function_code_unknown_length(fc), + helpers::client_pdu_length(pdu, sizeof(pdu)) == MIN_PDU_SIZE) + << "client_pdu_length disagrees for fc 0x" << std::hex << fc; + EXPECT_EQ(helpers::is_function_code_unknown_length(fc), + helpers::server_pdu_length(pdu, sizeof(pdu)) == MIN_PDU_SIZE) + << "server_pdu_length disagrees for fc 0x" << std::hex << fc; + } +} + +// A response with a function code outside the user-defined ranges (0x49) has no length case in +// server_pdu_length(), so the parser must find the frame end by CRC scan - the same way it already +// handles user-defined codes. Frame: address + FC 0x49 + 3 data bytes + CRC = 7 bytes. Without the +// scan the parser assumes a 4-byte frame, fails the CRC, and the response never reaches the device. +TEST(ModbusUnknownFunction, ClientParsesUnknownLengthResponse) { + InjectableUART uart; + ModbusClientHub hub; + hub.set_uart_parent(&uart); + hub.setup(); // computes frame timing from the baud rate + CustomRecordingDevice device(&hub, 0x02); + + const uint8_t request[] = {0x49, 0x01}; + ASSERT_TRUE(device.queue_pdu(request)); + hub.loop(); // transmit + ASSERT_FALSE(uart.written.empty()); + + const uint8_t response_pdu[] = {0x49, 0x02, 0xAA, 0xBB}; + uart.inject_frame(0x02, response_pdu); + hub.loop(); // receive + parse + match + dispatch + + ASSERT_EQ(device.responses.size(), 1u); + EXPECT_EQ(device.requests[0], std::vector(request, request + sizeof(request))); + EXPECT_EQ(device.responses[0], std::vector(response_pdu, response_pdu + sizeof(response_pdu))); + EXPECT_FALSE(device.statuses[0].has_value()); +} + +// The server side of the same gap: a request with FC 0x49 for a registered device must parse (CRC +// scan again) so the hub can answer ILLEGAL_FUNCTION per the spec. Without the scan the frame fails +// to parse and the client gets silence instead of the exception. +TEST(ModbusUnknownFunction, ServerRepliesIllegalFunctionToUnknownLengthRequest) { + TestServerHub hub; + RecordingUART uart; + hub.set_uart_parent(&uart); + + SilentServerDevice device; + device.set_address(0x02); + hub.register_device(&device); + + const uint8_t data[] = {0x02, 0xAA, 0xBB}; + ASSERT_TRUE(hub.run_receive_parser_for_test(0x02, 0x49, data)); + + // Expected reply: address + FC with exception flag + ILLEGAL_FUNCTION + CRC. + std::vector expected = {0x02, 0xC9, 0x01}; + uint16_t crc = crc16(expected.data(), expected.size()); + expected.push_back(crc & 0xFF); + expected.push_back(crc >> 8); + EXPECT_EQ(uart.written, expected); +} + +} // namespace esphome::modbus::testing From 5c9d050ebe2ef415484e2c3dc1de61cb26f6b09a Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:02:54 +0000 Subject: [PATCH 121/160] Bump aioesphomeapi from 45.10.3 to 45.11.0 (#18493) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a986646230..3d25440671 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.10.3 +aioesphomeapi==45.11.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 804e8fb856ce5a56a1dd1216f3f3e38273d8b4df Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 15:44:38 -0500 Subject: [PATCH 122/160] [socket] Remove constant duplicated by the beta merge (#18496) --- esphome/components/socket/lwip_raw_tcp_impl.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index b20f79fba1..8d00dbede2 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -50,11 +50,6 @@ static const char *const TAG = "socket"; static constexpr uint32_t ESP8266_YIELD_INTERVAL_US = 1000; #endif -#ifdef USE_ESP8266 -// optimistic_yield() rate limit in microseconds of CONT time; cheap when hot. -static constexpr uint32_t ESP8266_YIELD_INTERVAL_US = 1000; -#endif - // set to 1 to enable verbose lwip logging #if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if) #define LWIP_LOG(msg, ...) ESP_LOGVV(TAG, "socket %p: " msg, this, ##__VA_ARGS__) From 0a88c81d95897db3504aca4e623c8fe27daa9a76 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:38:23 -0500 Subject: [PATCH 123/160] Bump aioesphomeapi from 45.11.0 to 45.12.0 (#18501) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d25440671..e4521859e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==45.11.0 +aioesphomeapi==45.12.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 7a999f9a48f89d9d6561ed5cd46853d5c64f9828 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:06:55 -0500 Subject: [PATCH 124/160] [ci] Install requirements_dev.txt when the venv cache misses (#18502) --- .github/actions/restore-python/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/restore-python/action.yml b/.github/actions/restore-python/action.yml index 6279a26dc4..ce14b0152a 100644 --- a/.github/actions/restore-python/action.yml +++ b/.github/actions/restore-python/action.yml @@ -49,7 +49,7 @@ runs: python -m venv venv source venv/bin/activate python --version - uv pip install -r requirements.txt -r requirements_test.txt + uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' && runner.os == 'Windows' @@ -58,5 +58,5 @@ runs: python -m venv venv source ./venv/Scripts/activate python --version - uv pip install -r requirements.txt -r requirements_test.txt + uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . From 4f866c563b5721a1a9ed1225b6b4fe50ef4f2637 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:36:57 -0500 Subject: [PATCH 125/160] [platformio] Give the ccache wrapper a cmd.exe safe path (#18495) --- esphome/platformio/ccache.py.script | 9 +- esphome/platformio/toolchain.py | 63 +++-- tests/unit_tests/test_platformio_toolchain.py | 240 +++++++++++++++++- 3 files changed, 286 insertions(+), 26 deletions(-) diff --git a/esphome/platformio/ccache.py.script b/esphome/platformio/ccache.py.script index cc08a8c044..22592a2398 100644 --- a/esphome/platformio/ccache.py.script +++ b/esphome/platformio/ccache.py.script @@ -1,5 +1,4 @@ import os -import shutil # pylint: disable=E0602 Import("env") # noqa @@ -9,15 +8,17 @@ Import("env") # noqa # esphome/platformio/toolchain.py); this script only supplies the SCons-level # mechanism. # +# The binary comes pre-resolved in ESPHOME_CCACHE_PATH; _ccache_env() has +# already stripped the Windows \\?\ prefix that cmd.exe cannot run. +# # This is a "pre" script, so the platform's builder (which sets CC/CXX and # clones the construction environment for framework and library builds) runs # after it. Replacing CC/CXX here would be overwritten, and replacing them in # a "post" script would miss the already-cloned library environments. Wrapping # SPAWN instead is ordering-proof: clones copy the wrapper, and every compiler # invocation from every environment funnels through it at execution time. -if ( - os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" - and (ccache_path := shutil.which("ccache")) is not None +if os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" and ( + ccache_path := os.environ.get("ESPHOME_CCACHE_PATH") ): original_spawn = env["SPAWN"] diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 08a4fcff78..d76581d032 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -60,6 +60,9 @@ def _strip_win_long_path_prefix(path: str) -> str: "The system cannot find the path specified." Stripping the prefix early keeps the path shell-quotable. + Also applied to the ccache path exported by ``_ccache_env()``, which + ``shutil.which`` can return with the same prefix. + No-op on non-Windows platforms. """ if sys.platform != "win32": @@ -235,8 +238,8 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) -def _ccache_usable() -> bool: - """Return True when the ``ccache`` on PATH actually runs. +def _ccache_runs(ccache: str) -> bool: + """Return True when the ``ccache`` found on PATH actually runs. ``shutil.which`` proves existence, not runnability: on Windows it also matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose @@ -244,9 +247,6 @@ def _ccache_usable() -> bool: step with an opaque OS error, so probe once and fall back to compiling without ccache when the probe fails. """ - ccache = shutil.which("ccache") - if ccache is None: - return False try: subprocess.run( [ccache, "--version"], @@ -265,14 +265,29 @@ def _ccache_usable() -> bool: def _ccache_env() -> dict[str, str]: - """Return ccache settings for PlatformIO builds. + r"""Return ccache settings for PlatformIO builds. Enabled by default whenever the ``ccache`` binary is on PATH; set ``ESPHOME_CCACHE_ENABLE=0`` in the environment to opt out (or ``1`` to - force it on). The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` - so platform build scripts (e.g. the esp8266 ``ccache.py`` extra script, - which wraps compiler invocations inside SCons) only have to check for - ``"1"`` instead of re-implementing the policy. + force it on without the runnability probe; a binary is still needed). + The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` and the + binary's location into ``ESPHOME_CCACHE_PATH`` so platform build scripts + (the shared ``ccache.py`` extra script, which wraps compiler invocations + inside SCons) only have to check for ``"1"`` and use the path as given + instead of re-implementing the policy. + + The path is exported rather than looked up again inside SCons because + ``shutil.which`` can return a Windows extended-length ``\\?\`` path + (ESPHome Desktop puts its bundled ccache on PATH that way). Such a path + runs fine through ``CreateProcess``, which is how ESP-IDF invokes it, + but SCons runs every compile through ``cmd.exe``, which fails on it with + "The system cannot find the path specified." (#18399), so the prefix is + stripped here with ``_strip_win_long_path_prefix()`` before the + runnability probe, which therefore validates the exact string the build + will execute. + ``ESPHOME_CCACHE_PATH`` is an internal channel, not a user setting: the + script only honours it together with ``ESPHOME_CCACHE_ENABLE=1``, and this + function always sets both or neither. The returned values are merged into the environment of the PlatformIO subprocess only, never into ``os.environ``: a long-running process @@ -293,13 +308,27 @@ def _ccache_env() -> dict[str, str]: build dir. The other ``CCACHE_*`` values the user already set in the environment are respected. """ - if "ESPHOME_CCACHE_ENABLE" in os.environ: - enabled = get_bool_env("ESPHOME_CCACHE_ENABLE") - else: - enabled = _ccache_usable() - env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"} - if not enabled: - return env + explicit = "ESPHOME_CCACHE_ENABLE" in os.environ + if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): + return {"ESPHOME_CCACHE_ENABLE": "0"} + ccache_path = shutil.which("ccache") + if ccache_path is None: + if explicit: + _LOGGER.warning( + "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " + "compiling without ccache" + ) + return {"ESPHOME_CCACHE_ENABLE": "0"} + # Strip before probing so the probe validates (and the failure warning + # names) the exact string the build will execute through cmd.exe. + ccache_path = _strip_win_long_path_prefix(ccache_path) + # An explicit opt-in skips the runnability probe. + if not explicit and not _ccache_runs(ccache_path): + return {"ESPHOME_CCACHE_ENABLE": "0"} + env = { + "ESPHOME_CCACHE_ENABLE": "1", + "ESPHOME_CCACHE_PATH": ccache_path, + } # build_path is set during preload for every config-loading command, so it # being unset means a caller built the environment too early; fail loudly # rather than with an opaque TypeError from Path(None). diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index eebb0b8cd7..172b288c25 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -2,7 +2,7 @@ # pylint: disable=protected-access -from collections.abc import Generator +from collections.abc import Callable, Generator from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json @@ -437,6 +437,7 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: env = toolchain._ccache_env() assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) assert env["CCACHE_DIR"].endswith("platformio-ccache") assert env["CCACHE_NOHASHDIR"] == "true" @@ -446,17 +447,35 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: assert "ESPHOME_CCACHE_ENABLE" not in os.environ -def test_ccache_env_disabled_without_binary(setup_core: Path) -> None: - """Ccache stays off when the binary is not on PATH.""" +@pytest.mark.parametrize( + ("env_vars", "expect_warning"), + [ + pytest.param({}, False, id="default"), + pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, True, id="forced-on"), + ], +) +def test_ccache_env_disabled_without_binary( + setup_core: Path, + caplog: pytest.LogCaptureFixture, + env_vars: dict[str, str], + expect_warning: bool, +) -> None: + """Ccache stays off when the binary is not on PATH, even when forced on. + + A deliberate opt-in that finds no binary is downgraded with a warning so + the user can tell why it had no effect; the default path stays quiet. + """ CORE.build_path = setup_core / "build" / "test" with ( - patch.dict(os.environ, {}, clear=True), + patch.dict(os.environ, env_vars, clear=True), patch.object(toolchain.shutil, "which", return_value=None), + caplog.at_level("WARNING"), ): env = toolchain._ccache_env() assert env == {"ESPHOME_CCACHE_ENABLE": "0"} + assert ("no ccache binary is on PATH" in caplog.text) is expect_warning @pytest.mark.parametrize( @@ -489,14 +508,47 @@ def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), patch.object(toolchain.subprocess, "run") as mock_probe, ): env = toolchain._ccache_env() assert env["ESPHOME_CCACHE_ENABLE"] == "1" + # The binary's location is still handed to the build script. + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" mock_probe.assert_not_called() +def test_ccache_env_strips_win_long_path_prefix(setup_core: Path) -> None: + r"""A ``\\?\`` ccache path from PATH is exported without the prefix. + + That is the shape ESPHome Desktop puts on PATH (#18399); see ``_ccache_env``. + """ + CORE.build_path = setup_core / "build" / "test" + prefixed = ( + "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder" + "\\ccache\\ccache.exe" + ) + stripped = ( + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder\\ccache\\ccache.exe" + ) + + with ( + patch.dict(os.environ, {}, clear=True), + # shutil.which is patched, so the win32 code path of the real + # implementation (which crashes on a POSIX host) is never reached. + patch("esphome.platformio.toolchain.sys.platform", "win32"), + patch.object(toolchain.shutil, "which", return_value=prefixed), + patch.object(toolchain.subprocess, "run") as mock_probe, + ): + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == stripped + # The probe validates the exact string the build will execute. + assert mock_probe.call_args[0][0] == [stripped, "--version"] + + def test_ccache_env_opt_out(setup_core: Path) -> None: """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present.""" CORE.build_path = setup_core / "build" / "test" @@ -516,7 +568,7 @@ def test_ccache_env_normalizes_enable_value(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "yes"}, clear=True), - patch.object(toolchain.shutil, "which", return_value=None), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), ): env = toolchain._ccache_env() @@ -563,8 +615,10 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only( env = mock_run_external_process.call_args[1]["env"] assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) assert "ESPHOME_CCACHE_ENABLE" not in os.environ + assert "ESPHOME_CCACHE_PATH" not in os.environ assert "CCACHE_BASEDIR" not in os.environ @@ -613,6 +667,182 @@ def test_copy_ccache_script(setup_core: Path) -> None: assert dest.read_text() == source.read_text() +class _FakeSConsEnv(dict): + """Just enough of a SCons construction environment for ccache.py.""" + + def Replace(self, **kwargs: object) -> None: # noqa: N802 + self.update(kwargs) + + +def _load_ccache_script( + env_vars: dict[str, str], original_spawn: Callable[..., int] | None = None +) -> tuple[_FakeSConsEnv, Callable[..., int]]: + """Run ccache.py.script against a fake SCons env and return (env, original SPAWN).""" + if original_spawn is None: + original_spawn = Mock(name="original_spawn", return_value=0) + scons_env = _FakeSConsEnv(SPAWN=original_spawn) + source = (Path(toolchain.__file__).parent / "ccache.py.script").read_text() + with patch.dict(os.environ, env_vars, clear=True): + exec( # noqa: S102 + compile(source, "ccache.py", "exec"), + {"Import": lambda *_names: None, "env": scons_env}, + ) + return scons_env, original_spawn + + +def _scons_win32_escape(x: str) -> str: + """Copy of ``SCons.Platform.win32.escape``: quote, guarding a trailing backslash.""" + if x[-1] == "\\": + x = x + "\\" + return '"' + x + '"' + + +def test_ccache_script_wraps_compiles_with_exported_path() -> None: + """The SCons script uses ESPHOME_CCACHE_PATH as given, without a PATH lookup.""" + ccache_path = "C:\\Users\\jesse\\ESPHome Device Builder\\ccache\\ccache.exe" + scons_env, original_spawn = _load_ccache_script( + {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": ccache_path} + ) + spawn = scons_env["SPAWN"] + assert spawn is not original_spawn + + # A compile step is routed through ccache, with the same path used for + # the program and (escaped) as the first argument. + compile_args = ["xtensa-lx106-elf-g++", "-o", "main.o", "-c", "main.cpp"] + spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", compile_args, {}) + original_spawn.assert_called_once_with( + "cmd.exe", + _scons_win32_escape, + ccache_path, + [_scons_win32_escape(ccache_path), *compile_args], + {}, + ) + + # Link steps pass through untouched. + original_spawn.reset_mock() + link_args = ["xtensa-lx106-elf-g++", "-o", "firmware.elf", "main.o"] + spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {}) + original_spawn.assert_called_once_with( + "cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {} + ) + + +@pytest.mark.parametrize( + "env_vars", + [ + pytest.param({"ESPHOME_CCACHE_ENABLE": "0"}, id="disabled"), + pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, id="enabled-without-path"), + pytest.param({}, id="unset"), + ], +) +def test_ccache_script_leaves_spawn_alone_without_path( + env_vars: dict[str, str], +) -> None: + """Without both the enable flag and a path, SPAWN is not replaced.""" + scons_env, original_spawn = _load_ccache_script(env_vars) + assert scons_env["SPAWN"] is original_spawn + + +def _scons_win32_spawn( + sh: str, escape: Callable[[str], str], cmd: str, args: list[str], env: dict +) -> int: + r"""Mirror of ``SCons.Platform.win32.spawn``: every command runs via ``cmd.exe /C``. + + SCons is not importable in the test environment (PlatformIO fetches it at + build time), so the lines that matter are mirrored here. The command line + SCons hands ``os.spawnve`` goes to ``CreateProcess`` via ``subprocess`` + instead (identical on Windows, where a string passes through untouched); + ``spawnve`` itself crashes inside pytest. + """ + return subprocess.run( + " ".join([sh, "/C", escape(" ".join(args))]), env=env, check=False + ).returncode + + +_MARKER_ENV = "ESPHOME_TEST_CCACHE_MARKER" +# Stands in for a compile: the "ccache" is really the Python interpreter, and +# the compile "flags" make it write a marker file so the test can tell whether +# the wrapped command actually ran to completion. +_FAKE_COMPILE_ARGS = [ + "-c", + f"import os, pathlib; pathlib.Path(os.environ['{_MARKER_ENV}']).write_text('compiled')", +] + + +def _spawn_fake_compile_via_cmd_exe(scons_env: _FakeSConsEnv, marker: Path) -> int: + """Run one wrapped compile step the way SCons does on Windows.""" + child_env = {**os.environ, _MARKER_ENV: str(marker)} + return scons_env["SPAWN"]( + os.environ.get("COMSPEC", "cmd.exe"), + _scons_win32_escape, + "xtensa-lx106-elf-gcc", + [_scons_win32_escape(arg) if " " in arg else arg for arg in _FAKE_COMPILE_ARGS], + child_env, + ) + + +_WINDOWS_ONLY = pytest.mark.skipif( + sys.platform != "win32", reason="drives cmd.exe, which SCons uses only on Windows" +) + + +@_WINDOWS_ONLY +def test_ccache_env_real_probe_runs_stripped_path(setup_core: Path) -> None: + r"""With a ``\\?\`` which result, the real probe runs the stripped binary. + + The probe therefore validates the exact string the build will execute + through ``cmd.exe``; probing the verbatim path instead would pass even + when the stripped path is unusable (``CreateProcess`` accepts + extended-length paths, ``cmd.exe`` does not). + """ + CORE.build_path = setup_core / "build" / "test" + assert not sys.executable.startswith("\\\\?\\") + + with ( + patch.dict(os.environ, {}, clear=False), + patch.object( + toolchain.shutil, "which", return_value="\\\\?\\" + sys.executable + ), + ): + os.environ.pop("ESPHOME_CCACHE_ENABLE", None) + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == sys.executable + + +@_WINDOWS_ONLY +@pytest.mark.parametrize( + ("prefix", "expect_ok"), + [ + pytest.param("", True, id="stripped-path-compiles"), + pytest.param("\\\\?\\", False, id="verbatim-path-fails"), + ], +) +def test_ccache_wrapper_through_cmd_exe( + tmp_path: Path, prefix: str, expect_ok: bool +) -> None: + r"""End to end through ``cmd.exe``: the exported path works, a ``\\?\`` one does not. + + The interpreter stands in for ccache; the spawn mirrors SCons on Windows. + The failing case is the mechanism behind #18399 ("The system cannot find + the path specified." on every compile step); should it ever start passing, + ``cmd.exe`` learned extended-length paths and the strip is no longer needed. + """ + marker = tmp_path / "compiled.txt" + scons_env, _ = _load_ccache_script( + {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": prefix + sys.executable}, + original_spawn=_scons_win32_spawn, + ) + assert scons_env["SPAWN"] is not _scons_win32_spawn + + rc = _spawn_fake_compile_via_cmd_exe(scons_env, marker) + assert (rc == 0) is expect_ok + assert marker.exists() is expect_ok + if expect_ok: + assert marker.read_text() == "compiled" + + @pytest.mark.parametrize( ("platform", "input_path", "expected"), [ From 8b637b339b109ceaab298dad6f748a4671afd420 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:37:10 -0500 Subject: [PATCH 126/160] [vscode] Report the origin of an unexpected exception during validation (#18494) --- esphome/vscode.py | 20 +++++++++- tests/unit_tests/test_vscode.py | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/esphome/vscode.py b/esphome/vscode.py index f404f02f00..ba7b4e727b 100644 --- a/esphome/vscode.py +++ b/esphome/vscode.py @@ -3,12 +3,14 @@ from __future__ import annotations from io import StringIO import json from pathlib import Path +import sys +import traceback from typing import Any from esphome.config import Config, _format_vol_invalid, validate_config import esphome.config_validation as cv from esphome.const import __version__ as ESPHOME_VERSION -from esphome.core import CORE, DocumentRange +from esphome.core import CORE, DocumentRange, EsphomeError from esphome.yaml_util import parse_yaml @@ -97,6 +99,16 @@ def _ace_loader(fname: Path) -> dict[str, Any]: return parse_yaml(fname, raw_yaml_stream) +def _format_unexpected_error(err: Exception) -> str: + """Describe a crash inside validation with the frame it came from.""" + message = f"Unexpected error while validating: {type(err).__name__}: {err}" + frames = traceback.extract_tb(err.__traceback__) + if not frames: + return message + frame = frames[-1] + return f"{message} ({frame.filename}:{frame.lineno} in {frame.name})" + + def _print_version(): """Print ESPHome version.""" print( @@ -134,8 +146,12 @@ def read_config(args): try: config = loader(file_name) res = validate_config(config, command_line_substitutions) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + except (EsphomeError, cv.Invalid) as err: vs.add_yaml_error(str(err)) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + # stdout carries the JSON protocol; the full chain goes to stderr. + traceback.print_exc(file=sys.stderr) + vs.add_yaml_error(_format_unexpected_error(err)) else: for err in res.errors: try: diff --git a/tests/unit_tests/test_vscode.py b/tests/unit_tests/test_vscode.py index 63bdf3e255..9b7d1e9504 100644 --- a/tests/unit_tests/test_vscode.py +++ b/tests/unit_tests/test_vscode.py @@ -3,6 +3,8 @@ from pathlib import Path from unittest.mock import Mock, patch from esphome import vscode +import esphome.config_validation as cv +from esphome.core import EsphomeError def _run_repl_test(input_data): @@ -126,3 +128,67 @@ packages: assert range["start_col"] == 2 assert range["end_line"] == 1 assert range["end_col"] == 7 + + +def _explode(*_args: object, **_kwargs: object) -> None: + raise AttributeError("'NoneType' object has no attribute 'get'") + + +def test_unexpected_error_reports_origin() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", _explode): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["validation_errors"] == [] + (error,) = result["yaml_errors"] + assert error["message"].startswith( + "Unexpected error while validating: AttributeError: " + "'NoneType' object has no attribute 'get' (" + ) + assert "test_vscode.py" in error["message"] + assert error["message"].endswith(" in _explode)") + + +def test_esphome_error_stays_plain() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", side_effect=EsphomeError("boom")): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["yaml_errors"] == [{"message": "boom"}] + + +def test_invalid_stays_plain() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", side_effect=cv.Invalid("bad value")): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["yaml_errors"] == [{"message": "bad value"}] + + +def test_format_unexpected_error_without_traceback() -> None: + message = vscode._format_unexpected_error(ValueError("boom")) + assert message == "Unexpected error while validating: ValueError: boom" From 8aa7db15e52e7842edb4e0ce634c58028b20f4fa Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:25:09 -0400 Subject: [PATCH 127/160] [esp32] Fix ESP32-P4 bootloop on rev3 (v3.x) chips when only variant is set (#18500) --- esphome/components/esp32/__init__.py | 78 +++++++++++++++------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 7d43c3ac07..3065cdadad 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1073,6 +1073,26 @@ def _parse_pio_platform_version(value): return value +def _normalize_p4_engineering_sample(value: ConfigType) -> bool: + """Fill in CONF_ENGINEERING_SAMPLE when unset, warning that production + silicon (rev3) is assumed. Returns the normalized flag.""" + if (engineering_sample := value.get(CONF_ENGINEERING_SAMPLE)) is None: + _LOGGER.warning( + "Defaulting to ESP32-P4 production silicon (rev3).\n" + "If you have an early engineering sample (pre-rev3), add this to your config:\n" + "\n" + " esp32:\n" + " engineering_sample: true\n" + "\n" + "To check your chip revision, look for 'chip revision: vX.Y' in the boot log.\n" + "Engineering samples will show a revision below v3.0.\n" + "The 'debug:' component also reports the revision (e.g. Revision: 100 = v1.0, 300 = v3.0)." + ) + engineering_sample = False + value[CONF_ENGINEERING_SAMPLE] = engineering_sample + return engineering_sample + + def _detect_variant(value): board = value.get(CONF_BOARD) variant = value.get(CONF_VARIANT) @@ -1085,6 +1105,8 @@ def _detect_variant(value): # name rather than carrying a PIO board name through the IDF build. if CORE.using_toolchain_esp_idf: value = value.copy() + if variant == VARIANT_ESP32P4: + _normalize_p4_engineering_sample(value) value[CONF_BOARD] = VARIANT_FRIENDLY[variant].lower() return value if variant not in STANDARD_BOARDS: @@ -1095,22 +1117,8 @@ def _detect_variant(value): ) value = value.copy() value[CONF_BOARD] = STANDARD_BOARDS[variant] - if variant == VARIANT_ESP32P4: - engineering_sample = value.get(CONF_ENGINEERING_SAMPLE) - if engineering_sample is None: - _LOGGER.warning( - "No board specified for ESP32-P4. Defaulting to production silicon (rev3).\n" - "If you have an early engineering sample (pre-rev3), add this to your config:\n" - "\n" - " esp32:\n" - " engineering_sample: true\n" - "\n" - "To check your chip revision, look for 'chip revision: vX.Y' in the boot log.\n" - "Engineering samples will show a revision below v3.0.\n" - "The 'debug:' component also reports the revision (e.g. Revision: 100 = v1.0, 300 = v3.0)." - ) - elif engineering_sample: - value[CONF_BOARD] = "esp32-p4-evboard" + if variant == VARIANT_ESP32P4 and _normalize_p4_engineering_sample(value): + value[CONF_BOARD] = "esp32-p4-evboard" elif board in BOARDS: variant = variant or BOARDS[board][KEY_VARIANT] if variant != BOARDS[board][KEY_VARIANT]: @@ -1120,6 +1128,14 @@ def _detect_variant(value): ) value = value.copy() value[CONF_VARIANT] = variant + if variant == VARIANT_ESP32P4: + board_is_es = BOARDS[board].get("engineering_sample", False) + engineering_sample = value.setdefault(CONF_ENGINEERING_SAMPLE, board_is_es) + if engineering_sample != board_is_es: + raise cv.Invalid( + f"'{CONF_ENGINEERING_SAMPLE}' does not match board '{board}'", + path=[CONF_ENGINEERING_SAMPLE], + ) elif not variant: raise cv.Invalid( "This board is unknown, if you are sure you want to compile with this board selection, " @@ -1131,6 +1147,9 @@ def _detect_variant(value): "This board is unknown; the specified variant '%s' will be used but this may not work as expected.", variant, ) + if variant == VARIANT_ESP32P4: + value = value.copy() + _normalize_p4_engineering_sample(value) return value @@ -1434,20 +1453,6 @@ def final_validate(config) -> None: path=[CONF_ENGINEERING_SAMPLE], ) ) - if ( - config[CONF_VARIANT] == VARIANT_ESP32P4 - and config.get(CONF_ENGINEERING_SAMPLE) is not None - ): - board_is_es = BOARDS.get(config[CONF_BOARD], {}).get( - "engineering_sample", False - ) - if config[CONF_ENGINEERING_SAMPLE] != board_is_es: - errs.append( - cv.Invalid( - f"'{CONF_ENGINEERING_SAMPLE}' does not match board '{config[CONF_BOARD]}'", - path=[CONF_ENGINEERING_SAMPLE], - ) - ) if advanced[CONF_EXECUTE_FROM_PSRAM]: if config[CONF_VARIANT] not in {VARIANT_ESP32S3, VARIANT_ESP32P4}: errs.append( @@ -2518,15 +2523,14 @@ async def to_code(config): f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_frequency[:-3]}M", True ) - # ESP32-P4: ESP-IDF 5.5.3 changed the default of ESP32P4_SELECTS_REV_LESS_V3 - # from y to n. PlatformIO uses sections.ld.in (for rev <3) or - # sections.rev3.ld.in (for rev >=3) based on board definition. - # Set the sdkconfig option to match the board's chip revision. + # ESP32-P4: pre-v3 and rev3 (v3.0+) silicon are not binary compatible. + # CONFIG_ESP32P4_SELECTS_REV_LESS_V3 selects which layout ESP-IDF links; + # validation normalizes CONF_ENGINEERING_SAMPLE from the board when unset. if variant == VARIANT_ESP32P4: - is_eng_sample = BOARDS.get(config[CONF_BOARD], {}).get( - "engineering_sample", False + add_idf_sdkconfig_option( + "CONFIG_ESP32P4_SELECTS_REV_LESS_V3", + config.get(CONF_ENGINEERING_SAMPLE, False), ) - add_idf_sdkconfig_option("CONFIG_ESP32P4_SELECTS_REV_LESS_V3", is_eng_sample) # Set minimum chip revision for ESP32 variant # Setting this to 3.0 or higher reduces flash size by excluding workaround code, From 07fa16e2e74f9964da91147028b630a7436b5d0e Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:38:18 -0400 Subject: [PATCH 128/160] [ci] Stop persisting the integration test ccache (#18504) --- .github/workflows/ci.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c81c783b5..c3f830a5aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -341,18 +341,6 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -y --no-install-recommends ccache - - name: Restore ccache (restore-only) - # esphome stores the PlatformIO ccache under the machine-global cache - # dir (see _ccache_env() in esphome/platformio/toolchain.py). The - # bucket-name prefix prefers a same-bucket seed; the bare prefix falls - # back to any seed when the bucket layout differs from dev. - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/esphome/platformio-ccache - key: integration-ccache-${{ matrix.bucket.name }}-${{ github.sha }} - restore-keys: | - integration-ccache-${{ matrix.bucket.name }}- - integration-ccache- - name: Set up Python 3.13 id: python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -401,14 +389,6 @@ jobs: # esphome stores the PlatformIO ccache under the machine-global cache # dir (see _ccache_env() in esphome/platformio/toolchain.py). run: CCACHE_DIR="$HOME/.cache/esphome/platformio-ccache" ccache -s - - name: Save ccache - # Pull request saves land in per-PR scopes nothing else can reuse; - # dev pushes seed the shared copy instead. - if: github.event_name != 'pull_request' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/esphome/platformio-ccache - key: integration-ccache-${{ matrix.bucket.name }}-${{ github.sha }} import-time: name: Check import esphome.__main__ time From b7121940c85ca166fd344d5c6b3c37f49e228a24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:08:23 -0400 Subject: [PATCH 129/160] Update wheel requirement from <0.48,>=0.43 to >=0.43,<0.49 (#18459) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index afa6208cae..3185fe0a9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools==84.0.0", "wheel>=0.43,<0.48"] +requires = ["setuptools==84.0.0", "wheel>=0.43,<0.49"] build-backend = "setuptools.build_meta" [project] From 17eed7055bf516797478e3c12724758e05e8f94c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:22:51 -0400 Subject: [PATCH 130/160] Bump resvg-py from 0.3.4 to 0.4.0 (#18460) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e4521859e7..740a8c1a79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import esphome-glyphsets==0.2.0 pillow==12.3.0 -resvg-py==0.3.4 +resvg-py==0.4.0 freetype-py==2.5.1 jinja2==3.1.6 bleak==3.0.2 From b7cc271219467909b28f81f244710bc06b81f79f Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:57:51 -0500 Subject: [PATCH 131/160] Bump bundled esphome-device-builder to 1.11.3 (#18505) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4a8daeaaf6..50b698224c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.3 RUN \ platformio settings set enable_telemetry No \ From d2c3f749abb87fdc4f7740ef5f05b4d33772de77 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:24:23 -0500 Subject: [PATCH 132/160] Bump bundled esphome-device-builder to 1.11.4 (#18506) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 50b698224c..5c21e07618 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.4 RUN \ platformio settings set enable_telemetry No \ From e26237e57d66ea9d8e064323bd9d12a83790499f Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:42:05 -0500 Subject: [PATCH 133/160] Bump bundled esphome-device-builder to 1.11.5 (#18507) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5c21e07618..1be10db3af 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.5 RUN \ platformio settings set enable_telemetry No \ From f90b7760714a96a396bcda158bd7465b65888b69 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 11:39:01 -0500 Subject: [PATCH 134/160] [ci] Key PlatformIO cache on the Python version so a runner image bump does not serve a broken LibreTiny venv (#18512) --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3f830a5aa..6afb8a9d22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -545,24 +545,29 @@ jobs: fetch-depth: 2 - name: Restore Python + id: restore-python uses: ./.github/actions/restore-python with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + # Key on the exact Python version as well: LibreTiny creates a venv under + # ~/.platformio/penv whose interpreter is a symlink into the runner's + # hosted toolcache, so a cache saved on an older runner image breaks once + # a new image ships a newer patch release and drops the old interpreter. - name: Cache platformio if: github.ref == 'refs/heads/dev' && matrix.pio_cache_key uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.platformio - key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} + key: platformio-${{ matrix.pio_cache_key }}-${{ steps.restore-python.outputs.python-version }}-${{ hashFiles('platformio.ini') }} - name: Cache platformio if: github.ref != 'refs/heads/dev' && matrix.pio_cache_key uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.platformio - key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} + key: platformio-${{ matrix.pio_cache_key }}-${{ steps.restore-python.outputs.python-version }}-${{ hashFiles('platformio.ini') }} - name: Cache ESP-IDF install if: matrix.cache_idf From 7b7107556f4637c157a6bbdbae5bbd80cbd5f3ee Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:21:58 -0500 Subject: [PATCH 135/160] Bump bundled esphome-device-builder to 1.12.0 (#18514) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1be10db3af..18f705b501 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.12.0 RUN \ platformio settings set enable_telemetry No \ From 470226ca03dfee7b8b6d08589a15a237bba12499 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:30:12 -0700 Subject: [PATCH 136/160] [image] Restore defaults:/files: support for platform entries (#18032) Co-authored-by: Claude Sonnet 5 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/components/file/image.py | 3 +- esphome/components/image/__init__.py | 152 +++++++- esphome/components/runtime_image/__init__.py | 5 +- esphome/config.py | 17 + esphome/loader.py | 8 + tests/component_tests/image/test_init.py | 328 +++++++++++++++++- .../validate-platform-defaults.host.yaml | 21 ++ .../validate-platform-defaults.host.yaml | 24 ++ tests/unit_tests/test_config_normalization.py | 124 ++++++- 9 files changed, 657 insertions(+), 25 deletions(-) create mode 100644 tests/components/animation/validate-platform-defaults.host.yaml create mode 100644 tests/components/image/validate-platform-defaults.host.yaml diff --git a/esphome/components/file/image.py b/esphome/components/file/image.py index d340d21490..feced063d0 100644 --- a/esphome/components/file/image.py +++ b/esphome/components/file/image.py @@ -23,6 +23,7 @@ from esphome.components.image import ( get_image_type_enum, get_transparency_enum, is_svg_file, + validate_byte_order, validate_settings, validate_transparency, validate_type, @@ -200,7 +201,7 @@ OPTIONS_SCHEMA = { "NONE", "FLOYDSTEINBERG", upper=True ), cv.Optional(CONF_INVERT_ALPHA, default=False): cv.boolean, - cv.Optional(CONF_BYTE_ORDER): cv.one_of("BIG_ENDIAN", "LITTLE_ENDIAN", upper=True), + cv.Optional(CONF_BYTE_ORDER): validate_byte_order, cv.Optional(CONF_TRANSPARENCY, default=CONF_OPAQUE): validate_transparency(), } diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 37a9afb84d..eaee31a1c7 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -10,7 +10,14 @@ from PIL import Image, UnidentifiedImageError import esphome.codegen as cg from esphome.components.const import CONF_BYTE_ORDER, KEY_METADATA import esphome.config_validation as cv -from esphome.const import CONF_DEFAULTS, CONF_FILE, CONF_ID, CONF_PLATFORM, CONF_TYPE +from esphome.const import ( + CONF_DEFAULTS, + CONF_FILE, + CONF_FILES, + CONF_ID, + CONF_PLATFORM, + CONF_TYPE, +) from esphome.core import CORE from esphome.types import ConfigType @@ -48,6 +55,9 @@ TRANSPARENCY_TYPES = ( CONF_ALPHA_CHANNEL, ) +# Shared validator for the image platform schemas and `_drop_incompatible_byte_order`. +validate_byte_order = cv.one_of("BIG_ENDIAN", "LITTLE_ENDIAN", upper=True) + def get_image_type_enum(type): return getattr(ImageType, f"IMAGE_TYPE_{type.upper()}") @@ -404,6 +414,120 @@ def get_image_metadata(image_id: str) -> ImageMetaData | None: return get_all_image_metadata().get(image_id) +# --------------------------------------------------------------------------- +# `defaults:`/`files:` expansion: a `platform:` entry merges shared `defaults:` +# into every `files:` entry; the platform's CONFIG_SCHEMA validates each. +# Permanent, unlike the legacy migration below. +# --------------------------------------------------------------------------- + + +def _drop_incompatible_byte_order( + merged: dict, explicit: dict, *, index: int | None = None +) -> dict: + """Drop `byte_order` when the resolved type doesn't support it, unless written directly on `explicit`. + + With `index`, inherited values are validated before being dropped (the legacy flattener always drops). + """ + if CONF_BYTE_ORDER in explicit: + return merged + type_class = IMAGE_TYPE.get(str(merged.get(CONF_TYPE, "")).upper()) + if ( + CONF_BYTE_ORDER in merged + and isinstance(type_class, type) + and issubclass(type_class, ImageEncoder) + and not type_class.is_endian() + ): + if index is not None: + try: + validate_byte_order(merged[CONF_BYTE_ORDER]) + except cv.Invalid as exc: + exc.prepend([index]) + raise + del merged[CONF_BYTE_ORDER] + return merged + + +def _expand_platform_entry(index: int, entry: dict) -> list[dict]: + if CONF_FILES not in entry: + if CONF_DEFAULTS in entry: + raise cv.Invalid( + f"'{CONF_DEFAULTS}' may only be used together with '{CONF_FILES}'", + path=[index], + ) + return [entry] + + extra_keys = set(entry) - {CONF_PLATFORM, CONF_DEFAULTS, CONF_FILES} + if extra_keys: + raise cv.Invalid( + f"'{CONF_FILES}' cannot be combined with " + f"{', '.join(sorted(extra_keys))} on the same entry", + path=[index], + ) + + files = entry[CONF_FILES] + if files is None: + raise cv.Invalid(f"'{CONF_FILES}' must not be empty", path=[index]) + if not isinstance(files, list): + raise cv.Invalid(f"'{CONF_FILES}' must be a list", path=[index]) + if not files: + raise cv.Invalid(f"'{CONF_FILES}' must not be empty", path=[index]) + + defaults = entry.get(CONF_DEFAULTS, {}) + if defaults is None: + defaults = {} + if not isinstance(defaults, dict): + raise cv.Invalid(f"'{CONF_DEFAULTS}' must be a mapping", path=[index]) + # Neither `id:` nor `platform:` makes sense inside `defaults:`. + for disallowed in (CONF_ID, CONF_PLATFORM): + if disallowed in defaults: + raise cv.Invalid( + f"'{disallowed}' is not allowed inside '{CONF_DEFAULTS}'", + path=[index], + ) + + from esphome import yaml_util + + platform = entry[CONF_PLATFORM] + result: list[dict] = [] + for file_entry in files: + if not isinstance(file_entry, dict): + raise cv.Invalid( + f"each entry in '{CONF_FILES}' must be a mapping", path=[index] + ) + # The platform is chosen by the entry's own `platform:` key, not per file. + if CONF_PLATFORM in file_entry: + raise cv.Invalid( + f"'{CONF_PLATFORM}' is not allowed inside '{CONF_FILES}'", + path=[index], + ) + # Keep the `files:` item's source range so whole-entry errors anchor there; + # `make_data_base` needs a real ESPHomeDataBase, so skip it for plain dicts. + source = ( + file_entry if isinstance(file_entry, yaml_util.ESPHomeDataBase) else None + ) + merged = yaml_util.make_data_base( + {CONF_PLATFORM: platform, **defaults, **file_entry}, source + ) + result.append(_drop_incompatible_byte_order(merged, file_entry, index=index)) + return result + + +def expand_platform_config(config: list) -> list: + """Expand `defaults:`/`files:` entries; the platform's own CONFIG_SCHEMA validates each result.""" + result = [] + for i, entry in enumerate(config): + if isinstance(entry, dict) and CONF_PLATFORM in entry: + result.extend(_expand_platform_entry(i, entry)) + else: + result.append(entry) + return result + + +EXPAND_PLATFORM_CONFIG = expand_platform_config + +# --------------------- end defaults/files expansion ------------------------- + + # --------------------------------------------------------------------------- # Legacy top-level component -> `image:` platform deprecation helpers # -- REMOVE after 2027.1.0 together with the `animation:`/`online_image:` shims. @@ -496,11 +620,17 @@ def _is_legacy_image_format(config: object) -> bool: proper error instead of the migration silently dropping the input. """ if isinstance(config, list): - # A bare list of (not-yet-platform-tagged) image dicts. + # Exclude `files:` entries -- the list branch would otherwise silently + # migrate them to `platform: file` instead of raising the missing-platform error. return bool(config) and all( - isinstance(entry, dict) and CONF_PLATFORM not in entry for entry in config + isinstance(entry, dict) + and CONF_PLATFORM not in entry + and CONF_FILES not in entry + for entry in config ) - if not isinstance(config, dict): + if not isinstance(config, dict) or CONF_PLATFORM in config or CONF_FILES in config: + # `platform:`/`files:` dicts are new-format (left for list-wrapping + + # expansion); the legacy flattener has no `files:` branch and would drop them. return False # A single image dict, or the grouped `defaults:`/`images:`/type-key form. return ( @@ -532,18 +662,8 @@ def _flatten_legacy_image_config(config: object) -> list[dict]: def _add(entry: dict, extra: dict) -> None: merged = {**defaults, **extra, **entry} - # The legacy `defaults:`/type-grouped forms only applied `byte_order` to - # types that support it. Replicate that so an endian default merged into - # e.g. a binary image stays valid. - type_class = IMAGE_TYPE.get(str(merged.get(CONF_TYPE, "")).upper()) - if ( - CONF_BYTE_ORDER in merged - and isinstance(type_class, type) - and issubclass(type_class, ImageEncoder) - and not type_class.is_endian() - ): - del merged[CONF_BYTE_ORDER] - result.append(merged) + # Always drop, matching the pre-platform behavior -- see `_drop_incompatible_byte_order`. + result.append(_drop_incompatible_byte_order(merged, {})) def _add_entries(entries: object, extra: dict) -> None: # `entries` may be a single image dict or a list of them; non-dict diff --git a/esphome/components/runtime_image/__init__.py b/esphome/components/runtime_image/__init__.py index d8517d4493..9fa32a5a65 100644 --- a/esphome/components/runtime_image/__init__.py +++ b/esphome/components/runtime_image/__init__.py @@ -5,6 +5,7 @@ from esphome.components.const import CONF_BYTE_ORDER from esphome.components.image import ( IMAGE_TYPE, Image_, + validate_byte_order, validate_settings, validate_transparency, validate_type, @@ -128,9 +129,7 @@ def runtime_image_schema(image_class: cg.MockObjClass = RuntimeImage) -> cv.Sche cv.Required(CONF_FORMAT): cv.one_of(*IMAGE_FORMATS, upper=True), cv.Optional(CONF_RESIZE): cv.dimensions, cv.Required(CONF_TYPE): validate_type(IMAGE_TYPE), - cv.Optional(CONF_BYTE_ORDER): cv.one_of( - "BIG_ENDIAN", "LITTLE_ENDIAN", upper=True - ), + cv.Optional(CONF_BYTE_ORDER): validate_byte_order, cv.Optional(CONF_TRANSPARENCY, default="OPAQUE"): validate_transparency(), cv.Optional(CONF_PLACEHOLDER): cv.use_id(Image_), } diff --git a/esphome/config.py b/esphome/config.py index 987bb9c96a..13ec744ce4 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -620,6 +620,23 @@ class LoadValidationStep(ConfigValidationStep): elif not isinstance(self.conf, list): result[self.domain] = self.conf = [self.conf] + # Permanent expansion hook: a platform-tagged entry may expand into + # several (e.g. `image`'s `defaults:`/`files:`), for `platform:`-tagged dicts only. + if (expand := component.expand_platform_config) is not None and all( + isinstance(entry, dict) and CONF_PLATFORM in entry + for entry in self.conf + ): + with result.catch_error(path): + expanded = expand(self.conf) + if not isinstance(expanded, list): + # A non-list return is a component bug (not a user error): + # raise explicitly (survives -O/-OO) so it escapes catch_error. + raise TypeError( + f"{self.domain}: EXPAND_PLATFORM_CONFIG must " + f"return a list, got {type(expanded).__name__}" + ) + result[self.domain] = self.conf = expanded + # Process AUTO_LOAD _process_auto_load(result, component, path) diff --git a/esphome/loader.py b/esphome/loader.py index f994f0c5eb..23c6d1bfa5 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -164,6 +164,14 @@ class ComponentManifest: """ return getattr(self.module, "LEGACY_CONFIG_MIGRATE", None) + @property + def expand_platform_config( + self, + ) -> Callable[[list[ConfigType]], list[ConfigType]] | None: + """Optional `EXPAND_PLATFORM_CONFIG` callable; runs on the normalized `platform:`-tagged + entry list before per-entry CONFIG_SCHEMA. Must return a list (raise `cv.Invalid` for user errors).""" + return getattr(self.module, "EXPAND_PLATFORM_CONFIG", None) + @property def resources(self) -> list[FileResource]: """Return a list of all file resources defined in the package of this component. diff --git a/tests/component_tests/image/test_init.py b/tests/component_tests/image/test_init.py index f52c477c85..fad8b7df09 100644 --- a/tests/component_tests/image/test_init.py +++ b/tests/component_tests/image/test_init.py @@ -21,16 +21,20 @@ from esphome.components.image import ( CONF_OPAQUE, CONF_TRANSPARENCY, PLATFORM_FILE, + _expand_platform_entry, _flatten_legacy_image_config, _is_legacy_image_format, _is_new_image_format, _migrate_legacy_image_config, + expand_platform_config, get_all_image_metadata, get_image_metadata, ) from esphome.const import ( + CONF_DEFAULTS, CONF_DITHER, CONF_FILE, + CONF_FILES, CONF_ID, CONF_PLATFORM, CONF_RAW_DATA_ID, @@ -259,6 +263,15 @@ def test_flatten_keeps_byte_order_for_endian_type() -> None: assert out[0][CONF_BYTE_ORDER] == "little_endian" +def test_flatten_drops_byte_order_written_directly_on_legacy_entry() -> None: + """The legacy flattener drops an incompatible byte_order even when written directly on the entry.""" + out = _flatten_legacy_image_config( + {"binary": [{"id": "a", "file": "x.png", "byte_order": "little_endian"}]} + ) + assert out == [{"id": "a", "file": "x.png", "type": "binary"}] + assert CONF_BYTE_ORDER not in out[0] + + def test_flatten_skips_meta_and_unknown_keys() -> None: out = _flatten_legacy_image_config( { @@ -342,6 +355,42 @@ def test_migrate_legacy_warns_and_prepends_platform( ), pytest.param({"foo": 1}, False, id="dict_unknown_keys"), pytest.param("a string", False, id="scalar"), + # A `platform:`-tagged dict is the new format written without list brackets. + pytest.param( + {CONF_PLATFORM: "file", "id": "a", "file": "x.png"}, + False, + id="platform_tagged_flat_dict", + ), + pytest.param( + { + CONF_PLATFORM: "file", + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + }, + False, + id="platform_tagged_defaults_files_dict", + ), + # `files:` without `platform:` is not legacy either -- the flattener has no branch for it. + pytest.param( + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + }, + False, + id="defaults_files_dict_without_platform", + ), + # Same as above in a list -- without this exclusion it would be silently + # migrated to a hard-coded `platform: file` instead of raising the error. + pytest.param( + [ + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + } + ], + False, + id="defaults_files_list_entry_without_platform", + ), ], ) def test_is_legacy_image_format(config: object, expected: bool) -> None: @@ -359,17 +408,290 @@ def test_is_legacy_image_format(config: object, expected: bool) -> None: def test_migrate_returns_none_for_invalid_legacy_shapes( config: object, caplog: pytest.LogCaptureFixture ) -> None: - """Unrecognised shapes are not migrated (and emit no warning) so normal - platform validation surfaces a proper error instead of silently dropping - the offending input.""" + """Unrecognised shapes are not migrated (and emit no warning), so normal platform validation reports them.""" with caplog.at_level(logging.WARNING): assert _migrate_legacy_image_config(config) is None assert "deprecated" not in caplog.text +def test_migrate_returns_none_for_mapping_form_defaults_files() -> None: + """A `platform:`-tagged `defaults:`/`files:` mapping must not be swallowed by the legacy migrator.""" + config = { + CONF_PLATFORM: "file", + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + assert _migrate_legacy_image_config(config) is None + + +def test_migrate_returns_none_for_defaults_files_dict_without_platform() -> None: + """`defaults:`/`files:` without `platform:` must not be swallowed either -- the flattener has + no `files:` branch and would silently return `[]`.""" + config = { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + assert _migrate_legacy_image_config(config) is None + + +def test_migrate_returns_none_for_defaults_files_list_entry_without_platform() -> None: + """Same, in a list -- previously the list branch migrated it to a hard-coded + `platform: file` instead of raising a missing-platform error.""" + config = [ + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + ] + assert _migrate_legacy_image_config(config) is None + + # --------------------------- end legacy migration -------------------------- +def test_expand_platform_entry_passes_through_plain_entry() -> None: + entry = {CONF_PLATFORM: "file", "id": "a", "file": "x.png"} + assert _expand_platform_entry(0, entry) == [entry] + + +def test_expand_platform_entry_expands_files_with_defaults() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565", "transparency": "opaque"}, + CONF_FILES: [ + {"id": "img1", "file": "foo.png"}, + {"id": "img2", "file": "bar.png", "type": "GRAYSCALE"}, + ], + } + assert _expand_platform_entry(0, entry) == [ + { + CONF_PLATFORM: "file", + "id": "img1", + "file": "foo.png", + "type": "RGB565", + "transparency": "opaque", + }, + { + CONF_PLATFORM: "file", + "id": "img2", + "file": "bar.png", + "type": "GRAYSCALE", + "transparency": "opaque", + }, + ] + + +def test_expand_platform_entry_files_without_defaults() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "img1", "file": "foo.png"}], + } + assert _expand_platform_entry(0, entry) == [ + {CONF_PLATFORM: "file", "id": "img1", "file": "foo.png"} + ] + + +def test_expand_platform_entry_preserves_source_range() -> None: + """A merged entry keeps the source range of its `files:` item so whole-entry errors anchor there.""" + from esphome import yaml_util + + file_entry = yaml_util.make_data_base({"id": "img1", "file": "foo.png"}) + file_entry._esp_range = "sentinel-range" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [file_entry], + } + [out] = _expand_platform_entry(0, entry) + assert isinstance(out, yaml_util.ESPHomeDataBase) + assert out.esp_range == "sentinel-range" + + +def test_expand_platform_entry_plain_dict_file_entry_has_no_source_range() -> None: + """Plain-dict `files:` items must not crash -- `from_database` reads `.esp_range` unconditionally.""" + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "img1", "file": "foo.png"}], + } + [out] = _expand_platform_entry(0, entry) + assert out == {CONF_PLATFORM: "file", "id": "img1", "file": "foo.png"} + + +def test_expand_platform_entry_per_file_overrides_win() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [{"id": "img1", "file": "foo.png", "type": "BINARY"}], + } + [out] = _expand_platform_entry(0, entry) + assert out["type"] == "BINARY" + + +def test_expand_platform_entry_drops_byte_order_for_non_endian_override() -> None: + """A `byte_order` default merged into a non-endian override is dropped, as the legacy flattener did.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "little_endian"}, + CONF_FILES: [ + {"id": "a", "file": "x.png"}, + {"id": "b", "file": "y.png", "type": "binary"}, + ], + } + out = _expand_platform_entry(0, entry) + assert out[0]["byte_order"] == "little_endian" + assert "byte_order" not in out[1] + + +def test_expand_platform_entry_invalid_byte_order_in_defaults_raises() -> None: + """A dropped `byte_order` inherited from `defaults:` is still validated, so a typo raises.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "little_andian"}, + CONF_FILES: [{"id": "a", "file": "x.png", "type": "binary"}], + } + with pytest.raises(cv.Invalid, match="did you mean") as excinfo: + _expand_platform_entry(0, entry) + assert excinfo.value.path == [0] + + +def test_expand_platform_entry_keeps_byte_order_for_endian_override() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "big_endian"}, + CONF_FILES: [{"id": "a", "file": "x.png", "type": "rgb565"}], + } + [out] = _expand_platform_entry(0, entry) + assert out["byte_order"] == "big_endian" + + +def test_expand_platform_entry_keeps_explicit_byte_order_conflict() -> None: + """A `byte_order` written directly on the entry is kept so validate_settings raises the normal error.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565"}, + CONF_FILES: [ + { + "id": "a", + "file": "x.png", + "type": "binary", + "byte_order": "little_endian", + } + ], + } + [out] = _expand_platform_entry(0, entry) + assert out["byte_order"] == "little_endian" + + +def test_expand_platform_entry_defaults_without_files_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_DEFAULTS: {"type": "RGB565"}} + with pytest.raises(cv.Invalid, match="may only be used together with") as excinfo: + _expand_platform_entry(0, entry) + assert excinfo.value.path == [0] + + +def test_expand_platform_entry_null_files_raises_not_empty() -> None: + """A `files:` key with no value parses to `None` and must be reported clearly.""" + entry = {CONF_PLATFORM: "file", CONF_DEFAULTS: {"type": "RGB565"}, CONF_FILES: None} + with pytest.raises(cv.Invalid, match="must not be empty"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_empty_files_list_raises_not_empty() -> None: + """An explicit `files: []` must not silently drop the whole platform entry.""" + entry = {CONF_PLATFORM: "file", CONF_FILES: []} + with pytest.raises(cv.Invalid, match="must not be empty"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_files_with_stray_key_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "a", "file": "x.png"}], + "extra": 1, + } + with pytest.raises(cv.Invalid, match="cannot be combined with"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_id_in_defaults_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {CONF_ID: "a"}, + CONF_FILES: [{"file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_platform_in_defaults_raises() -> None: + """`platform:` inside `defaults:` would silently reassign every file's platform.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {CONF_PLATFORM: "animation"}, + CONF_FILES: [{"id": "a", "file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_platform_in_file_entry_raises() -> None: + """`platform:` on a `files:` item must not silently override the entry's platform.""" + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "a", "file": "x.png", CONF_PLATFORM: "animation"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_files_not_list_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_FILES: "not-a-list"} + with pytest.raises(cv.Invalid, match="must be a list"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_defaults_not_mapping_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: "not-a-mapping", + CONF_FILES: [{"id": "a", "file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="must be a mapping"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_file_item_not_mapping_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_FILES: [1, 2]} + with pytest.raises(cv.Invalid, match="must be a mapping"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_config_mixes_plain_and_expanded_entries() -> None: + config = [ + { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [ + {"id": "img1", "file": "foo.png"}, + {"id": "img2", "file": "bar.png"}, + ], + }, + {CONF_PLATFORM: "file", "id": "plain", "file": "baz.png", "type": "BINARY"}, + ] + out = expand_platform_config(config) + assert [entry["id"] for entry in out] == ["img1", "img2", "plain"] + + +def test_expand_platform_config_ignores_non_platform_entries() -> None: + # Not expanded here -- legacy_config_migrate runs before this hook and is + # responsible for tagging/flattening pre-platform shapes. + config = ["not-a-platform-entry"] + assert expand_platform_config(config) == config + + +# --------------------- end defaults/files expansion ------------------------- + + def test_validate_image_final_defaults_to_little_endian() -> None: config = {CONF_FILE: "x.png"} validate_image_final(config) diff --git a/tests/components/animation/validate-platform-defaults.host.yaml b/tests/components/animation/validate-platform-defaults.host.yaml new file mode 100644 index 0000000000..034497c548 --- /dev/null +++ b/tests/components/animation/validate-platform-defaults.host.yaml @@ -0,0 +1,21 @@ +# `platform: animation` entry exercising the shared `defaults:`/`files:` expansion. +display: + - platform: sdl + id: animation_display + auto_clear_enabled: false + dimensions: + width: 480 + height: 480 + +image: + - platform: animation + defaults: + type: rgb565 + transparency: opaque + resize: 50x50 + files: + - id: platform_defaults_animation + file: $component_dir/anim.gif + - id: platform_defaults_animation_rgb + file: $component_dir/anim.apng + type: rgb diff --git a/tests/components/image/validate-platform-defaults.host.yaml b/tests/components/image/validate-platform-defaults.host.yaml new file mode 100644 index 0000000000..e1b3037cc3 --- /dev/null +++ b/tests/components/image/validate-platform-defaults.host.yaml @@ -0,0 +1,24 @@ +# `platform: file` entry using the `defaults:`/`files:` shape, including the +# per-type byte_order drop when an entry overrides to a non-endian type. +display: + - platform: sdl + id: image_display + auto_clear_enabled: false + dimensions: + width: 480 + height: 480 + +image: + - platform: file + defaults: + type: rgb565 + transparency: opaque + byte_order: little_endian + resize: 50x50 + dither: FloydSteinberg + files: + - id: platform_defaults_image + file: ../../pnglogo.png + - id: platform_defaults_binary + file: ../../pnglogo.png + type: binary diff --git a/tests/unit_tests/test_config_normalization.py b/tests/unit_tests/test_config_normalization.py index c8b7b63094..04363ad45b 100644 --- a/tests/unit_tests/test_config_normalization.py +++ b/tests/unit_tests/test_config_normalization.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest -from esphome import config, yaml_util +from esphome import config, config_validation as cv, yaml_util from esphome.core import CORE, AutoLoad from esphome.types import ConfigType @@ -127,12 +127,14 @@ def _run_load_step( domain: str, conf: object, migrate: Callable[[ConfigType], list | None] | None, + expand: Callable[[list], list] | None = None, ) -> config.Config: - """Run a LoadValidationStep for a platform component with a given migrate hook.""" + """Run a LoadValidationStep for a platform component with given hooks.""" component = Mock() component.is_platform_component = True component.multi_conf_no_default = False component.legacy_config_migrate = migrate + component.expand_platform_config = expand result = config.Config() with ( @@ -197,6 +199,124 @@ def test_legacy_migrate_skipped_for_autoload() -> None: assert result["image"] == [auto] +# --------------------------------------------------------------------------- +# EXPAND_PLATFORM_CONFIG hook on LoadValidationStep -- permanent counterpart +# to legacy_config_migrate; runs after legacy migration/list normalization. +# --------------------------------------------------------------------------- + + +def test_expand_hook_rewrites_conf() -> None: + """A config the expand hook rewrites is replaced with the expanded list.""" + expanded = [{"platform": "file", "id": "a"}, {"platform": "file", "id": "b"}] + expand = Mock(return_value=expanded) + + result = _run_load_step("image", [{"platform": "file", "id": "a"}], None, expand) + + expand.assert_called_once_with([{"platform": "file", "id": "a"}]) + assert result["image"] == expanded + + +def test_expand_hook_absent_is_noop() -> None: + """A platform component without the hook is left as normalized by the + existing list-wrapping logic.""" + result = _run_load_step("image", [{"platform": "file", "id": "a"}], None, None) + + assert result["image"] == [{"platform": "file", "id": "a"}] + + +def test_expand_hook_runs_after_legacy_migrate() -> None: + """The expand hook sees the already-migrated list, not the raw legacy conf.""" + migrated = [{"platform": "file", "id": "a"}] + migrate = Mock(return_value=migrated) + expand = Mock(side_effect=lambda conf: conf) + + _run_load_step("image", [{"id": "a", "file": "x.png"}], migrate, expand) + + expand.assert_called_once_with(migrated) + + +def test_expand_hook_skipped_for_non_dict_entry() -> None: + """Malformed entries are left alone; the hook only sees `platform:`-tagged dicts.""" + expand = Mock(side_effect=lambda conf: conf) + + result = _run_load_step("image", ["not-a-dict"], None, expand) + + expand.assert_not_called() + assert result["image"] == ["not-a-dict"] + + +def test_expand_hook_skipped_for_entry_missing_platform_key() -> None: + """A dict entry missing the `platform:` key is left alone -- the normal + per-entry error reporting further down catches this case instead.""" + expand = Mock(side_effect=lambda conf: conf) + + result = _run_load_step("image", [{"id": "a"}], None, expand) + + expand.assert_not_called() + assert result["image"] == [{"id": "a"}] + + +def test_expand_hook_skipped_for_autoload() -> None: + """A non-empty AutoLoad reaching the hook stage is left alone.""" + expand = Mock(side_effect=lambda conf: conf) + auto = AutoLoad() + auto["id"] = "a" + + result = _run_load_step("image", auto, None, expand) + + expand.assert_not_called() + assert result["image"] == [auto] + + +def test_expand_hook_runs_when_all_entries_are_platform_tagged_dicts() -> None: + """The guard does not block the normal, well-formed case.""" + expand = Mock(side_effect=lambda conf: conf) + conf = [{"platform": "file", "id": "a"}, {"platform": "animation", "id": "b"}] + + result = _run_load_step("image", conf, None, expand) + + expand.assert_called_once_with(conf) + assert result["image"] == conf + + +def test_expand_hook_invalid_reports_single_error_at_domain_path() -> None: + """A `cv.Invalid` from the hook is reported once with the domain path prepended; no further validation runs.""" + expand = Mock(side_effect=cv.Invalid("bad shape")) + pre_expand_conf = [{"platform": "file", "id": "a"}] + + result = _run_load_step("image", pre_expand_conf, None, expand) + + assert len(result.errors) == 1 + assert result.errors[0].path == ["image"] + assert "bad shape" in str(result.errors[0]) + assert result["image"] == pre_expand_conf + + +def test_expand_hook_final_external_invalid_reports_without_path_prepend() -> None: + """`cv.FinalExternalInvalid` keeps its already-resolved path (no domain path prepended).""" + already_resolved_error = cv.FinalExternalInvalid( + "bad shape", path=["image", 3, "files"] + ) + expand = Mock(side_effect=already_resolved_error) + pre_expand_conf = [{"platform": "file", "id": "a"}] + + result = _run_load_step("image", pre_expand_conf, None, expand) + + assert len(result.errors) == 1 + assert result.errors[0] is already_resolved_error + assert result.errors[0].path == ["image", 3, "files"] + assert result["image"] == pre_expand_conf + + +def test_expand_hook_non_list_return_raises_type_error() -> None: + """A non-list return is a component bug: it escapes as an uncaught TypeError + (explicit raise survives -O/-OO).""" + expand = Mock(return_value={"not": "a list"}) + + with pytest.raises(TypeError, match="must return a list"): + _run_load_step("image", [{"platform": "file", "id": "a"}], None, expand) + + def _write_merge_conflict_config(tmp_path: Path, *, suppress: bool) -> Path: """Create a config where two `<<` includes both define `logger:`. From 47da743d11ec42374a9026d8b473174d20489077 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:09:12 -0700 Subject: [PATCH 137/160] [ai] Add instructions for concise comments (#18522) --- AGENTS.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index fa0f61c263..f006ee6087 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -763,3 +763,13 @@ The project uses English for non-code content. When drafting documentation, code PR descriptions, and similar text, avoid technical jargon. Instead, express concepts in plain English, using standard technical terms only when required. Ensure the text is readily comprehensible to a wide audience, including non-native English speakers. + +## 10. Code Comments + +Code comments on individual lines should be used only where necessary to flag issues that may not be obvious +on a simple reading of the code. Keep them short (e.g. 1 or 2 lines). + +Function and method comment blocks may include more detail as required to make +calling contracts clear and document parameter usage, but should still be kept concise. + +Avoid redundancy and repetition; comments should never simply restate what the code already says. From 0b1065feee095c82220fa4e9d1cd6b3b164aa82a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 16:30:34 -0500 Subject: [PATCH 138/160] [ci] Stop jobs hanging on apt by restoring the cached apt action and bounding raw apt calls (#18518) --- .github/workflows/ci-api-proto.yml | 28 +++++++++- .github/workflows/ci.yml | 89 +++++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-api-proto.yml b/.github/workflows/ci-api-proto.yml index 1ccff96f24..63219a1dbc 100644 --- a/.github/workflows/ci-api-proto.yml +++ b/.github/workflows/ci-api-proto.yml @@ -41,10 +41,32 @@ jobs: version: "0.11.15" - name: Install apt dependencies + # PR-only workflow, so nothing on dev could seed a shared apt cache + # entry; the cached apt action would save one copy per PR. Plain apt + # with every call bounded: the apt.conf.d timeouts make a dead + # mirror fail over in seconds, and timeout runs under sudo so it can + # kill apt-get itself. Install without update first: image lists are + # fresh, and the index refresh is what a congested mirror makes slow. + timeout-minutes: 15 run: | - sudo apt update - sudo apt-cache show protobuf-compiler - sudo apt install -y protobuf-compiler + sudo tee /etc/apt/apt.conf.d/99ci-acquire-timeouts >/dev/null <<'EOF' + Acquire::Retries "1"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + EOF + # Common path: the image's package lists are fresh enough. + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 \ + apt-get install -y protobuf-compiler; then + protoc --version + exit 0 + fi + # Rescue path: refresh the lists once with a generous bound; the + # apt config already fails a stalled mirror over quickly. + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 30 \ + dpkg --configure -a || true + sudo timeout -k 15 300 apt-get update + sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 \ + apt-get install -y protobuf-compiler protoc --version - name: Install python dependencies run: uv pip install --system aioesphomeapi -c requirements.txt -r requirements_dev.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6afb8a9d22..35148de0c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,22 @@ jobs: uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . + seed-apt-cache: + name: Seed apt package cache + runs-on: ubuntu-24.04 + # PR-branch cache saves are invisible to other PRs, so dev/beta/release + # pushes seed the one shared entry PR jobs restore. The key is derived + # only from the package list and version; keep both identical in every + # step that restores it. In ci-status needs so a broken seed fails dev. + if: github.event_name == 'push' + timeout-minutes: 10 + steps: + - name: Install apt packages (cached) + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 + determine-jobs: name: Determine which jobs to run runs-on: ubuntu-24.04 @@ -323,7 +339,8 @@ jobs: integration-tests: name: Run integration tests (${{ matrix.bucket.name }}) - runs-on: ubuntu-latest + # Must match seed-apt-cache's image: the apt cache key has no OS in it. + runs-on: ubuntu-24.04 needs: - common - determine-jobs @@ -335,12 +352,16 @@ jobs: steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install ccache - # Speeds up the host compiles: tests in a bucket compile overlapping - # component sets, so later tests reuse earlier tests' objects. - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends ccache + - name: Install apt packages (cached) + # ccache speeds up the host compiles. A cache hit never touches apt + # (mirror outages cannot hang the job); the timeout bounds the cold + # path. Packages and version must match seed-apt-cache exactly; + # libsdl2-dev is unused here and carried only for cache-key parity. + timeout-minutes: 10 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 - name: Set up Python 3.13 id: python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -421,6 +442,7 @@ jobs: benchmarks: name: Run CodSpeed benchmarks runs-on: ubuntu-24.04 + timeout-minutes: 30 needs: - common - determine-jobs @@ -461,6 +483,41 @@ jobs: fi echo "binary=$BINARY" >> $GITHUB_OUTPUT + - name: Bound apt fetches and pre-install libc6-dbg + # The CodSpeed runner installs valgrind + libc6-dbg via its own + # unbounded apt-get update; per-invocation apt options cannot reach + # it. The apt.conf.d timeouts below bound every later apt call in + # this job, the runner's included. Pre-installing libc6-dbg lets the + # runner skip apt once its valgrind cache is restored (it checks + # ``dpkg -s libc6-dbg``, so the cache action's unregistered restores + # would not count). Install without update first: image lists are + # fresh, and the index refresh is what a congested mirror makes + # slow. Best effort; the job timeout is the last backstop. + timeout-minutes: 15 + continue-on-error: true + run: | + sudo tee /etc/apt/apt.conf.d/99ci-acquire-timeouts >/dev/null <<'EOF' + Acquire::Retries "1"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + EOF + if dpkg -s libc6-dbg >/dev/null 2>&1; then + echo "libc6-dbg already installed" + exit 0 + fi + # Common path: the image's package lists are fresh enough. + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 \ + apt-get install -y libc6-dbg; then + exit 0 + fi + # Rescue path: refresh the lists once with a generous bound; the + # apt config already fails a stalled mirror over quickly. + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 30 \ + dpkg --configure -a || true + sudo timeout -k 15 300 apt-get update + sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 \ + apt-get install -y libc6-dbg + - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3 with: @@ -884,12 +941,17 @@ jobs: - name: List components run: echo ${{ matrix.batch.components }} - - name: Install apt packages - # Not cached: this job is pull-request-only, so a cache save could - # never be shared and would only consume quota. - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends libsdl2-dev ccache + - name: Install apt packages (cached) + # A cache hit (seeded on dev by seed-apt-cache) never touches apt, + # so mirror outages cannot hang this PR-only job; the timeout bounds + # the cold path. Packages and version must match seed-apt-cache + # exactly. The action has no --no-install-recommends; same package + # set this job used before #17463. + timeout-minutes: 10 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1424,6 +1486,7 @@ jobs: # this check. needs: - common + - seed-apt-cache - determine-jobs - ci-custom - pylint From 9daae377fca5eea6d1f39d13fa33e790d50b2f9c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:03:14 -0500 Subject: [PATCH 139/160] [api] Bump noise-c to 0.1.20 (#18482) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index cdc0d97c49..3e69d5842c 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.19") + cg.add_library("esphome/noise-c", "0.1.20") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 39600d622a..13bb5a556f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.19 ; api + esphome/noise-c@0.1.20 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.19 ; api + esphome/noise-c@0.1.20 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.19 ; used by api + esphome/noise-c@0.1.20 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 200a1644a5d12c30e4f0d562f1e0132b96482dc6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:03:33 -0500 Subject: [PATCH 140/160] [ci] Fail the benchmark job when the C++ benchmark build fails (#18480) --- .github/workflows/ci.yml | 17 ++++++++++++++--- tests/benchmarks/components/api/__init__.py | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b603e68ad7..2075fde9ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -460,10 +460,21 @@ jobs: - name: Build benchmarks id: build run: | + # pipefail: without it a failed build is masked by the grep/cut + # pipeline below, leaving BINARY empty and silently dropping every + # C++ benchmark from the run while the job still reports success. + set -o pipefail . venv/bin/activate - export BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) - # --build-only prints BUILD_BINARY= to stdout - BINARY=$(script/cpp_benchmark.py --all --build-only | grep '^BUILD_BINARY=' | tail -1 | cut -d= -f2-) + BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) + export BENCHMARK_LIB_CONFIG + # --build-only prints BUILD_BINARY= to stdout; the grep is + # non-fatal so a missing marker reaches the check below instead of + # tripping errexit at this assignment + BINARY=$(script/cpp_benchmark.py --all --build-only | { grep '^BUILD_BINARY=' || true; } | tail -1 | cut -d= -f2-) + if [ -z "$BINARY" ]; then + echo "::error::Benchmark build did not report a binary path" + exit 1 + fi echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/tests/benchmarks/components/api/__init__.py b/tests/benchmarks/components/api/__init__.py index 0d02e0b054..0565bc5330 100644 --- a/tests/benchmarks/components/api/__init__.py +++ b/tests/benchmarks/components/api/__init__.py @@ -15,6 +15,7 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # components have hardware dependencies (BLE/UART/RMT); lightweight # stub headers in tests/benchmarks/stubs/ satisfy the includes. cg.add_define("USE_BLUETOOTH_PROXY") + cg.add_define("USE_BLUETOOTH_PROXY_CONNECTIONS") cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", 3) cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) cg.add_define("USE_ZWAVE_PROXY") From a99a8f364e8bad032d89d65e18e2470fd2df2267 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:32:37 -0500 Subject: [PATCH 141/160] [api] Bump noise-c to 0.1.21 (#18484) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3e69d5842c..912d580a0f 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.20") + cg.add_library("esphome/noise-c", "0.1.21") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 13bb5a556f..4c372cc0bb 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.20 ; api + esphome/noise-c@0.1.21 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.20 ; api + esphome/noise-c@0.1.21 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.20 ; used by api + esphome/noise-c@0.1.21 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 10e592fa3a51b301644b5a742c75088cc3ab286a Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Tue, 18 Aug 2026 09:09:12 -0700 Subject: [PATCH 142/160] [modbus] CRC scan all unknown function codes (#18483) Co-authored-by: Claude Opus 4.8 (1M context) --- esphome/components/modbus/modbus.cpp | 33 ++-- esphome/components/modbus/modbus.h | 2 +- esphome/components/modbus/modbus_helpers.h | 32 ++++ tests/components/modbus/common.h | 36 +++++ .../modbus/modbus_unknown_function_test.cpp | 141 ++++++++++++++++++ 5 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 tests/components/modbus/modbus_unknown_function_test.cpp diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index 5305f6313f..e4bd51ad5a 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -219,14 +219,25 @@ void ModbusServerHub::parse_modbus_frames() { this->clear_rx_buffer_(LOG_STR("timeout after partial response"), true); } -uint16_t Modbus::find_custom_frame_end_(uint16_t min_length) const { - // Custom functions could be any length - we have to rely on the CRC to determine completeness. +uint16_t Modbus::find_frame_end_by_crc_(uint16_t min_length) const { + // Unknown-length functions (user-defined codes, unimplemented management codes, unassigned values) + // could be any length - we have to rely on the CRC to determine completeness. // If a CRC match is never found, the buffer will eventually overflow and be cleared. const uint8_t *raw = &this->rx_buffer_[0]; const size_t size = this->rx_buffer_.size(); - for (uint16_t len = min_length; len <= std::min(size, size_t(MAX_FRAME_SIZE)); len++) { - if (crc16(raw, len) == 0) - return len; + const auto max_len = static_cast(std::min(size, size_t(MAX_FRAME_SIZE))); + if (min_length > max_len) + return 0; + // The Modbus CRC (poly 0xa001, refin/refout false) keeps its running state in the returned value, + // so we seed once over the first min_length bytes and extend one byte at a time instead of + // recomputing the whole prefix for every candidate length. + uint16_t crc = crc16(raw, min_length); + if (crc == 0) + return min_length; + for (uint16_t len = min_length; len < max_len; len++) { + crc = crc16(&raw[len], 1, crc); + if (crc == 0) + return len + 1; } return 0; } @@ -241,11 +252,11 @@ bool Modbus::parse_modbus_server_frame_() { uint8_t address = this->rx_buffer_[0]; uint8_t function_code = this->rx_buffer_[1]; - if (helpers::is_function_code_custom(function_code)) { - frame_length = this->find_custom_frame_end_(frame_length); + if (helpers::is_function_code_unknown_length(function_code)) { + frame_length = this->find_frame_end_by_crc_(frame_length); if (frame_length == 0) return size < MAX_FRAME_SIZE; // Continue to parse until we hit max size - ESP_LOGD(TAG, "User-defined function %02X found", function_code); + ESP_LOGD(TAG, "Unknown-length function %02X found", function_code); } else { if (crc16(&this->rx_buffer_[0], frame_length) != 0) return false; @@ -272,11 +283,11 @@ bool ModbusServerHub::parse_modbus_client_frame_() { uint8_t address = this->rx_buffer_[0]; uint8_t function_code = this->rx_buffer_[1]; - if (helpers::is_function_code_custom(function_code)) { - frame_length = this->find_custom_frame_end_(frame_length); + if (helpers::is_function_code_unknown_length(function_code)) { + frame_length = this->find_frame_end_by_crc_(frame_length); if (frame_length == 0) return size < MAX_FRAME_SIZE; // Continue to parse until we hit max size - ESP_LOGD(TAG, "User-defined function %02X found", function_code); + ESP_LOGD(TAG, "Unknown-length function %02X found", function_code); } else { if (crc16(&this->rx_buffer_[0], frame_length) != 0) return false; diff --git a/esphome/components/modbus/modbus.h b/esphome/components/modbus/modbus.h index dfe4a4872d..bb303c43a8 100644 --- a/esphome/components/modbus/modbus.h +++ b/esphome/components/modbus/modbus.h @@ -82,7 +82,7 @@ class Modbus : public uart::UARTDevice, public Component { bool send_frame_(const ModbusFrame &frame); // Scans forward from min_length to find a frame boundary by CRC match for custom function codes. // Returns the matched frame length, or 0 if no valid CRC was found within MAX_FRAME_SIZE. - uint16_t find_custom_frame_end_(uint16_t min_length) const; + uint16_t find_frame_end_by_crc_(uint16_t min_length) const; uint32_t last_modbus_byte_{0}; uint32_t last_receive_check_{0}; diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index c737e206c0..b2454e6f14 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -55,6 +55,38 @@ inline bool is_function_code_custom(uint8_t function_code) { masked_function_code <= FUNCTION_CODE_USER_DEFINED_SPACE_2_END); } +/// True for any function code whose frame length the parsers cannot predict - everything the +/// server_pdu_length()/client_pdu_length() switches fall through to `default` on (keep the case list +/// in step with those switches). Deliberately wider than is_function_code_custom(): the user-defined +/// ranges are unknown to the parser too, but so are the assigned-but-unimplemented codes +/// (READ_EXCEPTION_STATUS, DIAGNOSTICS, GET_COMM_EVENT_*, REPORT_SERVER_ID) and every unassigned value. +/// The 0x80 exception flag is masked off first, so a frame with it set classifies by its base code - +/// even though a spec exception reply has a known 2-byte PDU. That is deliberate, matching what +/// is_function_code_custom() has always done: some vendors use codes with the 0x80 bit set as ordinary +/// codes with longer payloads, so the response parser CRC-scans these rather than assuming the spec +/// length. For an intact spec exception the scan matches at its first candidate, so only a corrupt one +/// pays (recovery by timeout instead of an immediate CRC failure). +inline bool is_function_code_unknown_length(uint8_t function_code) { + switch (static_cast(function_code & FUNCTION_CODE_MASK)) { + case FunctionCode::READ_COILS: + case FunctionCode::READ_DISCRETE_INPUTS: + case FunctionCode::READ_HOLDING_REGISTERS: + case FunctionCode::READ_INPUT_REGISTERS: + case FunctionCode::WRITE_SINGLE_COIL: + case FunctionCode::WRITE_SINGLE_REGISTER: + case FunctionCode::WRITE_MULTIPLE_COILS: + case FunctionCode::WRITE_MULTIPLE_REGISTERS: + case FunctionCode::READ_FILE_RECORD: + case FunctionCode::WRITE_FILE_RECORD: + case FunctionCode::MASK_WRITE_REGISTER: + case FunctionCode::READ_WRITE_MULTIPLE_REGISTERS: + case FunctionCode::READ_FIFO_QUEUE: + return false; + default: + return true; + } +} + // Returns the expected length of a server response PDU based on the function code. // If too few bytes have arrived to determine the length, returns the minimum length. `size` is the // number of bytes available so far, which may exceed the eventual PDU (e.g. include the frame's CRC diff --git a/tests/components/modbus/common.h b/tests/components/modbus/common.h index d03ccf8ec3..e6c37b0e6d 100644 --- a/tests/components/modbus/common.h +++ b/tests/components/modbus/common.h @@ -1,7 +1,10 @@ #pragma once #include +#include +#include #include #include "esphome/components/uart/uart_component.h" +#include "esphome/core/helpers.h" namespace esphome::modbus::testing { @@ -30,4 +33,37 @@ class RecordingUART : public NullUART { std::vector written; }; +// A UART the test can inject received bytes into, so frames travel the full receive path +// (receive_modbus_frames -> parse -> dispatch) through hub.loop(). Writes are recorded. +class InjectableUART : public RecordingUART { + public: + bool peek_byte(uint8_t *data) override { + if (this->rx_.empty()) + return false; + *data = this->rx_.front(); + return true; + } + bool read_array(uint8_t *data, size_t len) override { + if (len > this->rx_.size()) + return false; + memcpy(data, this->rx_.data(), len); + this->rx_.erase(this->rx_.begin(), this->rx_.begin() + len); + return true; + } + size_t available() override { return this->rx_.size(); } + + // Queues a complete wire frame: address + PDU + CRC16 (low byte first). + void inject_frame(uint8_t address, std::span pdu) { + size_t start = this->rx_.size(); + this->rx_.push_back(address); + this->rx_.insert(this->rx_.end(), pdu.begin(), pdu.end()); + uint16_t crc = crc16(this->rx_.data() + start, this->rx_.size() - start); + this->rx_.push_back(crc & 0xFF); + this->rx_.push_back(crc >> 8); + } + + private: + std::vector rx_; +}; + } // namespace esphome::modbus::testing diff --git a/tests/components/modbus/modbus_unknown_function_test.cpp b/tests/components/modbus/modbus_unknown_function_test.cpp new file mode 100644 index 0000000000..8b91d088b8 --- /dev/null +++ b/tests/components/modbus/modbus_unknown_function_test.cpp @@ -0,0 +1,141 @@ +#include + +#include +#include +#include + +#include "common.h" +#include "esphome/components/modbus/modbus.h" + +namespace esphome::modbus::testing { + +namespace { + +// Records custom-response dispatches so tests can assert an unknown-length frame reached the device. +class CustomRecordingDevice : public ModbusClientDevice { + public: + using ModbusClientDevice::ModbusClientDevice; + void on_custom_response(std::span request_pdu, std::span response_pdu, + ResponseStatus status) override { + this->requests.emplace_back(request_pdu.begin(), request_pdu.end()); + this->responses.emplace_back(response_pdu.begin(), response_pdu.end()); + this->statuses.push_back(status); + } + std::vector> requests; + std::vector> responses; + std::vector statuses; +}; + +// Every handler keeps its ILLEGAL_FUNCTION default; the hub's dispatch is what is under test. +class SilentServerDevice : public ModbusServerDevice {}; + +// Drives full client frames through the server hub's receive path (same shape as the broadcast tests). +class TestServerHub : public ModbusServerHub { + public: + bool tx_blocked() override { return false; } + + // Builds a complete client frame (address + FC + data + CRC) and runs the full receive-side parser. + // Returns true once the buffer has fully drained. + bool run_receive_parser_for_test(uint8_t address, uint8_t function_code, std::span data) { + this->rx_buffer_.clear(); + this->rx_buffer_.reserve(data.size() + 4); + this->rx_buffer_.push_back(address); + this->rx_buffer_.push_back(function_code); + this->rx_buffer_.insert(this->rx_buffer_.end(), data.begin(), data.end()); + uint16_t crc = crc16(this->rx_buffer_.data(), this->rx_buffer_.size()); + this->rx_buffer_.push_back(crc & 0xFF); + this->rx_buffer_.push_back(crc >> 8); + this->parse_modbus_frames(); + return this->rx_buffer_.empty(); + } +}; + +} // namespace + +// The frame-length parsers have explicit cases for exactly these 13 codes; every other value - the +// assigned-but-unimplemented management codes, both user-defined ranges, and all unassigned codes - +// must classify as unknown length. The exception flag masks off first. +TEST(ModbusUnknownFunction, HelperMatchesParserCoverage) { + for (uint8_t fc : {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x0F, 0x10, 0x14, 0x15, 0x16, 0x17, 0x18}) { + EXPECT_FALSE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); + } + for (uint8_t fc : {0x07, 0x08, 0x0B, 0x0C, 0x11, 0x2A, 0x41, 0x48, 0x49, 0x64, 0x6E, 0x00, 0x7F}) { + EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); + } + // Exception replies classify by their base code. + EXPECT_FALSE(helpers::is_function_code_unknown_length(0x83)); + EXPECT_TRUE(helpers::is_function_code_unknown_length(0x87)); + // Strictly wider than the user-defined ranges: every custom code is unknown-length, but not vice versa. + for (int fc = 0; fc <= 0xFF; fc++) { + if (helpers::is_function_code_custom(fc)) + EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << fc; + } + EXPECT_FALSE(helpers::is_function_code_custom(0x49)); + + // Derived contract check: the helper must say "unknown" exactly when both length parsers fall + // through to default. With a zero-filled max-size PDU every explicit case returns at least 2 + // (file records bottom out at 2, FIFO at 3) and only default returns MIN_PDU_SIZE, so comparing + // against MIN_PDU_SIZE detects a case added to either switch without updating the helper. The + // loop stops at 0x7F: above it the helper masks the exception flag off while client_pdu_length() + // switches on the unmasked byte and server_pdu_length() early-returns the exception length. + for (int fc = 0; fc <= 0x7F; fc++) { + const uint8_t pdu[MAX_PDU_SIZE] = {static_cast(fc)}; // zero header fields + EXPECT_EQ(helpers::is_function_code_unknown_length(fc), + helpers::client_pdu_length(pdu, sizeof(pdu)) == MIN_PDU_SIZE) + << "client_pdu_length disagrees for fc 0x" << std::hex << fc; + EXPECT_EQ(helpers::is_function_code_unknown_length(fc), + helpers::server_pdu_length(pdu, sizeof(pdu)) == MIN_PDU_SIZE) + << "server_pdu_length disagrees for fc 0x" << std::hex << fc; + } +} + +// A response with a function code outside the user-defined ranges (0x49) has no length case in +// server_pdu_length(), so the parser must find the frame end by CRC scan - the same way it already +// handles user-defined codes. Frame: address + FC 0x49 + 3 data bytes + CRC = 7 bytes. Without the +// scan the parser assumes a 4-byte frame, fails the CRC, and the response never reaches the device. +TEST(ModbusUnknownFunction, ClientParsesUnknownLengthResponse) { + InjectableUART uart; + ModbusClientHub hub; + hub.set_uart_parent(&uart); + hub.setup(); // computes frame timing from the baud rate + CustomRecordingDevice device(&hub, 0x02); + + const uint8_t request[] = {0x49, 0x01}; + ASSERT_TRUE(device.queue_pdu(request)); + hub.loop(); // transmit + ASSERT_FALSE(uart.written.empty()); + + const uint8_t response_pdu[] = {0x49, 0x02, 0xAA, 0xBB}; + uart.inject_frame(0x02, response_pdu); + hub.loop(); // receive + parse + match + dispatch + + ASSERT_EQ(device.responses.size(), 1u); + EXPECT_EQ(device.requests[0], std::vector(request, request + sizeof(request))); + EXPECT_EQ(device.responses[0], std::vector(response_pdu, response_pdu + sizeof(response_pdu))); + EXPECT_FALSE(device.statuses[0].has_value()); +} + +// The server side of the same gap: a request with FC 0x49 for a registered device must parse (CRC +// scan again) so the hub can answer ILLEGAL_FUNCTION per the spec. Without the scan the frame fails +// to parse and the client gets silence instead of the exception. +TEST(ModbusUnknownFunction, ServerRepliesIllegalFunctionToUnknownLengthRequest) { + TestServerHub hub; + RecordingUART uart; + hub.set_uart_parent(&uart); + + SilentServerDevice device; + device.set_address(0x02); + hub.register_device(&device); + + const uint8_t data[] = {0x02, 0xAA, 0xBB}; + ASSERT_TRUE(hub.run_receive_parser_for_test(0x02, 0x49, data)); + + // Expected reply: address + FC with exception flag + ILLEGAL_FUNCTION + CRC. + std::vector expected = {0x02, 0xC9, 0x01}; + uint16_t crc = crc16(expected.data(), expected.size()); + expected.push_back(crc & 0xFF); + expected.push_back(crc >> 8); + EXPECT_EQ(uart.written, expected); +} + +} // namespace esphome::modbus::testing From 2df953f3d7c0b022cd3a75c05ab2ca0d63cc39f9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:36:57 -0500 Subject: [PATCH 143/160] [platformio] Give the ccache wrapper a cmd.exe safe path (#18495) --- esphome/platformio/ccache.py.script | 9 +- esphome/platformio/toolchain.py | 63 +++-- tests/unit_tests/test_platformio_toolchain.py | 240 +++++++++++++++++- 3 files changed, 286 insertions(+), 26 deletions(-) diff --git a/esphome/platformio/ccache.py.script b/esphome/platformio/ccache.py.script index cc08a8c044..22592a2398 100644 --- a/esphome/platformio/ccache.py.script +++ b/esphome/platformio/ccache.py.script @@ -1,5 +1,4 @@ import os -import shutil # pylint: disable=E0602 Import("env") # noqa @@ -9,15 +8,17 @@ Import("env") # noqa # esphome/platformio/toolchain.py); this script only supplies the SCons-level # mechanism. # +# The binary comes pre-resolved in ESPHOME_CCACHE_PATH; _ccache_env() has +# already stripped the Windows \\?\ prefix that cmd.exe cannot run. +# # This is a "pre" script, so the platform's builder (which sets CC/CXX and # clones the construction environment for framework and library builds) runs # after it. Replacing CC/CXX here would be overwritten, and replacing them in # a "post" script would miss the already-cloned library environments. Wrapping # SPAWN instead is ordering-proof: clones copy the wrapper, and every compiler # invocation from every environment funnels through it at execution time. -if ( - os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" - and (ccache_path := shutil.which("ccache")) is not None +if os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" and ( + ccache_path := os.environ.get("ESPHOME_CCACHE_PATH") ): original_spawn = env["SPAWN"] diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 08a4fcff78..d76581d032 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -60,6 +60,9 @@ def _strip_win_long_path_prefix(path: str) -> str: "The system cannot find the path specified." Stripping the prefix early keeps the path shell-quotable. + Also applied to the ccache path exported by ``_ccache_env()``, which + ``shutil.which`` can return with the same prefix. + No-op on non-Windows platforms. """ if sys.platform != "win32": @@ -235,8 +238,8 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) -def _ccache_usable() -> bool: - """Return True when the ``ccache`` on PATH actually runs. +def _ccache_runs(ccache: str) -> bool: + """Return True when the ``ccache`` found on PATH actually runs. ``shutil.which`` proves existence, not runnability: on Windows it also matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose @@ -244,9 +247,6 @@ def _ccache_usable() -> bool: step with an opaque OS error, so probe once and fall back to compiling without ccache when the probe fails. """ - ccache = shutil.which("ccache") - if ccache is None: - return False try: subprocess.run( [ccache, "--version"], @@ -265,14 +265,29 @@ def _ccache_usable() -> bool: def _ccache_env() -> dict[str, str]: - """Return ccache settings for PlatformIO builds. + r"""Return ccache settings for PlatformIO builds. Enabled by default whenever the ``ccache`` binary is on PATH; set ``ESPHOME_CCACHE_ENABLE=0`` in the environment to opt out (or ``1`` to - force it on). The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` - so platform build scripts (e.g. the esp8266 ``ccache.py`` extra script, - which wraps compiler invocations inside SCons) only have to check for - ``"1"`` instead of re-implementing the policy. + force it on without the runnability probe; a binary is still needed). + The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` and the + binary's location into ``ESPHOME_CCACHE_PATH`` so platform build scripts + (the shared ``ccache.py`` extra script, which wraps compiler invocations + inside SCons) only have to check for ``"1"`` and use the path as given + instead of re-implementing the policy. + + The path is exported rather than looked up again inside SCons because + ``shutil.which`` can return a Windows extended-length ``\\?\`` path + (ESPHome Desktop puts its bundled ccache on PATH that way). Such a path + runs fine through ``CreateProcess``, which is how ESP-IDF invokes it, + but SCons runs every compile through ``cmd.exe``, which fails on it with + "The system cannot find the path specified." (#18399), so the prefix is + stripped here with ``_strip_win_long_path_prefix()`` before the + runnability probe, which therefore validates the exact string the build + will execute. + ``ESPHOME_CCACHE_PATH`` is an internal channel, not a user setting: the + script only honours it together with ``ESPHOME_CCACHE_ENABLE=1``, and this + function always sets both or neither. The returned values are merged into the environment of the PlatformIO subprocess only, never into ``os.environ``: a long-running process @@ -293,13 +308,27 @@ def _ccache_env() -> dict[str, str]: build dir. The other ``CCACHE_*`` values the user already set in the environment are respected. """ - if "ESPHOME_CCACHE_ENABLE" in os.environ: - enabled = get_bool_env("ESPHOME_CCACHE_ENABLE") - else: - enabled = _ccache_usable() - env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"} - if not enabled: - return env + explicit = "ESPHOME_CCACHE_ENABLE" in os.environ + if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): + return {"ESPHOME_CCACHE_ENABLE": "0"} + ccache_path = shutil.which("ccache") + if ccache_path is None: + if explicit: + _LOGGER.warning( + "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " + "compiling without ccache" + ) + return {"ESPHOME_CCACHE_ENABLE": "0"} + # Strip before probing so the probe validates (and the failure warning + # names) the exact string the build will execute through cmd.exe. + ccache_path = _strip_win_long_path_prefix(ccache_path) + # An explicit opt-in skips the runnability probe. + if not explicit and not _ccache_runs(ccache_path): + return {"ESPHOME_CCACHE_ENABLE": "0"} + env = { + "ESPHOME_CCACHE_ENABLE": "1", + "ESPHOME_CCACHE_PATH": ccache_path, + } # build_path is set during preload for every config-loading command, so it # being unset means a caller built the environment too early; fail loudly # rather than with an opaque TypeError from Path(None). diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index eebb0b8cd7..172b288c25 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -2,7 +2,7 @@ # pylint: disable=protected-access -from collections.abc import Generator +from collections.abc import Callable, Generator from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json @@ -437,6 +437,7 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: env = toolchain._ccache_env() assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) assert env["CCACHE_DIR"].endswith("platformio-ccache") assert env["CCACHE_NOHASHDIR"] == "true" @@ -446,17 +447,35 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: assert "ESPHOME_CCACHE_ENABLE" not in os.environ -def test_ccache_env_disabled_without_binary(setup_core: Path) -> None: - """Ccache stays off when the binary is not on PATH.""" +@pytest.mark.parametrize( + ("env_vars", "expect_warning"), + [ + pytest.param({}, False, id="default"), + pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, True, id="forced-on"), + ], +) +def test_ccache_env_disabled_without_binary( + setup_core: Path, + caplog: pytest.LogCaptureFixture, + env_vars: dict[str, str], + expect_warning: bool, +) -> None: + """Ccache stays off when the binary is not on PATH, even when forced on. + + A deliberate opt-in that finds no binary is downgraded with a warning so + the user can tell why it had no effect; the default path stays quiet. + """ CORE.build_path = setup_core / "build" / "test" with ( - patch.dict(os.environ, {}, clear=True), + patch.dict(os.environ, env_vars, clear=True), patch.object(toolchain.shutil, "which", return_value=None), + caplog.at_level("WARNING"), ): env = toolchain._ccache_env() assert env == {"ESPHOME_CCACHE_ENABLE": "0"} + assert ("no ccache binary is on PATH" in caplog.text) is expect_warning @pytest.mark.parametrize( @@ -489,14 +508,47 @@ def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), patch.object(toolchain.subprocess, "run") as mock_probe, ): env = toolchain._ccache_env() assert env["ESPHOME_CCACHE_ENABLE"] == "1" + # The binary's location is still handed to the build script. + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" mock_probe.assert_not_called() +def test_ccache_env_strips_win_long_path_prefix(setup_core: Path) -> None: + r"""A ``\\?\`` ccache path from PATH is exported without the prefix. + + That is the shape ESPHome Desktop puts on PATH (#18399); see ``_ccache_env``. + """ + CORE.build_path = setup_core / "build" / "test" + prefixed = ( + "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder" + "\\ccache\\ccache.exe" + ) + stripped = ( + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder\\ccache\\ccache.exe" + ) + + with ( + patch.dict(os.environ, {}, clear=True), + # shutil.which is patched, so the win32 code path of the real + # implementation (which crashes on a POSIX host) is never reached. + patch("esphome.platformio.toolchain.sys.platform", "win32"), + patch.object(toolchain.shutil, "which", return_value=prefixed), + patch.object(toolchain.subprocess, "run") as mock_probe, + ): + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == stripped + # The probe validates the exact string the build will execute. + assert mock_probe.call_args[0][0] == [stripped, "--version"] + + def test_ccache_env_opt_out(setup_core: Path) -> None: """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present.""" CORE.build_path = setup_core / "build" / "test" @@ -516,7 +568,7 @@ def test_ccache_env_normalizes_enable_value(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "yes"}, clear=True), - patch.object(toolchain.shutil, "which", return_value=None), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), ): env = toolchain._ccache_env() @@ -563,8 +615,10 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only( env = mock_run_external_process.call_args[1]["env"] assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) assert "ESPHOME_CCACHE_ENABLE" not in os.environ + assert "ESPHOME_CCACHE_PATH" not in os.environ assert "CCACHE_BASEDIR" not in os.environ @@ -613,6 +667,182 @@ def test_copy_ccache_script(setup_core: Path) -> None: assert dest.read_text() == source.read_text() +class _FakeSConsEnv(dict): + """Just enough of a SCons construction environment for ccache.py.""" + + def Replace(self, **kwargs: object) -> None: # noqa: N802 + self.update(kwargs) + + +def _load_ccache_script( + env_vars: dict[str, str], original_spawn: Callable[..., int] | None = None +) -> tuple[_FakeSConsEnv, Callable[..., int]]: + """Run ccache.py.script against a fake SCons env and return (env, original SPAWN).""" + if original_spawn is None: + original_spawn = Mock(name="original_spawn", return_value=0) + scons_env = _FakeSConsEnv(SPAWN=original_spawn) + source = (Path(toolchain.__file__).parent / "ccache.py.script").read_text() + with patch.dict(os.environ, env_vars, clear=True): + exec( # noqa: S102 + compile(source, "ccache.py", "exec"), + {"Import": lambda *_names: None, "env": scons_env}, + ) + return scons_env, original_spawn + + +def _scons_win32_escape(x: str) -> str: + """Copy of ``SCons.Platform.win32.escape``: quote, guarding a trailing backslash.""" + if x[-1] == "\\": + x = x + "\\" + return '"' + x + '"' + + +def test_ccache_script_wraps_compiles_with_exported_path() -> None: + """The SCons script uses ESPHOME_CCACHE_PATH as given, without a PATH lookup.""" + ccache_path = "C:\\Users\\jesse\\ESPHome Device Builder\\ccache\\ccache.exe" + scons_env, original_spawn = _load_ccache_script( + {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": ccache_path} + ) + spawn = scons_env["SPAWN"] + assert spawn is not original_spawn + + # A compile step is routed through ccache, with the same path used for + # the program and (escaped) as the first argument. + compile_args = ["xtensa-lx106-elf-g++", "-o", "main.o", "-c", "main.cpp"] + spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", compile_args, {}) + original_spawn.assert_called_once_with( + "cmd.exe", + _scons_win32_escape, + ccache_path, + [_scons_win32_escape(ccache_path), *compile_args], + {}, + ) + + # Link steps pass through untouched. + original_spawn.reset_mock() + link_args = ["xtensa-lx106-elf-g++", "-o", "firmware.elf", "main.o"] + spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {}) + original_spawn.assert_called_once_with( + "cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {} + ) + + +@pytest.mark.parametrize( + "env_vars", + [ + pytest.param({"ESPHOME_CCACHE_ENABLE": "0"}, id="disabled"), + pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, id="enabled-without-path"), + pytest.param({}, id="unset"), + ], +) +def test_ccache_script_leaves_spawn_alone_without_path( + env_vars: dict[str, str], +) -> None: + """Without both the enable flag and a path, SPAWN is not replaced.""" + scons_env, original_spawn = _load_ccache_script(env_vars) + assert scons_env["SPAWN"] is original_spawn + + +def _scons_win32_spawn( + sh: str, escape: Callable[[str], str], cmd: str, args: list[str], env: dict +) -> int: + r"""Mirror of ``SCons.Platform.win32.spawn``: every command runs via ``cmd.exe /C``. + + SCons is not importable in the test environment (PlatformIO fetches it at + build time), so the lines that matter are mirrored here. The command line + SCons hands ``os.spawnve`` goes to ``CreateProcess`` via ``subprocess`` + instead (identical on Windows, where a string passes through untouched); + ``spawnve`` itself crashes inside pytest. + """ + return subprocess.run( + " ".join([sh, "/C", escape(" ".join(args))]), env=env, check=False + ).returncode + + +_MARKER_ENV = "ESPHOME_TEST_CCACHE_MARKER" +# Stands in for a compile: the "ccache" is really the Python interpreter, and +# the compile "flags" make it write a marker file so the test can tell whether +# the wrapped command actually ran to completion. +_FAKE_COMPILE_ARGS = [ + "-c", + f"import os, pathlib; pathlib.Path(os.environ['{_MARKER_ENV}']).write_text('compiled')", +] + + +def _spawn_fake_compile_via_cmd_exe(scons_env: _FakeSConsEnv, marker: Path) -> int: + """Run one wrapped compile step the way SCons does on Windows.""" + child_env = {**os.environ, _MARKER_ENV: str(marker)} + return scons_env["SPAWN"]( + os.environ.get("COMSPEC", "cmd.exe"), + _scons_win32_escape, + "xtensa-lx106-elf-gcc", + [_scons_win32_escape(arg) if " " in arg else arg for arg in _FAKE_COMPILE_ARGS], + child_env, + ) + + +_WINDOWS_ONLY = pytest.mark.skipif( + sys.platform != "win32", reason="drives cmd.exe, which SCons uses only on Windows" +) + + +@_WINDOWS_ONLY +def test_ccache_env_real_probe_runs_stripped_path(setup_core: Path) -> None: + r"""With a ``\\?\`` which result, the real probe runs the stripped binary. + + The probe therefore validates the exact string the build will execute + through ``cmd.exe``; probing the verbatim path instead would pass even + when the stripped path is unusable (``CreateProcess`` accepts + extended-length paths, ``cmd.exe`` does not). + """ + CORE.build_path = setup_core / "build" / "test" + assert not sys.executable.startswith("\\\\?\\") + + with ( + patch.dict(os.environ, {}, clear=False), + patch.object( + toolchain.shutil, "which", return_value="\\\\?\\" + sys.executable + ), + ): + os.environ.pop("ESPHOME_CCACHE_ENABLE", None) + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == sys.executable + + +@_WINDOWS_ONLY +@pytest.mark.parametrize( + ("prefix", "expect_ok"), + [ + pytest.param("", True, id="stripped-path-compiles"), + pytest.param("\\\\?\\", False, id="verbatim-path-fails"), + ], +) +def test_ccache_wrapper_through_cmd_exe( + tmp_path: Path, prefix: str, expect_ok: bool +) -> None: + r"""End to end through ``cmd.exe``: the exported path works, a ``\\?\`` one does not. + + The interpreter stands in for ccache; the spawn mirrors SCons on Windows. + The failing case is the mechanism behind #18399 ("The system cannot find + the path specified." on every compile step); should it ever start passing, + ``cmd.exe`` learned extended-length paths and the strip is no longer needed. + """ + marker = tmp_path / "compiled.txt" + scons_env, _ = _load_ccache_script( + {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": prefix + sys.executable}, + original_spawn=_scons_win32_spawn, + ) + assert scons_env["SPAWN"] is not _scons_win32_spawn + + rc = _spawn_fake_compile_via_cmd_exe(scons_env, marker) + assert (rc == 0) is expect_ok + assert marker.exists() is expect_ok + if expect_ok: + assert marker.read_text() == "compiled" + + @pytest.mark.parametrize( ("platform", "input_path", "expected"), [ From 6084314cc9c029b4b6b131a92665d98d4046e464 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:37:10 -0500 Subject: [PATCH 144/160] [vscode] Report the origin of an unexpected exception during validation (#18494) --- esphome/vscode.py | 20 +++++++++- tests/unit_tests/test_vscode.py | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/esphome/vscode.py b/esphome/vscode.py index f404f02f00..ba7b4e727b 100644 --- a/esphome/vscode.py +++ b/esphome/vscode.py @@ -3,12 +3,14 @@ from __future__ import annotations from io import StringIO import json from pathlib import Path +import sys +import traceback from typing import Any from esphome.config import Config, _format_vol_invalid, validate_config import esphome.config_validation as cv from esphome.const import __version__ as ESPHOME_VERSION -from esphome.core import CORE, DocumentRange +from esphome.core import CORE, DocumentRange, EsphomeError from esphome.yaml_util import parse_yaml @@ -97,6 +99,16 @@ def _ace_loader(fname: Path) -> dict[str, Any]: return parse_yaml(fname, raw_yaml_stream) +def _format_unexpected_error(err: Exception) -> str: + """Describe a crash inside validation with the frame it came from.""" + message = f"Unexpected error while validating: {type(err).__name__}: {err}" + frames = traceback.extract_tb(err.__traceback__) + if not frames: + return message + frame = frames[-1] + return f"{message} ({frame.filename}:{frame.lineno} in {frame.name})" + + def _print_version(): """Print ESPHome version.""" print( @@ -134,8 +146,12 @@ def read_config(args): try: config = loader(file_name) res = validate_config(config, command_line_substitutions) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + except (EsphomeError, cv.Invalid) as err: vs.add_yaml_error(str(err)) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + # stdout carries the JSON protocol; the full chain goes to stderr. + traceback.print_exc(file=sys.stderr) + vs.add_yaml_error(_format_unexpected_error(err)) else: for err in res.errors: try: diff --git a/tests/unit_tests/test_vscode.py b/tests/unit_tests/test_vscode.py index 63bdf3e255..9b7d1e9504 100644 --- a/tests/unit_tests/test_vscode.py +++ b/tests/unit_tests/test_vscode.py @@ -3,6 +3,8 @@ from pathlib import Path from unittest.mock import Mock, patch from esphome import vscode +import esphome.config_validation as cv +from esphome.core import EsphomeError def _run_repl_test(input_data): @@ -126,3 +128,67 @@ packages: assert range["start_col"] == 2 assert range["end_line"] == 1 assert range["end_col"] == 7 + + +def _explode(*_args: object, **_kwargs: object) -> None: + raise AttributeError("'NoneType' object has no attribute 'get'") + + +def test_unexpected_error_reports_origin() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", _explode): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["validation_errors"] == [] + (error,) = result["yaml_errors"] + assert error["message"].startswith( + "Unexpected error while validating: AttributeError: " + "'NoneType' object has no attribute 'get' (" + ) + assert "test_vscode.py" in error["message"] + assert error["message"].endswith(" in _explode)") + + +def test_esphome_error_stays_plain() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", side_effect=EsphomeError("boom")): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["yaml_errors"] == [{"message": "boom"}] + + +def test_invalid_stays_plain() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", side_effect=cv.Invalid("bad value")): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["yaml_errors"] == [{"message": "bad value"}] + + +def test_format_unexpected_error_without_traceback() -> None: + message = vscode._format_unexpected_error(ValueError("boom")) + assert message == "Unexpected error while validating: ValueError: boom" From b768e2a1ce796f7055f9fcba8e8b3494798ce8fa Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:25:09 -0400 Subject: [PATCH 145/160] [esp32] Fix ESP32-P4 bootloop on rev3 (v3.x) chips when only variant is set (#18500) --- esphome/components/esp32/__init__.py | 78 +++++++++++++++------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index ada6d25db5..2c06ebac9a 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1070,6 +1070,26 @@ def _parse_pio_platform_version(value): return value +def _normalize_p4_engineering_sample(value: ConfigType) -> bool: + """Fill in CONF_ENGINEERING_SAMPLE when unset, warning that production + silicon (rev3) is assumed. Returns the normalized flag.""" + if (engineering_sample := value.get(CONF_ENGINEERING_SAMPLE)) is None: + _LOGGER.warning( + "Defaulting to ESP32-P4 production silicon (rev3).\n" + "If you have an early engineering sample (pre-rev3), add this to your config:\n" + "\n" + " esp32:\n" + " engineering_sample: true\n" + "\n" + "To check your chip revision, look for 'chip revision: vX.Y' in the boot log.\n" + "Engineering samples will show a revision below v3.0.\n" + "The 'debug:' component also reports the revision (e.g. Revision: 100 = v1.0, 300 = v3.0)." + ) + engineering_sample = False + value[CONF_ENGINEERING_SAMPLE] = engineering_sample + return engineering_sample + + def _detect_variant(value): board = value.get(CONF_BOARD) variant = value.get(CONF_VARIANT) @@ -1082,6 +1102,8 @@ def _detect_variant(value): # name rather than carrying a PIO board name through the IDF build. if CORE.using_toolchain_esp_idf: value = value.copy() + if variant == VARIANT_ESP32P4: + _normalize_p4_engineering_sample(value) value[CONF_BOARD] = VARIANT_FRIENDLY[variant].lower() return value if variant not in STANDARD_BOARDS: @@ -1092,22 +1114,8 @@ def _detect_variant(value): ) value = value.copy() value[CONF_BOARD] = STANDARD_BOARDS[variant] - if variant == VARIANT_ESP32P4: - engineering_sample = value.get(CONF_ENGINEERING_SAMPLE) - if engineering_sample is None: - _LOGGER.warning( - "No board specified for ESP32-P4. Defaulting to production silicon (rev3).\n" - "If you have an early engineering sample (pre-rev3), add this to your config:\n" - "\n" - " esp32:\n" - " engineering_sample: true\n" - "\n" - "To check your chip revision, look for 'chip revision: vX.Y' in the boot log.\n" - "Engineering samples will show a revision below v3.0.\n" - "The 'debug:' component also reports the revision (e.g. Revision: 100 = v1.0, 300 = v3.0)." - ) - elif engineering_sample: - value[CONF_BOARD] = "esp32-p4-evboard" + if variant == VARIANT_ESP32P4 and _normalize_p4_engineering_sample(value): + value[CONF_BOARD] = "esp32-p4-evboard" elif board in BOARDS: variant = variant or BOARDS[board][KEY_VARIANT] if variant != BOARDS[board][KEY_VARIANT]: @@ -1117,6 +1125,14 @@ def _detect_variant(value): ) value = value.copy() value[CONF_VARIANT] = variant + if variant == VARIANT_ESP32P4: + board_is_es = BOARDS[board].get("engineering_sample", False) + engineering_sample = value.setdefault(CONF_ENGINEERING_SAMPLE, board_is_es) + if engineering_sample != board_is_es: + raise cv.Invalid( + f"'{CONF_ENGINEERING_SAMPLE}' does not match board '{board}'", + path=[CONF_ENGINEERING_SAMPLE], + ) elif not variant: raise cv.Invalid( "This board is unknown, if you are sure you want to compile with this board selection, " @@ -1128,6 +1144,9 @@ def _detect_variant(value): "This board is unknown; the specified variant '%s' will be used but this may not work as expected.", variant, ) + if variant == VARIANT_ESP32P4: + value = value.copy() + _normalize_p4_engineering_sample(value) return value @@ -1431,20 +1450,6 @@ def final_validate(config): path=[CONF_ENGINEERING_SAMPLE], ) ) - if ( - config[CONF_VARIANT] == VARIANT_ESP32P4 - and config.get(CONF_ENGINEERING_SAMPLE) is not None - ): - board_is_es = BOARDS.get(config[CONF_BOARD], {}).get( - "engineering_sample", False - ) - if config[CONF_ENGINEERING_SAMPLE] != board_is_es: - errs.append( - cv.Invalid( - f"'{CONF_ENGINEERING_SAMPLE}' does not match board '{config[CONF_BOARD]}'", - path=[CONF_ENGINEERING_SAMPLE], - ) - ) if advanced[CONF_EXECUTE_FROM_PSRAM]: if config[CONF_VARIANT] not in {VARIANT_ESP32S3, VARIANT_ESP32P4}: errs.append( @@ -2517,15 +2522,14 @@ async def to_code(config): f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_frequency[:-3]}M", True ) - # ESP32-P4: ESP-IDF 5.5.3 changed the default of ESP32P4_SELECTS_REV_LESS_V3 - # from y to n. PlatformIO uses sections.ld.in (for rev <3) or - # sections.rev3.ld.in (for rev >=3) based on board definition. - # Set the sdkconfig option to match the board's chip revision. + # ESP32-P4: pre-v3 and rev3 (v3.0+) silicon are not binary compatible. + # CONFIG_ESP32P4_SELECTS_REV_LESS_V3 selects which layout ESP-IDF links; + # validation normalizes CONF_ENGINEERING_SAMPLE from the board when unset. if variant == VARIANT_ESP32P4: - is_eng_sample = BOARDS.get(config[CONF_BOARD], {}).get( - "engineering_sample", False + add_idf_sdkconfig_option( + "CONFIG_ESP32P4_SELECTS_REV_LESS_V3", + config.get(CONF_ENGINEERING_SAMPLE, False), ) - add_idf_sdkconfig_option("CONFIG_ESP32P4_SELECTS_REV_LESS_V3", is_eng_sample) # Set minimum chip revision for ESP32 variant # Setting this to 3.0 or higher reduces flash size by excluding workaround code, From 7418fcce8d8f154bceb088f1ad10782c6dadb4ca Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:38:18 -0400 Subject: [PATCH 146/160] [ci] Stop persisting the integration test ccache (#18504) --- .github/workflows/ci.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2075fde9ef..0d8f35ed83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -341,18 +341,6 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -y --no-install-recommends ccache - - name: Restore ccache (restore-only) - # esphome stores the PlatformIO ccache under the machine-global cache - # dir (see _ccache_env() in esphome/platformio/toolchain.py). The - # bucket-name prefix prefers a same-bucket seed; the bare prefix falls - # back to any seed when the bucket layout differs from dev. - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/esphome/platformio-ccache - key: integration-ccache-${{ matrix.bucket.name }}-${{ github.sha }} - restore-keys: | - integration-ccache-${{ matrix.bucket.name }}- - integration-ccache- - name: Set up Python 3.13 id: python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -401,14 +389,6 @@ jobs: # esphome stores the PlatformIO ccache under the machine-global cache # dir (see _ccache_env() in esphome/platformio/toolchain.py). run: CCACHE_DIR="$HOME/.cache/esphome/platformio-ccache" ccache -s - - name: Save ccache - # Pull request saves land in per-PR scopes nothing else can reuse; - # dev pushes seed the shared copy instead. - if: github.event_name != 'pull_request' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/esphome/platformio-ccache - key: integration-ccache-${{ matrix.bucket.name }}-${{ github.sha }} import-time: name: Check import esphome.__main__ time From e9e77d02a00d6d9b8f0661b0e4c4a025b4f697b9 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:57:51 -0500 Subject: [PATCH 147/160] Bump bundled esphome-device-builder to 1.11.3 (#18505) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4a8daeaaf6..50b698224c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.3 RUN \ platformio settings set enable_telemetry No \ From 74e22b5ad74308fbed86738bf63fbcaed9f0fd03 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:24:23 -0500 Subject: [PATCH 148/160] Bump bundled esphome-device-builder to 1.11.4 (#18506) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 50b698224c..5c21e07618 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.4 RUN \ platformio settings set enable_telemetry No \ From 2c92a2498e5fb5632554f485eedd3446155d7e83 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:42:05 -0500 Subject: [PATCH 149/160] Bump bundled esphome-device-builder to 1.11.5 (#18507) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5c21e07618..1be10db3af 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.5 RUN \ platformio settings set enable_telemetry No \ From 4a85c98285c1a2c38b2e5e9115fb4793b5b1d69f Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:21:58 -0500 Subject: [PATCH 150/160] Bump bundled esphome-device-builder to 1.12.0 (#18514) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1be10db3af..18f705b501 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.12.0 RUN \ platformio settings set enable_telemetry No \ From b3fda9973ebd67fcafb26b4f3b7a831427ecafff Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:30:12 -0700 Subject: [PATCH 151/160] [image] Restore defaults:/files: support for platform entries (#18032) Co-authored-by: Claude Sonnet 5 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/components/file/image.py | 3 +- esphome/components/image/__init__.py | 152 +++++++- esphome/components/runtime_image/__init__.py | 5 +- esphome/config.py | 17 + esphome/loader.py | 8 + tests/component_tests/image/test_init.py | 328 +++++++++++++++++- .../validate-platform-defaults.host.yaml | 21 ++ .../validate-platform-defaults.host.yaml | 24 ++ tests/unit_tests/test_config_normalization.py | 124 ++++++- 9 files changed, 657 insertions(+), 25 deletions(-) create mode 100644 tests/components/animation/validate-platform-defaults.host.yaml create mode 100644 tests/components/image/validate-platform-defaults.host.yaml diff --git a/esphome/components/file/image.py b/esphome/components/file/image.py index b54c3f2adf..212c778763 100644 --- a/esphome/components/file/image.py +++ b/esphome/components/file/image.py @@ -23,6 +23,7 @@ from esphome.components.image import ( get_image_type_enum, get_transparency_enum, is_svg_file, + validate_byte_order, validate_settings, validate_transparency, validate_type, @@ -200,7 +201,7 @@ OPTIONS_SCHEMA = { "NONE", "FLOYDSTEINBERG", upper=True ), cv.Optional(CONF_INVERT_ALPHA, default=False): cv.boolean, - cv.Optional(CONF_BYTE_ORDER): cv.one_of("BIG_ENDIAN", "LITTLE_ENDIAN", upper=True), + cv.Optional(CONF_BYTE_ORDER): validate_byte_order, cv.Optional(CONF_TRANSPARENCY, default=CONF_OPAQUE): validate_transparency(), } diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 37a9afb84d..eaee31a1c7 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -10,7 +10,14 @@ from PIL import Image, UnidentifiedImageError import esphome.codegen as cg from esphome.components.const import CONF_BYTE_ORDER, KEY_METADATA import esphome.config_validation as cv -from esphome.const import CONF_DEFAULTS, CONF_FILE, CONF_ID, CONF_PLATFORM, CONF_TYPE +from esphome.const import ( + CONF_DEFAULTS, + CONF_FILE, + CONF_FILES, + CONF_ID, + CONF_PLATFORM, + CONF_TYPE, +) from esphome.core import CORE from esphome.types import ConfigType @@ -48,6 +55,9 @@ TRANSPARENCY_TYPES = ( CONF_ALPHA_CHANNEL, ) +# Shared validator for the image platform schemas and `_drop_incompatible_byte_order`. +validate_byte_order = cv.one_of("BIG_ENDIAN", "LITTLE_ENDIAN", upper=True) + def get_image_type_enum(type): return getattr(ImageType, f"IMAGE_TYPE_{type.upper()}") @@ -404,6 +414,120 @@ def get_image_metadata(image_id: str) -> ImageMetaData | None: return get_all_image_metadata().get(image_id) +# --------------------------------------------------------------------------- +# `defaults:`/`files:` expansion: a `platform:` entry merges shared `defaults:` +# into every `files:` entry; the platform's CONFIG_SCHEMA validates each. +# Permanent, unlike the legacy migration below. +# --------------------------------------------------------------------------- + + +def _drop_incompatible_byte_order( + merged: dict, explicit: dict, *, index: int | None = None +) -> dict: + """Drop `byte_order` when the resolved type doesn't support it, unless written directly on `explicit`. + + With `index`, inherited values are validated before being dropped (the legacy flattener always drops). + """ + if CONF_BYTE_ORDER in explicit: + return merged + type_class = IMAGE_TYPE.get(str(merged.get(CONF_TYPE, "")).upper()) + if ( + CONF_BYTE_ORDER in merged + and isinstance(type_class, type) + and issubclass(type_class, ImageEncoder) + and not type_class.is_endian() + ): + if index is not None: + try: + validate_byte_order(merged[CONF_BYTE_ORDER]) + except cv.Invalid as exc: + exc.prepend([index]) + raise + del merged[CONF_BYTE_ORDER] + return merged + + +def _expand_platform_entry(index: int, entry: dict) -> list[dict]: + if CONF_FILES not in entry: + if CONF_DEFAULTS in entry: + raise cv.Invalid( + f"'{CONF_DEFAULTS}' may only be used together with '{CONF_FILES}'", + path=[index], + ) + return [entry] + + extra_keys = set(entry) - {CONF_PLATFORM, CONF_DEFAULTS, CONF_FILES} + if extra_keys: + raise cv.Invalid( + f"'{CONF_FILES}' cannot be combined with " + f"{', '.join(sorted(extra_keys))} on the same entry", + path=[index], + ) + + files = entry[CONF_FILES] + if files is None: + raise cv.Invalid(f"'{CONF_FILES}' must not be empty", path=[index]) + if not isinstance(files, list): + raise cv.Invalid(f"'{CONF_FILES}' must be a list", path=[index]) + if not files: + raise cv.Invalid(f"'{CONF_FILES}' must not be empty", path=[index]) + + defaults = entry.get(CONF_DEFAULTS, {}) + if defaults is None: + defaults = {} + if not isinstance(defaults, dict): + raise cv.Invalid(f"'{CONF_DEFAULTS}' must be a mapping", path=[index]) + # Neither `id:` nor `platform:` makes sense inside `defaults:`. + for disallowed in (CONF_ID, CONF_PLATFORM): + if disallowed in defaults: + raise cv.Invalid( + f"'{disallowed}' is not allowed inside '{CONF_DEFAULTS}'", + path=[index], + ) + + from esphome import yaml_util + + platform = entry[CONF_PLATFORM] + result: list[dict] = [] + for file_entry in files: + if not isinstance(file_entry, dict): + raise cv.Invalid( + f"each entry in '{CONF_FILES}' must be a mapping", path=[index] + ) + # The platform is chosen by the entry's own `platform:` key, not per file. + if CONF_PLATFORM in file_entry: + raise cv.Invalid( + f"'{CONF_PLATFORM}' is not allowed inside '{CONF_FILES}'", + path=[index], + ) + # Keep the `files:` item's source range so whole-entry errors anchor there; + # `make_data_base` needs a real ESPHomeDataBase, so skip it for plain dicts. + source = ( + file_entry if isinstance(file_entry, yaml_util.ESPHomeDataBase) else None + ) + merged = yaml_util.make_data_base( + {CONF_PLATFORM: platform, **defaults, **file_entry}, source + ) + result.append(_drop_incompatible_byte_order(merged, file_entry, index=index)) + return result + + +def expand_platform_config(config: list) -> list: + """Expand `defaults:`/`files:` entries; the platform's own CONFIG_SCHEMA validates each result.""" + result = [] + for i, entry in enumerate(config): + if isinstance(entry, dict) and CONF_PLATFORM in entry: + result.extend(_expand_platform_entry(i, entry)) + else: + result.append(entry) + return result + + +EXPAND_PLATFORM_CONFIG = expand_platform_config + +# --------------------- end defaults/files expansion ------------------------- + + # --------------------------------------------------------------------------- # Legacy top-level component -> `image:` platform deprecation helpers # -- REMOVE after 2027.1.0 together with the `animation:`/`online_image:` shims. @@ -496,11 +620,17 @@ def _is_legacy_image_format(config: object) -> bool: proper error instead of the migration silently dropping the input. """ if isinstance(config, list): - # A bare list of (not-yet-platform-tagged) image dicts. + # Exclude `files:` entries -- the list branch would otherwise silently + # migrate them to `platform: file` instead of raising the missing-platform error. return bool(config) and all( - isinstance(entry, dict) and CONF_PLATFORM not in entry for entry in config + isinstance(entry, dict) + and CONF_PLATFORM not in entry + and CONF_FILES not in entry + for entry in config ) - if not isinstance(config, dict): + if not isinstance(config, dict) or CONF_PLATFORM in config or CONF_FILES in config: + # `platform:`/`files:` dicts are new-format (left for list-wrapping + + # expansion); the legacy flattener has no `files:` branch and would drop them. return False # A single image dict, or the grouped `defaults:`/`images:`/type-key form. return ( @@ -532,18 +662,8 @@ def _flatten_legacy_image_config(config: object) -> list[dict]: def _add(entry: dict, extra: dict) -> None: merged = {**defaults, **extra, **entry} - # The legacy `defaults:`/type-grouped forms only applied `byte_order` to - # types that support it. Replicate that so an endian default merged into - # e.g. a binary image stays valid. - type_class = IMAGE_TYPE.get(str(merged.get(CONF_TYPE, "")).upper()) - if ( - CONF_BYTE_ORDER in merged - and isinstance(type_class, type) - and issubclass(type_class, ImageEncoder) - and not type_class.is_endian() - ): - del merged[CONF_BYTE_ORDER] - result.append(merged) + # Always drop, matching the pre-platform behavior -- see `_drop_incompatible_byte_order`. + result.append(_drop_incompatible_byte_order(merged, {})) def _add_entries(entries: object, extra: dict) -> None: # `entries` may be a single image dict or a list of them; non-dict diff --git a/esphome/components/runtime_image/__init__.py b/esphome/components/runtime_image/__init__.py index d8517d4493..9fa32a5a65 100644 --- a/esphome/components/runtime_image/__init__.py +++ b/esphome/components/runtime_image/__init__.py @@ -5,6 +5,7 @@ from esphome.components.const import CONF_BYTE_ORDER from esphome.components.image import ( IMAGE_TYPE, Image_, + validate_byte_order, validate_settings, validate_transparency, validate_type, @@ -128,9 +129,7 @@ def runtime_image_schema(image_class: cg.MockObjClass = RuntimeImage) -> cv.Sche cv.Required(CONF_FORMAT): cv.one_of(*IMAGE_FORMATS, upper=True), cv.Optional(CONF_RESIZE): cv.dimensions, cv.Required(CONF_TYPE): validate_type(IMAGE_TYPE), - cv.Optional(CONF_BYTE_ORDER): cv.one_of( - "BIG_ENDIAN", "LITTLE_ENDIAN", upper=True - ), + cv.Optional(CONF_BYTE_ORDER): validate_byte_order, cv.Optional(CONF_TRANSPARENCY, default="OPAQUE"): validate_transparency(), cv.Optional(CONF_PLACEHOLDER): cv.use_id(Image_), } diff --git a/esphome/config.py b/esphome/config.py index 987bb9c96a..13ec744ce4 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -620,6 +620,23 @@ class LoadValidationStep(ConfigValidationStep): elif not isinstance(self.conf, list): result[self.domain] = self.conf = [self.conf] + # Permanent expansion hook: a platform-tagged entry may expand into + # several (e.g. `image`'s `defaults:`/`files:`), for `platform:`-tagged dicts only. + if (expand := component.expand_platform_config) is not None and all( + isinstance(entry, dict) and CONF_PLATFORM in entry + for entry in self.conf + ): + with result.catch_error(path): + expanded = expand(self.conf) + if not isinstance(expanded, list): + # A non-list return is a component bug (not a user error): + # raise explicitly (survives -O/-OO) so it escapes catch_error. + raise TypeError( + f"{self.domain}: EXPAND_PLATFORM_CONFIG must " + f"return a list, got {type(expanded).__name__}" + ) + result[self.domain] = self.conf = expanded + # Process AUTO_LOAD _process_auto_load(result, component, path) diff --git a/esphome/loader.py b/esphome/loader.py index f994f0c5eb..23c6d1bfa5 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -164,6 +164,14 @@ class ComponentManifest: """ return getattr(self.module, "LEGACY_CONFIG_MIGRATE", None) + @property + def expand_platform_config( + self, + ) -> Callable[[list[ConfigType]], list[ConfigType]] | None: + """Optional `EXPAND_PLATFORM_CONFIG` callable; runs on the normalized `platform:`-tagged + entry list before per-entry CONFIG_SCHEMA. Must return a list (raise `cv.Invalid` for user errors).""" + return getattr(self.module, "EXPAND_PLATFORM_CONFIG", None) + @property def resources(self) -> list[FileResource]: """Return a list of all file resources defined in the package of this component. diff --git a/tests/component_tests/image/test_init.py b/tests/component_tests/image/test_init.py index 78462463b1..846c152cab 100644 --- a/tests/component_tests/image/test_init.py +++ b/tests/component_tests/image/test_init.py @@ -21,16 +21,20 @@ from esphome.components.image import ( CONF_OPAQUE, CONF_TRANSPARENCY, PLATFORM_FILE, + _expand_platform_entry, _flatten_legacy_image_config, _is_legacy_image_format, _is_new_image_format, _migrate_legacy_image_config, + expand_platform_config, get_all_image_metadata, get_image_metadata, ) from esphome.const import ( + CONF_DEFAULTS, CONF_DITHER, CONF_FILE, + CONF_FILES, CONF_ID, CONF_PLATFORM, CONF_RAW_DATA_ID, @@ -259,6 +263,15 @@ def test_flatten_keeps_byte_order_for_endian_type() -> None: assert out[0][CONF_BYTE_ORDER] == "little_endian" +def test_flatten_drops_byte_order_written_directly_on_legacy_entry() -> None: + """The legacy flattener drops an incompatible byte_order even when written directly on the entry.""" + out = _flatten_legacy_image_config( + {"binary": [{"id": "a", "file": "x.png", "byte_order": "little_endian"}]} + ) + assert out == [{"id": "a", "file": "x.png", "type": "binary"}] + assert CONF_BYTE_ORDER not in out[0] + + def test_flatten_skips_meta_and_unknown_keys() -> None: out = _flatten_legacy_image_config( { @@ -342,6 +355,42 @@ def test_migrate_legacy_warns_and_prepends_platform( ), pytest.param({"foo": 1}, False, id="dict_unknown_keys"), pytest.param("a string", False, id="scalar"), + # A `platform:`-tagged dict is the new format written without list brackets. + pytest.param( + {CONF_PLATFORM: "file", "id": "a", "file": "x.png"}, + False, + id="platform_tagged_flat_dict", + ), + pytest.param( + { + CONF_PLATFORM: "file", + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + }, + False, + id="platform_tagged_defaults_files_dict", + ), + # `files:` without `platform:` is not legacy either -- the flattener has no branch for it. + pytest.param( + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + }, + False, + id="defaults_files_dict_without_platform", + ), + # Same as above in a list -- without this exclusion it would be silently + # migrated to a hard-coded `platform: file` instead of raising the error. + pytest.param( + [ + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + } + ], + False, + id="defaults_files_list_entry_without_platform", + ), ], ) def test_is_legacy_image_format(config: object, expected: bool) -> None: @@ -359,17 +408,290 @@ def test_is_legacy_image_format(config: object, expected: bool) -> None: def test_migrate_returns_none_for_invalid_legacy_shapes( config: object, caplog: pytest.LogCaptureFixture ) -> None: - """Unrecognised shapes are not migrated (and emit no warning) so normal - platform validation surfaces a proper error instead of silently dropping - the offending input.""" + """Unrecognised shapes are not migrated (and emit no warning), so normal platform validation reports them.""" with caplog.at_level(logging.WARNING): assert _migrate_legacy_image_config(config) is None assert "deprecated" not in caplog.text +def test_migrate_returns_none_for_mapping_form_defaults_files() -> None: + """A `platform:`-tagged `defaults:`/`files:` mapping must not be swallowed by the legacy migrator.""" + config = { + CONF_PLATFORM: "file", + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + assert _migrate_legacy_image_config(config) is None + + +def test_migrate_returns_none_for_defaults_files_dict_without_platform() -> None: + """`defaults:`/`files:` without `platform:` must not be swallowed either -- the flattener has + no `files:` branch and would silently return `[]`.""" + config = { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + assert _migrate_legacy_image_config(config) is None + + +def test_migrate_returns_none_for_defaults_files_list_entry_without_platform() -> None: + """Same, in a list -- previously the list branch migrated it to a hard-coded + `platform: file` instead of raising a missing-platform error.""" + config = [ + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + ] + assert _migrate_legacy_image_config(config) is None + + # --------------------------- end legacy migration -------------------------- +def test_expand_platform_entry_passes_through_plain_entry() -> None: + entry = {CONF_PLATFORM: "file", "id": "a", "file": "x.png"} + assert _expand_platform_entry(0, entry) == [entry] + + +def test_expand_platform_entry_expands_files_with_defaults() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565", "transparency": "opaque"}, + CONF_FILES: [ + {"id": "img1", "file": "foo.png"}, + {"id": "img2", "file": "bar.png", "type": "GRAYSCALE"}, + ], + } + assert _expand_platform_entry(0, entry) == [ + { + CONF_PLATFORM: "file", + "id": "img1", + "file": "foo.png", + "type": "RGB565", + "transparency": "opaque", + }, + { + CONF_PLATFORM: "file", + "id": "img2", + "file": "bar.png", + "type": "GRAYSCALE", + "transparency": "opaque", + }, + ] + + +def test_expand_platform_entry_files_without_defaults() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "img1", "file": "foo.png"}], + } + assert _expand_platform_entry(0, entry) == [ + {CONF_PLATFORM: "file", "id": "img1", "file": "foo.png"} + ] + + +def test_expand_platform_entry_preserves_source_range() -> None: + """A merged entry keeps the source range of its `files:` item so whole-entry errors anchor there.""" + from esphome import yaml_util + + file_entry = yaml_util.make_data_base({"id": "img1", "file": "foo.png"}) + file_entry._esp_range = "sentinel-range" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [file_entry], + } + [out] = _expand_platform_entry(0, entry) + assert isinstance(out, yaml_util.ESPHomeDataBase) + assert out.esp_range == "sentinel-range" + + +def test_expand_platform_entry_plain_dict_file_entry_has_no_source_range() -> None: + """Plain-dict `files:` items must not crash -- `from_database` reads `.esp_range` unconditionally.""" + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "img1", "file": "foo.png"}], + } + [out] = _expand_platform_entry(0, entry) + assert out == {CONF_PLATFORM: "file", "id": "img1", "file": "foo.png"} + + +def test_expand_platform_entry_per_file_overrides_win() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [{"id": "img1", "file": "foo.png", "type": "BINARY"}], + } + [out] = _expand_platform_entry(0, entry) + assert out["type"] == "BINARY" + + +def test_expand_platform_entry_drops_byte_order_for_non_endian_override() -> None: + """A `byte_order` default merged into a non-endian override is dropped, as the legacy flattener did.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "little_endian"}, + CONF_FILES: [ + {"id": "a", "file": "x.png"}, + {"id": "b", "file": "y.png", "type": "binary"}, + ], + } + out = _expand_platform_entry(0, entry) + assert out[0]["byte_order"] == "little_endian" + assert "byte_order" not in out[1] + + +def test_expand_platform_entry_invalid_byte_order_in_defaults_raises() -> None: + """A dropped `byte_order` inherited from `defaults:` is still validated, so a typo raises.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "little_andian"}, + CONF_FILES: [{"id": "a", "file": "x.png", "type": "binary"}], + } + with pytest.raises(cv.Invalid, match="did you mean") as excinfo: + _expand_platform_entry(0, entry) + assert excinfo.value.path == [0] + + +def test_expand_platform_entry_keeps_byte_order_for_endian_override() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "big_endian"}, + CONF_FILES: [{"id": "a", "file": "x.png", "type": "rgb565"}], + } + [out] = _expand_platform_entry(0, entry) + assert out["byte_order"] == "big_endian" + + +def test_expand_platform_entry_keeps_explicit_byte_order_conflict() -> None: + """A `byte_order` written directly on the entry is kept so validate_settings raises the normal error.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565"}, + CONF_FILES: [ + { + "id": "a", + "file": "x.png", + "type": "binary", + "byte_order": "little_endian", + } + ], + } + [out] = _expand_platform_entry(0, entry) + assert out["byte_order"] == "little_endian" + + +def test_expand_platform_entry_defaults_without_files_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_DEFAULTS: {"type": "RGB565"}} + with pytest.raises(cv.Invalid, match="may only be used together with") as excinfo: + _expand_platform_entry(0, entry) + assert excinfo.value.path == [0] + + +def test_expand_platform_entry_null_files_raises_not_empty() -> None: + """A `files:` key with no value parses to `None` and must be reported clearly.""" + entry = {CONF_PLATFORM: "file", CONF_DEFAULTS: {"type": "RGB565"}, CONF_FILES: None} + with pytest.raises(cv.Invalid, match="must not be empty"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_empty_files_list_raises_not_empty() -> None: + """An explicit `files: []` must not silently drop the whole platform entry.""" + entry = {CONF_PLATFORM: "file", CONF_FILES: []} + with pytest.raises(cv.Invalid, match="must not be empty"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_files_with_stray_key_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "a", "file": "x.png"}], + "extra": 1, + } + with pytest.raises(cv.Invalid, match="cannot be combined with"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_id_in_defaults_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {CONF_ID: "a"}, + CONF_FILES: [{"file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_platform_in_defaults_raises() -> None: + """`platform:` inside `defaults:` would silently reassign every file's platform.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {CONF_PLATFORM: "animation"}, + CONF_FILES: [{"id": "a", "file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_platform_in_file_entry_raises() -> None: + """`platform:` on a `files:` item must not silently override the entry's platform.""" + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "a", "file": "x.png", CONF_PLATFORM: "animation"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_files_not_list_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_FILES: "not-a-list"} + with pytest.raises(cv.Invalid, match="must be a list"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_defaults_not_mapping_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: "not-a-mapping", + CONF_FILES: [{"id": "a", "file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="must be a mapping"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_file_item_not_mapping_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_FILES: [1, 2]} + with pytest.raises(cv.Invalid, match="must be a mapping"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_config_mixes_plain_and_expanded_entries() -> None: + config = [ + { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [ + {"id": "img1", "file": "foo.png"}, + {"id": "img2", "file": "bar.png"}, + ], + }, + {CONF_PLATFORM: "file", "id": "plain", "file": "baz.png", "type": "BINARY"}, + ] + out = expand_platform_config(config) + assert [entry["id"] for entry in out] == ["img1", "img2", "plain"] + + +def test_expand_platform_config_ignores_non_platform_entries() -> None: + # Not expanded here -- legacy_config_migrate runs before this hook and is + # responsible for tagging/flattening pre-platform shapes. + config = ["not-a-platform-entry"] + assert expand_platform_config(config) == config + + +# --------------------- end defaults/files expansion ------------------------- + + def test_validate_image_final_defaults_to_little_endian() -> None: out = validate_image_final({CONF_FILE: "x.png"}) assert out[CONF_BYTE_ORDER] == "LITTLE_ENDIAN" diff --git a/tests/components/animation/validate-platform-defaults.host.yaml b/tests/components/animation/validate-platform-defaults.host.yaml new file mode 100644 index 0000000000..034497c548 --- /dev/null +++ b/tests/components/animation/validate-platform-defaults.host.yaml @@ -0,0 +1,21 @@ +# `platform: animation` entry exercising the shared `defaults:`/`files:` expansion. +display: + - platform: sdl + id: animation_display + auto_clear_enabled: false + dimensions: + width: 480 + height: 480 + +image: + - platform: animation + defaults: + type: rgb565 + transparency: opaque + resize: 50x50 + files: + - id: platform_defaults_animation + file: $component_dir/anim.gif + - id: platform_defaults_animation_rgb + file: $component_dir/anim.apng + type: rgb diff --git a/tests/components/image/validate-platform-defaults.host.yaml b/tests/components/image/validate-platform-defaults.host.yaml new file mode 100644 index 0000000000..e1b3037cc3 --- /dev/null +++ b/tests/components/image/validate-platform-defaults.host.yaml @@ -0,0 +1,24 @@ +# `platform: file` entry using the `defaults:`/`files:` shape, including the +# per-type byte_order drop when an entry overrides to a non-endian type. +display: + - platform: sdl + id: image_display + auto_clear_enabled: false + dimensions: + width: 480 + height: 480 + +image: + - platform: file + defaults: + type: rgb565 + transparency: opaque + byte_order: little_endian + resize: 50x50 + dither: FloydSteinberg + files: + - id: platform_defaults_image + file: ../../pnglogo.png + - id: platform_defaults_binary + file: ../../pnglogo.png + type: binary diff --git a/tests/unit_tests/test_config_normalization.py b/tests/unit_tests/test_config_normalization.py index c8b7b63094..04363ad45b 100644 --- a/tests/unit_tests/test_config_normalization.py +++ b/tests/unit_tests/test_config_normalization.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest -from esphome import config, yaml_util +from esphome import config, config_validation as cv, yaml_util from esphome.core import CORE, AutoLoad from esphome.types import ConfigType @@ -127,12 +127,14 @@ def _run_load_step( domain: str, conf: object, migrate: Callable[[ConfigType], list | None] | None, + expand: Callable[[list], list] | None = None, ) -> config.Config: - """Run a LoadValidationStep for a platform component with a given migrate hook.""" + """Run a LoadValidationStep for a platform component with given hooks.""" component = Mock() component.is_platform_component = True component.multi_conf_no_default = False component.legacy_config_migrate = migrate + component.expand_platform_config = expand result = config.Config() with ( @@ -197,6 +199,124 @@ def test_legacy_migrate_skipped_for_autoload() -> None: assert result["image"] == [auto] +# --------------------------------------------------------------------------- +# EXPAND_PLATFORM_CONFIG hook on LoadValidationStep -- permanent counterpart +# to legacy_config_migrate; runs after legacy migration/list normalization. +# --------------------------------------------------------------------------- + + +def test_expand_hook_rewrites_conf() -> None: + """A config the expand hook rewrites is replaced with the expanded list.""" + expanded = [{"platform": "file", "id": "a"}, {"platform": "file", "id": "b"}] + expand = Mock(return_value=expanded) + + result = _run_load_step("image", [{"platform": "file", "id": "a"}], None, expand) + + expand.assert_called_once_with([{"platform": "file", "id": "a"}]) + assert result["image"] == expanded + + +def test_expand_hook_absent_is_noop() -> None: + """A platform component without the hook is left as normalized by the + existing list-wrapping logic.""" + result = _run_load_step("image", [{"platform": "file", "id": "a"}], None, None) + + assert result["image"] == [{"platform": "file", "id": "a"}] + + +def test_expand_hook_runs_after_legacy_migrate() -> None: + """The expand hook sees the already-migrated list, not the raw legacy conf.""" + migrated = [{"platform": "file", "id": "a"}] + migrate = Mock(return_value=migrated) + expand = Mock(side_effect=lambda conf: conf) + + _run_load_step("image", [{"id": "a", "file": "x.png"}], migrate, expand) + + expand.assert_called_once_with(migrated) + + +def test_expand_hook_skipped_for_non_dict_entry() -> None: + """Malformed entries are left alone; the hook only sees `platform:`-tagged dicts.""" + expand = Mock(side_effect=lambda conf: conf) + + result = _run_load_step("image", ["not-a-dict"], None, expand) + + expand.assert_not_called() + assert result["image"] == ["not-a-dict"] + + +def test_expand_hook_skipped_for_entry_missing_platform_key() -> None: + """A dict entry missing the `platform:` key is left alone -- the normal + per-entry error reporting further down catches this case instead.""" + expand = Mock(side_effect=lambda conf: conf) + + result = _run_load_step("image", [{"id": "a"}], None, expand) + + expand.assert_not_called() + assert result["image"] == [{"id": "a"}] + + +def test_expand_hook_skipped_for_autoload() -> None: + """A non-empty AutoLoad reaching the hook stage is left alone.""" + expand = Mock(side_effect=lambda conf: conf) + auto = AutoLoad() + auto["id"] = "a" + + result = _run_load_step("image", auto, None, expand) + + expand.assert_not_called() + assert result["image"] == [auto] + + +def test_expand_hook_runs_when_all_entries_are_platform_tagged_dicts() -> None: + """The guard does not block the normal, well-formed case.""" + expand = Mock(side_effect=lambda conf: conf) + conf = [{"platform": "file", "id": "a"}, {"platform": "animation", "id": "b"}] + + result = _run_load_step("image", conf, None, expand) + + expand.assert_called_once_with(conf) + assert result["image"] == conf + + +def test_expand_hook_invalid_reports_single_error_at_domain_path() -> None: + """A `cv.Invalid` from the hook is reported once with the domain path prepended; no further validation runs.""" + expand = Mock(side_effect=cv.Invalid("bad shape")) + pre_expand_conf = [{"platform": "file", "id": "a"}] + + result = _run_load_step("image", pre_expand_conf, None, expand) + + assert len(result.errors) == 1 + assert result.errors[0].path == ["image"] + assert "bad shape" in str(result.errors[0]) + assert result["image"] == pre_expand_conf + + +def test_expand_hook_final_external_invalid_reports_without_path_prepend() -> None: + """`cv.FinalExternalInvalid` keeps its already-resolved path (no domain path prepended).""" + already_resolved_error = cv.FinalExternalInvalid( + "bad shape", path=["image", 3, "files"] + ) + expand = Mock(side_effect=already_resolved_error) + pre_expand_conf = [{"platform": "file", "id": "a"}] + + result = _run_load_step("image", pre_expand_conf, None, expand) + + assert len(result.errors) == 1 + assert result.errors[0] is already_resolved_error + assert result.errors[0].path == ["image", 3, "files"] + assert result["image"] == pre_expand_conf + + +def test_expand_hook_non_list_return_raises_type_error() -> None: + """A non-list return is a component bug: it escapes as an uncaught TypeError + (explicit raise survives -O/-OO).""" + expand = Mock(return_value={"not": "a list"}) + + with pytest.raises(TypeError, match="must return a list"): + _run_load_step("image", [{"platform": "file", "id": "a"}], None, expand) + + def _write_merge_conflict_config(tmp_path: Path, *, suppress: bool) -> Path: """Create a config where two `<<` includes both define `logger:`. From 78a65eabdc6f33e6ac7f398a905217f61f779b64 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 16:30:34 -0500 Subject: [PATCH 152/160] [ci] Stop jobs hanging on apt by restoring the cached apt action and bounding raw apt calls (#18518) --- .github/workflows/ci-api-proto.yml | 28 +++++++++- .github/workflows/ci.yml | 89 +++++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-api-proto.yml b/.github/workflows/ci-api-proto.yml index 820081cc46..771b4cd94f 100644 --- a/.github/workflows/ci-api-proto.yml +++ b/.github/workflows/ci-api-proto.yml @@ -41,10 +41,32 @@ jobs: version: "0.11.15" - name: Install apt dependencies + # PR-only workflow, so nothing on dev could seed a shared apt cache + # entry; the cached apt action would save one copy per PR. Plain apt + # with every call bounded: the apt.conf.d timeouts make a dead + # mirror fail over in seconds, and timeout runs under sudo so it can + # kill apt-get itself. Install without update first: image lists are + # fresh, and the index refresh is what a congested mirror makes slow. + timeout-minutes: 15 run: | - sudo apt update - sudo apt-cache show protobuf-compiler - sudo apt install -y protobuf-compiler + sudo tee /etc/apt/apt.conf.d/99ci-acquire-timeouts >/dev/null <<'EOF' + Acquire::Retries "1"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + EOF + # Common path: the image's package lists are fresh enough. + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 \ + apt-get install -y protobuf-compiler; then + protoc --version + exit 0 + fi + # Rescue path: refresh the lists once with a generous bound; the + # apt config already fails a stalled mirror over quickly. + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 30 \ + dpkg --configure -a || true + sudo timeout -k 15 300 apt-get update + sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 \ + apt-get install -y protobuf-compiler protoc --version - name: Install python dependencies run: uv pip install --system aioesphomeapi -c requirements.txt -r requirements_dev.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d8f35ed83..d2d4c7a2a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,22 @@ jobs: uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . + seed-apt-cache: + name: Seed apt package cache + runs-on: ubuntu-24.04 + # PR-branch cache saves are invisible to other PRs, so dev/beta/release + # pushes seed the one shared entry PR jobs restore. The key is derived + # only from the package list and version; keep both identical in every + # step that restores it. In ci-status needs so a broken seed fails dev. + if: github.event_name == 'push' + timeout-minutes: 10 + steps: + - name: Install apt packages (cached) + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 + determine-jobs: name: Determine which jobs to run runs-on: ubuntu-24.04 @@ -323,7 +339,8 @@ jobs: integration-tests: name: Run integration tests (${{ matrix.bucket.name }}) - runs-on: ubuntu-latest + # Must match seed-apt-cache's image: the apt cache key has no OS in it. + runs-on: ubuntu-24.04 needs: - common - determine-jobs @@ -335,12 +352,16 @@ jobs: steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install ccache - # Speeds up the host compiles: tests in a bucket compile overlapping - # component sets, so later tests reuse earlier tests' objects. - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends ccache + - name: Install apt packages (cached) + # ccache speeds up the host compiles. A cache hit never touches apt + # (mirror outages cannot hang the job); the timeout bounds the cold + # path. Packages and version must match seed-apt-cache exactly; + # libsdl2-dev is unused here and carried only for cache-key parity. + timeout-minutes: 10 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 - name: Set up Python 3.13 id: python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -421,6 +442,7 @@ jobs: benchmarks: name: Run CodSpeed benchmarks runs-on: ubuntu-24.04 + timeout-minutes: 30 needs: - common - determine-jobs @@ -457,6 +479,41 @@ jobs: fi echo "binary=$BINARY" >> $GITHUB_OUTPUT + - name: Bound apt fetches and pre-install libc6-dbg + # The CodSpeed runner installs valgrind + libc6-dbg via its own + # unbounded apt-get update; per-invocation apt options cannot reach + # it. The apt.conf.d timeouts below bound every later apt call in + # this job, the runner's included. Pre-installing libc6-dbg lets the + # runner skip apt once its valgrind cache is restored (it checks + # ``dpkg -s libc6-dbg``, so the cache action's unregistered restores + # would not count). Install without update first: image lists are + # fresh, and the index refresh is what a congested mirror makes + # slow. Best effort; the job timeout is the last backstop. + timeout-minutes: 15 + continue-on-error: true + run: | + sudo tee /etc/apt/apt.conf.d/99ci-acquire-timeouts >/dev/null <<'EOF' + Acquire::Retries "1"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + EOF + if dpkg -s libc6-dbg >/dev/null 2>&1; then + echo "libc6-dbg already installed" + exit 0 + fi + # Common path: the image's package lists are fresh enough. + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 \ + apt-get install -y libc6-dbg; then + exit 0 + fi + # Rescue path: refresh the lists once with a generous bound; the + # apt config already fails a stalled mirror over quickly. + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 30 \ + dpkg --configure -a || true + sudo timeout -k 15 300 apt-get update + sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 \ + apt-get install -y libc6-dbg + - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3 with: @@ -875,12 +932,17 @@ jobs: - name: List components run: echo ${{ matrix.batch.components }} - - name: Install apt packages - # Not cached: this job is pull-request-only, so a cache save could - # never be shared and would only consume quota. - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends libsdl2-dev ccache + - name: Install apt packages (cached) + # A cache hit (seeded on dev by seed-apt-cache) never touches apt, + # so mirror outages cannot hang this PR-only job; the timeout bounds + # the cold path. Packages and version must match seed-apt-cache + # exactly. The action has no --no-install-recommends; same package + # set this job used before #17463. + timeout-minutes: 10 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1415,6 +1477,7 @@ jobs: # this check. needs: - common + - seed-apt-cache - determine-jobs - ci-custom - pylint From f735dcadc0c38f25eec83cf4f5eba97bc62e30d6 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:33:05 +1200 Subject: [PATCH 153/160] Bump version to 2026.8.0b6 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 3dad4629be..c83d95d0ef 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.0b5 +PROJECT_NUMBER = 2026.8.0b6 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index e86465f9a0..2296f8c0b7 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.0b5" +__version__ = "2026.8.0b6" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From c45599196235345e2f13415eccb537b2d6e13b49 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 11:39:01 -0500 Subject: [PATCH 154/160] [ci] Key PlatformIO cache on the Python version so a runner image bump does not serve a broken LibreTiny venv (#18512) --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2d4c7a2a1..fa119fb6d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -598,24 +598,29 @@ jobs: fetch-depth: 2 - name: Restore Python + id: restore-python uses: ./.github/actions/restore-python with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + # Key on the exact Python version as well: LibreTiny creates a venv under + # ~/.platformio/penv whose interpreter is a symlink into the runner's + # hosted toolcache, so a cache saved on an older runner image breaks once + # a new image ships a newer patch release and drops the old interpreter. - name: Cache platformio if: github.ref == 'refs/heads/dev' && matrix.pio_cache_key uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.platformio - key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} + key: platformio-${{ matrix.pio_cache_key }}-${{ steps.restore-python.outputs.python-version }}-${{ hashFiles('platformio.ini') }} - name: Cache platformio if: github.ref != 'refs/heads/dev' && matrix.pio_cache_key uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.platformio - key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} + key: platformio-${{ matrix.pio_cache_key }}-${{ steps.restore-python.outputs.python-version }}-${{ hashFiles('platformio.ini') }} - name: Cache ESP-IDF install if: matrix.cache_idf From 185f12266a3f9ae0248df1a38ce96f2cc6aae7b0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 17:29:10 -0500 Subject: [PATCH 155/160] [tests] Keep PlatformIO libdeps per xdist worker to stop a compile race (#18524) --- tests/integration/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1bf799b658..483d5392af 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -60,7 +60,11 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: env = os.environ.copy() env["PLATFORMIO_CORE_DIR"] = str(cache_dir) env["PLATFORMIO_CACHE_DIR"] = str(cache_dir / ".cache") - env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps") + # libdeps is keyed only by env name (the device name), and fixtures share + # names; two xdist workers first-compiling the same name race pio pkg + # install in the same directory. Keep libdeps per worker. + worker = os.environ.get("PYTEST_XDIST_WORKER", "master") + env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker) # Prevent cache cleaning during integration tests env["ESPHOME_SKIP_CLEAN_BUILD"] = "1" # Compile with THIS tree's esphome sources, not wherever the venv's editable From d9359a70c1ef82ab907aba6adad45a27cd5d5fab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 17:29:10 -0500 Subject: [PATCH 156/160] [tests] Keep PlatformIO libdeps per xdist worker to stop a compile race (#18524) --- tests/integration/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1bf799b658..483d5392af 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -60,7 +60,11 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: env = os.environ.copy() env["PLATFORMIO_CORE_DIR"] = str(cache_dir) env["PLATFORMIO_CACHE_DIR"] = str(cache_dir / ".cache") - env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps") + # libdeps is keyed only by env name (the device name), and fixtures share + # names; two xdist workers first-compiling the same name race pio pkg + # install in the same directory. Keep libdeps per worker. + worker = os.environ.get("PYTEST_XDIST_WORKER", "master") + env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker) # Prevent cache cleaning during integration tests env["ESPHOME_SKIP_CLEAN_BUILD"] = "1" # Compile with THIS tree's esphome sources, not wherever the venv's editable From 828eac90f36ffe9dd1fa714f41b27377fda2cafd Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:51:59 +1200 Subject: [PATCH 157/160] Bump version to 2026.8.0 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index c83d95d0ef..ed0670621d 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.0b6 +PROJECT_NUMBER = 2026.8.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 2296f8c0b7..17ff1e17d9 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.0b6" +__version__ = "2026.8.0" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From ca97c86d6580746e12e071351bf3d219ef7de51f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:06:55 -0500 Subject: [PATCH 158/160] [ci] Install requirements_dev.txt when the venv cache misses (#18502) --- .github/actions/restore-python/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/restore-python/action.yml b/.github/actions/restore-python/action.yml index daf041819c..fab6dc6ffb 100644 --- a/.github/actions/restore-python/action.yml +++ b/.github/actions/restore-python/action.yml @@ -49,7 +49,7 @@ runs: python -m venv venv source venv/bin/activate python --version - uv pip install -r requirements.txt -r requirements_test.txt + uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' && runner.os == 'Windows' @@ -58,5 +58,5 @@ runs: python -m venv venv source ./venv/Scripts/activate python --version - uv pip install -r requirements.txt -r requirements_test.txt + uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . From b7d0b676fc0cd5f93a772f6f11d398a157ae612d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20H=C3=A4ll?= Date: Thu, 20 Aug 2026 06:31:27 +0200 Subject: [PATCH 159/160] [wifi] Take the lwIP core lock around sntp_servermode_dhcp() (#18511) --- esphome/components/wifi/wifi_component_esp_idf.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 245390b097..24cb060edb 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -580,7 +580,14 @@ bool WiFiComponent::wifi_sta_ip_config_(const optional &manual_ip) { // lwIP starts the SNTP client if it gets an SNTP server from DHCP. We don't need the time, and more importantly, // the built-in SNTP client has a memory leak in certain situations. Disable this feature. // https://github.com/esphome/issues/issues/2299 - sntp_servermode_dhcp(false); + { +#if SNTP_GET_SERVERS_FROM_DHCP || SNTP_GET_SERVERS_FROM_DHCPV6 + // sntp_servermode_dhcp() is an empty macro unless lwIP is built with + // DHCP-supplied NTP servers, so only that build needs the core lock. + LwIPLock lock; +#endif + sntp_servermode_dhcp(false); + } // No manual IP is set; use DHCP client if (dhcp_status != ESP_NETIF_DHCP_STARTED) { From 5e9de7c94bde17b782e643a1d12745f67e23f3d6 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:00:44 -0500 Subject: [PATCH 160/160] Bump bundled esphome-device-builder to 1.12.1 (#18541) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 18f705b501..55aa0ac982 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.12.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.12.1 RUN \ platformio settings set enable_telemetry No \