From ea5832a410ced951c96b17794ce039a0265e6d42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:21:01 +0200 Subject: [PATCH 001/221] Bump pytest-mock from 3.15.1 to 3.16.0 (#19952) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 7f6d75f947..3ec864a340 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -8,7 +8,7 @@ yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests pytest==9.1.1 pytest-cov==7.1.0 -pytest-mock==3.15.1 +pytest-mock==3.16.0 pytest-asyncio==1.4.0 pytest-xdist==3.8.0 asyncmock==0.4.2 From 35474aae3d3bef713acd0977f7f00ae98b3e7480 Mon Sep 17 00:00:00 2001 From: Nate Clark Date: Wed, 30 Sep 2026 16:24:22 -0400 Subject: [PATCH 002/221] [mqtt] Stop disabling the mbedTLS hardware MPI accelerator for TLS (#19958) Co-authored-by: Claude Opus 5.5 (1M context) --- esphome/components/mqtt/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index b6badb4ef9..9418614346 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -3,7 +3,6 @@ import esphome.codegen as cg from esphome.components import logger, socket from esphome.components.esp32 import ( add_idf_component, - add_idf_sdkconfig_option, idf_version, include_builtin_idf_component, ) @@ -452,10 +451,6 @@ async def to_code(config): cg.add(var.set_cl_certificate(config[CONF_CLIENT_CERTIFICATE])) cg.add(var.set_cl_key(config[CONF_CLIENT_CERTIFICATE_KEY])) - # prevent error -0x428e - # See https://github.com/espressif/esp-idf/issues/139 - add_idf_sdkconfig_option("CONFIG_MBEDTLS_HARDWARE_MPI", False) - if CONF_IDF_SEND_ASYNC in config and config[CONF_IDF_SEND_ASYNC]: cg.add_define("USE_MQTT_IDF_ENQUEUE") # end esp-idf From cb47dfcf0e44e0ce060f5851f6d42000f37c3970 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 1 Oct 2026 07:22:41 +1000 Subject: [PATCH 003/221] [light] Fix white flash on turn-on during an effect, start strobe on its first color (#19916) Co-authored-by: Claude Opus 5.5 --- esphome/components/light/base_light_effects.h | 5 + esphome/components/light/light_call.cpp | 33 +++- esphome/components/light/light_call.h | 1 + .../fixtures/light_repeat_effect.yaml | 49 ++++++ tests/integration/test_light_repeat_effect.py | 152 ++++++++++++++++++ 5 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 tests/integration/fixtures/light_repeat_effect.yaml create mode 100644 tests/integration/test_light_repeat_effect.py diff --git a/esphome/components/light/base_light_effects.h b/esphome/components/light/base_light_effects.h index ba3fba6c12..e32dbde952 100644 --- a/esphome/components/light/base_light_effects.h +++ b/esphome/components/light/base_light_effects.h @@ -163,6 +163,11 @@ struct StrobeLightEffectColor { class StrobeLightEffect : public LightEffect { public: explicit StrobeLightEffect(const char *name) : LightEffect(name) {} + void start() override { + // Place the cycle at the end of the last color, so the first apply() switches straight to the first color + this->at_color_ = this->colors_.size() - 1; + this->last_switch_ = millis() - this->colors_.back().duration; + } void apply() override { const uint32_t now = millis(); if (now - this->last_switch_ < this->colors_[this->at_color_].duration) diff --git a/esphome/components/light/light_call.cpp b/esphome/components/light/light_call.cpp index f540d2f31f..7e54fccebc 100644 --- a/esphome/components/light/light_call.cpp +++ b/esphome/components/light/light_call.cpp @@ -156,11 +156,13 @@ void LightCall::perform() { ESP_LOGV(TAG, " Effect: '%.*s'", (int) effect_s.size(), effect_s.c_str()); } - this->parent_->start_effect_(this->effect_); + if (this->effect_ != this->parent_->active_effect_index_) { + this->parent_->start_effect_(this->effect_); - // Also set light color values when starting an effect - // For example to turn off the light - this->parent_->set_immediately_(v, true); + // Also set light color values when starting an effect + // For example to turn off the light + this->parent_->set_immediately_(v, true); + } } else { // INSTANT CHANGE this->parent_->set_immediately_(v, publish); @@ -193,10 +195,9 @@ LightColorValues LightCall::validate_() { auto *name = this->parent_->get_name().c_str(); auto traits = this->parent_->get_traits(); -#ifdef USE_LIGHT_RESUME_EFFECT // Snapshot before the adjustments below add flags of their own + const bool sets_values = (this->flags_ & VALUE_FLAGS_MASK) != 0; const bool plain_turn_on = this->has_state() && this->state_ && (this->flags_ & ~STATE_ONLY_FLAGS_MASK) == 0; -#endif // USE_LIGHT_RESUME_EFFECT // Color mode check if (this->has_color_mode() && !traits.supports_color_mode(this->color_mode_)) { @@ -348,9 +349,25 @@ LightColorValues LightCall::validate_() { } #endif // USE_LIGHT_RESUME_EFFECT - // If effect is already active, remove effect start + // A plain turn-on of a lit light keeps a running effect as it is. + // Effects' own calls don't publish, so they are not caught here. + if (plain_turn_on && this->get_publish_() && this->parent_->remote_values.is_on() && + this->parent_->active_effect_index_ != 0) { + this->effect_ = this->parent_->active_effect_index_; + this->set_flag_(FLAG_HAS_EFFECT); + } + + // If effect is already active, remove effect start. When a lit light that stays on gets no new values or flash, + // keep the flag and drop any transition, which has nothing to fade to; perform() then leaves the running effect + // undisturbed. A call that turns the light off keeps no flag, so the turn-off block below stops the effect. + // has_brightness() catches the brightness added above to make the turn-on visible. if (this->has_effect_() && this->effect_ == this->parent_->active_effect_index_) { - this->clear_flag_(FLAG_HAS_EFFECT); + if (sets_values || this->has_brightness() || this->has_flash_() || this->effect_ == 0 || + !this->parent_->remote_values.is_on() || !v.is_on()) { + this->clear_flag_(FLAG_HAS_EFFECT); + } else { + this->clear_flag_(FLAG_HAS_TRANSITION); + } } // validate effect index diff --git a/esphome/components/light/light_call.h b/esphome/components/light/light_call.h index c9f6af7c91..a226690853 100644 --- a/esphome/components/light/light_call.h +++ b/esphome/components/light/light_call.h @@ -217,6 +217,7 @@ class LightCall { static constexpr uint16_t CLAMP_FLAGS_MASK = 0x00FFu; // bits 0-7 // Flags a plain turn-on may carry; any other flag means the caller asked for something specific static constexpr uint16_t STATE_ONLY_FLAGS_MASK = FLAG_HAS_STATE | FLAG_PUBLISH | FLAG_SAVE; + static constexpr uint16_t VALUE_FLAGS_MASK = CLAMP_FLAGS_MASK | FLAG_HAS_COLOR_TEMPERATURE | FLAG_HAS_COLOR_MODE; inline bool has_transition_() { return (this->flags_ & FLAG_HAS_TRANSITION) != 0; } inline bool has_flash_() { return (this->flags_ & FLAG_HAS_FLASH) != 0; } diff --git a/tests/integration/fixtures/light_repeat_effect.yaml b/tests/integration/fixtures/light_repeat_effect.yaml new file mode 100644 index 0000000000..85dc346f2c --- /dev/null +++ b/tests/integration/fixtures/light_repeat_effect.yaml @@ -0,0 +1,49 @@ +esphome: + name: light-repeat-effect +host: +api: # Port will be automatically injected +logger: + level: DEBUG + +output: + - platform: template + id: red_output + type: float + write_action: + - logger.log: + format: "RED_OUTPUT:%.4f" + args: [state] + - platform: template + id: green_output + type: float + write_action: + - logger.log: + format: "GREEN_OUTPUT:%.4f" + args: [state] + - platform: template + id: blue_output + type: float + write_action: + - logger.log: + format: "BLUE_OUTPUT:%.4f" + args: [state] + +light: + - platform: rgb + name: "Test RGB Light" + id: test_rgb_light + red: red_output + green: green_output + blue: blue_output + effects: + - strobe: + name: "Slow Strobe" + colors: + - red: 100% + green: 0% + blue: 0% + duration: 1s + - red: 0% + green: 0% + blue: 100% + duration: 1s diff --git a/tests/integration/test_light_repeat_effect.py b/tests/integration/test_light_repeat_effect.py new file mode 100644 index 0000000000..e8e4cf8e12 --- /dev/null +++ b/tests/integration/test_light_repeat_effect.py @@ -0,0 +1,152 @@ +"""Integration test for starting a strobe effect and for turn-ons sent while it runs. + +Starting the strobe used to leave the last published colour (white here) on the output until +the first color's duration had passed since boot, then skip to the second color. + +A turn-on naming the effect that is already running, or a plain turn-on of a lit light, used +to get the default transition (or the requested one). That faded the output towards white, which the strobe never +updates, until the next strobe step snapped it back. +""" + +from __future__ import annotations + +import asyncio +import re +from typing import Any + +from aioesphomeapi import EntityState, LightState +import pytest + +from .state_utils import InitialStateHelper +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_light_repeat_effect( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """The strobe starts on its first color, and turn-ons while it runs don't fade to white.""" + output_pattern = re.compile(r"(RED|GREEN|BLUE)_OUTPUT:([\d.]+)") + outputs: dict[str, list[float]] = {"RED": [], "GREEN": [], "BLUE": []} + green = outputs["GREEN"] + blue = outputs["BLUE"] + warnings: list[str] = [] + + def on_log_line(line: str) -> None: + if match := output_pattern.search(line): + outputs[match.group(1)].append(float(match.group(2))) + elif "effect cannot be used with" in line: + warnings.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + light = next(e for e in entities if e.object_id == "test_rgb_light") + + state_futures: dict[int, asyncio.Future[LightState]] = {} + + def on_state(state: EntityState) -> None: + if isinstance(state, LightState) and state.key in state_futures: + future = state_futures[state.key] + if not future.done(): + future.set_result(state) + + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + await initial_state_helper.wait_for_initial_states() + + async def send_and_wait(timeout: float = 5.0, **kwargs: Any) -> LightState: + """Send a light command and wait for the matching state response.""" + state_futures[light.key] = asyncio.get_running_loop().create_future() + client.light_command(key=light.key, **kwargs) + return await asyncio.wait_for(state_futures[light.key], timeout=timeout) + + # Plain turn-on leaves the published colour at the white default + state = await send_and_wait(state=True, transition_length=0.0) + assert state.state is True + + green.clear() + blue.clear() + state = await send_and_wait(state=True, effect="Slow Strobe") + assert state.effect == "Slow Strobe" + await asyncio.sleep(0.3) + + # The first color is red; white shows green, and the second color is blue + assert green and blue, "No output observed after starting the strobe" + assert max(green) == pytest.approx(0.0, abs=0.01), ( + f"Starting the strobe showed white: green={green}" + ) + assert max(blue) == pytest.approx(0.0, abs=0.01), ( + f"Starting the strobe skipped its first color: blue={blue}" + ) + + for description, command in ( + ("Repeating the running effect", {"effect": "Slow Strobe"}), + ( + "Repeating the running effect with a transition", + {"effect": "Slow Strobe", "transition_length": 2.0}, + ), + ("A plain turn-on", {}), + ): + green.clear() + state = await send_and_wait(state=True, **command) + assert state.effect == "Slow Strobe", f"{description} changed the effect" + # Cover the rest of the current strobe step and the start of the next one + await asyncio.sleep(1.0) + + assert green, "No green output observed while the strobe was running" + assert max(green) == pytest.approx(0.0, abs=0.01), ( + f"{description} faded the output towards white: {green}" + ) + + assert not warnings, f"Unexpected warnings: {warnings}" + + # A flash requested with the running effect still flashes, here to white + green.clear() + state = await send_and_wait(state=True, effect="Slow Strobe", flash_length=0.5) + assert not warnings, f"Unexpected warnings: {warnings}" + await asyncio.sleep(0.2) + assert max(green) == pytest.approx(1.0, abs=0.01), ( + f"The flash was dropped: green={green}" + ) + await asyncio.sleep(0.5) + + # Brightness 0 with no state turns the light off but leaves the effect + # running; naming that effect must still turn the light on + state = await send_and_wait(brightness=0.0) + assert state.state is False + state = await send_and_wait(state=True, effect="Slow Strobe") + assert state.state is True, "Turn-on with the running effect was dropped" + assert state.brightness == pytest.approx(1.0) + assert state.effect == "Slow Strobe" + + # A lit light at brightness 0 is made visible by a turn-on naming the effect + state = await send_and_wait(state=True, brightness=0.0) + assert state.state is True + assert state.brightness == pytest.approx(0.0) + state = await send_and_wait(state=True, effect="Slow Strobe") + assert state.brightness == pytest.approx(1.0), ( + "Turn-on with the running effect did not make the light visible" + ) + assert state.effect == "Slow Strobe" + + # Turning the light off while naming the running effect must stop the effect, not just + # publish the light as off and leave the effect driving the outputs + warnings.clear() + state = await send_and_wait(state=False, effect="Slow Strobe") + assert state.state is False + # Let the default turn-off transition finish, then watch a full strobe cycle + await asyncio.sleep(1.5) + for values in outputs.values(): + values.clear() + await asyncio.sleep(2.2) + assert max((v for values in outputs.values() for v in values), default=0.0) == ( + pytest.approx(0.0, abs=0.01) + ), f"The effect kept driving the outputs after turn-off: {outputs}" + assert not warnings, f"Unexpected warnings: {warnings}" + + client.light_command(key=light.key, effect="None") From 568c1ae494a00d3e49c5cd3d0b69d01aaf0c6027 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:45:50 +0200 Subject: [PATCH 004/221] [zigbee] Implement switch support for esp32 (#17579) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/zigbee/__init__.py | 92 ++++++++++++------ esphome/components/zigbee/const_esp32.py | 4 + .../zigbee/zigbee_attribute_esp32.h | 22 +++++ esphome/components/zigbee/zigbee_ep_esp32.py | 49 ++++++++++ esphome/components/zigbee/zigbee_esp32.cpp | 57 +++++++++++ esphome/components/zigbee/zigbee_esp32.h | 18 +++- esphome/components/zigbee/zigbee_esp32.py | 20 ++++ .../components/zigbee/zigbee_event_esp32.h | 97 +++++++++++++++++++ .../components/zigbee/zigbee_helpers_esp32.c | 14 +++ esphome/components/zigbee/zigbee_zephyr.py | 6 +- tests/components/zigbee/common_esp32.yaml | 11 +++ 11 files changed, 349 insertions(+), 41 deletions(-) create mode 100644 esphome/components/zigbee/zigbee_event_esp32.h diff --git a/esphome/components/zigbee/__init__.py b/esphome/components/zigbee/__init__.py index e9fef59f35..02f05ef8a2 100644 --- a/esphome/components/zigbee/__init__.py +++ b/esphome/components/zigbee/__init__.py @@ -43,6 +43,7 @@ from .zigbee_esp32 import ( final_validate_esp32, validate_binary_sensor_esp32, validate_sensor_esp32, + validate_switch_esp32, zigbee_require_vfs_select, ) from .zigbee_zephyr import ( @@ -67,34 +68,58 @@ def _check_report_deprecation(value: str) -> str: return value -BASE_SCHEMA = cv.Schema( - { - cv.Optional(CONF_REPORT): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - _check_report_deprecation, - cv.enum(REPORT, lower=True), - ), - cv.OnlyWith(CONF_CLUSTER, ["esp32", "zigbee"], default="generic"): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - cv.one_of(*["generic", "device_class"], lower=True), - ), - cv.Optional(CONF_ENDPOINT): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - cv.int_range(1, CONF_MAX_EP_NUMBER), - ), - cv.Optional(CONF_USE_DEVICE_TYPE): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - cv.boolean, - ), - } +def _get_base_schema(cluster_options: list[str] | None = None) -> cv.Schema: + schema = cv.Schema( + { + cv.Optional(CONF_REPORT): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + _check_report_deprecation, + cv.enum(REPORT, lower=True), + ), + cv.Optional(CONF_ENDPOINT): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + cv.int_range(1, CONF_MAX_EP_NUMBER), + ), + cv.Optional(CONF_USE_DEVICE_TYPE): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + cv.boolean, + ), + } + ) + if cluster_options: + schema = cv.Schema( + { + cv.OnlyWith( + CONF_CLUSTER, ["esp32", "zigbee"], default=cluster_options[0] + ): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + cv.one_of(*cluster_options, lower=True), + ), + } + ).extend(schema) + return schema + + +# set BASE_SCHEMA for CI and backwards compatibility +BASE_SCHEMA = _get_base_schema() + +BINARY_SENSOR_SCHEMA = ( + cv.Schema({}) + .extend(_get_base_schema(["generic", "device_class"])) + .extend(zephyr_binary_sensor) +) +SENSOR_SCHEMA = ( + cv.Schema({}) + .extend(_get_base_schema(["generic", "device_class"])) + .extend(zephyr_sensor) +) +SWITCH_SCHEMA = ( + cv.Schema({}).extend(_get_base_schema(["generic", "on_off"])).extend(zephyr_switch) ) -BINARY_SENSOR_SCHEMA = cv.Schema({}).extend(BASE_SCHEMA).extend(zephyr_binary_sensor) -SENSOR_SCHEMA = cv.Schema({}).extend(BASE_SCHEMA).extend(zephyr_sensor) -SWITCH_SCHEMA = cv.Schema({}).extend(zephyr_switch) NUMBER_SCHEMA = cv.Schema({}).extend(zephyr_number) @@ -226,12 +251,15 @@ async def setup_sensor(entity: cg.MockObj, config: ConfigType) -> None: async def setup_switch(entity: cg.MockObj, config: ConfigType) -> None: - if not config.get(CONF_ZIGBEE_ID) or config.get(CONF_INTERNAL): + if "zigbee" not in CORE.loaded_integrations or config.get(CONF_INTERNAL): return if CORE.using_zephyr: - from .zigbee_zephyr import zephyr_setup_switch - - await zephyr_setup_switch(entity, config) + if not config.get(CONF_ZIGBEE_ID): + return + from .zigbee_zephyr import add_switch + else: + from .zigbee_esp32 import add_component as add_switch + CORE.add_job(add_switch, entity, config) async def setup_number( @@ -284,7 +312,7 @@ def validate_switch(config: ConfigType) -> ConfigType: if "zigbee" not in CORE.loaded_integrations or config.get(CONF_INTERNAL): return config if CORE.is_esp32: - return config + return validate_switch_esp32(config) return consume_endpoint(config) diff --git a/esphome/components/zigbee/const_esp32.py b/esphome/components/zigbee/const_esp32.py index 32ae3e3734..b8122d1441 100644 --- a/esphome/components/zigbee/const_esp32.py +++ b/esphome/components/zigbee/const_esp32.py @@ -14,7 +14,9 @@ KEY_ZIGBEE_EP_NO_NUM = "zigbee_ep_no_num" KEY_ZIGBEE_FIRST_EP_CL = "zigbee_first_ep_cl" DEVICE_ID = { + "ON_OFF_OUTPUT": cg.RawExpression("EZB_ZHA_ON_OFF_OUTPUT_DEVICE_ID"), "RANGE_EXTENDER": cg.RawExpression("EZB_ZHA_RANGE_EXTENDER_DEVICE_ID"), + "MAINS_POWER_OUTLET": cg.RawExpression("EZB_ZHA_MAINS_POWER_OUTLET_DEVICE_ID"), "SIMPLE_SENSOR": cg.RawExpression("EZB_ZHA_SIMPLE_SENSOR_DEVICE_ID"), "LIGHT_SENSOR": cg.RawExpression("EZB_ZHA_LIGHT_SENSOR_DEVICE_ID"), "OCCUPANCY_SENSOR": cg.RawExpression("EZB_ZHA_OCCUPANCY_SENSOR_DEVICE_ID"), @@ -26,8 +28,10 @@ DEVICE_ID = { cluster_id = cg.esphome_ns.enum("ezb_zcl_cluster_id_e") CLUSTER_ID = { "BASIC": cluster_id.EZB_ZCL_CLUSTER_ID_BASIC, + "ON_OFF": cluster_id.EZB_ZCL_CLUSTER_ID_ON_OFF, "TIME": cluster_id.EZB_ZCL_CLUSTER_ID_TIME, "BINARY_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_BINARY_INPUT, + "BINARY_OUTPUT": cluster_id.EZB_ZCL_CLUSTER_ID_BINARY_OUTPUT, "ANALOG_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_ANALOG_INPUT, "ILLUMINANCE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT, "TEMPERATURE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT, diff --git a/esphome/components/zigbee/zigbee_attribute_esp32.h b/esphome/components/zigbee/zigbee_attribute_esp32.h index 47f460406c..2680fc2203 100644 --- a/esphome/components/zigbee/zigbee_attribute_esp32.h +++ b/esphome/components/zigbee/zigbee_attribute_esp32.h @@ -20,6 +20,9 @@ #ifdef USE_BINARY_SENSOR #include "esphome/components/binary_sensor/binary_sensor.h" #endif +#ifdef USE_SWITCH +#include "esphome/components/switch/switch.h" +#endif namespace esphome::zigbee { @@ -47,12 +50,23 @@ class ZigbeeAttribute final : public Component { template void set_attr(const T &value); uint8_t attr_type() { return attr_type_; } void set_report(ZigbeeReportT report); + + template void add_on_value_callback(F &&callback) { on_value_callback_.add(std::forward(callback)); } + void on_value(ezb_zcl_attribute_t attribute) { + if (attribute.data.type == this->attr_type() && attribute.data.value) { + this->on_value_callback_.call(attribute); + } + } + #ifdef USE_SENSOR template void connect(sensor::Sensor *sensor); template void connect(sensor::Sensor *sensor, F &&f); #endif #ifdef USE_BINARY_SENSOR template void connect(binary_sensor::BinarySensor *sensor); +#endif +#ifdef USE_SWITCH + template void connect(switch_::Switch *device); #endif bool report_enabled = false; @@ -71,6 +85,7 @@ class ZigbeeAttribute final : public Component { bool set_attr_requested_{false}; bool report_requested_{false}; bool force_report_{false}; + LazyCallbackManager on_value_callback_{}; template T scale_value_(float value); template T invalid_value_(); }; @@ -135,6 +150,13 @@ template void ZigbeeAttribute::connect(binary_sensor::BinarySensor * sensor->add_on_state_callback([this](bool value) { this->set_attr((T) (this->scale_ * value)); }); } #endif +#ifdef USE_SWITCH +template void ZigbeeAttribute::connect(switch_::Switch *device) { + this->add_on_value_callback( + [device](ezb_zcl_attribute_t attribute) { device->control(*(T *) attribute.data.value); }); + device->add_on_state_callback([this](bool value) { this->set_attr((T) (this->scale_ * value)); }); +} +#endif } // namespace esphome::zigbee diff --git a/esphome/components/zigbee/zigbee_ep_esp32.py b/esphome/components/zigbee/zigbee_ep_esp32.py index c45f580045..b0f3d17bd6 100644 --- a/esphome/components/zigbee/zigbee_ep_esp32.py +++ b/esphome/components/zigbee/zigbee_ep_esp32.py @@ -108,6 +108,35 @@ BINARY_INPUT_EP = { ], } +BINARY_OUTPUT_EP = { + CONF_CLUSTERS: [ + { + CONF_ID: "BINARY_OUTPUT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x55, + CONF_TYPE: "BOOL", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + }, + { + CONF_ATTRIBUTE_ID: 0x51, + CONF_TYPE: "BOOL", + }, + { + CONF_ATTRIBUTE_ID: 0x6F, + CONF_TYPE: "MAP8", + }, + { + CONF_ATTRIBUTE_ID: 0x1C, + CONF_TYPE: "STRING", + }, + ], + }, + ], +} + def _pressure_ep(device_type: bool = False) -> dict[str, Any]: ep = { @@ -300,6 +329,26 @@ BINARY_SENSOR_EP_CONFIGS: dict[str, dict[str, Any]] = { }, } +SWITCH_EP_CONFIGS: dict[str, dict[str, Any]] = { + "on_off": { + DEVICE_TYPE: "ON_OFF_OUTPUT", + CONF_CLUSTERS: [ + { + CONF_ID: "ON_OFF", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x0, + CONF_TYPE: "BOOL", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + }, + ], + }, + ], + }, +} + def _get_next_ep_num(eps: list[int]) -> int: try: diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index c00b13cffd..c6597ebfd6 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -154,6 +154,31 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { return true; } +void load_zb_event(ZBEvent *event, ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + event->load_set_attr_value_event(info, attribute); +} + +template void enqueue_zb_event(Args... args) { + // Allocate an event from the pool + ZBEvent *event = global_zigbee->zb_event_pool_.allocate(); + if (event == nullptr) { + // No events available - queue is full or we're out of memory + global_zigbee->zb_events_.increment_dropped_count(); + return; + } + + // Load new event data (replaces previous event) + load_zb_event(event, args...); + + // Push the event to the queue + global_zigbee->zb_events_.push(event); + // Push always succeeds because we're the only producer and the pool ensures we never exceed queue size + global_zigbee->enable_loop_soon_any_context(); +} + +// Explicit template instantiations for the friend function +template void enqueue_zb_event(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute); + static void zb_attribute_handler(ezb_zcl_set_attr_value_message_t *message) { ESP_RETURN_ON_FALSE(message, , TAG, "Empty message"); ESP_RETURN_ON_FALSE(message->info.status == EZB_ZCL_STATUS_SUCCESS, , TAG, "Received message: error status(%d)", @@ -161,6 +186,7 @@ static void zb_attribute_handler(ezb_zcl_set_attr_value_message_t *message) { ESP_LOGD(TAG, "ZCL SetAttributeValue message for endpoint(%d) cluster(0x%04x) %s with status(0x%02x)", message->info.dst_ep, message->info.cluster_id, message->info.cluster_role == EZB_ZCL_CLUSTER_SERVER ? "server" : "client", message->info.status); + enqueue_zb_event(message->info, message->in.attribute); } static void zb_action_handler(ezb_zcl_core_action_callback_id_t callback_id, void *message) { @@ -180,6 +206,13 @@ static void zb_action_handler(ezb_zcl_core_action_callback_id_t callback_id, voi } } +void ZigbeeComponent::handle_attribute_(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + auto it = this->attributes_.find({info.dst_ep, info.cluster_id, info.cluster_role, attribute.id}); + if (it != this->attributes_.end()) { + it->second->on_value(attribute); + } +} + void ZigbeeComponent::create_default_cluster(uint8_t endpoint_id, uint16_t device_id) { ezb_af_ep_desc_t ep_desc = esphome_zb_zha_default_ep_desc_create(endpoint_id, device_id, this->basic_cluster_data_.power_source); @@ -349,6 +382,30 @@ void ZigbeeComponent::setup() { } void ZigbeeComponent::loop() { + // Process all pending events + ZBEvent *event = this->zb_events_.pop(); + while (event != nullptr) { + // Handle the event + switch (event->callback_id_) { + case EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID: + this->handle_attribute_(event->event_.set_attr.info, event->event_.set_attr.attribute); + break; + default: + ESP_LOGW(TAG, "Received event with unhandled callback id: 0x%x", static_cast(event->callback_id_)); + break; + } + + // Free the event back to the pool + this->zb_event_pool_.release(event); + // Get the next event + event = this->zb_events_.pop(); + } + // Log dropped events periodically + uint16_t dropped = this->zb_events_.get_and_reset_dropped_count(); + if (dropped > 0) { + ESP_LOGW(TAG, "Dropped %u Zigbee events due to buffer overflow", dropped); + } + if (!this->start_reported_ && this->started_) { this->start_cb_.call(); this->start_reported_ = true; diff --git a/esphome/components/zigbee/zigbee_esp32.h b/esphome/components/zigbee/zigbee_esp32.h index e84ed9817c..093ae95939 100644 --- a/esphome/components/zigbee/zigbee_esp32.h +++ b/esphome/components/zigbee/zigbee_esp32.h @@ -12,11 +12,10 @@ #include "ezbee/zha.h" #include "esphome/core/automation.h" #include "esphome/core/component.h" +#include "esphome/core/lock_free_queue.h" +#include "esphome/core/event_pool.h" #include "zigbee_helpers_esp32.h" - -#ifdef USE_BINARY_SENSOR -#include "esphome/components/binary_sensor/binary_sensor.h" -#endif +#include "zigbee_event_esp32.h" namespace esphome::zigbee { @@ -24,6 +23,7 @@ namespace esphome::zigbee { static const uint16_t ED_KEEP_ALIVE = 3000; /* 3000 millisecond */ static const uint8_t MAX_CHILDREN = 10; static const uint32_t EZB_PRIMARY_CHANNEL_MASK = 0x07FFF800U; /* channels 11-26 */ +static constexpr uint8_t MAX_ZB_QUEUE_SIZE = 32; #define EZB_DEFAULT_RADIO_CONFIG() \ { .radio_mode = ESP_ZIGBEE_RADIO_MODE_NATIVE, } @@ -88,6 +88,7 @@ class ZigbeeComponent final : public Component { template void add_attr_(ZigbeeAttribute *attr, uint8_t endpoint_id, uint16_t cluster_id, uint8_t role, uint16_t attr_id, T *value_p); + void handle_attribute_(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute); // attributes_ will be used during operation in zigbee callbacks to update the attribute values and trigger // automations // key tuple could be replaced by single 64 (48) bit int with bit fields for endpoint, cluster, role and attr_id @@ -95,6 +96,15 @@ class ZigbeeComponent final : public Component { std::vector attr_values_; ezb_af_device_desc_t dev_desc_ = ezb_af_create_device_desc(); CallbackManager join_cb_{}; + template friend void enqueue_zb_event(Args... args); + esphome::LockFreeQueue zb_events_; + // Pool sized to queue capacity (SIZE-1) because LockFreeQueue is a ring + // buffer that holds N-1 elements (one slot distinguishes full from empty). + // This guarantees allocate() returns nullptr before push() can fail, which: + // 1. Prevents leaking a pool slot (the Nth allocate succeeds but push fails) + // 2. Avoids needing release() on the producer path after a failed push(), + // preserving the SPSC contract on the pool's internal free list + esphome::EventPool zb_event_pool_; LazyCallbackManager start_cb_{}; bool start_reported_{false}; std::atomic started_ = false; diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index d5e8ebbf5c..f48bf9ef0b 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -27,6 +27,7 @@ from esphome.const import ( CONF_UNIT_OF_MEASUREMENT, CONF_VALUE, CONF_WIFI, + DEVICE_CLASS_OUTLET, ) from esphome.core import CORE from esphome.coroutine import CoroPriority, coroutine_with_priority @@ -67,8 +68,10 @@ from .const_esp32 import ( from .zigbee_ep_esp32 import ( ANALOG_INPUT_EP, BINARY_INPUT_EP, + BINARY_OUTPUT_EP, BINARY_SENSOR_EP_CONFIGS, SENSOR_EP_CONFIGS, + SWITCH_EP_CONFIGS, add_ep, create_ep, ) @@ -275,6 +278,23 @@ def validate_binary_sensor_esp32(config: ConfigType) -> ConfigType: return config +def validate_switch_esp32(config: ConfigType) -> ConfigType: + if config[CONF_CLUSTER] == "on_off": + ep = copy.deepcopy(SWITCH_EP_CONFIGS["on_off"]) + if config.get(CONF_DEVICE_CLASS) == DEVICE_CLASS_OUTLET: + ep[DEVICE_TYPE] = "MAINS_POWER_OUTLET" + else: + if config.get(CONF_DEVICE_CLASS) == DEVICE_CLASS_OUTLET: + _LOGGER.warning( + "'device_class: outlet' has no effect with 'cluster: generic', " + "use 'cluster: on_off' to expose a MAINS_POWER_OUTLET device type" + ) + ep = copy.deepcopy(BINARY_OUTPUT_EP) + setup_attributes(config, ep[CONF_CLUSTERS]) + add_ep(ep, config.get(CONF_ENDPOINT), config.get(CONF_USE_DEVICE_TYPE)) + return config + + def zigbee_require_vfs_select(config: ConfigType) -> ConfigType: """Register VFS select requirement during config validation.""" # Zigbee uses esp_vfs_eventfd which requires VFS select support diff --git a/esphome/components/zigbee/zigbee_event_esp32.h b/esphome/components/zigbee/zigbee_event_esp32.h new file mode 100644 index 0000000000..b8b3e97b4c --- /dev/null +++ b/esphome/components/zigbee/zigbee_event_esp32.h @@ -0,0 +1,97 @@ +#pragma once + +#include "esphome/core/defines.h" +#ifdef USE_ESP32 +#ifdef USE_ZIGBEE + +#include // for offsetof +#include // for memcpy +#include "esp_zigbee.h" +#include "ezbee/zha.h" + +namespace esphome::zigbee { + +class ZBEvent { + public: + // IMPORTANT: We MUST copy all values because the pointer from ESP-IDF + // is only valid during the callback execution. Since ZB events are processed + // asynchronously in the main loop, we store our own copy inline to ensure + // the data remains valid until the event is processed. + ZBEvent(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) + : event_{}, callback_id_(EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID) { + this->init_set_attr_value_data_(info, attribute); + } + + // Destructor to clean up heap allocations + ~ZBEvent() { this->release(); } + + // Default constructor for pre-allocation in pool + ZBEvent() : event_{}, callback_id_(EZB_ZCL_CORE_CB_ID_END) {} + + // Invoked on return to EventPool - clean up any heap-allocated data + void release() { + // Free any allocated memory within the event + switch (this->callback_id_) { + case EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID: + if (!this->event_.set_attr.is_inline && this->event_.set_attr.data.heap_data != nullptr) { + delete[] this->event_.set_attr.data.heap_data; + this->event_.set_attr.data.heap_data = nullptr; + } + break; + default: + break; + } + } + + // Load new event data for reuse (replaces previous event data) + // Note: release() is NOT called here because EventPool::release() already + // calls event->release() before returning to the free list. Every event + // from allocate() is already in a clean state. + void load_set_attr_value_event(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + this->callback_id_ = EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID; + this->init_set_attr_value_data_(info, attribute); + } + + // Disable copy to prevent double-delete + ZBEvent(const ZBEvent &) = delete; + ZBEvent &operator=(const ZBEvent &) = delete; + + union { + // NOLINTNEXTLINE(readability-identifier-naming) + struct set_attr_event { + ezb_zcl_message_info_t info; + ezb_zcl_attribute_t attribute; + union { + uint8_t *heap_data; + uint8_t inline_data[4]; // For small data types (<= 32 bit) + } data; + bool is_inline; + } set_attr; + } event_; + + ezb_zcl_core_action_callback_id_t callback_id_; + + private: + void init_set_attr_value_data_(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + this->event_.set_attr.info = info; + this->event_.set_attr.attribute = attribute; + // get attribute.data.value with correct type + if (attribute.data.value != nullptr) { + // Copy the attribute value to avoid dangling pointer issues + size_t value_size = ezb_zcl_get_attr_value_size(attribute.data.type, attribute.data.value); + if (value_size > 4) { + this->event_.set_attr.data.heap_data = new uint8_t[value_size]; + memcpy(this->event_.set_attr.data.heap_data, attribute.data.value, value_size); + this->event_.set_attr.attribute.data.value = this->event_.set_attr.data.heap_data; + this->event_.set_attr.is_inline = false; + } else { + memcpy(this->event_.set_attr.data.inline_data, attribute.data.value, value_size); + this->event_.set_attr.attribute.data.value = this->event_.set_attr.data.inline_data; + this->event_.set_attr.is_inline = true; + } + } + } +}; +} // namespace esphome::zigbee +#endif // USE_ZIGBEE +#endif // USE_ESP32 diff --git a/esphome/components/zigbee/zigbee_helpers_esp32.c b/esphome/components/zigbee/zigbee_helpers_esp32.c index 53f2c31e8d..22f014f3ef 100644 --- a/esphome/components/zigbee/zigbee_helpers_esp32.c +++ b/esphome/components/zigbee/zigbee_helpers_esp32.c @@ -27,6 +27,12 @@ ezb_err_t esphome_zb_add_or_update_cluster(uint16_t cluster_id, ezb_af_ep_desc_t ezb_af_ep_desc_t esphome_zb_zha_default_ep_desc_create(uint8_t ep_id, uint16_t device_id, uint8_t power_source) { ezb_af_ep_desc_t ep_desc; switch (device_id) { + case EZB_ZHA_MAINS_POWER_OUTLET_DEVICE_ID: { + ezb_zha_mains_power_outlet_config_t config = EZB_ZHA_MAINS_POWER_OUTLET_CONFIG(); + config.basic_cfg.power_source = power_source; + ep_desc = ezb_zha_create_mains_power_outlet(ep_id, &config); + break; + } case EZB_ZHA_LIGHT_SENSOR_DEVICE_ID: { ezb_zha_light_sensor_config_t config = EZB_ZHA_LIGHT_SENSOR_CONFIG(); config.basic_cfg.power_source = power_source; @@ -60,12 +66,16 @@ ezb_zcl_cluster_desc_t esphome_zb_default_cluster_dscr_create(uint16_t cluster_i return ezb_zcl_basic_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_IDENTIFY: return ezb_zcl_identify_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_ON_OFF: + return ezb_zcl_on_off_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_TIME: return ezb_zcl_time_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_ANALOG_INPUT: return ezb_zcl_analog_input_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: return ezb_zcl_binary_input_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_BINARY_OUTPUT: + return ezb_zcl_binary_output_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT: return ezb_zcl_illuminance_measurement_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT: @@ -97,12 +107,16 @@ ezb_err_t esphome_zb_cluster_add_attr(uint16_t cluster_id, ezb_zcl_cluster_desc_ return ezb_zcl_basic_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_IDENTIFY: return ezb_zcl_identify_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_ON_OFF: + return ezb_zcl_on_off_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_TIME: return ezb_zcl_time_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_ANALOG_INPUT: return ezb_zcl_analog_input_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: return ezb_zcl_binary_input_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_BINARY_OUTPUT: + return ezb_zcl_binary_output_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT: return ezb_zcl_illuminance_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT: diff --git a/esphome/components/zigbee/zigbee_zephyr.py b/esphome/components/zigbee/zigbee_zephyr.py index 588305da30..3f863140e6 100644 --- a/esphome/components/zigbee/zigbee_zephyr.py +++ b/esphome/components/zigbee/zigbee_zephyr.py @@ -314,10 +314,6 @@ async def _ctx_to_code(config: ConfigType) -> None: cg.add(cg.RawExpression("ZB_AF_REGISTER_DEVICE_CTX(&zb_device_ctx)")) -async def zephyr_setup_switch(entity: cg.MockObj, config: ConfigType) -> None: - CORE.add_job(_add_switch, entity, config) - - async def zephyr_setup_number( entity: cg.MockObj, config: ConfigType, @@ -425,7 +421,7 @@ async def add_sensor(entity: cg.MockObj, config: ConfigType) -> None: ) -async def _add_switch(entity: cg.MockObj, config: ConfigType) -> None: +async def add_switch(entity: cg.MockObj, config: ConfigType) -> None: await _add_zigbee_ep( entity, config, diff --git a/tests/components/zigbee/common_esp32.yaml b/tests/components/zigbee/common_esp32.yaml index 706da1f10c..3d19180c06 100644 --- a/tests/components/zigbee/common_esp32.yaml +++ b/tests/components/zigbee/common_esp32.yaml @@ -45,6 +45,17 @@ sensor: device_class: volume_flow_rate cluster: device_class +switch: + - platform: template + name: "Template Switch 1" + optimistic: true + cluster: on_off + - platform: template + name: "Template Switch 2" + optimistic: true + cluster: on_off + device_class: outlet + zigbee: model: zigbee_test router: false From 6f5d7475f01013080d05ec4fec58603503a19dc7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 00:50:40 +0200 Subject: [PATCH 005/221] [esp8266] Derive the native image header flash size from the linker script (#19961) --- esphome/build_gen/arduino8266.py | 31 +++++++++- esphome/core/config.py | 14 +++-- .../unit_tests/build_gen/test_arduino8266.py | 56 ++++++++++++++++++- tests/unit_tests/core/test_config.py | 13 +++++ 4 files changed, 106 insertions(+), 8 deletions(-) diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index fd198b2e0c..5b14af5a1d 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -1065,10 +1065,10 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: " rspfile_content = $in_newline", " description = LINK $out", "rule elf2bin", - # --flash_size deliberately stays board-derived, as under - # PlatformIO (which reads upload.maximum_size, not the ldscript). + # --flash_size follows PlatformIO: ldscript filename first, then + # upload.maximum_size, so the header always matches the layout. # -W: the framework's own elf2bin.py trips SyntaxWarning on 3.12+. - f" command = $python -W ignore::SyntaxWarning {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {flash_mode} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_flash_size_str(BOARDS[board][KEY_FLASH_SIZE])} --path {_q(toolchain_bin)} --out $out", + f" command = $python -W ignore::SyntaxWarning {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {flash_mode} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_elf2bin_flash_size(board, flash_ld_name)} --path {_q(toolchain_bin)} --out $out", " description = BIN $out", "rule copy", " command = $python $buildtool copy $in $out", @@ -1189,3 +1189,28 @@ def _flash_size_str(flash_size: int) -> str: """Flash size argument for elf2bin (e.g. ``4M``, ``512K``).""" mb = 1024 * 1024 return f"{flash_size // mb}M" if flash_size >= mb else f"{flash_size // 1024}K" + + +# Same pattern PlatformIO's _get_flash_size applies to the ldscript path +_LD_FLASH_SIZE_RE = re.compile(r"\.flash\.(\d+[mk]).*\.ld") +# The framework elf2bin.py's --flash_size choices +_ELF2BIN_FLASH_SIZES = frozenset({"256K", "512K", "1M", "2M", "4M", "8M", "16M"}) + + +def _elf2bin_flash_size(board: str, flash_ld_name: str) -> str: + """Image-header flash size as PlatformIO derives it: ldscript filename, + else board_upload.maximum_size, else the board table. The SDK clamps the + chip to the header size at boot, so a header smaller than the linked + layout breaks OTA writes on the running device.""" + if match := _LD_FLASH_SIZE_RE.search(flash_ld_name): + token = match.group(1) + base = 1024 if token[-1] == "k" else 1024 * 1024 + return _flash_size_str(int(token[:-1]) * base) + if max_size := _pio_option("board_upload.maximum_size", ""): + if ( + not max_size.isdigit() + or (size := _flash_size_str(int(max_size))) not in _ELF2BIN_FLASH_SIZES + ): + raise EsphomeError(f"Invalid board_upload.maximum_size value {max_size!r}") + return size + return _flash_size_str(BOARDS[board][KEY_FLASH_SIZE]) diff --git a/esphome/core/config.py b/esphome/core/config.py index bb3f03d366..281d430410 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -549,7 +549,9 @@ def _add_library_str(lib: str) -> None: # platformio_options keys the native ESP8266 Arduino generator (a later PR # in this chain) will honor; its ignored-option warning will consume the same # list so the two cannot drift -NATIVE_ARDUINO_PIO_OPTIONS = frozenset({"board_build.f_cpu", "board_build.ldscript"}) +NATIVE_ARDUINO_PIO_OPTIONS = frozenset( + {"board_build.f_cpu", "board_build.ldscript", "board_upload.maximum_size"} +) # The full set that survives into CORE.platformio_options under the native # arduino toolchain: lib_ignore is the only specially-translated key below # that is stored rather than translated away. Consumed by the esp8266 native @@ -609,9 +611,13 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No # through to the ignored-option warning). Other native # toolchains have no equivalent and fall through too. cg.add_platformio_option(key, vals[-1]) - elif key != "upload_speed": - # upload_speed needs no handling: it is read from the raw - # config at upload time (upload_using_esptool) + elif key != "upload_speed" and not ( + key == "board_upload.flash_size" and CORE.using_toolchain_arduino + ): + # upload_speed is read from the raw config at upload time. + # board_upload.flash_size is dropped silently on arduino: + # PlatformIO's esp8266 builder never reads it either, and + # published configs (Athom) commonly set it. _LOGGER.warning( "esphome->platformio_options->%s is ignored when building with " "the native '%s' toolchain", diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index 4a0355fd01..066746d36d 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -21,12 +21,18 @@ from esphome.arduino8266.framework import InstalledPaths, toolchain_tool from esphome.build_gen import arduino8266 from esphome.build_gen.arduino8266 import ( _defines_flags, + _elf2bin_flash_size, _flag_defines, _flash_size_str, _resolve_build_config, get_flash_ld_path, ) -from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD +from esphome.components.esp8266.boards import ( + BOARDS, + ESP8266_BOARD_BUILD, + KEY_FLASH_SIZE, + board_ld_script, +) from esphome.components.esp8266.build_surgery import RATETABLE_RULE from esphome.components.esp8266.const import KEY_BOARD, KEY_ESP8266, KEY_SCANF_FLOAT import esphome.config_validation as cv @@ -747,6 +753,54 @@ def test_flash_size_str() -> None: assert _flash_size_str(512 * 1024) == "512K" +def test_elf2bin_flash_size() -> None: + """The image-header size follows the ldscript filename like PlatformIO, + falling back to board_upload.maximum_size and then the board table.""" + assert _elf2bin_flash_size("esp8285", "eagle.flash.2m.ld") == "2M" + assert _elf2bin_flash_size("esp01", "eagle.flash.512k.ld") == "512K" + assert _elf2bin_flash_size("nodemcuv2", "eagle.flash.4m1m.ld") == "4M" + # The testing-mode prefix still matches (search, not match) + assert _elf2bin_flash_size("esp8285", "testing_eagle.flash.2m.ld") == "2M" + # Custom ldscript name: board_upload.maximum_size wins over the board + CORE.platformio_options["board_upload.maximum_size"] = "2097152" + assert _elf2bin_flash_size("esp8285", "custom.ld") == "2M" + del CORE.platformio_options["board_upload.maximum_size"] + assert _elf2bin_flash_size("esp8285", "custom.ld") == "1M" + + +@pytest.mark.parametrize("bad", ["2MB", "3145728", "-1"]) +def test_elf2bin_flash_size_rejects_bad_maximum_size(bad: str) -> None: + """A non-numeric or unsupported board_upload.maximum_size fails by name + instead of a ValueError or a late elf2bin choices error.""" + CORE.platformio_options["board_upload.maximum_size"] = bad + with pytest.raises(EsphomeError, match="board_upload.maximum_size"): + _elf2bin_flash_size("esp8285", "custom.ld") + + +def test_elf2bin_flash_size_default_matches_board_table() -> None: + """Without an ldscript override, every board's own ldscript parses to + the board-table size, so the emitted --flash_size is unchanged.""" + for board, entry in BOARDS.items(): + assert _elf2bin_flash_size(board, board_ld_script(entry)) == _flash_size_str( + entry[KEY_FLASH_SIZE] + ), board + + +def test_write_project_flash_size_follows_ldscript_override( + tmp_path: Path, +) -> None: + """An ldscript overriding the board's flash size drives the image header + too (the Athom shape: esp8285 with eagle.flash.2m.ld). A 1M header over + a 2M layout clamps the chip below the OTA scratch area and bricks OTA.""" + paths = _make_framework(tmp_path) + (paths.framework / "variants" / "esp8285").mkdir() + CORE.data[KEY_ESP8266][KEY_BOARD] = "esp8285" + CORE.platformio_options["board_build.ldscript"] = "eagle.flash.2m.ld" + content = _write_ninja(paths) + assert "--flash_size 2M" in content + assert "eagle.flash.2m.ld" in content + + def test_write_project_testing_mode(tmp_path: Path) -> None: paths = _make_framework(tmp_path) CORE.testing_mode = True diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 189c024b3a..cfb9651e6b 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1341,6 +1341,8 @@ async def test_add_platformio_options_native_idf( "lib_ignore": "libsodium", "upload_speed": "115200", "board_build.f_flash": "80000000L", + # Silently dropped on arduino only; warns here + "board_upload.flash_size": "2MB", } ) @@ -1351,6 +1353,9 @@ async def test_add_platformio_options_native_idf( # nothing else lands in platformio_options on the native toolchain. assert CORE.platformio_options == {"lib_ignore": ["libsodium"]} assert "esphome->platformio_options->board_build.f_flash is ignored" in caplog.text + assert ( + "esphome->platformio_options->board_upload.flash_size is ignored" in caplog.text + ) assert "upload_speed" not in caplog.text # build_flags has a first-class esphome equivalent, so it is deprecated. # lib_deps/lib_ignore are kept as valid platformio_options (no warning). @@ -1464,13 +1469,21 @@ async def test_add_platformio_options_native_arduino( "board_build.ldscript": ["eagle.flash.2m.ld", "eagle.flash.4m2m.ld"], "board_build.filesystem": "littlefs", "upload_speed": "115200", + # The Athom shape: maximum_size is the elf2bin fallback, + # flash_size is dropped silently (PlatformIO never reads it) + "board_upload.maximum_size": "2097152", + "board_upload.flash_size": "2MB", } ) assert CORE.platformio_options["board_build.f_cpu"] == "160000000L" assert CORE.platformio_options["board_build.ldscript"] == "eagle.flash.4m2m.ld" + assert CORE.platformio_options["board_upload.maximum_size"] == "2097152" + assert "board_upload.flash_size" not in CORE.platformio_options assert "board_build.f_cpu is ignored" not in caplog.text assert "board_build.ldscript is ignored" not in caplog.text + assert "board_upload.maximum_size is ignored" not in caplog.text + assert "board_upload.flash_size is ignored" not in caplog.text assert ( "esphome->platformio_options->board_build.filesystem is ignored" in caplog.text ) From 94ed17815c64d4469a1b801aa7b2d652788d810d Mon Sep 17 00:00:00 2001 From: mtmatteo79 <74114415+mtmatteo79@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:55:39 +0200 Subject: [PATCH 006/221] [bthome_mithermometer] Decode 0.1C temperature and 1% humidity objects (#19956) --- .../bthome_mithermometer/bthome_ble.cpp | 15 +++++ .../sensor/bthome_objects_test.cpp | 66 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp diff --git a/esphome/components/bthome_mithermometer/bthome_ble.cpp b/esphome/components/bthome_mithermometer/bthome_ble.cpp index 1ebabea0a3..51f5d866a4 100644 --- a/esphome/components/bthome_mithermometer/bthome_ble.cpp +++ b/esphome/components/bthome_mithermometer/bthome_ble.cpp @@ -422,6 +422,21 @@ bool BTHomeMiThermometer::handle_service_data_(const ble_device_base::ServiceDat } break; } + case 0x45: { // temperature (0.1C), e.g. Shelly BLU H&T + if (this->temperature_ != nullptr) { + const int16_t raw = encode_uint16(value[1], value[0]); + this->temperature_->publish_state(raw * 0.1f); + reported = true; + } + break; + } + case 0x2E: { // humidity (uint8, 1%), e.g. Shelly BLU H&T + if (this->humidity_ != nullptr) { + this->humidity_->publish_state(value[0]); + reported = true; + } + break; + } default: break; } diff --git a/tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp b/tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp new file mode 100644 index 0000000000..e3903212be --- /dev/null +++ b/tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp @@ -0,0 +1,66 @@ +#include + +#include +#include +#include + +#include "esphome/components/bthome_mithermometer/bthome_ble.h" + +namespace esphome::bthome_mithermometer::testing { + +namespace { + +constexpr uint64_t SENSOR_ADDRESS = 0xA4C1384E1678ULL; + +// Unencrypted BTHome v2 service data (UUID 0xFCD2) sent from SENSOR_ADDRESS. +ble_device_base::ESPBTDevice advert(std::initializer_list service_data) { + std::vector adv = {static_cast(service_data.size() + 3), 0x16, 0xD2, 0xFC}; + adv.insert(adv.end(), service_data.begin(), service_data.end()); + uint8_t mac[6]; + for (size_t i = 0; i < 6; i++) + mac[i] = static_cast(SENSOR_ADDRESS >> (i * 8)); + ble_device_base::ESPBTDevice device; + device.from_scan_result(mac, -60, 0, adv.data(), static_cast(adv.size())); + return device; +} + +struct Harness { + Harness() { + this->thermometer.set_address(SENSOR_ADDRESS); + this->thermometer.set_temperature(&this->temperature); + this->thermometer.set_humidity(&this->humidity); + this->thermometer.set_battery_level(&this->battery_level); + } + + BTHomeMiThermometer thermometer; + sensor::Sensor temperature, humidity, battery_level; +}; + +} // namespace + +// Shelly BLU H&T: packet id, battery 90 %, humidity 55 % (0x2E), temperature 22.5 C (0x45) +TEST(BTHomeMiThermometerObjects, DecodesShellyStyleTemperatureAndHumidity) { + Harness h; + ASSERT_TRUE(h.thermometer.parse_device(advert({0x40, 0x00, 0x01, 0x01, 0x5A, 0x2E, 0x37, 0x45, 0xE1, 0x00}))); + EXPECT_NEAR(h.temperature.state, 22.5f, 0.001f); + EXPECT_FLOAT_EQ(h.humidity.state, 55.0f); + EXPECT_FLOAT_EQ(h.battery_level.state, 90.0f); +} + +TEST(BTHomeMiThermometerObjects, DecodesNegativeTenthDegreeTemperature) { + Harness h; + // -5.3 C = -53 = 0xFFCB + ASSERT_TRUE(h.thermometer.parse_device(advert({0x40, 0x45, 0xCB, 0xFF}))); + EXPECT_NEAR(h.temperature.state, -5.3f, 0.001f); + EXPECT_FALSE(h.humidity.has_state()); +} + +// PVVX firmware: temperature 22.50 C (0x02) and humidity 45.67 % (0x03) keep working +TEST(BTHomeMiThermometerObjects, StillDecodesHundredthDegreeObjects) { + Harness h; + ASSERT_TRUE(h.thermometer.parse_device(advert({0x40, 0x02, 0xCA, 0x08, 0x03, 0xD7, 0x11}))); + EXPECT_NEAR(h.temperature.state, 22.5f, 0.001f); + EXPECT_NEAR(h.humidity.state, 45.67f, 0.001f); +} + +} // namespace esphome::bthome_mithermometer::testing From 5a11f8d7c3b0728e526d0040f6c6ff53b5ea21e2 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:58:06 +0000 Subject: [PATCH 007/221] Bump bundled esphome-device-builder to 1.18.0 (#19968) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4c82b8eba9..e68ea3ee72 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.17.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.18.0 RUN \ platformio settings set enable_telemetry No \ From 3dc608c7721cdc27e9d6957362b2a111aaf56f9f Mon Sep 17 00:00:00 2001 From: Panda <735591750@QQ.COM> Date: Thu, 1 Oct 2026 19:56:32 +0800 Subject: [PATCH 008/221] [core] Parse ESPHOME_SERIAL_LOGGING_RESET as a boolean (#19963) --- esphome/__main__.py | 4 ++-- tests/unit_tests/test_main.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 49fc7a3020..e4213e8edb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2343,7 +2343,7 @@ def parse_args(argv): "-r", action="store_true", help="Reset the device before starting serial logs.", - default=os.getenv("ESPHOME_SERIAL_LOGGING_RESET"), + default=get_bool_env("ESPHOME_SERIAL_LOGGING_RESET"), ) _add_states_args(parser_logs) @@ -2384,7 +2384,7 @@ def parse_args(argv): "-r", action="store_true", help="Reset the device before starting serial logs.", - default=os.getenv("ESPHOME_SERIAL_LOGGING_RESET"), + default=get_bool_env("ESPHOME_SERIAL_LOGGING_RESET"), ) parser_run.add_argument( "--ota-platform", diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 837643417d..633ac1b3e1 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -6437,6 +6437,30 @@ def test_parse_args_logs_states() -> None: assert args.states is True +@pytest.mark.parametrize("command", ["logs", "run"]) +@pytest.mark.parametrize( + ("env_value", "expected"), + [(None, False), ("false", False), ("0", False), ("true", True), ("1", True)], +) +def test_parse_args_serial_logging_reset_env( + monkeypatch: pytest.MonkeyPatch, + command: str, + env_value: str | None, + expected: bool, +) -> None: + """The serial reset environment default must be a boolean for both commands.""" + if env_value is None: + monkeypatch.delenv("ESPHOME_SERIAL_LOGGING_RESET", raising=False) + else: + monkeypatch.setenv("ESPHOME_SERIAL_LOGGING_RESET", env_value) + + args = parse_args(["esphome", command, "device.yaml"]) + assert args.reset is expected + if not expected: + args = parse_args(["esphome", command, "--reset", "device.yaml"]) + assert args.reset is True + + def test_parse_args_argcomplete_only_runs_when_completing() -> None: """Only import and invoke argcomplete when _ARGCOMPLETE is set. From 7e32770c5a927ac73301b8c74503fbdb1ac4c6a6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 14:37:42 +0200 Subject: [PATCH 009/221] [ci] Pin SOURCE_DATE_EPOCH in the idf.py equivalence check (#19970) --- script/check_idf_py_equivalence.py | 4 ++++ tests/script/test_check_idf_py_equivalence.py | 22 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index fa349e223a..ff7c58a5a6 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse import hashlib import json +import os from pathlib import Path import re import subprocess @@ -106,6 +107,9 @@ def check(build_path: Path) -> list[str]: (build_path / "build" / "project_description.json").read_text(encoding="utf-8") ) name, version = _setup_core(build_path, description) + # ESP-IDF's openthread stamps the configure time into its compile flags; + # pin it before the env is cached so both configures get the same value. + os.environ["SOURCE_DATE_EPOCH"] = "0" env = toolchain._get_idf_env(version) python = toolchain._get_idf_tool("python") idf_py = toolchain._get_idf_path(version) / "tools" / "idf.py" diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index f8135772d0..6ded9a3a81 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Iterator import json +import os from pathlib import Path import subprocess import sys @@ -55,22 +56,32 @@ def _run_check( side_effect: Callable[[list[str]], None] = lambda cmd: None, rc: int = 0, esphome_rcs: tuple[int, int] = (0, 0), + envs: list[dict[str, str]] | None = None, ) -> tuple[list[str], list[list[str]]]: - """Run check() with idf.py replaced by ``side_effect``; return problems, calls.""" + """Run check() with idf.py replaced by ``side_effect``; return problems, calls. + + ``envs`` collects the env each idf.py call receives. + """ calls: list[list[str]] = [] def run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess: calls.append(cmd) + if envs is not None: + envs.append(kwargs["env"]) side_effect(cmd) return subprocess.CompletedProcess(cmd, rc, "out\n", "err\n") with ( - patch.object(toolchain, "_get_idf_env", return_value={}), + # Snapshot at call time like the real cached env. + patch.object( + toolchain, "_get_idf_env", side_effect=lambda *_: dict(os.environ) + ), patch.object(toolchain, "_get_idf_tool", return_value="/py"), patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), patch.object(guard.subprocess, "run", side_effect=run), + patch.dict(os.environ), ): return guard.check(tree), calls @@ -86,6 +97,13 @@ def test_check_passes_when_idf_py_changes_nothing(tmp_path: Path) -> None: ] +def test_check_pins_source_date_epoch(tmp_path: Path) -> None: + """ESP-IDF's openthread bakes the configure time into its compile flags.""" + envs: list[dict[str, str]] = [] + _run_check(_make_tree(tmp_path), envs=envs) + assert [env.get("SOURCE_DATE_EPOCH") for env in envs] == ["0", "0"] + + def test_check_reports_changed_files_and_rebuilt_outputs(tmp_path: Path) -> None: tree = _make_tree(tmp_path) build = tree / "build" From 5a3e95b161f887ddc7010737955333e788e24bd9 Mon Sep 17 00:00:00 2001 From: tomaszduda23 Date: Thu, 1 Oct 2026 14:41:18 +0200 Subject: [PATCH 010/221] [nrf52] Add nRF Connect SDK 3.4.0 / Zephyr 4.x support (#19863) Co-authored-by: Claude Sonnet 4.6 --- esphome/components/logger/logger_zephyr.cpp | 11 +- esphome/components/nrf52/__init__.py | 34 +++-- esphome/components/nrf52/framework.py | 91 +++++++++--- tests/unit_tests/components/nrf52/test_pch.py | 2 + .../components/nrf52/test_run_compile_hex.py | 130 ++++++++++++++++++ tests/unit_tests/test_nrf52_framework.py | 96 +++++++++++++ tests/unit_tests/test_nrf52_toolchain.py | 16 +++ 7 files changed, 346 insertions(+), 34 deletions(-) create mode 100644 tests/unit_tests/components/nrf52/test_run_compile_hex.py diff --git a/esphome/components/logger/logger_zephyr.cpp b/esphome/components/logger/logger_zephyr.cpp index 69ef6167da..396728ceaf 100644 --- a/esphome/components/logger/logger_zephyr.cpp +++ b/esphome/components/logger/logger_zephyr.cpp @@ -20,6 +20,13 @@ __attribute__((weak)) void print_coredump() {} namespace esphome::logger { +// Zephyr 3.7 renamed z_arch_esf_t to struct arch_esf; the old name was later removed. +#if KERNEL_VERSION_NUMBER >= 0x030700 +using FatalErrorEsf = ::arch_esf; +#else +using FatalErrorEsf = z_arch_esf_t; +#endif + __attribute__((section(".noinit"))) struct { uint32_t magic; uint32_t reason; @@ -168,7 +175,7 @@ void Logger::dump_crash_() { } } -void k_sys_fatal_error_handler(unsigned int reason, const z_arch_esf_t *esf) { +void k_sys_fatal_error_handler(unsigned int reason, const FatalErrorEsf *esf) { crash_buf.magic = App.get_config_hash(); crash_buf.reason = reason; if (esf) { @@ -192,7 +199,7 @@ void k_sys_fatal_error_handler(unsigned int reason, const z_arch_esf_t *esf) { extern "C" { -void k_sys_fatal_error_handler(unsigned int reason, const z_arch_esf_t *esf) { +void k_sys_fatal_error_handler(unsigned int reason, const esphome::logger::FatalErrorEsf *esf) { esphome::logger::k_sys_fatal_error_handler(reason, esf); } } diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 8e91cff049..5ebf5b747f 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -438,7 +438,8 @@ async def to_code(config: ConfigType) -> None: zephyr_add_prj_conf("SERIAL", False, False) # disable stuff to make image smaller by default - zephyr_add_prj_conf("NCS_BOOT_BANNER", False, False) + if framework_ver >= cv.Version(2, 9, 2): + zephyr_add_prj_conf("NCS_BOOT_BANNER", False, False) zephyr_add_prj_conf("BOOT_BANNER", False, False) zephyr_add_prj_conf("PRINTK", False, False) zephyr_add_prj_conf("CONSOLE", False, False) @@ -495,8 +496,8 @@ def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]: types = [] UF2_PATH = "zephyr/zephyr.uf2" DFU_PATH = "firmware.zip" - HEX_PATH = "zephyr/zephyr.hex" # SDK 2.6.1, only generated when OTA is disabled - HEX_MERGED_PATH = "zephyr/merged.hex" # SDK 2.9.2, always generated + HEX_PATH = "zephyr/zephyr.hex" # SDK 2.6.1 without OTA, SDK 3.4.0+ + HEX_MERGED_PATH = "zephyr/merged.hex" # SDK 2.9.2 to 3.3.x, always generated APP_IMAGE_PATH = "zephyr/app_update.bin" build_dir = Path(storage_json.firmware_bin_path).parent if (build_dir / UF2_PATH).is_file(): @@ -968,6 +969,11 @@ def run_compile(args, config: ConfigType) -> bool: _LOGGER.info("Build inputs changed, cleaning %s", build_dir) rmtree(build_dir) + # SDK 3.4.0+ no longer generates merged.hex; drop one left by an older SDK + # build so it is never packaged or offered for download. + for stale_hex in (build_dir / "merged.hex", build_dir / "zephyr" / "merged.hex"): + stale_hex.unlink(missing_ok=True) + if pch_on: pch.log_pch_in_use() # Zephyr turns ccache on by itself when it is installed @@ -1012,13 +1018,18 @@ def run_compile(args, config: ConfigType) -> bool: west_out = zephyr_dir / "zephyr" _copy_if_exists(west_out / "zephyr.uf2", zephyr_dir / "zephyr.uf2") _copy_if_exists(west_out / "zephyr.signed.bin", zephyr_dir / "app_update.bin") + _copy_if_exists(west_out / "zephyr.hex", zephyr_dir / "zephyr.hex") _copy_if_exists(build_dir / "merged.hex", zephyr_dir / "merged.hex") - # For Adafruit bootloader builds, regenerate the UF2 from merged.hex, - # whose records carry the correct flash addresses. The build's own - # zephyr.uf2 uses the board's default offset, which is wrong in some cases. - merged_hex = zephyr_dir / "merged.hex" - if bootloader in _UF2_FAMILY_IDS and merged_hex.is_file(): + # For Adafruit bootloader builds, regenerate the UF2 from a hex file. + # merged.hex carries the correct flash addresses; SDK 3.4.0+ no longer + # generates it, so use zephyr.hex there. Chosen by version so a merged.hex + # left by an older SDK build is never picked. + if framework_ver >= cv.Version(3, 4, 0): + hex_file = zephyr_dir / "zephyr.hex" + else: + hex_file = zephyr_dir / "merged.hex" + if bootloader in _UF2_FAMILY_IDS and hex_file.is_file(): # Drop the build's own wrong-offset UF2 so it isn't shipped alongside. app_uf2 = west_out / "zephyr.uf2" if app_uf2.is_file(): @@ -1035,12 +1046,12 @@ def run_compile(args, config: ConfigType) -> bool: "-c", "-o", str(zephyr_dir / "zephyr.uf2"), - str(merged_hex), + str(hex_file), ], env=env, stream_output=True, ): - raise EsphomeError("Failed to generate UF2 from merged hex") + raise EsphomeError(f"Failed to generate UF2 from {hex_file.name}") if bootloader in ( BOOTLOADER_ADAFRUIT, @@ -1048,9 +1059,6 @@ def run_compile(args, config: ConfigType) -> bool: BOOTLOADER_ADAFRUIT_NRF52_SD140_V6, BOOTLOADER_ADAFRUIT_NRF52_SD140_V7, ): - # no fallback is needed for adafruit case. merged merged.hex is always generated. - # get_download_types needs fallback for mcuboot (non adafruit) - hex_file = zephyr_dir / "merged.hex" dfu_package = build_dir / "firmware.zip" genpkg_cmd = [ str(paths["python_executable"]), diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 48af47e8ff..58c87619bf 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -28,6 +28,26 @@ _LOGGER = logging.getLogger(__name__) _REQUIREMENTS = Path(__file__).parent / "requirements.txt" TOOLCHAIN_VERSION = "0.17.4" +# Zephyr SDK used by nRF Connect SDK 3.4.0 and newer. +_TOOLCHAIN_VERSION_NCS_3_4_0 = "1.0.1" +_TOOLCHAIN_VERSIONS = (TOOLCHAIN_VERSION, _TOOLCHAIN_VERSION_NCS_3_4_0) + + +def _uses_sdk_ng_1_toolchain() -> bool: + """True when the framework needs Zephyr SDK 1.0+. + + SDK 1.0 moved the GNU toolchain under gnu/ and renamed its archive + to toolchain_gnu_*. + """ + return CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(3, 4, 0) + + +def _get_toolchain_version() -> str: + """Return the Zephyr SDK toolchain version for the current framework.""" + if _uses_sdk_ng_1_toolchain(): + return _TOOLCHAIN_VERSION_NCS_3_4_0 + return TOOLCHAIN_VERSION + # Packages the PlatformIO toolchain's Zephyr build script needs beyond west # (which comes from requirements.txt). Keep the pin in sync with @@ -40,6 +60,20 @@ SDK_NG_TOOLCHAIN_MIRRORS = str_to_lst_of_str( "https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v{VERSION}/toolchain_{sysname}-{machine}_arm-zephyr-eabi.{extension}", ) ) +_SDK_NG_TOOLCHAIN_GNU_MIRRORS = str_to_lst_of_str( + os.environ.get( + "ESPHOME_SDK_NG_TOOLCHAIN_GNU_MIRRORS", + "https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v{VERSION}/toolchain_gnu_{sysname}-{machine}_arm-zephyr-eabi.{extension}", + ) +) + + +def _get_sdk_ng_toolchain_mirrors() -> list[str]: + """Return toolchain mirror URLs for the current framework version.""" + if _uses_sdk_ng_1_toolchain(): + return _SDK_NG_TOOLCHAIN_GNU_MIRRORS + return SDK_NG_TOOLCHAIN_MIRRORS + # Minimal SDK provides cmake discovery files (Zephyr-sdkConfig.cmake) and # host tools (dtc etc.) required by the Zephyr cmake build system. @@ -85,15 +119,22 @@ def _get_toolchain_path(version: str) -> Path: return get_sdk_nrf_tools_path() / "toolchains" / version -def toolchain_tool(name: str) -> Path: - """Path to one of the pinned Zephyr SDK's tools (objdump, readelf, ...). +def _get_arm_toolchain_path() -> Path: + """The arm-zephyr-eabi directory inside the pinned Zephyr SDK. - The single owner of the ``arm-zephyr-eabi/bin/arm-zephyr-eabi-`` - layout and the Windows suffix. + The single owner of the SDK 0.x (arm-zephyr-eabi/) and SDK 1.0+ + (gnu/arm-zephyr-eabi/) layouts. """ + toolchain_root = _get_toolchain_path(_get_toolchain_version()) + if _uses_sdk_ng_1_toolchain(): + return toolchain_root / "gnu" / "arm-zephyr-eabi" + return toolchain_root / "arm-zephyr-eabi" + + +def toolchain_tool(name: str) -> Path: + """Path to one of the pinned Zephyr SDK's tools (objdump, readelf, ...).""" suffix = ".exe" if os.name == "nt" else "" - bin_path = _get_toolchain_path(TOOLCHAIN_VERSION) / "arm-zephyr-eabi" / "bin" - return bin_path / f"arm-zephyr-eabi-{name}{suffix}" + return _get_arm_toolchain_path() / "bin" / f"arm-zephyr-eabi-{name}{suffix}" _SITECUSTOMIZE = """\ @@ -169,7 +210,7 @@ def get_build_env() -> dict: # "Zephyr-sdk_DIR" environment hint proved unreliable here: containerized # non-root builds failed to locate the SDK with it, while # ZEPHYR_SDK_INSTALL_DIR fixed the same invocation. - env["ZEPHYR_SDK_INSTALL_DIR"] = str(_get_toolchain_path(TOOLCHAIN_VERSION)) + env["ZEPHYR_SDK_INSTALL_DIR"] = str(_get_toolchain_path(_get_toolchain_version())) return env @@ -561,36 +602,41 @@ def _check_and_install(version: str) -> None: raise EsphomeError(f"Install Zephyr requirements for {version} failure") zephyr_sentinel.touch() - # Shared by every SDK version; locked only while missing - if not (_get_toolchain_path(TOOLCHAIN_VERSION) / ".ready").exists(): - with _install_lock(f"toolchain-{TOOLCHAIN_VERSION}"): + # Shared by every SDK version that uses the same toolchain; locked only + # while missing + toolchain_version = _get_toolchain_version() + if not (_get_toolchain_path(toolchain_version) / ".ready").exists(): + with _install_lock(f"toolchain-{toolchain_version}"): _install_toolchain() def _install_toolchain() -> None: - toolchains_dir = _get_toolchain_path(TOOLCHAIN_VERSION) + toolchain_version = _get_toolchain_version() + toolchains_dir = _get_toolchain_path(toolchain_version) sentinel = toolchains_dir / ".ready" if not sentinel.exists(): - rmdir(toolchains_dir, msg=f"Clean up {TOOLCHAIN_VERSION} toolchain environment") + rmdir(toolchains_dir, msg=f"Clean up {toolchain_version} toolchain environment") sysname, machine, extension = _get_toolchain_platform_info() substitutions = { - "VERSION": TOOLCHAIN_VERSION, + "VERSION": toolchain_version, "sysname": sysname, "machine": machine, "extension": extension, } # Downloaded next to the destination (not a temp file) so an # interrupted download's .part file resumes on the next run. + # SDK 1.0+ Zephyr-sdkConfig.cmake looks for the toolchain in + # gnu/arm-zephyr-eabi/; extraction strips the archive's single root. for mirrors, extract_dir, what, slug in ( (SDK_NG_MINIMAL_MIRRORS, toolchains_dir, "Zephyr SDK minimal", "minimal"), ( - SDK_NG_TOOLCHAIN_MIRRORS, - toolchains_dir / "arm-zephyr-eabi", + _get_sdk_ng_toolchain_mirrors(), + _get_arm_toolchain_path(), "toolchain", "toolchain", ), ): - _LOGGER.info("Downloading %s %s ...", TOOLCHAIN_VERSION, what) + _LOGGER.info("Downloading %s %s ...", toolchain_version, what) download_and_extract( mirrors, substitutions, @@ -598,10 +644,17 @@ def _install_toolchain() -> None: extract_dir, progress_header="Extracting", ) - # Best-effort prune of resume leftovers, including a previous - # TOOLCHAIN_VERSION's orphans; the SDK archives are hundreds of MB. - # A locked file must not discard the just-completed install. + # Best-effort prune of resume leftovers, including orphans of retired + # toolchain versions; the SDK archives are hundreds of MB. The other + # toolchain still in use may be downloading under its own lock, so its + # leftovers are kept. A locked file must not discard the just-completed + # install. + other_versions = tuple( + f"{v}." for v in _TOOLCHAIN_VERSIONS if v != toolchain_version + ) for leftover in toolchains_dir.parent.glob("*.archive.part*"): + if leftover.name.startswith(other_versions): + continue try: leftover.unlink() except OSError as err: diff --git a/tests/unit_tests/components/nrf52/test_pch.py b/tests/unit_tests/components/nrf52/test_pch.py index 5af55586c3..0dcac3cad4 100644 --- a/tests/unit_tests/components/nrf52/test_pch.py +++ b/tests/unit_tests/components/nrf52/test_pch.py @@ -7,6 +7,7 @@ import pytest from esphome.components import nrf52 from esphome.components.zephyr.const import KEY_BOARD +import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain from esphome.core import CORE, EsphomeError @@ -110,6 +111,7 @@ def run_cmd(tmp_path: Path) -> Mock: CORE.build_path = tmp_path / "build" CORE.name = "livingroom" CORE.toolchain = Toolchain.SDK_NRF + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 2, 0)} with ( patch.object(nrf52, "check_and_install"), patch.object(nrf52, "_generate_cmake_lists", return_value=False), diff --git a/tests/unit_tests/components/nrf52/test_run_compile_hex.py b/tests/unit_tests/components/nrf52/test_run_compile_hex.py new file mode 100644 index 0000000000..f86a29a9d2 --- /dev/null +++ b/tests/unit_tests/components/nrf52/test_run_compile_hex.py @@ -0,0 +1,130 @@ +"""nrf52 sdk-nrf build: the hex file used for the Adafruit UF2 and DFU package.""" + +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from esphome.components import nrf52 +from esphome.components.nrf52.const import BOOTLOADER_ADAFRUIT_NRF52_SD140_V7 +from esphome.components.zephyr.const import KEY_BOARD, KEY_BOOTLOADER +import esphome.config_validation as cv +from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain +from esphome.core import CORE + + +@pytest.fixture +def build_dir(tmp_path: Path) -> Path: + CORE.config_path = tmp_path / "test.yaml" + CORE.build_path = tmp_path / "build" + CORE.name = "livingroom" + CORE.toolchain = Toolchain.SDK_NRF + return CORE.relative_pioenvs_path(CORE.name) + + +@pytest.fixture +def run_cmd(tmp_path: Path) -> Iterator[Mock]: + with ( + patch.object(nrf52, "check_and_install"), + patch.object(nrf52, "_generate_cmake_lists", return_value=False), + patch.object( + nrf52, + "get_build_paths", + return_value={"python_executable": "python3", "framework_path": tmp_path}, + ), + patch.object(nrf52, "get_build_env", return_value={}), + patch.object( + nrf52, + "zephyr_data", + return_value={ + KEY_BOARD: "board", + KEY_BOOTLOADER: BOOTLOADER_ADAFRUIT_NRF52_SD140_V7, + }, + ), + patch.object(nrf52.pch, "pch_enabled", return_value=False), + patch.object(nrf52, "run_command_ok", return_value=True) as run, + ): + yield run + + +def _uf2_hex(run_cmd: Mock) -> str: + cmd = next(c.args[0] for c in run_cmd.call_args_list if "-f" in c.args[0]) + return cmd[-1] + + +def _dfu_hex(run_cmd: Mock) -> str: + cmd = next(c.args[0] for c in run_cmd.call_args_list if "genpkg" in c.args[0]) + return cmd[cmd.index("--application") + 1] + + +def _build( + build_dir: Path, + version: cv.Version, + outputs: list[str], + run_cmd: Mock, + leftovers: tuple[str, ...] = (), +) -> None: + """Run run_compile; *outputs* appear when west builds, *leftovers* before.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: version} + (build_dir / "zephyr" / "zephyr").mkdir(parents=True) + # Without a CMake cache run_compile wipes the build dir first + for name in ["CMakeCache.txt", *leftovers]: + (build_dir / name).write_text("") + + def west_build(cmd: list[str], **kwargs: object) -> bool: + if "west" in cmd: + for name in outputs: + (build_dir / name).write_text("") + return True + + run_cmd.side_effect = west_build + nrf52.run_compile(None, {}) + + +def test_uf2_and_dfu_use_merged_hex(build_dir: Path, run_cmd: Mock) -> None: + _build( + build_dir, + cv.Version(2, 9, 2), + ["merged.hex", "zephyr/zephyr/zephyr.hex"], + run_cmd, + ) + merged = str(build_dir / "zephyr" / "merged.hex") + assert _uf2_hex(run_cmd) == merged + assert _dfu_hex(run_cmd) == merged + + +def test_uf2_and_dfu_fall_back_to_zephyr_hex(build_dir: Path, run_cmd: Mock) -> None: + """The nRF Connect SDK 3.4.0 build produces no merged.hex.""" + _build(build_dir, cv.Version(3, 4, 0), ["zephyr/zephyr/zephyr.hex"], run_cmd) + app_hex = build_dir / "zephyr" / "zephyr.hex" + assert app_hex.is_file() + assert _uf2_hex(run_cmd) == str(app_hex) + assert _dfu_hex(run_cmd) == str(app_hex) + + +def test_older_sdk_does_not_fall_back_to_zephyr_hex( + build_dir: Path, run_cmd: Mock +) -> None: + """Before SDK 3.4.0 a missing merged.hex must not be replaced by zephyr.hex.""" + _build(build_dir, cv.Version(3, 2, 0), ["zephyr/zephyr/zephyr.hex"], run_cmd) + assert not any("-f" in c.args[0] for c in run_cmd.call_args_list) + assert _dfu_hex(run_cmd) == str(build_dir / "zephyr" / "merged.hex") + + +def test_stale_merged_hex_is_removed_before_building( + build_dir: Path, run_cmd: Mock +) -> None: + """A merged.hex left by an older SDK build must not be packaged.""" + _build( + build_dir, + cv.Version(3, 4, 0), + ["zephyr/zephyr/zephyr.hex"], + run_cmd, + leftovers=("merged.hex", "zephyr/merged.hex"), + ) + assert not (build_dir / "merged.hex").exists() + assert not (build_dir / "zephyr" / "merged.hex").exists() + app_hex = str(build_dir / "zephyr" / "zephyr.hex") + assert _uf2_hex(run_cmd) == app_hex + assert _dfu_hex(run_cmd) == app_hex diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index d6b27bdbef..17d583d139 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -20,6 +20,7 @@ from esphome.components.nrf52.framework import ( _get_penv_site_packages, _get_platformio_penv_path, _get_toolchain_platform_info, + _install_toolchain, _needs_venv_rebuild, _wanted_west_projects, check_and_install, @@ -808,6 +809,63 @@ class TestCheckAndInstall: assert substitutions["machine"] == "x86_64" assert substitutions["extension"] == "tar.xz" + def test_toolchain_download_uses_gnu_url_for_sdk_3_4_0( + self, + tmp_path: Path, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """For nRF Connect SDK >= 3.4.0 the toolchain archive name includes 'toolchain_gnu_'.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + sdk_version = "3.4.0" + tools = get_sdk_nrf_tools_path() + python_env = tools / "penvs" / f"v{sdk_version}" + framework = tools / "frameworks" / f"v{sdk_version}" + toolchain_dir = tools / "toolchains" / "1.0.1" + for d in (python_env, framework, toolchain_dir): + d.mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts").mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts" / "requirements.txt").touch() + _mark_venv_ready(python_env) + (framework / ".ready").touch() + + check_and_install() + + # Two download calls: minimal SDK first, toolchain second + toolchain_call = mock_nrf52_ops.download_from_mirrors.call_args_list[1] + mirrors = toolchain_call.args[0] + assert all("toolchain_gnu_" in m for m in mirrors) + + def test_toolchain_extracts_under_gnu_for_sdk_3_4_0( + self, + tmp_path: Path, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """SDK 1.0+ toolchain archive must land in gnu/arm-zephyr-eabi/. + + Zephyr-sdkConfig.cmake validates the toolchain at gnu/arm-zephyr-eabi/ + in SDK 1.0+; if the archive is extracted to arm-zephyr-eabi/ instead, + cmake reports the package as not found. + """ + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + sdk_version = "3.4.0" + tools = get_sdk_nrf_tools_path() + python_env = tools / "penvs" / f"v{sdk_version}" + framework = tools / "frameworks" / f"v{sdk_version}" + toolchain_dir = tools / "toolchains" / "1.0.1" + for d in (python_env, framework, toolchain_dir): + d.mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts").mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts" / "requirements.txt").touch() + _mark_venv_ready(python_env) + (framework / ".ready").touch() + + check_and_install() + + # Two extract calls: minimal SDK first (to toolchain root), toolchain second + extract_calls = mock_nrf52_ops.archive_extract_all.call_args_list + _, toolchain_extract_dir = extract_calls[1].args[:2] + assert toolchain_extract_dir == toolchain_dir / "gnu" / "arm-zephyr-eabi" + # --------------------------------------------------------------------------- # setup_platformio_python_env tests @@ -1040,6 +1098,44 @@ def test_get_build_env( assert env["SOME_PREEXISTING_VAR"] == "kept" +def test_get_build_env_sdk_3_4_0_uses_toolchain_root( + setup_core: Path, +) -> None: + """For NCS >= 3.4.0, ZEPHYR_SDK_INSTALL_DIR still points at the toolchain root.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + + env = get_build_env() + + tools = get_sdk_nrf_tools_path() + assert env["ZEPHYR_SDK_INSTALL_DIR"] == str(tools / "toolchains" / "1.0.1") + assert "Zephyr-sdk_DIR" not in env + + +def test_install_toolchain_keeps_other_live_toolchain_leftovers( + setup_core: Path, +) -> None: + """Pruning must not delete a partial download of the other toolchain in use.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + toolchains = get_sdk_nrf_tools_path() / "toolchains" + toolchains.mkdir(parents=True) + own = toolchains / "1.0.1.toolchain.archive.part" + other = toolchains / f"{TOOLCHAIN_VERSION}.toolchain.archive.part" + retired = toolchains / "0.16.8.toolchain.archive.part" + for leftover in (own, other, retired): + leftover.write_text("") + + with patch( + "esphome.components.nrf52.framework.download_and_extract", + side_effect=lambda *args, **kwargs: args[3].mkdir(parents=True), + ): + _install_toolchain() + + assert not own.exists() + assert other.exists() + assert not retired.exists() + assert (toolchains / "1.0.1" / ".ready").exists() + + # --------------------------------------------------------------------------- # get_sdk_nrf_tools_path tests # --------------------------------------------------------------------------- diff --git a/tests/unit_tests/test_nrf52_toolchain.py b/tests/unit_tests/test_nrf52_toolchain.py index ee8f6eaf49..b09540bb80 100644 --- a/tests/unit_tests/test_nrf52_toolchain.py +++ b/tests/unit_tests/test_nrf52_toolchain.py @@ -24,6 +24,7 @@ def _set_sdk_version(version: cv.Version) -> None: def test_binutils_come_from_the_pinned_zephyr_sdk() -> None: + _set_sdk_version(cv.Version(2, 9, 0)) bin_path = ( get_sdk_nrf_tools_path() / "toolchains" @@ -37,6 +38,21 @@ def test_binutils_come_from_the_pinned_zephyr_sdk() -> None: assert toolchain.get_readelf_path() == bin_path / f"arm-zephyr-eabi-readelf{suffix}" +def test_binutils_use_new_toolchain_for_sdk_3_4_0() -> None: + _set_sdk_version(cv.Version(3, 4, 0)) + bin_path = ( + get_sdk_nrf_tools_path() + / "toolchains" + / "1.0.1" + / "gnu" + / "arm-zephyr-eabi" + / "bin" + ) + suffix = ".exe" if os.name == "nt" else "" + assert toolchain.get_objdump_path() == bin_path / f"arm-zephyr-eabi-objdump{suffix}" + assert toolchain.get_readelf_path() == bin_path / f"arm-zephyr-eabi-readelf{suffix}" + + def test_elf_uses_the_nested_layout_from_sdk_2_9_2(nrf52_build: Path) -> None: _set_sdk_version(cv.Version(2, 9, 2)) # A flat image left by an older SDK must not be picked over the current From 2accef437ff94503d418779acbaf63f4782b89ab Mon Sep 17 00:00:00 2001 From: Jeff Brown Date: Thu, 1 Oct 2026 05:46:49 -0700 Subject: [PATCH 011/221] [pmsa003i] Fix spurious component failures and warnings on power up (#19933) Co-authored-by: J. Nick Koston --- esphome/components/pmsa003i/pmsa003i.cpp | 40 ++++++++++++++++-------- esphome/components/pmsa003i/pmsa003i.h | 1 + 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/esphome/components/pmsa003i/pmsa003i.cpp b/esphome/components/pmsa003i/pmsa003i.cpp index 0b5c72a94d..f20b93fff8 100644 --- a/esphome/components/pmsa003i/pmsa003i.cpp +++ b/esphome/components/pmsa003i/pmsa003i.cpp @@ -15,24 +15,38 @@ static const uint8_t CHECKSUM_START_INDEX = COUNT_DATA_BYTES - 2; static const uint8_t COUNT_16_BIT_VALUES = (COUNT_PAYLOAD_LENGTH_BYTES + COUNT_PAYLOAD_BYTES) / 2; static const uint8_t START_CHARACTER_1 = 0x42; static const uint8_t START_CHARACTER_2 = 0x4D; -static const uint8_t READ_DATA_RETRY_COUNT = 3; + +// Timeout for determining when the device is ready for use, in milliseconds. +// The PMSA003I typically takes 2.3 seconds to perform its first measurement after a cold power up and +// I2C requests performed during that time will be NACKed. Use a slightly longer timeout to tolerate +// timing variation. Note that although we consider the device ready as soon as it responds to I2C +// requests, it may take 30 seconds or more for the data to stabilize according to the datasheet. +static const uint32_t READY_TIMEOUT_MS = 3000; + +// Poll interval for determining when the device is ready for use, in milliseconds. +static const uint32_t READY_POLL_INTERVAL_MS = 100; void PMSA003IComponent::setup() { - PM25AQIData data; - bool successful_read = this->read_data_(&data); + // Stop polling until the device is actually ready to prevent spurious I2C warnings during premature updates. + this->stop_poller(); + this->poll_until_ready_or_timeout_(millis()); +} - if (!successful_read) { - for (uint8_t i = 0; i < READ_DATA_RETRY_COUNT; i++) { - successful_read = this->read_data_(&data); - if (successful_read) { - break; - } - } +void PMSA003IComponent::poll_until_ready_or_timeout_(uint32_t start_time) { + // Check whether the device is responding and identifies itself as expected. + // We don't care about the actual sensor readings or the packet CRC in this case. + uint8_t buffer[2]; + if (this->read(buffer, sizeof(buffer)) == i2c::ERROR_OK && buffer[0] == START_CHARACTER_1 && + buffer[1] == START_CHARACTER_2) { + ESP_LOGD(TAG, "PMSA003I is ready"); + this->start_poller(); + return; } - if (!successful_read) { - this->mark_failed(); - return; + if (millis() - start_time < READY_TIMEOUT_MS) { + this->set_timeout(READY_POLL_INTERVAL_MS, [this, start_time]() { this->poll_until_ready_or_timeout_(start_time); }); + } else { + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); } } diff --git a/esphome/components/pmsa003i/pmsa003i.h b/esphome/components/pmsa003i/pmsa003i.h index 908b073be1..6eccde3665 100644 --- a/esphome/components/pmsa003i/pmsa003i.h +++ b/esphome/components/pmsa003i/pmsa003i.h @@ -46,6 +46,7 @@ class PMSA003IComponent final : public PollingComponent, public i2c::I2CDevice { void set_pmc_10_0_sensor(sensor::Sensor *pmc_10_0) { this->pmc_10_0_sensor_ = pmc_10_0; } protected: + void poll_until_ready_or_timeout_(uint32_t start_time); bool read_data_(PM25AQIData *data); bool standard_units_; From 56a7a1116210b2c936a7d7015016b16f12505d3c Mon Sep 17 00:00:00 2001 From: rwrozelle Date: Thu, 1 Oct 2026 08:48:31 -0400 Subject: [PATCH 012/221] [openthread] rewrite openthread esp.cpp to use new esp functions (#16334) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: Claude Sonnet 4.6 Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- esphome/components/openthread/openthread.cpp | 44 ++-- esphome/components/openthread/openthread.h | 7 +- .../components/openthread/openthread_esp.cpp | 196 +++++++----------- .../openthread/openthread_zephyr.cpp | 5 +- 4 files changed, 105 insertions(+), 147 deletions(-) diff --git a/esphome/components/openthread/openthread.cpp b/esphome/components/openthread/openthread.cpp index c788b1f968..7d98522cd2 100644 --- a/esphome/components/openthread/openthread.cpp +++ b/esphome/components/openthread/openthread.cpp @@ -230,23 +230,27 @@ void *OpenThreadSrpComponent::pool_alloc_(size_t size) { bool OpenThreadComponent::teardown() { switch (this->teardown_stage_) { case TeardownStage::TEARDOWN_STAGE_NOT_STARTED: { - auto lock = InstanceLock::try_acquire(100); - if (!lock) { - // Try again on next teardown loop - ESP_LOGV(TAG, "Failed to acquire OpenThread lock during teardown"); - return false; - } - // Start tearing down - this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_STOP_IN_PROCESS; - ESP_LOGV(TAG, "Clear SRP"); - otInstance *instance = lock.get_instance(); - otSrpClientClearHostAndServices(instance); - otSrpClientBuffersFreeAllServices(instance); - if (otThreadSetEnabled(instance, false) != OT_ERROR_NONE) { - ESP_LOGW(TAG, "Failed to disable Thread during teardown"); - } - if (otIp6SetEnabled(instance, false) != OT_ERROR_NONE) { - ESP_LOGW(TAG, "Failed to disable IPv6 during teardown"); + { + auto lock = InstanceLock::try_acquire(100); + // The OT task may still be starting up; stay pending and retry on + // the next call rather than giving up after a single failed attempt. + if (!lock) { + ESP_LOGV(TAG, "Failed to acquire OpenThread lock during teardown"); + return false; + } + this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_STOP_IN_PROCESS; + ESP_LOGV(TAG, "Clear SRP"); + otInstance *instance = lock.get_instance(); + otSrpClientClearHostAndServices(instance); + otSrpClientBuffersFreeAllServices(instance); + if (otThreadSetEnabled(instance, false) != OT_ERROR_NONE) { + ESP_LOGW(TAG, "Failed to disable Thread during teardown"); + } + if (otIp6SetEnabled(instance, false) != OT_ERROR_NONE) { + ESP_LOGW(TAG, "Failed to disable IPv6 during teardown"); + } + // Release the lock before stopping -- openthread_stop_() (esp_openthread_stop() on + // ESP32) acquires it internally, and the lock is not recursive. } // Stop OpenThread global_openthread_component = nullptr; @@ -254,11 +258,11 @@ bool OpenThreadComponent::teardown() { int error = this->openthread_stop_(); if (error != 0) { ESP_LOGW(TAG, "Failed attempt to stop OpenThread %d", error); - this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_COMPLETED; } } break; case TeardownStage::TEARDOWN_STAGE_STOP_IN_PROCESS: - // Waiting on OpenThread stop + // Unreachable today; teardown is synchronous on both platforms. Kept for a future + // graceful-exit path, or a platform whose teardown() cannot be made synchronous. break; case TeardownStage::TEARDOWN_STAGE_COMPLETED: ESP_LOGV(TAG, "OpenThreadComponent Teardown Complete"); @@ -286,7 +290,7 @@ void OpenThreadComponent::apply_poll_period(uint32_t poll_period) { #if CONFIG_OPENTHREAD_MTD this->set_poll_period(poll_period); if (!this->is_lock_initialized()) { - // The action may run before the stack is up, e.g. from a restore mode; ot_main applies the stored value. + // The action may run before the stack is up, e.g. from a restore mode; setup() applies the stored value. ESP_LOGD(TAG, "Not (yet) ready to apply"); return; } diff --git a/esphome/components/openthread/openthread.h b/esphome/components/openthread/openthread.h index 8bccae7f85..8599304c89 100644 --- a/esphome/components/openthread/openthread.h +++ b/esphome/components/openthread/openthread.h @@ -35,11 +35,10 @@ class OpenThreadComponent final : public Component { float get_setup_priority() const override { return setup_priority::WIFI; } bool is_connected() const { return this->connected_; } - /// Returns true once esp_openthread_init() has completed and the OT lock is usable. + /// Returns true once esp_openthread_start() has completed and the OT lock is usable. bool is_lock_initialized() const { return this->lock_initialized_; } network::IPAddresses get_ip_addresses(); std::optional get_omr_address(); - void ot_main(); void on_factory_reset(std::function callback); void defer_factory_reset_external_callback(); @@ -60,7 +59,6 @@ class OpenThreadComponent final : public Component { protected: /** Apply Link Mode settings (incl poll period). * Callers running outside the OpenThread task must hold InstanceLock. - * ot_main() runs on the OpenThread task itself and must not acquire the lock. */ void apply_linkmode_(otInstance *instance); @@ -73,7 +71,8 @@ class OpenThreadComponent final : public Component { #endif std::optional output_power_{}; std::atomic lock_initialized_{false}; - std::atomic teardown_stage_{TeardownStage::TEARDOWN_STAGE_NOT_STARTED}; + // Only ever written from teardown(), on the main task -- no atomic needed. + TeardownStage teardown_stage_{TeardownStage::TEARDOWN_STAGE_NOT_STARTED}; std::atomic connected_{false}; private: diff --git a/esphome/components/openthread/openthread_esp.cpp b/esphome/components/openthread/openthread_esp.cpp index 881bbea3c9..526fc961f2 100644 --- a/esphome/components/openthread/openthread_esp.cpp +++ b/esphome/components/openthread/openthread_esp.cpp @@ -1,6 +1,5 @@ #include "esphome/core/defines.h" #if defined(USE_OPENTHREAD) && defined(USE_ESP32) -#include #include "openthread.h" #include "esp_log.h" @@ -13,14 +12,9 @@ #include "esphome/core/log.h" #include "esp_err.h" -#include "esp_event.h" #include "esp_netif.h" -#include "esp_netif_types.h" -#include "esp_openthread_cli.h" #include "esp_openthread_netif_glue.h" #include "esp_vfs_eventfd.h" -#include "freertos/FreeRTOS.h" -#include "freertos/task.h" #include "nvs_flash.h" static const char *const TAG = "openthread"; @@ -39,140 +33,94 @@ void OpenThreadComponent::setup() { ESP_ERROR_CHECK(nvs_flash_init()); ESP_ERROR_CHECK(esp_vfs_eventfd_register(&eventfd_config)); - xTaskCreate( - [](void *arg) { - static_cast(arg)->ot_main(); - vTaskDelete(nullptr); - }, - "ot_main", 10240, this, 5, nullptr); -} + esp_openthread_config_t config = {.netif_config = ESP_NETIF_DEFAULT_OPENTHREAD(), + .platform_config = { + .radio_config = + { + .radio_mode = RADIO_MODE_NATIVE, + .radio_uart_config = {}, + }, + .host_config = + { + // There is a conflict between esphome's logger which also + // claims the usb serial jtag device. + // .host_connection_mode = HOST_CONNECTION_MODE_CLI_USB, + // .host_usb_config = USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT(), + }, + .port_config = + { + .storage_partition_name = "nvs", + .netif_queue_size = 10, + .task_queue_size = 10, + }, + }}; -static esp_netif_t *init_openthread_netif(const esp_openthread_platform_config_t *config) { - esp_netif_config_t cfg = ESP_NETIF_DEFAULT_OPENTHREAD(); - esp_netif_t *netif = esp_netif_new(&cfg); - assert(netif != nullptr); - ESP_ERROR_CHECK(esp_netif_attach(netif, esp_openthread_netif_glue_init(config))); - - return netif; -} - -void OpenThreadComponent::ot_main() { - esp_openthread_platform_config_t config = { - .radio_config = - { - .radio_mode = RADIO_MODE_NATIVE, - .radio_uart_config = {}, - }, - .host_config = - { - // There is a conflict between esphome's logger which also - // claims the usb serial jtag device. - // .host_connection_mode = HOST_CONNECTION_MODE_CLI_USB, - // .host_usb_config = USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT(), - }, - .port_config = - { - .storage_partition_name = "nvs", - .netif_queue_size = 10, - .task_queue_size = 10, - }, - }; - - // Initialize the OpenThread stack - // otLoggingSetLevel(OT_LOG_LEVEL_DEBG); - ESP_ERROR_CHECK(esp_openthread_init(&config)); + ESP_ERROR_CHECK(esp_openthread_start(&config)); // Mark lock as initialized so InstanceLock callers know it's safe to acquire. - // Must be set after esp_openthread_init() which creates the internal semaphore. + // Must be set after esp_openthread_start() which creates the internal semaphore. this->lock_initialized_ = true; // Fetch OT instance once to avoid repeated call into OT stack otInstance *instance = esp_openthread_get_instance(); + { + InstanceLock lock = InstanceLock::acquire(); -#if CONFIG_OPENTHREAD_STATE_INDICATOR_ENABLE - ESP_ERROR_CHECK(esp_openthread_state_indicator_init(instance)); -#endif + this->apply_linkmode_(instance); -#if CONFIG_OPENTHREAD_LOG_LEVEL_DYNAMIC - // The OpenThread log level directly matches ESP log level - (void) otLoggingSetLevel(CONFIG_LOG_DEFAULT_LEVEL); -#endif - // Initialize the OpenThread cli -#if CONFIG_OPENTHREAD_CLI - esp_openthread_cli_init(); -#endif - - esp_netif_t *openthread_netif; - // Initialize the esp_netif bindings - openthread_netif = init_openthread_netif(&config); - esp_netif_set_default_netif(openthread_netif); - -#if CONFIG_OPENTHREAD_CLI_ESP_EXTENSION - esp_cli_custom_command_init(); -#endif // CONFIG_OPENTHREAD_CLI_ESP_EXTENSION - - ESP_LOGD(TAG, "Thread Version: %" PRIu16, otThreadGetVersion()); - - this->apply_linkmode_(instance); - - if (this->output_power_.has_value()) { - if (const auto err = otPlatRadioSetTransmitPower(instance, *this->output_power_); err != OT_ERROR_NONE) { - ESP_LOGE(TAG, "Failed to set power: %s", otThreadErrorToString(err)); + if (this->output_power_.has_value()) { + if (const auto err = otPlatRadioSetTransmitPower(instance, *this->output_power_); err != OT_ERROR_NONE) { + ESP_LOGE(TAG, "Failed to set power: %s", otThreadErrorToString(err)); + } } - } - - // Run the main loop -#if CONFIG_OPENTHREAD_CLI - esp_openthread_cli_create_task(); -#endif - ESP_LOGI(TAG, "Activating dataset..."); - otOperationalDatasetTlvs dataset = {}; + ESP_LOGI(TAG, "Activating dataset..."); + otOperationalDatasetTlvs dataset = {}; #ifndef USE_OPENTHREAD_FORCE_DATASET - // Check if openthread has a valid dataset from a previous execution - otError error = otDatasetGetActiveTlvs(instance, &dataset); - if (error != OT_ERROR_NONE) { - // Make sure the length is 0 so we fallback to the configuration - dataset.mLength = 0; - } else { - ESP_LOGI(TAG, "Found existing dataset, ignoring config (force_dataset: true to override)"); - } + // Check if openthread has a valid dataset from a previous execution + otError error = otDatasetGetActiveTlvs(instance, &dataset); + if (error != OT_ERROR_NONE) { + // Make sure the length is 0 so we fallback to the configuration + dataset.mLength = 0; + } else { + ESP_LOGI(TAG, "Found existing dataset, ignoring config (force_dataset: true to override)"); + } #endif #ifdef USE_OPENTHREAD_TLVS - if (dataset.mLength == 0) { - // If we didn't have an active dataset, and we have tlvs, parse it and pass it to esp_openthread_auto_start - size_t len = (sizeof(USE_OPENTHREAD_TLVS) - 1) / 2; - if (len > sizeof(dataset.mTlvs)) { - ESP_LOGW(TAG, "TLV buffer too small, truncating"); - len = sizeof(dataset.mTlvs); + if (dataset.mLength == 0) { + // If we didn't have an active dataset, and we have tlvs, parse it and pass it to esp_openthread_auto_start + size_t len = (sizeof(USE_OPENTHREAD_TLVS) - 1) / 2; + if (len > sizeof(dataset.mTlvs)) { + ESP_LOGW(TAG, "TLV buffer too small, truncating"); + len = sizeof(dataset.mTlvs); + } + parse_hex(USE_OPENTHREAD_TLVS, sizeof(USE_OPENTHREAD_TLVS) - 1, dataset.mTlvs, len); + dataset.mLength = len; } - parse_hex(USE_OPENTHREAD_TLVS, sizeof(USE_OPENTHREAD_TLVS) - 1, dataset.mTlvs, len); - dataset.mLength = len; - } #endif - // Pass the existing dataset, or NULL which will use the preprocessor definitions - ESP_ERROR_CHECK(esp_openthread_auto_start(dataset.mLength > 0 ? &dataset : nullptr)); + // Pass the existing dataset, or NULL which will use the preprocessor definitions + ESP_ERROR_CHECK(esp_openthread_auto_start(dataset.mLength > 0 ? &dataset : nullptr)); - // Register state change callback to update connected_ reactively instead of polling - otError ot_err = otSetStateChangedCallback(instance, OpenThreadComponent::on_state_changed, this); - if (ot_err != OT_ERROR_NONE) { - ESP_LOGW(TAG, "Failed to register state change callback: %d", ot_err); + // Register state change callback to update connected_ reactively instead of polling + otError ot_err = otSetStateChangedCallback(instance, OpenThreadComponent::on_state_changed, this); + if (ot_err != OT_ERROR_NONE) { + ESP_LOGW(TAG, "Failed to register state change callback: %d", ot_err); + } } - esp_openthread_launch_mainloop(); - - // Clean up - reset lock flag before deinit destroys the semaphore - this->lock_initialized_ = false; - esp_openthread_deinit(); - esp_openthread_netif_glue_deinit(); - esp_netif_destroy(openthread_netif); - - esp_vfs_eventfd_unregister(); - this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_COMPLETED; - vTaskDelete(NULL); + ESP_LOGD(TAG, "Thread Version: %" PRIu16, otThreadGetVersion()); } -int OpenThreadComponent::openthread_stop_() { return esp_openthread_mainloop_exit(); } +int OpenThreadComponent::openthread_stop_() { + // Clean up - reset lock flag before deinit destroys the semaphore + this->lock_initialized_ = false; + int error = esp_openthread_stop(); + // Mark complete even on failure: we're already mid-shutdown/reboot, so there's no + // recovery path to retry into -- leaving the stage stuck would only burn the full + // teardown timeout for no benefit. + this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_COMPLETED; + return error; +} network::IPAddresses OpenThreadComponent::get_ip_addresses() { network::IPAddresses addresses; @@ -199,8 +147,14 @@ InstanceLock InstanceLock::try_acquire(int delay) { } InstanceLock InstanceLock::acquire() { - // Wait for the lock to be created by ot_main() before attempting to acquire it. - // esp_openthread_lock_acquire() will assert-crash if called before esp_openthread_init(). + // teardown() clears global_openthread_component before the stack fully stops; a caller + // racing teardown would otherwise dereference a null pointer below. + if (global_openthread_component == nullptr) { + ESP_LOGE(TAG, "OpenThread component torn down, cannot acquire instance lock"); + abort(); + } + // Wait for the lock to be created before attempting to acquire it. + // esp_openthread_lock_acquire() will assert-crash if called before esp_openthread_start(). constexpr uint32_t lock_init_timeout_ms = 10000; uint32_t start = millis(); while (!global_openthread_component->is_lock_initialized()) { diff --git a/esphome/components/openthread/openthread_zephyr.cpp b/esphome/components/openthread/openthread_zephyr.cpp index cacb4c0122..0f474fc040 100644 --- a/esphome/components/openthread/openthread_zephyr.cpp +++ b/esphome/components/openthread/openthread_zephyr.cpp @@ -83,9 +83,10 @@ void OpenThreadComponent::setup() { } openthread_state_changed_cb_register(context, &ot_state_changed_cb); openthread_start(context); -} -void OpenThreadComponent::ot_main() {} + InstanceLock lock = InstanceLock::acquire(); + this->apply_linkmode_(lock.get_instance()); +} otInstance *OpenThreadComponent::get_openthread_instance_() { return openthread_get_default_instance(); } From 826571518acd50e3194fa390f629f3ac58fedd79 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:15:31 -0500 Subject: [PATCH 013/221] [noise] Generate the responder ephemeral key ahead of the handshake (#19000) --- esphome/components/api/__init__.py | 2 + esphome/components/api/api_connection.h | 6 ++ esphome/components/api/api_server.cpp | 23 ++++++- esphome/components/api/api_server.h | 5 +- esphome/components/noise/__init__.py | 5 ++ esphome/components/noise/noise.cpp | 34 ++++++++++ esphome/components/noise/noise.h | 18 +++++ esphome/components/noise/noise_handshake.cpp | 7 ++ esphome/components/noise/noise_handshake.h | 3 +- esphome/core/defines.h | 1 + tests/components/noise/__init__.py | 8 +++ .../components/noise/test_noise_handshake.cpp | 66 +++++++++++++++++++ 12 files changed, 174 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 2291dc54f8..40f520a057 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -13,6 +13,7 @@ from esphome.components.logger import request_log_listener from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, + enable_spare_ephemeral, encryption_schema, new_psk_progmem, validate_encryption_key, @@ -613,6 +614,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") + enable_spare_ephemeral() else: cg.add_define("USE_API_PLAINTEXT") diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index f03c4c42e8..0b8a1dc2b2 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -317,9 +317,15 @@ class APIConnection final : public APIServerConnectionBase { void on_noise_encryption_set_key_request(const NoiseEncryptionSetKeyRequest &msg); #endif + // How long a new connection holds off the spare ephemeral refill + static constexpr uint32_t CONNECT_GRACE_MS = 1000; bool is_authenticated() { return static_cast(this->flags_.connection_state) == ConnectionState::AUTHENTICATED; } + // An older unauthenticated connection is a stale half open client and does not count + bool is_still_connecting(uint32_t now) { + return !this->is_authenticated() && now - this->last_traffic_ < CONNECT_GRACE_MS; + } bool is_connection_setup() { return static_cast(this->flags_.connection_state) == ConnectionState::CONNECTED || this->is_authenticated(); diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 2a89d31a74..da9ca8bc79 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -162,6 +162,13 @@ void APIServer::loop() { this->accept_new_connections_(); } + const bool connected = network::is_connected(); +#ifdef USE_NOISE_SPARE_EPHEMERAL + if (connected && !noise::has_spare_ephemeral()) { + this->refill_spare_ephemeral_(); + } +#endif + if (this->api_connection_count_ == 0) { // Check reboot timeout - done in loop to avoid scheduler heap churn // (cancelled scheduler items sit in heap memory until their scheduled time). @@ -178,8 +185,7 @@ void APIServer::loop() { } // Process clients and remove disconnected ones in a single pass - // Check network connectivity once for all clients - if (!network::is_connected()) { + if (!connected) { // Network is down - disconnect all clients for (auto &client : this->active_clients()) { client->on_fatal_error(); @@ -207,6 +213,19 @@ void APIServer::loop() { } } +#ifdef USE_NOISE_SPARE_EPHEMERAL +// An OTA handshake is not visible here and just pays the refill it triggered +void APIServer::refill_spare_ephemeral_() { + const uint32_t now = App.get_loop_component_start_time(); + for (auto &client : this->active_clients()) { + if (client->is_still_connecting(now)) { + return; + } + } + noise::prepare_spare_ephemeral(); +} +#endif + void APIServer::remove_client_(uint8_t client_index) { auto &client = this->clients_[client_index]; diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 49cf77ca5a..865cd17156 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -5,7 +5,7 @@ #include "api_buffer.h" // Must precede clients_ so APIConnection is complete for default_delete (libc++). #include "api_connection.h" -#ifdef USE_API_NOISE +#if defined(USE_API_NOISE) || defined(USE_NOISE_SPARE_EPHEMERAL) // Only present in the build when the noise component is loaded #include "esphome/components/noise/noise.h" #endif @@ -367,6 +367,9 @@ class APIServer final : public Component uint8_t provisioning_source_{0}; #endif +#ifdef USE_NOISE_SPARE_EPHEMERAL + void refill_spare_ephemeral_(); +#endif #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; #ifndef USE_API_NOISE_PSK_FROM_YAML diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 47cd4cfc67..c894c147cd 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -92,6 +92,11 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: return ENCRYPTION_SCHEMA(config) +def enable_spare_ephemeral() -> None: + """Compile the spare ephemeral key slot; the component that refills it calls this.""" + cg.add_define("USE_NOISE_SPARE_EPHEMERAL") + + async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") cg.add_library("esphome/noise-c", "0.1.30") diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 86ea1ec874..118bfb234f 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,6 +1,7 @@ #include "noise.h" #ifdef USE_NOISE #include "esphome/core/hal.h" +#include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/progmem.h" @@ -8,6 +9,7 @@ #include #include +#include namespace esphome::noise { @@ -21,6 +23,38 @@ void NoiseContext::load_psk(psk_t &out) const { progmem_memcpy(out.data(), this->psk_, out.size()); } +#ifdef USE_NOISE_SPARE_EPHEMERAL +static constexpr size_t PRIVATE_KEY_SIZE = SPARE_EPHEMERAL_KEY_SIZE; +static constexpr size_t PUBLIC_KEY_SIZE = SPARE_EPHEMERAL_KEY_SIZE; +uint8_t spare_ephemeral[SPARE_EPHEMERAL_SIZE]; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +void prepare_spare_ephemeral() { + uint8_t *private_key = spare_ephemeral; + uint8_t *public_key = spare_ephemeral + PRIVATE_KEY_SIZE; + // Same steps as noise-c's keygen; the clamp sets the ready bit + if (!random_bytes(private_key, PRIVATE_KEY_SIZE)) { + sodium_memzero(spare_ephemeral, sizeof(spare_ephemeral)); + return; + } + private_key[0] &= 0xF8; + private_key[PRIVATE_KEY_SIZE - 1] = (private_key[PRIVATE_KEY_SIZE - 1] & 0x7F) | 0x40; + if (crypto_scalarmult_curve25519_base(public_key, private_key) != 0) { + sodium_memzero(spare_ephemeral, sizeof(spare_ephemeral)); + } +} + +int consume_spare_ephemeral(NoiseHandshakeState *state) { + if (!has_spare_ephemeral()) { + return 0; + } + // noise-c keeps its own copy, so the slot is wiped either way + int err = noise_handshakestate_set_local_ephemeral(state, spare_ephemeral, PRIVATE_KEY_SIZE, + spare_ephemeral + PRIVATE_KEY_SIZE, PUBLIC_KEY_SIZE); + sodium_memzero(spare_ephemeral, sizeof(spare_ephemeral)); + return err; +} +#endif // USE_NOISE_SPARE_EPHEMERAL + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index 1033d5423c..271a829944 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -6,6 +6,9 @@ #include #include "esphome/core/log.h" +// Forward decl from +using NoiseHandshakeState = struct NoiseHandshakeState_s; + namespace esphome::noise { using psk_t = std::array; @@ -38,6 +41,21 @@ class NoiseContext { /// Convert a noise error code to a readable error const LogString *noise_err_to_logstr(int err); +#ifdef USE_NOISE_SPARE_EPHEMERAL +// One responder ephemeral key pair made ahead of time (about 60 ms on ESP8266), refilled by the api server +// while idle and consumed by the next noise handshake of any transport; empty means the handshake generates +// its own. Private key then public key; the private key stays in RAM until consumed. +static constexpr size_t SPARE_EPHEMERAL_KEY_SIZE = 32; +static constexpr size_t SPARE_EPHEMERAL_SIZE = 2 * SPARE_EPHEMERAL_KEY_SIZE; +extern uint8_t spare_ephemeral[SPARE_EPHEMERAL_SIZE]; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +// The clamp always sets bit 254, so it doubles as the ready flag +inline bool has_spare_ephemeral() { return (spare_ephemeral[SPARE_EPHEMERAL_KEY_SIZE - 1] & 0x40) != 0; } +/// Fill the slot; blocks for the base point multiply +void prepare_spare_ephemeral(); +/// Move the key pair into a handshake that has not started and wipe the slot; 0 unless noise-c rejects it +int consume_spare_ephemeral(NoiseHandshakeState *state); +#endif + // Shared wire format for the noise transports (api and ota): every frame is // FRAME_INDICATOR, a 16-bit big-endian payload length, then the payload. // Handshake payloads start with a status byte; transport payloads end with diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 64846dcb67..bd1afa198d 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -57,6 +57,13 @@ int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prolog HANDSHAKE_STEP_LOG("noise_handshakestate_set_prologue", err); return this->fail_init_(err); } +#ifdef USE_NOISE_SPARE_EPHEMERAL + err = consume_spare_ephemeral(this->handshake_); + // Not fatal: the handshake generates its own key instead + if (err != 0) { + HANDSHAKE_STEP_LOG("noise_handshakestate_set_local_ephemeral", err); + } +#endif err = noise_handshakestate_start(this->handshake_); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_start", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index bf1aa8cb7f..ef556a26b0 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -37,7 +37,8 @@ class NoiseResponderHandshake { NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; /// Create and start the handshake with the context's PSK and the prologue. - /// A repeated call frees the previous handshake state and starts over. + /// A repeated call frees the previous handshake state and starts over. A + /// spare ephemeral key, when one is ready, is used instead of generating. [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. diff --git a/esphome/core/defines.h b/esphome/core/defines.h index f845154389..88f35dac46 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -281,6 +281,7 @@ #define USE_IMPROV_NEXT_URL #define USE_MD5 #define USE_NOISE +#define USE_NOISE_SPARE_EPHEMERAL #define USE_SHA256 #ifndef USE_RP2 // no MQTT backend or esp_wireguard library on RP2 #define USE_MQTT diff --git a/tests/components/noise/__init__.py b/tests/components/noise/__init__.py index 60a5740a83..aeae988f44 100644 --- a/tests/components/noise/__init__.py +++ b/tests/components/noise/__init__.py @@ -1,3 +1,4 @@ +import esphome.codegen as cg from tests.testing_helpers import ComponentManifestOverride @@ -5,3 +6,10 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # to_code must run: it defines USE_NOISE and adds the noise-c library # the component sources under test need. manifest.enable_codegen() + real_to_code = manifest.to_code + + async def to_code_testing(config): + await real_to_code(config) + cg.add_define("USE_NOISE_SPARE_EPHEMERAL") + + manifest.to_code = to_code_testing diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index f2081f2965..f9548a61f0 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -157,6 +157,72 @@ TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { noise_cipherstate_free(recv_cipher); } +// One full NNpsk0 handshake; responder_e gets the ephemeral public key the responder put on the wire, read +// before the initiator consumes the buffer in place +static void run_handshake(NoiseResponderHandshake &responder, uint8_t responder_e[SPARE_EPHEMERAL_KEY_SIZE]) { + const psk_t psk = make_psk(7); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); + Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); + uint8_t msg[MAX_HANDSHAKE_SIZE]; + size_t msg_len = initiator.write_message(msg, sizeof(msg)); + ASSERT_EQ(responder.read_message(msg, msg_len), 0); + size_t reply_len = 0; + ASSERT_EQ(responder.write_message(msg, sizeof(msg), reply_len), 0); + ASSERT_GE(reply_len, SPARE_EPHEMERAL_KEY_SIZE); + std::memcpy(responder_e, msg, SPARE_EPHEMERAL_KEY_SIZE); + ASSERT_EQ(initiator.read_message(msg, reply_len), 0); + ASSERT_EQ(responder.action(), Action::ACTION_SPLIT); +} + +TEST(SpareEphemeralTest, EmptySlotLeavesHandshakeToGenerate) { + ASSERT_FALSE(has_spare_ephemeral()); + NoiseResponderHandshake responder; + uint8_t responder_e[SPARE_EPHEMERAL_KEY_SIZE]; + run_handshake(responder, responder_e); + EXPECT_FALSE(has_spare_ephemeral()); +} + +TEST(SpareEphemeralTest, ConsumeHandsTheKeyToANewState) { + prepare_spare_ephemeral(); + ASSERT_TRUE(has_spare_ephemeral()); + const NoiseProtocolId nid = { + .prefix_id = NOISE_PREFIX_STANDARD, + .pattern_id = NOISE_PATTERN_NN, + .modifier_ids = {NOISE_MODIFIER_PSK0}, + .dh_id = NOISE_DH_CURVE25519, + .cipher_id = NOISE_CIPHER_CHACHAPOLY, + .hash_id = NOISE_HASH_SHA256, + .hybrid_id = NOISE_DH_NONE, + }; + NoiseHandshakeState *state = nullptr; + ASSERT_EQ(noise_handshakestate_new_by_id(&state, &nid, NOISE_ROLE_RESPONDER), 0); + const psk_t psk = make_psk(7); + ASSERT_EQ(noise_handshakestate_set_pre_shared_key(state, psk.data(), psk.size()), 0); + ASSERT_EQ(noise_handshakestate_set_prologue(state, PROLOGUE, sizeof(PROLOGUE)), 0); + EXPECT_EQ(consume_spare_ephemeral(state), 0); + EXPECT_FALSE(has_spare_ephemeral()); + noise_handshakestate_free(state); +} + +TEST(SpareEphemeralTest, SlotKeyIsOnTheWireAndConsumedOnce) { + prepare_spare_ephemeral(); + ASSERT_TRUE(has_spare_ephemeral()); + uint8_t expected_pub[SPARE_EPHEMERAL_KEY_SIZE]; + std::memcpy(expected_pub, spare_ephemeral + SPARE_EPHEMERAL_KEY_SIZE, sizeof(expected_pub)); + + NoiseResponderHandshake first; + uint8_t responder_e[SPARE_EPHEMERAL_KEY_SIZE]; + run_handshake(first, responder_e); + // The spare, not a generated key, went out; and it went out once + EXPECT_EQ(std::memcmp(responder_e, expected_pub, sizeof(expected_pub)), 0); + EXPECT_FALSE(has_spare_ephemeral()); + + NoiseResponderHandshake second; + run_handshake(second, responder_e); + EXPECT_NE(std::memcmp(responder_e, expected_pub, sizeof(expected_pub)), 0); + EXPECT_FALSE(has_spare_ephemeral()); +} + TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // The documented retry shape: a repeated init() frees the previous state // and starts over. The first message under the new key authenticating From 54f6fb0eb2e3e42865ebf3a4b21eddf105e50725 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:25:48 -0500 Subject: [PATCH 014/221] [espidf] Skip ldgen when only app code changes (#18857) --- .github/workflows/ci-docker.yml | 1 + esphome/build_gen/espidf.py | 62 ++++++++- esphome/espidf/toolchain.py | 65 +++++++++- tests/unit_tests/build_gen/test_espidf.py | 47 +++++++ tests/unit_tests/test_espidf_toolchain.py | 148 ++++++++++++++++++++++ 5 files changed, 321 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 2784f13511..5bcdb22cb8 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -219,5 +219,6 @@ jobs: run: | docker run --rm \ -v "${{ github.workspace }}/docker/test_configs:/config" \ + -e ESPHOME_LDGEN_STRICT=1 \ "ghcr.io/esphome/esphome-amd64:${{ needs.check-docker.outputs.tag }}" \ compile "${{ matrix.id }}.yaml" diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 9f48013a4e..124fa3ddb1 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -20,7 +20,7 @@ from esphome.framework_helpers import ( get_project_cxx_compile_flags, get_project_link_flags, ) -from esphome.helpers import mkdir_p, write_file_if_changed +from esphome.helpers import get_bool_env, mkdir_p, write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -35,6 +35,46 @@ list(FILTER esphome_cxx_compile_options EXCLUDE REGEX "^-std=") list(APPEND esphome_cxx_compile_options "-std={standard}") idf_build_set_property(CXX_COMPILE_OPTIONS "${{esphome_cxx_compile_options}}")""" +# Drops the app archive from ldgen's inputs so app-only edits skip the +# sections.ld regeneration. Safe: no mapping fragment references it +# (run_compile re-checks each build). Filters only the top-level call; +# the prior definition stays reachable with an underscore prefix. +_LDGEN_OVERRIDE = """\ +if(COMMAND __ldgen_get_lib_deps_of_target) + set_property(GLOBAL PROPERTY ESPHOME_LDGEN_ARMED 1) + function(__ldgen_get_lib_deps_of_target target out_list_var) + if(NOT COMMAND ___ldgen_get_lib_deps_of_target) + message(FATAL_ERROR "ESPHome ldgen override lost the original " + "implementation; set ESPHOME_LDGEN_FULL_DEPS=1 and rebuild.") + endif() + ___ldgen_get_lib_deps_of_target(${target} ${out_list_var}) + if(out_list_var STREQUAL "ldgen_libraries") + set_property(GLOBAL PROPERTY ESPHOME_LDGEN_FILTERED 1) + list(LENGTH ${out_list_var} esphome_ldgen_before) + list(REMOVE_ITEM ${out_list_var} idf::src __idf_src) + list(LENGTH ${out_list_var} esphome_ldgen_after) + if(esphome_ldgen_before EQUAL esphome_ldgen_after) + message(@SEVERITY@ "ESPHome ldgen app archive exclusion matched " + "nothing; app edits will regenerate sections.ld.") + endif() + endif() + set(${out_list_var} "${${out_list_var}}" PARENT_SCOPE) + endfunction() +else() + message(@MISSING@ "ESPHome ldgen override target not found; " + "app edits will regenerate sections.ld.") +endif()""" + +# Runs after project() so the walk has happened; catches the remaining +# silent path where the top-level out-var was renamed. +_LDGEN_OVERRIDE_CHECK = """\ +get_property(esphome_ldgen_armed GLOBAL PROPERTY ESPHOME_LDGEN_ARMED) +get_property(esphome_ldgen_filtered GLOBAL PROPERTY ESPHOME_LDGEN_FILTERED) +if(esphome_ldgen_armed AND NOT esphome_ldgen_filtered) + message(@SEVERITY@ "ESPHome ldgen override never filtered the app " + "archive; app edits will regenerate sections.ld.") +endif()""" + def get_available_components() -> list[str] | None: """List the built-in ESP-IDF components from ``project_description.json``. @@ -125,6 +165,22 @@ def get_project_cmakelists( else "" ) + # Stops the ~3s sections.ld regeneration on app-only edits; see + # _LDGEN_OVERRIDE. ESPHOME_LDGEN_FULL_DEPS=1 restores stock behavior; + # ESPHOME_LDGEN_STRICT=1 (CI) fails the configure when an IDF bump + # breaks the override instead of degrading to stock deps. + if get_bool_env("ESPHOME_LDGEN_FULL_DEPS"): + ldgen_override = "" + ldgen_override_check = "" + else: + strict = get_bool_env("ESPHOME_LDGEN_STRICT") + severity = "FATAL_ERROR" if strict else "WARNING" + missing = "FATAL_ERROR" if strict else "STATUS" + ldgen_override = _LDGEN_OVERRIDE.replace("@SEVERITY@", severity).replace( + "@MISSING@", missing + ) + ldgen_override_check = _LDGEN_OVERRIDE_CHECK.replace("@SEVERITY@", severity) + # CMake variables registered via cg.add_cmake_arg(). Emitted before # include(project.cmake) so values like EXCLUDE_COMPONENTS are already # set when project.cmake seeds the component list, and on minimal @@ -202,6 +258,8 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src) include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +{ldgen_override} + {cpp_standard_options} {cxx_compile_options} @@ -214,6 +272,8 @@ include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) project({CORE.name}) +{ldgen_override_check} + # Emit per-memory-type JSON size data for ESPHome to read post-build. # json2 stays small; raw dumps every symbol (~2s on a large map) and # this command runs inside the link edge, blocking everything downstream. diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 4243eff0ba..d73dbf6b7e 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -1,6 +1,7 @@ """ESP-IDF direct build API for ESPHome.""" from dataclasses import dataclass, field +import fnmatch import hashlib import json import logging @@ -25,7 +26,7 @@ from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary -from esphome.helpers import add_git_ceiling_directory, write_file +from esphome.helpers import add_git_ceiling_directory, get_bool_env, write_file _LOGGER = logging.getLogger(__name__) @@ -647,6 +648,65 @@ def _patch_memory_segments(): _LOGGER.warning("Could not patch memory segments in %s", memory_ld) +_LDGEN_FRAGMENTS_RE = re.compile(r'--fragments-list\s+"([^"]+)"') +_LDGEN_ARCHIVE_RE = re.compile(r"^\s*archive:\s*(\S+)", re.MULTILINE) + + +def _fragment_maps_app_archive(text: str) -> bool: + """True when an archive: spec selects libsrc.a, the archive of the src + component excluded as idf::src/__idf_src in build_gen/espidf.py. + + The bare * is IDF's stock catch-all; its archive-level entries resolve + in the linker against all link inputs, so it stays safe when the + archive is excluded from ldgen's own inputs. + """ + return any( + value != "*" and fnmatch.fnmatch("libsrc.a", value) + for value in _LDGEN_ARCHIVE_RE.findall(text) + ) + + +def _ldgen_check_skip(msg: str, strict: bool) -> None: + """A skipped fragment check is debug for users, fatal under strict.""" + if strict: + raise EsphomeError(f"ldgen fragment check: {msg} (ESPHOME_LDGEN_STRICT)") + _LOGGER.debug("Skipping ldgen fragment check: %s", msg) + + +def _warn_if_app_archive_mapped() -> None: + """Belt for the ldgen exclusion (see build_gen/espidf.py): warn if any + linker fragment names the app archive, since ldgen would silently skip + remapping it rather than fail. + """ + strict = get_bool_env("ESPHOME_LDGEN_STRICT") + build_ninja = CORE.relative_build_path("build", "build.ninja") + try: + ninja_text = build_ninja.read_text(encoding="utf-8", errors="replace") + except OSError as e: + _ldgen_check_skip(f"could not read {build_ninja}: {e}", strict) + return + match = _LDGEN_FRAGMENTS_RE.search(ninja_text) + if match is None: + _ldgen_check_skip(f"no --fragments-list in {build_ninja}", strict) + return + for fragment in match.group(1).split(";"): + try: + text = Path(fragment).read_text(encoding="utf-8", errors="replace") + except OSError as e: + _ldgen_check_skip(f"could not read {fragment}: {e}", strict) + continue + if _fragment_maps_app_archive(text): + msg = ( + f"Linker fragment {fragment} maps the app archive; its " + "entries may be skipped. Set ESPHOME_LDGEN_FULL_DEPS=1 " + "and rebuild." + ) + if strict: + raise EsphomeError(msg) + _LOGGER.warning("%s", msg) + return + + def run_compile(config, verbose: bool) -> int: """Compile the ESP-IDF project. @@ -677,6 +737,9 @@ def run_compile(config, verbose: bool) -> int: else: _LOGGER.info("Build configuration is up to date") + if not get_bool_env("ESPHOME_LDGEN_FULL_DEPS"): + _warn_if_app_archive_mapped() + # In testing mode, generate the linker script first, patch DRAM/IRAM sizes, # then build. memory.ld is regenerated by ninja during the build phase, # so we must patch after it's generated but before linking (same timing diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 4a84284796..7d53ee1e5e 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -163,6 +163,53 @@ def test_has_discovered_components_after_configure(tmp_path: Path) -> None: assert has_discovered_components() +@pytest.mark.parametrize("minimal", [False, True]) +def test_get_project_cmakelists_emits_ldgen_override( + minimal: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """Both renders override the ldgen dep walker to drop the app archive, + after include(project.cmake) which defines the original.""" + monkeypatch.delenv("ESPHOME_LDGEN_FULL_DEPS", raising=False) + monkeypatch.delenv("ESPHOME_LDGEN_STRICT", raising=False) + content = _render(minimal=minimal) + assert "REMOVE_ITEM ${out_list_var} idf::src __idf_src" in content + # Quoted so spaced elements survive and an empty list stays defined + assert 'set(${out_list_var} "${${out_list_var}}" PARENT_SCOPE)' in content + assert 'message(WARNING "ESPHome ldgen app archive exclusion' in content + assert 'message(STATUS "ESPHome ldgen override target not found' in content + assert 'message(WARNING "ESPHome ldgen override never filtered' in content + assert content.index("tools/cmake/project.cmake") < content.index( + "function(__ldgen_get_lib_deps_of_target" + ) + # The never-filtered check must run after project() has walked the deps + assert content.index("project(test)") < content.index("esphome_ldgen_armed GLOBAL") + + +def test_get_project_cmakelists_ldgen_strict_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ESPHOME_LDGEN_STRICT turns both degradation paths into hard errors so + CI fails right away when an IDF bump breaks the override.""" + monkeypatch.delenv("ESPHOME_LDGEN_FULL_DEPS", raising=False) + monkeypatch.setenv("ESPHOME_LDGEN_STRICT", "1") + content = _render() + assert 'message(FATAL_ERROR "ESPHome ldgen app archive exclusion' in content + assert 'message(FATAL_ERROR "ESPHome ldgen override target not found' in content + assert 'message(FATAL_ERROR "ESPHome ldgen override never filtered' in content + assert "@SEVERITY@" not in content + assert "@MISSING@" not in content + + +def test_get_project_cmakelists_ldgen_full_deps_escape_hatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ESPHOME_LDGEN_FULL_DEPS restores stock ldgen behavior.""" + monkeypatch.setenv("ESPHOME_LDGEN_FULL_DEPS", "true") + content = _render() + assert "__ldgen_get_lib_deps_of_target" not in content + assert "esphome_ldgen_armed" not in content + + def test_get_project_cmakelists_size_command_uses_json2() -> None: """The POST_BUILD size command uses the cheap json2 format, with --ng only on the 1.x tool bundled with IDF < 6.""" diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index c012785e89..1b40460b91 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -630,6 +630,154 @@ def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) assert toolchain.load_cached_builtin_components() is None +@pytest.fixture(autouse=True) +def _clear_ldgen_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Isolate tests from ambient ldgen escape hatch and strict knobs.""" + monkeypatch.delenv("ESPHOME_LDGEN_STRICT", raising=False) + monkeypatch.delenv("ESPHOME_LDGEN_FULL_DEPS", raising=False) + + +def _write_fragments_build_ninja(tmp_path: Path, fragments: list[Path]) -> None: + build_dir = CORE.relative_build_path("build") + build_dir.mkdir(parents=True, exist_ok=True) + frag_list = ";".join(str(f) for f in fragments) + (build_dir / "build.ninja").write_text( + f' COMMAND = python ldgen.py --fragments-list "{frag_list}" --input x\n' + ) + + +def test_warn_if_app_archive_mapped_warns( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A fragment naming the app archive, even with trailing text or leading + whitespace, triggers the loud warning.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\n archive: libsrc.a # app\nentries:\n") + _write_fragments_build_ninja(tmp_path, [frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" in caplog.text + + +def test_warn_if_app_archive_mapped_scans_past_unreadable( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unreadable fragment doesn't stop later fragments being checked.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\narchive: libsrc.a\n") + _write_fragments_build_ninja(tmp_path, [tmp_path / "missing.lf", frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" in caplog.text + + +def test_warn_if_app_archive_mapped_strict_no_fragments_list( + setup_core: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Under strict, a build.ninja the check can't parse fails the build.""" + monkeypatch.setenv("ESPHOME_LDGEN_STRICT", "1") + _setup_build(setup_core) + build_dir = CORE.relative_build_path("build") + build_dir.mkdir(parents=True, exist_ok=True) + (build_dir / "build.ninja").write_text("rule CXX\n command = gcc\n") + with pytest.raises(EsphomeError, match="no --fragments-list"): + toolchain._warn_if_app_archive_mapped() + + +def test_warn_if_app_archive_mapped_strict_raises( + setup_core: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Under ESPHOME_LDGEN_STRICT a mapped app archive fails the build.""" + monkeypatch.setenv("ESPHOME_LDGEN_STRICT", "1") + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\narchive: libsrc.a\n") + _write_fragments_build_ninja(tmp_path, [frag]) + with pytest.raises(EsphomeError, match="maps the app archive"): + toolchain._warn_if_app_archive_mapped() + + +def test_warn_if_app_archive_mapped_glob( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A glob archive spec that selects the app archive is also flagged.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\narchive: lib*\n") + _write_fragments_build_ninja(tmp_path, [frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" in caplog.text + + +def test_warn_if_app_archive_mapped_clean( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Normal fragments, including IDF's stock archive: * catch-all, + produce no warning.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text( + "[mapping:freertos]\narchive: libfreertos.a\n[mapping:default]\narchive: *\n" + ) + _write_fragments_build_ninja(tmp_path, [frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" not in caplog.text + + +def test_warn_if_app_archive_mapped_missing_fragment( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unreadable fragment file is non-fatal.""" + _setup_build(setup_core) + _write_fragments_build_ninja(tmp_path, [tmp_path / "missing.lf"]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" not in caplog.text + + +def test_warn_if_app_archive_mapped_no_build_ninja(setup_core: Path) -> None: + """No build.ninja yet is a quiet no-op.""" + _setup_build(setup_core) + toolchain._warn_if_app_archive_mapped() + + +def test_warn_if_app_archive_mapped_no_fragments_list(setup_core: Path) -> None: + """A build.ninja without a fragments-list argument is a quiet no-op.""" + _setup_build(setup_core) + build_dir = CORE.relative_build_path("build") + build_dir.mkdir(parents=True, exist_ok=True) + (build_dir / "build.ninja").write_text("rule CXX\n command = gcc\n") + toolchain._warn_if_app_archive_mapped() + + +def test_run_compile_runs_fragment_check(setup_core: Path) -> None: + """The fragment belt runs by default on every compile.""" + _setup_build(setup_core) + + with ( + _up_to_date_compile(), + patch.object(toolchain, "_warn_if_app_archive_mapped") as mock_check, + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + + mock_check.assert_called_once() + + +def test_run_compile_full_deps_skips_fragment_check( + setup_core: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ESPHOME_LDGEN_FULL_DEPS disables the fragment belt with the override.""" + monkeypatch.setenv("ESPHOME_LDGEN_FULL_DEPS", "1") + _setup_build(setup_core) + + with ( + _up_to_date_compile(), + patch.object(toolchain, "_warn_if_app_archive_mapped") as mock_check, + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + + mock_check.assert_not_called() + + @pytest.mark.parametrize("limit", [1, None]) def test_run_compile_passes_compile_process_limit( setup_core: Path, limit: int | None From eb423be9d25a895cbd18e84c99fae3e63fa2ceef Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:36:09 +0200 Subject: [PATCH 015/221] [socket] Add an IPv4 lookup next to set_sockaddr (#19909) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- esphome/components/socket/ipv4_resolve.cpp | 205 ++++++++++++++++++ esphome/components/socket/ipv4_resolve.h | 103 +++++++++ .../socket/test_ipv4_resolve_host.cpp | 71 ++++++ .../ipv4_resolve_test_component/__init__.py | 22 ++ .../ipv4_resolve_test_component.cpp | 46 ++++ .../ipv4_resolve_test_component.h | 12 + .../fixtures/socket_ipv4_resolve.yaml | 17 ++ tests/integration/test_socket_ipv4_resolve.py | 51 +++++ 8 files changed, 527 insertions(+) create mode 100644 esphome/components/socket/ipv4_resolve.cpp create mode 100644 esphome/components/socket/ipv4_resolve.h create mode 100644 tests/components/socket/test_ipv4_resolve_host.cpp create mode 100644 tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py create mode 100644 tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp create mode 100644 tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h create mode 100644 tests/integration/fixtures/socket_ipv4_resolve.yaml create mode 100644 tests/integration/test_socket_ipv4_resolve.py diff --git a/esphome/components/socket/ipv4_resolve.cpp b/esphome/components/socket/ipv4_resolve.cpp new file mode 100644 index 0000000000..d2316cd064 --- /dev/null +++ b/esphome/components/socket/ipv4_resolve.cpp @@ -0,0 +1,205 @@ +#include "ipv4_resolve.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include "socket.h" +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +#include + +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) +#include "lwip/dns.h" +#else +#include +#endif + +namespace esphome::socket { + +static const char *const TAG = "socket"; + +bool Ipv4Resolve::consume_failure() { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + uint8_t expected = STATE_FAILED; + return this->state_word_.compare_exchange_strong(expected, STATE_IDLE); +#else + if (this->state_word_ != STATE_FAILED) { + return false; + } + this->state_word_ = STATE_IDLE; + return true; +#endif +} + +void Ipv4Resolve::forget() { +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + // The generation moves first; RESOLVING stays set until the callback + // drops the stale result, so start() cannot replace that lookup. + this->epoch_ = this->epoch_ + 1; + this->addr_word_ = 0; + if (this->state_word_ != STATE_RESOLVING) { + this->state_word_ = STATE_IDLE; + } +#elif defined(IPV4_RESOLVE_ATOMIC_STATE) + // PUBLISHING stays, so start() cannot queue a second callback + // while this one is storing the address. + uint8_t expected = STATE_RESOLVING; + if (this->state_word_.compare_exchange_strong(expected, STATE_IDLE)) { + this->set_addr_(0); + return; + } + if (this->state_() == STATE_PUBLISHING) { + return; + } + this->set_state_(STATE_IDLE); + this->set_addr_(0); +#else + this->set_state_(STATE_IDLE); + this->set_addr_(0); +#endif +} + +socklen_t Ipv4Resolve::to_sockaddr(struct sockaddr *dest, socklen_t destlen, uint16_t port) const { + if (this->state_() != STATE_RESOLVED || destlen < sizeof(sockaddr_in)) { + return 0; + } + auto *in = reinterpret_cast(dest); + memset(in, 0, sizeof(sockaddr_in)); + in->sin_family = AF_INET; + in->sin_port = htons(port); + in->sin_addr.s_addr = this->addr_(); + return sizeof(sockaddr_in); +} + +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) +bool Ipv4Resolve::drop_stale_(uint32_t expected) { + if (this->epoch_ == expected) { + return false; + } + this->state_word_ = STATE_IDLE; + this->addr_word_ = 0; + return true; +} +#endif + +void Ipv4Resolve::dns_found(const char *name, const ip_addr_t *addr, void *arg) { + auto *self = static_cast(arg); +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + const uint32_t expected = self->pending_epoch_; + if (self->drop_stale_(expected)) { + return; + } +#endif + if (addr != nullptr && IP_IS_V4(addr)) { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + // Only the callback that wins the exchange owns the lookup. + uint8_t expected = STATE_RESOLVING; + if (!self->state_word_.compare_exchange_strong(expected, STATE_PUBLISHING)) { + return; + } + self->set_addr_(ip4_addr_get_u32(ip_2_ip4(addr))); + expected = STATE_PUBLISHING; + // Lost ownership; the leftover address is gated by to_sockaddr()'s state check. + if (!self->state_word_.compare_exchange_strong(expected, STATE_RESOLVED)) { + return; + } +#elif defined(IPV4_RESOLVE_VOLATILE_EPOCH) + self->addr_word_ = ip4_addr_get_u32(ip_2_ip4(addr)); + self->state_word_ = STATE_RESOLVED; + if (self->drop_stale_(expected)) { + return; + } +#else + // Do not publish over a newer start(). + if (self->state_() != STATE_RESOLVING) { + return; + } + self->set_addr_(ip4_addr_get_u32(ip_2_ip4(addr))); + self->set_state_(STATE_RESOLVED); +#endif + } else { + ESP_LOGW(self->tag_ != nullptr ? self->tag_ : TAG, "DNS failed for %s", name); +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + uint8_t expected = STATE_RESOLVING; + self->state_word_.compare_exchange_strong(expected, STATE_FAILED); +#elif defined(IPV4_RESOLVE_VOLATILE_EPOCH) + self->state_word_ = STATE_FAILED; + if (self->drop_stale_(expected)) { + return; + } +#else + if (self->state_() != STATE_RESOLVING) { + return; + } + self->set_state_(STATE_FAILED); +#endif + } +} +#endif + +void Ipv4Resolve::start(const char *host, uint16_t port, const char *tag) { + const uint8_t state = this->state_(); + if (state == STATE_RESOLVED || state == STATE_RESOLVING || state == STATE_PUBLISHING) { + return; + } + this->set_state_(STATE_IDLE); + this->tag_ = tag; + struct sockaddr_storage literal; + if (set_sockaddr(reinterpret_cast(&literal), sizeof(literal), host, port) != 0) { + if (literal.ss_family == AF_INET) { + auto *in = reinterpret_cast(&literal); + this->set_addr_(in->sin_addr.s_addr); + this->set_state_(STATE_RESOLVED); + return; + } + this->set_state_(STATE_FAILED); + ESP_LOGW(tag, "Not an IPv4 address: %s", host); + return; + } +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) + ip_addr_t cached; + err_t err; + { + LwIPLock lock; +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + this->pending_epoch_ = this->epoch_; +#endif + this->set_state_(STATE_RESOLVING); + err = dns_gethostbyname_addrtype(host, &cached, &Ipv4Resolve::dns_found, this, LWIP_DNS_ADDRTYPE_IPV4); + if (err != ERR_INPROGRESS && this->state_() == STATE_RESOLVING) { + this->set_state_(STATE_IDLE); + } + } + if (err == ERR_OK && IP_IS_V4(&cached)) { + this->set_addr_(ip4_addr_get_u32(ip_2_ip4(&cached))); + this->set_state_(STATE_RESOLVED); + return; + } + if (err == ERR_INPROGRESS || this->state_() == STATE_RESOLVED || this->state_() == STATE_PUBLISHING) { + return; + } +#else + struct addrinfo hints {}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo *res = nullptr; + if (getaddrinfo(host, nullptr, &hints, &res) == 0 && res != nullptr) { + auto *in = reinterpret_cast(res->ai_addr); + if (res->ai_family == AF_INET) { + this->set_addr_(in->sin_addr.s_addr); + this->set_state_(STATE_RESOLVED); + } + freeaddrinfo(res); + if (this->ready()) { + return; + } + } +#endif + this->set_state_(STATE_FAILED); + ESP_LOGW(tag, "Could not resolve %s", host); +} + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/socket/ipv4_resolve.h b/esphome/components/socket/ipv4_resolve.h new file mode 100644 index 0000000000..628efb32d9 --- /dev/null +++ b/esphome/components/socket/ipv4_resolve.h @@ -0,0 +1,103 @@ +#pragma once + +#include "headers.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include + +// MULTI_ATOMICS: one atomic state word, races closed by compare_exchange. +// SINGLE: volatile state, the callback never runs beside loop(). +// MULTI_NO_ATOMICS: volatile state plus a generation; BK72xx has no +// compare_exchange and the DNS callback runs on the tcpip thread. +#if defined(ESPHOME_THREAD_MULTI_NO_ATOMICS) +#define IPV4_RESOLVE_VOLATILE_EPOCH +#elif defined(ESPHOME_THREAD_SINGLE) +#define IPV4_RESOLVE_VOLATILE +#else +#define IPV4_RESOLVE_ATOMIC_STATE +#include +#endif + +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) +#include "lwip/ip_addr.h" +#endif + +namespace esphome::socket { + +/// One IPv4 literal or hostname. Must outlive a pending lookup. +class Ipv4Resolve { + public: + static constexpr uint8_t STATE_IDLE = 0; + static constexpr uint8_t STATE_RESOLVING = 1; + static constexpr uint8_t STATE_RESOLVED = 2; + static constexpr uint8_t STATE_FAILED = 3; + // The callback holds this between winning the lookup and storing the address. + static constexpr uint8_t STATE_PUBLISHING = 4; + + /// Drop the stored address so the next start() resolves again. + /// A result already publishing may still land, so ready() can be true + /// right after this; after changing hosts, forget() until ready() is false. + void forget(); + /// Drop a failed lookup so the next start() tries again. + bool consume_failure(); + bool ready() const { return this->state_() == STATE_RESOLVED; } + /// Write the stored address into dest. Returns 0 until ready() is true. + socklen_t to_sockaddr(struct sockaddr *dest, socklen_t destlen, uint16_t port) const; + /// Resolve host; tag names the failure log. On host and Zephyr this + /// blocks in getaddrinfo(). + void start(const char *host, uint16_t port, const char *tag); + + private: +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) + static void dns_found(const char *name, const ip_addr_t *addr, void *arg); +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + bool drop_stale_(uint32_t expected); +#endif +#endif + uint8_t state_() const { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + return this->state_word_.load(); +#else + return this->state_word_; +#endif + } + uint32_t addr_() const { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + return this->addr_word_.load(); +#else + return this->addr_word_; +#endif + } + void set_state_(uint8_t state) { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + this->state_word_.store(state); +#else + this->state_word_ = state; +#endif + } + void set_addr_(uint32_t addr) { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + this->addr_word_.store(addr); +#else + this->addr_word_ = addr; +#endif + } + const char *tag_{nullptr}; +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + std::atomic addr_word_{0}; + std::atomic state_word_{STATE_IDLE}; +#elif defined(IPV4_RESOLVE_VOLATILE_EPOCH) + volatile uint32_t addr_word_{0}; + volatile uint32_t epoch_{0}; + volatile uint32_t pending_epoch_{0}; + volatile uint8_t state_word_{STATE_IDLE}; +#else + volatile uint32_t addr_word_{0}; + volatile uint8_t state_word_{STATE_IDLE}; +#endif +}; + +} // namespace esphome::socket + +#endif diff --git a/tests/components/socket/test_ipv4_resolve_host.cpp b/tests/components/socket/test_ipv4_resolve_host.cpp new file mode 100644 index 0000000000..6381135c40 --- /dev/null +++ b/tests/components/socket/test_ipv4_resolve_host.cpp @@ -0,0 +1,71 @@ +#include + +#include + +#include "esphome/components/socket/ipv4_resolve.h" + +#ifdef USE_HOST + +namespace esphome::socket::testing { + +TEST(Ipv4Resolve, LiteralIsReadyWithoutDns) { + Ipv4Resolve lookup; + lookup.start("192.168.1.1", 1, "test"); + EXPECT_TRUE(lookup.ready()); + + struct sockaddr_storage addr {}; + socklen_t len = lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), 6053); + ASSERT_EQ(len, sizeof(sockaddr_in)); + auto *in = reinterpret_cast(&addr); + EXPECT_EQ(in->sin_family, AF_INET); + EXPECT_EQ(ntohs(in->sin_port), 6053); + EXPECT_EQ(in->sin_addr.s_addr, htonl(0xC0A80101)); +} + +TEST(Ipv4Resolve, ForgetDropsTheLiteral) { + Ipv4Resolve lookup; + lookup.start("10.0.0.5", 80, "test"); + ASSERT_TRUE(lookup.ready()); + lookup.forget(); + EXPECT_FALSE(lookup.ready()); + + struct sockaddr_storage addr {}; + EXPECT_EQ(lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), 80), 0u); +} + +TEST(Ipv4Resolve, Ipv6LiteralIsRejected) { + Ipv4Resolve lookup; + lookup.start("::1", 443, "test"); + EXPECT_FALSE(lookup.ready()); + EXPECT_TRUE(lookup.consume_failure()); +} + +TEST(Ipv4Resolve, ShortBufferWritesNothing) { + Ipv4Resolve lookup; + lookup.start("192.0.2.10", 502, "test"); + ASSERT_TRUE(lookup.ready()); + struct sockaddr_in addr {}; + EXPECT_EQ(lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr) - 1, 502), 0u); +} + +TEST(Ipv4Resolve, RetryAfterFailureKeepsTheAddress) { + Ipv4Resolve lookup; + lookup.start("::1", 443, "test"); + EXPECT_FALSE(lookup.ready()); + lookup.start("192.168.1.1", 1, "test"); + EXPECT_TRUE(lookup.ready()); + EXPECT_FALSE(lookup.consume_failure()); + EXPECT_TRUE(lookup.ready()); +} + +TEST(Ipv4Resolve, ForgetDropsAStaleFailure) { + Ipv4Resolve lookup; + lookup.start("::1", 443, "test"); + lookup.forget(); + EXPECT_FALSE(lookup.consume_failure()); + EXPECT_FALSE(lookup.ready()); +} + +} // namespace esphome::socket::testing + +#endif diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py new file mode 100644 index 0000000000..3e397d6186 --- /dev/null +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py @@ -0,0 +1,22 @@ +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_ID +from esphome.types import ConfigType + +AUTO_LOAD = ["socket"] + +ipv4_resolve_test_component_ns = cg.esphome_ns.namespace("ipv4_resolve_test_component") +Ipv4ResolveTestComponent = ipv4_resolve_test_component_ns.class_( + "Ipv4ResolveTestComponent", cg.Component +) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(Ipv4ResolveTestComponent), + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp new file mode 100644 index 0000000000..e4a6e914e8 --- /dev/null +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp @@ -0,0 +1,46 @@ +#include "ipv4_resolve_test_component.h" +#include "esphome/components/socket/ipv4_resolve.h" +#include "esphome/core/log.h" + +#include + +namespace esphome::ipv4_resolve_test_component { + +static const char *const TAG = "ipv4_resolve_test"; + +static bool check_sockaddr(socket::Ipv4Resolve &lookup, uint16_t port, uint32_t expected) { + struct sockaddr_storage addr {}; + socklen_t len = lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), port); + if (len != sizeof(sockaddr_in)) { + return false; + } + auto *in = reinterpret_cast(&addr); + return in->sin_family == AF_INET && ntohs(in->sin_port) == port && in->sin_addr.s_addr == htonl(expected); +} + +void Ipv4ResolveTestComponent::setup() { + ESP_LOGI(TAG, "IPv4 resolve test starting"); + + socket::Ipv4Resolve lookup; + struct sockaddr_storage addr {}; + + lookup.start("192.168.1.1", 1, TAG); + bool ok = lookup.ready() && check_sockaddr(lookup, 6053, 0xC0A80101); + ESP_LOGI(TAG, "Literal resolve: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + lookup.forget(); + ok = !lookup.ready() && lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), 80) == 0; + ESP_LOGI(TAG, "Forget drops address: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + lookup.start("::1", 443, TAG); + ok = !lookup.ready() && lookup.consume_failure(); + ESP_LOGI(TAG, "IPv6 literal rejected: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + lookup.start("localhost", 6053, TAG); + ok = lookup.ready() && check_sockaddr(lookup, 6053, 0x7F000001); + ESP_LOGI(TAG, "Hostname resolve: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + ESP_LOGI(TAG, "IPv4 resolve test complete"); +} + +} // namespace esphome::ipv4_resolve_test_component diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h new file mode 100644 index 0000000000..b223f228e5 --- /dev/null +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h @@ -0,0 +1,12 @@ +#pragma once + +#include "esphome/core/component.h" + +namespace esphome::ipv4_resolve_test_component { + +class Ipv4ResolveTestComponent : public Component { + public: + void setup() override; +}; + +} // namespace esphome::ipv4_resolve_test_component diff --git a/tests/integration/fixtures/socket_ipv4_resolve.yaml b/tests/integration/fixtures/socket_ipv4_resolve.yaml new file mode 100644 index 0000000000..14931d0918 --- /dev/null +++ b/tests/integration/fixtures/socket_ipv4_resolve.yaml @@ -0,0 +1,17 @@ +esphome: + name: socket-ipv4-resolve-test + +host: + +api: + +logger: + level: INFO + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + components: [ipv4_resolve_test_component] + +ipv4_resolve_test_component: diff --git a/tests/integration/test_socket_ipv4_resolve.py b/tests/integration/test_socket_ipv4_resolve.py new file mode 100644 index 0000000000..5d7bd5a100 --- /dev/null +++ b/tests/integration/test_socket_ipv4_resolve.py @@ -0,0 +1,51 @@ +"""Integration test for the socket Ipv4Resolve helper on host.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + +CHECKS = ( + "Literal resolve", + "Forget drops address", + "IPv6 literal rejected", + "Hostname resolve", +) + + +@pytest.mark.asyncio +async def test_socket_ipv4_resolve( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Exercise Ipv4Resolve literals, forget, failure, and getaddrinfo on host.""" + test_complete = asyncio.Event() + results: dict[str, bool] = {} + + def on_log_line(line: str) -> None: + if "IPv4 resolve test complete" in line: + test_complete.set() + return + for check in CHECKS: + if f"{check}:" in line: + results[check] = "PASSED" in line + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "socket-ipv4-resolve-test" + + try: + await asyncio.wait_for(test_complete.wait(), timeout=10.0) + except TimeoutError: + pytest.fail("IPv4 resolve test timed out") + + for check in CHECKS: + assert results.get(check), f"{check} check failed or never ran" From 9a855b43773d193e318dcd48c4bf836ba46dba2f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:36:37 -0500 Subject: [PATCH 016/221] [ina260] Use the sub_* helpers for optional child entities (#19977) --- esphome/components/ina260/sensor.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/esphome/components/ina260/sensor.py b/esphome/components/ina260/sensor.py index b7b94a248b..20cf1fee45 100644 --- a/esphome/components/ina260/sensor.py +++ b/esphome/components/ina260/sensor.py @@ -59,14 +59,7 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_BUS_VOLTAGE in config: - sens = await sensor.new_sensor(config[CONF_BUS_VOLTAGE]) - cg.add(var.set_bus_voltage_sensor(sens)) - - if CONF_CURRENT in config: - sens = await sensor.new_sensor(config[CONF_CURRENT]) - cg.add(var.set_current_sensor(sens)) - - if CONF_POWER in config: - sens = await sensor.new_sensor(config[CONF_POWER]) - cg.add(var.set_power_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_BUS_VOLTAGE, var.set_bus_voltage_sensor) + await sensors(CONF_CURRENT, var.set_current_sensor) + await sensors(CONF_POWER, var.set_power_sensor) From 852fb0c5a5279defb5aae7941590d48a4842eb06 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:09 -0500 Subject: [PATCH 017/221] [midea] Use the sub_* helpers for optional child entities (#19976) --- esphome/components/midea/climate.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/midea/climate.py b/esphome/components/midea/climate.py index e7a684babe..da952ab87b 100644 --- a/esphome/components/midea/climate.py +++ b/esphome/components/midea/climate.py @@ -214,15 +214,10 @@ async def to_code(config): cg.add(var.set_custom_presets(config[CONF_CUSTOM_PRESETS])) if CONF_CUSTOM_FAN_MODES in config: cg.add(var.set_custom_fan_modes(config[CONF_CUSTOM_FAN_MODES])) - if CONF_OUTDOOR_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_OUTDOOR_TEMPERATURE]) - cg.add(var.set_outdoor_temperature_sensor(sens)) - if CONF_POWER_USAGE in config: - sens = await sensor.new_sensor(config[CONF_POWER_USAGE]) - cg.add(var.set_power_sensor(sens)) - if CONF_HUMIDITY_SETPOINT in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY_SETPOINT]) - cg.add(var.set_humidity_setpoint_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_OUTDOOR_TEMPERATURE, var.set_outdoor_temperature_sensor) + await sensors(CONF_POWER_USAGE, var.set_power_sensor) + await sensors(CONF_HUMIDITY_SETPOINT, var.set_humidity_setpoint_sensor) # MideaUART uses the Arduino WiFi API for the network-notify frame # (WiFi auto-enables Network via dependency mapping). On ESP-IDF the # library talks to esp_wifi directly, so no library entry is needed. From af9c4cb778ff59d9cc4dae1268f39c20fbd94870 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:12 -0500 Subject: [PATCH 018/221] [xiaomi_mjyd02yla] Use the sub_* helpers for optional child entities (#19975) --- .../xiaomi_mjyd02yla/binary_sensor.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/esphome/components/xiaomi_mjyd02yla/binary_sensor.py b/esphome/components/xiaomi_mjyd02yla/binary_sensor.py index 5d562a2ca9..be5827c171 100644 --- a/esphome/components/xiaomi_mjyd02yla/binary_sensor.py +++ b/esphome/components/xiaomi_mjyd02yla/binary_sensor.py @@ -77,15 +77,9 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_IDLE_TIME in config: - sens = await sensor.new_sensor(config[CONF_IDLE_TIME]) - cg.add(var.set_idle_time(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) - if CONF_ILLUMINANCE in config: - sens = await sensor.new_sensor(config[CONF_ILLUMINANCE]) - cg.add(var.set_illuminance(sens)) - if CONF_LIGHT in config: - sens = await binary_sensor.new_binary_sensor(config[CONF_LIGHT]) - cg.add(var.set_light(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_IDLE_TIME, var.set_idle_time) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) + await sensors(CONF_ILLUMINANCE, var.set_illuminance) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_LIGHT, var.set_light) From b4cb164e5d09df85d55689cb73458fea54dd56f7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:39 -0500 Subject: [PATCH 019/221] [xiaomi_jqjcy01ym] Use the sub_* helpers for optional child entities (#19974) --- esphome/components/xiaomi_jqjcy01ym/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/xiaomi_jqjcy01ym/sensor.py b/esphome/components/xiaomi_jqjcy01ym/sensor.py index b791c40dca..4bfe9c8f75 100644 --- a/esphome/components/xiaomi_jqjcy01ym/sensor.py +++ b/esphome/components/xiaomi_jqjcy01ym/sensor.py @@ -72,15 +72,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_FORMALDEHYDE in config: - sens = await sensor.new_sensor(config[CONF_FORMALDEHYDE]) - cg.add(var.set_formaldehyde(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_FORMALDEHYDE, var.set_formaldehyde) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 879c5cfc5b33c756f695acc9aa907ec6d10bf71d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:55 -0500 Subject: [PATCH 020/221] [xiaomi_gcls002] Use the sub_* helpers for optional child entities (#19973) --- esphome/components/xiaomi_gcls002/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/xiaomi_gcls002/sensor.py b/esphome/components/xiaomi_gcls002/sensor.py index 63ba5da597..7513897024 100644 --- a/esphome/components/xiaomi_gcls002/sensor.py +++ b/esphome/components/xiaomi_gcls002/sensor.py @@ -71,15 +71,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_MOISTURE in config: - sens = await sensor.new_sensor(config[CONF_MOISTURE]) - cg.add(var.set_moisture(sens)) - if CONF_ILLUMINANCE in config: - sens = await sensor.new_sensor(config[CONF_ILLUMINANCE]) - cg.add(var.set_illuminance(sens)) - if CONF_CONDUCTIVITY in config: - sens = await sensor.new_sensor(config[CONF_CONDUCTIVITY]) - cg.add(var.set_conductivity(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_MOISTURE, var.set_moisture) + await sensors(CONF_ILLUMINANCE, var.set_illuminance) + await sensors(CONF_CONDUCTIVITY, var.set_conductivity) From 5e672588189a5b9dbe78bf0204863e0d03edc29a Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:59:00 +1300 Subject: [PATCH 021/221] [noise] Import noise-c and libsodium as ESP-IDF managed components (#18351) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/components/esp32/__init__.py | 38 +++- esphome/components/noise/__init__.py | 37 +++- esphome/espidf/clang_tidy.py | 32 +++- esphome/espidf/component.py | 13 +- esphome/idf_component.yml | 11 ++ esphome/platformio/library.py | 14 +- platformio.ini | 5 +- .../components/noise/test_noise_to_code.py | 163 ++++++++++++++++ .../test_esp32_write_idf_component_yml.py | 175 ++++++++++++++++++ tests/unit_tests/test_espidf_clang_tidy.py | 159 +++++++++++++++- tests/unit_tests/test_espidf_component.py | 106 +++++++++++ 11 files changed, 726 insertions(+), 27 deletions(-) create mode 100644 tests/unit_tests/components/noise/test_noise_to_code.py create mode 100644 tests/unit_tests/components/test_esp32_write_idf_component_yml.py diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index c3da73f510..49a115c9c5 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -322,6 +322,34 @@ ARDUINO_EXCLUDED_IDF_COMPONENTS = ( "joltwallet__littlefs", # LittleFS - ESPHome doesn't use filesystem ) +# Entries arduino-esp32 only declares below the given IDF version; stubbing one past +# it clashes with ESPHome's own managed component of the same short name. +ARDUINO_EXCLUDED_IDF_COMPONENTS_MAX_IDF: dict[str, cv.Version] = { + "espressif__libsodium": cv.Version(6, 0, 0), +} + + +def arduino_bundles_libsodium() -> bool: + """arduino-esp32 ships its own libsodium below IDF 6.0.""" + return ( + CORE.using_arduino + and idf_version() + < ARDUINO_EXCLUDED_IDF_COMPONENTS_MAX_IDF["espressif__libsodium"] + ) + + +def arduino_excluded_idf_components() -> set[str]: + """The arduino-bundled components to stub for this build's IDF version.""" + version = idf_version() + return { + component + for component in ARDUINO_EXCLUDED_IDF_COMPONENTS + if (max_version := ARDUINO_EXCLUDED_IDF_COMPONENTS_MAX_IDF.get(component)) + is None + or version < max_version + } + + # Mapping of Arduino libraries to IDF managed components they require # When an Arduino library is enabled via cg.add_library(), these components # are automatically un-stubbed from ARDUINO_EXCLUDED_IDF_COMPONENTS. @@ -3513,9 +3541,7 @@ def _write_idf_component_yml(): } # Only stub components that are not required by any enabled Arduino library - components_to_stub = ( - set(ARDUINO_EXCLUDED_IDF_COMPONENTS) - required_idf_components - ) + components_to_stub = arduino_excluded_idf_components() - required_idf_components stubs_dir = CORE.relative_build_path("component_stubs") stubs_dir.mkdir(exist_ok=True) @@ -3580,7 +3606,11 @@ def _write_idf_component_yml(): # Don't process arduino libraries if name not in ARDUINO_DISABLED_LIBRARIES ] - for component in generate_idf_components(libraries): + # A library also declared as a managed component is not converted too, or + # IDF sees the same requirement twice; converted components reach it through + # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}. + managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {})) + for component in generate_idf_components(libraries, managed=managed): dependencies[component.get_sanitized_name()] = { "override_path": str(component.path) } diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index c894c147cd..19ad36612d 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -12,6 +12,11 @@ from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] DOMAIN = "noise" +# Keep in sync with platformio.ini and esphome/idf_component.yml. +# LIBSODIUM_VERSION must match the version noise-c pins in its manifests. +NOISE_C_VERSION = "0.1.30" +LIBSODIUM_VERSION = "1.10021.11" + noise_ns = cg.esphome_ns.namespace("noise") CONFIG_SCHEMA = cv.Schema({}) @@ -92,6 +97,22 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: return ENCRYPTION_SCHEMA(config) +def _use_managed_components() -> bool: + """Whether noise-c and libsodium come from the ESP-IDF component registry. + + Both build as ESP-IDF components, so on ESP32 they skip the PlatformIO library + converter unless arduino-esp32 bundles its own libsodium. Not toolchain + dependent: the PlatformIO toolchain reads the project manifest too, and every + consumer of libsodium must make the same choice or a second copy appears. + """ + if not CORE.is_esp32: + return False + + from esphome.components.esp32 import arduino_bundles_libsodium + + return not arduino_bundles_libsodium() + + def enable_spare_ephemeral() -> None: """Compile the spare ephemeral key slot; the component that refills it calls this.""" cg.add_define("USE_NOISE_SPARE_EPHEMERAL") @@ -99,12 +120,16 @@ def enable_spare_ephemeral() -> None: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.30") - # noise-c depends on libsodium, but declaring it here too lets the - # library manager see the full set up front instead of discovering - # libsodium only after noise-c has downloaded, so the two can download - # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.11") + # libsodium is declared next to noise-c so the library manager sees both up front + # and nothing else pulls a second copy; the version must match noise-c's own pin + if _use_managed_components(): + from esphome.components.esp32 import add_idf_component + + add_idf_component(name="esphome/noise-c", ref=NOISE_C_VERSION) + add_idf_component(name="esphome/libsodium", ref=LIBSODIUM_VERSION) + else: + cg.add_library("esphome/noise-c", NOISE_C_VERSION) + cg.add_library("esphome/libsodium", LIBSODIUM_VERSION) # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/esphome/espidf/clang_tidy.py b/esphome/espidf/clang_tidy.py index 3e1a510dee..623b352e97 100644 --- a/esphome/espidf/clang_tidy.py +++ b/esphome/espidf/clang_tidy.py @@ -238,6 +238,17 @@ def _parse_lib_deps(platformio_ini: Path, framework: str): return libs +def _esphome_manifest_deps() -> set[str]: + """Names of the managed components declared in ``esphome/idf_component.yml``.""" + import yaml + + esphome_dir = Path(__file__).resolve().parent.parent + manifest = yaml.safe_load( + (esphome_dir / "idf_component.yml").read_text(encoding="utf-8") + ) + return set(manifest.get("dependencies") or {}) + + def _convert_pio_libs( platformio_ini: Path, framework: str ) -> dict[str, dict[str, str]]: @@ -250,12 +261,19 @@ def _convert_pio_libs( The whole library set is resolved as a single batch so a shared transitive dependency (e.g. esphome/libsodium pulled by both noise-c and esp_wireguard) is deduplicated to one component instead of clashing override_path entries. + + Libraries the manifest already provides as managed components are skipped, as + in the real esp32 build; converting them too would give IDF the same requirement + twice. Arduino below IDF 6.0 has them rule-gated off, so there they convert. """ + from esphome.components.esp32 import arduino_bundles_libsodium from esphome.espidf.component import generate_idf_components libraries = _parse_lib_deps(platformio_ini, framework) + # Same rule as noise._use_managed_components and the manifest + managed = set() if arduino_bundles_libsodium() else _esphome_manifest_deps() deps: dict[str, dict[str, str]] = {} - for component in generate_idf_components(libraries): + for component in generate_idf_components(libraries, managed=managed): deps[component.get_sanitized_name()] = {"override_path": str(component.path)} return deps @@ -273,24 +291,18 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: ethernet) are NOT stubbed -- those are real deps we need, and arduino-esp32 resolves to the same component rather than conflicting. """ - import yaml - from esphome.components.esp32 import ( - ARDUINO_EXCLUDED_IDF_COMPONENTS, _idf_component_dep_name, _idf_component_stub_name, + arduino_excluded_idf_components, ) - esphome_dir = Path(__file__).resolve().parent.parent - base_manifest = yaml.safe_load( - (esphome_dir / "idf_component.yml").read_text(encoding="utf-8") - ) - esphome_deps = set(base_manifest.get("dependencies") or {}) + esphome_deps = _esphome_manifest_deps() stubs_dir = work_dir / "component_stubs" stubs_dir.mkdir(parents=True, exist_ok=True) deps: dict[str, dict] = {} - for component in sorted(ARDUINO_EXCLUDED_IDF_COMPONENTS): + for component in sorted(arduino_excluded_idf_components()): if _idf_component_dep_name(component) in esphome_deps: continue # ESPHome needs this one for real (don't stub it away) stub_path = stubs_dir / _idf_component_stub_name(component) diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index bed999656b..c53dbe17db 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -287,12 +287,19 @@ def _emit_idf_component(component: IDFComponent) -> None: ) -def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: - """Resolve and convert a batch of PlatformIO libraries to IDF components.""" +def generate_idf_components( + libraries: list[Library], managed: set[str] | None = None +) -> list[IDFComponent]: + """Resolve and convert a batch of PlatformIO libraries to IDF components. + + ``managed`` names registry components already in the project manifest; they + are not converted, since IDF rejects the same requirement from two components. + Converted components reach them through ``${ESPHOME_PROJECT_MANAGED_COMPONENTS}``. + """ backend = LibraryBackend( platform=ESP32_PLATFORM, framework=_idf_framework(), emit=_emit_idf_component, cache_key="idf", ) - return convert_libraries(libraries, backend) + return convert_libraries(libraries, backend, provided=managed) diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 1a972896c3..fc8090a322 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -106,3 +106,14 @@ dependencies: version: d44c800a9e876a8394caefc2ce4915dd96dac77b rules: - if: "$ESPHOME_ARDUINO_COMPONENT == 1" + # noise (api, ota). Arduino below IDF 6.0 bundles espressif/libsodium, which + # clashes with esphome/libsodium, so those builds use the PlatformIO libraries. + esphome/noise-c: + version: 0.1.30 + rules: + - if: "$ESPHOME_ARDUINO_COMPONENT == 0 || idf_version >= 6.0.0" + # Declared alongside noise-c so the library converter skips esp_wireguard's copy + esphome/libsodium: + version: 1.10021.11 + rules: + - if: "$ESPHOME_ARDUINO_COMPONENT == 0 || idf_version >= 6.0.0" diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index e5e4aa7245..12aaae315e 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -1120,7 +1120,9 @@ def _prefetch_wave( def convert_libraries( - libraries: list[Library], backend: LibraryBackend + libraries: list[Library], + backend: LibraryBackend, + provided: set[str] | None = None, ) -> list[ConvertedLibrary]: """Resolve and convert a batch of PlatformIO libraries for ``backend``. @@ -1141,10 +1143,18 @@ def convert_libraries( ``lib_ignore`` from ``esphome->platformio_options`` excludes libraries by short name (part after the ``/``), matched against both the top-level libraries and every dependency discovered during the graph walk. + + ``provided`` names libraries the toolchain supplies by other means (for ESP-IDF, + managed components from ``add_idf_component``); unlike ``backend.provides`` they + carry an owner. They are excluded like ``lib_ignore`` so nothing is both + converted and managed, which ESP-IDF refuses to build. """ nodes: dict[str, _LibNode] = {} - lib_ignore = lib_ignore_set() + # Folded into one set so every is_lib_ignored() call site honors both. + lib_ignore = lib_ignore_set() | { + name.split("/")[-1].lower() for name in provided or () + } # The generated build files inside the shared cache bake in the dependency # wiring, which lib_ignore changes; salt the cache path so configs with diff --git a/platformio.ini b/platformio.ini index dd2a572493..00fad1ff66 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,6 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.30 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / improv_ble kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -77,6 +76,10 @@ lib_compat_mode = strict extends = common lib_deps = ${common.lib_deps} + ; noise (api, ota) -- on the ESP-IDF framework these come from the + ; component registry instead (see esphome/idf_component.yml), so they + ; are not in [common]. + esphome/noise-c@0.1.30 ; noise (api, ota) SPI ; spi (Arduino built-in) Wire ; i2c (Arduino built-int) heman/AsyncMqttClient-esphome@1.0.0 ; mqtt diff --git a/tests/unit_tests/components/noise/test_noise_to_code.py b/tests/unit_tests/components/noise/test_noise_to_code.py new file mode 100644 index 0000000000..a7e0a45e5c --- /dev/null +++ b/tests/unit_tests/components/noise/test_noise_to_code.py @@ -0,0 +1,163 @@ +"""Tests for the noise-c/libsodium library wiring in the noise component. + +Drives the real to_code() so every branch of the managed-versus-converted +decision runs end to end. +""" + +from __future__ import annotations + +import asyncio +import re + +import pytest + +import esphome.codegen as cg +from esphome.components import esp32, noise +import esphome.config_validation as cv +from esphome.const import ( + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Framework, + Platform, + Toolchain, +) +from esphome.core import CORE + +DEFAULT_IDF_VERSION = cv.Version(5, 5, 4) + + +def _setup_core( + platform: Platform, + framework: Framework, + toolchain: Toolchain, + idf_version: cv.Version = DEFAULT_IDF_VERSION, +) -> None: + CORE.reset() + CORE.toolchain = toolchain + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: str(platform), + KEY_TARGET_FRAMEWORK: str(framework), + } + if platform == Platform.ESP32: + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_VARIANT: "ESP32", + esp32.KEY_IDF_VERSION: idf_version, + } + + +def _record_calls( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[list[dict], list[tuple]]: + """Capture both wiring paths so each test can assert one ran and one did not.""" + idf_calls: list[dict] = [] + lib_calls: list[tuple] = [] + monkeypatch.setattr( + esp32, "add_idf_component", lambda **kwargs: idf_calls.append(kwargs) + ) + monkeypatch.setattr( + cg, + "add_library", + lambda name, version, repository=None: lib_calls.append((name, version)), + ) + return idf_calls, lib_calls + + +@pytest.mark.parametrize("toolchain", [Toolchain.ESP_IDF, Toolchain.PLATFORMIO]) +def test_to_code_esp32_idf_uses_managed_idf_components( + toolchain: Toolchain, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ESP32 + ESP-IDF declares both as managed IDF components on either toolchain.""" + _setup_core(Platform.ESP32, Framework.ESP_IDF, toolchain) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert idf_calls == [ + {"name": "esphome/noise-c", "ref": noise.NOISE_C_VERSION}, + {"name": "esphome/libsodium", "ref": noise.LIBSODIUM_VERSION}, + ] + assert lib_calls == [] + + +def test_to_code_esp32_arduino_below_idf6_uses_add_library( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Arduino below IDF 6.0 bundles espressif/libsodium, so both stay PlatformIO + libraries.""" + _setup_core( + Platform.ESP32, Framework.ARDUINO, Toolchain.ESP_IDF, cv.Version(5, 5, 4) + ) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert lib_calls == [ + ("esphome/noise-c", noise.NOISE_C_VERSION), + ("esphome/libsodium", noise.LIBSODIUM_VERSION), + ] + assert idf_calls == [] + + +@pytest.mark.parametrize("idf_version", [cv.Version(6, 0, 0), cv.Version(6, 1, 0)]) +def test_to_code_esp32_arduino_idf6_uses_managed_idf_components( + idf_version: cv.Version, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """From IDF 6.0 there is no clash, so Arduino uses managed components too.""" + _setup_core(Platform.ESP32, Framework.ARDUINO, Toolchain.ESP_IDF, idf_version) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert idf_calls == [ + {"name": "esphome/noise-c", "ref": noise.NOISE_C_VERSION}, + {"name": "esphome/libsodium", "ref": noise.LIBSODIUM_VERSION}, + ] + assert lib_calls == [] + + +def test_to_code_non_esp32_uses_add_library( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Off ESP32 entirely (e.g. host) there are no IDF components at all.""" + _setup_core(Platform.HOST, Framework.NATIVE, Toolchain.PLATFORMIO) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert lib_calls == [ + ("esphome/noise-c", noise.NOISE_C_VERSION), + ("esphome/libsodium", noise.LIBSODIUM_VERSION), + ] + assert idf_calls == [] + + +def test_versions_match_the_repo_manifests() -> None: + """A bump that misses one of the duplicated pins would ship two libsodium versions.""" + from pathlib import Path + + import yaml + + repo_root = Path(__file__).resolve().parents[4] + manifest = yaml.safe_load( + (repo_root / "esphome" / "idf_component.yml").read_text(encoding="utf-8") + ) + deps = manifest["dependencies"] + + assert deps["esphome/noise-c"]["version"] == noise.NOISE_C_VERSION + assert deps["esphome/libsodium"]["version"] == noise.LIBSODIUM_VERSION + # Both are skipped on Arduino below IDF 6.0, where the PlatformIO library + # path is used instead (see noise._use_managed_components). + for name in ("esphome/noise-c", "esphome/libsodium"): + assert deps[name]["rules"] == [ + {"if": "$ESPHOME_ARDUINO_COMPONENT == 0 || idf_version >= 6.0.0"} + ] + # Every noise-c pin in platformio.ini, not just one of them + pins = re.findall( + r"esphome/noise-c@(\S+)", + (repo_root / "platformio.ini").read_text(encoding="utf-8"), + ) + assert set(pins) == {noise.NOISE_C_VERSION} + assert len(pins) >= 1 diff --git a/tests/unit_tests/components/test_esp32_write_idf_component_yml.py b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py new file mode 100644 index 0000000000..931de627a1 --- /dev/null +++ b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py @@ -0,0 +1,175 @@ +"""Tests for esp32's _write_idf_component_yml() managed-component wiring. + +A library already declared as a managed IDF component must not also be converted, +or ESP-IDF sees the same requirement twice and refuses to build. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from esphome.components import esp32 +import esphome.config_validation as cv +from esphome.const import ( + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Framework, + Platform, + Toolchain, +) +from esphome.core import CORE + + +def _setup_core(tmp_path: Path) -> None: + CORE.reset() + CORE.name = "testdevice" + CORE.build_path = tmp_path + CORE.toolchain = Toolchain.ESP_IDF + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: str(Platform.ESP32), + KEY_TARGET_FRAMEWORK: str(Framework.ESP_IDF), + } + + +def test_write_idf_component_yml_passes_managed_components( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Names registered via add_idf_component are passed as ``managed`` so the + converter skips them.""" + _setup_core(tmp_path) + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_COMPONENTS: { + "esphome/noise-c": { + esp32.KEY_REPO: None, + esp32.KEY_REF: "0.1.15", + esp32.KEY_PATH: None, + }, + }, + } + + captured: dict[str, set[str] | None] = {} + + # A converted (non-managed) library the batch still resolves, so the loop + # wiring its override_path into the manifest is exercised for real too. + converted = MagicMock() + converted.get_sanitized_name.return_value = "esphome/other-lib" + converted.path = tmp_path / "pio_components" / "other-lib" + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [converted] + + monkeypatch.setattr(esp32, "generate_idf_components", fake_generate_idf_components) + + esp32._write_idf_component_yml() + + assert captured["managed"] == {"esphome/noise-c"} + # The managed component itself is still written into the manifest deps + # directly (from KEY_COMPONENTS), just not converted a second time. + yml_path = tmp_path / "src" / "idf_component.yml" + assert yml_path.is_file() + contents = yml_path.read_text(encoding="utf-8") + assert "esphome/noise-c" in contents + assert "0.1.15" in contents + # The converted library the batch DID return is still wired in. + assert "esphome/other-lib" in contents + assert str(converted.path) in contents + + +def test_write_idf_component_yml_empty_managed_when_no_components( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No add_idf_component calls means an empty managed set, the old behavior.""" + _setup_core(tmp_path) + CORE.data[esp32.KEY_ESP32] = {esp32.KEY_COMPONENTS: {}} + + captured: dict[str, set[str] | None] = {} + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [] + + monkeypatch.setattr(esp32, "generate_idf_components", fake_generate_idf_components) + + esp32._write_idf_component_yml() + + assert captured["managed"] == set() + + +@pytest.mark.parametrize( + ("version", "libsodium_stubbed"), + [ + (cv.Version(5, 5, 4), True), + (cv.Version(5, 99, 99), True), + (cv.Version(6, 0, 0), False), + (cv.Version(6, 1, 0), False), + ], +) +def test_arduino_excluded_idf_components_depends_on_idf_version( + version: cv.Version, libsodium_stubbed: bool +) -> None: + """espressif/libsodium is stubbed below IDF 6.0 only; unmapped entries always.""" + CORE.reset() + CORE.data[esp32.KEY_ESP32] = {esp32.KEY_IDF_VERSION: version} + + excluded = esp32.arduino_excluded_idf_components() + + assert ("espressif__libsodium" in excluded) is libsodium_stubbed + assert "espressif__cbor" in excluded + + +@pytest.mark.parametrize( + ("framework", "version", "bundled"), + [ + (Framework.ARDUINO, cv.Version(5, 5, 5), True), + (Framework.ARDUINO, cv.Version(6, 0, 0), False), + (Framework.ESP_IDF, cv.Version(5, 5, 5), False), + ], +) +def test_arduino_bundles_libsodium( + framework: Framework, version: cv.Version, bundled: bool, tmp_path: Path +) -> None: + """Only Arduino below IDF 6.0 brings its own libsodium.""" + _setup_core(tmp_path) + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = str(framework) + CORE.data[esp32.KEY_ESP32] = {esp32.KEY_IDF_VERSION: version} + + assert esp32.arduino_bundles_libsodium() is bundled + + +@pytest.mark.parametrize( + ("version", "libsodium_stubbed"), + [(cv.Version(5, 5, 4), True), (cv.Version(6, 0, 0), False)], +) +def test_write_idf_component_yml_arduino_stubs_follow_idf_version( + version: cv.Version, + libsodium_stubbed: bool, + tmp_path: Path, +) -> None: + """The manifest stubs follow the IDF version: espressif/libsodium below 6.0 only.""" + _setup_core(tmp_path) + CORE.toolchain = Toolchain.PLATFORMIO + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = str(Framework.ARDUINO) + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_COMPONENTS: {}, + esp32.KEY_IDF_VERSION: version, + esp32.KEY_ARDUINO_LIBRARIES: set(), + } + + esp32._write_idf_component_yml() + + contents = (tmp_path / "src" / "idf_component.yml").read_text(encoding="utf-8") + assert ("espressif/libsodium" in contents) is libsodium_stubbed + assert "espressif/cbor" in contents + stub_dir = ( + tmp_path + / "component_stubs" + / esp32._idf_component_stub_name("espressif__libsodium") + ) + assert stub_dir.is_dir() is libsodium_stubbed diff --git a/tests/unit_tests/test_espidf_clang_tidy.py b/tests/unit_tests/test_espidf_clang_tidy.py index e0cfec88c8..b0d016b1b4 100644 --- a/tests/unit_tests/test_espidf_clang_tidy.py +++ b/tests/unit_tests/test_espidf_clang_tidy.py @@ -5,12 +5,26 @@ import json import os from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch import pytest +import yaml +from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION +import esphome.config_validation as cv +from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM +from esphome.core import CORE from esphome.espidf import clang_tidy -from esphome.espidf.clang_tidy import _Settings, _setup_core, _write_tidy_project +from esphome.espidf.clang_tidy import ( + _arduino_excluded_stubs, + _convert_pio_libs, + _esphome_manifest_deps, + _Settings, + _setup_core, + _write_tidy_project, +) +import esphome.espidf.component as espidf_component REPO_ROOT = Path(__file__).resolve().parents[2] @@ -137,3 +151,146 @@ def test_generate_compile_commands_configures_twice( ) assert result == tmp_path / "build" / "compile_commands.json" assert mock_write.call_args_list[1].args[1] == ["esp_timer", "lwip"] + + +def test_esphome_manifest_deps_reads_repo_manifest() -> None: + """Top-level dependency names, independent of the per-dependency rules.""" + manifest = yaml.safe_load( + (REPO_ROOT / "esphome" / "idf_component.yml").read_text(encoding="utf-8") + ) + + deps = _esphome_manifest_deps() + + assert isinstance(deps, set) + assert "esphome/noise-c" in deps + assert "esphome/libsodium" in deps + # Cross-check against a fresh parse instead of hardcoding the manifest's + # whole key list, so this doesn't need updating whenever a dependency is + # added or removed. + assert deps == set(manifest["dependencies"]) + + +def test_convert_pio_libs_arduino_framework_passes_empty_managed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Below IDF 6.0 the manifest gates noise-c/libsodium off on Arduino, so they + still go through the converter.""" + _set_idf_version(cv.Version(5, 5, 5)) + monkeypatch.setattr(clang_tidy, "_parse_lib_deps", lambda ini, framework: []) + + captured: dict[str, set[str] | None] = {} + + # A converted library the batch resolves, so the loop wiring its + # override_path into the returned deps mapping is exercised for real too. + converted = SimpleNamespace( + get_sanitized_name=lambda: "esphome/other-lib", + path=tmp_path / "other-lib", + ) + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [converted] + + monkeypatch.setattr( + espidf_component, "generate_idf_components", fake_generate_idf_components + ) + + result = _convert_pio_libs(tmp_path / "platformio.ini", "arduino") + + assert captured["managed"] == set() + assert result == { + "esphome/other-lib": {"override_path": str(tmp_path / "other-lib")} + } + + +def test_convert_pio_libs_arduino_idf_6_passes_manifest_deps( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """From IDF 6.0 the manifest enables them on Arduino too, so the converter + must skip them.""" + _set_idf_version(cv.Version(6, 0, 0)) + monkeypatch.setattr(clang_tidy, "_parse_lib_deps", lambda ini, framework: []) + + captured: dict[str, set[str] | None] = {} + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [] + + monkeypatch.setattr( + espidf_component, "generate_idf_components", fake_generate_idf_components + ) + + assert _convert_pio_libs(tmp_path / "platformio.ini", "arduino") == {} + assert captured["managed"] == _esphome_manifest_deps() + + +def test_convert_pio_libs_espidf_framework_passes_manifest_deps( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Manifest-provided components are passed as ``managed`` so the converter + skips them.""" + _set_idf_version(cv.Version(5, 5, 5), "espidf") + monkeypatch.setattr(clang_tidy, "_parse_lib_deps", lambda ini, framework: []) + + captured: dict[str, set[str] | None] = {} + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [] + + monkeypatch.setattr( + espidf_component, "generate_idf_components", fake_generate_idf_components + ) + + result = _convert_pio_libs(tmp_path / "platformio.ini", "espidf") + + assert captured["managed"] == _esphome_manifest_deps() + assert "esphome/noise-c" in captured["managed"] + assert result == {} + + +def _set_idf_version(version: cv.Version, framework: str = "arduino") -> None: + CORE.reset() + CORE.data[KEY_ESP32] = {KEY_IDF_VERSION: version} + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp32", + KEY_TARGET_FRAMEWORK: framework, + } + + +def test_arduino_excluded_stubs_skips_components_esphome_manifest_provides( + tmp_path: Path, +) -> None: + """A component ESPHome's own manifest declares (espressif/lan867x for ethernet) + is not stubbed; one only arduino-esp32 bundles still is.""" + _set_idf_version(cv.Version(5, 5, 4)) + + deps = _arduino_excluded_stubs(tmp_path) + + # lan867x is a real ESPHome dependency (esphome/idf_component.yml), so it + # must be excluded from the stub set. + assert "espressif/lan867x" not in deps + # espressif/libsodium (arduino-esp32's bundled copy) is a different + # package from ESPHome's own esphome/libsodium, so below IDF 6.0, where + # arduino-esp32 still declares it, it is stubbed. + assert "espressif/libsodium" in deps + stub_info = deps["espressif/libsodium"] + assert stub_info["version"] == "*" + stub_path = Path(stub_info["override_path"]) + assert (stub_path / "CMakeLists.txt").is_file() + + +def test_arduino_excluded_stubs_skips_libsodium_from_idf_6(tmp_path: Path) -> None: + """From IDF 6.0 arduino-esp32 drops espressif/libsodium; a stub would clash + with esphome/libsodium.""" + _set_idf_version(cv.Version(6, 0, 0)) + + deps = _arduino_excluded_stubs(tmp_path) + + assert "espressif/libsodium" not in deps + # Other arduino-bundled components are still stubbed. + assert "espressif/cbor" in deps diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 3789eefc64..67da0f4f6d 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -803,6 +803,112 @@ def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies( assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]] +def test_generate_idf_components_managed_filters_top_level_and_dependencies( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # managed (e.g. noise-c/libsodium already declared via add_idf_component) + # must drop B at the top level and C when discovered as a dependency of A, + # exactly like lib_ignore -- neither may be resolved, downloaded, or wired + # into a manifest. + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [ + {"owner": "esphome", "name": "C", "version": "==1.10021.0"} + ], + }, + "esphome/B": {"name": "B"}, + } + + download_salts: list[str] = [] + + def fake_download(self, force=False, salt="", namespace=""): + download_salts.append(salt) + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + + resolve_calls: list[str] = [] + + def fake_resolve(owner, pkgname, requirements): + resolve_calls.append(pkgname) + return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None + + monkeypatch.setattr( + esphome.platformio.library, "_resolve_registry_version", fake_resolve + ) + + top = generate_idf_components( + [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)], + managed={"esphome/B", "esphome/C"}, + ) + + assert [c.name for c in top] == ["esphome/A"] + # Managed libraries were never resolved (and therefore never downloaded). + assert resolve_calls == ["A"] + # The managed dependency is not wired into A's manifest. + assert top[0].dependencies == [] + # managed changes the generated wiring just like lib_ignore, so the cache + # path is salted the same way. + assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]] + + +def test_generate_idf_components_lib_ignore_and_managed_combine_into_salt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # lib_ignore and managed both contribute to the same exclusion set, so a + # config using both gets a salt reflecting the union of the two sources + # rather than either alone. + manifests = { + "esphome/A": {"name": "A"}, + "esphome/D": {"name": "D"}, + "esphome/E": {"name": "E"}, + } + + download_salts: list[str] = [] + + def fake_download(self, force=False, salt="", namespace=""): + download_salts.append(salt) + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + + resolve_calls: list[str] = [] + + def fake_resolve(owner, pkgname, requirements): + resolve_calls.append(pkgname) + return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None + + monkeypatch.setattr( + esphome.platformio.library, "_resolve_registry_version", fake_resolve + ) + monkeypatch.setattr(CORE, "platformio_options", {"lib_ignore": ["D"]}) + + top = generate_idf_components( + [ + Library("esphome/A", "1.0.0", None), + Library("esphome/D", "1.0.0", None), + Library("esphome/E", "1.0.0", None), + ], + managed={"esphome/E"}, + ) + + assert [c.name for c in top] == ["esphome/A"] + assert resolve_calls == ["A"] + # The salt reflects BOTH lib_ignore's "D" and managed's "E" together. + assert download_salts == [hashlib.sha256(b"d,e").hexdigest()[:8]] + + def test_generate_idf_components_handles_dependency_cycle( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From afba43160095d97b812770818138e65487029fef Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Thu, 1 Oct 2026 20:15:13 +0200 Subject: [PATCH 022/221] [resampler] Add resampler microphone platform (#19953) --- esphome/components/resampler/__init__.py | 15 ++ .../resampler/microphone/__init__.py | 77 ++++++++ .../microphone/resampler_microphone.cpp | 174 ++++++++++++++++++ .../microphone/resampler_microphone.h | 62 +++++++ .../components/resampler/speaker/__init__.py | 16 +- .../resampler/common-microphone.yaml | 29 +++ .../resampler/test-microphone.esp32-idf.yaml | 6 + .../test-microphone.esp32-s3-idf.yaml | 6 + 8 files changed, 372 insertions(+), 13 deletions(-) create mode 100644 esphome/components/resampler/microphone/__init__.py create mode 100644 esphome/components/resampler/microphone/resampler_microphone.cpp create mode 100644 esphome/components/resampler/microphone/resampler_microphone.h create mode 100644 tests/components/resampler/common-microphone.yaml create mode 100644 tests/components/resampler/test-microphone.esp32-idf.yaml create mode 100644 tests/components/resampler/test-microphone.esp32-s3-idf.yaml diff --git a/esphome/components/resampler/__init__.py b/esphome/components/resampler/__init__.py index e69de29bb2..b9b070e015 100644 --- a/esphome/components/resampler/__init__.py +++ b/esphome/components/resampler/__init__.py @@ -0,0 +1,15 @@ +from typing import Any + +import esphome.codegen as cg +import esphome.config_validation as cv + +resampler_ns = cg.esphome_ns.namespace("resampler") + +CONF_TAPS = "taps" + + +def validate_taps(taps: Any) -> int: + value = cv.int_range(min=16, max=128)(taps) + if value % 4 != 0: + raise cv.Invalid("Number of taps must be divisible by 4") + return value diff --git a/esphome/components/resampler/microphone/__init__.py b/esphome/components/resampler/microphone/__init__.py new file mode 100644 index 0000000000..0a8d0c1ca3 --- /dev/null +++ b/esphome/components/resampler/microphone/__init__.py @@ -0,0 +1,77 @@ +import esphome.codegen as cg +from esphome.components import audio, microphone +import esphome.config_validation as cv +from esphome.const import ( + CONF_BITS_PER_SAMPLE, + CONF_CHANNELS, + CONF_FILTERS, + CONF_ID, + CONF_MICROPHONE, + CONF_SAMPLE_RATE, + PLATFORM_ESP32, +) +from esphome.types import ConfigType + +from .. import CONF_TAPS, resampler_ns, validate_taps + +AUTO_LOAD = ["audio"] +DEPENDENCIES = ["microphone"] + +ResamplerMicrophone = resampler_ns.class_( + "ResamplerMicrophone", cg.Component, microphone.Microphone +) + + +def _set_stream_limits(config: ConfigType) -> ConfigType: + # Only the sample rate changes; the bits and channels are those selected from the source microphone + source = config[CONF_MICROPHONE] + audio.set_stream_limits( + min_bits_per_sample=source[CONF_BITS_PER_SAMPLE], + max_bits_per_sample=source[CONF_BITS_PER_SAMPLE], + min_channels=len(source[CONF_CHANNELS]), + max_channels=len(source[CONF_CHANNELS]), + min_sample_rate=config[CONF_SAMPLE_RATE], + max_sample_rate=config[CONF_SAMPLE_RATE], + )(config) + return config + + +CONFIG_SCHEMA = cv.All( + microphone.MICROPHONE_SCHEMA.extend( + { + cv.GenerateID(): cv.declare_id(ResamplerMicrophone), + cv.Required(CONF_MICROPHONE): microphone.microphone_source_schema( + min_bits_per_sample=16, + max_bits_per_sample=32, + min_channels=1, + max_channels=2, + ), + cv.Optional(CONF_SAMPLE_RATE, default=16000): cv.int_range(8000, 48000), + cv.Optional(CONF_FILTERS, default=16): cv.int_range(min=2, max=1024), + cv.Optional(CONF_TAPS, default=16): validate_taps, + } + ).extend(cv.COMPONENT_SCHEMA), + cv.only_on([PLATFORM_ESP32]), + _set_stream_limits, +) + + +FINAL_VALIDATE_SCHEMA = cv.Schema( + { + cv.Required( + CONF_MICROPHONE + ): microphone.final_validate_microphone_source_schema("resampler"), + }, + extra=cv.ALLOW_EXTRA, +) + + +async def to_code(config: ConfigType) -> None: + mic_source = await microphone.microphone_source_to_code(config[CONF_MICROPHONE]) + var = cg.new_Pvariable(config[CONF_ID], mic_source) + await cg.register_component(var, config) + await microphone.register_microphone(var, config) + + cg.add(var.set_target_sample_rate(config[CONF_SAMPLE_RATE])) + cg.add(var.set_filters(config[CONF_FILTERS])) + cg.add(var.set_taps(config[CONF_TAPS])) diff --git a/esphome/components/resampler/microphone/resampler_microphone.cpp b/esphome/components/resampler/microphone/resampler_microphone.cpp new file mode 100644 index 0000000000..0dcd523380 --- /dev/null +++ b/esphome/components/resampler/microphone/resampler_microphone.cpp @@ -0,0 +1,174 @@ +#include "resampler_microphone.h" + +#ifdef USE_ESP32 + +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +#include + +namespace esphome::resampler { + +static const char *const TAG = "resampler.microphone"; + +// Duration of audio the resampler converts per step; longer source chunks are processed in several steps +static constexpr uint32_t BUFFER_DURATION_MS = 16; + +void ResamplerMicrophone::setup() { + const audio::AudioStreamInfo input_stream_info = this->source_->get_audio_stream_info(); + this->audio_stream_info_ = audio::AudioStreamInfo(input_stream_info.get_bits_per_sample(), + input_stream_info.get_channels(), this->target_sample_rate_); + + // Allocate now for the expected source format; process_audio_ only sets up again if that format changes + if (!this->init_resampler_(input_stream_info)) { + this->mark_failed(); + return; + } + + this->source_->add_data_callback([this](const std::vector &data) { this->process_audio_(data); }); + + this->disable_loop(); +} + +void ResamplerMicrophone::dump_config() { + ESP_LOGCONFIG(TAG, + "Resampler Microphone:\n" + " Target Sample Rate: %" PRIu32 " Hz\n" + " Taps: %u\n" + " Filters: %u", + this->target_sample_rate_, this->taps_, this->filters_); +} + +void ResamplerMicrophone::start() { + if (this->is_failed() || this->active_listeners_ == UINT8_MAX) + return; + ++this->active_listeners_; + this->enable_loop(); +} + +void ResamplerMicrophone::stop() { + if (this->active_listeners_ == 0) + return; + --this->active_listeners_; + this->enable_loop(); +} + +void ResamplerMicrophone::loop() { + if (this->active_listeners_ == 0) { + if (this->state_ != microphone::STATE_STOPPED) { + this->source_->stop(); + this->state_ = microphone::STATE_STOPPED; + } + this->disable_loop(); + return; + } + + switch (this->state_) { + case microphone::STATE_STOPPED: + this->source_->start(); + this->state_ = microphone::STATE_STARTING; + break; + case microphone::STATE_STARTING: + if (this->source_->is_running()) { + this->state_ = microphone::STATE_RUNNING; + } + break; + case microphone::STATE_RUNNING: + // Follow the source if it restarts, e.g. after a driver error + if (!this->source_->is_running()) { + this->state_ = microphone::STATE_STARTING; + } + break; + case microphone::STATE_STOPPING: + break; + } +} + +bool ResamplerMicrophone::init_resampler_(const audio::AudioStreamInfo &input_stream_info) { + this->resampler_.reset(); + this->resampler_ready_ = false; + + if (input_stream_info.get_sample_rate() == this->target_sample_rate_) { + // The source already delivers the target sample rate, so its audio is passed through unchanged + this->input_stream_info_ = input_stream_info; + this->resampler_ready_ = true; + return true; + } + + const audio::AudioStreamInfo output_stream_info(input_stream_info.get_bits_per_sample(), + input_stream_info.get_channels(), this->target_sample_rate_); + + auto resampler = make_unique( + input_stream_info.ms_to_samples(BUFFER_DURATION_MS), output_stream_info.ms_to_samples(BUFFER_DURATION_MS)); + + esp_audio_libs::resampler::ResamplerConfiguration resample_config = { + .source_sample_rate = static_cast(input_stream_info.get_sample_rate()), + .target_sample_rate = static_cast(this->target_sample_rate_), + .source_bits_per_sample = input_stream_info.get_bits_per_sample(), + .target_bits_per_sample = input_stream_info.get_bits_per_sample(), + .channels = input_stream_info.get_channels(), + // Filters out frequencies above the new Nyquist limit when downsampling, to avoid aliasing + .use_pre_or_post_filter = this->target_sample_rate_ < input_stream_info.get_sample_rate(), + .subsample_interpolate = false, // Doubles the CPU load; more filters is a better alternative + .number_of_taps = this->taps_, + .number_of_filters = this->filters_, + }; + + if (!resampler->initialize(resample_config)) { + ESP_LOGE(TAG, "Not enough memory to resample"); + return false; + } + + this->output_buffer_.reserve(output_stream_info.ms_to_bytes(BUFFER_DURATION_MS)); + this->resampler_ = std::move(resampler); + // Only set on success, so a failed set up is retried with the next chunk + this->input_stream_info_ = input_stream_info; + this->resampler_ready_ = true; + return true; +} + +void ResamplerMicrophone::process_audio_(const std::vector &data) { + const audio::AudioStreamInfo input_stream_info = this->source_->get_audio_stream_info(); + if (input_stream_info != this->input_stream_info_) { + this->init_resampler_(input_stream_info); + } + if (!this->resampler_ready_) { + return; + } + + if (this->resampler_ == nullptr) { + this->data_callbacks_.call(data); + return; + } + + const size_t input_bytes_per_frame = input_stream_info.frames_to_bytes(1); + const uint32_t max_input_frames = input_stream_info.ms_to_frames(BUFFER_DURATION_MS); + // Both limits match the sizes the resampler's internal buffers were allocated with in init_resampler_ + const uint32_t max_output_frames = this->audio_stream_info_.ms_to_frames(BUFFER_DURATION_MS); + + const uint8_t *input = data.data(); + uint32_t input_frames = input_stream_info.bytes_to_frames(data.size()); + while (input_frames > 0) { + // Stays within the reserved capacity, so this never reallocates + this->output_buffer_.resize(this->audio_stream_info_.frames_to_bytes(max_output_frames)); + + // The resampler's internal buffers hold at most BUFFER_DURATION_MS of audio, so feed it in steps of that size. + // 0 dB keeps the microphone level that downstream detectors are tuned for; overshoot saturates instead of wrapping. + esp_audio_libs::resampler::ResamplerResults results = this->resampler_->resample( + input, this->output_buffer_.data(), std::min(input_frames, max_input_frames), max_output_frames, 0.0f); + + input += results.frames_used * input_bytes_per_frame; + input_frames -= results.frames_used; + + if (results.frames_generated > 0) { + this->output_buffer_.resize(this->audio_stream_info_.frames_to_bytes(results.frames_generated)); + this->data_callbacks_.call(this->output_buffer_); + } else if (results.frames_used == 0) { + break; // No progress; drop the rest of the chunk instead of spinning + } + } +} + +} // namespace esphome::resampler + +#endif // USE_ESP32 diff --git a/esphome/components/resampler/microphone/resampler_microphone.h b/esphome/components/resampler/microphone/resampler_microphone.h new file mode 100644 index 0000000000..780f22e429 --- /dev/null +++ b/esphome/components/resampler/microphone/resampler_microphone.h @@ -0,0 +1,62 @@ +#pragma once + +#ifdef USE_ESP32 + +#include "esphome/components/audio/audio.h" +#include "esphome/components/microphone/microphone.h" +#include "esphome/components/microphone/microphone_source.h" + +#include "esphome/core/component.h" + +#include // esp-audio-libs + +#include +#include + +namespace esphome::resampler { + +/// @brief Microphone that converts the audio of a source microphone to a different sample rate. +/// The bits per sample and channels are selected by the source's ``MicrophoneSource``; only the sample rate changes. +/// Resampling runs in the source microphone's data callback, so it needs no task or ring buffer of its own. +class ResamplerMicrophone final : public Component, public microphone::Microphone { + public: + explicit ResamplerMicrophone(microphone::MicrophoneSource *source) : source_(source) {} + + void setup() override; + void loop() override; + void dump_config() override; + + void start() override; + void stop() override; + + void set_target_sample_rate(uint32_t target_sample_rate) { this->target_sample_rate_ = target_sample_rate; } + void set_filters(uint16_t filters) { this->filters_ = filters; } + void set_taps(uint16_t taps) { this->taps_ = taps; } + + protected: + /// @brief Sets up the resampler for the given input format. No resampler is needed if the sample rates match. + /// @return false if the resampler failed to allocate; the audio is then dropped + bool init_resampler_(const audio::AudioStreamInfo &input_stream_info); + + /// @brief Resamples a chunk of source audio and passes it to the data callbacks. Source microphone task only. + void process_audio_(const std::vector &data); + + microphone::MicrophoneSource *source_; + std::unique_ptr resampler_; + // Reused for every chunk so resampling does not allocate + std::vector output_buffer_; + + // Format the resampler is set up for + audio::AudioStreamInfo input_stream_info_; + + uint32_t target_sample_rate_; + uint16_t taps_; + uint16_t filters_; + + uint8_t active_listeners_{0}; + bool resampler_ready_{false}; +}; + +} // namespace esphome::resampler + +#endif // USE_ESP32 diff --git a/esphome/components/resampler/speaker/__init__.py b/esphome/components/resampler/speaker/__init__.py index 7de468cb50..8fa8aeb61c 100644 --- a/esphome/components/resampler/speaker/__init__.py +++ b/esphome/components/resampler/speaker/__init__.py @@ -1,5 +1,3 @@ -from typing import Any - import esphome.codegen as cg from esphome.components import audio, psram, speaker import esphome.config_validation as cv @@ -17,16 +15,15 @@ from esphome.const import ( from esphome.core.entity_helpers import inherit_property_from from esphome.types import ConfigType +from .. import CONF_TAPS, resampler_ns, validate_taps + AUTO_LOAD = ["audio"] CODEOWNERS = ["@kahrendt"] -resampler_ns = cg.esphome_ns.namespace("resampler") ResamplerSpeaker = resampler_ns.class_( "ResamplerSpeaker", cg.Component, speaker.Speaker ) -CONF_TAPS = "taps" - PASSTHROUGH = "passthrough" @@ -60,13 +57,6 @@ def _validate_audio_compatibility(config: ConfigType) -> None: )(config) -def _validate_taps(taps: Any) -> int: - value = cv.int_range(min=16, max=128)(taps) - if value % 4 != 0: - raise cv.Invalid("Number of taps must be divisible by 4") - return value - - CONFIG_SCHEMA = cv.All( speaker.SPEAKER_SCHEMA.extend( { @@ -80,7 +70,7 @@ CONFIG_SCHEMA = cv.All( ): cv.positive_time_period_milliseconds, cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_FILTERS, default=16): cv.int_range(min=2, max=1024), - cv.Optional(CONF_TAPS, default=16): _validate_taps, + cv.Optional(CONF_TAPS, default=16): validate_taps, } ).extend(cv.COMPONENT_SCHEMA), cv.only_on([PLATFORM_ESP32]), diff --git a/tests/components/resampler/common-microphone.yaml b/tests/components/resampler/common-microphone.yaml new file mode 100644 index 0000000000..123ba87edc --- /dev/null +++ b/tests/components/resampler/common-microphone.yaml @@ -0,0 +1,29 @@ +microphone: + - platform: i2s_audio + id: resampler_i2s_mic_id + i2s_audio_id: i2s_audio_bus + adc_type: external + i2s_din_pin: ${din_pin} + sample_rate: 48000 + bits_per_sample: 32bit + channel: stereo + - platform: resampler + id: resampler_mic_id + microphone: + microphone: resampler_i2s_mic_id + channels: 0 + bits_per_sample: 16 + sample_rate: 16000 + on_data: + - logger.log: + format: "Received %u bytes" + args: [x.size()] + - platform: resampler + id: resampler_mic_stereo_id + microphone: + microphone: resampler_i2s_mic_id + channels: [0, 1] + bits_per_sample: 32 + sample_rate: 44100 + filters: 32 + taps: 32 diff --git a/tests/components/resampler/test-microphone.esp32-idf.yaml b/tests/components/resampler/test-microphone.esp32-idf.yaml new file mode 100644 index 0000000000..9ca0ce7f05 --- /dev/null +++ b/tests/components/resampler/test-microphone.esp32-idf.yaml @@ -0,0 +1,6 @@ +substitutions: + din_pin: GPIO21 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml + resampler: !include common-microphone.yaml diff --git a/tests/components/resampler/test-microphone.esp32-s3-idf.yaml b/tests/components/resampler/test-microphone.esp32-s3-idf.yaml new file mode 100644 index 0000000000..5ff29672a0 --- /dev/null +++ b/tests/components/resampler/test-microphone.esp32-s3-idf.yaml @@ -0,0 +1,6 @@ +substitutions: + din_pin: GPIO16 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-s3-idf.yaml + resampler: !include common-microphone.yaml From 788d2bc6316d8d08eb8e46ead9e713e6fe1e82f9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:19:30 -0500 Subject: [PATCH 023/221] [ci] Balance component test batches by estimated build time (#19980) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- script/determine-jobs.py | 5 - script/split_components_for_ci.py | 216 +++++++++++++------ tests/script/test_determine_jobs.py | 19 +- tests/script/test_split_components_for_ci.py | 157 ++++++++++++++ 4 files changed, 318 insertions(+), 79 deletions(-) create mode 100644 tests/script/test_split_components_for_ci.py diff --git a/script/determine-jobs.py b/script/determine-jobs.py index bd127a7a43..9c8f3cd3b4 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -99,10 +99,6 @@ from split_components_for_ci import create_intelligent_batches # For large PRs (>= 65 files), use split for better parallelization CLANG_TIDY_SPLIT_THRESHOLD = 65 -# Component test batch size (weighted) -# Isolated components count as 10x, groupable components count as 1x -COMPONENT_TEST_BATCH_SIZE = 40 - # Above the threshold, fan out across up to this many jobs, balanced by the # recorded per-file durations. The target is serial junit-time weight per # bucket, not wall time (calibrated with the conftest compile cap); it @@ -1534,7 +1530,6 @@ def main() -> None: batches, _ = create_intelligent_batches( components=changed_components_with_tests, tests_dir=tests_dir, - batch_size=COMPONENT_TEST_BATCH_SIZE, directly_changed=batch_directly_changed, ) # Convert batches to CI matrix entries: the component list plus which diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index 7f06f50f48..cc458d5d66 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -13,7 +13,9 @@ from __future__ import annotations import argparse from collections import defaultdict +from dataclasses import dataclass, field import json +import math from pathlib import Path import sys @@ -28,13 +30,34 @@ from script.analyze_component_buses import ( create_grouping_signature, merge_compatible_bus_groups, ) -from script.helpers import get_component_test_files, split_conflicting_groups +from script.helpers import ( + get_component_test_files, + parse_test_filename, + split_conflicting_groups, +) -# Weighting for batch creation -# Isolated components can't be grouped/merged, so they count as 10x -# Groupable components can be merged into single builds, so they count as 1x -ISOLATED_WEIGHT = 10 -GROUPABLE_WEIGHT = 1 +# Estimated CI seconds for one build per test platform, read from batch job +# logs. Absolute accuracy does not matter, only the ratios between platforms. +# A full run takes over an hour, so recording real timings is not practical. +PLATFORM_BUILD_SECONDS = { + "host": 10, + "esp8266-ard": 30, + "esp32-idf": 45, + "rtl87xx-ard": 35, + "ln882x-ard": 50, + "rp2040-ard": 50, + "rp2350-ard": 50, + "bk72xx-ard": 60, +} +NRF52_BUILD_SECONDS = 45 +# Other ESP32 chips and Arduino on ESP32 share less of the ccache +DEFAULT_BUILD_SECONDS = 90 +# Each extra component merged into a grouped build makes it larger +GROUPED_COMPONENT_SECONDS = 5 +# Estimated build seconds per CI runner; sets how many runners are used. +# Approximate: the runner count charges each grouped build once, but a large +# group spreads over several runners, which each pay for that build. +TARGET_BATCH_SECONDS = 600 # Platform used for batching (platform-agnostic batching) # Batches are split across CI runners and each runner tests all platforms @@ -61,10 +84,115 @@ def has_test_files(component_name: str, tests_dir: Path) -> bool: ) +def build_seconds(platform: str) -> int: + """Return the estimated CI seconds for one build on a test platform.""" + if (seconds := PLATFORM_BUILD_SECONDS.get(platform)) is not None: + return seconds + if platform.startswith("nrf52"): + return NRF52_BUILD_SECONDS + return DEFAULT_BUILD_SECONDS + + +@dataclass +class _BatchItem: + """A component and the builds it adds to the batch it lands in.""" + + component: str + # Builds that always run on their own (isolated tests, variants on a + # platform without a base test) + own_seconds: int + # Per (signature, platform) shared build: the seconds of this component's + # files on that platform when no other batch member shares the build + grouped_builds: dict[tuple[str, str], int] = field(default_factory=dict) + + def standalone_seconds(self) -> int: + return self.own_seconds + sum(self.grouped_builds.values()) + + +def _grouped_build_seconds(platform: str, solo_seconds: list[int]) -> int: + """Return the seconds of one (signature, platform) build in a batch. + + test_build_components only groups when two or more members share the + build; a grouped member then skips its variants on that platform. + """ + if len(solo_seconds) <= 1: + return sum(solo_seconds) + return build_seconds(platform) + GROUPED_COMPONENT_SECONDS * (len(solo_seconds) - 1) + + +def _make_item( + tests_dir: Path, component: str, signature: str, is_isolated: bool +) -> _BatchItem: + files_by_platform: dict[str, list[str]] = defaultdict(list) + for test_file in (tests_dir / component).glob("test[.-]*.yaml"): + test_name, platform = parse_test_filename(test_file) + files_by_platform[platform].append(test_name) + own_seconds = 0 + grouped: dict[tuple[str, str], int] = {} + for platform, test_names in files_by_platform.items(): + seconds = build_seconds(platform) * len(test_names) + if is_isolated or "test" not in test_names: + own_seconds += seconds + else: + grouped[(signature, platform)] = seconds + return _BatchItem(component, own_seconds, grouped) + + +class _Batch: + """A CI runner's components and its estimated build seconds.""" + + def __init__(self) -> None: + self.components: list[str] = [] + self.seconds = 0 + self.grouped_builds: dict[tuple[str, str], list[int]] = defaultdict(list) + + def added_seconds(self, item: _BatchItem) -> int: + """Return the seconds item would add; joining an existing build is cheap.""" + added = item.own_seconds + for (signature, platform), solo in item.grouped_builds.items(): + members = self.grouped_builds.get((signature, platform), []) + added += _grouped_build_seconds( + platform, [*members, solo] + ) - _grouped_build_seconds(platform, members) + return added + + def add(self, item: _BatchItem) -> None: + self.seconds += self.added_seconds(item) + for build, solo in item.grouped_builds.items(): + self.grouped_builds[build].append(solo) + self.components.append(item.component) + + +def balance_batches(items: list[_BatchItem], target_seconds: int) -> list[list[str]]: + """Spread items over enough runners to stay near target_seconds each. + + The runner count comes from the total estimate with every grouped build + counted once. Heaviest items go first, each to the runner that ends up + lightest, so grouped components follow the builds they can join. + """ + if not items: + return [] + grouped_builds: dict[tuple[str, str], list[int]] = defaultdict(list) + for item in items: + for build, solo in item.grouped_builds.items(): + grouped_builds[build].append(solo) + total = sum(item.own_seconds for item in items) + sum( + _grouped_build_seconds(platform, solos) + for (_, platform), solos in grouped_builds.items() + ) + count = min(len(items), max(1, math.ceil(total / target_seconds))) + batches = [_Batch() for _ in range(count)] + for item in sorted(items, key=lambda i: (-i.standalone_seconds(), i.component)): + min(batches, key=lambda b: (b.seconds + b.added_seconds(item), b.seconds)).add( + item + ) + return [batch.components for batch in batches if batch.components] + + def create_intelligent_batches( components: list[str], tests_dir: Path, - batch_size: int = 40, + target_seconds: int = TARGET_BATCH_SECONDS, directly_changed: set[str] | None = None, ) -> tuple[list[list[str]], dict[tuple[str, str], list[str]]]: """Create batches optimized for component grouping. @@ -76,7 +204,7 @@ def create_intelligent_batches( Args: components: List of component names to batch tests_dir: Path to tests/components directory - batch_size: Target size for each batch + target_seconds: Estimated build seconds per batch directly_changed: Set of directly changed components (for logging only) Returns: @@ -157,51 +285,17 @@ def create_intelligent_batches( # actually be split into two at build time -- throwing off CI distribution. signature_groups = split_conflicting_groups(signature_groups) - # Create batches by keeping signature groups together - # Components with the same signature stay in the same batches - batches = [] - - # Sort signature groups to prioritize groupable components - # 1. Put "isolated_*" signatures last (can't be grouped with others) - # 2. Sort groupable signatures by size (largest first) - # 3. "no_buses" components CAN be grouped together - def sort_key(item): - (_platform, signature), components = item - is_isolated = signature.startswith(ISOLATED_SIGNATURE_PREFIX) - # Put "isolated_*" last (1), groupable first (0) - # Within each category, sort by size (largest first) - return (is_isolated, -len(components)) - - sorted_groups = sorted(signature_groups.items(), key=sort_key) - - # Strategy: Create batches using weighted sizes - # - Isolated components count as 10x (since they can't be grouped/merged) - # - Groupable components count as 1x (can be merged into single builds) - # - This distributes isolated components across more runners - # - Ensures each runner has a good mix of groupable vs isolated components - - current_batch = [] - current_weight = 0 - - for (_platform, signature), group_components in sorted_groups: - is_isolated = signature.startswith(ISOLATED_SIGNATURE_PREFIX) - weight_per_component = ISOLATED_WEIGHT if is_isolated else GROUPABLE_WEIGHT - - for component in group_components: - # Check if adding this component would exceed the batch size - if current_weight + weight_per_component > batch_size and current_batch: - # Start a new batch - batches.append(current_batch) - current_batch = [] - current_weight = 0 - - # Add component to current batch - current_batch.append(component) - current_weight += weight_per_component - - # Don't forget the last batch - if current_batch: - batches.append(current_batch) + items = [ + _make_item( + tests_dir, + component, + signature, + signature.startswith(ISOLATED_SIGNATURE_PREFIX), + ) + for (_platform, signature), group_components in sorted(signature_groups.items()) + for component in group_components + ] + batches = balance_batches(items, target_seconds) return batches, signature_groups @@ -218,11 +312,11 @@ def main() -> int: help="JSON array of component names", ) parser.add_argument( - "--batch-size", - "-b", + "--target-seconds", + "-t", type=int, - default=40, - help="Target batch size (default: 40, weighted)", + default=TARGET_BATCH_SECONDS, + help=f"Estimated build seconds per batch (default: {TARGET_BATCH_SECONDS})", ) parser.add_argument( "--tests-dir", @@ -268,7 +362,7 @@ def main() -> int: batches, signature_groups = create_intelligent_batches( components=components, tests_dir=args.tests_dir, - batch_size=args.batch_size, + target_seconds=args.target_seconds, directly_changed=directly_changed, ) @@ -378,15 +472,15 @@ def main() -> int: file=sys.stderr, ) - print(f" - Groupable (weight=1): {groupable_count}", file=sys.stderr) - print(f" - Isolated (weight=10): {isolated_count}", file=sys.stderr) + print(f" - Groupable: {groupable_count}", file=sys.stderr) + print(f" - Isolated: {isolated_count}", file=sys.stderr) if actual_components < len(components): print( f"Components skipped (no test files): {len(components) - actual_components}", file=sys.stderr, ) print(f"Number of batches: {len(batches)}", file=sys.stderr) - print(f"Batch size target (weighted): {args.batch_size}", file=sys.stderr) + print(f"Target build seconds per batch: {args.target_seconds}", file=sys.stderr) if len(batches) > 0: print( f"Average components per batch: {actual_components / len(batches):.1f}", diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index cea2a2bfe6..da688703fc 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -2594,7 +2594,7 @@ def test_detect_platform_hint_from_filename_case_insensitive( assert result == expected_platform -def test_component_batching_beta_branch_40_per_batch( +def test_component_batching_beta_branch_groups_evenly( tmp_path: Path, mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, @@ -2604,11 +2604,10 @@ def test_component_batching_beta_branch_40_per_batch( mock_determine_cpp_unit_tests: Mock, capsys: pytest.CaptureFixture[str], ) -> None: - """Test that beta/release branches create batches with 40 actual components each. + """Test that beta/release branches group every component and split evenly. For beta/release branches, all components should be groupable (not isolated), - and each batch should contain 40 actual components with weight 1 each. - This matches the original behavior before consolidation. + so they share one grouped build per runner and spread evenly. """ # Create 120 test components with test files component_names = [f"comp_{i:03d}" for i in range(120)] @@ -2672,15 +2671,9 @@ def test_component_batching_beta_branch_40_per_batch( assert "component_test_batches" in output batches = output["component_test_batches"] - # Should have 3 batches (120 components / 40 per batch = 3) - assert len(batches) == 3, f"Expected 3 batches, got {len(batches)}" - - # Each batch should have approximately 40 components (all weight=1, groupable) - for i, batch in enumerate(batches): - batch_components = batch["components"].split() - assert len(batch_components) == 40, ( - f"Batch {i} should have 40 components, got {len(batch_components)}" - ) + # One grouped esp32-idf build plus 5 s per extra component is 640 s, + # so two runners of 60 components each + assert [len(batch["components"].split()) for batch in batches] == [60, 60] # Verify all 120 components are in batches all_components = [] diff --git a/tests/script/test_split_components_for_ci.py b/tests/script/test_split_components_for_ci.py new file mode 100644 index 0000000000..88a1a17b33 --- /dev/null +++ b/tests/script/test_split_components_for_ci.py @@ -0,0 +1,157 @@ +"""Unit tests for script/split_components_for_ci.py module.""" + +from collections.abc import Generator +from pathlib import Path +import sys +from unittest.mock import patch + +import pytest + +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) +sys.path.insert(0, script_dir) + +import split_components_for_ci # noqa: E402 + +import script.helpers # noqa: E402 + + +@pytest.fixture +def tests_dir(tmp_path: Path) -> Generator[Path, None, None]: + """Point the helpers at an empty tests/components tree.""" + path = tmp_path / "tests" / "components" + path.mkdir(parents=True) + with patch.object(script.helpers, "root_path", str(tmp_path)): + yield path + + +def _add_component(tests_dir: Path, name: str, files: list[str]) -> None: + comp_dir = tests_dir / name + comp_dir.mkdir() + for file in files: + (comp_dir / file).write_text(f"# {name}\n") + + +def test_build_seconds() -> None: + """Known platforms use the table; nrf52 boards share one estimate.""" + build_seconds = split_components_for_ci.build_seconds + assert build_seconds("host") == 10 + assert build_seconds("esp32-idf") == 45 + assert ( + build_seconds("nrf52-xiao-ble") == split_components_for_ci.NRF52_BUILD_SECONDS + ) + assert ( + build_seconds("esp32-s3-idf") == split_components_for_ci.DEFAULT_BUILD_SECONDS + ) + + +def test_make_item_isolated_and_grouped(tests_dir: Path) -> None: + """Isolated components own every build; groupable ones join shared builds.""" + _add_component( + tests_dir, + "comp", + [ + "test.esp32-idf.yaml", + "test.host.yaml", + "test-extra.esp32-idf.yaml", + "test-only.rp2040-ard.yaml", + "validate.esp32-idf.yaml", + ], + ) + + isolated = split_components_for_ci._make_item( + tests_dir, "comp", "isolated_comp", True + ) + assert isolated.own_seconds == 45 + 10 + 45 + 50 + assert isolated.grouped_builds == {} + + # The rp2040 variant has no base test to group with, so it always runs + grouped = split_components_for_ci._make_item(tests_dir, "comp", "i2c", False) + assert grouped.own_seconds == 50 + assert grouped.grouped_builds == {("i2c", "esp32-idf"): 90, ("i2c", "host"): 10} + + +def test_grouped_component_joins_existing_build() -> None: + """A second member turns a lone build into a group that skips variants.""" + item = split_components_for_ci._BatchItem + batch = split_components_for_ci._Batch() + batch.add(item("a", 0, {("i2c", "esp32-idf"): 90})) + assert batch.seconds == 90 + + other = item("b", 0, {("i2c", "esp32-idf"): 45, ("i2c", "host"): 10}) + grouped = 45 + split_components_for_ci.GROUPED_COMPONENT_SECONDS + assert batch.added_seconds(other) == grouped - 90 + 10 + batch.add(other) + assert batch.seconds == grouped + 10 + + +def test_isolated_components_balance_across_runners(tests_dir: Path) -> None: + """Heavy isolated components spread out and light ones fill the gaps.""" + platforms = ["esp32-idf", "esp8266-ard", "rp2040-ard", "bk72xx-ard"] + for name in ("big_a", "big_b"): + _add_component( + tests_dir, + name, + [f"test.{p}.yaml" for p in platforms] + + [f"test-variant.{p}.yaml" for p in platforms], + ) + for name in ("small_a", "small_b"): + _add_component(tests_dir, name, ["test.esp32-idf.yaml"]) + + batches, _ = split_components_for_ci.create_intelligent_batches( + components=["big_a", "big_b", "small_a", "small_b"], + tests_dir=tests_dir, + target_seconds=500, + directly_changed={"big_a", "big_b", "small_a", "small_b"}, + ) + + assert sorted(sorted(batch) for batch in batches) == [ + ["big_a", "small_a"], + ["big_b", "small_b"], + ] + + +def test_groupable_variants_skipped_when_grouped(tests_dir: Path) -> None: + """Grouped members skip their variants, so the group fits one runner.""" + names = [f"comp_{i:02d}" for i in range(12)] + for name in names: + _add_component( + tests_dir, + name, + ["test.esp32-idf.yaml", "test-a.esp32-idf.yaml", "test-b.esp32-idf.yaml"], + ) + + batches, _ = split_components_for_ci.create_intelligent_batches( + components=names, tests_dir=tests_dir, target_seconds=400 + ) + + assert batches == [names] + + +def test_groupable_components_split_evenly(tests_dir: Path) -> None: + """A group too large for one runner spreads evenly.""" + platforms = ["esp32-idf", "esp8266-ard", "rp2040-ard", "bk72xx-ard"] + names = [f"comp_{i:02d}" for i in range(12)] + for name in names: + _add_component(tests_dir, name, [f"test.{p}.yaml" for p in platforms]) + + batches, _ = split_components_for_ci.create_intelligent_batches( + components=names, tests_dir=tests_dir, target_seconds=100 + ) + + assert sorted(len(batch) for batch in batches) == [2, 2, 2, 3, 3] + assert sorted(c for batch in batches for c in batch) == names + + +def test_make_item_reads_tests_dir(tmp_path: Path) -> None: + """Costs come from the given tests_dir, not the repository tree.""" + other = tmp_path / "other" + other.mkdir() + _add_component(other, "comp", ["test.esp32-idf.yaml"]) + + item = split_components_for_ci._make_item(other, "comp", "isolated_comp", True) + assert item.own_seconds == 45 + + +def test_balance_batches_empty() -> None: + """No components means no runners.""" + assert split_components_for_ci.balance_batches([], 600) == [] From 3bae9e4ee2fddbd46711386d3fad917051f8dfcb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:33:50 -0500 Subject: [PATCH 024/221] [climate] Fix the restore state memset warning on every platform (#19983) --- esphome/components/climate/climate.cpp | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 494efd8edc..d08952115b 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -368,19 +368,11 @@ optional Climate::restore_state_() { } void Climate::save_state_(const ClimateTraits &traits) { -#if (defined(USE_ESP32) || defined(USE_ESP8266)) && !defined(CLANG_TIDY) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" -#define TEMP_IGNORE_MEMACCESS -#endif ClimateDeviceRestoreState state{}; - // initialize as zero (including padding) to prevent random data on stack triggering erase + // initialize as zero (including padding) to prevent random data on stack triggering erase; + // the void * cast tells GCC the bytewise clear of a non-trivial type is intentional // NOLINTNEXTLINE(bugprone-raw-memory-call-on-non-trivial-type) -- intentional bytewise zero for RTC save - memset(&state, 0, sizeof(ClimateDeviceRestoreState)); -#ifdef TEMP_IGNORE_MEMACCESS -#pragma GCC diagnostic pop -#undef TEMP_IGNORE_MEMACCESS -#endif + memset(static_cast(&state), 0, sizeof(ClimateDeviceRestoreState)); state.mode = this->mode; if (traits.has_feature_flags(CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE | From e9f0a53ec97788b3aa8f5032837aab2ede76df35 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:34:17 -0500 Subject: [PATCH 025/221] [display] Fix the null pointer comparison warning in the display log macro (#19984) --- esphome/components/display/display.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/components/display/display.h b/esphome/components/display/display.h index bebeba8dcb..cc75eb70b8 100644 --- a/esphome/components/display/display.h +++ b/esphome/components/display/display.h @@ -287,13 +287,15 @@ template class DisplayWriter { // Type alias for Display writer - uses optimized DisplayWriter instead of std::function using display_writer_t = DisplayWriter; +// Callers usually pass `this`; checking a copy keeps GCC's -Wnonnull-compare quiet. #define LOG_DISPLAY(prefix, type, obj) \ - if ((obj) != nullptr) { \ + if (auto *log_display_obj = (obj); log_display_obj != nullptr) { \ ESP_LOGCONFIG(TAG, \ prefix type "\n" \ "%s Rotations: %d °\n" \ "%s Dimensions: %dpx x %dpx", \ - prefix, (obj)->rotation_, prefix, (obj)->get_width(), (obj)->get_height()); \ + prefix, log_display_obj->rotation_, prefix, log_display_obj->get_width(), \ + log_display_obj->get_height()); \ } /// Turn the pixel OFF. From 2de4f4cf8151bf7084145e8756b82b2379e6af8d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:34:43 -0500 Subject: [PATCH 026/221] [speaker] Fix the fall through warning in the media player turn off command (#19987) --- .../components/speaker/media_player/speaker_media_player.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/speaker/media_player/speaker_media_player.cpp b/esphome/components/speaker/media_player/speaker_media_player.cpp index 06525235d4..cac2f6649c 100644 --- a/esphome/components/speaker/media_player/speaker_media_player.cpp +++ b/esphome/components/speaker/media_player/speaker_media_player.cpp @@ -224,7 +224,7 @@ void SpeakerMediaPlayer::watch_media_commands_() { break; case media_player::MEDIA_PLAYER_COMMAND_TURN_OFF: this->is_turn_off_ = true; - // Intentional Fall-through + [[fallthrough]]; #endif case media_player::MEDIA_PLAYER_COMMAND_STOP: // Pipelines do not stop immediately after calling the stop command, so confirm its stopped before unpausing. From e500ece4d5ca69471d61f50d31f9f15fa4e66965 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:01 -0500 Subject: [PATCH 027/221] [uart] Fix a maybe uninitialized warning in the fixed size read (#19988) --- esphome/components/uart/uart.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/uart/uart.h b/esphome/components/uart/uart.h index eda5b72ea8..d1c6773363 100644 --- a/esphome/components/uart/uart.h +++ b/esphome/components/uart/uart.h @@ -37,9 +37,10 @@ class UARTDevice { bool read_array(uint8_t *data, size_t len) { return this->parent_->read_array(data, len); } template optional> read_array() { // NOLINT - std::array res; - if (!this->read_array(res.data(), N)) { - return {}; + // One object returned on both paths; a separate empty optional trips GCC's -Wmaybe-uninitialized. + optional> res(std::in_place); + if (!this->read_array(res->data(), N)) { + res.reset(); } return res; } From 36bcce563ab7f72d2b8864dd1c26bb807c701fbc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:15 -0500 Subject: [PATCH 028/221] [sensor] Fix a double promotion warning in the NTC resistance filter (#19985) --- esphome/components/sensor/filter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index dbd6f4d34b..9b9d7c033a 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -450,7 +450,7 @@ optional ToNTCResistanceFilter::new_value(float value) { } double k = 273.15; // https://de.wikipedia.org/wiki/Steinhart-Hart-Gleichung#cite_note-stein2_s4-3 - double t = value + k; + double t = static_cast(value) + k; double y = (this->a_ - 1 / (t)) / (2 * this->c_); double x = sqrt(pow(this->b_ / (3 * this->c_), 3) + y * y); double resistance = exp(pow(x - y, 1 / 3.0) - pow(x + y, 1 / 3.0)); From 13463422507ebd7eee7a97403a65d1e15337b8df Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:41 -0500 Subject: [PATCH 029/221] [light] Fix a double promotion warning in the light test config (#19989) --- tests/components/light/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/light/common.yaml b/tests/components/light/common.yaml index c40cfd5e3a..046acef9ce 100644 --- a/tests/components/light/common.yaml +++ b/tests/components/light/common.yaml @@ -188,7 +188,7 @@ number: then: - light.turn_on: id: test_monochromatic_light - brightness: !lambda "return x / 100.0;" + brightness: !lambda "return x / 100.0f;" output: - platform: template From 5d4b436fc003da8201a52a52f0aa54e181ca3165 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:56 -0500 Subject: [PATCH 030/221] [mk2pvrouter] Fix an ignored qualifiers warning in the group length check (#19986) --- esphome/components/mk2pvrouter/mk2pvrouter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.cpp b/esphome/components/mk2pvrouter/mk2pvrouter.cpp index 0c0476fb11..f8aa1003bc 100644 --- a/esphome/components/mk2pvrouter/mk2pvrouter.cpp +++ b/esphome/components/mk2pvrouter/mk2pvrouter.cpp @@ -1,5 +1,6 @@ #include "mk2pvrouter.h" #include "esphome/core/log.h" +#include #include namespace esphome::mk2pvrouter { @@ -48,7 +49,7 @@ uint8_t Mk2PVRouter::calculate_crc_(const char *grp, size_t grp_len) { // Verifies the CRC of a group against its trailing CRC byte. bool Mk2PVRouter::check_crc_(const char *grp, const char *grp_end) { const auto grp_len = grp_end - grp; - if (grp_len < static_cast(CRC_SUFFIX_LEN)) { + if (grp_len < static_cast(CRC_SUFFIX_LEN)) { ESP_LOGE(TAG, "Empty or too short group"); return false; } From 3ed1bc7e2077b666ae36c3c2f9581e458eeff0b0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:50:46 -0500 Subject: [PATCH 031/221] [esp32_ble_tracker] Guard Bluedroid against stale queued connection requests (#19375) --- .../components/esp32_ble_tracker/__init__.py | 24 ++++++ .../esp32_ble_tracker/bluedroid_stubs.cpp | 48 ++++++++++++ esphome/core/defines.h | 1 + .../test_direct_conn_guard.py | 73 +++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp create mode 100644 tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 205acc55a5..fc52b92ee7 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -436,6 +436,25 @@ async def to_code(config: ConfigType) -> None: cg.add_define("USE_ESP32_BLE_SOFTWARE_COEXISTENCE") +# First tagged release per series with espressif/esp-idf@82e71c1767 (see bluedroid_stubs.cpp). +# A series without an entry keeps the guard until a fixed release is tagged; the guard is +# harmless on fixed sources. The 5.4, 5.5 and 6.1 branches carry the fix but have no tag yet. +DIRECT_CONN_FIX_VERSIONS = { + (5, 2): cv.Version(5, 2, 8), + (5, 3): cv.Version(5, 3, 6), + (6, 0): cv.Version(6, 0, 3), +} +DIRECT_CONN_FIX_ALL_FROM = cv.Version(6, 2, 0) + + +def _needs_direct_conn_guard() -> bool: + ver = idf_version() + if ver >= DIRECT_CONN_FIX_ALL_FROM: + return False + fixed = DIRECT_CONN_FIX_VERSIONS.get((ver.major, ver.minor)) + return fixed is None or ver < fixed + + # This needs to be run as a job with very low priority so that all components have # chance to call register_ble_tracker and register_client before the list is checked # and added to the global defines list. @@ -452,6 +471,11 @@ async def _add_ble_features() -> None: if BLEFeatures.ESP_BT_DEVICE in required_features: cg.add_define("USE_ESP32_BLE_DEVICE") cg.add_define("USE_ESP32_BLE_UUID") + if cg.get_slot_count(CLIENT_COUNT_DEFINE) and _needs_direct_conn_guard(): + # --undefined keeps the wrapper, libsrc.a is scanned before the IDF libraries + cg.add_define("USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD") + cg.add_build_flag("-Wl,--wrap=l2cble_init_direct_conn") + cg.add_build_flag("-Wl,--undefined=__wrap_l2cble_init_direct_conn") ESP32_BLE_START_SCAN_ACTION_SCHEMA = cv.Schema( diff --git a/esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp b/esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp new file mode 100644 index 0000000000..5cedc22d06 --- /dev/null +++ b/esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp @@ -0,0 +1,48 @@ +/* + * Bluedroid queues outgoing BLE connections as raw link block pointers and does + * not drop them when the block is released, so btm_send_pending_direct_conn() + * can start a connect on a released block and l2c_link_timeout() later crashes + * on its null timer parameter. Mirrors espressif/esp-idf@82e71c1767; codegen + * only enables it for releases without that commit. + */ + +#include "esphome/core/defines.h" + +#ifdef USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD + +#include +#include +#include "esphome/core/log.h" + +namespace esphome::esp32_ble_tracker { +static const char *const TAG = "esp32_ble_tracker"; +} // namespace esphome::esp32_ble_tracker + +static_assert(ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(6, 2, 0), + "ESP-IDF 6.2 and later have the fix, this guard should not be enabled (esphome/esphome#19373)"); + +// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) +extern "C" { + +bool __real_l2cble_init_direct_conn(void *p_lcb); +void l2cu_release_lcb(void *p_lcb); + +bool __wrap_l2cble_init_direct_conn(void *p_lcb) { + // in_use is the first member of the private tL2C_LCB (checked ESP-IDF 5.0 to 6.1) + const auto *in_use = static_cast(p_lcb); + if (p_lcb == nullptr || *in_use == 0) { + ESP_LOGW(esphome::esp32_ble_tracker::TAG, "Dropped queued connect on a released link block"); + return false; + } + const bool started = __real_l2cble_init_direct_conn(p_lcb); + // Every failure path releases the block except unknown device, also fixed upstream + if (!started && *in_use != 0) { + l2cu_release_lcb(p_lcb); + } + return started; +} + +} // extern "C" +// NOLINTEND(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) + +#endif // USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 88f35dac46..9b3252fa05 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -389,6 +389,7 @@ #define USE_ESP32_BLE_SERVER_ON_CONNECT #define USE_ESP32_BLE_SERVER_ON_DISCONNECT #define USE_ESP32_BLE_TRACKER +#define USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD #define USE_BLE_GATT_CLIENT #define ESPHOME_BLE_GATT_CLIENT_COUNT 1 #define ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT 1 diff --git a/tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py b/tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py new file mode 100644 index 0000000000..d3aa2ba878 --- /dev/null +++ b/tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py @@ -0,0 +1,73 @@ +"""The Bluedroid queued connection guard is emitted only for client builds on unfixed ESP-IDF.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome import config_validation as cv +from esphome.components import esp32_ble_tracker +from esphome.core import CORE + +# Spelled out so a typo in the component's flags fails here +_GUARD_FLAGS = { + "-Wl,--wrap=l2cble_init_direct_conn", + "-Wl,--undefined=__wrap_l2cble_init_direct_conn", +} + + +def _pin_idf(monkeypatch: pytest.MonkeyPatch, idf: str) -> None: + monkeypatch.setattr(esp32_ble_tracker, "idf_version", lambda: cv.Version.parse(idf)) + + +@pytest.mark.parametrize( + ("config_file", "idf", "expected"), + [ + pytest.param("scan_window_raised.yaml", "5.5.5", True, id="client"), + pytest.param("scan_window_raised.yaml", "6.0.3", False, id="client_fixed_idf"), + pytest.param("scan_window_scan_only.yaml", "5.5.5", False, id="scan_only"), + ], +) +def test_guard_only_in_client_builds_on_unfixed_idf( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + monkeypatch: pytest.MonkeyPatch, + config_file: str, + idf: str, + expected: bool, +) -> None: + _pin_idf(monkeypatch, idf) + generate_main(component_config_path(config_file)) + assert (CORE.build_flags >= _GUARD_FLAGS) is expected + assert CORE.build_flags.isdisjoint(_GUARD_FLAGS) is not expected + defines = {define.name for define in CORE.defines} + assert ("USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD" in defines) is expected + + +@pytest.mark.parametrize( + ("idf", "expected"), + [ + ("5.1.6", True), # series that never got the fix + ("5.2.7", True), + ("5.2.8", False), + ("5.3.5", True), + ("5.3.6", False), + ("5.4.4", True), + ("5.4.5", True), # no fixed 5.4, 5.5 or 6.1 tag yet + ("5.5.5", True), + ("5.5.6", True), + ("6.0.2", True), + ("6.0.3", False), + ("6.1.0", True), + ("6.1.1", True), + ("6.2.0", False), + ("7.0.0", False), + ], +) +def test_needs_direct_conn_guard( + monkeypatch: pytest.MonkeyPatch, idf: str, expected: bool +) -> None: + _pin_idf(monkeypatch, idf) + assert esp32_ble_tracker._needs_direct_conn_guard() is expected From 0ed997da926cccfd14ce043d4fc10c1597abc25f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:55:30 -0500 Subject: [PATCH 032/221] [espidf] Add --skip-bootloader for OTA only builds (#19925) --- esphome/__main__.py | 47 +++- esphome/build_gen/espidf.py | 74 ++++++ esphome/components/esp32/__init__.py | 27 ++- esphome/core/__init__.py | 3 + esphome/espidf/toolchain.py | 82 ++++++- script/check_idf_py_equivalence.py | 77 +++++-- tests/script/test_check_idf_py_equivalence.py | 80 ++++++- tests/unit_tests/build_gen/test_espidf.py | 72 ++++++ tests/unit_tests/test_download_types.py | 29 +++ tests/unit_tests/test_espidf_toolchain.py | 218 +++++++++++++++++- tests/unit_tests/test_main.py | 104 +++++++++ 11 files changed, 765 insertions(+), 48 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index e4213e8edb..cbeb9ff01c 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -148,6 +148,7 @@ class ArgsProtocol(Protocol): file: str | None no_logs: bool only_generate: bool + skip_bootloader: bool show_secrets: bool dashboard: bool configuration: str @@ -827,6 +828,14 @@ def write_cpp_file() -> int: def compile_program(args: ArgsProtocol, config: ConfigType) -> int: + if CORE.skip_bootloader and not (CORE.is_esp32 and CORE.using_toolchain_esp_idf): + # Info, not a warning: an orchestrator cannot see YAML toolchain + # overrides, this is its expected no-op, and a full build is safe. + _LOGGER.info( + "--skip-bootloader ignored: only supported on ESP32 with the " + "esp-idf toolchain" + ) + CORE.skip_bootloader = False # Keep this gate here, NOT in config validation: device-builder needs # `esphome config` to keep succeeding with placeholders so onboarding can run. if CONF_WIFI in config: @@ -857,7 +866,10 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: return rc # Create factory.bin, ota.bin, and firmware.elf copy - toolchain.create_factory_bin() + if not toolchain.create_factory_bin(): + # A build whose factory image could not be produced must not + # exit 0; downloads would serve an image from an older build. + return 1 toolchain.create_ota_bin() toolchain.create_elf_copy() from esphome.build_helpers.idedata import warn_if_idedata_missing @@ -975,8 +987,9 @@ def upload_using_esptool( # and partitions included where the target needs them) image = native.get_factory_firmware_path() if not image.is_file(): + hint = getattr(native, "missing_image_hint", lambda: None)() raise EsphomeError( - f"{image} does not exist; compile the configuration first" + hint or f"{image} does not exist; compile the configuration first" ) flash_images = [FlashImage(path=image, offset="0x0")] else: @@ -1379,6 +1392,12 @@ def _upload_via_native_api( ota_type = espota2.OTA_TYPE_UPDATE_PARTITION_TABLE elif getattr(args, "bootloader", False): check_partition_access("--bootloader") + if ( + getattr(args, "file", None) is None + and (native := native_backend()) + and (hint := getattr(native, "missing_image_hint", lambda: None)()) + ): + raise EsphomeError(hint) binary = CORE.bootloader_bin ota_type = espota2.OTA_TYPE_UPDATE_BOOTLOADER if getattr(args, "file", None) is not None: @@ -1754,6 +1773,18 @@ def command_logs(args: ArgsProtocol, config: ConfigType) -> int | None: def command_run(args: ArgsProtocol, config: ConfigType) -> int | None: + if ( + CORE.skip_bootloader + and CORE.is_esp32 + and CORE.using_toolchain_esp_idf + and any( + get_port_type(device) == PortType.SERIAL for device in (args.device or []) + ) + ): + # Fail before the compile: the result could never flash over serial. + # Elsewhere the flag is ignored, so serial stays fine there. + _LOGGER.error("--skip-bootloader builds cannot be flashed over serial") + return 1 exit_code = write_cpp(config) if exit_code != 0: return exit_code @@ -2139,6 +2170,15 @@ SIMPLE_CONFIG_ACTIONS = [ ] +def _add_skip_bootloader_arg(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--skip-bootloader", + help="Do not build the bootloader or the factory image; " + "the result can only be flashed over OTA.", + action="store_true", + ) + + def _add_states_args(parser: argparse.ArgumentParser) -> None: """Add mutually exclusive ``--states``/``--no-states`` flags to a parser. @@ -2280,6 +2320,7 @@ def parse_args(argv): help="Only generate source code, do not compile.", action="store_true", ) + _add_skip_bootloader_arg(parser_compile) parser_upload = subparsers.add_parser( "upload", @@ -2376,6 +2417,7 @@ def parse_args(argv): parser_run.add_argument( "--no-logs", help="Disable starting logs.", action="store_true" ) + _add_skip_bootloader_arg(parser_run) _add_states_args(parser_run) @@ -2635,6 +2677,7 @@ def run_esphome(argv): CORE.config_path = conf_path CORE.dashboard = args.dashboard + CORE.skip_bootloader = getattr(args, "skip_bootloader", False) if args.toolchain is not None: # CLI toolchain wins over esp32.toolchain in YAML. CORE.toolchain = args.toolchain diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 124fa3ddb1..3838d07602 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -3,6 +3,7 @@ import json import logging from pathlib import Path +import re import textwrap from esphome.build_helpers import pch @@ -120,6 +121,78 @@ def _cmake_quote(value: str) -> str: return f'"{escaped}"' +# CONFIG_APP_BUILD_BOOTLOADER is hidden and force-selected, so it can only be +# cleared at the CMake level (the same state IDF's RAM-app build type uses). +# The macro is IDF's __build_process_project_includes plus two added lines; +# the flag is ignored and the bootloader builds as usual if IDF changes it. +IDF_BOOTLOADER_OVERRIDE = """\ +# ESPHome bootloader skip switch; see esphome/espidf/toolchain.py. +if(ESPHOME_SKIP_BOOTLOADER) + macro(__build_process_project_includes) + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + set(CONFIG_APP_BUILD_BOOTLOADER "") + # bt's CMakeLists reads the lowercase idf_target that the (now + # skipped) bootloader project_include leaks; keep it defined, or + # its empty TARGET_SRC_NAME sends file(GLOB_RECURSE) across /. + idf_build_get_property(idf_target IDF_TARGET) + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(build_property ${build_properties}) + idf_build_get_property(val ${build_property}) + set(${build_property} "${val}") + endforeach() + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_PATH ${dir}) + if(EXISTS ${COMPONENT_DIR}/project_include.cmake) + include(${COMPONENT_DIR}/project_include.cmake) + endif() + endforeach() + endmacro() +endif() +""" + +# The lines the override adds to IDF's macro; idf_macro_matches() below +# strips them before comparing with the live macro. +BOOTLOADER_OVERRIDE_ADDED_LINES = ( + 'set(CONFIG_APP_BUILD_BOOTLOADER "")', + "idf_build_get_property(idf_target IDF_TARGET)", +) + +_MACRO = re.compile( + r"macro\(__build_process_project_includes\)(.*?)endmacro\(\)", re.DOTALL +) + + +def _normalized_macro(text: str) -> list[str] | None: + """The macro body as comment-free, whitespace-collapsed lines.""" + if (match := _MACRO.search(text)) is None: + return None + return [ + re.sub(r"\s+", " ", line) + for raw in match.group(1).splitlines() + if (line := raw.split("#", 1)[0].strip()) + ] + + +_EXPECTED_MACRO = [ + line + for line in _normalized_macro(IDF_BOOTLOADER_OVERRIDE) + if line not in BOOTLOADER_OVERRIDE_ADDED_LINES +] + + +def idf_macro_matches(idf_path: Path) -> bool: + """Whether IDF's macro still matches the copy the override replays.""" + build_cmake = idf_path / "tools" / "cmake" / "build.cmake" + live = _normalized_macro(build_cmake.read_text(encoding="utf-8")) + return live == _EXPECTED_MACRO + + def get_project_cmakelists( minimal: bool = False, builtin_components: list[str] | None = None ) -> str: @@ -258,6 +331,7 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src) include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +{IDF_BOOTLOADER_OVERRIDE} {ldgen_override} {cpp_standard_options} diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 49a115c9c5..6a693eff89 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -670,20 +670,29 @@ def get_download_types(storage_json): # No recorded firmware path means nothing was built; no downloads. if storage_json.firmware_bin_path is None: return [] - return [ - { - "title": "Factory format (Previously Modern)", - "description": "For use with ESPHome Web and other tools.", - "file": "firmware.factory.bin", - "download": f"{storage_json.name}.factory.bin", - }, + from esphome.espidf.toolchain import tree_skips_bootloader + + types = [] + # A --skip-bootloader tree deliberately has no factory image; an + # unreadable tree (PlatformIO, capability probes) reads as full. + if not tree_skips_bootloader(Path(storage_json.firmware_bin_path).parent): + types.append( + { + "title": "Factory format (Previously Modern)", + "description": "For use with ESPHome Web and other tools.", + "file": "firmware.factory.bin", + "download": f"{storage_json.name}.factory.bin", + } + ) + types.append( { "title": "OTA format (Previously Legacy)", "description": "For OTA updating a device.", "file": "firmware.ota.bin", "download": f"{storage_json.name}.ota.bin", - }, - ] + } + ) + return types def only_on_variant(*, supported=None, unsupported=None, msg_prefix="This feature"): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index fa3d6bb64e..30c0372066 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -589,6 +589,8 @@ class EsphomeCore: self.vscode = False # True if running in testing mode (disables validation checks for grouped testing) self.testing_mode = False + # True if this build skips the bootloader and factory image (OTA only) + self.skip_bootloader = False # The name of the node self.name: str | None = None # The friendly name of the node @@ -692,6 +694,7 @@ class EsphomeCore: from esphome.pins import PIN_SCHEMA_REGISTRY self.dashboard = False + self.skip_bootloader = False self.name = None self.friendly_name = None self.area = None diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index d73dbf6b7e..ffd8082515 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -26,11 +26,14 @@ from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary -from esphome.helpers import add_git_ceiling_directory, get_bool_env, write_file +from esphome.helpers import add_git_ceiling_directory, get_bool_env, rmtree, write_file _LOGGER = logging.getLogger(__name__) DOMAIN = "espidf_toolchain" +# The -D that tells the generated CMakeLists to skip the in-tree bootloader +# build; also read back from CMakeCache.txt to identify a tree. +SKIP_BOOTLOADER_DEFINE = "ESPHOME_SKIP_BOOTLOADER" @dataclass @@ -38,6 +41,7 @@ class _CacheData: paths: dict[str, tuple] = field(default_factory=dict) env: dict[str, dict[str, str]] = field(default_factory=dict) cmake_output: dict[Path, str] = field(default_factory=dict) + skip_bootloader: bool | None = None cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict) @@ -285,6 +289,48 @@ def _cache_entries_changed() -> bool: return any(cache.get(k) != v for k, v in _configure_defines().items()) +def tree_skips_bootloader(build_dir: Path) -> bool: + """Whether a configured tree was set up to skip the bootloader build. + + Total: an unreadable tree reads as the stock full build. + """ + try: + cache = _parse_cmakecache(build_dir / "CMakeCache.txt") + except (OSError, ValueError) as err: + _LOGGER.debug("Cannot read %s, assuming a full build: %s", build_dir, err) + return False + return cache.get(SKIP_BOOTLOADER_DEFINE) == "1" + + +def _skip_bootloader() -> bool: + """Whether this tree should not build a bootloader at all; per-run memo.""" + cache = _cache() + if cache.skip_bootloader is None: + cache.skip_bootloader = _compute_skip_bootloader() + return cache.skip_bootloader + + +def _compute_skip_bootloader() -> bool: + if not CORE.skip_bootloader: + return False + from esphome.build_gen.espidf import idf_macro_matches + + if not idf_macro_matches(_get_idf_path()): + _LOGGER.warning( + "--skip-bootloader ignored: IDF changed its bootloader macro; " + "building the bootloader" + ) + return False + return True + + +def missing_image_hint() -> str | None: + """Why an expected flash image is absent, for upload error messages.""" + if tree_skips_bootloader(_build_dir()): + return "this build was compiled with --skip-bootloader; recompile without it" + return None + + def _configure_defines() -> dict[str, str]: """Every ``-D`` idf.py passes to cmake, in its order.""" return { @@ -292,6 +338,9 @@ def _configure_defines() -> dict[str, str]: "PYTHON": _get_idf_tool("python"), "ESP_PLATFORM": "1", **_cache_entries(), + # ESPHome's own switch; idf.py never passes it and cmake keeps the + # cached value, so idf.py runs against the tree stay in the same mode. + SKIP_BOOTLOADER_DEFINE: "1" if _skip_bootloader() else "0", } @@ -312,6 +361,14 @@ def run_reconfigure(verbose: bool = False) -> int: """Run the CMake configure, with the arguments idf.py uses.""" build_dir = _build_dir() build_dir.mkdir(parents=True, exist_ok=True) + if _skip_bootloader() and not tree_skips_bootloader(build_dir): + # Flipping into skip mode: full-mode leftovers are stale for + # OTA --bootloader and downloads, and a partial cleanup would + # poison the flip back (deleted byproducts never regenerate). + for stale in ("bootloader", "bootloader-prefix"): + if (path := build_dir / stale).is_dir(): + rmtree(path) + get_factory_firmware_path().unlink(missing_ok=True) cmd = [_get_idf_tool("cmake"), "-G", "Ninja"] if _idf_py().binary_dir_arg: cmd += ["-B", str(build_dir)] @@ -861,6 +918,20 @@ def get_idedata() -> dict | None: def create_factory_bin() -> bool: """Create factory.bin by merging bootloader, partition table, and app.""" build_dir = CORE.relative_build_path("build") + if tree_skips_bootloader(build_dir): + # Nothing to merge, and nothing stale: the flip into skip mode + # already removed the factory image and the sub-build. + _LOGGER.info("Bootloader skipped; no factory image") + return True + if _merge_factory_bin(build_dir): + return True + # Never leave an image that does not match this build. + get_factory_firmware_path().unlink(missing_ok=True) + return False + + +def _merge_factory_bin(build_dir: Path) -> bool: + """Run the esptool merge for a full-build tree.""" flasher_args_path = build_dir / "flasher_args.json" if not flasher_args_path.is_file(): @@ -883,10 +954,11 @@ def create_factory_bin() -> bool: flash_data.get("flash_files", {}).items(), key=lambda kv: int(kv[0], 16) ): file_path = build_dir / fname - if file_path.is_file(): - sections.extend([addr, str(file_path)]) - else: - _LOGGER.warning("Flash file not found: %s", file_path) + if not file_path.is_file(): + # A partial factory image would not boot; never write one. + _LOGGER.error("Flash file not found: %s", file_path) + return False + sections.extend([addr, str(file_path)]) if not sections: _LOGGER.warning("No flash sections found") diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index ff7c58a5a6..c0aa3c0d5e 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -30,14 +30,31 @@ WATCHED = ( "build/compile_commands.json", "build/project_description.json", "build/config/sdkconfig.h", - "build/bootloader/bootloader.bin", ) -# Ninja logs whose outputs mean real work when their recorded mtime changes. -# The top level re-logs the bootloader step's byproducts on every build, so -# the bootloader is judged by its own sub-build log instead. +# Only a tree that builds the bootloader has one to watch. +BOOTLOADER_BIN = "build/bootloader/bootloader.bin" +OVERRIDE_INEFFECTIVE = ( + "skip-mode tree built a bootloader; the IDF_BOOTLOADER_OVERRIDE macro in " + "esphome/build_gen/espidf.py is not taking effect" +) +# Ninja logs whose outputs mean real work when their recorded mtime changes; +# the bootloader is judged by its own sub-build log when one exists. TOP_NINJA_LOG = "build/.ninja_log" -NINJA_LOGS = (TOP_NINJA_LOG, "build/bootloader/.ninja_log") + + +def _ninja_logs(skip_bootloader: bool) -> list[str]: + """A missing sub-build log stays an error in the mode that requires one.""" + logs = [TOP_NINJA_LOG] + if not skip_bootloader: + logs.append("build/bootloader/.ninja_log") + return logs + + BOOTLOADER_BYPRODUCT = re.compile(r"(^|/build/)bootloader/") +MACRO_CHANGED = ( + "IDF changed __build_process_project_includes; update " + "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py" +) WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld") DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*" @@ -46,19 +63,24 @@ def _digest(path: Path) -> str | None: return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None -def watched(name: str) -> list[str]: - """Files that change if idf.py configures or builds differently.""" - return [*WATCHED, f"build/{name}.elf", f"build/{name}.bin"] +def watched(name: str, skip_bootloader: bool) -> list[str]: + """The files idf.py must leave untouched for this tree's mode.""" + files = [*WATCHED, f"build/{name}.elf", f"build/{name}.bin"] + if not skip_bootloader: + files.append(BOOTLOADER_BIN) + return files -def _snapshot(build_path: Path, name: str) -> dict[str, str | None]: - return {f: _digest(build_path / f) for f in watched(name)} +def _snapshot( + build_path: Path, name: str, skip_bootloader: bool +) -> dict[str, str | None]: + return {f: _digest(build_path / f) for f in watched(name, skip_bootloader)} -def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]: +def _ninja_mtimes(build_path: Path, logs: list[str]) -> dict[tuple[str, str], str]: """(log, output) -> recorded mtime; compaction-safe, unlike a line count.""" mtimes = {} - for name in NINJA_LOGS: + for name in logs: log = build_path / name lines = log.read_text(encoding="utf-8").splitlines() if log.is_file() else [] for fields in (line.split("\t") for line in lines if not line.startswith("#")): @@ -69,10 +91,12 @@ def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]: return mtimes -def _log_problems(build_path: Path, mtimes: dict[tuple[str, str], str]) -> list[str]: +def _log_problems( + build_path: Path, mtimes: dict[tuple[str, str], str], logs: list[str] +) -> list[str]: """A missing or unparsable ninja log would otherwise compare as unchanged.""" problems = [] - for log in NINJA_LOGS: + for log in logs: if not (build_path / log).is_file(): problems.append(f"missing {log}") elif not any(k[0] == log and k[1].endswith(WORK_SUFFIXES) for k in mtimes): @@ -101,12 +125,22 @@ def _setup_core(build_path: Path, description: dict) -> tuple[str, str]: def check(build_path: Path) -> list[str]: """Return the problems found in one build tree.""" # pylint: disable=protected-access + from esphome.build_gen.espidf import idf_macro_matches + from esphome.core import CORE from esphome.espidf import toolchain description = json.loads( (build_path / "build" / "project_description.json").read_text(encoding="utf-8") ) name, version = _setup_core(build_path, description) + # Reconfiguring must not flip the tree's bootloader mode: the check + # validates the shape the build produced, not this process's flags. + skip_bootloader = toolchain.tree_skips_bootloader(build_path / "build") + CORE.skip_bootloader = skip_bootloader + # A prior tree's memoized decision must not leak into this one. + toolchain._cache().skip_bootloader = None + if not idf_macro_matches(toolchain._get_idf_path(version)): + return [MACRO_CHANGED] # ESP-IDF's openthread stamps the configure time into its compile flags; # pin it before the env is cached so both configures get the same value. os.environ["SOURCE_DATE_EPOCH"] = "0" @@ -122,11 +156,14 @@ def check(build_path: Path) -> list[str]: return [f"ESPHome's CMake configure failed with exit code {rc}"] if (rc := toolchain._run_ninja("all", verbose=False, jobs=None)) != 0: return [f"ESPHome's ninja build failed with exit code {rc}"] - before = _snapshot(build_path, name) - mtimes_before = _ninja_mtimes(build_path) + before = _snapshot(build_path, name, skip_bootloader) + logs = _ninja_logs(skip_bootloader) + mtimes_before = _ninja_mtimes(build_path, logs) # A moved or renamed output would otherwise compare as "unchanged". problems = [f"missing {f}" for f, digest in before.items() if digest is None] - if problems := problems + _log_problems(build_path, mtimes_before): + if skip_bootloader and (build_path / BOOTLOADER_BIN).is_file(): + problems.append(OVERRIDE_INEFFECTIVE) + if problems := problems + _log_problems(build_path, mtimes_before, logs): return problems for action in ("reconfigure", "build"): result = subprocess.run( @@ -139,10 +176,10 @@ def check(build_path: Path) -> list[str]: ) if result.returncode != 0: return [f"idf.py {action} failed:\n{result.stdout}{result.stderr}"] - after = _snapshot(build_path, name) - mtimes_after = _ninja_mtimes(build_path) + after = _snapshot(build_path, name, skip_bootloader) + mtimes_after = _ninja_mtimes(build_path, logs) problems = [f"idf.py changed {f}" for f in before if before[f] != after[f]] - problems += _log_problems(build_path, mtimes_after) + problems += _log_problems(build_path, mtimes_after, logs) for key in sorted(mtimes_before.keys() | mtimes_after.keys()): log, out = key if not out.endswith(WORK_SUFFIXES): diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index 6ded9a3a81..42dbdf4dc7 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "script")) import check_idf_py_equivalence as guard # noqa: E402 +from esphome.build_gen import espidf as build_gen # noqa: E402 from esphome.core import CORE # noqa: E402 from esphome.espidf import toolchain # noqa: E402 @@ -25,12 +26,26 @@ def _reset_core() -> Iterator[None]: CORE.reset() -def _make_tree(tmp_path: Path) -> Path: +BOOTLOADER_LOG = "build/bootloader/.ninja_log" +ALL_LOGS = (guard.TOP_NINJA_LOG, BOOTLOADER_LOG) + + +def _make_tree(tmp_path: Path, skip_bootloader: bool = False) -> Path: + """A fake build tree; stock shape by default, or the skip shape + (define set to 1, no bootloader bin, no sub-build).""" tree = tmp_path / "config" / ".esphome" / "build" / "dev" build = tree / "build" - for name in (*guard.watched("dev"), *guard.NINJA_LOGS): + files = [ + *guard.watched("dev", skip_bootloader), + *guard._ninja_logs(skip_bootloader), + ] + for name in files: (tree / name).parent.mkdir(parents=True, exist_ok=True) (tree / name).write_bytes(b"x") + define = "1" if skip_bootloader else "0" + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED={define}\n" + ) (build / "project_description.json").write_text( json.dumps( { @@ -44,9 +59,10 @@ def _make_tree(tmp_path: Path) -> Path: "# ninja log v7\n1\t2\t10\tesp-idf/a.obj\t0\n" "1\t2\t10\tbootloader/bootloader.bin\t0\n" ) - (build / "bootloader" / ".ninja_log").write_text( - "# ninja log v7\n1\t2\t10\tbootloader.elf\t0\n" - ) + if not skip_bootloader: + (build / "bootloader" / ".ninja_log").write_text( + "# ninja log v7\n1\t2\t10\tbootloader.elf\t0\n" + ) (tree / "sdkconfig.dev").write_text("") return tree @@ -56,6 +72,7 @@ def _run_check( side_effect: Callable[[list[str]], None] = lambda cmd: None, rc: int = 0, esphome_rcs: tuple[int, int] = (0, 0), + macro_matches: bool = True, envs: list[dict[str, str]] | None = None, ) -> tuple[list[str], list[list[str]]]: """Run check() with idf.py replaced by ``side_effect``; return problems, calls. @@ -80,6 +97,7 @@ def _run_check( patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), + patch.object(build_gen, "idf_macro_matches", return_value=macro_matches), patch.object(guard.subprocess, "run", side_effect=run), patch.dict(os.environ), ): @@ -168,9 +186,7 @@ def test_check_stops_when_idf_py_fails(tmp_path: Path) -> None: assert len(calls) == 1 -@pytest.mark.parametrize( - "remove", ["build/build.ninja", "build/dev.bin", *guard.NINJA_LOGS] -) +@pytest.mark.parametrize("remove", ["build/build.ninja", "build/dev.bin", *ALL_LOGS]) def test_check_fails_when_an_input_is_missing(tmp_path: Path, remove: str) -> None: """A moved or renamed output must not compare as unchanged.""" tree = _make_tree(tmp_path) @@ -197,7 +213,7 @@ def test_check_stops_when_the_esphome_baseline_fails( assert calls == [] -@pytest.mark.parametrize("log", guard.NINJA_LOGS) +@pytest.mark.parametrize("log", ALL_LOGS) def test_check_fails_when_a_ninja_log_has_no_entries(tmp_path: Path, log: str) -> None: """A log format change must not leave the rebuild check with nothing to compare.""" tree = _make_tree(tmp_path) @@ -223,6 +239,52 @@ def test_main_rejects_a_path_that_is_not_a_tree( mock_check.assert_not_called() +def test_check_resets_the_skip_memo_per_tree(tmp_path: Path) -> None: + """A second tree must not inherit the first tree's memoized mode.""" + tree = _make_tree(tmp_path) + CORE.skip_bootloader = True + toolchain._cache().skip_bootloader = True # leftover from a prior tree + problems, _ = _run_check(tree) + assert problems == [] + assert toolchain._skip_bootloader() is False + + +def test_check_accepts_a_skip_bootloader_tree(tmp_path: Path) -> None: + """No bootloader bin or sub-build is the skip shape, not missing input.""" + tree = _make_tree(tmp_path, skip_bootloader=True) + problems, calls = _run_check(tree) + assert problems == [] + assert len(calls) == 2 + # The baseline reconfigure must not flip the tree's mode. + assert CORE.skip_bootloader is True + + +def test_check_flags_an_ineffective_override(tmp_path: Path) -> None: + """A skip-mode tree that still built a bootloader must fail CI.""" + tree = _make_tree(tmp_path, skip_bootloader=True) + (tree / guard.BOOTLOADER_BIN).parent.mkdir(parents=True) + (tree / guard.BOOTLOADER_BIN).write_bytes(b"x") + problems, _ = _run_check(tree) + assert problems == [guard.OVERRIDE_INEFFECTIVE] + + +def test_check_requires_the_sub_log_on_a_stock_tree(tmp_path: Path) -> None: + """The mode comes from the define, so a vanished sub-build stays an error.""" + tree = _make_tree(tmp_path) + (tree / BOOTLOADER_LOG).unlink() + problems, calls = _run_check(tree) + assert problems == [f"missing {BOOTLOADER_LOG}"] + assert calls == [] + + +def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None: + """An IDF bump that rewrites the overridden macro must fail CI.""" + tree = _make_tree(tmp_path) + problems, calls = _run_check(tree, macro_matches=False) + assert problems == [guard.MACRO_CHANGED] + assert calls == [] + + def test_main_without_build_trees( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 7d53ee1e5e..7ed6201b6a 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -670,3 +670,75 @@ def test_pch_checksum_disabled_writes_nothing( _make_pch_project(tmp_path) write_pch_checksum() assert not CORE.relative_build_path(_PCH_SUM_PATH).exists() + + +# The macro body as shipped in tools/cmake/build.cmake; byte identical in +# IDF 5.5.5 and 6.1.0, so one fixture covers both supported versions. +IDF_BUILD_CMAKE = """\ +some_other_cmake() + +macro(__build_process_project_includes) + # Include the sdkconfig cmake file, since the following operations require + # knowledge of config values. + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + + # Make each build property available as a read-only variable + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(build_property ${build_properties}) + idf_build_get_property(val ${build_property}) + set(${build_property} "${val}") + endforeach() + + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + + # Include each component's project_include.cmake + foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_PATH ${dir}) # this is deprecated, users are encouraged to use COMPONENT_DIR; + # retained for compatibility + if(EXISTS ${COMPONENT_DIR}/project_include.cmake) + include(${COMPONENT_DIR}/project_include.cmake) + endif() + endforeach() +endmacro() +""" + + +def _write_idf_build_cmake(tmp_path: Path, text: str = IDF_BUILD_CMAKE) -> Path: + idf = tmp_path / "idf" + (idf / "tools" / "cmake").mkdir(parents=True) + (idf / "tools" / "cmake" / "build.cmake").write_text(text) + return idf + + +def test_normalized_macro_strips_comments_and_whitespace() -> None: + from esphome.build_gen.espidf import _normalized_macro + + text = "macro(__build_process_project_includes)\n a( b ) # tail\n\n # only\n c(d)\nendmacro()" + assert _normalized_macro(text) == ["a( b )", "c(d)"] + + +def test_normalized_macro_none_without_macro() -> None: + from esphome.build_gen.espidf import _normalized_macro + + assert _normalized_macro("nothing here") is None + + +def test_idf_macro_matches_the_shipped_body(tmp_path: Path) -> None: + """The override's embedded copy must equal what build.cmake ships.""" + from esphome.build_gen.espidf import idf_macro_matches + + assert idf_macro_matches(_write_idf_build_cmake(tmp_path)) is True + + +def test_idf_macro_mismatch_detected(tmp_path: Path) -> None: + from esphome.build_gen.espidf import idf_macro_matches + + changed = IDF_BUILD_CMAKE.replace( + "include(${sdkconfig_cmake})", "include(${sdkconfig_cmake} NEW_ARG)" + ) + assert idf_macro_matches(_write_idf_build_cmake(tmp_path, changed)) is False diff --git a/tests/unit_tests/test_download_types.py b/tests/unit_tests/test_download_types.py index 0d664b8db6..0f5aae444d 100644 --- a/tests/unit_tests/test_download_types.py +++ b/tests/unit_tests/test_download_types.py @@ -52,6 +52,35 @@ def test_recorded_firmware_path_yields_downloads(platform: str, tmp_path: Path) ) +def test_esp32_skip_bootloader_tree_offers_no_factory_entry(tmp_path: Path) -> None: + """A --skip-bootloader tree has no factory image; do not offer one. + + An unreadable tree (PlatformIO, capability probes passing a bare + path) reads as a full build and keeps the entry. + """ + from esphome.espidf.toolchain import SKIP_BOOTLOADER_DEFINE + + storage = _wizard_storage() + storage.firmware_bin_path = tmp_path / "firmware.bin" + + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.factory.bin", "firmware.ota.bin"] + + cache = tmp_path / "CMakeCache.txt" + cache.write_text(f"{SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n") + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.ota.bin"] + + # A capability probe passes the path as a plain string. + storage.firmware_bin_path = str(tmp_path / "firmware.bin") + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.ota.bin"] + + cache.write_text(f"{SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n") + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.factory.bin", "firmware.ota.bin"] + + def _nrf52_files(tmp_path: Path, *built: str) -> list[str]: """The files nrf52 offers for a build directory holding *built*.""" (tmp_path / "zephyr").mkdir() diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 1b40460b91..63d15a694c 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -8,11 +8,16 @@ import json import os from pathlib import Path import subprocess -from unittest.mock import call, patch +from unittest.mock import MagicMock, call, patch import pytest -from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT +from esphome.components.esp32.const import ( + KEY_ESP32, + KEY_FLASH_SIZE, + KEY_IDF_VERSION, + KEY_VARIANT, +) import esphome.config_validation as cv from esphome.const import ( CONF_COMPILE_PROCESS_LIMIT, @@ -893,6 +898,7 @@ def test_run_reconfigure_cmake_argv_matches_idf_py(setup_core: Path) -> None: "-DESP_PLATFORM=1", f"-DSDKCONFIG={sdkconfig}", "-DCCACHE_ENABLE=0", + "-DESPHOME_SKIP_BOOTLOADER=0", project, ] kwargs = mock_run.call_args.kwargs @@ -927,6 +933,7 @@ def test_run_reconfigure_cmake_argv_matches_idf6_py( "-DPYTHON=/tools/python", "-DESP_PLATFORM=1", f"-DCCACHE_ENABLE={expected}", + "-DESPHOME_SKIP_BOOTLOADER=0", str(build_dir.parent), ] @@ -938,7 +945,8 @@ def test_run_reconfigure_without_sdkconfig_or_filter(setup_core: Path) -> None: assert toolchain.run_reconfigure(verbose=True) == 0 cmd = mock_run.call_args.args[0] assert not any(arg.startswith("-DSDKCONFIG=") for arg in cmd) - assert cmd[-2] == "-DCCACHE_ENABLE=0" + assert cmd[-3] == "-DCCACHE_ENABLE=0" + assert cmd[-2] == "-DESPHOME_SKIP_BOOTLOADER=0" assert mock_run.call_args.kwargs["filter_lines"] is None @@ -1005,6 +1013,7 @@ _CONFIGURED = ( "PYTHON_DEPS_CHECKED:UNINITIALIZED=1\n" "PYTHON:UNINITIALIZED=/tools/python\n" "ESP_PLATFORM:UNINITIALIZED=1\n" + "ESPHOME_SKIP_BOOTLOADER:UNINITIALIZED=0\n" ) @@ -1256,3 +1265,206 @@ def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> N with patch.object(toolchain, "_get_cmake_output", return_value=output): assert toolchain.get_cmake_cache_value("ESPHOME_PCH") == "OFF" assert toolchain.get_cmake_cache_value("ESPHOME_MISSING") is None + + +def test_skip_bootloader_requires_flag_and_matching_macro( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The switch is explicit, and an IDF macro change wins over it.""" + from esphome.build_gen import espidf as build_gen + + assert toolchain._skip_bootloader() is False + CORE.skip_bootloader = True + toolchain._cache().skip_bootloader = None # decision is memoized per run + with ( + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(build_gen, "idf_macro_matches", return_value=True), + ): + assert toolchain._skip_bootloader() is True + toolchain._cache().skip_bootloader = None + with ( + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(build_gen, "idf_macro_matches", return_value=False) as mock_match, + caplog.at_level("WARNING"), + ): + assert toolchain._skip_bootloader() is False + assert toolchain._skip_bootloader() is False + mock_match.assert_called_once() # the memo also dedupes the warning + assert "--skip-bootloader ignored" in caplog.text + + +def test_configure_defines_follow_skip_bootloader(setup_core: Path) -> None: + with ( + patch.object(toolchain, "_get_idf_tool", return_value="/tools/python"), + patch.object(toolchain, "_cache_entries", return_value={}), + ): + with patch.object(toolchain, "_skip_bootloader", return_value=True): + assert ( + toolchain._configure_defines()[toolchain.SKIP_BOOTLOADER_DEFINE] == "1" + ) + with patch.object(toolchain, "_skip_bootloader", return_value=False): + assert ( + toolchain._configure_defines()[toolchain.SKIP_BOOTLOADER_DEFINE] == "0" + ) + + +def test_tree_skips_bootloader_reads_the_define(tmp_path: Path) -> None: + build = tmp_path / "build" + build.mkdir() + assert toolchain.tree_skips_bootloader(build) is False # not configured + cache = build / "CMakeCache.txt" + cache.write_text(f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n") + assert toolchain.tree_skips_bootloader(build) is True + cache.write_text(f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n") + assert toolchain.tree_skips_bootloader(build) is False + + +def test_tree_skips_bootloader_unreadable_cache_reads_full( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Unreadable trees fall back to the safe full build, with a trace.""" + build = tmp_path / "build" + (build / "CMakeCache.txt").mkdir(parents=True) # read raises OSError + with caplog.at_level("DEBUG"): + assert toolchain.tree_skips_bootloader(build) is False + assert "assuming a full build" in caplog.text + + +def test_create_factory_bin_merges_the_listed_flash_files( + setup_core: Path, +) -> None: + """Every listed file lands in the esptool merge argv, address sorted.""" + _setup_build(setup_core) + CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB" + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n" + ) + (build / "app.bin").write_bytes(b"app") + (build / "boot.bin").write_bytes(b"boot") + (build / "flasher_args.json").write_text( + '{"flash_files": {"0x10000": "app.bin", "0x0": "boot.bin"},' + ' "extra_esptool_args": {"chip": "esp32"}}' + ) + with ( + patch.object(toolchain, "_get_idf_env", return_value={}), + patch.object(toolchain, "_get_idf_tool", return_value="/py"), + patch.object( + toolchain.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert toolchain.create_factory_bin() is True + argv = mock_run.call_args.args[0] + boot = argv.index("0x0") + assert argv[boot + 1].endswith("boot.bin") + assert argv[boot + 2] == "0x10000" + + +def test_create_factory_bin_fails_on_a_missing_listed_flash_file( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A partial factory image would not boot; never write one.""" + _setup_build(setup_core) + CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB" + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n" + ) + (build / "flasher_args.json").write_text('{"flash_files": {"0x0": "missing.bin"}}') + stale = toolchain.get_factory_firmware_path() + stale.write_bytes(b"old") + with patch.object(toolchain.subprocess, "run") as mock_run: + assert toolchain.create_factory_bin() is False + mock_run.assert_not_called() + assert "Flash file not found" in caplog.text + # The image from an earlier build must not be served as this one. + assert not stale.exists() + + +def test_create_factory_bin_skip_mode_creates_nothing( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The flip into skip mode already cleaned up; the merge just no-ops.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + with ( + patch.object(toolchain.subprocess, "run") as mock_run, + caplog.at_level("INFO"), + ): + assert toolchain.create_factory_bin() is True + mock_run.assert_not_called() + assert "no factory image" in caplog.text + + +def test_run_reconfigure_flip_into_skip_mode_cleans_up(setup_core: Path) -> None: + """Full-mode leftovers are stale for OTA --bootloader and downloads, and + a partial cleanup would poison the flip back to full mode.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + (build / "bootloader").mkdir(parents=True) + (build / "bootloader" / "bootloader.bin").write_bytes(b"old") + # bootloader-prefix deliberately absent: cleanup skips what is not there. + stale = toolchain.get_factory_firmware_path() + stale.parent.mkdir(parents=True, exist_ok=True) + stale.write_bytes(b"old") + with ( + patch.object(toolchain, "_skip_bootloader", return_value=True), + patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "_tool_env", return_value={}), + patch.object(toolchain, "run_build_tool", return_value=0), + patch.object(toolchain, "_idf_py") as mock_idf_py, + ): + mock_idf_py.return_value.binary_dir_arg = False + assert toolchain.run_reconfigure() == 0 + assert not (build / "bootloader").exists() + assert not stale.exists() + + +def test_run_reconfigure_skip_steady_state_cleans_nothing(setup_core: Path) -> None: + """Cleanup belongs to the flip; a reconfigure of a skip tree touches nothing.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + marker = build / "bootloader" + marker.mkdir() + with ( + patch.object(toolchain, "_skip_bootloader", return_value=True), + patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "_tool_env", return_value={}), + patch.object(toolchain, "run_build_tool", return_value=0), + patch.object(toolchain, "_idf_py") as mock_idf_py, + ): + mock_idf_py.return_value.binary_dir_arg = False + assert toolchain.run_reconfigure() == 0 + assert marker.exists() + + +def test_create_factory_bin_full_mode_needs_flasher_args(setup_core: Path) -> None: + """Past the skip gate, a full tree without flasher_args fails as before.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n" + ) + assert toolchain.create_factory_bin() is False + + +def test_missing_image_hint_names_the_flag(setup_core: Path) -> None: + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + assert toolchain.missing_image_hint() is None # stock tree + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + assert "--skip-bootloader" in toolchain.missing_image_hint() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 633ac1b3e1..e46b61cfd5 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -1548,6 +1548,7 @@ class MockArgs: partition_table: bool = False bootloader: bool = False states: bool | None = None + device: list[str] | None = None def test_upload_program_serial_esp32( @@ -5357,6 +5358,28 @@ def _setup_build_info_test( return build_info_path, firmware_path +def test_compile_program_warns_and_ignores_skip_bootloader_elsewhere( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A full build is always safe; orchestrators cannot see YAML overrides.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device") + CORE.skip_bootloader = True + + config: dict[str, Any] = {CONF_ESPHOME: {CONF_NAME: "test_device"}} + with ( + patch( + "esphome.components.esp8266.check_rosetta", + side_effect=EsphomeError("stop here"), + ), + pytest.raises(EsphomeError, match="stop here"), + caplog.at_level("INFO"), + ): + compile_program(MockArgs(), config) + + assert "--skip-bootloader ignored" in caplog.text + assert CORE.skip_bootloader is False + + def test_compile_program_esp8266_runs_rosetta_check(tmp_path: Path) -> None: """Test that compile_program runs the Rosetta preflight for ESP8266 targets.""" setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device") @@ -6846,6 +6869,70 @@ def test_upload_using_esptool_arduino_toolchain( assert cmd_list[firmware_offset_idx + 1] == str(factory) +def test_upload_using_esptool_skip_bootloader_tree_names_the_flag( + tmp_path: Path, +) -> None: + """A serial flash needs the factory image the skip flag did not build.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test") + CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32"} + CORE.toolchain = Toolchain.ESP_IDF + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + from esphome.espidf import toolchain as espidf_toolchain + + (build / "CMakeCache.txt").write_text( + f"{espidf_toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + config = {CONF_ESPHOME: {"platformio_options": {}}} + with pytest.raises(EsphomeError, match="compiled with --skip-bootloader"): + upload_using_esptool(config, "/dev/ttyUSB0", None, None) + + +def test_command_run_rejects_serial_device_with_skip_bootloader( + tmp_path: Path, +) -> None: + """The compile could never be flashed over serial; fail before it runs.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test") + CORE.skip_bootloader = True + CORE.toolchain = Toolchain.ESP_IDF + args = MockArgs(device=["/dev/ttyUSB0"]) + with patch("esphome.__main__.write_cpp") as mock_write: + assert command_run(args, {}) == 1 + mock_write.assert_not_called() + + +def test_upload_program_ota_bootloader_skip_tree_names_the_flag( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """OTA --bootloader on a skip tree errors before picking the binary.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test") + CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32"} + CORE.toolchain = Toolchain.ESP_IDF + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + from esphome.espidf import toolchain as espidf_toolchain + + (build / "CMakeCache.txt").write_text( + f"{espidf_toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + mock_get_port_type.return_value = "NETWORK" + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + "allow_partition_access": True, + } + ] + } + args = MockArgs(bootloader=True) + with pytest.raises(EsphomeError, match="compiled with --skip-bootloader"): + upload_program(config, args, ["192.168.1.100"]) + mock_run_ota.assert_not_called() + + @pytest.mark.parametrize( ("toolchain", "pio_project_written"), [ @@ -7073,6 +7160,23 @@ def test_compile_program_espidf_idedata_success_is_silent( assert "idedata" not in caplog.text +def test_compile_program_espidf_failed_factory_bin_fails_the_build() -> None: + """A compile whose factory image could not be produced must not exit 0.""" + CORE.toolchain = Toolchain.ESP_IDF + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp32", + KEY_TARGET_FRAMEWORK: "esp-idf", + } + with ( + patch("esphome.espidf.toolchain.run_compile", return_value=0), + patch("esphome.espidf.toolchain.create_factory_bin", return_value=False), + patch("esphome.espidf.toolchain.create_ota_bin") as mock_ota, + patch("esphome.__main__._check_and_emit_build_info"), + ): + assert compile_program(MagicMock(), {}) == 1 + mock_ota.assert_not_called() + + def test_compile_program_espidf_idedata_none_warns( caplog: pytest.LogCaptureFixture, ) -> None: From bfb7e76b2e3bbc6db9374dd786e9aa92f82f13d6 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:02:58 +1000 Subject: [PATCH 033/221] [light] Add `color` option to strobe and other color state (#19874) Co-authored-by: Claude Sonnet 5 Co-authored-by: J. Nick Koston --- esphome/components/light/effects.py | 132 +++++++++++------- .../addressable_color_wipe_color_test.yaml | 22 +++ .../test_addressable_color_wipe_color.py | 40 ++++++ .../light/test_effect_color.py | 69 +++++++++ .../components/fastled_clockless/common.yaml | 2 + tests/components/light/common.yaml | 4 + 6 files changed, 222 insertions(+), 47 deletions(-) create mode 100644 tests/component_tests/light/config/addressable_color_wipe_color_test.yaml create mode 100644 tests/component_tests/light/test_addressable_color_wipe_color.py create mode 100644 tests/component_tests/light/test_effect_color.py diff --git a/esphome/components/light/effects.py b/esphome/components/light/effects.py index b12ec189b9..dc6266b2fd 100644 --- a/esphome/components/light/effects.py +++ b/esphome/components/light/effects.py @@ -31,9 +31,10 @@ from esphome.const import ( ) from esphome.cpp_generator import MockObjClass from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor +from esphome.types import ConfigType from esphome.util import Registry -from .automation import validate_light_state +from .automation import COLOR_SCHEMA, color_to_rgb, validate_light_state from .types import ( COLOR_MODES, AddressableColorWipeEffect, @@ -88,6 +89,28 @@ MONOCHROMATIC_EFFECTS = [] RGB_EFFECTS = [] ADDRESSABLE_EFFECTS = [] + +def _with_color_option(schema: cv.Schema, *default_keys: str) -> cv.Schema: + """Add the `color:` option to a per-item effect color schema. + + `color_to_rgb` must run before the keys are defaulted to full level below, so a + `color:` can populate `default_keys` (red/green/blue and, where the effect has + one, color_brightness) and a plain channel list still defaults them to full + level. Centralizing both steps here means a new effect schema only has to call + this once instead of repeating the ordering rule itself. + """ + schema = schema.extend(COLOR_SCHEMA) + schema.add_extra(color_to_rgb) + + def _default_to_full_level(config: ConfigType) -> ConfigType: + for key in default_keys: + config.setdefault(key, 1.0) + return config + + schema.add_extra(_default_to_full_level) + return schema + + EFFECTS_REGISTRY = Registry() @@ -267,6 +290,30 @@ async def random_effect_to_code(config, effect_id): return effect +STROBE_COLOR_SCHEMA = cv.Schema( + { + cv.Optional(CONF_STATE, default=True): validate_light_state, + cv.Optional(CONF_BRIGHTNESS, default=1.0): cv.percentage, + cv.Optional(CONF_COLOR_MODE): cv.enum(COLOR_MODES, upper=True, space="_"), + cv.Optional(CONF_COLOR_BRIGHTNESS): cv.percentage, + cv.Optional(CONF_RED): cv.percentage, + cv.Optional(CONF_GREEN): cv.percentage, + cv.Optional(CONF_BLUE): cv.percentage, + cv.Optional(CONF_WHITE, default=1.0): cv.percentage, + cv.Optional(CONF_COLOR_TEMPERATURE): cv.color_temperature, + cv.Optional(CONF_COLD_WHITE, default=1.0): cv.percentage, + cv.Optional(CONF_WARM_WHITE, default=1.0): cv.percentage, + cv.Required(CONF_DURATION): cv.positive_time_period_milliseconds, + cv.Optional( + CONF_TRANSITION_LENGTH, default="0s" + ): cv.positive_time_period_milliseconds, + } +) +STROBE_COLOR_SCHEMA = _with_color_option( + STROBE_COLOR_SCHEMA, CONF_COLOR_BRIGHTNESS, CONF_RED, CONF_GREEN, CONF_BLUE +) + + @register_binary_effect( "strobe", StrobeLightEffect, @@ -280,29 +327,7 @@ async def random_effect_to_code(config, effect_id): ], ): cv.All( cv.ensure_list( - cv.Schema( - { - cv.Optional(CONF_STATE, default=True): validate_light_state, - cv.Optional(CONF_BRIGHTNESS, default=1.0): cv.percentage, - cv.Optional(CONF_COLOR_MODE): cv.enum( - COLOR_MODES, upper=True, space="_" - ), - cv.Optional(CONF_COLOR_BRIGHTNESS, default=1.0): cv.percentage, - cv.Optional(CONF_RED, default=1.0): cv.percentage, - cv.Optional(CONF_GREEN, default=1.0): cv.percentage, - cv.Optional(CONF_BLUE, default=1.0): cv.percentage, - cv.Optional(CONF_WHITE, default=1.0): cv.percentage, - cv.Optional(CONF_COLOR_TEMPERATURE): cv.color_temperature, - cv.Optional(CONF_COLD_WHITE, default=1.0): cv.percentage, - cv.Optional(CONF_WARM_WHITE, default=1.0): cv.percentage, - cv.Required( - CONF_DURATION - ): cv.positive_time_period_milliseconds, - cv.Optional( - CONF_TRANSITION_LENGTH, default="0s" - ): cv.positive_time_period_milliseconds, - } - ), + STROBE_COLOR_SCHEMA, cv.has_at_least_one_key( CONF_STATE, CONF_BRIGHTNESS, @@ -406,6 +431,27 @@ async def addressable_rainbow_effect_to_code(config, effect_id): return var +ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA = cv.Schema( + { + cv.Optional(CONF_COLOR_BRIGHTNESS): cv.percentage, + cv.Optional(CONF_RED): cv.percentage, + cv.Optional(CONF_GREEN): cv.percentage, + cv.Optional(CONF_BLUE): cv.percentage, + cv.Optional(CONF_WHITE, default=1.0): cv.percentage, + cv.Optional(CONF_RANDOM, default=False): cv.boolean, + cv.Required(CONF_NUM_LEDS): cv.All(cv.uint32_t, cv.Range(min=1)), + cv.Optional(CONF_GRADIENT, default=False): cv.boolean, + } +) +ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA = _with_color_option( + ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA, + CONF_COLOR_BRIGHTNESS, + CONF_RED, + CONF_GREEN, + CONF_BLUE, +) + + @register_addressable_effect( "addressable_color_wipe", AddressableColorWipeEffect, @@ -413,17 +459,7 @@ async def addressable_rainbow_effect_to_code(config, effect_id): { cv.Optional( CONF_COLORS, default=[{CONF_NUM_LEDS: 1, CONF_RANDOM: True}] - ): cv.ensure_list( - { - cv.Optional(CONF_RED, default=1.0): cv.percentage, - cv.Optional(CONF_GREEN, default=1.0): cv.percentage, - cv.Optional(CONF_BLUE, default=1.0): cv.percentage, - cv.Optional(CONF_WHITE, default=1.0): cv.percentage, - cv.Optional(CONF_RANDOM, default=False): cv.boolean, - cv.Required(CONF_NUM_LEDS): cv.All(cv.uint32_t, cv.Range(min=1)), - cv.Optional(CONF_GRADIENT, default=False): cv.boolean, - } - ), + ): cv.ensure_list(ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA), cv.Optional( CONF_ADD_LED_INTERVAL, default="0.1s" ): cv.positive_time_period_milliseconds, @@ -434,19 +470,21 @@ async def addressable_color_wipe_effect_to_code(config, effect_id): var = cg.new_Pvariable(effect_id, config[CONF_NAME]) cg.add(var.set_add_led_interval(config[CONF_ADD_LED_INTERVAL])) cg.add(var.set_reverse(config[CONF_REVERSE])) - colors = [ - cg.StructInitializer( - AddressableColorWipeEffectColor, - ("r", int(round(color[CONF_RED] * 255))), - ("g", int(round(color[CONF_GREEN] * 255))), - ("b", int(round(color[CONF_BLUE] * 255))), - ("w", int(round(color[CONF_WHITE] * 255))), - ("random", color[CONF_RANDOM]), - ("num_leds", color[CONF_NUM_LEDS]), - ("gradient", color[CONF_GRADIENT]), + colors = [] + for color in config.get(CONF_COLORS, []): + color_brightness = color[CONF_COLOR_BRIGHTNESS] + colors.append( + cg.StructInitializer( + AddressableColorWipeEffectColor, + ("r", int(round(color[CONF_RED] * color_brightness * 255))), + ("g", int(round(color[CONF_GREEN] * color_brightness * 255))), + ("b", int(round(color[CONF_BLUE] * color_brightness * 255))), + ("w", int(round(color[CONF_WHITE] * 255))), + ("random", color[CONF_RANDOM]), + ("num_leds", color[CONF_NUM_LEDS]), + ("gradient", color[CONF_GRADIENT]), + ) ) - for color in config.get(CONF_COLORS, []) - ] cg.add(var.set_colors(colors)) return var diff --git a/tests/component_tests/light/config/addressable_color_wipe_color_test.yaml b/tests/component_tests/light/config/addressable_color_wipe_color_test.yaml new file mode 100644 index 0000000000..fc1e80688f --- /dev/null +++ b/tests/component_tests/light/config/addressable_color_wipe_color_test.yaml @@ -0,0 +1,22 @@ +esphome: + name: test-color-wipe-color + +esp32: + board: esp32dev + framework: + type: arduino + +light: + - platform: fastled_clockless + id: wipe_light + chipset: WS2811 + pin: GPIO13 + num_leds: 10 + effects: + - addressable_color_wipe: + name: Wipe + colors: + - color: darkred + num_leds: 1 + - red: 50% + num_leds: 1 diff --git a/tests/component_tests/light/test_addressable_color_wipe_color.py b/tests/component_tests/light/test_addressable_color_wipe_color.py new file mode 100644 index 0000000000..f1dd44a32c --- /dev/null +++ b/tests/component_tests/light/test_addressable_color_wipe_color.py @@ -0,0 +1,40 @@ +"""Regression test: addressable_color_wipe's `color:` must scale by color_brightness. + +The per-item schema tests in test_effect_color.py only check the validated config +dict; they would still pass if addressable_color_wipe_effect_to_code dropped +color_brightness on the floor (as it originally did), since a dark color's red/green/ +blue are already peak-normalized to 1.0 by the time codegen sees them. Generate the +actual C++ and check the AddressableColorWipeEffectColor byte values instead, so a +regression in the scaling multiplication itself is caught. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + + +@pytest.fixture +def main_cpp( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> str: + return generate_main( + component_config_path("addressable_color_wipe_color_test.yaml") + ) + + +def test_dark_color_name_is_scaled_by_color_brightness(main_cpp: str) -> None: + # color: darkred (0x8B0000) normalizes to r=1.0 but color_brightness=0x8B/0xFF, + # so the byte value must come back down to 0x8B (139), not 255. + assert ".r = 139," in main_cpp + assert ".g = 0," in main_cpp + assert ".b = 0," in main_cpp + + +def test_explicit_channel_is_unaffected(main_cpp: str) -> None: + # red: 50% with no color: or color_brightness: set must still scale by the + # default color_brightness of 1.0, i.e. come out as plain 50% of 255. + assert ".r = 128," in main_cpp diff --git a/tests/component_tests/light/test_effect_color.py b/tests/component_tests/light/test_effect_color.py new file mode 100644 index 0000000000..52e4e440dd --- /dev/null +++ b/tests/component_tests/light/test_effect_color.py @@ -0,0 +1,69 @@ +"""Tests for the `color` option in per-item effect color lists.""" + +from __future__ import annotations + +import pytest + +from esphome import config_validation as cv +from esphome.components.light.effects import ( + ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA, + STROBE_COLOR_SCHEMA, +) +from esphome.const import CONF_BLUE, CONF_COLOR_BRIGHTNESS, CONF_GREEN, CONF_RED + + +def test_strobe_plain_defaults_full_level() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s"}) + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 1.0, 1.0) + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_strobe_color_name() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s", "color": "darkred"}) + assert "color" not in result + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 0.0, 0.0) + assert result[CONF_COLOR_BRIGHTNESS] == pytest.approx(0x8B / 0xFF) + + +def test_strobe_hex_color() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s", "color": "0x223344"}) + assert result[CONF_BLUE] == 1.0 + assert result[CONF_COLOR_BRIGHTNESS] == pytest.approx(0x44 / 0xFF) + + +def test_strobe_explicit_channels_unaffected() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s", "red": "50%"}) + assert result[CONF_RED] == 0.5 + assert (result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 1.0) + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_strobe_color_conflicts_with_rgb() -> None: + with pytest.raises(cv.Invalid, match="cannot be used with"): + STROBE_COLOR_SCHEMA({"duration": "1s", "color": "red", "red": "10%"}) + + +def test_addressable_color_wipe_plain_defaults_full_level() -> None: + result = ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA({"num_leds": 1}) + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 1.0, 1.0) + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_addressable_color_wipe_color_name() -> None: + result = ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA({"num_leds": 1, "color": "darkred"}) + assert "color" not in result + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 0.0, 0.0) + assert result[CONF_COLOR_BRIGHTNESS] == pytest.approx(0x8B / 0xFF) + + +def test_addressable_color_wipe_explicit_channels_unaffected() -> None: + result = ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA({"num_leds": 1, "red": "50%"}) + assert result[CONF_RED] == 0.5 + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_addressable_color_wipe_color_conflicts_with_rgb() -> None: + with pytest.raises(cv.Invalid, match="cannot be used with"): + ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA( + {"num_leds": 1, "color": "blue", "blue": "10%"} + ) diff --git a/tests/components/fastled_clockless/common.yaml b/tests/components/fastled_clockless/common.yaml index a7ce7ed280..98bc65f4dd 100644 --- a/tests/components/fastled_clockless/common.yaml +++ b/tests/components/fastled_clockless/common.yaml @@ -21,6 +21,8 @@ light: green: 0% blue: 0% num_leds: 1 + - color: darkred + num_leds: 1 add_led_interval: 100ms reverse: false - addressable_scan: diff --git a/tests/components/light/common.yaml b/tests/components/light/common.yaml index 046acef9ce..3cf2c59223 100644 --- a/tests/components/light/common.yaml +++ b/tests/components/light/common.yaml @@ -429,6 +429,10 @@ light: blue: 0% duration: 1s transition_length: 500ms + - state: true + color: darkred + duration: 1s + transition_length: 500ms - platform: partition id: test_addressable_transition name: Addressable Transition Test From 7962b52d2a9aa93921f3cda6082472b272b2b594 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 14:05:15 -0500 Subject: [PATCH 034/221] [ci] Skip the bootloader in the component batch build jobs (#19931) --- .github/workflows/ci.yml | 9 ++- script/test_build_components.py | 27 ++++++++ tests/script/test_test_build_components.py | 81 ++++++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9507c5938..a7ed035063 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1225,7 +1225,14 @@ jobs: if [ -n "$compile_csv" ]; then # Run compilation with grouping and isolation - python3 script/test_build_components.py -e compile -c "$compile_csv" -f --isolate "$directly_changed_csv" + # The bootloader has no ESPHome code and these builds never + # flash; the check_idf_py batch keeps the full build so the + # native sub-build and the equivalence check stay covered. + skip_flag="--skip-bootloader" + if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then + skip_flag="" + fi + python3 script/test_build_components.py -e compile -c "$compile_csv" -f --isolate "$directly_changed_csv" $skip_flag if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then # The real idf.py must find nothing to configure or build in a # tree built above; catches drift on ESP-IDF bumps. diff --git a/script/test_build_components.py b/script/test_build_components.py index d3dfd36076..c955391a61 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -333,6 +333,12 @@ def extract_platform_with_version(base_file: Path) -> str: return base_file.stem.replace("build_components_base.", "") +def _wants_skip_bootloader(skip: bool, command: str, platform: str) -> bool: + """Every esp32 target: idf variants, and esp32-ard, whose Arduino + core builds as an ESP-IDF component under the native toolchain.""" + return skip and command == "compile" and platform.startswith("esp32") + + def run_esphome_test( component: str, test_file: Path, @@ -344,6 +350,7 @@ def run_esphome_test( continue_on_fail: bool, use_testing_mode: bool = False, toolchain: str | None = None, + skip_bootloader: bool = False, ) -> TestResult: """Run esphome test for a single component. @@ -408,6 +415,8 @@ def run_esphome_test( # Add command cmd.append(esphome_command) + if _wants_skip_bootloader(skip_bootloader, esphome_command, platform): + cmd.append("--skip-bootloader") # Add config file cmd.append(str(output_file)) @@ -469,6 +478,7 @@ def run_grouped_test( esphome_command: str, continue_on_fail: bool, toolchain: str | None = None, + skip_bootloader: bool = False, ) -> TestResult: """Run esphome test for a group of components with shared bus configs. @@ -554,6 +564,8 @@ def run_grouped_test( # Add command cmd.append(esphome_command) + if _wants_skip_bootloader(skip_bootloader, esphome_command, platform): + cmd.append("--skip-bootloader") cmd.append(str(output_file)) @@ -614,6 +626,7 @@ def run_grouped_component_tests( continue_on_fail: bool, additional_isolated: set[str] | None = None, toolchain: str | None = None, + skip_bootloader: bool = False, ) -> tuple[set[tuple[str, str]], list[TestResult]]: """Run grouped component tests. @@ -956,6 +969,7 @@ def run_grouped_component_tests( esphome_command=esphome_command, continue_on_fail=continue_on_fail, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) # Mark all components as tested @@ -980,6 +994,7 @@ def run_individual_component_test( tested_components: set[tuple[str, str]], test_results: list[TestResult], toolchain: str | None = None, + skip_bootloader: bool = False, ) -> None: """Run an individual component test if not already tested in a group. @@ -1014,6 +1029,7 @@ def run_individual_component_test( esphome_command=esphome_command, continue_on_fail=continue_on_fail, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) test_results.append(test_result) @@ -1027,6 +1043,7 @@ def test_components( isolated_components: set[str] | None = None, base_only: bool = False, toolchain: str | None = None, + skip_bootloader: bool = False, fail_on_no_tests: bool = False, ) -> int: """Test components with optional intelligent grouping. @@ -1127,6 +1144,7 @@ def test_components( continue_on_fail=continue_on_fail, additional_isolated=isolated_components, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) test_results.extend(grouped_results) @@ -1156,6 +1174,7 @@ def test_components( tested_components=tested_components, test_results=test_results, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) else: # Platform-specific test @@ -1189,6 +1208,7 @@ def test_components( tested_components=tested_components, test_results=test_results, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) silent: list[str] = [] @@ -1294,6 +1314,12 @@ def main() -> int: "--toolchain", help="Select toolchain for compiling.", ) + parser.add_argument( + "--skip-bootloader", + action="store_true", + help="Pass --skip-bootloader to esphome compile; component builds " + "never flash, and the bootloader is covered by the toolchain jobs", + ) parser.add_argument( "--fail-on-no-tests", action="store_true", @@ -1321,6 +1347,7 @@ def main() -> int: fail_on_no_tests=args.fail_on_no_tests, base_only=args.base_only, toolchain=args.toolchain, + skip_bootloader=args.skip_bootloader, ) diff --git a/tests/script/test_test_build_components.py b/tests/script/test_test_build_components.py index 1d21e5d943..cf15472888 100644 --- a/tests/script/test_test_build_components.py +++ b/tests/script/test_test_build_components.py @@ -94,6 +94,87 @@ def test_run_esphome_test_wraps_output_in_group( assert out.index("::group::") < out.index("> [foo]") < out.index("::endgroup::") +@pytest.mark.parametrize( + ("platform", "command", "skip", "expects_flag"), + [ + ("esp32-idf", "compile", True, True), + ("esp32-s3-idf", "compile", True, True), + ("esp32-ard", "compile", True, True), + ("esp32-idf", "compile", False, False), + ("esp32-idf", "config", True, False), + ("esp8266-ard", "compile", True, False), + ], + ids=[ + "idf-skip", + "idf-variant", + "arduino-as-idf-component", + "idf-off", + "config", + "esp8266", + ], +) +def test_run_esphome_test_skip_bootloader_argv( + _ci: None, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + platform: str, + command: str, + skip: bool, + expects_flag: bool, +) -> None: + """The flag lands on esp32 family compiles only.""" + seen: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + seen.append(cmd) + return _FakeCompleted(0) + + monkeypatch.setattr(tbc.subprocess, "run", fake_run) + repo_root = Path(tbc.__file__).parent.parent + test_file = repo_root / "tests" / "components" / "foo" / f"test.{platform}.yaml" + tbc.run_esphome_test( + component="foo", + test_file=test_file, + platform=platform, + platform_with_version=platform, + base_file=_make_base_file(tmp_path), + build_dir=tmp_path, + esphome_command=command, + continue_on_fail=True, + skip_bootloader=skip, + ) + assert ("--skip-bootloader" in seen[0]) is expects_flag + + +def test_run_grouped_test_skip_bootloader_argv( + _ci: None, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The grouped command site adds the flag the same way.""" + seen: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + seen.append(cmd) + return _FakeCompleted(0) + + monkeypatch.setattr(tbc.subprocess, "run", fake_run) + repo_root = Path(tbc.__file__).parent.parent + tests_dir = repo_root / "tests" / "components" + tbc.run_grouped_test( + components=["gpio"], + platform="esp32-idf", + platform_with_version="esp32-idf", + base_file=_make_base_file(tmp_path), + build_dir=tmp_path, + tests_dir=tests_dir, + esphome_command="compile", + continue_on_fail=True, + skip_bootloader=True, + ) + assert "--skip-bootloader" in seen[0] + + def test_run_esphome_test_closes_group_before_failure_report( _ci: None, monkeypatch: pytest.MonkeyPatch, From f7b4ea9f0e138f252d24c924d950e9a3623184dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:31:37 -0500 Subject: [PATCH 035/221] Bump filelock from 4.0.4 to 4.0.5 (#19991) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a0b7b31f4..940871224e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.12.0 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver -filelock==4.0.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==4.0.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.3 From 96c61208497b2964c1573c617fd1de3666f9b367 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:32:05 -0500 Subject: [PATCH 036/221] Bump zeroconf from 0.151.3 to 0.151.5 (#19992) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 940871224e..f05f092904 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.4.0 click==8.5.0 aioesphomeapi==46.6.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.151.3 +zeroconf==0.151.5 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From 8ac184228dde87642b61330036073d93adee3900 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:32:17 -0500 Subject: [PATCH 037/221] Bump platformdirs from 4.12.0 to 4.12.1 (#19994) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f05f092904..32931b650e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.12.0 # native esp-idf toolchain global cache dir +platformdirs==4.12.1 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver filelock==4.0.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From d49741dc10345014c68c3dc9c4c3c7a93f68560b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:48:16 -0500 Subject: [PATCH 038/221] Bump prek from 0.5.3 to 0.5.4 (#19993) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 3ec864a340..a66e05ba74 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.9 flake8==7.4.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.9 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -prek==0.5.3 # .github/workflows/ci.yml reads this pin +prek==0.5.4 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests From 2d37c6f58b78df01dd348e48a21ae5700af5add1 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:51:43 -0500 Subject: [PATCH 039/221] Bump bundled esphome-device-builder to 1.19.0 (#19995) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e68ea3ee72..8d3a5d2cc5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.18.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.19.0 RUN \ platformio settings set enable_telemetry No \ From 4075c15bea92e1b2fc9d8a4af639f01e567e8235 Mon Sep 17 00:00:00 2001 From: Jeff Brown Date: Thu, 1 Oct 2026 13:17:48 -0700 Subject: [PATCH 040/221] [stcc4] Add component for the Sensirion STCC4 CO2 sensor (#19434) Co-authored-by: will Co-authored-by: J. Nick Koston --- CODEOWNERS | 1 + .../sensirion_common/i2c_sensirion.cpp | 5 +- .../sensirion_common/i2c_sensirion.h | 7 +- esphome/components/stcc4/__init__.py | 0 esphome/components/stcc4/sensor.py | 141 +++++++++ esphome/components/stcc4/stcc4.cpp | 294 ++++++++++++++++++ esphome/components/stcc4/stcc4.h | 55 ++++ tests/components/stcc4/common.yaml | 36 +++ tests/components/stcc4/test.esp32-idf.yaml | 4 + tests/components/stcc4/test.esp8266-ard.yaml | 4 + tests/components/stcc4/test.rp2040-ard.yaml | 4 + 11 files changed, 548 insertions(+), 3 deletions(-) create mode 100644 esphome/components/stcc4/__init__.py create mode 100644 esphome/components/stcc4/sensor.py create mode 100644 esphome/components/stcc4/stcc4.cpp create mode 100644 esphome/components/stcc4/stcc4.h create mode 100644 tests/components/stcc4/common.yaml create mode 100644 tests/components/stcc4/test.esp32-idf.yaml create mode 100644 tests/components/stcc4/test.esp8266-ard.yaml create mode 100644 tests/components/stcc4/test.rp2040-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index e82158f390..12c0a54ac1 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -541,6 +541,7 @@ esphome/components/st7735/* @SenexCrenshaw esphome/components/st7789v/* @kbx81 esphome/components/st7920/* @marsjan155 esphome/components/statsd/* @Links2004 +esphome/components/stcc4/* @j9brown esphome/components/stts22h/* @B48D81EFCC esphome/components/substitutions/* @esphome/core esphome/components/sun/* @OttoWinter diff --git a/esphome/components/sensirion_common/i2c_sensirion.cpp b/esphome/components/sensirion_common/i2c_sensirion.cpp index f6ff4711d4..3325c27772 100644 --- a/esphome/components/sensirion_common/i2c_sensirion.cpp +++ b/esphome/components/sensirion_common/i2c_sensirion.cpp @@ -65,14 +65,15 @@ bool SensirionI2CDevice::write_command_(uint16_t command, CommandLen command_len } bool SensirionI2CDevice::get_register_(uint16_t reg, CommandLen command_len, uint16_t *data, const uint8_t len, - const uint8_t delay_ms) { + const uint8_t delay_ms, uint8_t sensirion_options) { if (!this->write_command_(reg, command_len, nullptr, 0)) { ESP_LOGE(TAG, "Write failed: reg=0x%X (%d) err=%d,", reg, command_len, this->last_error_); return false; } delay(delay_ms); bool result = this->read_data(data, len); - if (!result) { + if (!result && + (this->last_error_ != i2c::ERROR_NOT_ACKNOWLEDGED || !(sensirion_options & SENSIRION_OPTION_READ_MAY_NACK))) { ESP_LOGE(TAG, "Read failed: reg=0x%X err=%d,", reg, this->last_error_); } return result; diff --git a/esphome/components/sensirion_common/i2c_sensirion.h b/esphome/components/sensirion_common/i2c_sensirion.h index 558fbdbb12..1b1cd2ba15 100644 --- a/esphome/components/sensirion_common/i2c_sensirion.h +++ b/esphome/components/sensirion_common/i2c_sensirion.h @@ -16,6 +16,9 @@ namespace esphome::sensirion_common { */ static const uint8_t CRC_POLYNOMIAL = 0x31; // default for Sensirion +/// When reading a register and the device reports NACK because the value is not ready yet, don't log it as an error. +static constexpr uint8_t SENSIRION_OPTION_READ_MAY_NACK = 1u << 0; + class SensirionI2CDevice : public i2c::I2CDevice { public: enum CommandLen : uint8_t { ADDR_8_BIT = 1, ADDR_16_BIT = 2 }; @@ -139,9 +142,11 @@ class SensirionI2CDevice : public i2c::I2CDevice { * @param data pointer to raw result * @param len number of words to read * @param delay milliseconds to to wait between sending the I2C command and reading the result + * @param sensirion_options options for the request * @return true if reading succeeded */ - bool get_register_(uint16_t reg, CommandLen command_len, uint16_t *data, uint8_t len, uint8_t delay); + bool get_register_(uint16_t reg, CommandLen command_len, uint16_t *data, uint8_t len, uint8_t delay, + uint8_t sensirion_options = 0); /** last error code from I2C operation */ diff --git a/esphome/components/stcc4/__init__.py b/esphome/components/stcc4/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/stcc4/sensor.py b/esphome/components/stcc4/sensor.py new file mode 100644 index 0000000000..212fe6bb48 --- /dev/null +++ b/esphome/components/stcc4/sensor.py @@ -0,0 +1,141 @@ +import esphome.codegen as cg +from esphome.components import i2c, sensirion_common, sensor +from esphome.components.const import CONF_HUMIDITY_SOURCE +import esphome.config_validation as cv +from esphome.const import ( + CONF_AMBIENT_PRESSURE_COMPENSATION, + CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE, + CONF_CO2, + CONF_HUMIDITY, + CONF_ID, + CONF_MEASUREMENT_MODE, + CONF_TEMPERATURE, + CONF_TEMPERATURE_SOURCE, + CONF_UPDATE_INTERVAL, + DEVICE_CLASS_CARBON_DIOXIDE, + DEVICE_CLASS_HUMIDITY, + DEVICE_CLASS_TEMPERATURE, + ICON_MOLECULE_CO2, + ICON_THERMOMETER, + ICON_WATER_PERCENT, + STATE_CLASS_MEASUREMENT, + UNIT_CELSIUS, + UNIT_PARTS_PER_MILLION, + UNIT_PERCENT, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@j9brown"] +DEPENDENCIES = ["i2c"] +AUTO_LOAD = ["sensirion_common"] + +stcc4_ns = cg.esphome_ns.namespace("stcc4") +STCC4Component = stcc4_ns.class_( + "STCC4Component", cg.PollingComponent, sensirion_common.SensirionI2CDevice +) + +MeasurementMode = stcc4_ns.enum("MeasurementMode", is_class=True) + +MEASUREMENT_MODE_OPTIONS = { + "continuous": MeasurementMode.CONTINUOUS, + "single_shot": MeasurementMode.SINGLE_SHOT, +} + + +def validate_config(config: ConfigType) -> ConfigType: + if config[CONF_MEASUREMENT_MODE] == "continuous": + if CONF_UPDATE_INTERVAL in config: + raise cv.Invalid( + "update_interval must not be specified in continuous measurement mode" + ) + elif CONF_UPDATE_INTERVAL not in config: + config[CONF_UPDATE_INTERVAL] = cv.update_interval("60s") + return config + + +CONFIG_SCHEMA = ( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(STCC4Component), + cv.Optional(CONF_CO2): sensor.sensor_schema( + unit_of_measurement=UNIT_PARTS_PER_MILLION, + icon=ICON_MOLECULE_CO2, + accuracy_decimals=0, + device_class=DEVICE_CLASS_CARBON_DIOXIDE, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_TEMPERATURE): sensor.sensor_schema( + unit_of_measurement=UNIT_CELSIUS, + icon=ICON_THERMOMETER, + accuracy_decimals=2, + device_class=DEVICE_CLASS_TEMPERATURE, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_HUMIDITY): sensor.sensor_schema( + unit_of_measurement=UNIT_PERCENT, + icon=ICON_WATER_PERCENT, + accuracy_decimals=2, + device_class=DEVICE_CLASS_HUMIDITY, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Inclusive(CONF_TEMPERATURE_SOURCE, "rht_compensation"): cv.use_id( + sensor.Sensor + ), + cv.Inclusive(CONF_HUMIDITY_SOURCE, "rht_compensation"): cv.use_id( + sensor.Sensor + ), + cv.Exclusive( + CONF_AMBIENT_PRESSURE_COMPENSATION, "ambient_pressure_compensation" + ): cv.All(cv.pressure, cv.float_range(min=0.4, max=1.1)), + cv.Exclusive( + CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE, + "ambient_pressure_compensation", + ): cv.use_id(sensor.Sensor), + cv.Optional(CONF_MEASUREMENT_MODE, default="continuous"): cv.enum( + MEASUREMENT_MODE_OPTIONS, lower=True + ), + cv.Optional(CONF_UPDATE_INTERVAL): cv.update_interval, + } + ) + .extend(cv.COMPONENT_SCHEMA) + .extend(i2c.i2c_device_schema(0x64)) + .add_extra(validate_config) +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await i2c.register_i2c_device(var, config) + + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) + + if (temperature_source := config.get(CONF_TEMPERATURE_SOURCE)) is not None: + sens = await cg.get_variable(temperature_source) + cg.add(var.set_temperature_source(sens)) + + if (humidity_source := config.get(CONF_HUMIDITY_SOURCE)) is not None: + sens = await cg.get_variable(humidity_source) + cg.add(var.set_humidity_source(sens)) + + if ( + ambient_pressure_compensation := config.get(CONF_AMBIENT_PRESSURE_COMPENSATION) + ) is not None: + cg.add( + var.set_ambient_pressure_compensation( + ambient_pressure_compensation * 1000 # convert bar to hPa + ) + ) + + if ( + ambient_pressure_compensation_source := config.get( + CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE + ) + ) is not None: + sens = await cg.get_variable(ambient_pressure_compensation_source) + cg.add(var.set_ambient_pressure_source(sens)) + + cg.add(var.set_measurement_mode(config[CONF_MEASUREMENT_MODE])) diff --git a/esphome/components/stcc4/stcc4.cpp b/esphome/components/stcc4/stcc4.cpp new file mode 100644 index 0000000000..efc7a2b8ee --- /dev/null +++ b/esphome/components/stcc4/stcc4.cpp @@ -0,0 +1,294 @@ +#include + +#include "stcc4.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +namespace esphome::stcc4 { + +static const char *const TAG = "stcc4"; + +// I2C Commands +static constexpr uint16_t STCC4_CMD_START_CONTINUOUS_MEASUREMENT = 0x218b; +static constexpr uint16_t STCC4_CMD_STOP_CONTINUOUS_MEASUREMENT = 0x3f86; +static constexpr uint16_t STCC4_CMD_MEASURE_SINGLE_SHOT = 0x219d; +static constexpr uint16_t STCC4_CMD_READ_MEASUREMENT = 0xec05; +static constexpr uint16_t STCC4_CMD_GET_PRODUCT_ID = 0x365b; +static constexpr uint16_t STCC4_CMD_SET_RHT_COMPENSATION = 0xe000; +static constexpr uint16_t STCC4_CMD_SET_PRESSURE_COMPENSATION = 0xe016; + +// Exit sleep is an 8-bit command (single byte 0x00), not 16-bit +static constexpr uint8_t STCC4_CMD_EXIT_SLEEP_MODE = 0x00; + +static constexpr uint32_t STCC4_PRODUCT_ID = 0x0901018a; + +// Timeout for determining when the device is ready for use, in milliseconds. +// While waiting for the previous measurement to finish, the device will NACK all I2C requests +// and it can take up to 1200 ms for the operation to complete according to the datasheet. +static constexpr uint32_t READY_TIMEOUT_MS = 1200; + +// Poll interval for determining when the device is ready for use, in milliseconds. +static constexpr uint32_t READY_POLL_INTERVAL_MS = 100; + +// Convert units the the device's representation according to the datasheet. +constexpr uint16_t temperature_in_c_to_ticks(float temperature_in_c) { + return uint16_t((std::clamp(temperature_in_c, -45.f, 130.f) + 45.f) * 65535.f / 175.f); +} + +constexpr float temperature_in_ticks_to_c(uint16_t temperature_in_ticks) { + return temperature_in_ticks * 175.f / 65535.f - 45.f; +} + +constexpr uint16_t humidity_in_percent_to_ticks(float humidity_in_percent) { + return uint16_t((std::clamp(humidity_in_percent, 0.f, 100.f) + 6.f) * 65535.f / 125.f); +} + +constexpr float humidity_in_ticks_to_percent(uint16_t humidity_in_ticks) { + return humidity_in_ticks * 125.f / 65535.f - 6.f; +} + +constexpr uint16_t pressure_in_hpa_to_pa_2(float pressure_in_hpa) { + return uint16_t(std::clamp(pressure_in_hpa, 400.f, 1100.f) * 50.f); +} + +constexpr float pressure_in_pa_2_to_hpa(uint16_t pressure_in_pa_2) { return pressure_in_pa_2 / 50.f; } + +void STCC4Component::setup() { + this->stop_poller(); // not ready yet + + // Wait 100 ms after power up before attempting to communicate with the sensor + this->set_timeout(100, [this]() { + // Send exit sleep mode command (8-bit, NACK expected), wait 5 ms to exit sleep + this->write_command(STCC4_CMD_EXIT_SLEEP_MODE); + this->set_timeout(5, [this]() { this->poll_until_ready_for_setup_or_timeout_(millis()); }); + }); +} + +void STCC4Component::poll_until_ready_for_setup_or_timeout_(uint32_t start_time) { + // Stop continuous measurements in case they were previously running + // The device may NACK this request if it is not ready to communicate yet + if (this->write_command(STCC4_CMD_STOP_CONTINUOUS_MEASUREMENT)) { + // Read product ID to verify communication (6 words: 2 for product_id + 4 for serial) + // The device may NACK this request if it is not ready to communicate yet + uint16_t raw_product_id[6]; + if (this->get_register(STCC4_CMD_GET_PRODUCT_ID, raw_product_id, 6, 1)) { + uint32_t product_id = (uint32_t(raw_product_id[0]) << 16) | raw_product_id[1]; + uint64_t serial_number = (uint64_t(raw_product_id[2]) << 48) | (uint64_t(raw_product_id[3]) << 32) | + (uint64_t(raw_product_id[4]) << 16) | raw_product_id[5]; + ESP_LOGD(TAG, "Product ID: 0x%08" PRIX32 ", Serial: 0x%016" PRIX64, product_id, serial_number); + if (product_id != STCC4_PRODUCT_ID) { + ESP_LOGE(TAG, "Unsupported product ID"); + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); + return; + } + + // Set static ambient pressure compensation if configured + if (this->ambient_pressure_in_pa_2_ != 0) { + if (!this->write_ambient_pressure_compensation_(this->ambient_pressure_in_pa_2_)) { + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); + return; + } + } + + // Apply the dynamic compensation sources' current values, if configured + if (this->temperature_source_ != nullptr && this->humidity_source_ != nullptr) { + this->update_rht_compensation_from_source_(); + } + if (this->ambient_pressure_source_ != nullptr) { + this->update_ambient_pressure_compensation_from_source_(); + } + + if (this->measurement_mode_ == MeasurementMode::SINGLE_SHOT) { + this->start_poller(); + this->finish_setup_(); + return; + } + + // Start continuous measurement + if (!this->write_command(STCC4_CMD_START_CONTINUOUS_MEASUREMENT)) { + ESP_LOGE(TAG, "Failed to start continuous measurement"); + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); + return; + } + this->schedule_continuous_update_(false); + this->finish_setup_(); + return; + } + } + + if (millis() - start_time < READY_TIMEOUT_MS) { + ESP_LOGVV(TAG, "Retry sync"); + this->set_timeout(READY_POLL_INTERVAL_MS, + [this, start_time]() { this->poll_until_ready_for_setup_or_timeout_(start_time); }); + return; + } + + ESP_LOGE(TAG, "Failed to stop continuous measurements and read product ID"); + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); +} + +void STCC4Component::finish_setup_() { + this->ready_ = true; + // Follow the sources only once measuring started, so a failed setup stops writing to the device + if (this->temperature_source_ != nullptr && this->humidity_source_ != nullptr) { + this->temperature_source_->add_on_state_callback([this](float) { this->update_rht_compensation_from_source_(); }); + this->humidity_source_->add_on_state_callback([this](float) { this->update_rht_compensation_from_source_(); }); + } + if (this->ambient_pressure_source_ != nullptr) { + this->ambient_pressure_source_->add_on_state_callback( + [this](float) { this->update_ambient_pressure_compensation_from_source_(); }); + } +} + +void STCC4Component::dump_config() { + ESP_LOGCONFIG(TAG, "STCC4:"); + LOG_I2C_DEVICE(this); + if (this->is_failed()) { + ESP_LOGW(TAG, ESP_LOG_MSG_COMM_FAIL); + } + ESP_LOGCONFIG(TAG, " Measurement mode: %s", + this->measurement_mode_ == MeasurementMode::CONTINUOUS ? LOG_STR_LITERAL("Continuous (1s)") + : LOG_STR_LITERAL("Single shot")); + if (this->ambient_pressure_source_ != nullptr) { + ESP_LOGCONFIG(TAG, " Dynamic ambient pressure compensation using '%s'", + this->ambient_pressure_source_->get_name().c_str()); + } else if (this->ambient_pressure_in_pa_2_ != 0) { + ESP_LOGCONFIG(TAG, " Ambient pressure compensation: %f hPa", + pressure_in_pa_2_to_hpa(this->ambient_pressure_in_pa_2_)); + } + if (this->temperature_source_ != nullptr) { + ESP_LOGCONFIG(TAG, " Temperature compensation using '%s'", this->temperature_source_->get_name().c_str()); + } + if (this->humidity_source_ != nullptr) { + ESP_LOGCONFIG(TAG, " Humidity compensation using '%s'", this->humidity_source_->get_name().c_str()); + } + LOG_UPDATE_INTERVAL(this); + LOG_SENSOR(" ", "CO2", this->co2_sensor_); + LOG_SENSOR(" ", "Temperature", this->temperature_sensor_); + LOG_SENSOR(" ", "Humidity", this->humidity_sensor_); +} + +void STCC4Component::update() { + if (!this->ready_ || this->measurement_mode_ != MeasurementMode::SINGLE_SHOT) + return; + + // Perform single-shot measurement, wait 500 ms for the measurement to be ready + if (!this->write_command(STCC4_CMD_MEASURE_SINGLE_SHOT)) { + ESP_LOGW(TAG, "Failed to start single shot measurement"); + this->status_set_warning(); + return; + } + this->set_timeout(500, [this]() { + if (this->read_measurement_(0)) { + this->status_clear_warning(); + } else { + ESP_LOGW(TAG, "Failed to read measurement data"); + this->status_set_warning(); + } + }); +} + +void STCC4Component::schedule_continuous_update_(bool retry_for_clock_drift) { + // In continuous measurement mode, the STCC4 produces a sample every 1000 ms according to its + // internal clock. The datasheet recommends retrying 150 ms after a failed read to compensate + // for clock drift between the host and the device. + this->set_timeout(retry_for_clock_drift ? 150 : 1000, [this, retry_for_clock_drift]() { + if (this->read_measurement_(retry_for_clock_drift ? 0 : sensirion_common::SENSIRION_OPTION_READ_MAY_NACK)) { + this->status_clear_warning(); + this->schedule_continuous_update_(false); + } else if (!retry_for_clock_drift) { + this->schedule_continuous_update_(true); + } else { + this->status_set_warning(); + this->schedule_continuous_update_(false); + } + }); +} + +bool STCC4Component::read_measurement_(uint8_t sensirion_options) { + // Read measurement data: 4 words (CO2, temperature, humidity, status) + uint16_t raw_data[4]; + if (!this->get_register_(STCC4_CMD_READ_MEASUREMENT, ADDR_16_BIT, raw_data, 4, 1, sensirion_options)) { + return false; + } + + // CO2 value is in ppm as int16 (ignore negative values during warm-up) + const int16_t co2_raw = int16_t(raw_data[0]); + if (this->co2_sensor_ != nullptr && co2_raw >= 0) { + this->co2_sensor_->publish_state(co2_raw); + } + + if (this->temperature_sensor_ != nullptr) { + this->temperature_sensor_->publish_state(temperature_in_ticks_to_c(raw_data[1])); + } + + if (this->humidity_sensor_ != nullptr) { + this->humidity_sensor_->publish_state(humidity_in_ticks_to_percent(raw_data[2])); + } + return true; +} + +void STCC4Component::update_rht_compensation_from_source_() { + const float temperature_in_c = this->temperature_source_->state; + const float humidity_in_percent = this->humidity_source_->state; + if (std::isnan(temperature_in_c) || std::isnan(humidity_in_percent)) + return; + + const uint16_t temperature_in_ticks = temperature_in_c_to_ticks(temperature_in_c); + const uint16_t humidity_in_ticks = humidity_in_percent_to_ticks(humidity_in_percent); + if ((this->temperature_in_ticks_ != temperature_in_ticks || this->humidity_in_ticks_ != humidity_in_ticks) && + this->write_rht_compensation_(temperature_in_ticks, humidity_in_ticks)) { + this->temperature_in_ticks_ = temperature_in_ticks; + this->humidity_in_ticks_ = humidity_in_ticks; + } +} + +void STCC4Component::set_ambient_pressure_compensation(float pressure_in_hpa) { + this->ambient_pressure_in_pa_2_ = pressure_in_hpa_to_pa_2(pressure_in_hpa); +} + +void STCC4Component::update_ambient_pressure_compensation_from_source_() { + const float pressure_in_hpa = this->ambient_pressure_source_->state; + if (std::isnan(pressure_in_hpa)) + return; + + if (pressure_in_hpa < 100.f || pressure_in_hpa > 10000.f) { + // Some pressure sensors report values in Pa instead of hPa and there's no way to check at compile time. + // Warn if the value seems far outside of the expected range. + if (!this->ambient_pressure_unit_warning_logged_) { + this->ambient_pressure_unit_warning_logged_ = true; + ESP_LOGW(TAG, "Ambient pressure compensation sensor might have incompatible units: got %f hPa", pressure_in_hpa); + } + return; // skip this update + } else { + this->ambient_pressure_unit_warning_logged_ = false; + } + + const uint16_t ambient_pressure_in_pa_2 = pressure_in_hpa_to_pa_2(pressure_in_hpa); + if (this->ambient_pressure_in_pa_2_ != ambient_pressure_in_pa_2 && + this->write_ambient_pressure_compensation_(ambient_pressure_in_pa_2)) { + this->ambient_pressure_in_pa_2_ = ambient_pressure_in_pa_2; + } +} + +bool STCC4Component::write_rht_compensation_(uint16_t temperature_in_ticks, uint16_t humidity_in_ticks) { + ESP_LOGVV(TAG, "Set RHT compensation: %f °C, %f %%RH", temperature_in_ticks_to_c(temperature_in_ticks), + humidity_in_ticks_to_percent(humidity_in_ticks)); + const uint16_t data[2] = {temperature_in_ticks, humidity_in_ticks}; + if (!this->write_command(STCC4_CMD_SET_RHT_COMPENSATION, data, 2)) { + ESP_LOGE(TAG, "Failed to set RHT compensation"); + return false; + } + return true; +} + +bool STCC4Component::write_ambient_pressure_compensation_(uint16_t pressure_in_pa_2) { + ESP_LOGVV(TAG, "Set pressure compensation: %f hPa", pressure_in_pa_2_to_hpa(pressure_in_pa_2)); + if (!this->write_command(STCC4_CMD_SET_PRESSURE_COMPENSATION, pressure_in_pa_2)) { + ESP_LOGE(TAG, "Failed to set ambient pressure compensation"); + return false; + } + return true; +} + +} // namespace esphome::stcc4 diff --git a/esphome/components/stcc4/stcc4.h b/esphome/components/stcc4/stcc4.h new file mode 100644 index 0000000000..ad25130ec7 --- /dev/null +++ b/esphome/components/stcc4/stcc4.h @@ -0,0 +1,55 @@ +#pragma once + +#include "esphome/core/component.h" +#include "esphome/components/sensor/sensor.h" +#include "esphome/components/sensirion_common/i2c_sensirion.h" + +namespace esphome::stcc4 { + +enum class MeasurementMode : uint8_t { + CONTINUOUS = 0, + SINGLE_SHOT, +}; + +class STCC4Component final : public PollingComponent, public sensirion_common::SensirionI2CDevice { + public: + void setup() override; + void dump_config() override; + void update() override; + + void set_co2_sensor(sensor::Sensor *co2) { this->co2_sensor_ = co2; } + void set_temperature_sensor(sensor::Sensor *temperature) { this->temperature_sensor_ = temperature; } + void set_humidity_sensor(sensor::Sensor *humidity) { this->humidity_sensor_ = humidity; } + void set_temperature_source(sensor::Sensor *temperature) { this->temperature_source_ = temperature; } + void set_humidity_source(sensor::Sensor *humidity) { this->humidity_source_ = humidity; } + void set_ambient_pressure_compensation(float pressure_in_hpa); + void set_ambient_pressure_source(sensor::Sensor *pressure) { this->ambient_pressure_source_ = pressure; } + void set_measurement_mode(MeasurementMode mode) { this->measurement_mode_ = mode; } + + protected: + void poll_until_ready_for_setup_or_timeout_(uint32_t start_time); + void finish_setup_(); + void schedule_continuous_update_(bool retry_for_clock_drift); + bool read_measurement_(uint8_t sensirion_options); + void update_rht_compensation_from_source_(); + void update_ambient_pressure_compensation_from_source_(); + bool write_rht_compensation_(uint16_t temperature_in_ticks, uint16_t humidity_in_ticks); + bool write_ambient_pressure_compensation_(uint16_t pressure_in_pa_2); + + sensor::Sensor *co2_sensor_{nullptr}; + sensor::Sensor *temperature_sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; + sensor::Sensor *temperature_source_{nullptr}; + sensor::Sensor *humidity_source_{nullptr}; + sensor::Sensor *ambient_pressure_source_{nullptr}; + + MeasurementMode measurement_mode_{MeasurementMode::CONTINUOUS}; + + uint16_t temperature_in_ticks_{0}; + uint16_t humidity_in_ticks_{0}; + uint16_t ambient_pressure_in_pa_2_{0}; + bool ambient_pressure_unit_warning_logged_{false}; + bool ready_{false}; +}; + +} // namespace esphome::stcc4 diff --git a/tests/components/stcc4/common.yaml b/tests/components/stcc4/common.yaml new file mode 100644 index 0000000000..e87b719226 --- /dev/null +++ b/tests/components/stcc4/common.yaml @@ -0,0 +1,36 @@ +sensor: + # SHT4x for external RHT compensation (boards without onboard SHT4x) + - platform: sht4x + i2c_id: i2c_bus + temperature: + name: SHT4x Temperature + id: sht4x_temperature + humidity: + name: SHT4x Humidity + id: sht4x_humidity + - platform: template + id: ambient_pressure + unit_of_measurement: "hPa" + lambda: 'return 1013;' + + # STCC4 CO2 sensor with onboard temperature/humidity and external compensation + - platform: stcc4 + i2c_id: i2c_bus + id: stcc4_sensor_single_shot + co2: + name: STCC4 CO2 + temperature: + name: STCC4 Temperature + humidity: + name: STCC4 Humidity + temperature_source: sht4x_temperature + humidity_source: sht4x_humidity + measurement_mode: single_shot + ambient_pressure_compensation: 1013 mbar + update_interval: 60s + - platform: stcc4 + i2c_id: i2c_bus + address: 0x65 + id: stcc4_sensor_continuous + measurement_mode: continuous + ambient_pressure_compensation_source: ambient_pressure diff --git a/tests/components/stcc4/test.esp32-idf.yaml b/tests/components/stcc4/test.esp32-idf.yaml new file mode 100644 index 0000000000..b47e39c389 --- /dev/null +++ b/tests/components/stcc4/test.esp32-idf.yaml @@ -0,0 +1,4 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml + +<<: !include common.yaml diff --git a/tests/components/stcc4/test.esp8266-ard.yaml b/tests/components/stcc4/test.esp8266-ard.yaml new file mode 100644 index 0000000000..4a98b9388a --- /dev/null +++ b/tests/components/stcc4/test.esp8266-ard.yaml @@ -0,0 +1,4 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp8266-ard.yaml + +<<: !include common.yaml diff --git a/tests/components/stcc4/test.rp2040-ard.yaml b/tests/components/stcc4/test.rp2040-ard.yaml new file mode 100644 index 0000000000..319a7c71a6 --- /dev/null +++ b/tests/components/stcc4/test.rp2040-ard.yaml @@ -0,0 +1,4 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/rp2040-ard.yaml + +<<: !include common.yaml From dff793a9493a13ed44dd6f166a3a36a2661bf46a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 15:20:27 -0500 Subject: [PATCH 041/221] [espidf] Mirror registry components locally so configures skip the registry (#19949) --- esphome/espidf/component_mirror.py | 431 ++++++++++ esphome/espidf/toolchain.py | 38 +- esphome/framework_helpers.py | 6 +- .../test_espidf_component_mirror.py | 801 ++++++++++++++++++ tests/unit_tests/test_espidf_toolchain.py | 99 +++ tests/unit_tests/test_framework_helpers.py | 16 +- tests/unit_tests/test_lazy_imports.py | 4 +- 7 files changed, 1381 insertions(+), 14 deletions(-) create mode 100644 esphome/espidf/component_mirror.py create mode 100644 tests/unit_tests/test_espidf_component_mirror.py diff --git a/esphome/espidf/component_mirror.py b/esphome/espidf/component_mirror.py new file mode 100644 index 0000000000..a6bbf435fa --- /dev/null +++ b/esphome/espidf/component_mirror.py @@ -0,0 +1,431 @@ +"""Local mirror of IDF component-registry packages. + +The component manager contacts the registry on every cmake configure, even +with an unchanged ``dependencies.lock``, but checks +``IDF_COMPONENT_LOCAL_STORAGE_URL`` mirrors first and stops on a hit. +Mirroring the pinned components with its own ``registry sync`` removes all +registry traffic and makes builds work offline. +""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass, field +import json +import logging +import os +from pathlib import Path +import subprocess +from typing import TYPE_CHECKING, NamedTuple +import zipfile + +from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path +from esphome.core import EsphomeError +from esphome.framework_helpers import rename_with_retry +from esphome.helpers import rmtree, write_file + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping + +_LOGGER = logging.getLogger(__name__) + +# Bump the name if the manager's storage layout ever changes incompatibly; +# the stale directory is removed with the tools cache on clean-all. +_MIRROR_DIR_NAME = "component_mirror" + +_ENV_LOCAL_STORAGE_URL = "IDF_COMPONENT_LOCAL_STORAGE_URL" +_ENV_CHECK_NEW_VERSION = "IDF_COMPONENT_CHECK_NEW_VERSION" + +_DEFAULT_REGISTRY_URL = "https://components.espressif.com" +# Everything a missing, unreadable or wrongly-shaped index file can raise. +_BAD_INDEX_ERRORS = (OSError, ValueError, TypeError, KeyError, AttributeError) +_SYNC_LOCK_NAME = ".sync.lock" +_STAGING_DIR_NAME = ".staging" +_SYNC_TIMEOUT_S = 120 + + +class ServiceDep(NamedTuple): + """A registry ("service") dependency pinned to an exact version.""" + + namespace: str + name: str + version: str + + @property + def spec(self) -> str: + return f"{self.namespace}/{self.name}=={self.version}" + + +def get_mirror_path() -> Path: + """The machine-global mirror directory.""" + return tools_cache_path(*IDF_TOOLS_CACHE) / _MIRROR_DIR_NAME + + +def component_mirror_env() -> dict[str, str]: + """Environment additions that serve the mirror to the component manager. + + A user local storage list keeps precedence; the new-version check (an + extra solve whose answer exact pins make moot) is off unless user-set. + """ + mirror = get_mirror_path() + try: + mirror.mkdir(parents=True, exist_ok=True) + except OSError as err: + # Offline support is quietly lost otherwise; say so. + _LOGGER.warning("Component mirror unavailable at %s: %s", mirror, err) + return {} + _LOGGER.info("Serving IDF components from the local mirror at %s", mirror) + local_storage = mirror.as_uri() + if user_value := os.environ.get(_ENV_LOCAL_STORAGE_URL): + local_storage = f"{user_value};{local_storage}" + env = {_ENV_LOCAL_STORAGE_URL: local_storage} + if _ENV_CHECK_NEW_VERSION not in os.environ: + env[_ENV_CHECK_NEW_VERSION] = "0" + return env + + +def _load_yaml_dict(path: Path) -> dict | None: + """Read a small YAML mapping; missing file or bad content is None.""" + # Deferred: keeps pyyaml off the serial upload fast path. + import yaml + + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + return None + except (OSError, UnicodeDecodeError) as err: + _LOGGER.warning("Could not read %s: %s", path, err) + return None + try: + data = yaml.safe_load(text) + except yaml.YAMLError as err: + _LOGGER.warning("Could not parse %s: %s", path, err) + return None + return data if isinstance(data, dict) else None + + +def _iter_deps(path: Path) -> Iterator[tuple[str, str, dict]]: + """(namespace, name, entry) for each namespaced dependency in a YAML file.""" + data = _load_yaml_dict(path) + deps = data.get("dependencies") if data else None + if not isinstance(deps, dict): + return + for key, entry in deps.items(): + if isinstance(key, str) and "/" in key and isinstance(entry, dict): + # The registry stores lowercase paths; match a mixed-case key. + namespace, _, name = key.lower().partition("/") + yield namespace, name, entry + + +def parse_lock_service_deps(lock_path: Path) -> list[ServiceDep]: + """Pinned default-registry dependencies from a dependencies.lock. + + Git, local and idf sources cannot be mirrored; non-default registries + are left to the manager. + """ + deps: list[ServiceDep] = [] + for namespace, name, entry in _iter_deps(lock_path): + source = entry.get("source") + if not isinstance(source, dict) or source.get("type") != "service": + continue + registry_url = str(source.get("registry_url") or _DEFAULT_REGISTRY_URL) + if registry_url.rstrip("/") != _DEFAULT_REGISTRY_URL: + _LOGGER.debug("Not mirroring %s/%s: non-default registry", namespace, name) + continue + if isinstance(version := entry.get("version"), str): + deps.append(ServiceDep(namespace, name, version)) + return deps + + +def parse_manifest_service_deps(manifest_path: Path) -> list[ServiceDep]: + """Exactly-pinned registry dependencies from an idf_component.yml. + + The manifest exists before the first configure, so a fresh solve can + install from the mirror; range specs are left to the solver. + """ + deps: list[ServiceDep] = [] + for namespace, name, entry in _iter_deps(manifest_path): + if entry.keys() != {"version"} or not isinstance( + version := entry["version"], str + ): + continue + # The YAML shorthand keeps its operator: "==1.2.3" is an exact pin. + version = version.removeprefix("==") + # Exact versions start with a digit ("1.12.0", "1.3.3~1"); range + # operators are prefixes and wildcards contain "*". + if version[:1].isdigit() and "*" not in version: + deps.append(ServiceDep(namespace, name, version)) + return deps + + +def parse_manifest_ranged_specs(manifest_path: Path) -> list[str]: + """Compote specs for the manifest's range-pinned entries. + + Refreshed before every fresh solve, as a range resolves on dev; the + previous candidate stays the offline fallback. + """ + specs: list[str] = [] + for namespace, name, entry in _iter_deps(manifest_path): + if entry.keys() != {"version"} or not isinstance( + version := entry["version"], str + ): + continue + # [:1] alone would match "": the empty string is in every string. + if version.startswith(("^", "~", "<", ">", "!")) or ( + version.startswith("=") and not version.startswith("==") + ): + specs.append(f"{namespace}/{name}{version}") + return specs + + +def project_service_deps(lock_path: Path, manifest_path: Path) -> list[ServiceDep]: + """The mirrorable dependencies of a build; the lock wins, the manifest + covers the build whose lock has not been written yet.""" + deps = parse_lock_service_deps(lock_path) + seen = {(dep.namespace, dep.name) for dep in deps} + for dep in parse_manifest_service_deps(manifest_path): + if (dep.namespace, dep.name) not in seen: + deps.append(dep) + return deps + + +def _mirror_has(mirror: Path, dep: ServiceDep) -> bool: + """Whether the mirror holds the dependency's index entry and files. + + The manager fetches every file the entry names with no registry + fallback once the version is found locally. + """ + json_path = mirror / "components" / dep.namespace / f"{dep.name}.json" + try: + for entry in json.loads(json_path.read_text(encoding="utf-8"))["versions"]: + if entry.get("version") == dep.version: + checksums = entry.get("checksums") + # is_zipfile rejects an archive torn by a crash or an + # old race, so the next sync replaces it (self-heal). + archive = mirror / entry["url"] + if ( + archive.suffix == ".zip" + and archive.is_file() + and not (zipfile.is_zipfile(archive)) + ): + # Deleted, the manager falls back to the registry even + # when the heal sync cannot run; kept, it installs the + # torn file and the configure fails. + with suppress(OSError): + archive.unlink() + return False + return archive.is_file() and ( + not checksums or (mirror / checksums).is_file() + ) + except _BAD_INDEX_ERRORS: + return False + return False + + +def missing_deps(mirror: Path, deps: list[ServiceDep]) -> list[ServiceDep]: + return [dep for dep in deps if not _mirror_has(mirror, dep)] + + +def _read_versions(path: Path) -> list[dict]: + """The version entries of an index; [] when missing or corrupt, so + publishing heals it. An I/O error raises: the content may be valid, + so the caller retries later instead of dropping its versions.""" + try: + entries = json.loads(path.read_text(encoding="utf-8"))["versions"] + return [entry for entry in entries if isinstance(entry, dict)] + except FileNotFoundError: + return [] + except (ValueError, TypeError, KeyError) as err: + _LOGGER.debug("Ignoring the corrupt index %s: %s", path, err) + return [] + + +def _publish_index(src: Path, dst: Path) -> None: + """Publish the staged index merged with the live one, atomically. + + The staged index lists only the versions this sync fetched; the + atomic write keeps the live index intact when it fails, and raises. + """ + staged = json.loads(src.read_text(encoding="utf-8")) + versions = [entry for entry in staged["versions"] if isinstance(entry, dict)] + known = {entry.get("version") for entry in versions} + staged["versions"] = versions + [ + entry for entry in _read_versions(dst) if entry.get("version") not in known + ] + write_file(dst, json.dumps(staged)) + + +def _is_component_index(rel: Path) -> bool: + return ( + rel.parts[0] == "components" and len(rel.parts) == 3 and rel.suffix == ".json" + ) + + +def _promote(staging: Path, mirror: Path) -> None: + """Move the synced files into the mirror, atomically per file. + + Indexes go last, so a concurrent configure never reads a version + entry whose files have not landed yet. + """ + files = sorted( + (path for path in staging.rglob("*") if path.is_file()), + key=lambda path: (_is_component_index(path.relative_to(staging)), path), + ) + for src in files: + rel = src.relative_to(staging) + dst = mirror / rel + dst.parent.mkdir(parents=True, exist_ok=True) + if _is_component_index(rel): + _publish_index(src, dst) + else: + rename_with_retry(src, dst, overwrite=True) + + +@dataclass +class _SyncRun: + """Tools and progress of one registry sync.""" + + python: str + env: dict[str, str] + staging: Path + mirror: Path + synced: int = field(default=0, init=False) + + def _attempt(self, specs: list[str]) -> bool: + """One registry sync invocation; promotes its staging on success.""" + # Raises on an undeletable tree; never promote stale files. + rmtree(self.staging) + cmd = [self.python, "-m", "idf_component_manager", "registry", "sync"] + # latest: the default "all" follows ranged transitive specs and + # syncs every matching version. + cmd += ["--resolution", "latest"] + for spec in specs: + cmd += ["--component", spec] + cmd.append(str(self.staging)) + result = subprocess.run( + cmd, + env=self.env, + capture_output=True, + # Not text=True: the locale codec can raise UnicodeDecodeError. + encoding="utf-8", + errors="replace", + timeout=_SYNC_TIMEOUT_S, + check=False, + ) + if result.returncode != 0: + tail = "\n".join((result.stderr or result.stdout).strip().splitlines()[-5:]) + _LOGGER.warning( + "Could not mirror %s (exit %d):\n%s", + ", ".join(specs), + result.returncode, + tail, + ) + return False + _promote(self.staging, self.mirror) + return True + + def sync(self, specs: list[str]) -> bool: + """The whole set in one invocation, per component after a timeout.""" + try: + # One invocation for the whole set: one manager startup, which + # matters on slow hosts. + if not self._attempt(specs): + return False + self.synced = len(specs) + return True + except subprocess.TimeoutExpired: + if len(specs) == 1: + raise + _LOGGER.warning( + "Mirroring timed out after %d s; retrying one component at a time", + _SYNC_TIMEOUT_S, + ) + # Per component the timeout bounds each one, finished ones stay + # promoted, and the first failure stops the loop. + for spec in specs: + if not self._attempt([spec]): + return False + self.synced += 1 + return True + + +def sync_component_mirror( + lock_path: Path, + manifest_path: Path, + get_python: Callable[[], str], + get_env: Callable[[], Mapping[str, str]], +) -> bool: + """Mirror any of the build's registry components the mirror lacks. + + Returns False on a failed attempt so callers can skip retrying this + run; ``get_python``/``get_env`` are only called when a sync is needed. + """ + mirror = get_mirror_path() + to_sync = missing_deps(mirror, project_service_deps(lock_path, manifest_path)) + # Before a fresh solve, ranged pins re-resolve as they would on dev. + refresh = [] if lock_path.exists() else parse_manifest_ranged_specs(manifest_path) + if not to_sync and not refresh: + return True + try: + python = get_python() + env = dict(get_env()) + mirror.mkdir(parents=True, exist_ok=True) + except (OSError, EsphomeError) as err: + _LOGGER.warning("Could not mirror IDF components: %s", err) + return False + # Staged: the manager writes in place, and a configure in another + # process must never read a truncated file from the live mirror. + staging = mirror / _STAGING_DIR_NAME + # Lazy import, as in git.py: keeps filelock off the CLI startup path. + from filelock import FileLock, Timeout + + lock = FileLock(str(mirror / _SYNC_LOCK_NAME), fallback_to_soft=False) + try: + lock.acquire(blocking=False) + except Timeout: + # Another esphome process is already filling the shared mirror. + _LOGGER.debug("Component mirror sync skipped: already in progress") + return True + except OSError as err: + # A broken cache, not contention; the retry guard should apply. + _LOGGER.warning("Could not lock the component mirror: %s", err) + return False + specs = [dep.spec for dep in to_sync] + refresh + _LOGGER.info("Mirroring %d IDF component(s) for offline builds...", len(specs)) + run = _SyncRun(python, env, staging, mirror) + try: + if not run.sync(specs): + return False + if still := missing_deps(mirror, to_sync): + # A name the registry spells differently syncs clean yet + # covers nothing; retrying would loop forever. + _LOGGER.warning( + "Mirror sync left %d component(s) uncovered: %s", + len(still), + ", ".join(dep.spec for dep in still), + ) + return False + except subprocess.TimeoutExpired: + _LOGGER.warning( + "Mirroring timed out after %d s; kept %d of %d component(s)", + _SYNC_TIMEOUT_S, + run.synced, + len(specs), + ) + return False + except (*_BAD_INDEX_ERRORS, EsphomeError, subprocess.SubprocessError) as err: + # Includes a failed index publish; the live index is intact. + # A shape error is a bug: keep its traceback. + _LOGGER.warning( + "Could not mirror IDF components: %s", + err, + exc_info=isinstance(err, (TypeError, KeyError, AttributeError)), + ) + return False + finally: + # Cleanup only; a leftover tree is removed by the next attempt. + with suppress(OSError): + rmtree(staging) + lock.release() + _LOGGER.info("Mirrored %d IDF component(s) for offline builds", run.synced) + return True diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index ffd8082515..2e8cc0c632 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -24,6 +24,7 @@ from esphome.const import ( ) from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target +from esphome.espidf.component_mirror import component_mirror_env, sync_component_mirror from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary from esphome.helpers import add_git_ceiling_directory, get_bool_env, rmtree, write_file @@ -43,6 +44,7 @@ class _CacheData: cmake_output: dict[Path, str] = field(default_factory=dict) skip_bootloader: bool | None = None cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict) + mirror_sync_failed: bool = False def _cache() -> _CacheData: @@ -110,6 +112,11 @@ def _get_idf_path(version: str | None = None) -> Path | None: return Path(_get_esphome_esp_idf_paths(version)[0]) +def _esphome_manages_idf() -> bool: + """A checkout supplied through IDF_PATH is the user's, not ESPHome's.""" + return "IDF_PATH" not in os.environ + + def _get_idf_env(version: str | None = None) -> dict[str, str]: """Get environment variables needed for ESP-IDF build.""" version = version or _get_core_framework_version() @@ -120,10 +127,12 @@ def _get_idf_env(version: str | None = None) -> dict[str, str]: env_cache[version].pop("PYTHONPATH", None) # Use provided IDF framework if available - if "IDF_PATH" not in os.environ: + if _esphome_manages_idf(): env_cache[version] |= get_framework_env( *_get_esphome_esp_idf_paths(version) ) + # Serve the component manager from the local registry mirror. + env_cache[version] |= component_mirror_env() # Cap git's repo search at the config directory so ESP-IDF's # `git describe` for the app version can't error out on an @@ -369,6 +378,8 @@ def run_reconfigure(verbose: bool = False) -> int: if (path := build_dir / stale).is_dir(): rmtree(path) get_factory_firmware_path().unlink(missing_ok=True) + # First, so the configure (even a first solve) installs from the mirror. + _sync_component_mirror() cmd = [_get_idf_tool("cmake"), "-G", "Ninja"] if _idf_py().binary_dir_arg: cmd += ["-B", str(build_dir)] @@ -387,6 +398,9 @@ def run_reconfigure(verbose: bool = False) -> int: (build_dir / "CMakeCache.txt").unlink(missing_ok=True) _LOGGER.error("CMake configure failed with exit code %d", rc) _print_hints(log_path) + return rc + # Mirror what the solve added to the lock (transitive dependencies). + _sync_component_mirror() return rc @@ -481,6 +495,24 @@ def _print_hints(log_path: Path) -> None: _LOGGER.warning("%s", hints) +def _sync_component_mirror() -> None: + """Best-effort update of the local registry mirror; never fails the build.""" + if not _esphome_manages_idf(): + # _get_idf_env injects no mirror env, so nothing would read a sync. + return + cache = _cache() + if cache.mirror_sync_failed: + return + if not sync_component_mirror( + CORE.relative_build_path("dependencies.lock"), + CORE.relative_build_path("src/idf_component.yml"), + lambda: _get_idf_tool("python"), + _get_idf_env, + ): + # One failed attempt (e.g. offline) is enough per run. + cache.mirror_sync_failed = True + + def _builtin_component_cache_path() -> Path | None: """Cache file for this build's built-in component list. @@ -491,7 +523,7 @@ def _builtin_component_cache_path() -> Path | None: of CONFIG_* options and only gate their sources on them. A checkout supplied through IDF_PATH is not managed by ESPHome and is never cached. """ - if "IDF_PATH" in os.environ: + if not _esphome_manages_idf(): return None target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT]) excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "") @@ -793,6 +825,8 @@ def run_compile(config, verbose: bool) -> int: return rc else: _LOGGER.info("Build configuration is up to date") + # Ninja can still re-run cmake on its own; keep the mirror current. + _sync_component_mirror() if not get_bool_env("ESPHOME_LDGEN_FULL_DEPS"): _warn_if_app_archive_mapped() diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 86010e3065..0599b531c2 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -461,7 +461,7 @@ def _zip_extract_all( report(1) -def _rename_with_retry( +def rename_with_retry( src: Path, dst: Path, attempts: int = 5, overwrite: bool = False ) -> None: """Rename ``src`` to ``dst`` with backoff retries on Windows sharing violations. @@ -561,7 +561,7 @@ def _7z_extract_all( rmtree(dest) else: dest.unlink() - _rename_with_retry(item, dest) + rename_with_retry(item, dest) finally: # staging is created before the try, so it always exists here; the # guard is defensive cleanup and its False branch is unreachable. @@ -1193,7 +1193,7 @@ def download_with_resume( # retries fail, keep the verified part so the next attempt (or # run) only has to redo the rename, not the download. try: - _rename_with_retry(part, dest, overwrite=True) + rename_with_retry(part, dest, overwrite=True) except PermissionError as e: _LOGGER.debug("Could not move %s into place: %s", part, e) last_error = e diff --git a/tests/unit_tests/test_espidf_component_mirror.py b/tests/unit_tests/test_espidf_component_mirror.py new file mode 100644 index 0000000000..fec535393e --- /dev/null +++ b/tests/unit_tests/test_espidf_component_mirror.py @@ -0,0 +1,801 @@ +"""Tests for the local IDF component-registry mirror.""" + +from __future__ import annotations + +from collections.abc import Callable +import json +from pathlib import Path +import subprocess +from unittest.mock import MagicMock, patch +import zipfile + +from filelock import FileLock +import pytest + +from esphome.core import EsphomeError +from esphome.espidf import component_mirror + +_LOCK_TEXT = """\ +dependencies: + bblanchon/arduinojson: + component_hash: 5e6aff2bc5bc988b36c8e90c01e637495ee895ad3af93911860c21ec6da79b6a + dependencies: [] + source: + registry_url: https://components.espressif.com/ + type: service + version: 7.4.3 + espressif/mdns: + component_hash: 3ba256ac95e07c274be53cbd73f06cb846c403b61e8fbdf1be57bdb79db7a63e + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 1.12.0 + esphome/libsodium: + source: + path: /pio_components/idf/cb5ad5b7/esphome/libsodium + type: local + version: '*' + fastled/fastled: + source: + git: https://github.com/fastled/fastled + type: git + version: 3.7.0 + other/private: + source: + registry_url: https://registry.example.com + type: service + version: 1.0.0 + idf: + source: + type: idf + version: 5.5.5 +direct_dependencies: +- bblanchon/arduinojson +- espressif/mdns +manifest_hash: f4e38c031cbb70d5070c5806fc580cce194b8c9a0b919f52a7e952f7d496dda4 +target: esp32 +version: 2.0.0 +""" + +_MANIFEST_TEXT = """\ +dependencies: + espressif/mdns: + version: 1.12.0 + espressif/esp-tflite-micro: + version: 1.3.3~1 + fastled/fastled: + version: 3.7.0 + git: https://github.com/fastled/fastled + esphome/libsodium: + version: '*' + override_path: /stub/libsodium + espressif/ranged: + version: ^1.2.0 + esphome/shorthand: + version: ==2.1.0 + esphome/unquoted: + version: 2 +""" + +_ARDUINOJSON = component_mirror.ServiceDep("bblanchon", "arduinojson", "7.4.3") +_MDNS = component_mirror.ServiceDep("espressif", "mdns", "1.12.0") +_TFLITE = component_mirror.ServiceDep("espressif", "esp-tflite-micro", "1.3.3~1") +_SHORTHAND = component_mirror.ServiceDep("esphome", "shorthand", "2.1.0") +_MANIFEST_DEPS = [_MDNS, _TFLITE, _SHORTHAND] +_NS_CMP_1 = component_mirror.ServiceDep("ns", "cmp", "1.0.0") +_NS_CMP_2 = component_mirror.ServiceDep("ns", "cmp", "2.0.0") + + +@pytest.fixture(autouse=True) +def _isolate_mirror(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", str(tmp_path / "idf_install")) + + +def _write_lock(project_dir: Path, text: str = _LOCK_TEXT) -> Path: + lock = project_dir / "dependencies.lock" + lock.parent.mkdir(parents=True, exist_ok=True) + lock.write_text(text, encoding="utf-8") + return lock + + +def _ns_cmp_lock(version: str) -> str: + return ( + "dependencies:\n" + " ns/cmp:\n" + " source:\n" + " type: service\n" + f" version: {version}\n" + ) + + +def _write_manifest(project_dir: Path, text: str = _MANIFEST_TEXT) -> Path: + manifest = project_dir / "src" / "idf_component.yml" + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text(text, encoding="utf-8") + return manifest + + +def _add_to_mirror(mirror: Path, dep: component_mirror.ServiceDep) -> None: + """Lay out one component the way `registry sync` does.""" + version_dir = f"components/{dep.namespace}/{dep.name}/{dep.version}" + archive = f"{version_dir}/{dep.namespace}__{dep.name}-v{dep.version}.zip" + checksums = f"{version_dir}/CHECKSUMS.json" + json_path = mirror / "components" / dep.namespace / f"{dep.name}.json" + json_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text( + json.dumps( + { + "name": dep.name, + "namespace": dep.namespace, + "versions": [ + {"version": dep.version, "url": archive, "checksums": checksums} + ], + } + ), + encoding="utf-8", + ) + archive_path = mirror / archive + archive_path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(archive_path, "w") as zf: + zf.writestr("idf_component.yml", "") + (mirror / checksums).write_text("{}", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# parse_lock_service_deps / parse_manifest_service_deps / project_service_deps +# --------------------------------------------------------------------------- + + +def test_parse_lock_keeps_only_default_registry_service_deps(tmp_path: Path) -> None: + """git, local, idf and non-default registry sources cannot be mirrored.""" + deps = component_mirror.parse_lock_service_deps(_write_lock(tmp_path)) + assert deps == [_ARDUINOJSON, _MDNS] + + +def test_parse_lock_missing_file(tmp_path: Path) -> None: + assert component_mirror.parse_lock_service_deps(tmp_path / "none.lock") == [] + + +@pytest.mark.parametrize( + "text", + [ + pytest.param("{unbalanced", id="corrupt-yaml"), + pytest.param("[]", id="not-a-mapping"), + pytest.param("dependencies:\n espressif/mdns: not-a-dict\n", id="bad-entry"), + pytest.param(_ns_cmp_lock("1.2"), id="non-string-version"), + ], +) +def test_parse_lock_tolerates_bad_content(tmp_path: Path, text: str) -> None: + assert component_mirror.parse_lock_service_deps(_write_lock(tmp_path, text)) == [] + + +def test_parse_lock_unreadable_file_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Read trouble other than a missing file says so instead of raising.""" + lock = tmp_path / "dependencies.lock" + lock.mkdir() # read_text raises OSError, not FileNotFoundError + assert component_mirror.parse_lock_service_deps(lock) == [] + assert "Could not read" in caplog.text + + +def test_parse_lock_non_utf8_file_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A non-UTF-8 lock is a warning, never a failed build.""" + lock = tmp_path / "dependencies.lock" + lock.write_bytes(b"dependencies:\n # caf\xe9\n") + assert component_mirror.parse_lock_service_deps(lock) == [] + assert "Could not read" in caplog.text + + +def test_parse_lock_defaults_registry_url(tmp_path: Path) -> None: + """A service entry without registry_url is a default-registry dependency.""" + lock = _write_lock(tmp_path, _ns_cmp_lock("1.0.0")) + assert component_mirror.parse_lock_service_deps(lock) == [_NS_CMP_1] + + +def test_parse_manifest_keeps_only_exact_registry_pins(tmp_path: Path) -> None: + """Only exact pins survive, including the == shorthand prefix.""" + deps = component_mirror.parse_manifest_service_deps(_write_manifest(tmp_path)) + assert deps == _MANIFEST_DEPS + + +def test_parse_manifest_missing_file(tmp_path: Path) -> None: + assert component_mirror.parse_manifest_service_deps(tmp_path / "none.yml") == [] + + +def test_parse_manifest_lowercases_mixed_case_keys(tmp_path: Path) -> None: + """The registry stores lowercase paths; a mixed-case key must match them.""" + manifest = tmp_path / "idf_component.yml" + manifest.write_text("dependencies:\n Espressif/MDNS:\n version: 1.2.0\n") + deps = component_mirror.parse_manifest_service_deps(manifest) + assert deps == [component_mirror.ServiceDep("espressif", "mdns", "1.2.0")] + + +def test_parse_manifest_ranged_specs(tmp_path: Path) -> None: + """Ranges, not exact pins, git/override entries or an empty version, + become refresh specs.""" + manifest = _write_manifest( + tmp_path, _MANIFEST_TEXT + " ns/empty:\n version: ''\n" + ) + specs = component_mirror.parse_manifest_ranged_specs(manifest) + assert specs == ["espressif/ranged^1.2.0"] + + +def test_project_service_deps_merges_lock_and_manifest(tmp_path: Path) -> None: + """The lock wins for a component in both; the manifest fills the rest.""" + lock = _write_lock(tmp_path) + manifest = _write_manifest(tmp_path) + assert component_mirror.project_service_deps(lock, manifest) == [ + _ARDUINOJSON, + _MDNS, + _TFLITE, + _SHORTHAND, + ] + + +def test_project_service_deps_manifest_only(tmp_path: Path) -> None: + """A fresh build has no lock yet; the manifest alone drives the sync.""" + manifest = _write_manifest(tmp_path) + assert ( + component_mirror.project_service_deps(tmp_path / "dependencies.lock", manifest) + == _MANIFEST_DEPS + ) + + +# --------------------------------------------------------------------------- +# missing_deps +# --------------------------------------------------------------------------- + + +def test_missing_deps_covered_and_not(tmp_path: Path) -> None: + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + deps = [ + _MDNS, + component_mirror.ServiceDep("espressif", "mdns", "1.13.0"), + _ARDUINOJSON, + ] + assert component_mirror.missing_deps(mirror, deps) == deps[1:] + + +@pytest.mark.parametrize( + "damage", + [ + pytest.param( + lambda m: next(m.rglob("*.zip")).unlink(), + id="archive-gone", + ), + pytest.param( + lambda m: (m / "components/espressif/mdns/1.12.0/CHECKSUMS.json").unlink(), + id="checksums-gone", + ), + pytest.param( + lambda m: (m / "components/espressif/mdns.json").write_text("{broken"), + id="corrupt-index", + ), + pytest.param( + lambda m: next(m.rglob("*.zip")).write_bytes(b"PK\x03\x04torn"), + id="truncated-archive", + ), + ], +) +def test_missing_deps_damaged_mirror( + tmp_path: Path, damage: Callable[[Path], object] +) -> None: + """Every file the index references must exist, and a broken index + counts as missing, not covered.""" + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + damage(mirror) + assert component_mirror.missing_deps(mirror, [_MDNS]) == [_MDNS] + + +def test_missing_deps_deletes_a_torn_archive(tmp_path: Path) -> None: + """A torn archive is removed on detection, so the manager falls back to + the registry even before a heal sync runs.""" + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + torn = next(mirror.rglob("*.zip")) + torn.write_bytes(b"PK\x03\x04torn") + assert component_mirror.missing_deps(mirror, [_MDNS]) == [_MDNS] + assert not torn.exists() + + +def test_missing_deps_covered_without_checksums_field(tmp_path: Path) -> None: + """An older payload without a checksums field only needs its archive.""" + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + index = mirror / "components/espressif/mdns.json" + doc = json.loads(index.read_text()) + del doc["versions"][0]["checksums"] + index.write_text(json.dumps(doc)) + assert component_mirror.missing_deps(mirror, [_MDNS]) == [] + + +# --------------------------------------------------------------------------- +# _publish_index / _promote +# --------------------------------------------------------------------------- + + +def _staged_index(tmp_path: Path, versions: list) -> Path: + src = tmp_path / "staged.json" + src.write_text(json.dumps({"versions": versions}), encoding="utf-8") + return src + + +def test_publish_index_first_sync(tmp_path: Path) -> None: + """No live index yet: the staged one is published as is.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}]) + dst = tmp_path / "live.json" + component_mirror._publish_index(src, dst) + assert json.loads(dst.read_text()) == {"versions": [{"version": "2.0.0"}]} + + +@pytest.mark.parametrize( + "live", ["not json", '{"versions": null}', '{"versions": {"a": 1}}'] +) +def test_publish_index_replaces_a_corrupt_live_index(tmp_path: Path, live: str) -> None: + """An unreadable live index is replaced wholesale; that is the heal.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}]) + dst = tmp_path / "live.json" + dst.write_text(live) + component_mirror._publish_index(src, dst) + assert json.loads(dst.read_text()) == {"versions": [{"version": "2.0.0"}]} + + +def test_publish_index_merges_and_filters(tmp_path: Path) -> None: + """Unfetched live versions survive; non-dict entries drop, not raise.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}, "junk"]) + dst = tmp_path / "live.json" + dst.write_text( + json.dumps({"versions": [{"version": "1.0.0"}, {"version": "2.0.0"}, "bad"]}) + ) + component_mirror._publish_index(src, dst) + assert json.loads(dst.read_text()) == { + "versions": [{"version": "2.0.0"}, {"version": "1.0.0"}] + } + + +def test_publish_index_unreadable_live_index_raises(tmp_path: Path) -> None: + """An I/O error is not corruption; raising keeps the live versions and + a later run retries.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}]) + dst = tmp_path / "live.json" + dst.mkdir() # read_text raises IsADirectoryError, not FileNotFoundError + with pytest.raises(OSError): + component_mirror._publish_index(src, dst) + + +def test_promote_moves_archives_before_indexes(tmp_path: Path) -> None: + """A concurrent reader must never see an index entry without its files.""" + staging = tmp_path / "staging" + _add_to_mirror(staging, _NS_CMP_2) + order: list[str] = [] + real_rename = component_mirror.rename_with_retry + real_write = component_mirror.write_file + + def recording_rename(src: Path, dst: Path, **kwargs) -> None: + order.append(src.name) + real_rename(src, dst, **kwargs) + + def recording_write(path: Path, text: str) -> None: + order.append(path.name) + real_write(path, text) + + with ( + patch.object(component_mirror, "rename_with_retry", recording_rename), + patch.object(component_mirror, "write_file", recording_write), + ): + component_mirror._promote(staging, tmp_path / "mirror") + + assert order[-1] == "cmp.json" + assert set(order[:-1]) == {"CHECKSUMS.json", "ns__cmp-v2.0.0.zip"} + + +# --------------------------------------------------------------------------- +# component_mirror_env +# --------------------------------------------------------------------------- + + +def test_component_mirror_env_sets_file_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("IDF_COMPONENT_LOCAL_STORAGE_URL", raising=False) + monkeypatch.delenv("IDF_COMPONENT_CHECK_NEW_VERSION", raising=False) + env = component_mirror.component_mirror_env() + mirror = component_mirror.get_mirror_path() + assert env["IDF_COMPONENT_LOCAL_STORAGE_URL"] == mirror.as_uri() + assert env["IDF_COMPONENT_CHECK_NEW_VERSION"] == "0" + assert mirror.is_dir() + + +def test_component_mirror_env_preserves_user_values( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A user local storage list keeps precedence; their new-version choice wins.""" + monkeypatch.setenv("IDF_COMPONENT_LOCAL_STORAGE_URL", "file:///user/mirror") + monkeypatch.setenv("IDF_COMPONENT_CHECK_NEW_VERSION", "1") + env = component_mirror.component_mirror_env() + mirror_uri = component_mirror.get_mirror_path().as_uri() + assert env["IDF_COMPONENT_LOCAL_STORAGE_URL"] == f"file:///user/mirror;{mirror_uri}" + assert "IDF_COMPONENT_CHECK_NEW_VERSION" not in env + + +def test_component_mirror_env_unwritable_cache( + caplog: pytest.LogCaptureFixture, +) -> None: + """A read-only cache disables the feature with a warning, not a failure.""" + with patch.object(Path, "mkdir", side_effect=OSError("read-only")): + assert component_mirror.component_mirror_env() == {} + assert "Component mirror unavailable" in caplog.text + + +# --------------------------------------------------------------------------- +# sync_component_mirror +# --------------------------------------------------------------------------- + + +def _run_sync( + project_dir: Path, + *, + returncode: int = 0, + stderr: str = "", + side_effect: Exception | Callable | None = None, + get_python=lambda: "/penv/python", + get_env=lambda: {"PATH": "/penv"}, +) -> tuple[bool, MagicMock]: + with patch.object( + component_mirror.subprocess, + "run", + return_value=subprocess.CompletedProcess([], returncode, "", stderr), + side_effect=side_effect, + ) as mock_run: + ok = component_mirror.sync_component_mirror( + project_dir / "dependencies.lock", + project_dir / "src" / "idf_component.yml", + get_python, + get_env, + ) + return ok, mock_run + + +def _fake_registry_sync(returncode: int = 0): + """A subprocess.run stand-in that lays files out like `registry sync`.""" + + def run(cmd, **kwargs) -> subprocess.CompletedProcess: + _add_to_mirror(Path(cmd[-1]), _NS_CMP_2) + return subprocess.CompletedProcess(cmd, returncode, "", "sync failed") + + return run + + +def _assert_sync_lock_released() -> None: + lock = FileLock(str(component_mirror.get_mirror_path() / ".sync.lock")) + lock.acquire(blocking=False) + lock.release() + + +def test_sync_runs_the_manager_for_missing_deps( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + ok, mock_run = _run_sync(tmp_path) + # The stub stages nothing, so the post-promote recheck reports failure. + assert not ok + assert "uncovered" in caplog.text + mirror = component_mirror.get_mirror_path() + # The whole set goes in one invocation: one manager startup. + mock_run.assert_called_once() + assert mock_run.call_args.args[0] == [ + "/penv/python", + "-m", + "idf_component_manager", + "registry", + "sync", + "--resolution", + "latest", + "--component", + "bblanchon/arduinojson==7.4.3", + "--component", + "espressif/mdns==1.12.0", + str(mirror / ".staging"), + ] + assert mock_run.call_args.kwargs["env"] == {"PATH": "/penv"} + _assert_sync_lock_released() + + +def test_sync_skips_when_covered(tmp_path: Path) -> None: + """A covered project resolves no environment and runs no subprocess.""" + _write_lock(tmp_path) + mirror = component_mirror.get_mirror_path() + _add_to_mirror(mirror, _MDNS) + _add_to_mirror(mirror, _ARDUINOJSON) + + def _boom() -> str: + raise AssertionError("environment resolved for a covered mirror") + + ok, mock_run = _run_sync(tmp_path, get_python=_boom, get_env=_boom) + assert ok + mock_run.assert_not_called() + + +def test_sync_failure_is_tolerated( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + ok, mock_run = _run_sync(tmp_path, returncode=1, stderr="boom") + assert not ok + mock_run.assert_called_once() + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_subprocess_errors_are_tolerated( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + ok, _ = _run_sync(tmp_path, side_effect=OSError("no such file")) + assert not ok + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_refreshes_ranged_specs_only_before_a_fresh_solve( + tmp_path: Path, +) -> None: + """Without a lock the batch also re-resolves ranged pins, as dev would; + with a lock the ranged candidate is left alone.""" + _write_manifest(tmp_path) + ok, mock_run = _run_sync(tmp_path) + assert not ok # the stub stages nothing, so the recheck fails + assert "espressif/ranged^1.2.0" in mock_run.call_args.args[0] + + _write_lock(tmp_path) + ok, mock_run = _run_sync(tmp_path) + assert not ok + assert "espressif/ranged^1.2.0" not in mock_run.call_args.args[0] + + +def test_sync_ranged_refresh_runs_with_everything_else_covered( + tmp_path: Path, +) -> None: + """A covered build still refreshes its ranged pins when the lock is + gone, in one invocation.""" + manifest = _write_manifest(tmp_path) + mirror = component_mirror.get_mirror_path() + for dep in component_mirror.parse_manifest_service_deps(manifest): + _add_to_mirror(mirror, dep) + ok, mock_run = _run_sync(tmp_path) + assert ok + mock_run.assert_called_once() + assert "espressif/ranged^1.2.0" in mock_run.call_args.args[0] + + +def test_sync_timeout_retries_ranged_specs_too(tmp_path: Path) -> None: + """The fallback covers refresh specs, not only the exact pins.""" + _write_manifest( + tmp_path, + "dependencies:\n ns/one:\n version: ^1.0\n ns/two:\n version: ^2.0\n", + ) + effects = [ + subprocess.TimeoutExpired(cmd=[], timeout=120), + subprocess.CompletedProcess([], 0, "", ""), + subprocess.CompletedProcess([], 0, "", ""), + ] + ok, mock_run = _run_sync(tmp_path, side_effect=effects) + assert ok + retried = [call.args[0][-2] for call in mock_run.call_args_list[1:]] + assert retried == ["ns/one^1.0", "ns/two^2.0"] + + +def test_sync_timeout_falls_back_to_one_component_at_a_time( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A timed-out batch retries per component, so the ones that finish + stay promoted and later runs fetch only the rest.""" + _write_lock(tmp_path) + + def slow_sync(cmd, **kwargs): + specs = [arg for arg in cmd if "==" in arg] + if specs != ["bblanchon/arduinojson==7.4.3"]: + # The batch, and later the mdns retry, exceed the timeout. + raise subprocess.TimeoutExpired(cmd=cmd, timeout=120) + namespace, _, rest = specs[0].partition("/") + name, _, version = rest.partition("==") + _add_to_mirror( + Path(cmd[-1]), component_mirror.ServiceDep(namespace, name, version) + ) + return subprocess.CompletedProcess(cmd, 0, "", "") + + ok, mock_run = _run_sync(tmp_path, side_effect=slow_sync) + assert not ok + assert mock_run.call_count == 3 # batch, arduinojson, mdns + assert "retrying one component" in caplog.text + assert "kept 1 of 2" in caplog.text + mirror = component_mirror.get_mirror_path() + assert component_mirror.missing_deps(mirror, [_ARDUINOJSON, _MDNS]) == [_MDNS] + assert not (mirror / ".staging").exists() + _assert_sync_lock_released() + + +def test_sync_timeout_then_per_component_completes(tmp_path: Path) -> None: + """When every component fits on its own, the fallback finishes the job.""" + _write_lock(tmp_path) + + def slow_sync(cmd, **kwargs): + specs = [arg for arg in cmd if "==" in arg] + if len(specs) > 1: + raise subprocess.TimeoutExpired(cmd=cmd, timeout=120) + namespace, _, rest = specs[0].partition("/") + name, _, version = rest.partition("==") + _add_to_mirror( + Path(cmd[-1]), component_mirror.ServiceDep(namespace, name, version) + ) + return subprocess.CompletedProcess(cmd, 0, "", "") + + ok, mock_run = _run_sync(tmp_path, side_effect=slow_sync) + assert ok + assert mock_run.call_count == 3 # batch, then one call per component + mirror = component_mirror.get_mirror_path() + assert component_mirror.missing_deps(mirror, [_ARDUINOJSON, _MDNS]) == [] + + +def test_sync_timeout_then_component_error_stops( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A hard error in the fallback loop stops it like any other failure.""" + _write_lock(tmp_path) + effects = [ + subprocess.TimeoutExpired(cmd=[], timeout=120), + subprocess.CompletedProcess([], 1, "", "boom"), + ] + ok, mock_run = _run_sync(tmp_path, side_effect=effects) + assert not ok + assert mock_run.call_count == 2 + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_unexpected_error_logs_a_traceback( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An operational failure warns in one line; a bug keeps its traceback.""" + _write_lock(tmp_path) + with patch.object(component_mirror, "_promote", side_effect=KeyError("url")): + ok, _ = _run_sync(tmp_path) + assert not ok + record = next(r for r in caplog.records if "Could not mirror" in r.message) + assert record.exc_info + + +def test_sync_single_component_timeout_is_not_retried( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """With one missing component the batch was already the retry; a build + pays the timeout once.""" + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + ok, mock_run = _run_sync( + tmp_path, side_effect=subprocess.TimeoutExpired(cmd=[], timeout=120) + ) + assert not ok + mock_run.assert_called_once() + assert "kept 0 of 1" in caplog.text + _assert_sync_lock_released() + + +def test_sync_environment_resolution_failure_is_tolerated( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + + def _boom() -> str: + raise EsphomeError("python not found") + + ok, mock_run = _run_sync(tmp_path, get_python=_boom) + assert not ok + mock_run.assert_not_called() + assert "Could not mirror" in caplog.text + + +def test_sync_skips_when_another_process_holds_the_lock(tmp_path: Path) -> None: + _write_lock(tmp_path) + mirror = component_mirror.get_mirror_path() + mirror.mkdir(parents=True) + held = FileLock(str(mirror / ".sync.lock")) + held.acquire(blocking=False) + try: + ok, mock_run = _run_sync(tmp_path) + finally: + held.release() + assert ok + mock_run.assert_not_called() + + +def test_sync_ignores_a_leftover_lock_file(tmp_path: Path) -> None: + """A lock file from a dead process does not block: the OS lock is gone.""" + _write_lock(tmp_path) + mirror = component_mirror.get_mirror_path() + mirror.mkdir(parents=True) + (mirror / ".sync.lock").touch() + _, mock_run = _run_sync(tmp_path) + mock_run.assert_called() # the dead lock did not block the sync + + +def test_sync_lock_oserror_is_a_failure( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A broken cache is not contention; the retry guard should apply.""" + _write_lock(tmp_path) + with patch("filelock.FileLock.acquire", side_effect=PermissionError("ro")): + ok, mock_run = _run_sync(tmp_path) + assert not ok + mock_run.assert_not_called() + assert "Could not lock" in caplog.text + + +def test_sync_undeletable_staging_is_a_failure( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An undeletable staging tree aborts the sync; stale files must + never be promoted.""" + _write_lock(tmp_path) + with patch.object(component_mirror, "rmtree", side_effect=OSError("stuck staging")): + ok, mock_run = _run_sync(tmp_path) + assert not ok + mock_run.assert_not_called() + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_promotes_staged_files_and_merges_the_index(tmp_path: Path) -> None: + """New files land atomically and existing versions survive the merge.""" + mirror = component_mirror.get_mirror_path() + _add_to_mirror(mirror, _NS_CMP_1) + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + + ok, _ = _run_sync(tmp_path, side_effect=_fake_registry_sync()) + + assert ok + doc = json.loads((mirror / "components" / "ns" / "cmp.json").read_text()) + assert {entry["version"] for entry in doc["versions"]} == {"1.0.0", "2.0.0"} + assert (mirror / "components/ns/cmp/2.0.0/ns__cmp-v2.0.0.zip").is_file() + assert (mirror / "components/ns/cmp/1.0.0/ns__cmp-v1.0.0.zip").is_file() + assert not (mirror / ".staging").exists() + + +def test_sync_failure_leaves_no_staging_behind(tmp_path: Path) -> None: + """A failed sync promotes nothing and removes its staging directory.""" + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + + ok, _ = _run_sync(tmp_path, side_effect=_fake_registry_sync(returncode=1)) + + assert not ok + mirror = component_mirror.get_mirror_path() + assert not (mirror / ".staging").exists() + assert not (mirror / "components" / "ns").exists() + + +def test_sync_failed_index_publish_keeps_the_live_index( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failed publish keeps the live index; a later run retries.""" + mirror = component_mirror.get_mirror_path() + _add_to_mirror(mirror, _NS_CMP_1) + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + + with patch.object( + component_mirror, "write_file", side_effect=EsphomeError("disk full") + ): + ok, _ = _run_sync(tmp_path, side_effect=_fake_registry_sync()) + + assert not ok + assert "Could not mirror" in caplog.text + doc = json.loads((mirror / "components" / "ns" / "cmp.json").read_text()) + assert {entry["version"] for entry in doc["versions"]} == {"1.0.0"} + _assert_sync_lock_released() diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 63d15a694c..1aaff7e9ce 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -215,6 +215,38 @@ def test_get_idf_env_sets_git_ceiling_directories(setup_core: Path) -> None: assert str(CORE.config_dir) in env["GIT_CEILING_DIRECTORIES"].split(os.pathsep) +def test_get_idf_env_serves_the_component_mirror(setup_core: Path) -> None: + """The mirror env rides the managed-IDF branch, next to its sync gate.""" + toolchain._cache().env.clear() + with ( + patch.object( + toolchain, + "get_framework_env", + return_value={"PATH": "/penv"}, + ), + patch.object(toolchain, "_get_esphome_esp_idf_paths", return_value=((), {})), + patch.object( + toolchain, + "component_mirror_env", + return_value={"IDF_COMPONENT_LOCAL_STORAGE_URL": "file:///mirror"}, + ), + ): + env = toolchain._get_idf_env(version="5.5.4") + assert env["IDF_COMPONENT_LOCAL_STORAGE_URL"] == "file:///mirror" + + +def test_get_idf_env_user_idf_skips_the_mirror(setup_core: Path) -> None: + """A user-managed IDF gets neither the framework env nor the mirror.""" + toolchain._cache().env.clear() + with ( + patch.dict(os.environ, {"IDF_PATH": str(setup_core)}), + patch.object(toolchain, "component_mirror_env") as mock_env, + ): + env = toolchain._get_idf_env(version="5.5.4") + mock_env.assert_not_called() + assert "IDF_COMPONENT_LOCAL_STORAGE_URL" not in env + + def test_get_idf_env_pops_inherited_pythonpath(setup_core: Path) -> None: """A PYTHONPATH from the parent environment must not reach idf.py. @@ -362,6 +394,73 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: assert not CORE.relative_build_path("build/CMakeCache.txt").exists() +@pytest.mark.parametrize( + ("cmake_rc", "expected"), + [(0, ["sync", "cmake", "sync"]), (1, ["sync", "cmake"])], +) +def test_run_reconfigure_syncs_mirror_around_the_configure( + setup_core: Path, cmake_rc: int, expected: list[str] +) -> None: + """Every configure path syncs: before, so the solve installs from the + mirror; after a success, so solver-added lock entries are mirrored.""" + _setup_build(setup_core) + calls: list[str] = [] + + with ( + _fake_tools() as mock_run, + patch.object( + toolchain, + "_sync_component_mirror", + side_effect=lambda: calls.append("sync"), + ), + # The real hint printer resolves an IDF install (or downloads one). + patch.object(toolchain, "_print_hints"), + ): + mock_run.side_effect = lambda *a, **k: calls.append("cmake") or cmake_rc + assert toolchain.run_reconfigure() == cmake_rc + + assert calls == expected + + +def test_run_compile_syncs_mirror_when_up_to_date(setup_core: Path) -> None: + """No reconfigure still refreshes the mirror once for ninja-driven + cmake re-runs.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + + with ( + _up_to_date_compile(), + patch.object(toolchain, "_sync_component_mirror") as mock_sync, + ): + assert toolchain.run_compile(config, verbose=False) == 0 + + mock_sync.assert_called_once() + + +def test_sync_component_mirror_skips_user_idf( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A user-managed IDF never gets the mirror env, so a sync is unread.""" + monkeypatch.setenv("IDF_PATH", "/opt/esp-idf") + with patch.object(toolchain, "sync_component_mirror") as mock_sync: + toolchain._sync_component_mirror() + mock_sync.assert_not_called() + + +def test_sync_component_mirror_failure_not_retried( + setup_core: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """One failed attempt (e.g. offline) must not stall every later call.""" + monkeypatch.delenv("IDF_PATH", raising=False) + _setup_build(setup_core) + with patch.object( + toolchain, "sync_component_mirror", return_value=False + ) as mock_sync: + toolchain._sync_component_mirror() + toolchain._sync_component_mirror() + mock_sync.assert_called_once() + + def test_run_compile_reconfigures_after_full_write_outside_testing_mode( setup_core: Path, ) -> None: diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index f3b182073f..f63838a8e2 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -27,7 +27,6 @@ from esphome.framework_helpers import ( _7z_extract_all, _BatchDownloadProgress, _detect_archive_root, - _rename_with_retry, _tar_extract_all, _zip_extract_all, archive_extract_all, @@ -40,6 +39,7 @@ from esphome.framework_helpers import ( get_project_link_flags, get_python_env_executable_path, get_system_python_path, + rename_with_retry, rmdir, run_batch_downloads, run_command, @@ -1013,7 +1013,7 @@ class TestDownloadWithResume: with ( patch("requests.get", return_value=_mock_response(b"data")) as mock_get, patch( - "esphome.framework_helpers._rename_with_retry", + "esphome.framework_helpers.rename_with_retry", side_effect=[PermissionError("locked"), None], ) as rename, ): @@ -1030,7 +1030,7 @@ class TestDownloadWithResume: with ( patch("requests.get", return_value=_mock_response(b"data")), patch( - "esphome.framework_helpers._rename_with_retry", + "esphome.framework_helpers.rename_with_retry", side_effect=PermissionError("locked"), ), pytest.raises(EsphomeError, match="after 1 attempts"), @@ -2019,7 +2019,7 @@ class TestZipExtractAllBranches: # --------------------------------------------------------------------------- -# _rename_with_retry +# rename_with_retry # --------------------------------------------------------------------------- @@ -2028,7 +2028,7 @@ class TestRenameWithRetry: src = tmp_path / "src.txt" src.write_text("data") dst = tmp_path / "dst.txt" - _rename_with_retry(src, dst) + rename_with_retry(src, dst) assert dst.read_text() == "data" assert not src.exists() @@ -2050,7 +2050,7 @@ class TestRenameWithRetry: patch.object(Path, "rename", flaky_rename), patch("esphome.framework_helpers.time.sleep"), ): - _rename_with_retry(src, dst, attempts=3) + rename_with_retry(src, dst, attempts=3) assert dst.read_text() == "data" def test_raises_after_all_attempts_fail(self, tmp_path: Path) -> None: @@ -2062,14 +2062,14 @@ class TestRenameWithRetry: patch("esphome.framework_helpers.time.sleep"), pytest.raises(PermissionError), ): - _rename_with_retry(src, dst, attempts=3) + rename_with_retry(src, dst, attempts=3) def test_attempts_zero_is_noop(self, tmp_path: Path) -> None: """Zero attempts means the for-loop body never runs; src is untouched.""" src = tmp_path / "src.txt" src.write_text("data") dst = tmp_path / "dst.txt" - _rename_with_retry(src, dst, attempts=0) + rename_with_retry(src, dst, attempts=0) assert src.exists() assert not dst.exists() diff --git a/tests/unit_tests/test_lazy_imports.py b/tests/unit_tests/test_lazy_imports.py index 1eeddd80ea..28ca2394eb 100644 --- a/tests/unit_tests/test_lazy_imports.py +++ b/tests/unit_tests/test_lazy_imports.py @@ -192,7 +192,9 @@ def test_native_upload_fast_path_does_not_import_heavy_modules( fixture_path, probe_env, "native_upload_fast_path.py", - extra=("esphome.components.esp8266",), + # The cache-hit modules guard the espidf toolchain import chain: + # a top-level yaml import there would tax every upload subprocess. + extra=("esphome.components.esp8266", *CACHE_HIT_HEAVY_MODULES), ) assert not leaked, ( f"native upload_using_esptool pulls in heavy modules: {leaked}. " From 99070187b8edbc102668119b17b6b2b4707406e8 Mon Sep 17 00:00:00 2001 From: Michael Heichler <55435876+michaelheichler@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:15:23 +0200 Subject: [PATCH 042/221] [bluetooth_connection] Cancel pending Bluedroid opens on disconnect (#19859) Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- .../bluetooth_connection_bluedroid.cpp | 73 ++++++++++++++++++- .../bluetooth_connection_bluedroid.h | 4 + 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp index 986a67c7a8..d3af02780f 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp @@ -17,6 +17,13 @@ #include "esphome/core/log.h" #include +#include + +// cancel_open exists from ESP-IDF 5.5.5 and 6.0.1. +#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 5) && ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(6, 0, 0)) || \ + ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 1) +#define BLUEDROID_HAS_CANCEL_OPEN +#endif namespace esphome::bluetooth_connection { @@ -63,6 +70,12 @@ void BluedroidGattClient::loop() { // The one teardown safety net: a lost CLOSE_EVT, or a scheduled // teardown whose OPEN_EVT never arrives. if (millis() - this->disconnecting_started_ > ble_device_base::GATT_DISCONNECT_TIMEOUT_MS) { + if (this->conn_id_ == UNSET_CONN_ID) { + this->cancel_pending_open_(); + } else if (st == ClientState::CONNECTING) { + // CONNECT_EVT came but OPEN_EVT was lost, so our open still holds the ACL link. + this->check_and_log_error_("esp_ble_gap_disconnect", esp_ble_gap_disconnect(this->remote_bda_)); + } ESP_LOGE(TAG, "[%d] Timeout waiting for teardown, forcing IDLE", this->connection_index_); // Release before idling: a lost completion must not leak the cache. this->release_services(); @@ -140,6 +153,8 @@ void BluedroidGattClient::tracker_connect_() { this->services_released_ = false; this->seen_mtu_ = false; this->mtu_failed_ = false; + this->cancel_open_sent_ = false; + this->conn_id_ = UNSET_CONN_ID; this->enable_loop(); this->set_state(ClientState::CONNECTING); if (this->connection_type_ == ConnectionType::V3_WITHOUT_CACHE) { @@ -183,12 +198,28 @@ int BluedroidGattClient::gatt_disconnect() { // Arm the safety window: a lost OPEN_EVT must not leak the teardown. this->disconnecting_started_ = millis(); this->enable_loop(); + if (this->conn_id_ == UNSET_CONN_ID) { + // CANCEL_OPEN_EVT or a racing OPEN_EVT settles the slot. + this->cancel_pending_open_(); + } return 0; } this->unconditional_disconnect_(); return 0; } +void BluedroidGattClient::cancel_pending_open_() { +#ifdef BLUEDROID_HAS_CANCEL_OPEN + esp_ble_gattc_cancel_open_params_t params{}; + params.gattc_if = this->gattc_if_; + memcpy(params.remote_bda, this->remote_bda_, sizeof(esp_bd_addr_t)); + // A refused cancel schedules no CANCEL_OPEN_EVT, so the teardown stays cancellable. + if (this->check_and_log_error_("esp_ble_gattc_cancel_open", esp_ble_gattc_cancel_open(¶ms)) == ESP_OK) { + this->cancel_open_sent_ = true; + } +#endif +} + void BluedroidGattClient::unconditional_disconnect_() { ESP_LOGI(TAG, "[%d] Disconnecting (conn_id: %d)", this->connection_index_, this->conn_id_); if (this->conn_id_ == UNSET_CONN_ID) { @@ -214,6 +245,10 @@ bool BluedroidGattClient::cancel_gatt_disconnect() { if (this->state() != ClientState::CONNECTING || !this->disconnect_pending()) { return false; } + if (this->cancel_open_sent_) { + // The cancel already went out, so CANCEL_OPEN_EVT ends it. + return false; + } this->want_disconnect_ = false; return true; } @@ -574,6 +609,8 @@ void BluedroidGattClient::handle_open_evt_(esp_ble_gattc_cb_param_t *param) { this->listener_->on_connection_state(false, 0, param->open.status); return; } + // ALREADY_OPEN on an existing link sends no CONNECT_EVT, so take the id here. + this->conn_id_ = param->open.conn_id; if (this->disconnect_pending()) { // Open resolved with a teardown scheduled: close now (conn_id_ stays set // so CLOSE_EVT still matches). @@ -617,6 +654,11 @@ void BluedroidGattClient::handle_disconnect_evt_(esp_ble_gattc_cb_param_t *param // Active close delivers CLOSE_EVT first; never walk back to DISCONNECTING. return; } + if (this->conn_id_ == UNSET_CONN_ID) { + // No CONNECT_EVT for this attempt yet, so the event belongs to an earlier + // link, such as one the teardown timeout forced down. + return; + } // Passive disconnect: wait for CLOSE_EVT before going IDLE (reconnecting // earlier makes the controller reject with 133 or assert) and before // reporting - the wrapper frees the slot on the report, and a freed slot @@ -645,6 +687,10 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga case ESP_GATTC_CONNECT_EVT: { if (!this->check_addr_(param->connect.remote_bda)) return false; + // Every client interface gets CONNECT_EVT for every new link; only an + // attempt in flight owns it. An idle slot must not adopt another slot's link. + if (this->state() != ClientState::CONNECTING) + break; this->conn_id_ = param->connect.conn_id; // MTU request here rather than OPEN_EVT, matching the IDF examples. auto ret = esp_ble_gattc_send_mtu_req(this->gattc_if_, param->connect.conn_id); @@ -656,8 +702,15 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga break; } case ESP_GATTC_OPEN_EVT: { - if (!this->check_addr_(param->open.remote_bda)) + if (!this->check_addr_(param->open.remote_bda)) { + // Our interface (filtered above), stale address, so nobody tracks this link. + if (param->open.status == ESP_GATT_OK || param->open.status == ESP_GATT_ALREADY_OPEN) { + ESP_LOGW(TAG, "[%d] Closing link left by an abandoned open", this->connection_index_); + this->check_and_log_error_("esp_ble_gattc_close", esp_ble_gattc_close(this->gattc_if_, param->open.conn_id)); + return true; + } return false; + } this->handle_open_evt_(param); break; } @@ -748,6 +801,24 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga this->listener_->on_notify_data(param->notify.handle, param->notify.value, param->notify.value_len); break; } +#ifdef BLUEDROID_HAS_CANCEL_OPEN + case ESP_GATTC_CANCEL_OPEN_EVT: { + if (!this->check_addr_(param->cancel_open.remote_bda)) + return false; + if (param->cancel_open.status != ESP_GATT_OK) { + // Too late to cancel: OPEN_EVT follows and settles the slot. The latch + // stays set because the event cannot be tied to this attempt's cancel. + break; + } + if (this->state() == ClientState::CONNECTING && this->disconnect_pending()) { + ESP_LOGD(TAG, "[%d] Pending open cancelled", this->connection_index_); + this->release_services(); + this->set_idle_(); + this->listener_->on_connection_state(false, 0, ESP_GATT_CONN_CONN_CANCEL); + } + break; + } +#endif default: break; } diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h index a4e9edec23..ef0ca223cb 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h @@ -100,6 +100,7 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public int handle_search_cmpl_(esp_gatt_status_t status); void deliver_pending_search_(); void unconditional_disconnect_(); + void cancel_pending_open_(); void set_idle_(); void set_disconnecting_(); esp_err_t update_conn_params_(uint16_t min_interval, uint16_t max_interval, uint16_t latency, uint16_t timeout, @@ -133,6 +134,9 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public bool seen_mtu_ : 1 {false}; // The MTU request was refused at CONNECT_EVT; OPEN_EVT reports instead. bool mtu_failed_ : 1 {false}; + // esp_ble_gattc_cancel_open() was accepted; CANCEL_OPEN_EVT or OPEN_EVT ends + // the attempt, so the scheduled teardown can no longer be cancelled. + bool cancel_open_sent_ : 1 {false}; // Search issued at OPEN_EVT overlaps the MTU exchange; discover_services() // completes from it. Reset by set_idle_(). static_assert(static_cast(SearchState::REPORT_PENDING) < (1 << 4), "search_state_ bitfield too narrow"); From b324a7c6e050e935614d1c88eede3733cc24e8d6 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:40:09 +0200 Subject: [PATCH 043/221] [socket] Add a reconnecting TCP client link (#19999) Co-authored-by: J. Nick Koston --- esphome/components/socket/tcp_client_link.cpp | 150 ++++++++++++++++++ esphome/components/socket/tcp_client_link.h | 74 +++++++++ .../__init__.py | 35 ++++ .../tcp_client_link_test_component.cpp | 28 ++++ .../tcp_client_link_test_component.h | 24 +++ .../fixtures/socket_tcp_client_link.yaml | 20 +++ .../test_socket_tcp_client_link.py | 88 ++++++++++ 7 files changed, 419 insertions(+) create mode 100644 esphome/components/socket/tcp_client_link.cpp create mode 100644 esphome/components/socket/tcp_client_link.h create mode 100644 tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py create mode 100644 tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp create mode 100644 tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h create mode 100644 tests/integration/fixtures/socket_tcp_client_link.yaml create mode 100644 tests/integration/test_socket_tcp_client_link.py diff --git a/esphome/components/socket/tcp_client_link.cpp b/esphome/components/socket/tcp_client_link.cpp new file mode 100644 index 0000000000..f2c1b8e4c1 --- /dev/null +++ b/esphome/components/socket/tcp_client_link.cpp @@ -0,0 +1,150 @@ +#include "tcp_client_link.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +#include +#include + +namespace esphome::socket { + +// After this long in SYN, the stack's own retries are cut short. +static constexpr uint32_t CONNECT_TIMEOUT_MS = 10000; + +// Non-blocking options and TCP keepalive for a bridged stream socket. +// Keepalive is best-effort: the raw lwIP implementation (ESP8266, RP2040) +// rejects it, so a half-open link there is only detected by a failed write. +static void set_stream_options(Socket *sock) { + int yes = 1; + sock->setblocking(false); + sock->setsockopt(IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); + sock->setsockopt(SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)); +#ifdef TCP_KEEPIDLE + int idle = 30; + int interval = 10; + int count = 3; + sock->setsockopt(IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle)); + sock->setsockopt(IPPROTO_TCP, TCP_KEEPINTVL, &interval, sizeof(interval)); + sock->setsockopt(IPPROTO_TCP, TCP_KEEPCNT, &count, sizeof(count)); +#endif +} + +void TcpClientLink::begin(const char *tag) { + this->tag_ = tag; + // The first attempt must not wait out a full interval. + this->last_attempt_ms_ = App.get_loop_component_start_time() - this->reconnect_interval_ms_; +} + +void TcpClientLink::poll_slow_() { + if (this->sock_ == nullptr) { + this->try_connect_(); + return; + } + int err = 0; + switch (poll_connect(*this->sock_, err)) { + case ConnectPollResult::CONNECT_POLL_RESULT_PENDING: + // Give up before the stack's SYN retries do, so the interval stays honest + // and the next attempt resolves the host again. + if (App.get_loop_component_start_time() - this->last_attempt_ms_ >= + std::max(this->reconnect_interval_ms_, CONNECT_TIMEOUT_MS)) { + this->drop_(LOG_STR("Connect failed"), ETIMEDOUT); + } + return; + case ConnectPollResult::CONNECT_POLL_RESULT_ERROR: + this->drop_(LOG_STR("Connect failed"), err); + return; + default: + break; + } + this->connected_ = true; + ESP_LOGI(this->tag_, "Connected to %s:%u", this->host_.c_str(), this->port_); +} + +void TcpClientLink::try_connect_() { + if (this->resolved_.consume_failure()) { + this->note_attempt(); + return; + } + this->resolved_.start(this->host_.c_str(), this->port_, this->tag_); + if (!this->resolved_.ready()) { + return; + } + struct sockaddr_storage dest; + socklen_t dest_len = + this->resolved_.to_sockaddr(reinterpret_cast(&dest), sizeof(dest), this->port_); + if (dest_len == 0) { + this->note_attempt(); + return; + } + this->sock_ = socket_loop_monitored(dest.ss_family, SOCK_STREAM, IPPROTO_TCP); + if (this->sock_ == nullptr) { + this->drop_(LOG_STR("Connect failed"), errno); + return; + } + set_stream_options(this->sock_.get()); + // Starts the pending-connect clock that poll() times out against. + this->note_attempt(); + // An immediate success is reported by the next poll(); poll_connect() sees it writable. + if (this->sock_->connect(reinterpret_cast(&dest), dest_len) != 0 && errno != EINPROGRESS) { + this->drop_(LOG_STR("Connect failed"), errno); + } +} + +void TcpClientLink::adopt(std::unique_ptr sock) { + this->close(); + set_stream_options(sock.get()); + this->sock_ = std::move(sock); + this->connected_ = true; +} + +ssize_t TcpClientLink::read(uint8_t *buf, size_t len) { + if (!this->connected_) { + return 0; + } + ssize_t count = this->sock_->read(buf, len); + if (count > 0) { + return count; + } + if (count == 0 || (errno != EAGAIN && errno != EWOULDBLOCK)) { + this->drop_(LOG_STR("Connection lost"), count == 0 ? 0 : errno); + return -1; + } + return 0; +} + +ssize_t TcpClientLink::write(const uint8_t *buf, size_t len) { + if (!this->connected_ || len == 0) { + return 0; + } + ssize_t sent = this->sock_->write(buf, len); + if (sent >= 0) { + return sent; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return 0; + } + this->drop_(LOG_STR("Connection lost"), errno); + return -1; +} + +void TcpClientLink::close() { + if (this->sock_ != nullptr) { + this->sock_->shutdown(SHUT_RDWR); + this->sock_->close(); + this->sock_.reset(); + } + this->connected_ = false; + this->resolved_.forget(); +} + +void TcpClientLink::drop_(const LogString *what, int err) { + ESP_LOGW(this->tag_, "%s: %d", LOG_STR_ARG(what), err); + this->close(); + this->note_attempt(); +} + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/socket/tcp_client_link.h b/esphome/components/socket/tcp_client_link.h new file mode 100644 index 0000000000..065c4df562 --- /dev/null +++ b/esphome/components/socket/tcp_client_link.h @@ -0,0 +1,74 @@ +#pragma once + +#include "headers.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include "ipv4_resolve.h" +#include "socket.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" +#include "esphome/core/string_ref.h" + +#include +#include + +namespace esphome::socket { + +/// A reconnecting TCP stream driven from loop(). Owns the socket, the DNS +/// lookup and the retry backoff. A fatal read/write error closes the link +/// and schedules the next attempt; the caller sees the edge via connected(). +class TcpClientLink { + public: + void set_host(const char *host) { this->host_ = StringRef(host); } + void set_port(uint16_t port) { this->port_ = port; } + void set_reconnect_interval(uint32_t ms) { this->reconnect_interval_ms_ = ms; } + const char *host() const { return this->host_.c_str(); } + uint16_t port() const { return this->port_; } + uint32_t reconnect_interval() const { return this->reconnect_interval_ms_; } + + /// Call from setup(). tag names this link's log lines. + void begin(const char *tag); + /// Connect state machine; call every loop while acting as a client. + /// Inline no-op while connected or waiting out the backoff. + void poll() { + if (this->connected_ || (this->sock_ == nullptr && this->in_backoff())) { + return; + } + this->poll_slow_(); + } + /// Take over an accepted socket (the server side of a bridge). + void adopt(std::unique_ptr sock); + /// Returns bytes moved, 0 when nothing can move now, -1 when the link dropped. + ssize_t read(uint8_t *buf, size_t len); + ssize_t write(const uint8_t *buf, size_t len); + /// Close without scheduling a reconnect (shutdown). + void close(); + + bool connected() const { return this->connected_; } + bool ready() const { return this->sock_ != nullptr && this->sock_->ready(); } + /// Shared retry clock, also usable for a listen socket. + void note_attempt() { this->last_attempt_ms_ = App.get_loop_component_start_time(); } + bool in_backoff() const { + return App.get_loop_component_start_time() - this->last_attempt_ms_ < this->reconnect_interval_ms_; + } + + protected: + void poll_slow_(); + void try_connect_(); + /// Close after a failure, log what and errno, schedule the next attempt. + void drop_(const LogString *what, int err); + + StringRef host_; + std::unique_ptr sock_; + const char *tag_{nullptr}; + uint32_t last_attempt_ms_{0}; + uint32_t reconnect_interval_ms_{5000}; + Ipv4Resolve resolved_; + uint16_t port_{0}; + bool connected_{false}; +}; + +} // namespace esphome::socket + +#endif diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py new file mode 100644 index 0000000000..8a7703d025 --- /dev/null +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py @@ -0,0 +1,35 @@ +import esphome.codegen as cg +from esphome.components.const import CONF_HOST +import esphome.config_validation as cv +from esphome.const import CONF_ID, CONF_PORT +from esphome.types import ConfigType + +AUTO_LOAD = ["socket"] + +CONF_RECONNECT_INTERVAL = "reconnect_interval" + +tcp_client_link_test_component_ns = cg.esphome_ns.namespace( + "tcp_client_link_test_component" +) +TcpClientLinkTestComponent = tcp_client_link_test_component_ns.class_( + "TcpClientLinkTestComponent", cg.Component +) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(TcpClientLinkTestComponent), + cv.Required(CONF_HOST): cv.string, + cv.Required(CONF_PORT): cv.port, + cv.Optional( + CONF_RECONNECT_INTERVAL, default="1s" + ): cv.positive_time_period_milliseconds, + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + cg.add(var.set_host(config[CONF_HOST])) + cg.add(var.set_port(config[CONF_PORT])) + cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL])) diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp new file mode 100644 index 0000000000..7f2af8add8 --- /dev/null +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp @@ -0,0 +1,28 @@ +#include "tcp_client_link_test_component.h" +#include "esphome/core/log.h" + +namespace esphome::tcp_client_link_test_component { + +static const char *const TAG = "tcp_link_test"; + +void TcpClientLinkTestComponent::setup() { this->link_.begin(TAG); } + +void TcpClientLinkTestComponent::loop() { + this->link_.poll(); + bool up = this->link_.connected(); + if (up != this->was_up_) { + this->was_up_ = up; + ESP_LOGI(TAG, "Link %s", up ? LOG_STR_LITERAL("up") : LOG_STR_LITERAL("down")); + } + if (!up || !this->link_.ready()) { + return; + } + uint8_t buf[64]; + ssize_t count = this->link_.read(buf, sizeof(buf)); + if (count > 0) { + ESP_LOGI(TAG, "Echoing %d bytes", static_cast(count)); + this->link_.write(buf, static_cast(count)); + } +} + +} // namespace esphome::tcp_client_link_test_component diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h new file mode 100644 index 0000000000..829f15b8a3 --- /dev/null +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h @@ -0,0 +1,24 @@ +#pragma once + +#include "esphome/components/socket/tcp_client_link.h" +#include "esphome/core/component.h" + +namespace esphome::tcp_client_link_test_component { + +/// Echoes every byte the link receives back to the peer and logs link edges. +class TcpClientLinkTestComponent : public Component { + public: + void set_host(const char *host) { this->link_.set_host(host); } + void set_port(uint16_t port) { this->link_.set_port(port); } + void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); } + + void setup() override; + void loop() override; + void on_shutdown() override { this->link_.close(); } + + protected: + socket::TcpClientLink link_; + bool was_up_{false}; +}; + +} // namespace esphome::tcp_client_link_test_component diff --git a/tests/integration/fixtures/socket_tcp_client_link.yaml b/tests/integration/fixtures/socket_tcp_client_link.yaml new file mode 100644 index 0000000000..2ed7f775ff --- /dev/null +++ b/tests/integration/fixtures/socket_tcp_client_link.yaml @@ -0,0 +1,20 @@ +esphome: + name: socket-tcp-client-link-test + +host: + +api: + +logger: + level: INFO + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + components: [tcp_client_link_test_component] + +tcp_client_link_test_component: + host: 127.0.0.1 + port: 18123 + reconnect_interval: 1s diff --git a/tests/integration/test_socket_tcp_client_link.py b/tests/integration/test_socket_tcp_client_link.py new file mode 100644 index 0000000000..31cc344f67 --- /dev/null +++ b/tests/integration/test_socket_tcp_client_link.py @@ -0,0 +1,88 @@ +"""Integration test for socket::TcpClientLink on host. + +Pytest runs a real TCP server; the device echoes through the link. +Covers connect, read, write, a server-initiated drop and the reconnect. +""" + +from __future__ import annotations + +import asyncio +import contextlib + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + +PAYLOAD = b"hello link" + + +@pytest.mark.asyncio +async def test_socket_tcp_client_link( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + unused_tcp_port_factory, +) -> None: + server_port = unused_tcp_port_factory() + yaml_config = yaml_config.replace("port: 18123", f"port: {server_port}") + + echoed: list[bytes] = [] + echo_done = asyncio.Event() + reconnected = asyncio.Event() + link_down = asyncio.Event() + second_link_up = asyncio.Event() + link_up_count = 0 + + def on_log_line(line: str) -> None: + nonlocal link_up_count + if "Link up" in line: + link_up_count += 1 + if link_up_count >= 2: + second_link_up.set() + elif "Link down" in line: + link_down.set() + + async def handle( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + if not echo_done.is_set(): + writer.write(PAYLOAD) + await writer.drain() + with contextlib.suppress(TimeoutError, asyncio.IncompleteReadError): + echoed.append( + await asyncio.wait_for(reader.readexactly(len(PAYLOAD)), 10) + ) + echo_done.set() + # Drop the connection so the link has to reconnect. + writer.close() + return + reconnected.set() + + server = await asyncio.start_server(handle, "127.0.0.1", server_port) + try: + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "socket-tcp-client-link-test" + + try: + await asyncio.wait_for(echo_done.wait(), timeout=15.0) + except TimeoutError: + pytest.fail("Link never connected or echoed") + assert echoed and echoed[0] == PAYLOAD, "Echo payload mismatch" + + try: + await asyncio.wait_for(link_down.wait(), timeout=15.0) + except TimeoutError: + pytest.fail("Link never reported the drop") + try: + await asyncio.wait_for(reconnected.wait(), timeout=15.0) + await asyncio.wait_for(second_link_up.wait(), timeout=15.0) + except TimeoutError: + pytest.fail("Link did not reconnect after the server dropped it") + finally: + server.close() + await server.wait_closed() From ff973537672193bc6a7363d38e850cbafc6f18f7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 19:39:06 -0500 Subject: [PATCH 044/221] [espidf] Resolve the ESP-IDF tool paths once per build (#19996) --- esphome/espidf/framework.py | 33 +++++++++++++++++++++-- tests/unit_tests/test_espidf_framework.py | 24 +++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 8c377561ca..e9049815f6 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -2,6 +2,7 @@ from collections.abc import Callable from ctypes.util import find_library +from dataclasses import dataclass, field import json import logging import os @@ -18,7 +19,7 @@ from esphome.build_helpers.ccache import ( ) from esphome.build_helpers.pch import ccache_pch_env from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path -from esphome.core import Version +from esphome.core import CORE, Version from esphome.framework_helpers import ( PathType, create_venv, @@ -42,6 +43,21 @@ _LOGGER = logging.getLogger(__name__) _SCRIPTS_DIR = Path(__file__).parent +DOMAIN = "espidf_framework" + + +@dataclass +class _FrameworkCache: + tool_paths: dict[Path, tuple[list[str], dict[str, str]]] = field( + default_factory=dict + ) + + +def _cache() -> _FrameworkCache: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = _FrameworkCache() + return CORE.data[DOMAIN] + ESPHOME_STAMP_FILE = ".esphome.stamp.json" @@ -362,7 +378,17 @@ def _get_idf_tool_paths( Raises: RuntimeError: If ESP-IDF tool paths cannot be determined + + The install check and the build environment both resolve the same + framework, so the result is cached per run and the helper script runs + once per build instead of once per caller. The script also reads + ``IDF_TOOLS_PATH``; every caller sets it from ``get_idf_tools_path()``, + so the key leaves it out. """ + cache = _cache().tool_paths + key = Path(idf_framework_root) + if (cached := cache.get(key)) is not None: + return cached success, stdout, stderr = _run_idf_tools_script( idf_framework_root, "get_idf_tool_paths.py", "ESP-IDF tool paths", env=env @@ -373,11 +399,13 @@ def _get_idf_tool_paths( # Extract json values try: data = json.loads(stdout) - return data["paths_to_export"], data["export_vars"] + result = (data["paths_to_export"], data["export_vars"]) except Exception as e: raise RuntimeError( f"Can't extract ESP-IDF tool paths of {idf_framework_root}" ) from e + cache[key] = result + return result def _get_python_version( @@ -927,6 +955,7 @@ def _check_esphome_idf_framework_install( # Validate via the managed tool-path resolution, not ``idf_tools.py check``: # ``check`` probes tools on the system PATH and aborts if any fail to run (e.g. a # broken Homebrew openocd), which forced a toolchain reinstall on every build. + # The resolved paths stay cached for get_framework_env. try: _get_idf_tool_paths(framework_path, env) install = False diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index c785d56fa4..13aa8f7091 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -1511,6 +1511,30 @@ def test_get_idf_tool_paths_raises_on_failure(tmp_path: Path) -> None: _get_idf_tool_paths(tmp_path) +def test_get_idf_tool_paths_runs_the_script_once_per_build(tmp_path: Path) -> None: + payload = json.dumps({"paths_to_export": ["/a"], "export_vars": {"X": "1"}}) + env = {"IDF_TOOLS_PATH": str(tmp_path / "tools")} + with patch( + "esphome.espidf.framework.run_command", return_value=(True, payload, "") + ) as run: + first = _get_idf_tool_paths(tmp_path, env) + second = _get_idf_tool_paths(tmp_path, env) + assert run.call_count == 1 + assert first == second == (["/a"], {"X": "1"}) + + +def test_get_idf_tool_paths_does_not_cache_a_failure(tmp_path: Path) -> None: + payload = json.dumps({"paths_to_export": ["/a"], "export_vars": {}}) + with patch( + "esphome.espidf.framework.run_command", + side_effect=[(False, "", "err"), (True, payload, "")], + ) as run: + with pytest.raises(RuntimeError, match="Can't get ESP-IDF tool paths"): + _get_idf_tool_paths(tmp_path) + assert _get_idf_tool_paths(tmp_path) == (["/a"], {}) + assert run.call_count == 2 + + def test_get_python_version_parses_stdout(tmp_path: Path) -> None: with patch( "esphome.espidf.framework.run_command", return_value=(True, "3.11.0\n", "") From 6479e079d2331077c2eafb4aff0ac9a2b8ceeff7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 19:39:18 -0500 Subject: [PATCH 045/221] [espidf] Build all and size in one ninja run (#19997) --- esphome/build_gen/espidf.py | 4 ++ esphome/espidf/size_summary.py | 2 +- esphome/espidf/toolchain.py | 21 ++++----- tests/unit_tests/build_gen/test_espidf.py | 6 +++ tests/unit_tests/test_espidf_toolchain.py | 52 ++++++++++++----------- 5 files changed, 50 insertions(+), 35 deletions(-) diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 3838d07602..50c0abf1b2 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -351,11 +351,15 @@ project({CORE.name}) # Emit per-memory-type JSON size data for ESPHome to read post-build. # json2 stays small; raw dumps every symbol (~2s on a large map) and # this command runs inside the link edge, blocking everything downstream. +# The map is a BYPRODUCT so ninja knows the link writes it; IDF's size +# target depends on the map and can then be built in the same run as all. +# IDF's cmakev2 declares the map itself, so drop this line on that switch. add_custom_command( TARGET ${{CMAKE_PROJECT_NAME}}.elf POST_BUILD COMMAND ${{PYTHON}} -m esp_idf_size {size_ng_flag} --format=json2 -o ${{CMAKE_BINARY_DIR}}/esp_idf_size.json ${{CMAKE_PROJECT_NAME}}.map + BYPRODUCTS ${{CMAKE_BINARY_DIR}}/${{CMAKE_PROJECT_NAME}}.map WORKING_DIRECTORY ${{CMAKE_BINARY_DIR}} VERBATIM ) diff --git a/esphome/espidf/size_summary.py b/esphome/espidf/size_summary.py index ffe97ba618..9fc1a6b3a9 100644 --- a/esphome/espidf/size_summary.py +++ b/esphome/espidf/size_summary.py @@ -1,6 +1,6 @@ """PlatformIO-format RAM/Flash one-liners after a native ESP-IDF build. -The ninja ``size`` target (run after ``all`` in +The ninja ``size`` target (built together with ``all`` in ``toolchain.run_compile``) prints the per-region table inline as part of the build. This module adds two summary lines underneath, byte-identical to PlatformIO's output: diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 2e8cc0c632..4202ee71ce 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -428,21 +428,21 @@ def _build_jobs(config) -> int | None: def _run_ninja( - target: str, - *, + *targets: str, verbose: bool, jobs: int | None, progress: bool = False, extra_env: dict[str, str] | None = None, ) -> int: - """Build one ninja target, with the flags and env idf.py uses.""" + """Build ninja targets in one run, with the flags and env idf.py uses.""" cmd = [_get_idf_tool("ninja")] if jobs is not None: cmd += ["-j", str(jobs)] if verbose: cmd.append("-v") - cmd.append(target) - log_path = _build_dir() / "log" / f"ninja_{Path(target).name}_output.log" + cmd += targets + log_name = "_".join(Path(t).name for t in targets) + log_path = _build_dir() / "log" / f"ninja_{log_name}_output.log" rc = run_build_tool( cmd, cwd=_build_dir(), @@ -452,7 +452,7 @@ def _run_ninja( log_path=log_path, ) if rc != 0: - _LOGGER.error("ninja %s failed with exit code %d", target, rc) + _LOGGER.error("ninja %s failed with exit code %d", " ".join(targets), rc) _print_hints(log_path) return rc @@ -845,10 +845,11 @@ def run_compile(config, verbose: bool) -> int: write_pch_checksum() - # idf.py's ``build size``, minus the second ``ninja all`` it runs first. - rc = _run_ninja("all", verbose=verbose, jobs=jobs, progress=True) - if rc == 0: - rc = _run_ninja("size", verbose=verbose, jobs=jobs, extra_env=_size_env()) + # idf.py's ``build size`` in one ninja run; size needs the map, so it + # runs after the link. + rc = _run_ninja( + "all", "size", verbose=verbose, jobs=jobs, progress=True, extra_env=_size_env() + ) if rc == 0: size_json = CORE.relative_build_path("build", "esp_idf_size.json") partitions = CORE.relative_build_path("partitions.csv") diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 7ed6201b6a..f8e834a78d 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -222,6 +222,12 @@ def test_get_project_cmakelists_size_command_uses_json2() -> None: assert "--format=json2" in content +def test_get_project_cmakelists_declares_map_as_link_byproduct() -> None: + """The link declares the map so size can build in the same ninja run.""" + content = _render() + assert "BYPRODUCTS ${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}.map" in content + + def test_get_project_cmakelists_uses_supplied_builtin_components() -> None: """A cached list replaces project_description.json and is still filtered by EXCLUDE_COMPONENTS.""" diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 1aaff7e9ce..620d08fe3b 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -534,8 +534,8 @@ def _record_compile_calls( def record_save(components: list[str]) -> None: calls.append(("save", components)) - def record_ninja(target: str, **kwargs: object) -> int: - if target == "all": + def record_ninja(*targets: str, **kwargs: object) -> int: + if "all" in targets: calls.append(("build",)) return 0 @@ -886,7 +886,7 @@ def test_run_compile_full_deps_skips_fragment_check( def test_run_compile_passes_compile_process_limit( setup_core: Path, limit: int | None ) -> None: - """compile_process_limit is the job limit for both ninja runs.""" + """compile_process_limit is the job limit of the one ninja run.""" _setup_build(setup_core) esphome = {} if limit is None else {CONF_COMPILE_PROCESS_LIMIT: limit} @@ -894,8 +894,14 @@ def test_run_compile_passes_compile_process_limit( assert toolchain.run_compile({CONF_ESPHOME: esphome}, verbose=False) == 0 assert mock_run.call_args_list == [ - call("all", verbose=False, jobs=limit, progress=True), - call("size", verbose=False, jobs=limit, extra_env=toolchain._size_env()), + call( + "all", + "size", + verbose=False, + jobs=limit, + progress=True, + extra_env=toolchain._size_env(), + ), ] @@ -1185,14 +1191,17 @@ def test_run_ninja_filters_and_reports_failure( patch.object(toolchain, "_print_hints") as mock_hints, ): mock_run.return_value = 1 - assert toolchain._run_ninja("all", verbose=False, jobs=None, progress=True) == 1 + assert ( + toolchain._run_ninja("all", "size", verbose=False, jobs=None, progress=True) + == 1 + ) log_path = mock_run.call_args.kwargs["log_path"] - assert log_path.name == "ninja_all_output.log" + assert log_path.name == "ninja_all_size_output.log" mock_hints.assert_called_once_with(log_path) - assert mock_run.call_args.args[0] == ["/tools/ninja", "all"] + assert mock_run.call_args.args[0] == ["/tools/ninja", "all", "size"] assert mock_run.call_args.kwargs["filter_lines"] is toolchain.FILTER_IDF_LINES assert mock_run.call_args.kwargs["progress"] is True - assert "ninja all failed with exit code 1" in caplog.text + assert "ninja all size failed with exit code 1" in caplog.text @pytest.mark.parametrize("reconfigure_rc", [0, 5]) @@ -1215,17 +1224,12 @@ def test_run_compile_reconfigures_when_cache_entries_change( assert mock_ninja.called is (reconfigure_rc == 0) -@pytest.mark.parametrize("failing", ["all", "size"]) -def test_run_compile_stops_on_ninja_failure(setup_core: Path, failing: str) -> None: - """A failed build skips size; either failure skips the summary.""" +def test_run_compile_stops_on_ninja_failure(setup_core: Path) -> None: + """A failed ninja run skips the summary.""" _setup_build(setup_core) - with _up_to_date_compile(lambda target, **kw: 7 if target == failing else 0) as ( - mock_ninja, - mock_summary, - ): + with _up_to_date_compile(lambda *targets, **kw: 7) as (mock_ninja, mock_summary): assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 7 - targets = [c.args[0] for c in mock_ninja.call_args_list] - assert targets == (["all"] if failing == "all" else ["all", "size"]) + assert [c.args for c in mock_ninja.call_args_list] == [("all", "size")] mock_summary.assert_not_called() @@ -1236,11 +1240,11 @@ def test_run_compile_testing_mode_builds_memory_ld_first( """Testing mode builds and patches memory.ld before the main build.""" _setup_build(setup_core) CORE.testing_mode = True - targets: list[str] = [] + targets: list[tuple[str, ...]] = [] - def record(target: str, **kwargs: object) -> int: - targets.append(target) - return memory_ld_rc if target.endswith("memory.ld") else 0 + def record(*run_targets: str, **kwargs: object) -> int: + targets.append(run_targets) + return memory_ld_rc if run_targets[0].endswith("memory.ld") else 0 with ( _up_to_date_compile(record), @@ -1249,10 +1253,10 @@ def test_run_compile_testing_mode_builds_memory_ld_first( assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == memory_ld_rc memory_ld = str(Path("esp-idf", "esp_system", "ld", "memory.ld")) if memory_ld_rc: - assert targets == [memory_ld] + assert targets == [(memory_ld,)] mock_patch.assert_not_called() else: - assert targets == [memory_ld, "all", "size"] + assert targets == [(memory_ld,), ("all", "size")] mock_patch.assert_called_once() From 8617368294e43110d2139f6125c30cd7a95b5a2c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 19:48:00 -0500 Subject: [PATCH 046/221] [espidf] Read the ESP-IDF version in process instead of starting idf_tools (#19998) --- esphome/espidf/framework.py | 52 +++++++++++++++--- script/check_idf_py_equivalence.py | 18 ++++++- tests/script/test_check_idf_py_equivalence.py | 27 +++++++++- tests/unit_tests/test_espidf_framework.py | 53 ++++++++++++++++--- 4 files changed, 133 insertions(+), 17 deletions(-) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index e9049815f6..fdfc5c9b01 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -336,15 +336,42 @@ def _raise_script_failure(what: str, root: PathType, stderr: str | None) -> NoRe ) -def _get_idf_version( - idf_framework_root: PathType, env: dict[str, str] | None = None -) -> str: +# What idf_tools.get_idf_version() matches: ``version.txt`` first, then the +# version header. Both give major.minor only. +_IDF_VERSION_TXT_RE = re.compile(r"^v(\d+\.\d+)") +_IDF_VERSION_HEADER_RE = re.compile( + r"^#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+).+?^#define\s+ESP_IDF_VERSION_MINOR\s+(\d+)", + re.DOTALL | re.MULTILINE, +) + + +def _read_text(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.is_file() else "" + + +def read_idf_version_txt(idf_framework_root: Path) -> str | None: + """major.minor from ``version.txt``, as idf_tools reads it.""" + if match := _IDF_VERSION_TXT_RE.match( + _read_text(idf_framework_root / "version.txt") + ): + return match.group(1) + return None + + +def read_idf_version_header(idf_framework_root: Path) -> str | None: + """major.minor from ``esp_idf_version.h``, as idf_tools reads it.""" + header = idf_framework_root / "components" / "esp_common" / "include" + if match := _IDF_VERSION_HEADER_RE.search(_read_text(header / "esp_idf_version.h")): + return f"{match.group(1)}.{match.group(2)}" + return None + + +def _get_idf_version(idf_framework_root: PathType) -> str: """ Get the ESP-IDF version from the specified framework root. Args: idf_framework_root: Path to the ESP-IDF framework root directory - env: Optional dictionary of environment variables to set Returns: String containing ESP-IDF version @@ -352,9 +379,20 @@ def _get_idf_version( Raises: RuntimeError: If ESP-IDF version cannot be determined """ + root = Path(idf_framework_root) + try: + version = read_idf_version_txt(root) or read_idf_version_header(root) + except (OSError, UnicodeError) as e: + raise RuntimeError(f"Can't get ESP-IDF version of {root}: {e}") from e + if version is None: + raise RuntimeError(f"Can't get ESP-IDF version of {root}") + return version + +def idf_tools_version(idf_framework_root: PathType) -> str: + """The version from the framework's own ``idf_tools``, for the CI drift guard.""" success, stdout, stderr = _run_idf_tools_script( - idf_framework_root, "get_idf_version.py", "ESP-IDF version", env=env + idf_framework_root, "get_idf_version.py", "ESP-IDF version" ) if stdout: stdout = stdout.strip() @@ -1070,7 +1108,7 @@ def _check_esp_idf_python_env_install( create_venv(python_env_path, msg=f"ESP-IDF {version}") - esp_idf_version = _get_idf_version(framework_path, env=env) + esp_idf_version = _get_idf_version(framework_path) constraint_file_path = ( get_idf_tools_path() / f"espidf.constraints.v{esp_idf_version}.txt" ) @@ -1311,7 +1349,7 @@ def get_framework_env( # 4. Set framework-specific environment variables env["IDF_PATH"] = str(framework_path) - env["ESP_IDF_VERSION"] = _get_idf_version(framework_path, env) + env["ESP_IDF_VERSION"] = _get_idf_version(framework_path) # 5. Get and add tool paths and environment variables paths_to_export, export_vars = _get_idf_tool_paths(framework_path, env) diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index c0aa3c0d5e..8ef3577eed 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -55,6 +55,10 @@ MACRO_CHANGED = ( "IDF changed __build_process_project_includes; update " "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py" ) +VERSION_DRIFT = ( + "ESPHome reads ESP-IDF version {ours!r} from {source} but idf_tools reports " + "{theirs!r}; update read_idf_version_{source} in esphome/espidf/framework.py" +) WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld") DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*" @@ -127,7 +131,7 @@ def check(build_path: Path) -> list[str]: # pylint: disable=protected-access from esphome.build_gen.espidf import idf_macro_matches from esphome.core import CORE - from esphome.espidf import toolchain + from esphome.espidf import framework, toolchain description = json.loads( (build_path / "build" / "project_description.json").read_text(encoding="utf-8") @@ -139,8 +143,18 @@ def check(build_path: Path) -> list[str]: CORE.skip_bootloader = skip_bootloader # A prior tree's memoized decision must not leak into this one. toolchain._cache().skip_bootloader = None - if not idf_macro_matches(toolchain._get_idf_path(version)): + idf_path = toolchain._get_idf_path(version) + if not idf_macro_matches(idf_path): return [MACRO_CHANGED] + # A managed tree always has version.txt, so the header branch is + # compared on its own or it would never be exercised here. + theirs = framework.idf_tools_version(idf_path) + for source, read in ( + ("txt", framework.read_idf_version_txt), + ("header", framework.read_idf_version_header), + ): + if (ours := read(idf_path)) != theirs: + return [VERSION_DRIFT.format(ours=ours, source=source, theirs=theirs)] # ESP-IDF's openthread stamps the configure time into its compile flags; # pin it before the env is cached so both configures get the same value. os.environ["SOURCE_DATE_EPOCH"] = "0" diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index 42dbdf4dc7..9ce52a6df5 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -16,7 +16,7 @@ import check_idf_py_equivalence as guard # noqa: E402 from esphome.build_gen import espidf as build_gen # noqa: E402 from esphome.core import CORE # noqa: E402 -from esphome.espidf import toolchain # noqa: E402 +from esphome.espidf import framework, toolchain # noqa: E402 @pytest.fixture(autouse=True) @@ -74,10 +74,12 @@ def _run_check( esphome_rcs: tuple[int, int] = (0, 0), macro_matches: bool = True, envs: list[dict[str, str]] | None = None, + versions: tuple[str | None, str | None, str] = ("5.5", "5.5", "5.5"), ) -> tuple[list[str], list[list[str]]]: """Run check() with idf.py replaced by ``side_effect``; return problems, calls. - ``envs`` collects the env each idf.py call receives. + ``envs`` collects the env each idf.py call receives. ``versions`` is what + version.txt, the version header and idf_tools report for the framework. """ calls: list[list[str]] = [] @@ -98,6 +100,9 @@ def _run_check( patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), patch.object(build_gen, "idf_macro_matches", return_value=macro_matches), + patch.object(framework, "read_idf_version_txt", return_value=versions[0]), + patch.object(framework, "read_idf_version_header", return_value=versions[1]), + patch.object(framework, "idf_tools_version", return_value=versions[2]), patch.object(guard.subprocess, "run", side_effect=run), patch.dict(os.environ), ): @@ -285,6 +290,24 @@ def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None: assert calls == [] +@pytest.mark.parametrize( + ("versions", "source"), + [(("5.4", "5.5", "5.5"), "txt"), (("5.5", None, "5.5"), "header")], +) +def test_check_fails_loudly_when_the_version_read_drifts( + tmp_path: Path, versions: tuple[str | None, str | None, str], source: str +) -> None: + """An IDF bump that changes how idf_tools reads its version must fail CI; + both sources are checked since a managed tree never reaches the header.""" + tree = _make_tree(tmp_path) + problems, calls = _run_check(tree, versions=versions) + ours = versions[0] if source == "txt" else versions[1] + assert problems == [ + guard.VERSION_DRIFT.format(ours=ours, source=source, theirs="5.5") + ] + assert calls == [] + + def test_main_without_build_trees( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index 13aa8f7091..c3f29459c6 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -44,6 +44,7 @@ from esphome.espidf.framework import ( check_esp_idf_install, get_framework_env, get_idf_tools_path, + idf_tools_version, ) from esphome.framework_helpers import _tar_extract_all, get_python_env_executable_path @@ -1468,19 +1469,59 @@ def test_demote_unused_tools_already_patched_is_noop(tmp_path: Path) -> None: # --------------------------------------------------------------------------- -def test_get_idf_version_parses_stdout(tmp_path: Path) -> None: +def _write_idf_version_header(root: Path, major: int, minor: int) -> None: + include = root / "components" / "esp_common" / "include" + include.mkdir(parents=True) + (include / "esp_idf_version.h").write_text( + f"#define ESP_IDF_VERSION_MAJOR {major}\n" + "/** Minor version number (x.X.x) */\n" + f"#define ESP_IDF_VERSION_MINOR {minor}\n" + "#define ESP_IDF_VERSION_PATCH 0\n", + encoding="utf-8", + ) + + +def test_get_idf_version_reads_version_txt(tmp_path: Path) -> None: + """version.txt wins and gives major.minor, as idf_tools returns it.""" + (tmp_path / "version.txt").write_text("v5.5.5\n", encoding="utf-8") + _write_idf_version_header(tmp_path, 6, 1) + assert _get_idf_version(tmp_path) == "5.5" + + +def test_get_idf_version_falls_back_to_the_header(tmp_path: Path) -> None: + """A version.txt that does not match (a git ref) defers to the header.""" + (tmp_path / "version.txt").write_text("vrelease/v6.1\n", encoding="utf-8") + _write_idf_version_header(tmp_path, 6, 1) + assert _get_idf_version(tmp_path) == "6.1" + + +def test_get_idf_version_raises_without_a_source(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="Can't get ESP-IDF version"): + _get_idf_version(tmp_path) + + +def test_get_idf_version_wraps_an_unreadable_source(tmp_path: Path) -> None: + """A source that cannot be decoded keeps the RuntimeError contract.""" + (tmp_path / "version.txt").write_bytes(b"\xff\xfev") + with pytest.raises(RuntimeError, match="Can't get ESP-IDF version") as info: + _get_idf_version(tmp_path) + assert isinstance(info.value.__cause__, UnicodeError) + + +def test_idf_tools_version_runs_the_framework_script(tmp_path: Path) -> None: with patch( - "esphome.espidf.framework.run_command", return_value=(True, "5.1.2\n", "") - ): - assert _get_idf_version(tmp_path) == "5.1.2" + "esphome.espidf.framework.run_command", return_value=(True, "5.5\n", "") + ) as run: + assert idf_tools_version(tmp_path) == "5.5" + assert run.call_args.args[0][1].endswith("get_idf_version.py") -def test_get_idf_version_raises_on_failure(tmp_path: Path) -> None: +def test_idf_tools_version_raises_on_failure(tmp_path: Path) -> None: with ( patch("esphome.espidf.framework.run_command", return_value=(False, "", "boom")), pytest.raises(RuntimeError, match="Can't get ESP-IDF version"), ): - _get_idf_version(tmp_path) + idf_tools_version(tmp_path) def test_get_idf_tool_paths_parses_json(tmp_path: Path) -> None: From 12b016e3d27b03f88e87c301ca2c29a193be128a Mon Sep 17 00:00:00 2001 From: Jeff Brown Date: Fri, 2 Oct 2026 05:27:29 -0700 Subject: [PATCH 047/221] [substitutions] Support !include with a condition (#19455) Co-authored-by: J. Nick Koston --- esphome/components/substitutions/__init__.py | 19 +- esphome/yaml_util.py | 114 +++++++-- .../substitutions/01-include.approved.yaml | 7 + .../substitutions/01-include.input.yaml | 19 ++ .../substitutions/12-yaml-merge.input.yaml | 6 + .../fixtures/yaml_util/includes/true.yaml | 2 + .../fixtures/yaml_util/includetest.yaml | 6 +- .../includetest_invalid_condition_type.yaml | 3 + .../yaml_util/includetest_no_file.yaml | 3 + tests/unit_tests/test_bundle.py | 2 +- tests/unit_tests/test_substitutions.py | 95 +++++++- tests/unit_tests/test_yaml_util.py | 220 +++++++++++++++--- 12 files changed, 420 insertions(+), 76 deletions(-) create mode 100644 tests/unit_tests/fixtures/yaml_util/includes/true.yaml create mode 100644 tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml create mode 100644 tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index 5ef7a699eb..d59f39b9c2 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -353,7 +353,7 @@ def resolve_include( strict_undefined: bool = True, errors: ErrList | None = None, ) -> Any: - """Resolve an include, substituting the filename if needed. + """Resolve an include, substituting the condition and filename if needed. Note: no path-traversal validation is performed on the resolved filename. A substitution that resolves to an absolute path will bypass the parent @@ -362,6 +362,19 @@ def resolve_include( values (including command-line substitutions), so path restrictions are an explicit non-goal here. """ + if isinstance(original_condition := include.condition, str): + condition = str( + _expand_substitutions( + original_condition, + path + ["condition"], + context_vars, + strict_undefined, + errors, + ) + ) + if condition != original_condition: + include = include.with_condition(condition) + original = include.file filename = str( _expand_substitutions( @@ -372,8 +385,8 @@ def resolve_include( if substituted: include = include.with_file(filename) try: - return include.load() - except esphome.core.EsphomeError as err: + return include.load() if include.should_load() else {} + except (esphome.core.EsphomeError, cv.Invalid) as err: resolved = f" (expanded from '{original}')" if substituted else "" raise cv.Invalid( f"Error including file '{filename}'{resolved}: {err}" diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 7c6cf691b9..e892664724 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -35,7 +35,7 @@ from esphome.core import ( TimePeriod, ) from esphome.expression import has_substitution_or_expression -from esphome.helpers import add_class_to_obj +from esphome.helpers import FALSY_BOOL_STRINGS, TRUTHY_BOOL_STRINGS, add_class_to_obj from esphome.util import OrderedDict, filter_yaml_files _LOGGER = logging.getLogger(__name__) @@ -75,6 +75,12 @@ def take_dropped_merge_keys() -> list[tuple[str, str]]: return CORE.data.pop(_MERGE_WARNINGS_KEY, []) +def _raise_invalid(message: str) -> None: + from voluptuous import Invalid + + raise Invalid(message) + + class SensitiveStr(str): """Marker subclass for validated strings that should be masked in user-visible YAML output. ``cv.sensitive`` wraps validated values in this @@ -224,16 +230,18 @@ class IncludeFile: Created during YAML parsing instead of loading the file immediately, allowing substitution variables to appear in the filename path - (e.g. ``!include device-${platform}.yaml``). The actual file is - loaded on the first call to ``load()``, and the result is cached. + (e.g. ``!include device-${platform}.yaml``) and in an optional condition. + The actual file is loaded on the first call to ``load()``, and the result + is cached. The client is responsible for testing the condition if needed. """ def __init__( self, parent_file: Path, file: str, - vars: dict[str, Any] | None, yaml_loader: Callable[[Path], Any], + vars: dict[str, Any] | None = None, + condition: bool | str | None = None, ) -> None: self.parent_file = parent_file # The raw include text may be a substitution/Jinja expression, so it @@ -241,8 +249,9 @@ class IncludeFile: # rewrites "/" to "\", which Jinja then decodes as escapes like # "\b" -> backspace (issue #18545). self.file = file - self.vars = vars self.yaml_loader = yaml_loader + self.vars = vars + self.condition = condition self._content: Any = _UNSET def __repr__(self) -> str: @@ -256,23 +265,62 @@ class IncludeFile: """ if self._content is not _UNSET: return self._content - if self.has_unresolved_expressions(): - from voluptuous import Invalid - - raise Invalid( + if self.has_unresolved_file(): + _raise_invalid( f"Cannot load include with unresolved substitutions: {self.file}" ) self._content = self.yaml_loader(self.parent_file.parent / self.file) self._content = add_context(self._content, self.vars) return self._content - def has_unresolved_expressions(self) -> bool: + def has_unresolved_file(self) -> bool: """Check if the filename contains substitution variables or Jinja expressions.""" return has_substitution_or_expression(self.file) + def should_load(self) -> bool: + """Evaluates the condition and returns True if the file should be loaded.""" + if self.condition is None: + return True + if isinstance(self.condition, bool): + return self.condition + if self.has_unresolved_condition(): + _raise_invalid( + f"Cannot evaluate include condition for '{self.file}' with unresolved substitutions: {self.condition}" + ) + value = self.condition.lower() + if value in TRUTHY_BOOL_STRINGS: + return True + if value not in FALSY_BOOL_STRINGS: + _raise_invalid( + f"Cannot convert include condition for '{self.file}' to a boolean, please use 'true' or 'false': {self.condition}" + ) + return False + + def has_unresolved_condition(self) -> bool: + """Check if the condition contains substitution variables or Jinja expressions.""" + return isinstance(self.condition, str) and has_substitution_or_expression( + self.condition + ) + def with_file(self, file: str) -> IncludeFile: """Clone this include with *file* as the filename.""" - return IncludeFile(self.parent_file, file, self.vars, self.yaml_loader) + return IncludeFile( + self.parent_file, + file, + self.yaml_loader, + vars=self.vars, + condition=self.condition, + ) + + def with_condition(self, condition: bool | str | None) -> IncludeFile: + """Clone this include with *condition* as the condition.""" + return IncludeFile( + self.parent_file, + self.file, + self.yaml_loader, + vars=self.vars, + condition=condition, + ) def _is_visible_path(rel: Path) -> bool: @@ -434,7 +482,7 @@ def force_load_include_files( if id(obj) in _seen: return _seen.add(id(obj)) - if obj.has_unresolved_expressions(): + if obj.has_unresolved_file(): _load_include_candidates( obj, warn_on_unresolved=warn_on_unresolved, @@ -578,14 +626,21 @@ def _resolve_merge_include(value: Any, node: yaml.Node, value_node: yaml.Node) - for _ in range(_MAX_MERGE_INCLUDE_DEPTH): if not isinstance(value, IncludeFile): break - if value.has_unresolved_expressions(): + if value.has_unresolved_file(): raise yaml.constructor.ConstructorError( "While constructing a mapping", node.start_mark, "Substitution in include filename with merge keys is not supported yet.", value_node.start_mark, ) - value = value.load() + if value.has_unresolved_condition(): + raise yaml.constructor.ConstructorError( + "While constructing a mapping", + node.start_mark, + "Substitution in include condition with merge keys is not supported yet.", + value_node.start_mark, + ) + value = value.load() if value.should_load() else {} else: raise yaml.constructor.ConstructorError( "While constructing a mapping", @@ -791,11 +846,11 @@ class ESPHomeLoaderMixin: @_add_data_ref def construct_include(self, node: yaml.Node) -> Any: - from esphome.const import CONF_VARS + from esphome.const import CONF_CONDITION, CONF_FILE, CONF_VARS - def extract_file_vars(node): + def extract_fields(node): fields = self.construct_yaml_map(node) - file = fields.get("file") + file = fields.get(CONF_FILE) if file is None: raise yaml.MarkedYAMLError("Must include 'file'", node.start_mark) if not isinstance(file, str): @@ -803,14 +858,21 @@ class ESPHomeLoaderMixin: "Include 'file' must be a string", node.start_mark ) vars = fields.get(CONF_VARS) - return file, vars + condition = fields.get(CONF_CONDITION) + if condition is not None and not isinstance(condition, (bool, str)): + raise yaml.MarkedYAMLError( + "Include 'condition' must be a boolean or string", node.start_mark + ) + return file, vars, condition if isinstance(node, yaml.nodes.MappingNode): - file, vars = extract_file_vars(node) + file, vars, condition = extract_fields(node) else: - file, vars = node.value, None + file, vars, condition = node.value, None, None - return IncludeFile(self.name, file, vars, self.yaml_loader) + return IncludeFile( + self.name, file, self.yaml_loader, vars=vars, condition=condition + ) # Directory includes (!include_dir_*) load eagerly during YAML parsing # because their paths are directory names, not individual files, and @@ -967,7 +1029,7 @@ def _load_yaml_internal(fname: Path, *, track_document_range: bool = True) -> An # Top-level !include returns a deferred IncludeFile; resolve it so # callers always receive the final content. if isinstance(res, IncludeFile): - res = res.load() + res = res.load() if res.should_load() else {} return res @@ -1358,8 +1420,12 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_scalar(tag="!remove", value=value.value) def represent_include_file(self, value): - if value.vars: - mapping = {"file": value.file, "vars": value.vars} + if value.vars or value.condition is not None: + mapping = {"file": value.file} + if value.vars: + mapping["vars"] = value.vars + if value.condition is not None: + mapping["condition"] = value.condition return self.represent_mapping( tag="!include", mapping=mapping, flow_style=False ) diff --git a/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml b/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml index a812fedcfd..e13db65ef7 100644 --- a/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml +++ b/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml @@ -13,3 +13,10 @@ test_list: - a: alpha - b: beta - c: The value of C is $c + - {} + - values: + - var1: '1' + - a: alpha + - b: delta + - c: The value of C is saw + - {} diff --git a/tests/unit_tests/fixtures/substitutions/01-include.input.yaml b/tests/unit_tests/fixtures/substitutions/01-include.input.yaml index d3daa681a4..b30ae5f99e 100644 --- a/tests/unit_tests/fixtures/substitutions/01-include.input.yaml +++ b/tests/unit_tests/fixtures/substitutions/01-include.input.yaml @@ -13,3 +13,22 @@ test_list: file: inc1.yaml vars: b: "beta" + condition: true + - !include + file: inc1.yaml + vars: + b: "gamma" + c: "saw" + condition: false + - !include + file: inc1.yaml + vars: + b: "delta" + c: "saw" + condition: ${var1 == "1"} + - !include + file: inc1.yaml + vars: + b: "epsilon" + c: "saw" + condition: ${var1 == "2"} diff --git a/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml b/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml index a03e66e393..3152f8ddf0 100644 --- a/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml +++ b/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml @@ -7,4 +7,10 @@ test_list: file: inc3.yaml vars: y: 8 + condition: true + <<: !include + file: inc3.yaml + vars: + y: 10 + condition: false after: Content after diff --git a/tests/unit_tests/fixtures/yaml_util/includes/true.yaml b/tests/unit_tests/fixtures/yaml_util/includes/true.yaml new file mode 100644 index 0000000000..abd17b22f7 --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includes/true.yaml @@ -0,0 +1,2 @@ +--- +true diff --git a/tests/unit_tests/fixtures/yaml_util/includetest.yaml b/tests/unit_tests/fixtures/yaml_util/includetest.yaml index 7b01da6a17..6db5c294f4 100644 --- a/tests/unit_tests/fixtures/yaml_util/includetest.yaml +++ b/tests/unit_tests/fixtures/yaml_util/includetest.yaml @@ -11,8 +11,8 @@ esphome: # should be substituted as 'original', # not overwritten by vars in the !include above name: ${name} - name_add_mac_suffix: true - libraries: !include {file: includes/list.yaml, vars: {var1: Wire}} + name_add_mac_suffix: !include includes/true.yaml + libraries: !include {file: includes/list.yaml, vars: {var1: Wire}, condition: "true"} esp8266: - board: !include {file: includes/scalar.yaml, vars: {var1: nodemcu}} + board: !include {file: includes/scalar.yaml, vars: {var1: nodemcu}, condition: true} diff --git a/tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml b/tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml new file mode 100644 index 0000000000..94eb7d65ab --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml @@ -0,0 +1,3 @@ +--- +esphome: + libraries: !include {file: includes/list.yaml, vars: {var1: Wire}, condition: 1} diff --git a/tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml b/tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml new file mode 100644 index 0000000000..7724bb301f --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml @@ -0,0 +1,3 @@ +--- +esphome: + libraries: !include {vars: {var1: Wire}} diff --git a/tests/unit_tests/test_bundle.py b/tests/unit_tests/test_bundle.py index 1abc7a3ab8..8a2510cf0a 100644 --- a/tests/unit_tests/test_bundle.py +++ b/tests/unit_tests/test_bundle.py @@ -1384,7 +1384,7 @@ def test_force_load_skips_duplicate_include_file() -> None: self.parent_file = Path("root.yaml") self.load_calls = 0 - def has_unresolved_expressions(self) -> bool: + def has_unresolved_file(self) -> bool: return False def load(self) -> dict[str, Any]: diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index b94a70443c..198b002d43 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -573,6 +573,65 @@ def test_undefined_variable_warning( assert "'undefined_var' is undefined" in caplog.text +def test_undefined_variable_warning_avoided_by_include_condition( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Compare two different ways to conditionally enable an !include file + that defines and uses new variables. One way generates a undefined + variable warning when disabled and the other does not.""" + include_file = tmp_path / "include.yaml" + include_file.write_text("substitutions:\n var: 1\nresult: ${var}\n") + method_a_file = tmp_path / "method_a.yaml" + method_a_file.write_text( + "packages:\n" + " - !include\n" + " file: include.yaml\n" + " condition: ${enable == 'y'}\n" + ) + method_b_file = tmp_path / "method_b.yaml" + method_b_file.write_text( + "substitutions:\n" + " content: !include include.yaml\n" + "packages:\n" + " - ${content if enable == 'y' else {}}\n" + ) + + def process_file(file, enable): + with caplog.at_level(logging.WARNING): + args = {"enable": enable} + config = yaml_util.load_yaml(file) + config = do_packages_pass(config, command_line_substitutions=args) + return substitutions.do_substitution_pass( + config, command_line_substitutions=args + ) + + # Case: method A with include enabled produces a result and no warnings + config = process_file(method_a_file, "y") + assert caplog.text == "" + assert config["packages"][0]["result"] == 1 + + # Case: method A with include disabled produces no result and no warnings + # because the file was not even loaded + config = process_file(method_a_file, "n") + assert caplog.text == "" + assert config["packages"][0] == {} + + # Case: method B with include enabled produces a result and no warnings (same as method A) + config = process_file(method_b_file, "y") + assert caplog.text == "" + assert config["packages"][0]["result"] == 1 + + # Case: method B with include disabled produces no result and an undefined variable warning + # because the file is loaded into a variable and its own variables cannot be expanded + # (different from method A) + config = process_file(method_b_file, "n") + assert ( + "Could not resolve substitution variable 'content': 'var' is undefined" + in caplog.text + ) + assert config["packages"][0] == {} + + def test_password_field_warnings_suppressed( caplog: pytest.LogCaptureFixture, ) -> None: @@ -717,11 +776,9 @@ def test_resolve_package_max_depth_exceeded(tmp_path: Path) -> None: # Each call to the loader returns a fresh IncludeFile pointing at itself, # so PACKAGE_SCHEMA always sees an IncludeFile and never a dict. def always_returns_include(path: Path) -> yaml_util.IncludeFile: - return yaml_util.IncludeFile(parent, path.name, None, always_returns_include) + return yaml_util.IncludeFile(parent, path.name, always_returns_include) - package_config = yaml_util.IncludeFile( - parent, "test.yaml", None, always_returns_include - ) + package_config = yaml_util.IncludeFile(parent, "test.yaml", always_returns_include) processor = _PackageProcessor({}, None) with pytest.raises( cv.Invalid, @@ -791,6 +848,22 @@ def test_include_filename_jinja_expression_with_path_separator( assert config["result"] == {"value": 42} +def test_include_condition_substitution_undefined_var(tmp_path: Path) -> None: + """!include with an undefined substitution variable raises cv.Invalid. + + The error message must reference the unresolved condition template so the + user knows which include failed, rather than seeing a bare file-not-found. + """ + main_file = tmp_path / "main.yaml" + main_file.write_text( + "result: !include\n file: file.yaml\n condition: ${undefined_var}\n" + ) + + config = yaml_util.load_yaml(main_file) + with pytest.raises(cv.Invalid, match=r"\$\{undefined_var\}"): + substitutions.do_substitution_pass(config) + + def test_raise_first_undefined_logs_extras_at_debug( caplog: pytest.LogCaptureFixture, ) -> None: @@ -838,7 +911,7 @@ def test_do_substitution_pass_included_substitutions_must_be_mapping( def loader(path: Path): return ["not", "a", "mapping"] - include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + include = yaml_util.IncludeFile(parent, "subs.yaml", loader) config = OrderedDict({CONF_SUBSTITUTIONS: include}) with pytest.raises( @@ -861,7 +934,7 @@ def test_do_packages_pass_included_substitutions_must_be_mapping( def loader(path: Path): return ["not", "a", "mapping"] - include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + include = yaml_util.IncludeFile(parent, "subs.yaml", loader) config = OrderedDict( { CONF_SUBSTITUTIONS: include, @@ -888,9 +961,7 @@ def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> No def loader(path: Path): raise EsphomeError(f"Error reading file {path}: No such file") - package_config = yaml_util.IncludeFile( - parent, "${undefined_var}.yaml", None, loader - ) + package_config = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", loader) processor = _PackageProcessor({}, None) with pytest.raises(cv.Invalid, match="unresolved substitutions"): processor.resolve_package(package_config, substitutions.ContextVars(), []) @@ -906,7 +977,7 @@ def test_resolve_include_error_shows_expanded_from_when_substituted( def failing_loader(_path: Path) -> None: raise EsphomeError("File not found") - include = yaml_util.IncludeFile(parent, "${device}.yaml", None, failing_loader) + include = yaml_util.IncludeFile(parent, "${device}.yaml", failing_loader) context = substitutions.ContextVars({"device": "my_device"}) with pytest.raises(cv.Invalid) as exc_info: @@ -927,7 +998,7 @@ def test_resolve_include_error_no_expanded_from_for_literal_filename( def failing_loader(_path: Path) -> None: raise EsphomeError("File not found") - include = yaml_util.IncludeFile(parent, "literal.yaml", None, failing_loader) + include = yaml_util.IncludeFile(parent, "literal.yaml", failing_loader) with pytest.raises(cv.Invalid) as exc_info: substitutions.resolve_include(include, [], substitutions.ContextVars()) @@ -946,7 +1017,7 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None: included.write_text('!lambda |-\n return "${foo}";\n') include = yaml_util.IncludeFile( - tmp_path / "main.yaml", "lambda.yaml", {"foo": "bar"}, yaml_util.load_yaml + tmp_path / "main.yaml", "lambda.yaml", yaml_util.load_yaml, vars={"foo": "bar"} ) config = OrderedDict({"value": include.load()}) result = substitutions.do_substitution_pass(config) diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index f3f73ea4dc..e1c3080e29 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -43,17 +43,37 @@ def clear_core_frontmatter() -> None: core.CORE.frontmatter = {} -def test_include_with_vars(fixture_path: Path) -> None: +def test_include(fixture_path: Path) -> None: + """Test !include with and without vars, with and without conditions""" yaml_file = fixture_path / "yaml_util" / "includetest.yaml" actual = yaml_util.load_yaml(yaml_file) actual = substitutions.do_substitution_pass(actual, None) assert actual["esphome"]["name"] == "original" + assert actual["esphome"]["name_add_mac_suffix"] assert actual["esphome"]["libraries"][0] == "Wire" assert actual["esp8266"]["board"] == "nodemcu" assert actual["wifi"]["ssid"] == "my_custom_ssid" +def test_include_with_no_file(fixture_path: Path) -> None: + """Ensure that an error is emitted when the file field is missing.""" + yaml_file = fixture_path / "yaml_util" / "includetest_no_file.yaml" + + with pytest.raises(EsphomeError, match=r"Must include 'file'"): + yaml_util.load_yaml(yaml_file) + + +def test_include_with_invalid_condition_type(fixture_path: Path) -> None: + """Ensure that an error is emitted when the condition field is an invalid type.""" + yaml_file = fixture_path / "yaml_util" / "includetest_invalid_condition_type.yaml" + + with pytest.raises( + EsphomeError, match=r"Include 'condition' must be a boolean or string" + ): + yaml_util.load_yaml(yaml_file) + + def test_loading_a_missing_file(fixture_path): """We throw EsphomeError when loading a missing file.""" yaml_file = fixture_path / "yaml_util" / "missing.yaml" @@ -64,7 +84,7 @@ def test_loading_a_missing_file(fixture_path): def test_parsing_with_custom_loader(fixture_path): """Test custom loader used for vscode connection - Default loader is tested in test_include_with_vars + Default loader is tested in test_include """ yaml_file = fixture_path / "yaml_util" / "includetest.yaml" @@ -78,10 +98,11 @@ def test_parsing_with_custom_loader(fixture_path): # substitute config to expand includes: substitutions.substitute(config, [], substitutions.ContextVars(), False) - assert len(loader_calls) == 3 + assert len(loader_calls) == 4 assert loader_calls[0].parts[-2:] == ("includes", "included.yaml") - assert loader_calls[1].parts[-2:] == ("includes", "list.yaml") - assert loader_calls[2].parts[-2:] == ("includes", "scalar.yaml") + assert loader_calls[1].parts[-2:] == ("includes", "true.yaml") + assert loader_calls[2].parts[-2:] == ("includes", "list.yaml") + assert loader_calls[3].parts[-2:] == ("includes", "scalar.yaml") def test_construct_secret_simple(fixture_path: Path) -> None: @@ -570,7 +591,7 @@ def test_represent_remove() -> None: def test_represent_include_file() -> None: """Test that IncludeFile objects are dumped as !include scalars.""" include = yaml_util.IncludeFile( - Path("/fake/main.yaml"), "path/to/file.yaml", None, lambda _: {} + Path("/fake/main.yaml"), "path/to/file.yaml", lambda _: {} ) assert yaml_util.dump({"key": include}) == "key: !include 'path/to/file.yaml'\n" @@ -580,13 +601,27 @@ def test_represent_include_file_with_vars() -> None: include = yaml_util.IncludeFile( Path("/fake/main.yaml"), "path/to/file.yaml", - {"key": "value"}, lambda _: {}, + vars={"key": "value"}, + ) + assert ( + yaml_util.dump({"key": include}) + == "key: !include\n file: path/to/file.yaml\n vars:\n key: value\n" + ) + + +def test_represent_include_file_with_condition() -> None: + """Test that IncludeFile with condition is dumped as !include mapping form.""" + include = yaml_util.IncludeFile( + Path("/fake/main.yaml"), + "path/to/file.yaml", + lambda _: {}, + condition="true", + ) + assert ( + yaml_util.dump({"key": include}) + == "key: !include\n file: path/to/file.yaml\n condition: 'true'\n" ) - result = yaml_util.dump({"key": include}) - assert "!include" in result - assert "file: path/to/file.yaml" in result - assert "key: value" in result def test_represent_include_file_with_data_base_mixin() -> None: @@ -596,7 +631,7 @@ def test_represent_include_file_with_data_base_mixin() -> None: subclass. add_multi_representer must match this subclass through the MRO. """ include = yaml_util.IncludeFile( - Path("/fake/main.yaml"), "common/spi.yaml", None, lambda _: {} + Path("/fake/main.yaml"), "common/spi.yaml", lambda _: {} ) wrapped = yaml_util.make_data_base(include) assert isinstance(wrapped, yaml_util.ESPHomeDataBase) @@ -609,7 +644,7 @@ def test_represent_include_file_with_data_base_mixin() -> None: def test_include_file_repr(tmp_path: Path) -> None: """repr() includes the filename so it appears usefully in error messages.""" parent = tmp_path / "main.yaml" - include = yaml_util.IncludeFile(parent, "some/nested.yaml", None, lambda _: {}) + include = yaml_util.IncludeFile(parent, "some/nested.yaml", lambda _: {}) assert repr(include) == "IncludeFile(some/nested.yaml)" @@ -624,7 +659,7 @@ def test_include_file_load_caches_result(tmp_path: Path) -> None: call_count += 1 return content - include = yaml_util.IncludeFile(parent, "child.yaml", None, counting_loader) + include = yaml_util.IncludeFile(parent, "child.yaml", counting_loader) first = include.load() second = include.load() @@ -641,7 +676,7 @@ def test_include_file_load_caches_none_result(tmp_path: Path) -> None: nonlocal call_count call_count += 1 - include = yaml_util.IncludeFile(parent, "empty.yaml", None, counting_loader) + include = yaml_util.IncludeFile(parent, "empty.yaml", counting_loader) first = include.load() second = include.load() @@ -653,7 +688,7 @@ def test_include_file_load_caches_none_result(tmp_path: Path) -> None: def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> None: """load() raises if the filename contains unresolved substitutions or expressions.""" parent = tmp_path / "main.yaml" - include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", None, lambda _: {}) + include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", lambda _: {}) with pytest.raises(cv.Invalid, match="unresolved"): include.load() @@ -670,13 +705,83 @@ def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> N ("price-100$.yaml", False), # $ at end, not followed by valid substitution ], ) -def test_include_file_has_unresolved_expressions( +def test_include_file_has_unresolved_file( tmp_path: Path, filename: str, expected: bool ) -> None: - """has_unresolved_expressions() detects substitution patterns in the filename.""" + """has_unresolved_file() detects substitution patterns in the filename.""" parent = tmp_path / "main.yaml" - include = yaml_util.IncludeFile(parent, filename, None, lambda _: {}) - assert include.has_unresolved_expressions() == expected + include = yaml_util.IncludeFile(parent, filename, lambda _: {}) + assert include.has_unresolved_file() == expected + + +@pytest.mark.parametrize( + ("condition", "expected"), + [ + (None, False), + (True, False), + (False, False), + ("true", False), + ("false", False), + ("$has_feature", True), # whole substitution + ("${has_feature}", True), # whole substitution + ("tr$ue", True), # partial substitution + ("$.", False), # malformed substitution + ("${1 == 1}", True), # Jinja expression + ("${", False), # malformed expression + ], +) +def test_include_file_has_unresolved_condition( + tmp_path: Path, condition: bool | str | None, expected: bool +) -> None: + """has_unresolved_condition() detects substitution patterns in the condition.""" + parent = tmp_path / "main.yaml" + include = yaml_util.IncludeFile( + parent, "device.yaml", lambda _: {}, condition=condition + ) + assert include.has_unresolved_condition() == expected + + +@pytest.mark.parametrize( + ("condition", "expected_result_or_error"), + [ + (None, True), + (True, True), + (False, False), + ("true", True), + ("True", True), + ("TRUE", True), + ("yes", True), + ("on", True), + ("enable", True), + ("false", False), + ("False", False), + ("FALSE", False), + ("no", False), + ("off", False), + ("disable", False), + ( + "$sub", + "Cannot evaluate include condition for 'device.yaml' with unresolved substitutions", + ), + ("", "Cannot convert include condition for 'device.yaml' to a boolean"), + ("trues", "Cannot convert include condition for 'device.yaml' to a boolean"), + ], +) +def test_include_file_should_load( + tmp_path: Path, + condition: bool | str | None, + expected_result_or_error: bool | str, +) -> None: + """should_load() evaluates the condition and raises an error if it is malformed.""" + parent = tmp_path / "main.yaml" + include = yaml_util.IncludeFile( + parent, "device.yaml", lambda _: {}, condition=condition + ) + if isinstance(expected_result_or_error, bool): + assert include.should_load() == expected_result_or_error + else: + with pytest.raises(cv.Invalid, match=expected_result_or_error): + include.should_load() def test_mapping_include_non_string_file_rejected(tmp_path: Path) -> None: @@ -691,16 +796,16 @@ def test_include_file_templated_filename_stays_raw_string(tmp_path: Path) -> Non """A templated filename keeps its verbatim text (issue #18545).""" parent = tmp_path / "main.yaml" expr = '${ "bluetooth/proxy.yaml" if enable_bluetooth_proxy else "../empty.yaml" }' - include = yaml_util.IncludeFile(parent, expr, None, lambda _: {}) + include = yaml_util.IncludeFile(parent, expr, lambda _: {}) assert include.file == expr - assert include.has_unresolved_expressions() + assert include.has_unresolved_file() assert repr(include) == f"IncludeFile({expr})" def test_represent_include_file_templated() -> None: """Dumping a templated IncludeFile emits the raw expression unchanged.""" expr = '${ "a/b.yaml" if flag else "../c.yaml" }' - include = yaml_util.IncludeFile(Path("/fake/main.yaml"), expr, None, lambda _: {}) + include = yaml_util.IncludeFile(Path("/fake/main.yaml"), expr, lambda _: {}) assert yaml_util.dump({"key": include}) == f"key: !include '{expr}'\n" @@ -710,12 +815,12 @@ def test_include_in_list_context() -> None: parent = Path("/fake/main.yaml") # The nested IncludeFile resolves to a plain string value - inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: "gamma") + inner = yaml_util.IncludeFile(parent, "inner.yaml", lambda _: "gamma") # The outer IncludeFile returns a list whose last element is itself an IncludeFile, # exercising the substitution pass's ability to recurse into loaded content. outer = yaml_util.IncludeFile( - parent, "items.yaml", None, lambda _: ["alpha", "beta", inner] + parent, "items.yaml", lambda _: ["alpha", "beta", inner] ) config = OrderedDict({"values": outer}) @@ -736,15 +841,48 @@ def test_top_level_include_resolved_by_load_yaml(tmp_path: Path) -> None: assert result["key"] == "value" +@pytest.mark.parametrize( + ("condition", "expected_result_or_error"), + [ + ("true", True), + ("false", False), + ('"TRUE"', True), + ('"FALSE"', False), + ('"x"', "Cannot convert include condition for 'child.yaml' to a boolean"), + ( + "$sub", + "Cannot evaluate include condition for 'child.yaml' with unresolved substitutions", + ), + ], +) +def test_top_level_include_with_condition_resolved_by_load_yaml( + tmp_path: Path, condition: bool | str | None, expected_result_or_error: bool | str +) -> None: + """load_yaml evaluates the condition for a top-level !include.""" + child = tmp_path / "child.yaml" + child.write_text("key: value\n") + main = tmp_path / "main.yaml" + main.write_text(f"!include {{ file: child.yaml, condition: {condition} }}\n") + + if isinstance(expected_result_or_error, bool): + result = yaml_util.load_yaml(main) + assert isinstance(result, dict) + if expected_result_or_error: + assert result["key"] == "value" + else: + assert result == {} + else: + with pytest.raises(cv.Invalid, match=expected_result_or_error): + result = yaml_util.load_yaml(main) + + def test_include_plain_filename_loads_after_deferred_refactor() -> None: """!include with a plain filename (no $ expressions) still loads correctly. Regression guard: the deferred-loading refactor must not break the simple case. """ parent = Path("/fake/main.yaml") - include = yaml_util.IncludeFile( - parent, "child.yaml", None, lambda _: {"answer": 42} - ) + include = yaml_util.IncludeFile(parent, "child.yaml", lambda _: {"answer": 42}) config = OrderedDict({"result": include}) config = substitutions.do_substitution_pass(config) @@ -756,7 +894,23 @@ def test_yaml_merge_include_with_filename_substitution_raises() -> None: """<<: !include ${expr} raises a clear error — substitutions in merge-key filenames are not yet supported, and the error message must say so.""" yaml_text = "base:\n existing: value\n <<: !include ${filename}.yaml\n" - with pytest.raises(EsphomeError, match="not supported yet"): + with pytest.raises( + EsphomeError, + match="Substitution in include filename with merge keys is not supported yet", + ): + yaml_util.parse_yaml( + Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {} + ) + + +def test_yaml_merge_include_with_condition_substitution_raises() -> None: + """<<: !include { file: ${expr}, condition: {} } raises a clear error — substitutions in merge-key conditions + are not yet supported, and the error message must say so.""" + yaml_text = "base:\n existing: value\n <<: !include\n file: filename.yaml\n condition: ${expr}\n" + with pytest.raises( + EsphomeError, + match="Substitution in include condition with merge keys is not supported yet", + ): yaml_util.parse_yaml( Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {} ) @@ -775,8 +929,8 @@ def test_yaml_merge_chain_include_resolves() -> None: """Chained includes in merge keys resolve through multiple IncludeFile layers.""" parent = Path("/fake/main.yaml") - inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: {"x": 1}) - outer = yaml_util.IncludeFile(parent, "outer.yaml", None, lambda _: inner) + inner = yaml_util.IncludeFile(parent, "inner.yaml", lambda _: {"x": 1}) + outer = yaml_util.IncludeFile(parent, "outer.yaml", lambda _: inner) yaml_text = "base:\n existing: value\n <<: !include outer.yaml\n" config = yaml_util.parse_yaml(parent, io.StringIO(yaml_text), lambda _: outer) @@ -791,7 +945,7 @@ def test_yaml_merge_chain_include_depth_exceeded() -> None: parent = Path("/fake/main.yaml") def self_referencing_loader(path: Path) -> yaml_util.IncludeFile: - return yaml_util.IncludeFile(parent, path.name, None, self_referencing_loader) + return yaml_util.IncludeFile(parent, path.name, self_referencing_loader) yaml_text = "base:\n <<: !include loop.yaml\n" with pytest.raises(EsphomeError, match="Maximum include chain depth"): @@ -1061,7 +1215,7 @@ class _StubInclude: self._raise = raise_on_load self.load_calls = 0 - def has_unresolved_expressions(self) -> bool: + def has_unresolved_file(self) -> bool: return self._unresolved def load(self) -> object: From 3fd8f00e97a4e63bfd7c0c3260530b0e9ade29d9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 07:31:41 -0500 Subject: [PATCH 048/221] [socket] Compile the lookup and client link helpers only when required (#20002) --- esphome/components/socket/__init__.py | 16 +++++++- esphome/core/defines.h | 2 + tests/components/socket/__init__.py | 3 ++ .../ipv4_resolve_test_component/__init__.py | 2 + .../__init__.py | 2 + .../socket/test_socket_source_filter.py | 38 +++++++++++++++++++ 6 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 tests/unit_tests/components/socket/test_socket_source_filter.py diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index edebed5d40..a814f46e3b 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -140,6 +140,17 @@ def require_wake_loop_threadsafe() -> None: cg.add_define("USE_SOCKET_SELECT_SUPPORT") +def require_ipv4_resolve() -> None: + """Compile the shared IPv4 lookup; call from a consumer's to_code.""" + cg.add_define("USE_SOCKET_IPV4_RESOLVE") + + +def require_tcp_client_link() -> None: + """Compile the reconnecting TCP client link; call from a consumer's to_code.""" + require_ipv4_resolve() + cg.add_define("USE_SOCKET_TCP_CLIENT_LINK") + + CONFIG_SCHEMA = cv.Schema( { cv.SplitDefault( @@ -184,11 +195,14 @@ async def to_code(config: ConfigType) -> None: # Each implementation file is fully #ifdef'd on the define set in to_code -# for the selected implementation. +# for the selected implementation. The helper files compile only for +# consumers that called the matching require_ function. FILTER_SOURCE_FILES = filter_source_files_from_defines( { "lwip_raw_tcp_impl.cpp": "USE_SOCKET_IMPL_LWIP_TCP", "bsd_sockets_impl.cpp": "USE_SOCKET_IMPL_BSD_SOCKETS", "lwip_sockets_impl.cpp": "USE_SOCKET_IMPL_LWIP_SOCKETS", + "ipv4_resolve.cpp": "USE_SOCKET_IPV4_RESOLVE", + "tcp_client_link.cpp": "USE_SOCKET_TCP_CLIENT_LINK", } ) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9b3252fa05..88c82b0d7c 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -417,6 +417,8 @@ #define USE_SENDSPIN_VISUALIZER #define USE_SENDSPIN_PORT 8928 // NOLINT #define USE_SOCKET_IMPL_BSD_SOCKETS +#define USE_SOCKET_IPV4_RESOLVE +#define USE_SOCKET_TCP_CLIENT_LINK #define USE_LWIP_FAST_SELECT #define USE_SPEAKER diff --git a/tests/components/socket/__init__.py b/tests/components/socket/__init__.py index 8a3da97e23..6c1d2b1c08 100644 --- a/tests/components/socket/__init__.py +++ b/tests/components/socket/__init__.py @@ -2,6 +2,7 @@ # shadowing the stdlib module -- no test_*.py (Python-level) tests can live here. # required for cpp unit testing. import esphome.codegen as cg +from esphome.components import socket as socket_component from tests.testing_helpers import ComponentManifestOverride @@ -12,5 +13,7 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: async def to_code_testing(config): await real_to_code(config) cg.add_define("USE_NETWORK_IPV6", True) + # The gtests link against the filtered helper files. + socket_component.require_tcp_client_link() manifest.to_code = to_code_testing diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py index 3e397d6186..aee9e3ec13 100644 --- a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py @@ -1,4 +1,5 @@ import esphome.codegen as cg +from esphome.components import socket import esphome.config_validation as cv from esphome.const import CONF_ID from esphome.types import ConfigType @@ -18,5 +19,6 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: + socket.require_ipv4_resolve() var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py index 8a7703d025..24637ae586 100644 --- a/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py @@ -1,4 +1,5 @@ import esphome.codegen as cg +from esphome.components import socket from esphome.components.const import CONF_HOST import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_PORT @@ -28,6 +29,7 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: + socket.require_tcp_client_link() var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) cg.add(var.set_host(config[CONF_HOST])) diff --git a/tests/unit_tests/components/socket/test_socket_source_filter.py b/tests/unit_tests/components/socket/test_socket_source_filter.py new file mode 100644 index 0000000000..3967c9568d --- /dev/null +++ b/tests/unit_tests/components/socket/test_socket_source_filter.py @@ -0,0 +1,38 @@ +"""Tests for the socket component's source filtering and require functions.""" + +from unittest.mock import patch + +from esphome.components import socket +from esphome.core import Define + + +def test_helper_files_filtered_until_required() -> None: + """ipv4_resolve.cpp and tcp_client_link.cpp compile only when required.""" + with patch("esphome.config_helpers.CORE") as mock_core: + mock_core.defines = set() + filtered = socket.FILTER_SOURCE_FILES() + assert "ipv4_resolve.cpp" in filtered + assert "tcp_client_link.cpp" in filtered + + mock_core.defines = {Define("USE_SOCKET_IPV4_RESOLVE")} + filtered = socket.FILTER_SOURCE_FILES() + assert "ipv4_resolve.cpp" not in filtered + assert "tcp_client_link.cpp" in filtered + + mock_core.defines = { + Define("USE_SOCKET_IPV4_RESOLVE"), + Define("USE_SOCKET_TCP_CLIENT_LINK"), + } + filtered = socket.FILTER_SOURCE_FILES() + assert "ipv4_resolve.cpp" not in filtered + assert "tcp_client_link.cpp" not in filtered + + +def test_require_tcp_client_link_pulls_in_the_resolver() -> None: + """require_tcp_client_link() sets both defines; the link reads the resolver.""" + with patch.object(socket.cg, "add_define") as add_define: + socket.require_tcp_client_link() + assert {call.args[0] for call in add_define.call_args_list} == { + "USE_SOCKET_IPV4_RESOLVE", + "USE_SOCKET_TCP_CLIENT_LINK", + } From a58f3d34aa543724ec37dcb3646209c45eac4094 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 07:59:36 -0500 Subject: [PATCH 049/221] [core] Compile util.cpp only on MQTT builds (#20003) --- esphome/core/config.py | 15 +++++++++++++-- esphome/core/util.cpp | 17 +++++------------ esphome/core/util.h | 8 ++++++++ tests/unit_tests/core/test_config.py | 17 +++++++++++++++++ 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/esphome/core/config.py b/esphome/core/config.py index 281d430410..532ca7c36d 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -7,7 +7,10 @@ from pathlib import Path from esphome import automation, core import esphome.codegen as cg -from esphome.config_helpers import filter_source_files_from_platform +from esphome.config_helpers import ( + filter_source_files_from_defines, + filter_source_files_from_platform, +) import esphome.config_validation as cv from esphome.const import ( CONF_AREA, @@ -897,7 +900,7 @@ async def to_code(config: ConfigType) -> None: # Platform-specific source files for core -FILTER_SOURCE_FILES = filter_source_files_from_platform( +_filter_platform_sources = filter_source_files_from_platform( { "static_task.cpp": { PlatformFramework.ESP32_ARDUINO, @@ -950,3 +953,11 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( # as they are only included when needed by the preprocessor } ) + +# util.cpp holds only the out-of-line MQTT connectivity helpers; without +# USE_MQTT the header provides inline stubs and the file must not compile. +_filter_define_sources = filter_source_files_from_defines({"util.cpp": "USE_MQTT"}) + + +def FILTER_SOURCE_FILES() -> list[str]: + return _filter_platform_sources() + _filter_define_sources() diff --git a/esphome/core/util.cpp b/esphome/core/util.cpp index 54a7956163..4bca130e4a 100644 --- a/esphome/core/util.cpp +++ b/esphome/core/util.cpp @@ -1,23 +1,16 @@ #include "esphome/core/util.h" -#include "esphome/core/application.h" -#include "esphome/core/version.h" -#include "esphome/core/log.h" +// Compiled only on MQTT builds (core FILTER_SOURCE_FILES); without USE_MQTT +// the header provides inline stubs and this file must stay empty, so the +// guard protects builds that compile every source, such as clang-tidy. #ifdef USE_MQTT #include "esphome/components/mqtt/mqtt_client.h" -#endif namespace esphome { -bool mqtt_is_connected() { -#ifdef USE_MQTT - if (mqtt::global_mqtt_client != nullptr) { - return mqtt::global_mqtt_client->is_connected(); - } -#endif - return false; -} +bool mqtt_is_connected() { return mqtt::global_mqtt_client != nullptr && mqtt::global_mqtt_client->is_connected(); } bool remote_is_connected() { return api_is_connected() || mqtt_is_connected(); } } // namespace esphome +#endif diff --git a/esphome/core/util.h b/esphome/core/util.h index 8f90aa3411..aa3098ba65 100644 --- a/esphome/core/util.h +++ b/esphome/core/util.h @@ -25,9 +25,17 @@ ESPHOME_ALWAYS_INLINE inline bool api_is_connected() { return false; } #endif /// Return whether the node has an active connection to an MQTT broker +#ifdef USE_MQTT bool mqtt_is_connected(); +#else +inline bool mqtt_is_connected() { return false; } +#endif /// Return whether the node has any form of "remote" connection via the API or to an MQTT broker +#ifdef USE_MQTT bool remote_is_connected(); +#else +inline bool remote_is_connected() { return api_is_connected(); } +#endif } // namespace esphome diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index cfb9651e6b..7fbc2d1b39 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1493,3 +1493,20 @@ async def test_add_platformio_options_native_arduino( assert "board_build.ldscript is ignored" in caplog.text assert "'arduino' toolchain" in caplog.text assert "upload_speed" not in caplog.text + + +def test_filter_source_files_drops_util_cpp_without_mqtt() -> None: + """util.cpp compiles only on MQTT builds; the header stubs it otherwise.""" + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp8266", + KEY_TARGET_FRAMEWORK: "arduino", + } + CORE.defines = set() + + excluded = config.FILTER_SOURCE_FILES() + assert "util.cpp" in excluded + # The platform map still contributes through the composed function. + assert "static_task.cpp" in excluded + + CORE.defines = {core.Define("USE_API"), core.Define("USE_MQTT")} + assert "util.cpp" not in config.FILTER_SOURCE_FILES() From 797682cccc66a1bf7a7e35bafa23e750fb447001 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 07:59:52 -0500 Subject: [PATCH 050/221] [espidf] Skip the lwip sources that compile to empty objects (#20000) --- esphome/build_gen/espidf.py | 39 ++++++++ esphome/espidf/toolchain.py | 6 +- script/check_idf_py_equivalence.py | 57 ++++++++++- tests/script/test_check_idf_py_equivalence.py | 94 +++++++++++++++++++ tests/unit_tests/build_gen/test_espidf.py | 18 ++++ 5 files changed, 211 insertions(+), 3 deletions(-) diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 50c0abf1b2..42a1de5a4b 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -66,6 +66,43 @@ else() "app edits will regenerate sections.ld.") endif()""" +# lwip sources that compile to empty objects with the option off (their own +# #if guard). (option, regex valid for both Python and CMake); a source is +# only dropped when its option is defined and off, so a renamed option +# keeps it. +LWIP_EMPTY_SOURCES: tuple[tuple[str, str], ...] = ( + ("CONFIG_LWIP_PPP_SUPPORT", "/netif/ppp/"), + ("CONFIG_LWIP_IPV6", "/core/ipv6/"), + ("CONFIG_LWIP_AUTOIP", "/core/ipv4/autoip[.]c$"), + ("CONFIG_LWIP_STATS", "/core/stats[.]c$"), +) +# Drift guard only: keep every lwip source. +LWIP_FULL_SOURCES_ENV = "ESPHOME_LWIP_FULL_SOURCES" + +# Drops the empty objects after project(), once the lwip target exists. +_LWIP_EMPTY_SOURCES_FILTER = f"""\ +idf_build_get_property(esphome_build_components BUILD_COMPONENTS) +if(lwip IN_LIST esphome_build_components AND NOT DEFINED ENV{{{LWIP_FULL_SOURCES_ENV}}}) + idf_component_get_property(esphome_lwip_lib lwip COMPONENT_LIB) + get_target_property(esphome_lwip_srcs ${{esphome_lwip_lib}} SOURCES) +@FILTERS@ + set_property(TARGET ${{esphome_lwip_lib}} PROPERTY SOURCES ${{esphome_lwip_srcs}}) +endif()""" + + +def lwip_empty_source_gate(option: str, regex: str) -> str: + return ( + f" if(DEFINED {option} AND NOT {option})\n" + f' list(FILTER esphome_lwip_srcs EXCLUDE REGEX "{regex}")\n' + " endif()" + ) + + +def _lwip_empty_sources_filter() -> str: + gates = "\n".join(lwip_empty_source_gate(*entry) for entry in LWIP_EMPTY_SOURCES) + return _LWIP_EMPTY_SOURCES_FILTER.replace("@FILTERS@", gates) + + # Runs after project() so the walk has happened; catches the remaining # silent path where the top-level out-var was renamed. _LDGEN_OVERRIDE_CHECK = """\ @@ -348,6 +385,8 @@ project({CORE.name}) {ldgen_override_check} +{_lwip_empty_sources_filter()} + # Emit per-memory-type JSON size data for ESPHome to read post-build. # json2 stays small; raw dumps every symbol (~2s on a large map) and # this command runs inside the link edge, blocking everything downstream. diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 4202ee71ce..effb817f35 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -366,7 +366,9 @@ def _tool_env() -> dict[str, str]: return env -def run_reconfigure(verbose: bool = False) -> int: +def run_reconfigure( + verbose: bool = False, extra_env: dict[str, str] | None = None +) -> int: """Run the CMake configure, with the arguments idf.py uses.""" build_dir = _build_dir() build_dir.mkdir(parents=True, exist_ok=True) @@ -389,7 +391,7 @@ def run_reconfigure(verbose: bool = False) -> int: rc = run_build_tool( cmd, cwd=build_dir, - env=_tool_env(), + env={**_tool_env(), **(extra_env or {})}, filter_lines=None if verbose else FILTER_IDF_LINES, log_path=log_path, ) diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index 8ef3577eed..cbd55d57cc 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -59,6 +59,15 @@ VERSION_DRIFT = ( "ESPHome reads ESP-IDF version {ours!r} from {source} but idf_tools reports " "{theirs!r}; update read_idf_version_{source} in esphome/espidf/framework.py" ) +LWIP_NOT_EMPTY = ( + "lwip source {source} compiles to a non-empty object with {option} off; " + "drop it from LWIP_EMPTY_SOURCES in esphome/build_gen/espidf.py" +) +LWIP_NOTHING_MATCHED = ( + "no lwip object matched {regex!r} for {option}; the lwip layout or the " + "pattern in esphome/build_gen/espidf.py changed" +) +LWIP_NM_FAILED = "nm failed on lwip object {source}: {error}" WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld") DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*" @@ -108,6 +117,51 @@ def _log_problems( return problems +def _lwip_empty_source_problems(build_path: Path) -> list[str]: + """Compile the lwip sources the generated CMakeLists drops; any with + symbols is a problem. Leaves the tree configured with every source.""" + # pylint: disable=protected-access + from esphome.build_gen.espidf import LWIP_EMPTY_SOURCES, LWIP_FULL_SOURCES_ENV + from esphome.espidf import toolchain + + if (rc := toolchain.run_reconfigure(extra_env={LWIP_FULL_SOURCES_ENV: "1"})) != 0: + return [f"CMake configure with every lwip source failed with exit code {rc}"] + if rc := toolchain._run_ninja("esp-idf/lwip/liblwip.a", verbose=False, jobs=None): + return [f"building every lwip source failed with exit code {rc}"] + build = build_path / "build" + config = json.loads( + (build / "config" / "sdkconfig.json").read_text(encoding="utf-8") + ) + objects = [ + obj.as_posix().removesuffix(".obj") + for obj in (build / "esp-idf" / "lwip").rglob("*.obj") + ] + nm = toolchain._parse_cmakecache(build / "CMakeCache.txt")["CMAKE_NM"] + problems = [] + for option, regex in LWIP_EMPTY_SOURCES: + # Absent means the option is invisible here; the filter keeps those. + if config.get(option.removeprefix("CONFIG_"), True): + continue + matched = [source for source in objects if re.search(regex, source)] + if not matched: + problems.append(LWIP_NOTHING_MATCHED.format(regex=regex, option=option)) + for source in matched: + name = Path(source).name + result = subprocess.run( + [nm, "--defined-only", f"{source}.obj"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode: + problems.append( + LWIP_NM_FAILED.format(source=name, error=result.stderr.strip()) + ) + elif result.stdout.strip(): + problems.append(LWIP_NOT_EMPTY.format(source=name, option=option)) + return problems + + def _setup_core(build_path: Path, description: dict) -> tuple[str, str]: """Point CORE at the tree so ESPHome resolves the same IDF env as the build.""" from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT @@ -202,7 +256,8 @@ def check(build_path: Path) -> list[str]: problems.append(f"idf.py dropped {out} from {log}") elif mtimes_before.get(key) != mtimes_after[key]: problems.append(f"idf.py rebuilt {out}") - return problems + # Last: it reconfigures the tree, which would otherwise relink above. + return problems or _lwip_empty_source_problems(build_path) def main() -> int: diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index 9ce52a6df5..f2b798ff21 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -103,6 +103,7 @@ def _run_check( patch.object(framework, "read_idf_version_txt", return_value=versions[0]), patch.object(framework, "read_idf_version_header", return_value=versions[1]), patch.object(framework, "idf_tools_version", return_value=versions[2]), + patch.object(guard, "_lwip_empty_source_problems", return_value=[]), patch.object(guard.subprocess, "run", side_effect=run), patch.dict(os.environ), ): @@ -319,6 +320,99 @@ def test_main_without_build_trees( assert "No native ESP-IDF build tree found" in capsys.readouterr().out +def _make_lwip_tree(tmp_path: Path, objects: list[str], config: dict) -> Path: + """A tree with lwip objects, their sdkconfig.json and an nm in the cache.""" + tree = _make_tree(tmp_path) + objdir = tree / "build" / "esp-idf" / "lwip" / "CMakeFiles" / "__idf_lwip.dir" + for name in objects: + (objdir / name).parent.mkdir(parents=True, exist_ok=True) + (objdir / name).write_bytes(b"x") + (tree / "build" / "config").mkdir(parents=True, exist_ok=True) + (tree / "build" / "config" / "sdkconfig.json").write_text(json.dumps(config)) + with (tree / "build" / "CMakeCache.txt").open("a") as cache: + cache.write("CMAKE_NM:FILEPATH=/tools/nm\n") + return tree + + +def _run_lwip_check( + tree: Path, + non_empty: set[str] = frozenset(), + failing: set[str] = frozenset(), + calls: list[list[str]] | None = None, +) -> list[str]: + """Run the lwip check with nm faked; ``calls`` collects the nm commands.""" + + def run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess: + if calls is not None: + calls.append(cmd) + name = Path(cmd[-1]).name + if name in failing: + return subprocess.CompletedProcess(cmd, 1, "", "bad object") + return subprocess.CompletedProcess( + cmd, 0, "symbol\n" if name in non_empty else "", "" + ) + + with ( + patch.object(toolchain, "run_reconfigure", return_value=0) as reconfigure, + patch.object(toolchain, "_run_ninja", return_value=0), + patch.object(guard.subprocess, "run", side_effect=run), + ): + problems = guard._lwip_empty_source_problems(tree) + reconfigure.assert_called_once_with( + extra_env={build_gen.LWIP_FULL_SOURCES_ENV: "1"} + ) + return problems + + +def test_lwip_check_inspects_only_the_dropped_sources(tmp_path: Path) -> None: + """An option that is on, or absent (invisible), keeps its sources unchecked.""" + tree = _make_lwip_tree( + tmp_path, + [ + "lwip/src/netif/ppp/auth.c.obj", + "lwip/src/core/ipv6/ip6.c.obj", + "lwip/src/core/ipv4/autoip.c.obj", + ], + {"LWIP_PPP_SUPPORT": False, "LWIP_IPV6": True}, + ) + calls: list[list[str]] = [] + assert _run_lwip_check(tree, calls=calls) == [] + assert [Path(c[-1]).name for c in calls] == ["auth.c.obj"] + + +def test_lwip_check_flags_a_dropped_source_with_symbols(tmp_path: Path) -> None: + tree = _make_lwip_tree( + tmp_path, ["lwip/src/netif/ppp/auth.c.obj"], {"LWIP_PPP_SUPPORT": False} + ) + assert _run_lwip_check(tree, non_empty={"auth.c.obj"}) == [ + guard.LWIP_NOT_EMPTY.format(source="auth.c", option="CONFIG_LWIP_PPP_SUPPORT") + ] + + +def test_lwip_check_flags_a_failed_nm(tmp_path: Path) -> None: + """A broken nm must not pass as an empty object.""" + tree = _make_lwip_tree( + tmp_path, ["lwip/src/netif/ppp/auth.c.obj"], {"LWIP_PPP_SUPPORT": False} + ) + assert _run_lwip_check(tree, failing={"auth.c.obj"}) == [ + guard.LWIP_NM_FAILED.format(source="auth.c", error="bad object") + ] + + +def test_lwip_check_fails_per_pattern_that_matched_nothing(tmp_path: Path) -> None: + """A stale pattern is reported even while the others still match.""" + tree = _make_lwip_tree( + tmp_path, + ["lwip/src/netif/ppp/auth.c.obj"], + {"LWIP_PPP_SUPPORT": False, "LWIP_STATS": False}, + ) + assert _run_lwip_check(tree) == [ + guard.LWIP_NOTHING_MATCHED.format( + regex="/core/stats[.]c$", option="CONFIG_LWIP_STATS" + ) + ] + + @pytest.mark.parametrize(("problems", "rc"), [([], 0), (["idf.py changed x"], 1)]) def test_main_reports_each_tree( tmp_path: Path, diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index f8e834a78d..2c95c61b37 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -222,6 +222,24 @@ def test_get_project_cmakelists_size_command_uses_json2() -> None: assert "--format=json2" in content +def test_get_project_cmakelists_drops_empty_lwip_sources() -> None: + """The filter comes after project(), where the lwip target exists.""" + from esphome.build_gen.espidf import ( + LWIP_EMPTY_SOURCES, + LWIP_FULL_SOURCES_ENV, + lwip_empty_source_gate, + ) + + content = _render() + filter_at = content.index( + "set_property(TARGET ${esphome_lwip_lib} PROPERTY SOURCES" + ) + assert filter_at > content.index("project(") + assert f"NOT DEFINED ENV{{{LWIP_FULL_SOURCES_ENV}}}" in content + for entry in LWIP_EMPTY_SOURCES: + assert lwip_empty_source_gate(*entry) in content + + def test_get_project_cmakelists_declares_map_as_link_byproduct() -> None: """The link declares the map so size can build in the same ninja run.""" content = _render() From d970e9292e941c01bc8b5523c706f85599879463 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 08:00:10 -0500 Subject: [PATCH 051/221] [esphome][ota][core] Compress OTA uploads on ESP32, RP2040, LibreTiny and host (#19037) --- esphome/__main__.py | 7 +- esphome/components/esphome/ota/__init__.py | 15 +- .../components/esphome/ota/ota_esphome.cpp | 232 +++++--- esphome/components/esphome/ota/ota_esphome.h | 66 +++ .../esphome/ota/ota_esphome_inflate.c | 498 ++++++++++++++++++ .../esphome/ota/ota_esphome_inflate.h | 65 +++ .../ota/ota_esphome_inflate_session.cpp | 94 ++++ .../esphome/ota/ota_esphome_noise.cpp | 11 +- esphome/components/ota/ota_backend.h | 7 +- .../ota/ota_backend_arduino_libretiny.h | 2 +- .../components/ota/ota_backend_arduino_rp2.h | 5 +- esphome/components/ota/ota_backend_esp8266.h | 2 +- esphome/components/ota/ota_backend_esp_idf.h | 2 +- esphome/components/ota/ota_backend_factory.h | 5 +- esphome/components/ota/ota_backend_host.h | 2 +- esphome/core/defines.h | 1 + esphome/espota2.py | 38 +- script/build_helpers.py | 17 +- script/ci-custom.py | 2 + tests/components/esphome/__init__.py | 12 + tests/components/esphome/test_ota_inflate.cpp | 324 ++++++++++++ tests/components/main.cpp | 5 + tests/components/ota/test.bk72xx-ard.yaml | 1 + .../components/ota/test_backend_contract.cpp | 2 +- .../fixtures/host_ota_deflate.yaml | 9 + tests/integration/test_host_ota.py | 78 +++ tests/unit_tests/test_espota2.py | 51 +- 27 files changed, 1440 insertions(+), 113 deletions(-) create mode 100644 esphome/components/esphome/ota/ota_esphome_inflate.c create mode 100644 esphome/components/esphome/ota/ota_esphome_inflate.h create mode 100644 esphome/components/esphome/ota/ota_esphome_inflate_session.cpp create mode 100644 tests/components/esphome/__init__.py create mode 100644 tests/components/esphome/test_ota_inflate.cpp create mode 100644 tests/components/ota/test.bk72xx-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_deflate.yaml diff --git a/esphome/__main__.py b/esphome/__main__.py index cbeb9ff01c..40457dbcc2 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1300,10 +1300,9 @@ def _choose_ota_platform(config: ConfigType, requested: str | None) -> str: The native API uses challenge-response auth with MD5/SHA256 hashing of a server-issued nonce, so the password is never sent over the wire; the ``web_server`` path uses HTTP Basic auth which transmits credentials in - cleartext over the LAN. (The native path also supports gzip compression - on ESP8266, where flash space is tight; on ESP32/RP2040/LibreTiny the - backend reports ``supports_compression() == false`` and the firmware is - sent uncompressed regardless of which platform is used.) Falls back to + cleartext over the LAN. (The native path also compresses the upload: + gzip on ESP8266 and RP2040, which inflate it at reboot, and a deflate + stream on ESP32/LibreTiny, which inflate it as it arrives.) Falls back to ``web_server`` only when that is the only available platform. """ # Use a dict (insertion-ordered) instead of a list so error messages and diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index e14f5e7e86..6ecae24e29 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -315,10 +315,19 @@ FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate FILTER_SOURCE_FILES = filter_source_files_from_defines( - {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} + { + "ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION", + "ota_esphome_inflate_session.cpp": "USE_OTA_DEFLATE", + "ota_esphome_inflate.c": "USE_OTA_DEFLATE", + } ) +def enable_deflate() -> None: + """Compile the on-the-fly inflater for compressed uploads.""" + cg.add_define("USE_OTA_DEFLATE") + + @coroutine_with_priority(CoroPriority.OTA_UPDATES) async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) @@ -340,6 +349,10 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") + # ESP8266 and RP2040 inflate gzip at reboot; the rest inflate on the fly + if not (CORE.is_esp8266 or CORE.is_rp2): + enable_deflate() + # One key per device: an api encryption block supplies it (static or # runtime) and offers; the ota block only adds the requirement api_conf = CORE.config.get(CONF_API) or {} diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 22580343e8..c399c0901c 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -22,8 +22,10 @@ #include "esphome/core/lwip_fast_select.h" #endif +#include #include #include +#include #include namespace esphome { @@ -47,6 +49,8 @@ static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds // practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits // longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; +static constexpr uint32_t OTA_PROGRESS_INTERVAL_MS = 1000; +static constexpr size_t OTA_SIZE_FIELD_BYTES = 4; // sizes on the wire are 4 bytes MSB first // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) @@ -194,12 +198,23 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_DEFLATE = 0x10; // Noise needs the extended protocol: the prologue binds the 2-byte feature ack static constexpr uint8_t CLIENT_NOISE_FEATURES = CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +// Raw deflate, window <= OTA_INFLATE_WINDOW_SIZE. Binding once offered: the +// client must then send the image size frame and a deflate stream. +static constexpr uint8_t SERVER_FEATURE_SUPPORTS_DEFLATE = 0x08; + +#ifdef USE_OTA_ENCRYPTION +inline bool ESPHomeOTAComponent::noise_offered_() const { + return (this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES; +} +#endif inline bool ESPHomeOTAComponent::extended_proto_() const { #ifdef USE_OTA_ENCRYPTION_REQUIRED @@ -305,7 +320,7 @@ void ESPHomeOTAComponent::handle_handshake_() { this->transition_ota_state_(OTAState::FEATURE_ACK); const bool supports_compression = - (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_COMPRESSION) != 0 && this->backend_->supports_compression(); + (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_COMPRESSION) != 0 && ota::OTABackend::supports_compression(); // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte @@ -325,6 +340,28 @@ void ESPHomeOTAComponent::handle_handshake_() { #elif defined(USE_OTA_ENCRYPTION) // A yaml key always exists: validation rejects the all-zeros key this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; +#endif +#ifdef USE_OTA_ENCRYPTION + // Reserve the noise session before the optional inflate buffer, so the + // required allocation is not starved by the compression window + if (this->noise_offered_()) { + this->noise_reserve_session_(); + } +#endif +#ifdef USE_OTA_DEFLATE + // Offered only once the session memory is in hand; else uncompressed + if ((this->ota_features_ & CLIENT_FEATURE_SUPPORTS_DEFLATE) != 0) { + // Value initialized: a corrupt stream that back references the + // window before it is filled then copies zeros, never stale memory. + // Default placement, PSRAM first where present: the session lives for one + // upload and keeps 4.9 KB of internal heap free while it runs + this->inflate_ = RAMAllocator().make_unique(); + if (this->inflate_ != nullptr) { + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_DEFLATE; + } else { + ESP_LOGW(TAG, "No memory to inflate"); + } + } #endif } else { this->handshake_buf_[0] = @@ -343,8 +380,7 @@ void ESPHomeOTAComponent::handle_handshake_() { #ifdef USE_OTA_ENCRYPTION // Latch the offer actually sent: a key activating between the two // states must not start a session the client never expects - if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && - (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { + if (this->noise_offered_()) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -442,16 +478,11 @@ void ESPHomeOTAComponent::handle_data_() { // Backend calls overwrite this with OK; reset to UNKNOWN before any // goto error that follows a successful begin()/write() ota::OTAResponseTypes error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; - size_t total = 0; - uint32_t last_progress = 0; - uint32_t last_data_ms = 0; + DataTransfer xfer; uint8_t buf[OTA_BUFFER_SIZE]; char *sbuf = reinterpret_cast(buf); - size_t ota_size; + size_t image_size; ota::OTAType ota_type = ota::OTA_TYPE_UPDATE_APP; -#if USE_OTA_VERSION == 2 - size_t size_acknowledged = 0; -#endif // Set socket timeouts and blocking mode (see strategy table above) struct timeval tv; @@ -480,16 +511,20 @@ void ESPHomeOTAComponent::handle_data_() { ESP_LOGV(TAG, "OTA type is 0x%02x", ota_type); // Read size, 4 bytes MSB first - if (!this->data_readall_(buf, 4)) { + if (!this->read_size_(buf, xfer.ota_size, LOG_STR("size"))) { // The first request byte is the type on the extended protocol; a close after it was a cut-off request if (!this->extended_proto_() && this->client_left_before_start_()) return; this->log_read_error_(LOG_STR("size")); goto error; // NOLINT(cppcoreguidelines-avoid-goto) } - ota_size = (static_cast(buf[0]) << 24) | (static_cast(buf[1]) << 16) | - (static_cast(buf[2]) << 8) | buf[3]; - ESP_LOGV(TAG, "Size is %zu bytes", ota_size); + image_size = xfer.ota_size; +#ifdef USE_OTA_DEFLATE + if (this->inflate_ != nullptr && !this->read_size_(buf, image_size, LOG_STR("image size"))) { + this->log_read_error_(LOG_STR("image size")); + goto error; // NOLINT(cppcoreguidelines-avoid-goto) + } +#endif #ifndef USE_OTA_PARTITIONS if (ota_type != ota::OTA_TYPE_UPDATE_APP) { @@ -509,7 +544,7 @@ void ESPHomeOTAComponent::handle_data_() { #endif // begin() returns quickly; flash sectors are erased incrementally during write(). - error_code = this->backend_->begin(ota_size, ota_type); + error_code = this->backend_->begin(image_size, ota_type); if (error_code != ota::OTA_RESPONSE_OK) goto error; // NOLINT(cppcoreguidelines-avoid-goto) @@ -529,77 +564,25 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge MD5 OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_BIN_MD5_OK); - // Track when we last received data so a silently-vanished peer (no FIN/RST - // delivered, e.g. uploader killed mid-transfer or NAT/router dropped state) - // can't wedge the device indefinitely. Without this, the loop only exits - // on actual data, EOF, or a non-EWOULDBLOCK error from read(), and lwIP - // TCP keepalive isn't enabled here. - last_data_ms = millis(); - while (total < ota_size) { - if (millis() - last_data_ms > OTA_SOCKET_TIMEOUT_DATA) { - ESP_LOGW(TAG, "No data received for %u ms", (unsigned) OTA_SOCKET_TIMEOUT_DATA); - error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + xfer.last_data_ms = millis(); +#ifdef USE_OTA_DEFLATE + if (this->inflate_ != nullptr) { + error_code = this->inflate_data_(buf, image_size, xfer); + if (error_code != ota::OTA_RESPONSE_OK) goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } - size_t remaining = ota_size - total; - size_t requested = remaining < OTA_BUFFER_SIZE ? remaining : OTA_BUFFER_SIZE; - ssize_t read; -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ != nullptr) { - // One frame per call; noise_read_data_ waits internally (readall_), so - // there is no would-block retry here and failures are already logged. - read = this->noise_read_data_(buf, requested); - if (read <= 0) { - if (this->remote_closed_) - this->log_remote_closed_(LOG_STR("data")); + } else +#endif + { + while (xfer.total < xfer.ota_size) { + ssize_t read = this->receive_data_(buf, xfer); + if (read < 0) { error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; goto error; // NOLINT(cppcoreguidelines-avoid-goto) } - } else -#endif - { - read = this->client_->read(buf, requested); - if (read == -1) { - const int err = errno; - if (this->would_block_(err)) { - // read() already waited up to SO_RCVTIMEO for data, just feed WDT - App.feed_wdt(); - continue; - } - ESP_LOGW(TAG, "Read err %d", err); - error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + error_code = this->write_flash_(buf, read); + if (error_code != ota::OTA_RESPONSE_OK) goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } else if (read == 0) { - ESP_LOGW(TAG, "Remote closed"); - error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; - goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } - } - - last_data_ms = millis(); - error_code = this->backend_->write(buf, read); - if (error_code != ota::OTA_RESPONSE_OK) { - ESP_LOGW(TAG, "Flash write err %d", error_code); - goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } - total += read; -#if USE_OTA_VERSION == 2 - while (size_acknowledged + OTA_BLOCK_SIZE <= total || (total == ota_size && size_acknowledged < ota_size)) { - this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK); - size_acknowledged += OTA_BLOCK_SIZE; - } -#endif - - uint32_t now = millis(); - if (now - last_progress > 1000) { - last_progress = now; - float percentage = (total * 100.0f) / ota_size; - ESP_LOGD(TAG, "Progress: %0.1f%%", percentage); -#ifdef USE_OTA_STATE_LISTENER - this->notify_state_(ota::OTA_IN_PROGRESS, percentage, 0); -#endif - // feed watchdog and give other tasks a chance to run - this->yield_and_feed_watchdog_(); + this->ack_written_(xfer); } } @@ -815,6 +798,90 @@ bool ESPHomeOTAComponent::try_write_(size_t to_write, const LogString *desc) { return this->handshake_buf_pos_ >= to_write; } +bool ESPHomeOTAComponent::read_size_(uint8_t *buf, size_t &size, const LogString *desc) { + if (!this->data_readall_(buf, OTA_SIZE_FIELD_BYTES)) + return false; + size = encode_uint32(buf[0], buf[1], buf[2], buf[3]); + ESP_LOGV(TAG, "%s is %zu bytes", LOG_STR_ARG(desc), size); + return true; +} + +ota::OTAResponseTypes ESPHomeOTAComponent::write_flash_(uint8_t *data, size_t len) { + ota::OTAResponseTypes result = this->backend_->write(data, len); + if (result != ota::OTA_RESPONSE_OK) { + ESP_LOGW(TAG, "Flash write err %d", result); + } + return result; +} + +ssize_t ESPHomeOTAComponent::receive_data_(uint8_t *buf, DataTransfer &xfer) { + const size_t remaining = xfer.ota_size - xfer.total; + const size_t requested = std::min(remaining, OTA_BUFFER_SIZE); + ssize_t read; + for (;;) { + // A silently-vanished peer (no FIN/RST delivered, e.g. uploader killed + // mid-transfer or NAT/router dropped state) must not wedge the device: + // read() only fails on EOF or a real error, and lwIP TCP keepalive isn't + // enabled here. + if (millis() - xfer.last_data_ms > OTA_SOCKET_TIMEOUT_DATA) { + ESP_LOGW(TAG, "No data received for %u ms", (unsigned) OTA_SOCKET_TIMEOUT_DATA); + return -1; + } +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) { + // One frame per call; noise_read_data_ waits internally (readall_), so + // there is no would-block retry here and failures are already logged. + read = this->noise_read_data_(buf, requested); + if (read <= 0) { + if (this->remote_closed_) + this->log_remote_closed_(LOG_STR("data")); + return -1; + } + break; + } +#endif + read = this->client_->read(buf, requested); + if (read > 0) + break; + if (read == 0) { + this->log_remote_closed_(LOG_STR("data")); + return -1; + } + if (!this->would_block_(errno)) { + this->log_socket_error_(LOG_STR("data")); + return -1; + } + // read() already waited up to SO_RCVTIMEO for data, just feed WDT + App.feed_wdt(); + } + + const uint32_t now = millis(); + xfer.last_data_ms = now; + xfer.total += read; + this->ack_received_(xfer); + if (now - xfer.last_progress > OTA_PROGRESS_INTERVAL_MS) { + xfer.last_progress = now; + float percentage = (xfer.total * 100.0f) / xfer.ota_size; + ESP_LOGD(TAG, "Progress: %0.1f%%", percentage); +#ifdef USE_OTA_STATE_LISTENER + this->notify_state_(ota::OTA_IN_PROGRESS, percentage, 0); +#endif + // feed watchdog and give other tasks a chance to run + this->yield_and_feed_watchdog_(); + } + return read; +} + +void ESPHomeOTAComponent::send_chunk_acks_(DataTransfer &xfer) { +#if USE_OTA_VERSION == 2 + while (xfer.acknowledged + OTA_BLOCK_SIZE <= xfer.total || + (xfer.total == xfer.ota_size && xfer.acknowledged < xfer.ota_size)) { + this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK); + xfer.acknowledged += OTA_BLOCK_SIZE; + } +#endif +} + void ESPHomeOTAComponent::cleanup_connection_() { this->client_->close(); this->client_ = nullptr; @@ -829,6 +896,9 @@ void ESPHomeOTAComponent::cleanup_connection_() { #endif #ifdef USE_OTA_ENCRYPTION this->noise_ = nullptr; +#endif +#ifdef USE_OTA_DEFLATE + this->inflate_ = nullptr; #endif // Intentionally no disable_loop() — letting loop() run one more iteration catches // any connection that queued on the listener mid-session (otherwise the wake flag, diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index 6f04b78da5..910a5b3d99 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -7,6 +7,9 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise_handshake.h" #endif +#ifdef USE_OTA_DEFLATE +#include "ota_esphome_inflate.h" +#endif #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/preferences.h" @@ -89,6 +92,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { // The api server's live context when it exists, otherwise our own (a build // time key, or the saved key loaded in safe mode) const noise::NoiseContext &noise_context_() const; + // True once the feature ack offers noise and the client asked for it + bool noise_offered_() const; + void noise_reserve_session_(); bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); @@ -120,6 +126,38 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { return this->readall_(buf, len); } + // Upload accounting shared by the data loop and the inflate read callback + struct DataTransfer { + size_t ota_size{0}; // bytes the client sends + size_t total{0}; // bytes received so far +#if USE_OTA_VERSION == 2 + size_t acknowledged{0}; +#endif + uint32_t last_data_ms{0}; + uint32_t last_progress{0}; + }; + // Up to OTA_BUFFER_SIZE bytes into buf; returns bytes read, -1 on failure (logged) + ssize_t receive_data_(uint8_t *buf, DataTransfer &xfer); + // Raw lwIP cannot service the radio during a sector write, so the ack waits + // for the write there; a socket task lets the next block arrive meanwhile +#ifdef USE_SOCKET_IMPL_LWIP_TCP + static constexpr bool ACK_AFTER_WRITE = true; +#else + static constexpr bool ACK_AFTER_WRITE = false; +#endif + void send_chunk_acks_(DataTransfer &xfer); + inline void ack_received_(DataTransfer &xfer) { + if (!ACK_AFTER_WRITE) + this->send_chunk_acks_(xfer); + } + inline void ack_written_(DataTransfer &xfer) { + if (ACK_AFTER_WRITE) + this->send_chunk_acks_(xfer); + } + inline bool read_size_(uint8_t *buf, size_t &size, const LogString *desc); + // Writes to the backend and logs a failure + ota::OTAResponseTypes write_flash_(uint8_t *data, size_t len); + bool try_read_(size_t to_read, const LogString *desc); bool try_write_(size_t to_write, const LogString *desc); @@ -175,6 +213,34 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { static_assert(OTA_BUFFER_SIZE >= NOISE_CLIENT_MAX_PLAINTEXT + noise::MAC_SIZE, "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif +#ifdef USE_OTA_DEFLATE + // At least 1 << espota2.DEFLATE_WINDOW_BITS; also the inflate output buffer + static constexpr size_t OTA_INFLATE_WINDOW_SIZE = 4096; + // Heap-allocated only while a deflate upload is negotiated; the decoder + // state is the base so the read callback can recover the session + struct InflateSession : OtaInflateState { + // The session outlives the upload it serves, but these three are borrowed + // from inflate_data_'s caller and dangle once that call returns; only that + // call, and the flush and read callback it drives, may read them + ESPHomeOTAComponent *self; + DataTransfer *xfer; + uint8_t *in; // caller's buffer for the compressed input + size_t image_size; + size_t written; // inflated bytes in flash + size_t flushed; // bytes of the current window already in flash + ota::OTAResponseTypes error; // first failure inside the read callback + uint8_t window[OTA_INFLATE_WINDOW_SIZE]; + }; +#ifndef CLANG_TIDY // static analysis sets every define at once + static_assert(!ota::OTABackend::supports_compression(), + "USE_OTA_DEFLATE is for backends that cannot store a gzip image"); +#endif + // Writes the decoded bytes not yet in flash without moving dest + ota::OTAResponseTypes inflate_flush_(InflateSession &session); + ota::OTAResponseTypes inflate_data_(uint8_t *in, size_t image_size, DataTransfer &xfer); + RAMUniquePtr inflate_; +#endif + static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; // Derived from the feature byte; storing it would pad the trailing bytes bool extended_proto_() const; diff --git a/esphome/components/esphome/ota/ota_esphome_inflate.c b/esphome/components/esphome/ota/ota_esphome_inflate.c new file mode 100644 index 0000000000..1208f82ae8 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_inflate.c @@ -0,0 +1,498 @@ +/* + * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) + * + * Copyright (c) 2003 by Joergen Ibsen / Jibz + * All Rights Reserved + * http://www.ibsensoftware.com/ + * + * Copyright (c) 2014-2018 by Paul Sokolovsky + * + * This software is provided 'as-is', without any express + * or implied warranty. In no event will the authors be + * held liable for any damages arising from the use of + * this software. + * + * Permission is granted to anyone to use this software + * for any purpose, including commercial applications, + * and to alter it and redistribute it freely, subject to + * the following restrictions: + * + * 1. The origin of this software must not be + * misrepresented; you must not claim that you + * wrote the original software. If you use this + * software in a product, an acknowledgment in + * the product documentation would be appreciated + * but is not required. + * + * 2. Altered source versions must be plainly marked + * as such, and must not be misrepresented as + * being the original software. + * + * 3. This notice may not be removed or altered from + * any source distribution. + */ + +/* + * Altered for ESPHome: this is the raw deflate decoder from uzlib's + * tinflate.c (v2.9.5) with the gzip/zlib header parsers, checksums, + * runtime table builder and in-memory (non ring window) output path + * removed, and the public names prefixed with ota_inflate. + */ + +#include "ota_esphome_inflate.h" + +#include + +#define TINF_OK OTA_INFLATE_OK +#define TINF_DONE OTA_INFLATE_DONE +#define TINF_DATA_ERROR OTA_INFLATE_DATA_ERROR +#define TINF_DICT_ERROR OTA_INFLATE_DICT_ERROR +#define TINF_DATA struct OtaInflateState +#define TINF_TREE struct OtaInflateTree +#define TINF_ARRAY_SIZE(arr) (sizeof(arr) / sizeof(*(arr))) + +/* every output byte also goes into the ring window */ +#define TINF_PUT(d, c) \ + { \ + *d->dest++ = c; \ + d->dict_ring[d->dict_idx++] = c; \ + if (d->dict_idx == d->dict_size) \ + d->dict_idx = 0; \ + } + +/* --------------------------------------------------- * + * -- constant tables (upstream builds them at runtime) -- * + * --------------------------------------------------- */ + +static const unsigned char LENGTH_BITS[30] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, + 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5}; +static const unsigned short LENGTH_BASE[30] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, + 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258}; + +static const unsigned char DIST_BITS[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13}; +static const unsigned short DIST_BASE[30] = {1, 2, 3, 4, 5, 7, 9, 13, 17, 25, + 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, + 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577}; + +/* special ordering of code length codes */ +static const unsigned char CLCIDX[] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + +/* ----------------------- * + * -- utility functions -- * + * ----------------------- */ + +/* given an array of code lengths, build a tree */ +static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned int num) { + unsigned short offs[16]; + unsigned int i, sum; + + /* clear code length count table */ + for (i = 0; i < 16; ++i) + t->table[i] = 0; + + /* scan symbol lengths, and sum code length counts */ + for (i = 0; i < num; ++i) + t->table[lengths[i]]++; + + /* In the lengths array, 0 means unused code. So, t->table[0] now contains + number of unused codes. But table's purpose is to contain # of codes of + particular length, and there're 0 codes of length 0. */ + t->table[0] = 0; + + /* compute offset table for distribution sort */ + for (sum = 0, i = 0; i < 16; ++i) { + offs[i] = sum; + sum += t->table[i]; + } + + /* create code->symbol translation table (symbols sorted by code) */ + for (i = 0; i < num; ++i) { + if (lengths[i]) + t->trans[offs[lengths[i]]++] = i; + } +} + +/* ---------------------- * + * -- decode functions -- * + * ---------------------- */ + +static unsigned char uzlib_get_byte(TINF_DATA *d) { + /* If end of source buffer is not reached, return next byte from source + buffer. */ + if (d->source < d->source_limit) { + return *d->source++; + } + + /* Otherwise if there's callback and we haven't seen EOF yet, try to + read next byte using it. (Note: the callback can also update ->source + and ->source_limit). */ + if (!d->eof) { + int val = d->source_read_cb(d); + if (val >= 0) { + return (unsigned char) val; + } + } + + /* Otherwise, we hit EOF (either from ->source_read_cb() or from exhaustion + of the buffer), and it will be "sticky", i.e. further calls to this + function will end up here too. */ + d->eof = true; + + return 0; +} + +/* get one bit from source stream */ +static int tinf_getbit(TINF_DATA *d) { + unsigned int bit; + + /* check if tag is empty */ + if (!d->bitcount--) { + /* load next tag */ + d->tag = uzlib_get_byte(d); + d->bitcount = 7; + } + + /* shift bit out of tag */ + bit = d->tag & 0x01; + d->tag >>= 1; + + return bit; +} + +/* read a num bit value from a stream and add base */ +static unsigned int tinf_read_bits(TINF_DATA *d, int num, int base) { + unsigned int val = 0; + + /* read num bits */ + if (num) { + unsigned int limit = 1 << (num); + unsigned int mask; + + for (mask = 1; mask < limit; mask *= 2) + if (tinf_getbit(d)) + val += mask; + } + + return val + base; +} + +/* given a data stream and a tree, decode a symbol */ +static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) { + int sum = 0, cur = 0, len = 0; + + /* get more bits while code value is above sum */ + do { + cur = 2 * cur + tinf_getbit(d); + + if (++len == TINF_ARRAY_SIZE(t->table)) { + return TINF_DATA_ERROR; + } + + sum += t->table[len]; + cur -= t->table[len]; + + } while (cur >= 0); + + sum += cur; + if (sum < 0 || sum >= t->size) { + return TINF_DATA_ERROR; + } + + return t->trans[sum]; +} + +/* given a data stream, decode dynamic trees from it */ +static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) { + /* code lengths for 288 literal/len symbols and 32 dist symbols */ + unsigned char lengths[288 + 32]; + unsigned int hlit, hdist, hclen, hlimit; + unsigned int i, num, length; + + /* get 5 bits HLIT (257-286) */ + hlit = tinf_read_bits(d, 5, 257); + + /* get 5 bits HDIST (1-32) */ + hdist = tinf_read_bits(d, 5, 1); + + /* get 4 bits HCLEN (4-19) */ + hclen = tinf_read_bits(d, 4, 4); + + for (i = 0; i < 19; ++i) + lengths[i] = 0; + + /* read code lengths for code length alphabet */ + for (i = 0; i < hclen; ++i) { + /* get 3 bits code length (0-7) */ + unsigned int clen = tinf_read_bits(d, 3, 0); + + lengths[CLCIDX[i]] = clen; + } + + /* build code length tree, temporarily use length tree */ + tinf_build_tree(lt, lengths, 19); + + /* decode code lengths for the dynamic trees */ + hlimit = hlit + hdist; + for (num = 0; num < hlimit;) { + int sym = tinf_decode_symbol(d, lt); + unsigned char fill_value = 0; + int lbits, lbase = 3; + + /* error decoding */ + if (sym < 0) + return sym; + + switch (sym) { + case 16: + /* copy previous code length 3-6 times (read 2 bits) */ + if (num == 0) + return TINF_DATA_ERROR; + fill_value = lengths[num - 1]; + lbits = 2; + break; + case 17: + /* repeat code length 0 for 3-10 times (read 3 bits) */ + lbits = 3; + break; + case 18: + /* repeat code length 0 for 11-138 times (read 7 bits) */ + lbits = 7; + lbase = 11; + break; + default: + /* values 0-15 represent the actual code lengths */ + lengths[num++] = sym; + /* continue the for loop */ + continue; + } + + /* special code length 16-18 are handled here */ + length = tinf_read_bits(d, lbits, lbase); + if (num + length > hlimit) + return TINF_DATA_ERROR; + for (; length; --length) { + lengths[num++] = fill_value; + } + } + + /* Check that there's "end of block" symbol */ + if (lengths[256] == 0) { + return TINF_DATA_ERROR; + } + + /* build dynamic trees */ + tinf_build_tree(lt, lengths, hlit); + tinf_build_tree(dt, lengths + hlit, hdist); + + return TINF_OK; +} + +/* build the fixed huffman trees (RFC 1951 3.2.6) through the generic tree + builder; altered from upstream, which unrolls them by hand */ +static void tinf_build_fixed_trees(TINF_TREE *lt, TINF_TREE *dt) { + unsigned char lengths[288]; + unsigned int i; + + for (i = 0; i < 144; ++i) + lengths[i] = 8; + for (; i < 256; ++i) + lengths[i] = 9; + for (; i < 280; ++i) + lengths[i] = 7; + for (; i < 288; ++i) + lengths[i] = 8; + tinf_build_tree(lt, lengths, 288); + + for (i = 0; i < 32; ++i) + lengths[i] = 5; + tinf_build_tree(dt, lengths, 32); +} + +/* ----------------------------- * + * -- block inflate functions -- * + * ----------------------------- */ + +/* given a stream and two trees, inflate next chunk of output (a byte or more) */ +static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) { + if (d->curlen == 0) { + unsigned int offs; + int dist; + int sym = tinf_decode_symbol(d, lt); + + if (d->eof) { + return TINF_DATA_ERROR; + } + + if (sym < 0) { + return sym; + } + + /* literal byte */ + if (sym < 256) { + TINF_PUT(d, sym); + return TINF_OK; + } + + /* end of block */ + if (sym == 256) { + return TINF_DONE; + } + + /* substring from sliding dictionary */ + sym -= 257; + if (sym >= 29) { + return TINF_DATA_ERROR; + } + + /* possibly get more bits from length code */ + d->curlen = tinf_read_bits(d, LENGTH_BITS[sym], LENGTH_BASE[sym]); + + dist = tinf_decode_symbol(d, dt); + if (dist < 0 || dist >= 30) { + return TINF_DATA_ERROR; + } + + /* possibly get more bits from distance code */ + offs = tinf_read_bits(d, DIST_BITS[dist], DIST_BASE[dist]); + + /* calculate and validate actual LZ offset to use */ + if (offs > d->dict_size) { + return TINF_DICT_ERROR; + } + /* Note: we don't try to catch offset which points to not yet filled + part of the dictionary here. Doing so would require keeping another + variable to track "filled in" size of the dictionary. Appearance of + such an offset cannot lead to accessing memory outside of the + dictionary buffer, and clients which don't want to leak unrelated + information, should explicitly initialize dictionary buffer passed + to uzlib. */ + + d->lz_off = d->dict_idx - offs; + if (d->lz_off < 0) { + d->lz_off += d->dict_size; + } + } + + /* copy next byte from dict substring */ + TINF_PUT(d, d->dict_ring[d->lz_off]); + if ((unsigned) ++d->lz_off == d->dict_size) { + d->lz_off = 0; + } + d->curlen--; + return TINF_OK; +} + +/* inflate next byte from uncompressed block of data */ +static int tinf_inflate_uncompressed_block(TINF_DATA *d) { + if (d->curlen == 0) { + unsigned int length, invlength; + + /* get length */ + length = uzlib_get_byte(d); + length += 256 * uzlib_get_byte(d); + /* get one's complement of length */ + invlength = uzlib_get_byte(d); + invlength += 256 * uzlib_get_byte(d); + /* check length */ + if (length != (~invlength & 0x0000ffff)) + return TINF_DATA_ERROR; + + /* increment length to properly return TINF_DONE below, without + producing data at the same time */ + d->curlen = length + 1; + + /* make sure we start next block on a byte boundary */ + d->bitcount = 0; + } + + if (--d->curlen == 0) { + return TINF_DONE; + } + + unsigned char c = uzlib_get_byte(d); + TINF_PUT(d, c); + return TINF_OK; +} + +/* ---------------------- * + * -- public functions -- * + * ---------------------- */ + +/* initialize decompression structure */ +void ota_inflate_init(TINF_DATA *d, unsigned char *dict, unsigned int dict_len) { + d->source = NULL; + d->source_limit = NULL; + d->tag = 0; + d->eof = 0; + d->bitcount = 0; + d->lz_off = 0; + d->bfinal = 0; + d->btype = -1; + d->dict_size = dict_len; + d->dict_ring = dict; + d->dict_idx = 0; + d->curlen = 0; + d->ltree.trans = d->ltrans; + d->ltree.size = TINF_ARRAY_SIZE(d->ltrans); + d->dtree.trans = d->dtrans; + d->dtree.size = TINF_ARRAY_SIZE(d->dtrans); +} + +/* inflate next output bytes from compressed stream */ +int ota_inflate(TINF_DATA *d) { + do { + int res; + + /* start a new block */ + if (d->btype == -1) { + int old_btype; + next_blk: + old_btype = d->btype; + /* read final block flag */ + d->bfinal = tinf_getbit(d); + /* read block type (2 bits) */ + d->btype = tinf_read_bits(d, 2, 0); + + if (d->btype == 1 && old_btype != 1) { + /* build fixed huffman trees */ + tinf_build_fixed_trees(&d->ltree, &d->dtree); + } else if (d->btype == 2) { + /* decode trees from stream */ + res = tinf_decode_trees(d, &d->ltree, &d->dtree); + if (res != TINF_OK) { + return res; + } + } + } + + /* process current block */ + switch (d->btype) { + case 0: + /* decompress uncompressed block */ + res = tinf_inflate_uncompressed_block(d); + break; + case 1: + case 2: + /* decompress block with fixed/dynamic huffman trees */ + /* trees were decoded previously, so it's the same routine for both */ + res = tinf_inflate_block_data(d, &d->ltree, &d->dtree); + break; + default: + return TINF_DATA_ERROR; + } + + if (res == TINF_DONE && !d->bfinal) { + /* the block has ended (without producing more data), but we + can't return without data, so start procesing next block */ + goto next_blk; + } + + if (res != TINF_OK) { + return res; + } + + } while (d->dest < d->dest_limit); + + return TINF_OK; +} diff --git a/esphome/components/esphome/ota/ota_esphome_inflate.h b/esphome/components/esphome/ota/ota_esphome_inflate.h new file mode 100644 index 0000000000..9726508a30 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_inflate.h @@ -0,0 +1,65 @@ +#pragma once +// Raw deflate decoder cut down from uzlib (https://github.com/pfalcon/uzlib, +// zlib licence, see the .c file); output goes through a ring window. + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum OtaInflateResult { + OTA_INFLATE_OK = 0, /* more data produced, call again */ + OTA_INFLATE_DONE = 1, /* end of compressed stream reached */ + OTA_INFLATE_DATA_ERROR = -3, + OTA_INFLATE_DICT_ERROR = -5, +}; + +struct OtaInflateTree { + uint16_t table[16]; /* table of code length counts */ + uint16_t *trans; /* code -> symbol translation table, size entries */ + uint16_t size; +}; + +struct OtaInflateState { + /* Next byte in the input buffer and one past its end */ + const unsigned char *source; + const unsigned char *source_limit; + /* Called when source is exhausted; returns the next byte or -1 at EOF. + It may refill source/source_limit for buffered operation. */ + int (*source_read_cb)(struct OtaInflateState *d); + /* Output cursor and one past the end of the output buffer */ + unsigned char *dest; + unsigned char *dest_limit; + /* Ring window holding the last dict_size output bytes for back references */ + unsigned char *dict_ring; + + unsigned int tag; + unsigned int curlen; + int lz_off; + unsigned int dict_size; + unsigned int dict_idx; + + /* One word: btype is -1 between blocks, bitcount never exceeds 7 */ + int8_t btype; + uint8_t bfinal; + uint8_t bitcount; + bool eof; + + struct OtaInflateTree ltree; /* dynamic length/symbol tree */ + struct OtaInflateTree dtree; /* dynamic distance tree */ + uint16_t ltrans[288]; + uint16_t dtrans[32]; /* the distance alphabet has 30 symbols, so the tree is kept small */ +}; + +/* dict must cover the encoder's window (its largest back reference) */ +void ota_inflate_init(struct OtaInflateState *d, unsigned char *dict, unsigned int dict_len); +/* Fills dest up to dest_limit (OK) or to the end of the stream (DONE). dest may + alias dict only if dest_limit - dest == dict_len and dest is reset to dict + exactly when a call returns OK, so the ring index and dest stay in lockstep */ +int ota_inflate(struct OtaInflateState *d); + +#ifdef __cplusplus +} +#endif diff --git a/esphome/components/esphome/ota/ota_esphome_inflate_session.cpp b/esphome/components/esphome/ota/ota_esphome_inflate_session.cpp new file mode 100644 index 0000000000..f3005cac48 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_inflate_session.cpp @@ -0,0 +1,94 @@ +#include "ota_esphome.h" +#ifdef USE_OTA +#ifdef USE_OTA_DEFLATE +#include "esphome/components/ota/ota_backend.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +namespace esphome { + +static const char *const TAG = "esphome.ota"; + +// The window doubles as the output buffer; flushed bytes stay as back +// reference history for the next windowful. +ota::OTAResponseTypes ESPHomeOTAComponent::inflate_flush_(InflateSession &session) { + const size_t produced = session.dest - session.window; + const size_t pending = produced - session.flushed; + if (pending != 0) { + if (pending > session.image_size - session.written) { + ESP_LOGW(TAG, "Inflate overrun"); + return ota::OTA_RESPONSE_ERROR_UNKNOWN; + } + ota::OTAResponseTypes result = this->write_flash_(session.window + session.flushed, pending); + if (result != ota::OTA_RESPONSE_OK) + return result; + session.flushed = produced; + session.written += pending; + // A compressible region yields many windows per socket read + App.feed_wdt(); + } + // Even with nothing new written: a block boundary can fall inside a header + this->ack_written_(*session.xfer); + return ota::OTA_RESPONSE_OK; +} + +ota::OTAResponseTypes ESPHomeOTAComponent::inflate_data_(uint8_t *in, size_t image_size, DataTransfer &xfer) { + InflateSession &session = *this->inflate_; + session.self = this; + session.xfer = &xfer; + session.in = in; + session.image_size = image_size; + session.written = 0; + session.error = ota::OTA_RESPONSE_OK; + ota_inflate_init(&session, session.window, OTA_INFLATE_WINDOW_SIZE); + // Where the ack must follow the write, flush and ack before waiting for + // input, or the client waits for an ack while the decoder waits for data + session.source_read_cb = [](OtaInflateState *d) -> int { + auto *s = static_cast(d); + if (ACK_AFTER_WRITE) { + s->error = s->self->inflate_flush_(*s); + if (s->error != ota::OTA_RESPONSE_OK) + return -1; + } + // More input than announced; reported by the size check below + if (s->xfer->total >= s->xfer->ota_size) + return -1; + ssize_t read = s->self->receive_data_(s->in, *s->xfer); + if (read <= 0) { + // Already logged by receive_data_ + s->error = ota::OTA_RESPONSE_ERROR_UNKNOWN; + return -1; + } + d->source = s->in + 1; + d->source_limit = s->in + read; + return s->in[0]; + }; + + int res; + do { + // The ring index wrapped to 0 exactly when the window filled + session.dest = session.window; + session.dest_limit = session.window + OTA_INFLATE_WINDOW_SIZE; + session.flushed = 0; + res = ota_inflate(&session); + // A stored block keeps emitting zeros after a failed read, hence eof + if (res < 0 || session.eof) + break; + session.error = this->inflate_flush_(session); + } while (res != OTA_INFLATE_DONE && session.error == ota::OTA_RESPONSE_OK); + + // Transport and flash failures are logged where they happen + if (session.error != ota::OTA_RESPONSE_OK) + return session.error; + if (res != OTA_INFLATE_DONE || session.written != image_size || xfer.total != xfer.ota_size) { + ESP_LOGW(TAG, "Inflate err %d, %zu of %zu B from %zu of %zu", res, session.written, image_size, xfer.total, + xfer.ota_size); + return ota::OTA_RESPONSE_ERROR_UNKNOWN; + } + ESP_LOGD(TAG, "Inflated %zu bytes from %zu", session.written, xfer.total); + return ota::OTA_RESPONSE_OK; +} + +} // namespace esphome +#endif // USE_OTA_DEFLATE +#endif // USE_OTA diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 65476572a1..b3d8501f10 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -32,7 +32,12 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { } } -/** Allocate the session and start the responder handshake. +void ESPHomeOTAComponent::noise_reserve_session_() { + // Default placement, PSRAM first where present: the session lives for one upload + this->noise_ = RAMAllocator().make_unique(); +} + +/** Start the responder handshake, on the session reserved at offer time. * * The prologue binds the whole plaintext preamble, so any tampering with the * negotiation (a stripped feature flag, a changed version) breaks the first @@ -41,9 +46,7 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { // A provisioned key cleared between the offer and here is not guarded: the - // session runs on the zero key load_psk fills in and fails the client's MAC. - // Default placement, PSRAM first where present: the session only lives for one upload - this->noise_ = RAMAllocator().make_unique(); + // session runs on the zero key load_psk fills in and fails the client's MAC static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags diff --git a/esphome/components/ota/ota_backend.h b/esphome/components/ota/ota_backend.h index 7348a0ce90..a505d58efe 100644 --- a/esphome/components/ota/ota_backend.h +++ b/esphome/components/ota/ota_backend.h @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef USE_OTA_STATE_LISTENER #include @@ -102,6 +103,8 @@ enum OTAType : uint8_t { // - set_update_md5: expected digest of the incoming image, hex string. // - write: consume the next chunk; end: finalize and mark bootable. // - abort: safe to call in any state, including after end(). +// - supports_compression: constexpr, whether a gzip image is stored as is and +// inflated at reboot. template concept OTABackendContract = requires(T backend, size_t image_size, uint8_t *data, size_t len, const char *md5) { { backend.begin(image_size, OTA_TYPE_UPDATE_APP) } -> std::same_as; @@ -110,7 +113,9 @@ concept OTABackendContract = requires(T backend, size_t image_size, uint8_t *dat { backend.write(data, len) } -> std::same_as; { backend.end() } -> std::same_as; backend.abort(); - { backend.supports_compression() } -> std::same_as; + { T::supports_compression() } -> std::same_as; + // The value must be a constant expression + typename std::bool_constant; }; /** Listener interface for OTA state changes. diff --git a/esphome/components/ota/ota_backend_arduino_libretiny.h b/esphome/components/ota/ota_backend_arduino_libretiny.h index c2716a44d1..c322ed21f2 100644 --- a/esphome/components/ota/ota_backend_arduino_libretiny.h +++ b/esphome/components/ota/ota_backend_arduino_libretiny.h @@ -13,7 +13,7 @@ class ArduinoLibreTinyOTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } private: bool md5_set_{false}; diff --git a/esphome/components/ota/ota_backend_arduino_rp2.h b/esphome/components/ota/ota_backend_arduino_rp2.h index f7c0037bd2..15142869ac 100644 --- a/esphome/components/ota/ota_backend_arduino_rp2.h +++ b/esphome/components/ota/ota_backend_arduino_rp2.h @@ -15,7 +15,10 @@ class ArduinoRP2OTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + // The core's OTA stub inflates a staged gzip image at reboot, on every chip + // from 4.0.3 (ESPHome pins 6.0.0). begin() only sees the gzip size; the + // inflated size is known when the stub reads the trailer. + static constexpr bool supports_compression() { return USE_ARDUINO_VERSION_CODE >= VERSION_CODE(4, 0, 3); } private: bool md5_set_{false}; diff --git a/esphome/components/ota/ota_backend_esp8266.h b/esphome/components/ota/ota_backend_esp8266.h index 21b5c12c2d..1f1ec37eee 100644 --- a/esphome/components/ota/ota_backend_esp8266.h +++ b/esphome/components/ota/ota_backend_esp8266.h @@ -20,7 +20,7 @@ class ESP8266OTABackend final { OTAResponseTypes end(); void abort(); // Compression supported in all ESP8266 Arduino versions ESPHome supports (>= 2.7.0) - bool supports_compression() { return true; } + static constexpr bool supports_compression() { return true; } protected: /// Erase flash sector if current address is at sector boundary diff --git a/esphome/components/ota/ota_backend_esp_idf.h b/esphome/components/ota/ota_backend_esp_idf.h index c991f896e8..4f4093a594 100644 --- a/esphome/components/ota/ota_backend_esp_idf.h +++ b/esphome/components/ota/ota_backend_esp_idf.h @@ -33,7 +33,7 @@ class IDFOTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } protected: #ifdef USE_OTA_PARTITIONS diff --git a/esphome/components/ota/ota_backend_factory.h b/esphome/components/ota/ota_backend_factory.h index 82d001ed9e..06c58582b5 100644 --- a/esphome/components/ota/ota_backend_factory.h +++ b/esphome/components/ota/ota_backend_factory.h @@ -25,7 +25,7 @@ struct StubOTABackend { OTAResponseTypes write(uint8_t *data, size_t len) { return OTA_RESPONSE_ERROR_UNKNOWN; } OTAResponseTypes end() { return OTA_RESPONSE_ERROR_UNKNOWN; } void abort() {} - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } }; std::unique_ptr make_ota_backend(); } // namespace esphome::ota @@ -33,6 +33,7 @@ std::unique_ptr make_ota_backend(); namespace esphome::ota { using OTABackendPtr = decltype(make_ota_backend()); -static_assert(OTABackendContract, +using OTABackend = OTABackendPtr::element_type; +static_assert(OTABackendContract, "The platform's OTA backend is missing part of the backend surface (ota_backend.h)"); } // namespace esphome::ota diff --git a/esphome/components/ota/ota_backend_host.h b/esphome/components/ota/ota_backend_host.h index 51ffdaeda3..e53868f102 100644 --- a/esphome/components/ota/ota_backend_host.h +++ b/esphome/components/ota/ota_backend_host.h @@ -19,7 +19,7 @@ class HostOTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } protected: md5::MD5Digest md5_{}; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 88c82b0d7c..558bbd54cd 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -295,6 +295,7 @@ #define USE_RUNTIME_IMAGE_QOI #define USE_RUNTIME_STATS #define USE_OTA +#define USE_OTA_DEFLATE #define USE_OTA_ENCRYPTION #define USE_OTA_ENCRYPTION_PROVISIONED #define USE_OTA_ENCRYPTION_REQUIRED diff --git a/esphome/espota2.py b/esphome/espota2.py index 952f88fc4b..dbdf323366 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -65,9 +65,15 @@ CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01 CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02 CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04 CLIENT_FEATURE_SUPPORTS_NOISE = 0x08 +CLIENT_FEATURE_SUPPORTS_DEFLATE = 0x10 SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01 SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02 SERVER_FEATURE_SUPPORTS_NOISE = 0x04 +# Binding once offered: the device then expects the image size and a deflate stream +SERVER_FEATURE_SUPPORTS_DEFLATE = 0x08 + +# Wire constant: the deflate bit promises a 4 KB window (OTA_INFLATE_WINDOW_SIZE) +DEFLATE_WINDOW_BITS = 12 NOISE_FRAME_INDICATOR = 0x01 NOISE_HANDSHAKE_OK = 0x00 @@ -87,6 +93,9 @@ _SUPPORTED_OTA_TYPES: frozenset[int] = frozenset( ) UPLOAD_BLOCK_SIZE = 8192 +# Sizes on the wire are 4 bytes MSB first +SIZE_FIELD_BYTES = 4 +COMPRESS_LEVEL = 9 UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # Flaky Wi-Fi links often drop the first OTA attempt, and the device may need time @@ -573,6 +582,7 @@ def perform_ota( CLIENT_FEATURE_SUPPORTS_COMPRESSION | CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | CLIENT_FEATURE_SUPPORTS_DEFLATE ) if noise_psk: features_to_send |= CLIENT_FEATURE_SUPPORTS_NOISE @@ -675,8 +685,18 @@ def perform_ota( f"retry {flag_name}." ) - if features & SERVER_FEATURE_SUPPORTS_COMPRESSION: - upload_contents = gzip.compress(file_contents, compresslevel=9) + deflate = bool(extended_proto and features & SERVER_FEATURE_SUPPORTS_DEFLATE) + if deflate: + import zlib + + # The device inflates while receiving through a small ring window + upload_contents = zlib.compress( + file_contents, COMPRESS_LEVEL, wbits=-DEFLATE_WINDOW_BITS + ) + _LOGGER.info("Compressed to %s bytes (deflate)", len(upload_contents)) + elif features & SERVER_FEATURE_SUPPORTS_COMPRESSION: + # The device stores the gzip file and inflates it when it reboots + upload_contents = gzip.compress(file_contents, compresslevel=COMPRESS_LEVEL) _LOGGER.info("Compressed to %s bytes", len(upload_contents)) else: upload_contents = file_contents @@ -735,22 +755,20 @@ def perform_ota( send_check(sock, ota_type, "ota type") upload_size = len(upload_contents) - upload_size_encoded = [ - (upload_size >> 24) & 0xFF, - (upload_size >> 16) & 0xFF, - (upload_size >> 8) & 0xFF, - (upload_size >> 0) & 0xFF, - ] # The device erases flash between receiving the size and acking the # prepare, so this window shows the erase cost (near zero when the # device erases lazily during the upload) prepare_start = time.perf_counter() - send_check(sock, upload_size_encoded, "binary size") + send_check(sock, upload_size.to_bytes(SIZE_FIELD_BYTES, "big"), "binary size") + if deflate: + # Own frame: an encrypted session carries one field per frame + send_check(sock, file_size.to_bytes(SIZE_FIELD_BYTES, "big"), "image size") receive_exactly(sock, 1, "update prepare result", RESPONSE_UPDATE_PREPARE_OK) prepare_duration = time.perf_counter() - prepare_start _LOGGER.info("Preparing for upload took %.2f seconds", prepare_duration) - upload_md5 = hashlib.md5(upload_contents).hexdigest() + # The device hashes what it writes: the inflated image, else the received bytes + upload_md5 = hashlib.md5(file_contents if deflate else upload_contents).hexdigest() _LOGGER.debug("MD5 of upload is %s", upload_md5) send_check(sock, upload_md5, "file checksum") diff --git a/script/build_helpers.py b/script/build_helpers.py index f0b354a8a7..885a5a1475 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -46,6 +46,8 @@ EXIT_SKIPPED = 1 EXIT_COMPILE_ERROR = 2 EXIT_CONFIG_ERROR = 3 EXIT_NO_EXECUTABLE = 4 +# A test folder with this name would be synced into src/esphome and swept away with the core tree +CORE_TREE_DIR = "esphome" # Name of the per-component YAML config file in benchmark directories BENCHMARK_YAML_FILENAME = "benchmark.yaml" @@ -465,8 +467,19 @@ def build_and_run( components = sorted(components) - # Build include list: main entry point + component folders + extra dirs - includes: list[str] = [main_entry] + components + # Build include list: main entry point + component folders + extra dirs. The core tree + # folder is listed file by file, nested files included, since a folder include would + # land in src/esphome (see CORE_TREE_DIR) + includes: list[str] = [main_entry] + for component in components: + if component != CORE_TREE_DIR: + includes.append(component) + continue + includes.extend( + str(path.relative_to(tests_dir)) + for path in sorted((tests_dir / component).rglob("*")) + if path.suffix in (".cpp", ".h") + ) if extra_include_dirs: for d in extra_include_dirs: if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h"))): diff --git a/script/ci-custom.py b/script/ci-custom.py index aaf177c941..692393b221 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -904,6 +904,8 @@ def lint_relative_py_import(fname: Path, line, col, content): # neither can live in a C++ namespace. "esphome/components/esp32_hosted/esp_now_hosted.cpp", "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + # C header shared with the vendored decoder + "esphome/components/esphome/ota/ota_esphome_inflate.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: diff --git a/tests/components/esphome/__init__.py b/tests/components/esphome/__init__.py new file mode 100644 index 0000000000..713c0d5a63 --- /dev/null +++ b/tests/components/esphome/__init__.py @@ -0,0 +1,12 @@ +from esphome.loader import FileResource +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + # to_code emits the component count the application needs + manifest.enable_codegen() + # Only the decoder is under test; its ota platform is not in this build + manifest.resources = manifest.resources + [ + FileResource("esphome.components.esphome", "ota/ota_esphome_inflate.c"), + FileResource("esphome.components.esphome", "ota/ota_esphome_inflate.h"), + ] diff --git a/tests/components/esphome/test_ota_inflate.cpp b/tests/components/esphome/test_ota_inflate.cpp new file mode 100644 index 0000000000..e18c96440f --- /dev/null +++ b/tests/components/esphome/test_ota_inflate.cpp @@ -0,0 +1,324 @@ +#include + +#include +#include +#include + +#include "esphome/components/esphome/ota/ota_esphome_inflate.h" + +namespace esphome::testing { + +// build_plain() compressed with the CLI's window (espota2.DEFLATE_WINDOW_BITS): +// DEFLATED = zlib.compress(plain, 9, wbits=-12) +// STORED = zlib.compress(plain[:300], 0, wbits=-12) +static const uint8_t DEFLATED[] = { + 0xed, 0xc8, 0xf7, 0x3f, 0xd4, 0x0f, 0x03, 0x00, 0x70, 0x67, 0xaf, 0x4b, 0x67, 0x66, 0x9f, 0x90, 0x91, 0x11, 0xc2, + 0x11, 0x91, 0xb8, 0xb3, 0xf7, 0x3a, 0xd9, 0x5f, 0x4e, 0x99, 0x67, 0x1e, 0xce, 0x8a, 0xac, 0xec, 0x59, 0xb8, 0xc2, + 0x95, 0x5d, 0x56, 0x42, 0x67, 0x73, 0x46, 0xf6, 0xca, 0xce, 0xc8, 0xc8, 0xc8, 0x91, 0x8a, 0x7c, 0x2f, 0x7a, 0xfe, + 0x86, 0xe7, 0x87, 0xe7, 0x87, 0xe7, 0xf5, 0xfa, 0xbc, 0x7f, 0x7c, 0xbb, 0x07, 0xa2, 0x1f, 0xfa, 0xf9, 0xb8, 0x43, + 0xfd, 0x82, 0x5c, 0xa0, 0x6e, 0xee, 0x28, 0x6f, 0x97, 0x20, 0x77, 0xa8, 0x3b, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, + 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, + 0x70, 0xc0, 0x01, 0xf7, 0x5f, 0xdd, 0x40, 0xd4, 0x63, 0x2f, 0x03, 0xf2, 0x17, 0xb2, 0xe5, 0x69, 0xc7, 0x5a, 0x04, + 0xdf, 0x46, 0x22, 0xa8, 0x4b, 0x6e, 0xd5, 0x3a, 0x80, 0x05, 0xe1, 0x81, 0x4a, 0x44, 0x56, 0x27, 0xa9, 0x21, 0xf7, + 0xad, 0xd9, 0xb0, 0xd3, 0xf1, 0xa5, 0x0b, 0x6a, 0x96, 0x56, 0xb2, 0xca, 0xb5, 0xee, 0x0f, 0x96, 0x28, 0xc3, 0x9e, + 0x8f, 0x6e, 0xf2, 0x84, 0xa4, 0x3b, 0x2d, 0x16, 0x64, 0x08, 0x6a, 0x28, 0x91, 0xee, 0x87, 0x1a, 0x81, 0xff, 0xed, + 0x2c, 0x5f, 0xda, 0x4a, 0x48, 0x5f, 0x91, 0xf0, 0x31, 0xfe, 0x6b, 0xbd, 0x81, 0x86, 0x92, 0x33, 0x1a, 0x6c, 0x69, + 0x32, 0xfb, 0x19, 0x56, 0x2c, 0xc6, 0xaa, 0xef, 0xe8, 0x16, 0x98, 0xcf, 0x98, 0xbf, 0xa0, 0x2d, 0xde, 0x7f, 0xdf, + 0x4b, 0xcb, 0xf6, 0x5c, 0xaf, 0x11, 0x24, 0xf0, 0x46, 0x3e, 0x49, 0x0e, 0x67, 0x0e, 0xcf, 0xf3, 0x57, 0xca, 0xb0, + 0x39, 0x55, 0xe1, 0xe7, 0xfc, 0x37, 0x29, 0xe8, 0xd7, 0xf3, 0x08, 0x2e, 0xfb, 0x7b, 0x6d, 0x3e, 0xfe, 0xe9, 0x2c, + 0x68, 0xe4, 0x20, 0x88, 0x57, 0x56, 0x41, 0x3d, 0xab, 0x9b, 0xbf, 0x0c, 0x0c, 0xae, 0x51, 0x48, 0x8b, 0x72, 0xcc, + 0xb8, 0xbb, 0xf3, 0x30, 0xa8, 0x5c, 0xb5, 0xa1, 0x2f, 0x07, 0x52, 0xe9, 0x36, 0xb8, 0x3c, 0xbc, 0xee, 0xbc, 0xf1, + 0xa3, 0x8b, 0x81, 0xc2, 0x03, 0xbf, 0x29, 0x5a, 0x22, 0x24, 0x97, 0xf8, 0xc9, 0xb5, 0xbf, 0xd2, 0x24, 0x4a, 0x86, + 0xc1, 0x2b, 0xb7, 0xb8, 0xde, 0xa7, 0xea, 0xa5, 0x1d, 0x13, 0x1c, 0x1d, 0xd3, 0x3c, 0x81, 0x60, 0x2e, 0x2f, 0x93, + 0x5c, 0x78, 0x43, 0x2e, 0x39, 0x68, 0x09, 0x13, 0x09, 0x10, 0xce, 0xd6, 0x8d, 0xe9, 0x2f, 0xaf, 0x88, 0x6f, 0x99, + 0x4f, 0xcd, 0xdc, 0xaa, 0xf9, 0x47, 0xdb, 0x1c, 0xb5, 0x89, 0x53, 0x10, 0x3d, 0x77, 0xac, 0x26, 0x04, 0x3a, 0x3b, + 0x18, 0xf8, 0x47, 0x75, 0x56, 0xa5, 0xda, 0x70, 0xfb, 0xf7, 0x0d, 0x3b, 0x6e, 0x4b, 0x2a, 0xbc, 0x49, 0x32, 0x43, + 0x95, 0x62, 0x83, 0x3d, 0xdc, 0x0a, 0x1f, 0x1d, 0xf9, 0x59, 0x6b, 0x95, 0xf0, 0x9b, 0xf5, 0x53, 0x9e, 0xb5, 0x65, + 0xeb, 0x74, 0xfa, 0x81, 0x9b, 0x61, 0xa3, 0x57, 0x2f, 0xda, 0x2c, 0xcd, 0xf8, 0xb0, 0x74, 0xb9, 0x62, 0x39, 0x91, + 0xd9, 0x7d, 0xb3, 0x03, 0x65, 0x2e, 0x66, 0x20, 0x18, 0xac, 0xa2, 0xeb, 0x3b, 0x35, 0x98, 0xa3, 0x28, 0x46, 0x30, + 0xee, 0xd3, 0xeb, 0x47, 0x49, 0x11, 0xc5, 0xcd, 0xa0, 0xa5, 0x1c, 0x2e, 0x9d, 0x14, 0xd6, 0x46, 0x4a, 0x05, 0x7b, + 0xd3, 0xb9, 0x0d, 0xeb, 0xd7, 0x6f, 0xb5, 0xf4, 0xed, 0x3f, 0x27, 0xb8, 0xec, 0x51, 0xb1, 0x6e, 0xb0, 0x14, 0xed, + 0xdf, 0x90, 0x72, 0x59, 0x71, 0xd5, 0xf5, 0xf2, 0x61, 0x5f, 0xa7, 0x8a, 0xd7, 0x0f, 0x80, 0x2f, 0xe5, 0x0f, 0x45, + 0xf2, 0x4d, 0xd1, 0xdc, 0xdd, 0xce, 0x46, 0xdf, 0x77, 0xf2, 0xa6, 0xa2, 0x79, 0x77, 0xf0, 0x0e, 0xaa, 0x68, 0x14, + 0x85, 0x32, 0x05, 0x29, 0x75, 0x2b, 0x6c, 0xb0, 0x7c, 0x84, 0xc9, 0xec, 0x49, 0x70, 0x9b, 0x38, 0x36, 0x4c, 0xe9, + 0xa5, 0xdc, 0xb1, 0xb8, 0x28, 0xd6, 0x20, 0x89, 0x8a, 0x2a, 0x62, 0xc1, 0x90, 0x3a, 0x43, 0x48, 0xaa, 0xfc, 0xec, + 0x52, 0xf4, 0x02, 0xa7, 0xcd, 0x46, 0xf9, 0x78, 0x64, 0x4f, 0xad, 0x6b, 0x17, 0xdb, 0x56, 0xa2, 0xa1, 0x6a, 0xcf, + 0x3b, 0x66, 0x64, 0x01, 0xc2, 0xd6, 0xae, 0x23, 0x34, 0x82, 0x60, 0x0a, 0x3a, 0xe4, 0xdd, 0x7f, 0xaa, 0x97, 0x44, + 0x9a, 0x63, 0x8f, 0xaf, 0xa9, 0x97, 0x5a, 0x55, 0xcc, 0x81, 0x48, 0x83, 0xf8, 0xe6, 0xc5, 0xef, 0xdf, 0xfa, 0x5a, + 0x4f, 0x7f, 0x18, 0x56, 0xc0, 0x66, 0x36, 0xad, 0x8a, 0x79, 0xab, 0xde, 0xf4, 0x7b, 0x70, 0x98, 0xea, 0xfa, 0xf4, + 0xc7, 0x01, 0x31, 0x16, 0xea, 0xf1, 0x70, 0x8d, 0xee, 0x87, 0x52, 0x13, 0x01, 0x9e, 0x0c, 0xbd, 0x5a, 0x14, 0x01, + 0x8e, 0xf7, 0x6e, 0xff, 0xce, 0x7f, 0xa4, 0xd1, 0xa1, 0x6a, 0x9b, 0x95, 0xb2, 0x31, 0x8a, 0x6f, 0xc8, 0xfe, 0x8c, + 0x29, 0x55, 0xbc, 0xfc, 0x61, 0xd0, 0xde, 0xb0, 0xa9, 0x0c, 0x01, 0x0f, 0xba, 0x77, 0x7e, 0x77, 0xd9, 0x76, 0xa4, + 0xfd, 0xab, 0x38, 0x18, 0x92, 0xec, 0xf0, 0xd3, 0x57, 0x7f, 0xf4, 0xbd, 0x65, 0xd4, 0x77, 0x8e, 0xa1, 0x92, 0x82, + 0x0c, 0x7b, 0x34, 0xd3, 0x11, 0xfb, 0xc0, 0x36, 0x23, 0x8e, 0xbb, 0x89, 0xdf, 0x70, 0xdc, 0x9a, 0x76, 0x45, 0x0b, + 0x22, 0xfa, 0x35, 0xdf, 0x8f, 0x45, 0x48, 0x7d, 0xed, 0xdc, 0x06, 0x01, 0x5e, 0xbb, 0xe9, 0xf9, 0x07, 0x93, 0xf6, + 0x69, 0x6f, 0xf1, 0xfc, 0xfd, 0xfa, 0x41, 0xc0, 0x13, 0x9e, 0x74, 0x2d, 0x76, 0xfe, 0xa7, 0xbe, 0x4e, 0xd9, 0xaa, + 0x64, 0xa6, 0xed, 0xd3, 0xad, 0xee, 0x62, 0x9d, 0x39, 0xc9, 0xb8, 0xae, 0x86, 0x31, 0x4f, 0x62, 0x57, 0xea, 0xea, + 0x5a, 0xe3, 0x59, 0xd8, 0x99, 0xb3, 0xcd, 0x9f, 0x3b, 0xea, 0x58, 0x99, 0x11, 0xdc, 0x3d, 0xac, 0x58, 0xd9, 0xa6, + 0xae, 0x2d, 0x07, 0x1d, 0xd7, 0xfa, 0x87, 0x78, 0xa7, 0x7f, 0x2a, 0x4f, 0x25, 0x58, 0xef, 0x53, 0x78, 0x2e, 0x93, + 0xdc, 0x44, 0xcc, 0x53, 0x88, 0x77, 0x1c, 0xda, 0x14, 0xaf, 0xe1, 0x67, 0x92, 0xff, 0x36, 0x96, 0x20, 0x6f, 0x9d, + 0x2c, 0x7f, 0xea, 0x31, 0xf2, 0x34, 0x50, 0xc2, 0x39, 0x84, 0xee, 0x4c, 0xbe, 0xca, 0x06, 0x6f, 0x67, 0x42, 0xea, + 0x13, 0x58, 0xee, 0xdd, 0x8e, 0x29, 0x4f, 0xee, 0xd8, 0x93, 0x0d, 0x45, 0x80, 0x4d, 0xf3, 0x12, 0x79, 0xbb, 0x36, + 0xa3, 0x73, 0x95, 0x95, 0xb6, 0xfc, 0x54, 0x60, 0xb2, 0xcc, 0x71, 0xaa, 0xf1, 0x6b, 0x66, 0xed, 0xba, 0x8b, 0xd6, + 0x6d, 0x61, 0x79, 0x61, 0x1d, 0xb3, 0xba, 0xa6, 0x2f, 0xaa, 0xdc, 0x1d, 0xb8, 0x22, 0xd8, 0x98, 0x58, 0xed, 0x4d, + 0x3c, 0xea, 0xa9, 0x37, 0x5e, 0x5e, 0x7b, 0x47, 0xa1, 0x7a, 0x39, 0x42, 0xe4, 0x3b, 0xbb, 0x69, 0x0a, 0x8b, 0x32, + 0x6e, 0x63, 0xeb, 0x87, 0xf6, 0x5d, 0xaa, 0xbf, 0xbe, 0xc5, 0xb2, 0x85, 0x60, 0xdc, 0x32, 0x07, 0x85, 0x73, 0x3d, + 0x96, 0x8b, 0x89, 0x71, 0x52, 0xb4, 0x93, 0xe6, 0x18, 0xad, 0xbf, 0xce, 0x21, 0x1d, 0x33, 0xb5, 0xb3, 0x35, 0x6a, + 0x5b, 0xe7, 0x47, 0x13, 0x19, 0x8f, 0x53, 0xf4, 0x0c, 0x2a, 0x39, 0x16, 0x37, 0x39, 0x3b, 0x5f, 0x81, 0x51, 0xbc, + 0x23, 0x92, 0x2c, 0x8d, 0xbf, 0x2f, 0xee, 0xbf, 0xed, 0x9d, 0x3f, 0xe0, 0x16, 0x21, 0x5a, 0x57, 0xa6, 0x8c, 0x58, + 0x7a, 0xd5, 0xa1, 0x6d, 0xed, 0xe8, 0x90, 0x97, 0x14, 0xb4, 0x6b, 0xa5, 0x3b, 0xd7, 0x90, 0x84, 0x9e, 0x07, 0xc2, + 0x7f, 0x1e, 0x08, 0xa0, 0x6f, 0x69, 0xf1, 0xcc, 0x4e, 0xca, 0x07, 0x64, 0xa2, 0xb4, 0xbc, 0x5f, 0xe0, 0xa7, 0x0e, + 0x31, 0x77, 0x6f, 0x35, 0xd7, 0x66, 0x83, 0x5d, 0x64, 0x4e, 0xf1, 0x3c, 0x7a, 0xcb, 0xa5, 0xfc, 0xc9, 0x95, 0xab, + 0x27, 0x30, 0x6b, 0x82, 0x57, 0xcd, 0xb0, 0x85, 0x9d, 0xc5, 0xa9, 0x0f, 0xdb, 0xe3, 0x5a, 0xc1, 0xb2, 0x3f, 0xfa, + 0xea, 0xf3, 0x3a, 0xa0, 0xe4, 0xad, 0xd7, 0x14, 0xcf, 0x4f, 0xc9, 0x46, 0x09, 0x6e, 0x70, 0x27, 0xe4, 0x9b, 0x5f, + 0x61, 0x4a, 0xb2, 0xfb, 0xc3, 0xf0, 0x3a, 0x53, 0x38, 0x2a, 0x59, 0x48, 0x2b, 0xab, 0x21, 0x64, 0x1a, 0x1c, 0x90, + 0xb6, 0x69, 0xbb, 0x89, 0xe3, 0x9a, 0x12, 0xb7, 0xfd, 0x47, 0xcc, 0x57, 0xdb, 0x2b, 0xf6, 0x35, 0xe1, 0x79, 0xe0, + 0xbf, 0x6d, 0x7f, 0x71, 0x3c, 0x21, 0x27, 0x82, 0x12, 0x05, 0xae, 0x43, 0x11, 0x0f, 0xc8, 0xe7, 0x3c, 0xf0, 0x0b, + 0xe4, 0x69, 0xff, 0xb2, 0xda, 0x9e, 0x7f, 0xfe, 0xa4, 0x3f, 0xdf, 0x02, 0x58, 0xaa, 0x60, 0xd2, 0x8e, 0xd5, 0x6c, + 0x92, 0x22, 0x38, 0xb1, 0xd2, 0x84, 0x80, 0xea, 0xce, 0xfe, 0x34, 0x79, 0xd6, 0x94, 0xb0, 0xad, 0x51, 0x67, 0x65, + 0xf5, 0xab, 0xd5, 0x18, 0x05, 0x92, 0x8a, 0x24, 0x36, 0xc2, 0xc4, 0x99, 0xad, 0x0d, 0x26, 0xb8, 0x70, 0xea, 0x5b, + 0x74, 0x35, 0x6a, 0xfa, 0xac, 0x97, 0xd8, 0x4c, 0x08, 0x6d, 0xc1, 0xfe, 0x60, 0xe4, 0xd4, 0x8f, 0x93, 0xd4, 0xaf, + 0xa2, 0xec, 0xb8, 0x71, 0xf6, 0xcb, 0x3f, 0xdc, 0x3e, 0x42, 0xca, 0xe2, 0x8d, 0xb9, 0x0a, 0x2c, 0xef, 0xb6, 0x18, + 0x75, 0x46, 0x8a, 0x2b, 0x21, 0xeb, 0xc5, 0x12, 0x4d, 0xd7, 0xb8, 0x8e, 0x6c, 0xec, 0x9d, 0x12, 0xde, 0xa5, 0x45, + 0xab, 0xc4, 0x26, 0x47, 0xfa, 0x57, 0xad, 0x78, 0x4c, 0x5e, 0x87, 0x0b, 0x42, 0x6a, 0xbe, 0xef, 0xbd, 0xbc, 0xe5, + 0xd7, 0x4c, 0x3a, 0x0f, 0x96, 0x29, 0xcb, 0x45, 0x46, 0x3f, 0x8a, 0x31, 0xda, 0x8d, 0x7f, 0x89, 0xa4, 0xd6, 0xb4, + 0xe5, 0x99, 0x27, 0xa7, 0xab, 0x15, 0x82, 0x75, 0x3e, 0xc0, 0x9d, 0xae, 0x8c, 0x7f, 0x19, 0x80, 0xf5, 0xa6, 0x89, + 0xbc, 0xb7, 0x36, 0x50, 0x1f, 0x63, 0xca, 0x40, 0x3e, 0xe1, 0x4c, 0x2d, 0x1e, 0x15, 0x51, 0x93, 0xf2, 0x1a, 0x97, + 0x3e, 0xad, 0x5e, 0xba, 0xe2, 0xd6, 0xa7, 0x7c, 0x51, 0x45, 0xdf, 0x92, 0xbe, 0x35, 0xf3, 0xb4, 0xe8, 0x5c, 0x2e, + 0x6a, 0x5c, 0xda, 0xe8, 0xf7, 0x51, 0xdc, 0xcb, 0x06, 0xd2, 0xd2, 0xa4, 0x7c, 0x7d, 0x5f, 0x20, 0xb8, 0x50, 0x04, + 0xb5, 0xb1, 0x9a, 0x4e, 0xc9, 0x94, 0xe4, 0x40, 0x2b, 0x85, 0xc8, 0xd3, 0x71, 0xfa, 0x83, 0x81, 0x28, 0x53, 0xef, + 0xa0, 0xb0, 0xf0, 0xc8, 0xb9, 0x52, 0xb2, 0x0f, 0x8c, 0x49, 0x93, 0xeb, 0x2a, 0x95, 0x83, 0x4f, 0x47, 0xe8, 0x0c, + 0x83, 0x7b, 0x88, 0x18, 0x63, 0xa9, 0x87, 0x7f, 0xe3, 0x57, 0xa6, 0x57, 0xdd, 0x6d, 0x2a, 0x85, 0xe9, 0x79, 0x16, + 0x3b, 0x07, 0x59, 0x4f, 0xed, 0x9a, 0x17, 0xe7, 0xcd, 0x35, 0xa6, 0x9b, 0x06, 0x1d, 0x18, 0x5e, 0xb2, 0x9b, 0xac, + 0x18, 0xc6, 0x5b, 0xec, 0xbe, 0x29, 0xa3, 0x57, 0xc2, 0x6b, 0x3f, 0x11, 0x89, 0xd1, 0x57, 0xe4, 0xb8, 0xaf, 0x11, + 0xba, 0x90, 0x93, 0x4f, 0x5e, 0x0c, 0x6b, 0x9f, 0x61, 0x5c, 0xdd, 0xb6, 0xcf, 0x18, 0xcc, 0x9a, 0x95, 0x32, 0x5f, + 0x6f, 0xf1, 0x43, 0x17, 0x2d, 0xbc, 0x70, 0xbe, 0xd4, 0x76, 0x38, 0x42, 0xcc, 0xdd, 0xf0, 0xff, 0x5b, 0xe8, 0x19, + 0xc9, 0xf4, 0x25, 0xda, 0xdb, 0x5f, 0xd2, 0x3c, 0xf3, 0xe0, 0xed, 0xb0, 0xa2, 0x4c, 0xdf, 0x05, 0xf2, 0x0c, 0xc4, + 0x21, 0xb9, 0xc4, 0x02, 0xd2, 0x1e, 0x46, 0x2c, 0x57, 0x78, 0xb0, 0xf1, 0x85, 0x33, 0x51, 0xc5, 0x3e, 0xce, 0x69, + 0x68, 0x77, 0x3e, 0xb4, 0x5a, 0x4e, 0x43, 0x89, 0x85, 0x71, 0x36, 0x84, 0x56, 0x5c, 0x6b, 0xce, 0x78, 0x4e, 0x86, + 0x6a, 0x5f, 0xf1, 0x2d, 0x73, 0x4c, 0xba, 0xc1, 0x5e, 0x2b, 0x6e, 0x0d, 0xdd, 0x45, 0x1d, 0x92, 0xb4, 0x31, 0xd9, + 0xce, 0x0b, 0x58, 0x67, 0xc1, 0xa6, 0x47, 0x2e, 0x5e, 0x6d, 0x6d, 0xa3, 0x78, 0xed, 0xc5, 0xcb, 0xac, 0x20, 0xb8, + 0x6a, 0x14, 0x32, 0x55, 0xe8, 0xfb, 0xe0, 0xee, 0x31, 0xfc, 0x47, 0x33, 0xf6, 0xe7, 0x54, 0xd1, 0x61, 0x91, 0x74, + 0x90, 0xfa, 0x44, 0x0a, 0xc8, 0xaf, 0xa2, 0xdb, 0x2c, 0x83, 0xda, 0xe8, 0xd0, 0xb7, 0x34, 0x8a, 0xc8, 0xbb, 0x86, + 0x58, 0x82, 0x1a, 0x2f, 0xd6, 0xc5, 0x92, 0xe9, 0xd0, 0x21, 0x11, 0xf6, 0xeb, 0x9b, 0xed, 0xbf, 0xa9, 0x71, 0x89, + 0xb2, 0x59, 0x76, 0xb3, 0xeb, 0x9c, 0xe1, 0x73, 0x79, 0x7b, 0x11, 0x3d, 0x4e, 0x33, 0xca, 0xf6, 0x7e, 0xd8, 0xcf, + 0x06, 0x41, 0x39, 0xef, 0x78, 0x5a, 0x89, 0x69, 0x0c, 0xff, 0xda, 0x31, 0x16, 0x32, 0xd4, 0xe5, 0x2d, 0x8b, 0x22, + 0xb9, 0x82, 0x38, 0xd8, 0x22, 0xfc, 0xe8, 0xbd, 0xbf, 0x0d, 0x4b, 0xc9, 0x0d, 0x74, 0x5e, 0x2d, 0xb2, 0x1f, 0x23, + 0xa8, 0xf8, 0xa7, 0x27, 0xe1, 0x02, 0x61, 0x83, 0xc4, 0xd0, 0x63, 0x0e, 0x23, 0xf2, 0x34, 0x06, 0x69, 0xd1, 0xa3, + 0xa2, 0x76, 0x76, 0x94, 0xbc, 0x1f, 0x78, 0x73, 0x5a, 0x48, 0x82, 0xb4, 0x68, 0x3d, 0x24, 0x52, 0xe8, 0x92, 0x4e, + 0x58, 0xdf, 0x55, 0x3f, 0xa8, 0x48, 0x59, 0x35, 0x90, 0x9b, 0x55, 0xc9, 0xdc, 0xff, 0xcc, 0x22, 0x7c, 0xd1, 0x29, + 0xe0, 0xd5, 0xd8, 0x31, 0x3d, 0x9b, 0xae, 0x38, 0x52, 0x2a, 0xa1, 0xc5, 0x86, 0x19, 0x93, 0xe7, 0x2d, 0x9b, 0xa9, + 0x09, 0xbf, 0xdd, 0x89, 0x43, 0x35, 0xb2, 0x6f, 0x91, 0xba, 0x28, 0xfa, 0x2f, 0x2c, 0xab, 0xe8, 0x58, 0xe6, 0x4c, + 0xbf, 0xd5, 0xaf, 0x49, 0xcd, 0x7f, 0x18, 0xa4, 0x69, 0xd3, 0xff, 0x99, 0xad, 0x9d, 0xb0, 0x93, 0x3b, 0x04, 0x01, + 0xd5, 0x7d, 0x33, 0xe0, 0x66, 0xe2, 0xd4, 0xeb, 0xae, 0xcd, 0x7f, 0x66, 0xf2, 0x2c, 0xa7, 0xfe, 0x7d, 0x51, 0x8b, + 0x2a, 0x25, 0xa8, 0x90, 0xb2, 0x71, 0xe5, 0x98, 0x40, 0xb8, 0xe9, 0x1f, 0x64, 0xd9, 0x90, 0xf2, 0x39, 0x25, 0x6f, + 0x58, 0x35, 0x9e, 0x40, 0x25, 0x92, 0xbb, 0xa2, 0x4d, 0x45, 0xf6, 0xc3, 0x1a, 0xc6, 0xc0, 0xf6, 0xc2, 0x9b, 0x2b, + 0xb5, 0xc7, 0xe9, 0xc8, 0x75, 0x03, 0x71, 0x1c, 0x76, 0x0e, 0xde, 0x5c, 0xe7, 0xf4, 0x91, 0xc4, 0x1d, 0x9f, 0xcb, + 0x67, 0x1c, 0x1b, 0xc9, 0x25, 0x6d, 0x95, 0x05, 0xd7, 0x3b, 0xbf, 0x2d, 0x90, 0x96, 0xc3, 0x44, 0xa3, 0x0a, 0xee, + 0x7a, 0x4c, 0x7e, 0x7f, 0xc2, 0x4e, 0x4f, 0x38, 0x88, 0xe9, 0x9f, 0x93, 0xd8, 0xd1, 0x10, 0x8c, 0x85, 0x89, 0xb2, + 0xd0, 0x95, 0xc3, 0xe4, 0xd2, 0x8f, 0x2b, 0x0c, 0x82, 0xbd, 0x25, 0x74, 0x39, 0xe7, 0x3e, 0xef, 0xd2, 0x2f, 0x6f, + 0xdc, 0xe0, 0x75, 0x6b, 0xf2, 0x66, 0x7d, 0x65, 0x18, 0x26, 0xb0, 0x7a, 0x7a, 0x45, 0xf0, 0xf2, 0xc4, 0xbe, 0x70, + 0x6e, 0x58, 0x3a, 0x54, 0x31, 0x64, 0xf6, 0x49, 0xb5, 0xdc, 0x1c, 0x9e, 0x87, 0xdb, 0x6a, 0x92, 0x2a, 0x59, 0x73, + 0x35, 0x4d, 0x59, 0xed, 0xd0, 0x8a, 0xd9, 0xac, 0xea, 0x7c, 0xc0, 0x0a, 0xfe, 0x8f, 0x29, 0xb7, 0xda, 0xdc, 0x8a, + 0x55, 0x0f, 0x61, 0xc0, 0x82, 0x49, 0x4c, 0xc2, 0x51, 0x4b, 0x41, 0x7d, 0x1f, 0xf1, 0x07, 0x13, 0x23, 0x16, 0xe4, + 0x88, 0x76, 0x38, 0x92, 0xbd, 0x3d, 0xe2, 0x7b, 0xab, 0x1c, 0x53, 0x35, 0xfb, 0x85, 0x76, 0x35, 0xf7, 0xfc, 0xdc, + 0x3e, 0xc9, 0xae, 0x6a, 0x7c, 0x10, 0x44, 0xa5, 0xfb, 0xc4, 0x81, 0x9a, 0x6d, 0x9b, 0xf2, 0x87, 0x0a, 0x9e, 0xcd, + 0x75, 0xc2, 0xd5, 0xe7, 0x0b, 0xcd, 0x5c, 0x6e, 0xf2, 0x51, 0x96, 0xa3, 0xe8, 0xd5, 0x5a, 0x17, 0xf5, 0x1a, 0x28, + 0xff, 0xc8, 0xf0, 0xfc, 0x7e, 0x3b, 0xd3, 0xf3, 0x12, 0x61, 0xe9, 0x02, 0xfa, 0x7b, 0x8e, 0x6b, 0x34, 0x4a, 0x5a, + 0xb6, 0xb1, 0x71, 0x7d, 0xad, 0xbc, 0x88, 0x22, 0xf6, 0x14, 0xc5, 0x57, 0x1a, 0x0b, 0xb6, 0x8b, 0xbb, 0x9a, 0x09, + 0xf2, 0xe8, 0x8d, 0x32, 0xd1, 0x54, 0xa9, 0xd2, 0x9f, 0xc5, 0xf4, 0x12, 0xad, 0xb3, 0xfd, 0x52, 0xb7, 0xff, 0x0e, + 0x72, 0x59, 0xcb, 0x1d, 0x8d, 0xa4, 0xf2, 0x0f, 0x39, 0x8f, 0xc9, 0x84, 0xe0, 0x5c, 0xdd, 0x9c, 0xe4, 0x42, 0x4f, + 0xf2, 0x09, 0x1e, 0x05, 0xcc, 0x41, 0xdd, 0xbd, 0xc1, 0xed, 0xdc, 0x8e, 0x64, 0xc3, 0x67, 0x51, 0xe5, 0x8a, 0x9a, + 0x0d, 0x09, 0xb8, 0x4b, 0xa1, 0xb2, 0xdb, 0x01, 0xde, 0xf5, 0xd1, 0x51, 0xf7, 0x9d, 0x88, 0x59, 0xfd, 0xd5, 0x13, + 0x0d, 0x05, 0xd5, 0x5e, 0xfa, 0x63, 0x26, 0x25, 0xad, 0x01, 0x83, 0x6f, 0xf3, 0x7f, 0x56, 0x11, 0x89, 0xaa, 0xdb, + 0x49, 0xaa, 0x8f, 0x7d, 0x10, 0x99, 0xbf, 0x8b, 0xcf, 0xf4, 0xf8, 0xc2, 0x4e, 0x36, 0xa7, 0xb1, 0xfe, 0xc5, 0xf0, + 0x07, 0xc3, 0x57, 0xe3, 0xbd, 0x1e, 0xa1, 0x56, 0x46, 0x32, 0x43, 0x8f, 0x8d, 0xc7, 0x73, 0x0a, 0xa5, 0x35, 0xa9, + 0x93, 0x7c, 0x53, 0x3c, 0xf2, 0x58, 0xf3, 0x6a, 0x37, 0x41, 0xb4, 0xc7, 0x9d, 0xcd, 0xb1, 0x67, 0xb7, 0xaa, 0x5f, + 0x25, 0xb9, 0xc5, 0xb6, 0xa2, 0xdb, 0xce, 0x34, 0x2c, 0xea, 0x93, 0x45, 0x6e, 0x7b, 0x6f, 0xb6, 0xf6, 0x89, 0x83, + 0xd1, 0x9e, 0x08, 0x4b, 0x57, 0x4e, 0xd9, 0xf3, 0xe8, 0x81, 0x03, 0x5c, 0xd6, 0x6c, 0x2c, 0xea, 0x27, 0x2d, 0xfb, + 0x39, 0xbb, 0x25, 0x81, 0xbb, 0x1a, 0xc3, 0xcb, 0xf5, 0x5e, 0x69, 0x1c, 0xa4, 0x8a, 0x0e, 0x4a, 0x69, 0xdf, 0xcd, + 0x6c, 0xc7, 0xce, 0x48, 0xdb, 0x7a, 0x0b, 0xcb, 0x40, 0xc8, 0x59, 0x81, 0xde, 0x71, 0x3e, 0x0d, 0x41, 0xc9, 0xa2, + 0x57, 0xae, 0xb7, 0x3f, 0x8b, 0x4e, 0x7c, 0xcb, 0xf9, 0xa5, 0xcb, 0xf8, 0xee, 0x52, 0xbe, 0x54, 0xa8, 0xef, 0x99, + 0xb7, 0xa1, 0x0b, 0x77, 0xa2, 0xa7, 0x89, 0x47, 0x7f, 0x48, 0xaa, 0x02, 0x36, 0xbf, 0x9d, 0x97, 0xba, 0x43, 0x5e, + 0x39, 0xa6, 0x99, 0x81, 0xde, 0xe4, 0x98, 0x06, 0x6a, 0xc8, 0x2c, 0x96, 0xf5, 0x51, 0x8c, 0x95, 0xd3, 0xe4, 0x2b, + 0x94, 0x9a, 0x15, 0x71, 0x9f, 0xa4, 0x24, 0xfc, 0x92, 0x97, 0x18, 0xb7, 0x56, 0xe5, 0x59, 0xe1, 0x1a, 0x79, 0x3a, + 0x47, 0x77, 0x92, 0xf5, 0x75, 0x42, 0xbd, 0x2d, 0x55, 0x05, 0xcf, 0xe8, 0xea, 0x64, 0x8b, 0xff, 0x30, 0xa1, 0x1e, + 0x28, 0x2c, 0xa9, 0xaa, 0xd4, 0x19, 0x1d, 0x24, 0x7a, 0xcf, 0x5d, 0x39, 0xf2, 0xe3, 0xd2, 0xab, 0x26, 0x97, 0x35, + 0xf6, 0x70, 0x8b, 0x09, 0x8b, 0x00, 0xb9, 0xb8, 0xa8, 0xb7, 0x78, 0xf1, 0x96, 0x03, 0xd2, 0x72, 0x46, 0x26, 0x82, + 0x41, 0xce, 0xe3, 0x73, 0xb2, 0x31, 0x4f, 0x49, 0xc8, 0xa8, 0xcf, 0xcc, 0xb8, 0xcc, 0x5d, 0xab, 0xd1, 0x1e, 0xf5, + 0x5a, 0x9a, 0x54, 0xaf, 0x15, 0xed, 0x5f, 0x96, 0x2b, 0x62, 0x0a, 0x4c, 0x67, 0x21, 0x98, 0xfd, 0x29, 0xf8, 0x12, + 0x9d, 0xc2, 0x34, 0x4d, 0x70, 0x0e, 0x3b, 0x5e, 0xe3, 0x91, 0x58, 0x82, 0xa8, 0x9c, 0x7d, 0x11, 0xfb, 0x5d, 0x81, + 0x0c, 0x0f, 0x9d, 0x84, 0xbb, 0xeb, 0x65, 0x1d, 0xeb, 0x6e, 0x18, 0x8e, 0x93, 0x70, 0xbb, 0x9d, 0xed, 0x58, 0xb2, + 0x64, 0xf2, 0x7b, 0x8f, 0x05, 0xd8, 0xfa, 0x55, 0x50, 0xef, 0x7a, 0x49, 0x0d, 0xdf, 0xca, 0x63, 0xff, 0x41, 0x1d, + 0x89, 0x79, 0xca, 0x8c, 0x1b, 0xbe, 0x75, 0x6f, 0x0e, 0x6b, 0x68, 0x8c, 0x5c, 0x93, 0xc8, 0xf3, 0x62, 0x0b, 0x68, + 0xb1, 0x32, 0x2f, 0x1e, 0xda, 0xe5, 0x6d, 0x45, 0xaf, 0x3d, 0xa7, 0xaf, 0x81, 0x0f, 0xda, 0x82, 0x85, 0x94, 0x06, + 0xc5, 0x3f, 0xf5, 0x73, 0x5f, 0xa4, 0x43, 0x5f, 0xb3, 0xac, 0x3d, 0x16, 0xe8, 0x6f, 0x84, 0xb3, 0xb6, 0x75, 0xf7, + 0x13, 0x03, 0x46, 0x67, 0x87, 0x9a, 0xf1, 0x6c, 0x2e, 0x33, 0x33, 0x14, 0x85, 0x18, 0xe9, 0xd6, 0x1b, 0xee, 0x17, + 0xe9, 0xe7, 0x6b, 0xd4, 0x46, 0xaf, 0x26, 0x55, 0x8d, 0xfc, 0x3a, 0xb4, 0x61, 0x1a, 0x9e, 0x2d, 0x85, 0xa2, 0x86, + 0x8c, 0x92, 0xbd, 0x8d, 0x0e, 0x4b, 0x4e, 0x0d, 0xdf, 0x4a, 0xc6, 0x66, 0xcd, 0x28, 0xff, 0xc4, 0x44, 0xba, 0x39, + 0x4a, 0x04, 0x43, 0xce, 0x95, 0x47, 0x99, 0x8c, 0x9b, 0xb7, 0x72, 0x48, 0xdd, 0x9b, 0x98, 0x4f, 0xd5, 0x18, 0x81, + 0xb4, 0xef, 0xfa, 0x96, 0x54, 0x9b, 0xda, 0x73, 0xe2, 0xe0, 0x27, 0x2f, 0x10, 0x0f, 0xf9, 0x0e, 0x24, 0xb4, 0xd2, + 0x39, 0xf7, 0x49, 0x8c, 0x5c, 0x05, 0x37, 0xdd, 0xa7, 0x73, 0xac, 0x16, 0x85, 0x71, 0x46, 0xd1, 0x6f, 0x7a, 0x25, + 0xd2, 0x5c, 0x4e, 0x70, 0x26, 0x93, 0x36, 0x22, 0x5f, 0xe5, 0x42, 0x4e, 0x89, 0xf2, 0x4c, 0x55, 0x9b, 0xd1, 0xa5, + 0xaf, 0x87, 0xd8, 0x7e, 0xe8, 0xdd, 0xb9, 0x47, 0xb3, 0xda, 0xf7, 0x98, 0x52, 0x73, 0x8b, 0x3c, 0x0b, 0xe7, 0xfa, + 0x14, 0x7f, 0x9a, 0xa6, 0x8e, 0xf4, 0x7c, 0xad, 0x58, 0x11, 0x73, 0xe3, 0x3a, 0x56, 0x47, 0x9b, 0x95, 0x24, 0x68, + 0x35, 0xa9, 0xd9, 0x6e, 0x5c, 0x4d, 0xe8, 0xa4, 0xb8, 0x80, 0x0b, 0x73, 0xc4, 0x0f, 0x87, 0x0c, 0xa1, 0xf4, 0x5a, + 0x3c, 0xce, 0x3f, 0x30, 0x3d, 0xb5, 0xc1, 0x0b, 0x43, 0x2a, 0xb1, 0xbe, 0x6b, 0x12, 0xc4, 0x4b, 0xdb, 0xcb, 0x2f, + 0x77, 0x8a, 0x3a, 0xba, 0xf4, 0xa3, 0x9b, 0x05, 0xee, 0x98, 0x50, 0x77, 0xfe, 0x82, 0x5b, 0x29, 0x6d, 0x7c, 0x0c, + 0x4b, 0x41, 0xa2, 0x6a, 0xb2, 0x27, 0x32, 0xf1, 0x0c, 0x1d, 0xd7, 0x6c, 0x02, 0x4d, 0xbd, 0xf2, 0x5a, 0x66, 0xd3, + 0xb5, 0x30, 0xab, 0x4b, 0x7e, 0x72, 0x77, 0x30, 0x42, 0xd7, 0x70, 0x05, 0x9f, 0xeb, 0x2f, 0x22, 0x7f, 0x49, 0x2d, + 0x5d, 0xcf, 0x4b, 0x84, 0x0c, 0xa5, 0xb9, 0xe5, 0x49, 0xae, 0x54, 0x2c, 0x86, 0x53, 0x4c, 0x9c, 0x42, 0x29, 0x78, + 0xf0, 0x0a, 0x8a, 0x36, 0x53, 0x4d, 0x2b, 0xc9, 0xe2, 0x69, 0x0e, 0x9a, 0x37, 0x0c, 0x91, 0x2d, 0xac, 0x2f, 0x43, + 0x6a, 0xf1, 0xe9, 0xfb, 0xb4, 0x68, 0xec, 0x90, 0x00, 0xff, 0x32, 0xf8, 0x58, 0xab, 0xd1, 0x73, 0x9f, 0x77, 0x72, + 0x22, 0xed, 0x64, 0xac, 0x7d, 0xd0, 0xa8, 0x1f, 0x01, 0x99, 0x9f, 0xfd, 0xa6, 0xcb, 0x62, 0xca, 0x65, 0xd8, 0xf8, + 0x31, 0xf0, 0xc1, 0x5e, 0xef, 0xf1, 0x4d, 0x79, 0xa9, 0xfc, 0x8f, 0x57, 0xcb, 0xc1, 0xb7, 0x03, 0x30, 0xbb, 0xad, + 0xdb, 0x88, 0xf7, 0xcc, 0xa1, 0x25, 0xfd, 0x37, 0x03, 0xc5, 0xdb, 0x7f, 0xb4, 0xe0, 0xe6, 0xa3, 0x7e, 0xa4, 0x52, + 0x68, 0xa7, 0x2e, 0xe6, 0x1c, 0xea, 0x91, 0x48, 0x67, 0xbd, 0x3d, 0x6d, 0xcc, 0x09, 0x49, 0x17, 0x50, 0x16, 0xb5, + 0x65, 0x63, 0xd4, 0x84, 0x2c, 0x3a, 0x5f, 0x6a, 0x3c, 0x66, 0xfa, 0x24, 0xaf, 0xd1, 0xfd, 0x18, 0x66, 0xe4, 0xe8, + 0x3c, 0x6c, 0x74, 0xb6, 0xc5, 0x4c, 0xf8, 0xa8, 0x15, 0x3c, 0xd0, 0xb8, 0x4a, 0x53, 0x15, 0x4b, 0x1e, 0x56, 0x7f, + 0xd8, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xfe, 0xe7, 0xdc, 0x03, 0xd1, 0x0f, 0xfd, 0x7c, 0xdc, 0xa1, 0x7e, 0x41, 0x2e, 0x50, 0x37, 0x77, 0x94, 0xb7, 0x4b, + 0x90, 0x3b, 0x14, 0x38, 0xe0, 0x80, 0x03, 0x0e, 0x38, 0xe0, 0x80, 0xfb, 0xff, 0xba, 0xff, 0x00, +}; + +static const uint8_t STORED[] = { + 0x01, 0x2c, 0x01, 0xd3, 0xfe, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, + 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, + 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, + 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, + 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, + 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, + 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, + 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, + 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, + 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, + 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, + 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, + 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, + 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, + 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, + 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, + 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, +}; + +static constexpr size_t WINDOW = 4096; +static constexpr size_t PLAIN_SIZE = 16000; + +static uint8_t lcg_next(uint32_t &x) { + x = (x * 1103515245u + 12345u) & 0x7fffffffu; + return (x >> 16) & 0xff; +} + +static std::vector build_plain() { + std::vector plain; + const char *text = "esphome ota deflate "; + for (int i = 0; i < 300; i++) + plain.insert(plain.end(), text, text + strlen(text)); + uint32_t x = 1; + for (int i = 0; i < 3000; i++) + plain.push_back(lcg_next(x)); + plain.insert(plain.end(), 5000, 0); + for (int i = 0; i < 100; i++) + plain.insert(plain.end(), text, text + strlen(text)); + return plain; +} + +// Mirrors the OTA session: chunked input through the read callback, window as output +struct Session : OtaInflateState { + const uint8_t *in; + size_t in_len; + size_t in_pos; + size_t chunk; + std::vector out; + uint8_t window[WINDOW]; +}; + +static int read_cb(OtaInflateState *d) { + auto *s = static_cast(d); + if (s->in_pos >= s->in_len) + return -1; + size_t n = std::min(s->chunk, s->in_len - s->in_pos); + d->source = s->in + s->in_pos + 1; + d->source_limit = s->in + s->in_pos + n; + s->in_pos += n; + return s->in[s->in_pos - n]; +} + +// Inflates the whole input; returns the decoder result and fills s.out +static int inflate_all(Session &s, const uint8_t *in, size_t in_len, size_t chunk) { + s.in = in; + s.in_len = in_len; + s.in_pos = 0; + s.chunk = chunk; + s.out.clear(); + memset(s.window, 0, sizeof(s.window)); + ota_inflate_init(&s, s.window, WINDOW); + s.source_read_cb = read_cb; + int res; + do { + s.dest = s.window; + s.dest_limit = s.window + WINDOW; + res = ota_inflate(&s); + if (res < 0 || s.eof) + return res < 0 ? res : OTA_INFLATE_DATA_ERROR; + s.out.insert(s.out.end(), s.window, s.dest); + if (s.out.size() > PLAIN_SIZE) + return OTA_INFLATE_DATA_ERROR; + } while (res != OTA_INFLATE_DONE); + return res; +} + +TEST(OtaInflate, RoundTripThroughWindow) { + auto s = std::make_unique(); + ASSERT_EQ(inflate_all(*s, DEFLATED, sizeof(DEFLATED), 1040), OTA_INFLATE_DONE); + EXPECT_EQ(s->out, build_plain()); + EXPECT_EQ(s->in_pos, sizeof(DEFLATED)); +} + +TEST(OtaInflate, SmallReadChunks) { + auto s = std::make_unique(); + ASSERT_EQ(inflate_all(*s, DEFLATED, sizeof(DEFLATED), 7), OTA_INFLATE_DONE); + EXPECT_EQ(s->out, build_plain()); +} + +TEST(OtaInflate, StoredBlock) { + auto s = std::make_unique(); + ASSERT_EQ(inflate_all(*s, STORED, sizeof(STORED), 64), OTA_INFLATE_DONE); + auto plain = build_plain(); + plain.resize(300); + EXPECT_EQ(s->out, plain); +} + +TEST(OtaInflate, TruncatedStreamFails) { + auto s = std::make_unique(); + for (size_t cut : {size_t{1}, size_t{100}, size_t{1000}, sizeof(DEFLATED) - 1}) { + EXPECT_LT(inflate_all(*s, DEFLATED, cut, 1040), 0) << "cut at " << cut; + EXPECT_LE(s->out.size(), PLAIN_SIZE); + } +} + +TEST(OtaInflate, TruncatedStoredBlockFails) { + auto s = std::make_unique(); + EXPECT_LT(inflate_all(*s, STORED, sizeof(STORED) - 50, 64), 0); +} + +TEST(OtaInflate, CorruptStreamsNeverEscapeTheWindow) { + // Flipped bytes and garbage; the sanitizers check the decoder stays in bounds + auto s = std::make_unique(); + std::vector bad(DEFLATED, DEFLATED + sizeof(DEFLATED)); + // A coarse, non-aligned stride: neighbouring offsets hit the same paths + for (size_t i = 0; i < bad.size(); i += 29) { + bad[i] ^= 0x5a; + inflate_all(*s, bad.data(), bad.size(), 1040); + bad[i] ^= 0x5a; + } + uint32_t x = 99; + std::vector garbage(2000); + for (int round = 0; round < 50; round++) { + for (auto &b : garbage) + b = lcg_next(x); + inflate_all(*s, garbage.data(), garbage.size(), 1040); + } +} + +} // namespace esphome::testing diff --git a/tests/components/main.cpp b/tests/components/main.cpp index 373fde7151..aa0ceb5abe 100644 --- a/tests/components/main.cpp +++ b/tests/components/main.cpp @@ -28,6 +28,11 @@ void setup() { ::testing::InitGoogleTest(); int exit_code = RUN_ALL_TESTS(); + // A test folder that never reached the build would otherwise pass as an empty run + if (::testing::UnitTest::GetInstance()->total_test_count() == 0) { + fprintf(stderr, "No tests were linked into this binary\n"); + exit_code = 1; + } exit(exit_code); } diff --git a/tests/components/ota/test.bk72xx-ard.yaml b/tests/components/ota/test.bk72xx-ard.yaml new file mode 100644 index 0000000000..dade44d145 --- /dev/null +++ b/tests/components/ota/test.bk72xx-ard.yaml @@ -0,0 +1 @@ +<<: !include common.yaml diff --git a/tests/components/ota/test_backend_contract.cpp b/tests/components/ota/test_backend_contract.cpp index 1b4fbbc32d..36b7a793c3 100644 --- a/tests/components/ota/test_backend_contract.cpp +++ b/tests/components/ota/test_backend_contract.cpp @@ -14,7 +14,7 @@ struct MinimalBackend { OTAResponseTypes write(uint8_t *data, size_t len) { return OTA_RESPONSE_OK; } OTAResponseTypes end() { return OTA_RESPONSE_OK; } void abort() {} - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } }; static_assert(OTABackendContract); diff --git a/tests/integration/fixtures/host_ota_deflate.yaml b/tests/integration/fixtures/host_ota_deflate.yaml new file mode 100644 index 0000000000..ebf7977123 --- /dev/null +++ b/tests/integration/fixtures/host_ota_deflate.yaml @@ -0,0 +1,9 @@ +esphome: + name: host-ota-test +host: +api: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 56a685eac3..17c4dea25f 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -15,6 +15,7 @@ from dataclasses import dataclass import functools from pathlib import Path import socket +import zlib import pytest @@ -123,6 +124,7 @@ class _Device: binary_path: Path proc: asyncio.subprocess.Process | None = None reboots: int = 0 + inflates: int = 0 def __post_init__(self) -> None: self._rebooted = asyncio.Event() @@ -131,6 +133,8 @@ class _Device: if "Rebooting safely" in line: self.reboots += 1 self._rebooted.set() + if "Inflated " in line and " bytes from " in line: + self.inflates += 1 async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: async with asyncio.timeout(timeout): @@ -241,6 +245,80 @@ async def test_host_ota_self_update( await dev.ota(None, None, "second OTA failed -- listener leaked across execv") +@pytest.mark.asyncio +async def test_host_ota_deflate( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Deflate is negotiated by default, an old client gets an uncompressed + upload, and a corrupt stream is rejected without taking the device down.""" + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + errors: list[str] = [] + + def on_log(line: str) -> None: + # A corrupt stream is caught by the decoder, by the size check or by + # the MD5 at the end, depending on where the damage lands + if any( + text in line + for text in ("Inflate err", "Inflate overrun", "End update err") + ): + errors.append(line) + dev.on_log(line) + + real_compress = zlib.compress + + def corrupt_compress(data: bytes, *args: object, **kwargs: object) -> bytes: + # Reserved block type in the first header: rejected by the decoder on + # every build, unlike a flipped data bit that may only fail the MD5 + out = bytearray(real_compress(data, *args, **kwargs)) + out[0] |= 0x06 + return bytes(out) + + def overlong_compress(data: bytes, *args: object, **kwargs: object) -> bytes: + """A stream that inflates past the size the client announced.""" + return real_compress(data + bytes(8192), *args, **kwargs) + + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + + # Default: the host backend cannot store gzip, so the CLI sends deflate + await dev.ota(None, None, "deflate upload failed") + assert dev.inflates == 1, "device did not inflate the upload" + + # A client that does not offer deflate is served uncompressed + with monkeypatch.context() as m: + m.setattr(espota2, "CLIENT_FEATURE_SUPPORTS_DEFLATE", 0) + await dev.ota(None, None, "uncompressed upload failed") + assert dev.inflates == 1, "device inflated without a client offer" + + # A corrupt stream fails the upload and leaves the device running + with monkeypatch.context() as m: + m.setattr(zlib, "compress", corrupt_compress) + await dev.refused_ota(None, None, "corrupt deflate stream was accepted") + assert errors, "device did not report the corrupt stream" + + # So does a stream that inflates past the announced image size + errors.clear() + with monkeypatch.context() as m: + m.setattr(zlib, "compress", overlong_compress) + await dev.refused_ota(None, None, "overlong deflate stream was accepted") + assert any("Inflate overrun" in line for line in errors), ( + "device wrote past the announced size" + ) + + # and it still takes a good upload afterwards + await dev.ota(None, None, "upload after a rejected stream failed") + assert dev.inflates == 2 + + @pytest.mark.asyncio async def test_host_ota_encrypted( yaml_config: str, diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 2d65e8e079..f4c5935059 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -12,6 +12,7 @@ from pathlib import Path import socket import struct from unittest.mock import Mock, call, patch +import zlib import pytest from pytest import CaptureFixture @@ -354,6 +355,7 @@ def test_perform_ota_successful_md5_auth( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -601,12 +603,16 @@ def test_perform_ota_upload_error(mock_socket: Mock, mock_file: io.BytesIO) -> N espota2.perform_ota(mock_socket, None, mock_file, "test.bin") -def _no_auth_handshake(version: int) -> list[bytes]: +def _no_auth_handshake(version: int, server_features: int | None = None) -> list[bytes]: """Recv responses for a handshake without auth, up to the MD5 check.""" + if server_features is None: + features = [bytes([espota2.RESPONSE_HEADER_OK])] + else: + features = [bytes([espota2.RESPONSE_FEATURE_FLAGS]), bytes([server_features])] return [ bytes([espota2.RESPONSE_OK]), # First byte of version response bytes([version]), # Version number - bytes([espota2.RESPONSE_HEADER_OK]), # Features response + *features, bytes([espota2.RESPONSE_AUTH_OK]), # No auth required bytes([espota2.RESPONSE_UPDATE_PREPARE_OK]), # Binary size OK bytes([espota2.RESPONSE_BIN_MD5_OK]), # MD5 checksum OK @@ -1054,6 +1060,7 @@ def test_perform_ota_successful_sha256_auth( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1110,6 +1117,7 @@ def test_perform_ota_sha256_fallback_to_md5( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1219,6 +1227,7 @@ def test_perform_ota_extended_protocol_app( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1279,6 +1288,7 @@ def test_perform_ota_successful_partition_table( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1507,3 +1517,40 @@ def test_check_error_passes_non_error_when_expect_is_none() -> None: espota2.check_error([espota2.RESPONSE_OK], None) espota2.check_error([espota2.RESPONSE_HEADER_OK], None) espota2.check_error([espota2.RESPONSE_FEATURE_FLAGS], None) + + +# Device replies after the MD5 check for a one-chunk upload +_UPLOAD_TAIL = [ + bytes([espota2.RESPONSE_CHUNK_OK]), + bytes([espota2.RESPONSE_RECEIVE_OK]), + bytes([espota2.RESPONSE_UPDATE_END_OK]), +] + + +@pytest.mark.usefixtures("mock_time") +@pytest.mark.parametrize( + "server_features", + [ + espota2.SERVER_FEATURE_SUPPORTS_DEFLATE, + # Binding offer: deflate wins over gzip + espota2.SERVER_FEATURE_SUPPORTS_DEFLATE + | espota2.SERVER_FEATURE_SUPPORTS_COMPRESSION, + ], +) +def test_perform_ota_with_deflate(mock_socket: Mock, server_features: int) -> None: + """The device gets a raw deflate stream, both sizes and the image MD5.""" + original_content = b"firmware" * 100 + mock_socket.recv.side_effect = ( + _no_auth_handshake(espota2.OTA_VERSION_2_0, server_features) + _UPLOAD_TAIL + ) + + espota2.perform_ota(mock_socket, None, io.BytesIO(original_content), "test.bin") + + sent = [c[0][0] for c in mock_socket.sendall.call_args_list] + # magic, features, ota type, size, image size, md5, data, end ack + sent_size = struct.unpack(">I", sent[3])[0] + assert sent[4] == len(original_content).to_bytes(espota2.SIZE_FIELD_BYTES, "big") + payload = sent[6] + assert len(payload) == sent_size < len(original_content) + assert zlib.decompress(payload, -espota2.DEFLATE_WINDOW_BITS) == original_content + assert sent[5] == hashlib.md5(original_content).hexdigest().encode() From 4724aea42e2b9e6480ee48a276b543d6472ead29 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:00:40 +0200 Subject: [PATCH 052/221] [tcp_uart] Add a TCP client that implements a UART (#19884) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston --- CODEOWNERS | 1 + esphome/components/tcp_uart/__init__.py | 67 +++++++++ esphome/components/tcp_uart/tcp_uart.cpp | 131 ++++++++++++++++++ esphome/components/tcp_uart/tcp_uart.h | 66 +++++++++ tests/components/tcp_uart/common.yaml | 20 +++ .../components/tcp_uart/test.bk72xx-ard.yaml | 2 + tests/components/tcp_uart/test.esp32-idf.yaml | 2 + .../components/tcp_uart/test.esp8266-ard.yaml | 2 + tests/components/tcp_uart/test.host.yaml | 4 + .../components/tcp_uart/test.ln882x-ard.yaml | 2 + .../components/tcp_uart/test.rp2040-ard.yaml | 2 + .../components/tcp_uart/test.rtl87xx-ard.yaml | 2 + 12 files changed, 301 insertions(+) create mode 100644 esphome/components/tcp_uart/__init__.py create mode 100644 esphome/components/tcp_uart/tcp_uart.cpp create mode 100644 esphome/components/tcp_uart/tcp_uart.h create mode 100644 tests/components/tcp_uart/common.yaml create mode 100644 tests/components/tcp_uart/test.bk72xx-ard.yaml create mode 100644 tests/components/tcp_uart/test.esp32-idf.yaml create mode 100644 tests/components/tcp_uart/test.esp8266-ard.yaml create mode 100644 tests/components/tcp_uart/test.host.yaml create mode 100644 tests/components/tcp_uart/test.ln882x-ard.yaml create mode 100644 tests/components/tcp_uart/test.rp2040-ard.yaml create mode 100644 tests/components/tcp_uart/test.rtl87xx-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 12c0a54ac1..5f89315d23 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -560,6 +560,7 @@ esphome/components/tc74/* @sethgirvan esphome/components/tca9548a/* @andreashergert1984 esphome/components/tca9555/* @mobrembski esphome/components/tcl112/* @glmnet +esphome/components/tcp_uart/* @Bascht74 esphome/components/tee501/* @Stock-M esphome/components/teleinfo/* @0hax esphome/components/tem3200/* @bakerkj diff --git a/esphome/components/tcp_uart/__init__.py b/esphome/components/tcp_uart/__init__.py new file mode 100644 index 0000000000..20866d1a0c --- /dev/null +++ b/esphome/components/tcp_uart/__init__.py @@ -0,0 +1,67 @@ +import esphome.codegen as cg +from esphome.components import binary_sensor, socket, uart +from esphome.components.const import ( + CONF_DATA_BITS, + CONF_HOST, + CONF_PARITY, + CONF_STOP_BITS, +) +import esphome.config_validation as cv +from esphome.const import ( + CONF_BAUD_RATE, + CONF_ID, + CONF_PORT, + DEVICE_CLASS_CONNECTIVITY, + ENTITY_CATEGORY_DIAGNOSTIC, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@Bascht74"] +DEPENDENCIES = ["network"] +AUTO_LOAD = ["uart", "binary_sensor", "socket"] +MULTI_CONF = True + +tcp_uart_ns = cg.esphome_ns.namespace("tcp_uart") +TcpUart = tcp_uart_ns.class_("TcpUart", uart.UARTComponent, cg.Component) + +CONF_RECONNECT_INTERVAL = "reconnect_interval" +CONF_CONNECTED = "connected" + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(TcpUart), + cv.Required(CONF_HOST): cv.string, + cv.Required(CONF_PORT): cv.port, + cv.Optional(CONF_BAUD_RATE, default=9600): cv.int_range(min=1), + cv.Optional(CONF_DATA_BITS, default=8): cv.int_range(min=5, max=8), + cv.Optional(CONF_PARITY, default="NONE"): cv.enum( + uart.UART_PARITY_OPTIONS, upper=True + ), + cv.Optional(CONF_STOP_BITS, default=1): cv.one_of(1, 2, int=True), + cv.Optional( + CONF_RECONNECT_INTERVAL, default="5s" + ): cv.positive_time_period_milliseconds, + cv.Optional(CONF_CONNECTED): binary_sensor.binary_sensor_schema( + device_class=DEVICE_CLASS_CONNECTIVITY, + entity_category=ENTITY_CATEGORY_DIAGNOSTIC, + ), + } + ).extend(cv.COMPONENT_SCHEMA), + socket.consume_sockets(1, "tcp_uart"), +) + + +async def to_code(config: ConfigType) -> None: + socket.require_tcp_client_link() + var = cg.new_Pvariable(config[CONF_ID], config[CONF_HOST], config[CONF_PORT]) + await cg.register_component(var, config) + cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL])) + # The socket is not clocked. These only satisfy UARTComponent and a consumer check. + cg.add(var.set_baud_rate(config[CONF_BAUD_RATE])) + cg.add(var.set_data_bits(config[CONF_DATA_BITS])) + cg.add(var.set_stop_bits(config[CONF_STOP_BITS])) + cg.add(var.set_parity(config[CONF_PARITY])) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_CONNECTED, var.set_connected_sensor) diff --git a/esphome/components/tcp_uart/tcp_uart.cpp b/esphome/components/tcp_uart/tcp_uart.cpp new file mode 100644 index 0000000000..fc81f6ecfe --- /dev/null +++ b/esphome/components/tcp_uart/tcp_uart.cpp @@ -0,0 +1,131 @@ +#include "tcp_uart.h" + +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +#include +#include + +namespace esphome::tcp_uart { + +static const char *const TAG = "tcp_uart"; + +static constexpr uint32_t DROP_LOG_INTERVAL_MS = 5000; + +void TcpUart::setup() { + this->link_.begin(TAG); + if (this->connected_sensor_ != nullptr) { + this->connected_sensor_->publish_state(false); + } +} + +void TcpUart::dump_config() { + ESP_LOGCONFIG(TAG, + "TCP UART:\n" + " Host: %s:%u\n" + " Reconnect Interval: %" PRIu32 "ms", + this->link_.host(), this->link_.port(), this->link_.reconnect_interval()); + LOG_BINARY_SENSOR(" ", "Connected", this->connected_sensor_); +} + +void TcpUart::sync_link_() { + bool up = this->link_.connected(); + this->link_was_up_ = up; + if (!up) { + this->rx_start_ = this->rx_end_ = 0; + this->tx_len_ = 0; + } + if (this->connected_sensor_ != nullptr) { + this->connected_sensor_->publish_state(up); + } +} + +void TcpUart::read_socket_() { + if (this->rx_start_ != 0) { + this->rx_end_ -= this->rx_start_; + std::memmove(this->rx_, this->rx_ + this->rx_start_, this->rx_end_); + this->rx_start_ = 0; + } + size_t room = RX_BUFFER_SIZE - this->rx_end_; + if (room == 0) { + // Only a read that filled all free space gets here, so rx_pending_ is already set. + return; + } + ssize_t count = this->link_.read(this->rx_ + this->rx_end_, room); + if (count <= 0) { + // A dropped link (-1) is cleaned up by sync_link_() on the next loop. + if (count == 0) { + this->rx_pending_ = false; + } + return; + } + this->rx_end_ += static_cast(count); + this->rx_pending_ = static_cast(count) == room; +} + +void TcpUart::flush_tx_() { + ssize_t sent = this->link_.write(this->tx_, this->tx_len_); + if (sent > 0) { + this->tx_len_ -= static_cast(sent); + std::memmove(this->tx_, this->tx_ + sent, this->tx_len_); + } +} + +void TcpUart::loop() { + this->link_.poll(); + if (this->link_.connected() != this->link_was_up_) { + this->sync_link_(); + } + if (!this->link_was_up_) { + return; + } + if (this->rx_pending_ || this->link_.ready()) { + this->read_socket_(); + } + if (this->tx_len_ != 0) { + this->flush_tx_(); + } +} + +void TcpUart::write_array(const uint8_t *data, size_t len) { + size_t room = this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0; + if (len > room) { + uint32_t now = App.get_loop_component_start_time(); + if (this->last_drop_log_ms_ == 0 || now - this->last_drop_log_ms_ >= DROP_LOG_INTERVAL_MS) { + ESP_LOGW(TAG, "%s, dropped %u bytes", + this->link_.connected() ? LOG_STR_LITERAL("TX buffer full") : LOG_STR_LITERAL("Not connected"), + static_cast(len - room)); + this->last_drop_log_ms_ = now; + } + len = room; + } + std::memcpy(this->tx_ + this->tx_len_, data, len); + this->tx_len_ += static_cast(len); +} + +bool TcpUart::peek_byte(uint8_t *data) { + if (this->rx_start_ == this->rx_end_) { + return false; + } + *data = this->rx_[this->rx_start_]; + return true; +} + +bool TcpUart::read_array(uint8_t *data, size_t len) { + if (this->available() < len) { + return false; + } + std::memcpy(data, this->rx_ + this->rx_start_, len); + this->rx_start_ += static_cast(len); + return true; +} + +uart::UARTFlushResult TcpUart::flush() { + this->flush_tx_(); + if (this->tx_len_ == 0) { + return uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS; + } + return uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT; +} + +} // namespace esphome::tcp_uart diff --git a/esphome/components/tcp_uart/tcp_uart.h b/esphome/components/tcp_uart/tcp_uart.h new file mode 100644 index 0000000000..c13d2ef6f0 --- /dev/null +++ b/esphome/components/tcp_uart/tcp_uart.h @@ -0,0 +1,66 @@ +#pragma once + +#include "esphome/components/binary_sensor/binary_sensor.h" +#include "esphome/components/socket/tcp_client_link.h" +#include "esphome/components/uart/uart_component.h" +#include "esphome/core/component.h" + +#include + +namespace esphome::tcp_uart { + +/// TCP client presented as a UART. Bytes are copied unchanged. +class TcpUart : public uart::UARTComponent, public Component { + public: + TcpUart(const char *host, uint16_t port) { + this->link_.set_host(host); + this->link_.set_port(port); + this->rx_buffer_size_ = RX_BUFFER_SIZE; + } + + void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); } + void set_connected_sensor(binary_sensor::BinarySensor *sensor) { this->connected_sensor_ = sensor; } + + void setup() override; + void loop() override; + void dump_config() override; + void on_shutdown() override { this->link_.close(); } + float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } + + void write_array(const uint8_t *data, size_t len) override; + bool peek_byte(uint8_t *data) override; + bool read_array(uint8_t *data, size_t len) override; + size_t available() override { return static_cast(this->rx_end_ - this->rx_start_); } + // Same room write_array() grants, so consumers can apply backpressure. + size_t available_for_write() override { return this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0; } + uart::UARTFlushResult flush() override; + bool is_connected() override { return this->link_.connected(); } +#if defined(USE_ESP8266) || defined(USE_ESP32) + void load_settings(bool dump_config) override {} +#endif + + protected: + void check_logger_conflict() override {} + void sync_link_(); + void read_socket_(); + void flush_tx_(); + + static constexpr size_t RX_BUFFER_SIZE = 1024; + static constexpr size_t TX_BUFFER_SIZE = 1024; + + socket::TcpClientLink link_; + binary_sensor::BinarySensor *connected_sensor_{nullptr}; + uint32_t last_drop_log_ms_{0}; + uint16_t tx_len_{0}; + // rx_[rx_start_, rx_end_) holds unread bytes; read_socket_() compacts to the front. + uint16_t rx_start_{0}; + uint16_t rx_end_{0}; + // The link state loop() saw last; edges clear the buffers and publish the sensor. + bool link_was_up_{false}; + // A read stopped before EAGAIN. ready() stays false until new data arrives. + bool rx_pending_{false}; + uint8_t rx_[RX_BUFFER_SIZE]{}; + uint8_t tx_[TX_BUFFER_SIZE]{}; +}; + +} // namespace esphome::tcp_uart diff --git a/tests/components/tcp_uart/common.yaml b/tests/components/tcp_uart/common.yaml new file mode 100644 index 0000000000..f3c1d061aa --- /dev/null +++ b/tests/components/tcp_uart/common.yaml @@ -0,0 +1,20 @@ +wifi: + ssid: MySSID + password: password1 + +tcp_uart: + - id: tcp_uart_1 + host: 192.0.2.10 + port: 502 + reconnect_interval: 10s + connected: + name: TCP UART Connected + +interval: + - interval: 60s + then: + - lambda: |- + uint8_t byte; + if (id(tcp_uart_1).available() && id(tcp_uart_1).read_byte(&byte)) { + id(tcp_uart_1).write_byte(byte); + } diff --git a/tests/components/tcp_uart/test.bk72xx-ard.yaml b/tests/components/tcp_uart/test.bk72xx-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.bk72xx-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.esp32-idf.yaml b/tests/components/tcp_uart/test.esp32-idf.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.esp8266-ard.yaml b/tests/components/tcp_uart/test.esp8266-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.host.yaml b/tests/components/tcp_uart/test.host.yaml new file mode 100644 index 0000000000..22d7f7fbaa --- /dev/null +++ b/tests/components/tcp_uart/test.host.yaml @@ -0,0 +1,4 @@ +packages: + tcp_uart: !include common.yaml + +wifi: !remove diff --git a/tests/components/tcp_uart/test.ln882x-ard.yaml b/tests/components/tcp_uart/test.ln882x-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.ln882x-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.rp2040-ard.yaml b/tests/components/tcp_uart/test.rp2040-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.rp2040-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.rtl87xx-ard.yaml b/tests/components/tcp_uart/test.rtl87xx-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.rtl87xx-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml From 0c0015ef88daf762d9c1b9bb7f6b3fb75de14e20 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 08:35:07 -0500 Subject: [PATCH 053/221] [const] Move the role and reconnect interval keys to shared constants (#20009) --- esphome/components/const/__init__.py | 2 ++ esphome/components/modbus/__init__.py | 2 +- esphome/components/openthread_info/text_sensor.py | 2 +- esphome/components/tcp_uart/__init__.py | 2 +- .../tcp_client_link_test_component/__init__.py | 4 +--- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 97d757c9d4..b67964d47f 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -41,7 +41,9 @@ CONF_ON_SCAN_END = "on_scan_end" CONF_ON_STATE_CHANGE = "on_state_change" CONF_PARITY = "parity" CONF_RECEIVER_FREQUENCY = "receiver_frequency" +CONF_RECONNECT_INTERVAL = "reconnect_interval" CONF_REQUEST_HEADERS = "request_headers" +CONF_ROLE = "role" CONF_ROWS = "rows" CONF_SCAN_PARAMETERS = "scan_parameters" CONF_SHA256 = "sha256" diff --git a/esphome/components/modbus/__init__.py b/esphome/components/modbus/__init__.py index fe93758726..db7484e306 100644 --- a/esphome/components/modbus/__init__.py +++ b/esphome/components/modbus/__init__.py @@ -7,6 +7,7 @@ from typing import Any, Literal, NamedTuple from esphome import pins import esphome.codegen as cg from esphome.components import uart +from esphome.components.const import CONF_ROLE import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_CONTINUOUS, CONF_FLOW_CONTROL_PIN, CONF_ID from esphome.cpp_generator import MockObj @@ -45,7 +46,6 @@ MULTI_CONF = True CONF_ALLOW_BROADCAST_READ = "allow_broadcast_read" CONF_EXPECT_BROADCAST_WRITE_RESPONSE = "expect_broadcast_write_response" -CONF_ROLE = "role" CONF_MODBUS_ID = "modbus_id" CONF_SEND_WAIT_TIME = "send_wait_time" CONF_TURNAROUND_TIME = "turnaround_time" diff --git a/esphome/components/openthread_info/text_sensor.py b/esphome/components/openthread_info/text_sensor.py index da789ae706..4d3e8cd6c3 100644 --- a/esphome/components/openthread_info/text_sensor.py +++ b/esphome/components/openthread_info/text_sensor.py @@ -1,5 +1,6 @@ import esphome.codegen as cg from esphome.components import text_sensor +from esphome.components.const import CONF_ROLE from esphome.components.openthread.const import ( CONF_EXT_PAN_ID, CONF_NETWORK_KEY, @@ -10,7 +11,6 @@ import esphome.config_validation as cv from esphome.const import CONF_CHANNEL, CONF_IP_ADDRESS, ENTITY_CATEGORY_DIAGNOSTIC from esphome.types import ConfigType -CONF_ROLE = "role" CONF_RLOC16 = "rloc16" CONF_EUI64 = "eui64" CONF_EXT_ADDR = "ext_addr" diff --git a/esphome/components/tcp_uart/__init__.py b/esphome/components/tcp_uart/__init__.py index 20866d1a0c..e46e4dbfd0 100644 --- a/esphome/components/tcp_uart/__init__.py +++ b/esphome/components/tcp_uart/__init__.py @@ -4,6 +4,7 @@ from esphome.components.const import ( CONF_DATA_BITS, CONF_HOST, CONF_PARITY, + CONF_RECONNECT_INTERVAL, CONF_STOP_BITS, ) import esphome.config_validation as cv @@ -24,7 +25,6 @@ MULTI_CONF = True tcp_uart_ns = cg.esphome_ns.namespace("tcp_uart") TcpUart = tcp_uart_ns.class_("TcpUart", uart.UARTComponent, cg.Component) -CONF_RECONNECT_INTERVAL = "reconnect_interval" CONF_CONNECTED = "connected" diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py index 24637ae586..58affd23b7 100644 --- a/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py @@ -1,14 +1,12 @@ import esphome.codegen as cg from esphome.components import socket -from esphome.components.const import CONF_HOST +from esphome.components.const import CONF_HOST, CONF_RECONNECT_INTERVAL import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_PORT from esphome.types import ConfigType AUTO_LOAD = ["socket"] -CONF_RECONNECT_INTERVAL = "reconnect_interval" - tcp_client_link_test_component_ns = cg.esphome_ns.namespace( "tcp_client_link_test_component" ) From a27c73b44e26b41332e039a63581f6c6f17efc77 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:38:20 +0100 Subject: [PATCH 054/221] [wifi] Warn when eap is configured with esp32_hosted (#20011) --- esphome/components/wifi/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 1ba8946841..fa552da969 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -368,6 +368,13 @@ def final_validate(config): "The AP will not be usable for configuration or monitoring. " "Add 'captive_portal:' or 'web_server:' to your configuration." ) + if "esp32_hosted" in full_config and any( + CONF_EAP in net for net in config.get(CONF_NETWORKS, []) + ): + _LOGGER.warning( + "WPA2 Enterprise ('eap:') is not supported by the esp32_hosted coprocessor " + "firmware ESPHome provides" + ) def _consume_wifi_sockets(config: ConfigType) -> ConfigType: From d938bc9913f6cf133e52122aadc7eb44fdd48855 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:06:56 -0500 Subject: [PATCH 055/221] [ee895] Use the sub_* helpers for optional child entities (#20005) --- esphome/components/ee895/sensor.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/esphome/components/ee895/sensor.py b/esphome/components/ee895/sensor.py index fdad47fb05..a7110bd6d2 100644 --- a/esphome/components/ee895/sensor.py +++ b/esphome/components/ee895/sensor.py @@ -57,14 +57,7 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature_sensor(sens)) - - if co2 := config.get(CONF_CO2): - sens = await sensor.new_sensor(co2) - cg.add(var.set_co2_sensor(sens)) - - if pressure := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure) - cg.add(var.set_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_CO2, var.set_co2_sensor) + await sensors(CONF_PRESSURE, var.set_pressure_sensor) From 87d4bea24b973e95ed9f917de8be52de92f12c52 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:07:14 -0500 Subject: [PATCH 056/221] [pm2005] Use the sub_* helpers for optional child entities (#20004) --- esphome/components/pm2005/sensor.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/esphome/components/pm2005/sensor.py b/esphome/components/pm2005/sensor.py index f16c100e5e..a0c133eba1 100644 --- a/esphome/components/pm2005/sensor.py +++ b/esphome/components/pm2005/sensor.py @@ -74,14 +74,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_sensor_type(config[CONF_TYPE])) - if pm_1_0_config := config.get(CONF_PM_1_0): - sens = await sensor.new_sensor(pm_1_0_config) - cg.add(var.set_pm_1_0_sensor(sens)) - - if pm_2_5_config := config.get(CONF_PM_2_5): - sens = await sensor.new_sensor(pm_2_5_config) - cg.add(var.set_pm_2_5_sensor(sens)) - - if pm_10_0_config := config.get(CONF_PM_10_0): - sens = await sensor.new_sensor(pm_10_0_config) - cg.add(var.set_pm_10_0_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_PM_1_0, var.set_pm_1_0_sensor) + await sensors(CONF_PM_2_5, var.set_pm_2_5_sensor) + await sensors(CONF_PM_10_0, var.set_pm_10_0_sensor) From 65fef7d5d862b1f202d032b05ac990d6718e880a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:07:26 -0500 Subject: [PATCH 057/221] [cs5460a] Use the sub_* helpers for optional child entities (#20006) --- esphome/components/cs5460a/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/cs5460a/sensor.py b/esphome/components/cs5460a/sensor.py index 17ea41ed04..d087c99bf1 100644 --- a/esphome/components/cs5460a/sensor.py +++ b/esphome/components/cs5460a/sensor.py @@ -117,15 +117,10 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_hpf_enable(config[CONF_CURRENT_HPF], config[CONF_VOLTAGE_HPF])) cg.add(var.set_pulse_energy_wh(config[CONF_PULSE_ENERGY])) - if voltage_config := config.get(CONF_VOLTAGE): - sens = await sensor.new_sensor(voltage_config) - cg.add(var.set_voltage_sensor(sens)) - if current_config := config.get(CONF_CURRENT): - sens = await sensor.new_sensor(current_config) - cg.add(var.set_current_sensor(sens)) - if power_config := config.get(CONF_POWER): - sens = await sensor.new_sensor(power_config) - cg.add(var.set_power_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_VOLTAGE, var.set_voltage_sensor) + await sensors(CONF_CURRENT, var.set_current_sensor) + await sensors(CONF_POWER, var.set_power_sensor) automation.register_apply_action( From 7b018b9dcad0dec6af34f52a964315b8f6717d8f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:07:35 -0500 Subject: [PATCH 058/221] [socket] Move the outgoing buffer into the TCP client link (#20010) --- esphome/components/socket/tcp_client_link.cpp | 24 ++++++++++++++- esphome/components/socket/tcp_client_link.h | 28 ++++++++++++++++-- esphome/components/tcp_uart/tcp_uart.cpp | 29 ++++--------------- esphome/components/tcp_uart/tcp_uart.h | 8 ++--- .../tcp_client_link_test_component.cpp | 10 +++++-- 5 files changed, 64 insertions(+), 35 deletions(-) diff --git a/esphome/components/socket/tcp_client_link.cpp b/esphome/components/socket/tcp_client_link.cpp index f2c1b8e4c1..141695a4bb 100644 --- a/esphome/components/socket/tcp_client_link.cpp +++ b/esphome/components/socket/tcp_client_link.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace esphome::socket { @@ -114,7 +115,7 @@ ssize_t TcpClientLink::read(uint8_t *buf, size_t len) { return 0; } -ssize_t TcpClientLink::write(const uint8_t *buf, size_t len) { +ssize_t TcpClientLink::write_(const uint8_t *buf, size_t len) { if (!this->connected_ || len == 0) { return 0; } @@ -129,6 +130,26 @@ ssize_t TcpClientLink::write(const uint8_t *buf, size_t len) { return -1; } +size_t TcpClientLink::queue(const uint8_t *data, size_t len) { + size_t room = this->tx_free(); + if (len > room) { + len = room; + } + std::memcpy(this->tx_ + this->tx_len_, data, len); + this->tx_len_ += static_cast(len); + return len; +} + +void TcpClientLink::flush_tx_slow_() { + ssize_t sent = this->write_(this->tx_, this->tx_len_); + if (sent > 0) { + this->tx_len_ -= static_cast(sent); + if (this->tx_len_ != 0) { + std::memmove(this->tx_, this->tx_ + sent, this->tx_len_); + } + } +} + void TcpClientLink::close() { if (this->sock_ != nullptr) { this->sock_->shutdown(SHUT_RDWR); @@ -136,6 +157,7 @@ void TcpClientLink::close() { this->sock_.reset(); } this->connected_ = false; + this->tx_len_ = 0; this->resolved_.forget(); } diff --git a/esphome/components/socket/tcp_client_link.h b/esphome/components/socket/tcp_client_link.h index 065c4df562..f77ee05055 100644 --- a/esphome/components/socket/tcp_client_link.h +++ b/esphome/components/socket/tcp_client_link.h @@ -16,8 +16,9 @@ namespace esphome::socket { /// A reconnecting TCP stream driven from loop(). Owns the socket, the DNS -/// lookup and the retry backoff. A fatal read/write error closes the link -/// and schedules the next attempt; the caller sees the edge via connected(). +/// lookup, the retry backoff and the outgoing buffer. A fatal read/write +/// error closes the link and schedules the next attempt; the caller sees +/// the edge via connected(). class TcpClientLink { public: void set_host(const char *host) { this->host_ = StringRef(host); } @@ -41,7 +42,21 @@ class TcpClientLink { void adopt(std::unique_ptr sock); /// Returns bytes moved, 0 when nothing can move now, -1 when the link dropped. ssize_t read(uint8_t *buf, size_t len); - ssize_t write(const uint8_t *buf, size_t len); + /// Copy into the outgoing buffer; returns how many bytes fit. + size_t queue(const uint8_t *data, size_t len); + /// Direct access to the buffer's free tail. Fill at most tx_free() bytes, + /// then tx_commit() the count; neither is bounds checked. + uint8_t *tx_tail() { return this->tx_ + this->tx_len_; } + void tx_commit(size_t len) { this->tx_len_ += static_cast(len); } + size_t tx_free() const { return this->connected_ ? TX_BUFFER_SIZE - this->tx_len_ : 0; } + /// Send the front of the buffer; true once it is empty. + /// A partial write keeps the rest; inline no-op while nothing is queued. + bool flush_tx() { + if (this->tx_len_ != 0) { + this->flush_tx_slow_(); + } + return this->tx_len_ == 0; + } /// Close without scheduling a reconnect (shutdown). void close(); @@ -54,6 +69,11 @@ class TcpClientLink { } protected: + static constexpr size_t TX_BUFFER_SIZE = 1024; + + /// The raw stream write behind flush_tx(); drops the link on a fatal error. + ssize_t write_(const uint8_t *buf, size_t len); + void flush_tx_slow_(); void poll_slow_(); void try_connect_(); /// Close after a failure, log what and errno, schedule the next attempt. @@ -66,7 +86,9 @@ class TcpClientLink { uint32_t reconnect_interval_ms_{5000}; Ipv4Resolve resolved_; uint16_t port_{0}; + uint16_t tx_len_{0}; bool connected_{false}; + uint8_t tx_[TX_BUFFER_SIZE]{}; }; } // namespace esphome::socket diff --git a/esphome/components/tcp_uart/tcp_uart.cpp b/esphome/components/tcp_uart/tcp_uart.cpp index fc81f6ecfe..00b59d82ba 100644 --- a/esphome/components/tcp_uart/tcp_uart.cpp +++ b/esphome/components/tcp_uart/tcp_uart.cpp @@ -33,7 +33,6 @@ void TcpUart::sync_link_() { this->link_was_up_ = up; if (!up) { this->rx_start_ = this->rx_end_ = 0; - this->tx_len_ = 0; } if (this->connected_sensor_ != nullptr) { this->connected_sensor_->publish_state(up); @@ -63,14 +62,6 @@ void TcpUart::read_socket_() { this->rx_pending_ = static_cast(count) == room; } -void TcpUart::flush_tx_() { - ssize_t sent = this->link_.write(this->tx_, this->tx_len_); - if (sent > 0) { - this->tx_len_ -= static_cast(sent); - std::memmove(this->tx_, this->tx_ + sent, this->tx_len_); - } -} - void TcpUart::loop() { this->link_.poll(); if (this->link_.connected() != this->link_was_up_) { @@ -82,25 +73,20 @@ void TcpUart::loop() { if (this->rx_pending_ || this->link_.ready()) { this->read_socket_(); } - if (this->tx_len_ != 0) { - this->flush_tx_(); - } + this->link_.flush_tx(); } void TcpUart::write_array(const uint8_t *data, size_t len) { - size_t room = this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0; - if (len > room) { + size_t queued = this->link_.queue(data, len); + if (queued < len) { uint32_t now = App.get_loop_component_start_time(); if (this->last_drop_log_ms_ == 0 || now - this->last_drop_log_ms_ >= DROP_LOG_INTERVAL_MS) { ESP_LOGW(TAG, "%s, dropped %u bytes", this->link_.connected() ? LOG_STR_LITERAL("TX buffer full") : LOG_STR_LITERAL("Not connected"), - static_cast(len - room)); + static_cast(len - queued)); this->last_drop_log_ms_ = now; } - len = room; } - std::memcpy(this->tx_ + this->tx_len_, data, len); - this->tx_len_ += static_cast(len); } bool TcpUart::peek_byte(uint8_t *data) { @@ -121,11 +107,8 @@ bool TcpUart::read_array(uint8_t *data, size_t len) { } uart::UARTFlushResult TcpUart::flush() { - this->flush_tx_(); - if (this->tx_len_ == 0) { - return uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS; - } - return uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT; + return this->link_.flush_tx() ? uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS + : uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT; } } // namespace esphome::tcp_uart diff --git a/esphome/components/tcp_uart/tcp_uart.h b/esphome/components/tcp_uart/tcp_uart.h index c13d2ef6f0..7ce822c2fb 100644 --- a/esphome/components/tcp_uart/tcp_uart.h +++ b/esphome/components/tcp_uart/tcp_uart.h @@ -32,7 +32,7 @@ class TcpUart : public uart::UARTComponent, public Component { bool read_array(uint8_t *data, size_t len) override; size_t available() override { return static_cast(this->rx_end_ - this->rx_start_); } // Same room write_array() grants, so consumers can apply backpressure. - size_t available_for_write() override { return this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0; } + size_t available_for_write() override { return this->link_.tx_free(); } uart::UARTFlushResult flush() override; bool is_connected() override { return this->link_.connected(); } #if defined(USE_ESP8266) || defined(USE_ESP32) @@ -43,24 +43,20 @@ class TcpUart : public uart::UARTComponent, public Component { void check_logger_conflict() override {} void sync_link_(); void read_socket_(); - void flush_tx_(); static constexpr size_t RX_BUFFER_SIZE = 1024; - static constexpr size_t TX_BUFFER_SIZE = 1024; socket::TcpClientLink link_; binary_sensor::BinarySensor *connected_sensor_{nullptr}; uint32_t last_drop_log_ms_{0}; - uint16_t tx_len_{0}; // rx_[rx_start_, rx_end_) holds unread bytes; read_socket_() compacts to the front. uint16_t rx_start_{0}; uint16_t rx_end_{0}; - // The link state loop() saw last; edges clear the buffers and publish the sensor. + // The link state loop() saw last; edges clear rx_ and publish the sensor. bool link_was_up_{false}; // A read stopped before EAGAIN. ready() stays false until new data arrives. bool rx_pending_{false}; uint8_t rx_[RX_BUFFER_SIZE]{}; - uint8_t tx_[TX_BUFFER_SIZE]{}; }; } // namespace esphome::tcp_uart diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp index 7f2af8add8..bd45e60a00 100644 --- a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp @@ -14,14 +14,20 @@ void TcpClientLinkTestComponent::loop() { this->was_up_ = up; ESP_LOGI(TAG, "Link %s", up ? LOG_STR_LITERAL("up") : LOG_STR_LITERAL("down")); } - if (!up || !this->link_.ready()) { + if (!up) { + return; + } + // Retries a partial echo; inline no-op when nothing is queued. + this->link_.flush_tx(); + if (!this->link_.ready()) { return; } uint8_t buf[64]; ssize_t count = this->link_.read(buf, sizeof(buf)); if (count > 0) { ESP_LOGI(TAG, "Echoing %d bytes", static_cast(count)); - this->link_.write(buf, static_cast(count)); + this->link_.queue(buf, static_cast(count)); + this->link_.flush_tx(); } } From 54cb4c711a3118d9a92a673612ae753fe8c17eef Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:07:39 -0500 Subject: [PATCH 059/221] [mmc5983] Use the sub_* helpers for optional child entities (#20007) --- esphome/components/mmc5983/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/mmc5983/sensor.py b/esphome/components/mmc5983/sensor.py index 797181690f..48c532b0c6 100644 --- a/esphome/components/mmc5983/sensor.py +++ b/esphome/components/mmc5983/sensor.py @@ -45,12 +45,7 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if x_config := config.get(CONF_FIELD_STRENGTH_X): - sens = await sensor.new_sensor(x_config) - cg.add(var.set_x_sensor(sens)) - if y_config := config.get(CONF_FIELD_STRENGTH_Y): - sens = await sensor.new_sensor(y_config) - cg.add(var.set_y_sensor(sens)) - if z_config := config.get(CONF_FIELD_STRENGTH_Z): - sens = await sensor.new_sensor(z_config) - cg.add(var.set_z_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_FIELD_STRENGTH_X, var.set_x_sensor) + await sensors(CONF_FIELD_STRENGTH_Y, var.set_y_sensor) + await sensors(CONF_FIELD_STRENGTH_Z, var.set_z_sensor) From 4c9a44301d6022048c123db0173f54b771f1cdd5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:07:52 -0500 Subject: [PATCH 060/221] [zyaura] Use the sub_* helpers for optional child entities (#20008) --- esphome/components/zyaura/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/zyaura/sensor.py b/esphome/components/zyaura/sensor.py index 0508f456e4..cfe980f037 100644 --- a/esphome/components/zyaura/sensor.py +++ b/esphome/components/zyaura/sensor.py @@ -61,12 +61,7 @@ async def to_code(config: ConfigType) -> None: pin_data = await gpio_pin_expression(config[CONF_DATA_PIN]) cg.add(var.set_pin_data(pin_data)) - if CONF_CO2 in config: - sens = await sensor.new_sensor(config[CONF_CO2]) - cg.add(var.set_co2_sensor(sens)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) From 1440998f3c1f65e61a65ad786d6424c8339bb69f Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:09:14 +0200 Subject: [PATCH 061/221] [core] Add new options suspend_loop and loop_interval (#18136) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/config.py | 1 + esphome/core/application.h | 44 +++++++-- esphome/core/config.py | 63 +++++++++++++ esphome/core/defines.h | 4 + tests/components/esphome/common.yaml | 2 + tests/components/esphome/test.esp32-idf.yaml | 6 +- .../components/esphome/test.esp8266-ard.yaml | 6 +- .../esphome/test.nrf52-adafruit.yaml | 6 +- tests/unit_tests/core/test_config.py | 92 +++++++++++++++++++ .../core/config/loop_interval_bk72xx.yaml | 7 ++ .../core/config/loop_interval_esp32.yaml | 7 ++ .../core/config/loop_interval_nrf52.yaml | 6 ++ .../core/config/suspend_loop_esp32.yaml | 9 ++ .../core/config/suspend_loop_host.yaml | 5 + .../core/config/suspend_loop_rp2.yaml | 6 ++ 15 files changed, 255 insertions(+), 9 deletions(-) create mode 100644 tests/unit_tests/fixtures/core/config/loop_interval_bk72xx.yaml create mode 100644 tests/unit_tests/fixtures/core/config/loop_interval_esp32.yaml create mode 100644 tests/unit_tests/fixtures/core/config/loop_interval_nrf52.yaml create mode 100644 tests/unit_tests/fixtures/core/config/suspend_loop_esp32.yaml create mode 100644 tests/unit_tests/fixtures/core/config/suspend_loop_host.yaml create mode 100644 tests/unit_tests/fixtures/core/config/suspend_loop_rp2.yaml diff --git a/esphome/config.py b/esphome/config.py index 13ec744ce4..bc69661d8f 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -1226,6 +1226,7 @@ class CoreFinalValidateStep(ConfigValidationStep): with result.catch_error([CONF_ESPHOME]): if CONF_ESPHOME in result: core_config.validate_ids_and_references(result[CONF_ESPHOME]) + core_config.validate_loop_interval(result[CONF_ESPHOME]) fv.full_config.reset(token) diff --git a/esphome/core/application.h b/esphome/core/application.h index 8ed4c09096..2f399cbdcd 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -208,8 +208,8 @@ class Application { * Each component can request a high frequency loop execution by using the HighFrequencyLoopRequester * helper in helpers.h * - * Note: This method is not called by ESPHome core code. It is only used by lambda functions - * in YAML configurations or by external components. + * Sleep per wake is capped at 2 * WDT_FEED_INTERVAL_MS (except host and ESP8266); + * raise the platform watchdog timeout to sleep longer. * * @param loop_interval The interval in milliseconds to run the core loop at. Defaults to 16 milliseconds. */ @@ -232,6 +232,7 @@ class Application { /// - ESP8266 soft WDT (~1.6 s): ~16x <-- 100 ms feed (see USE_ESP8266 below) /// - ESP8266 HW WDT (~6 s): ~60x /// - BK72xx HW WDT (10 s): ~5x <-- platform override below + /// Important: if these are modified align validate_loop_interval in config.py #ifdef USE_BK72XX // BDK busy-waits 200us per WDT reload (sctrl_dpll_delay200us). LibreTiny // sets HW WDT to 10s; 2000ms keeps ~5x margin. See wdt_ctrl WCMD_RELOAD_PERIOD: @@ -775,8 +776,8 @@ inline void ESPHOME_ALWAYS_INLINE Application::loop() { } #endif - // Compute sleep: bounded by time-until-next-component-phase and the - // scheduler's next deadline. When a scheduler timer fires it re-enters + // Compute sleep: bounded by time-until-next-component-phase if there are + // components with loop enabled and the scheduler's next deadline. When a scheduler timer fires it re-enters // loop(), Phase A services it, and the component phase stays gated by // loop_interval_. When a background producer calls wake_loop_threadsafe() // it sets the wake_request flag and wakes select() / the task notification; @@ -795,16 +796,47 @@ inline void ESPHOME_ALWAYS_INLINE Application::loop() { uint32_t delay_time = 0; if (!HighFrequencyLoopRequester::is_high_frequency()) { const uint32_t elapsed_since_phase = now - this->last_loop_; - const uint32_t until_phase = +#ifdef ESPHOME_SUSPEND_LOOP + const bool has_loop_work = + this->looping_components_active_end_ > 0 || this->dump_config_at_ < this->components_.size(); + uint32_t until_phase = std::numeric_limits::max(); + if (has_loop_work) { + until_phase = (elapsed_since_phase >= this->loop_interval_) ? 0 : (this->loop_interval_ - elapsed_since_phase); + } +#else + uint32_t until_phase = (elapsed_since_phase >= this->loop_interval_) ? 0 : (this->loop_interval_ - elapsed_since_phase); +#endif const uint32_t until_sched = this->scheduler.next_schedule_in(now).value_or(until_phase); delay_time = std::min(until_phase, until_sched); } // All platforms route loop yields through the platform wake primitive. // On host this drains the loopback wake socket via select(); on FreeRTOS // targets it uses task notifications; on ESP8266/RP2040 it uses esp_delay/WFE. - esphome::internal::wakeable_delay(delay_time); + // Cap the sleep so the WDT feed and status-LED dispatch rate limits still get + // exercised even when loop_interval is raised or the scheduler and component + // phases are gated out for a long sleep. Waking every 2*WDT_FEED_INTERVAL_MS + // clears the feed rate limit on every wake, so the WDT is fed at least that + // often -- well inside every platform's timeout. +#if defined(USE_ESP8266) + // SDK os_timer_arm() accepts at most 0x68D7A3 ms without system_timer_reinit(); + // the SDK feeds both watchdogs while the cont task is suspended, so no WDT cap needed. + static constexpr uint32_t MAX_SLEEP_BASE = 0x68D7A3; +#elif defined(USE_HOST) + // arch_feed_wdt() is a no-op on host and ESPHOME_SUSPEND_LOOP is rejected by + // the config validator, so delay_time is already bounded by loop_interval_. + static constexpr uint32_t MAX_SLEEP_BASE = std::numeric_limits::max(); +#else + static constexpr uint32_t MAX_SLEEP_BASE = WDT_FEED_INTERVAL_MS * 2; +#endif + uint32_t max_sleep = MAX_SLEEP_BASE; +#ifdef USE_STATUS_LED + if ((this->app_state_ & STATUS_LED_MASK) != 0) { + max_sleep = std::min(max_sleep, STATUS_LED_DISPATCH_INTERVAL_MS); + } +#endif + esphome::internal::wakeable_delay(std::min(delay_time, max_sleep)); if (this->dump_config_at_ < this->components_.size()) { this->process_dump_config_(); } diff --git a/esphome/core/config.py b/esphome/core/config.py index 532ca7c36d..531a027647 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -43,7 +43,11 @@ from esphome.const import ( CONF_PROJECT, CONF_TRIGGER_ID, CONF_VERSION, + CONF_WATCHDOG_TIMEOUT, KEY_CORE, + PLATFORM_ESP32, + PLATFORM_HOST, + PLATFORM_RP2, PlatformFramework, __version__ as ESPHOME_VERSION, ) @@ -53,6 +57,7 @@ from esphome.core import ( CoroPriority, coroutine_with_priority, ) +import esphome.final_validate as fv from esphome.helpers import ( copy_file_if_changed, cpp_string_escape, @@ -63,6 +68,9 @@ from esphome.helpers import ( ) from esphome.types import ConfigType +CONF_LOOP_INTERVAL = "loop_interval" +CONF_SUSPEND_LOOP = "suspend_loop" + _LOGGER = logging.getLogger(__name__) # C++ variable names and separators for app name buffers (used with MAC suffix) @@ -186,6 +194,35 @@ def validate_ids_and_references(config: ConfigType) -> ConfigType: return config +def validate_loop_interval(config: ConfigType) -> ConfigType: + if CONF_LOOP_INTERVAL in config and not (CORE.is_host or CORE.is_esp8266): + # max_loop interval is calculated from WDT_FEED_INTERVAL_MS. Make sure to align with application.h + max_loop = 600 # 2 * 300ms default for other platforms + if CORE.is_esp32: + max_loop = ( + 2 + * fv.full_config.get()[PLATFORM_ESP32][ + CONF_WATCHDOG_TIMEOUT + ].total_milliseconds + // 5 + ) + elif CORE.is_bk72xx: + max_loop = 4000 # 10000ms / 5 * 2 default value + if config[CONF_LOOP_INTERVAL].total_milliseconds > max_loop: + _LOGGER.warning( + "%s of %s exceeds the %sms maximum sleep on this platform; the loop will still " + "wake every %sms.%s", + CONF_LOOP_INTERVAL, + config[CONF_LOOP_INTERVAL], + max_loop, + max_loop, + " Raise esp32.watchdog_timeout to sleep longer." + if CORE.is_esp32 + else "", + ) + return config + + def valid_include(value: str) -> str: # Look for "<...>" includes if value.startswith("<") and value.endswith(">"): @@ -266,6 +303,15 @@ def validate_area_config(config: dict | str) -> dict[str, str | core.ID]: return cv.maybe_simple_value(AREA_SCHEMA, key=CONF_NAME)(config) +def _validate_suspend_loop(value: bool) -> bool: + # host and RP2 platforms have unwakeable delay fallbacks, so suspending the main loop is unsafe + if value and CORE.target_platform in [PLATFORM_HOST, PLATFORM_RP2]: + raise cv.Invalid( + f"Suspend loop is not available on {CORE.target_platform} platform" + ) + return value + + CONFIG_SCHEMA = cv.All( cv.Schema( { @@ -331,6 +377,19 @@ CONFIG_SCHEMA = cv.All( cv.Optional( CONF_DEBUG_SCHEDULER, default=False, visibility=cv.Visibility.YAML_ONLY ): cv.boolean, + cv.Optional(CONF_LOOP_INTERVAL, visibility=cv.Visibility.YAML_ONLY): cv.All( + cv.positive_time_period_milliseconds, + cv.Range( + min=cv.TimePeriod(milliseconds=1), + max=cv.TimePeriod(milliseconds=65535), + ), + ), + cv.Optional( + CONF_SUSPEND_LOOP, default=False, visibility=cv.Visibility.YAML_ONLY + ): cv.All( + cv.boolean, + _validate_suspend_loop, + ), cv.Optional(CONF_PROJECT): cv.Schema( { cv.Required(CONF_NAME): cv.All( @@ -806,6 +865,10 @@ async def to_code(config: ConfigType) -> None: cg.add_cxx_build_flag("-Wno-volatile") if config[CONF_DEBUG_SCHEDULER]: cg.add_define("ESPHOME_DEBUG_SCHEDULER") + if config[CONF_SUSPEND_LOOP]: + cg.add_define("ESPHOME_SUSPEND_LOOP") + if CONF_LOOP_INTERVAL in config: + cg.add(cg.App.set_loop_interval(config[CONF_LOOP_INTERVAL])) if CORE.using_arduino: CORE.add_job(add_arduino_global_workaround) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 558bbd54cd..21c4e3d996 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -403,6 +403,7 @@ #define ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT 2 #define ESPHOME_ESP32_BLE_EXTENDED_AUTH_PARAMS #define ESPHOME_LOOP_TASK_STACK_SIZE 8192 +#define ESPHOME_SUSPEND_LOOP #define USE_ESP32_CAMERA_JPEG_ENCODER #define USE_HTTP_REQUEST_RESPONSE #define USE_I2C @@ -520,6 +521,7 @@ // ESP8266-specific feature flags #ifdef USE_ESP8266 +#define ESPHOME_SUSPEND_LOOP #define USE_ADC_SENSOR_VCC #define USE_ESP8266_CRASH_HANDLER #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 1, 2) @@ -597,6 +599,7 @@ #define USE_BK72XX_BLE_TRACKER #endif #define ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT 1 +#define ESPHOME_SUSPEND_LOOP #define USE_BLE_SCAN_RESPONSE_MERGER #define USE_CAPTIVE_PORTAL #define USE_WIFI_SCAN_RESULTS_LOCK @@ -622,6 +625,7 @@ #ifdef USE_NRF52 #define ESPHOME_BLE_NUS_TX_RING_BUFFER_SIZE 512 #define ESPHOME_BLE_NUS_RX_RING_BUFFER_SIZE 512 +#define ESPHOME_SUSPEND_LOOP #define USE_ESPHOME_TASK_LOG_BUFFER #define ESPHOME_TASK_LOG_BUFFER_SIZE 768 #define USE_LOGGER_EARLY_MESSAGE diff --git a/tests/components/esphome/common.yaml b/tests/components/esphome/common.yaml index 93f82824e6..fd3d6a0efe 100644 --- a/tests/components/esphome/common.yaml +++ b/tests/components/esphome/common.yaml @@ -1,5 +1,7 @@ esphome: debug_scheduler: true + loop_interval: 50ms + suspend_loop: false platformio_options: board_build.flash_mode: dio build_flags: diff --git a/tests/components/esphome/test.esp32-idf.yaml b/tests/components/esphome/test.esp32-idf.yaml index dade44d145..7c571c7f51 100644 --- a/tests/components/esphome/test.esp32-idf.yaml +++ b/tests/components/esphome/test.esp32-idf.yaml @@ -1 +1,5 @@ -<<: !include common.yaml +packages: + base: !include common.yaml + +esphome: + suspend_loop: true diff --git a/tests/components/esphome/test.esp8266-ard.yaml b/tests/components/esphome/test.esp8266-ard.yaml index dade44d145..7c571c7f51 100644 --- a/tests/components/esphome/test.esp8266-ard.yaml +++ b/tests/components/esphome/test.esp8266-ard.yaml @@ -1 +1,5 @@ -<<: !include common.yaml +packages: + base: !include common.yaml + +esphome: + suspend_loop: true diff --git a/tests/components/esphome/test.nrf52-adafruit.yaml b/tests/components/esphome/test.nrf52-adafruit.yaml index dade44d145..7c571c7f51 100644 --- a/tests/components/esphome/test.nrf52-adafruit.yaml +++ b/tests/components/esphome/test.nrf52-adafruit.yaml @@ -1 +1,5 @@ -<<: !include common.yaml +packages: + base: !include common.yaml + +esphome: + suspend_loop: true diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 7fbc2d1b39..1a36fdc967 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -26,6 +26,7 @@ from esphome.const import ( ) from esphome.core import CORE, KEY_CONTROLLER_REGISTRY_CONTROLLERS, config from esphome.core.config import ( + CONF_SUSPEND_LOOP, Area, make_app_name_cpp, preload_core_config, @@ -248,6 +249,97 @@ def test_area_id_collision( assert "ID duplicate_id redefined! Check esphome->area->id." in captured.out +@pytest.mark.parametrize( + ("fixture", "expected_platform"), + [ + ("suspend_loop_host.yaml", "host"), + ("suspend_loop_rp2.yaml", "rp2"), + ], +) +def test_suspend_loop_fail( + yaml_file: Callable[[str], str], + capsys: pytest.CaptureFixture[str], + fixture: str, + expected_platform: str, +) -> None: + """Test that suspend_loop fails.""" + result = load_config_from_fixture(yaml_file, fixture, FIXTURES_DIR) + assert result is None + + # Check for the specific error message in stdout + captured = capsys.readouterr() + assert ( + f"Suspend loop is not available on {expected_platform} platform" in captured.out + ) + + +def test_loop_interval_warn_esp32( + yaml_file: Callable[[str], str], + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that too high loop_interval prints warning.""" + result = load_config_from_fixture( + yaml_file, "loop_interval_esp32.yaml", FIXTURES_DIR + ) + assert result is not None + + assert ( + "loop_interval of 7s exceeds the 2400ms maximum sleep on this platform; the loop will still " + "wake every 2400ms. Raise esp32.watchdog_timeout to sleep longer." + in caplog.text + ) + + +@pytest.mark.parametrize( + ("fixture", "interval", "max_loop"), + [ + ("loop_interval_bk72xx.yaml", "5000ms", "4000"), + ("loop_interval_nrf52.yaml", "700ms", "600"), + ], +) +def test_loop_interval_warn( + yaml_file: Callable[[str], str], + caplog: pytest.LogCaptureFixture, + fixture: str, + interval: str, + max_loop: str, +) -> None: + """Test that too high loop_interval prints warning.""" + result = load_config_from_fixture(yaml_file, fixture, FIXTURES_DIR) + assert result is not None + + assert ( + f"loop_interval of {interval} exceeds the {max_loop}ms maximum sleep on this platform; the loop will still " + f"wake every {max_loop}ms." in caplog.text + ) + + +@pytest.mark.asyncio +@pytest.mark.filterwarnings("ignore::RuntimeWarning") +async def test_suspend_loop_and_loop_interval( + yaml_file: Callable[[str], Path], +) -> None: + """Test suspend_loop and loop_interval on esp32""" + result = load_config_from_fixture( + yaml_file, "suspend_loop_esp32.yaml", FIXTURES_DIR + ) + assert result is not None + + esphome_config = result["esphome"] + assert esphome_config.get(CONF_SUSPEND_LOOP) + + with patch("esphome.core.config.cg") as mock_cg: + mock_cg.RawStatement.side_effect = lambda *args, **kwargs: MagicMock() + mock_cg.RawExpression.side_effect = lambda *args, **kwargs: MagicMock() + await config.to_code(result[CONF_ESPHOME]) + + mock_cg.add_define.assert_any_call("ESPHOME_SUSPEND_LOOP") + mock_cg.add_define.assert_any_call("ESPHOME_DEBUG_SCHEDULER") + mock_cg.App.set_loop_interval.assert_called_once_with( + cv.TimePeriodMilliseconds(milliseconds=50) + ) + + def test_device_without_area(yaml_file: Callable[[str], str]) -> None: """Test that devices without area_id work correctly.""" result = load_config_from_fixture( diff --git a/tests/unit_tests/fixtures/core/config/loop_interval_bk72xx.yaml b/tests/unit_tests/fixtures/core/config/loop_interval_bk72xx.yaml new file mode 100644 index 0000000000..a29a8e4c93 --- /dev/null +++ b/tests/unit_tests/fixtures/core/config/loop_interval_bk72xx.yaml @@ -0,0 +1,7 @@ +esphome: + name: test + suspend_loop: true + loop_interval: 5000ms + +bk72xx: + board: generic-bk7252 diff --git a/tests/unit_tests/fixtures/core/config/loop_interval_esp32.yaml b/tests/unit_tests/fixtures/core/config/loop_interval_esp32.yaml new file mode 100644 index 0000000000..491b14d464 --- /dev/null +++ b/tests/unit_tests/fixtures/core/config/loop_interval_esp32.yaml @@ -0,0 +1,7 @@ +esphome: + name: test + loop_interval: 7s + +esp32: + board: esp32dev + watchdog_timeout: 6s diff --git a/tests/unit_tests/fixtures/core/config/loop_interval_nrf52.yaml b/tests/unit_tests/fixtures/core/config/loop_interval_nrf52.yaml new file mode 100644 index 0000000000..9dd38bf4da --- /dev/null +++ b/tests/unit_tests/fixtures/core/config/loop_interval_nrf52.yaml @@ -0,0 +1,6 @@ +esphome: + name: test + loop_interval: 700ms + +nrf52: + board: adafruit_feather_nrf52840 diff --git a/tests/unit_tests/fixtures/core/config/suspend_loop_esp32.yaml b/tests/unit_tests/fixtures/core/config/suspend_loop_esp32.yaml new file mode 100644 index 0000000000..ec18aa74b4 --- /dev/null +++ b/tests/unit_tests/fixtures/core/config/suspend_loop_esp32.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + debug_scheduler: true + suspend_loop: true + loop_interval: 50ms + +esp32: + board: esp32dev + watchdog_timeout: 6s diff --git a/tests/unit_tests/fixtures/core/config/suspend_loop_host.yaml b/tests/unit_tests/fixtures/core/config/suspend_loop_host.yaml new file mode 100644 index 0000000000..c75f4f8b4d --- /dev/null +++ b/tests/unit_tests/fixtures/core/config/suspend_loop_host.yaml @@ -0,0 +1,5 @@ +esphome: + name: test + suspend_loop: true + +host: diff --git a/tests/unit_tests/fixtures/core/config/suspend_loop_rp2.yaml b/tests/unit_tests/fixtures/core/config/suspend_loop_rp2.yaml new file mode 100644 index 0000000000..8a2af21996 --- /dev/null +++ b/tests/unit_tests/fixtures/core/config/suspend_loop_rp2.yaml @@ -0,0 +1,6 @@ +esphome: + name: test + suspend_loop: true + +rp2: + board: rpipicow From 61446e3e4fd10e90931f8df22b05257508a1423d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:49:00 -0500 Subject: [PATCH 062/221] [xiaomi_lywsd02mmc] Use the sub_* helpers for optional child entities (#20014) --- esphome/components/xiaomi_lywsd02mmc/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_lywsd02mmc/sensor.py b/esphome/components/xiaomi_lywsd02mmc/sensor.py index 05e0204def..5b088318ae 100644 --- a/esphome/components/xiaomi_lywsd02mmc/sensor.py +++ b/esphome/components/xiaomi_lywsd02mmc/sensor.py @@ -67,12 +67,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature(sens)) - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity(sens)) - if battery_level_config := config.get(CONF_BATTERY_LEVEL): - sens = await sensor.new_sensor(battery_level_config) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 97235a20b8d608f69c11fc9c2c2dc0726465624a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:49:07 -0500 Subject: [PATCH 063/221] [xiaomi_lywsd02] Use the sub_* helpers for optional child entities (#20015) --- esphome/components/xiaomi_lywsd02/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_lywsd02/sensor.py b/esphome/components/xiaomi_lywsd02/sensor.py index f79b81dce2..d37d7540c7 100644 --- a/esphome/components/xiaomi_lywsd02/sensor.py +++ b/esphome/components/xiaomi_lywsd02/sensor.py @@ -63,12 +63,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 416d0c46ac1cca608f0fe4cbd2112611ef05c06b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:50:37 -0500 Subject: [PATCH 064/221] [xiaomi_lywsdcgq] Use the sub_* helpers for optional child entities (#20016) --- esphome/components/xiaomi_lywsdcgq/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_lywsdcgq/sensor.py b/esphome/components/xiaomi_lywsdcgq/sensor.py index 8f0b93a22e..75a8a2d847 100644 --- a/esphome/components/xiaomi_lywsdcgq/sensor.py +++ b/esphome/components/xiaomi_lywsdcgq/sensor.py @@ -63,12 +63,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From f595d8677ccdaa77441bdb3a8f3c73da8be15817 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:50:48 -0500 Subject: [PATCH 065/221] [xiaomi_cgg1] Use the sub_* helpers for optional child entities (#20017) --- esphome/components/xiaomi_cgg1/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_cgg1/sensor.py b/esphome/components/xiaomi_cgg1/sensor.py index 3f0df9eafe..01ca8d1f84 100644 --- a/esphome/components/xiaomi_cgg1/sensor.py +++ b/esphome/components/xiaomi_cgg1/sensor.py @@ -67,12 +67,7 @@ async def to_code(config: ConfigType) -> None: if CONF_BINDKEY in config: cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 1c31e9d28d15fd3049275bfaa78e7c52b1e9f64c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 09:50:58 -0500 Subject: [PATCH 066/221] [xiaomi_cgd1] Use the sub_* helpers for optional child entities (#20018) --- esphome/components/xiaomi_cgd1/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_cgd1/sensor.py b/esphome/components/xiaomi_cgd1/sensor.py index 09337df5ca..162ba7016b 100644 --- a/esphome/components/xiaomi_cgd1/sensor.py +++ b/esphome/components/xiaomi_cgd1/sensor.py @@ -66,12 +66,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 7096546d83c6702a6783b3f3ccb59eab16f2933a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 10:07:13 -0500 Subject: [PATCH 067/221] [nrf52] Create the precompiled header in ccache depend mode (#20019) --- esphome/components/nrf52/__init__.py | 3 +++ tests/unit_tests/components/nrf52/test_pch.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 5ebf5b747f..07507901ba 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -978,6 +978,9 @@ def run_compile(args, config: ConfigType) -> bool: pch.log_pch_in_use() # Zephyr turns ccache on by itself when it is installed env.update(pch.ccache_pch_env()) + # Depend mode, or a Kconfig flip reuses a stale .gch: autoconf.h is + # all #defines, which vanish from the preprocessed creation hash. + env.setdefault("CCACHE_DEPEND", "1") _write_pch_checksum(build_dir, source_dir) west_cmd = _west_build_command( diff --git a/tests/unit_tests/components/nrf52/test_pch.py b/tests/unit_tests/components/nrf52/test_pch.py index 0dcac3cad4..04932e76cc 100644 --- a/tests/unit_tests/components/nrf52/test_pch.py +++ b/tests/unit_tests/components/nrf52/test_pch.py @@ -138,6 +138,18 @@ def test_ccache_pch_settings_reach_west(run_cmd: Mock) -> None: env = run_cmd.call_args.kwargs["env"] assert env["CCACHE_PCH_EXTSUM"] == "true" assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros" + # Without depend mode a Kconfig flip reuses a stale .gch + assert env["CCACHE_DEPEND"] == "1" + + +def test_ccache_depend_respects_a_user_override(run_cmd: Mock) -> None: + with ( + patch.object(nrf52, "get_build_env", return_value={"CCACHE_DEPEND": "0"}), + patch.dict("os.environ", {"ESPHOME_PCH_ENABLE": "1"}, clear=True), + pytest.raises(EsphomeError, match="nRF52 native build failed"), + ): + nrf52.run_compile(None, {}) + assert run_cmd.call_args.kwargs["env"]["CCACHE_DEPEND"] == "0" def test_disabled_leaves_the_west_environment_alone( @@ -147,3 +159,4 @@ def test_disabled_leaves_the_west_environment_alone( with pytest.raises(EsphomeError, match="nRF52 native build failed"): nrf52.run_compile(None, {}) assert "CCACHE_PCH_EXTSUM" not in run_cmd.call_args.kwargs["env"] + assert "CCACHE_DEPEND" not in run_cmd.call_args.kwargs["env"] From 6491eaeaae689c8589d332f1bf578c889924ed11 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 10:10:28 -0500 Subject: [PATCH 068/221] [tcp_uart] Report a failed flush when the flush drops the link (#20013) --- esphome/components/tcp_uart/tcp_uart.cpp | 9 +- .../socket/test_tcp_client_link_host.cpp | 77 +++++++++++++ tests/components/tcp_uart/__init__.py | 7 ++ tests/components/tcp_uart/test_flush_host.cpp | 69 ++++++++++++ .../tcp_client_link_test_component.cpp | 5 +- .../integration/fixtures/tcp_uart_bridge.yaml | 33 ++++++ .../test_socket_tcp_client_link.py | 16 ++- tests/integration/test_tcp_uart_bridge.py | 102 ++++++++++++++++++ 8 files changed, 312 insertions(+), 6 deletions(-) create mode 100644 tests/components/socket/test_tcp_client_link_host.cpp create mode 100644 tests/components/tcp_uart/__init__.py create mode 100644 tests/components/tcp_uart/test_flush_host.cpp create mode 100644 tests/integration/fixtures/tcp_uart_bridge.yaml create mode 100644 tests/integration/test_tcp_uart_bridge.py diff --git a/esphome/components/tcp_uart/tcp_uart.cpp b/esphome/components/tcp_uart/tcp_uart.cpp index 00b59d82ba..b24280c8b1 100644 --- a/esphome/components/tcp_uart/tcp_uart.cpp +++ b/esphome/components/tcp_uart/tcp_uart.cpp @@ -107,8 +107,13 @@ bool TcpUart::read_array(uint8_t *data, size_t len) { } uart::UARTFlushResult TcpUart::flush() { - return this->link_.flush_tx() ? uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS - : uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT; + bool emptied = this->link_.flush_tx(); + if (!this->link_.connected()) { + // A down link cannot have delivered anything, whether this flush dropped + // it or an earlier loop() write did. + return uart::UARTFlushResult::UART_FLUSH_RESULT_FAILED; + } + return emptied ? uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS : uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT; } } // namespace esphome::tcp_uart diff --git a/tests/components/socket/test_tcp_client_link_host.cpp b/tests/components/socket/test_tcp_client_link_host.cpp new file mode 100644 index 0000000000..9a6d388b72 --- /dev/null +++ b/tests/components/socket/test_tcp_client_link_host.cpp @@ -0,0 +1,77 @@ +#include + +#include +#include +#include +#include +#include + +#include "esphome/components/socket/tcp_client_link.h" + +#ifdef USE_HOST + +namespace esphome::socket::testing { + +class LinkPeer { + public: + LinkPeer() { + // EPIPE must come back as an errno, not a signal. + signal(SIGPIPE, SIG_IGN); + int fds[2]; + EXPECT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + this->peer_fd_ = fds[1]; + this->link_.set_host("peer"); + this->link_.set_port(1); + this->link_.begin("link_test"); + this->link_.adopt(std::make_unique(fds[0])); + } + ~LinkPeer() { + if (this->peer_fd_ >= 0) { + ::close(this->peer_fd_); + } + this->link_.close(); + } + void close_peer() { + ::close(this->peer_fd_); + this->peer_fd_ = -1; + } + + TcpClientLink link_; + int peer_fd_{-1}; +}; + +TEST(TcpClientLink, AdoptedSocketFlushesQueuedBytes) { + LinkPeer p; + ASSERT_TRUE(p.link_.connected()); + EXPECT_EQ(p.link_.queue(reinterpret_cast("ping"), 4), 4u); + EXPECT_TRUE(p.link_.flush_tx()); + char buf[8]; + EXPECT_EQ(::read(p.peer_fd_, buf, sizeof(buf)), 4); + EXPECT_EQ(std::memcmp(buf, "ping", 4), 0); +} + +TEST(TcpClientLink, CloseClearsQueuedBytes) { + LinkPeer p; + EXPECT_EQ(p.link_.queue(reinterpret_cast("MARKER"), 6), 6u); + p.link_.close(); + EXPECT_FALSE(p.link_.connected()); + EXPECT_EQ(p.link_.tx_free(), 0u); + // An uncleared buffer would make flush_tx() report it as still pending. + EXPECT_TRUE(p.link_.flush_tx()); +} + +TEST(TcpClientLink, FatalWriteInsideFlushDropsTheLink) { + LinkPeer p; + EXPECT_EQ(p.link_.queue(reinterpret_cast("MARKER"), 6), 6u); + p.close_peer(); + // Still connected from the link's point of view: the drop must happen + // inside this flush, the exact ordering TcpUart::flush() reports FAILED. + ASSERT_TRUE(p.link_.connected()); + bool emptied = p.link_.flush_tx(); + EXPECT_TRUE(emptied); + EXPECT_FALSE(p.link_.connected()); +} + +} // namespace esphome::socket::testing + +#endif diff --git a/tests/components/tcp_uart/__init__.py b/tests/components/tcp_uart/__init__.py new file mode 100644 index 0000000000..1e4de448b7 --- /dev/null +++ b/tests/components/tcp_uart/__init__.py @@ -0,0 +1,7 @@ +# This file's presence makes pytest treat this directory as a package named +# "tcp_uart"; required for cpp unit testing. +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + manifest.multi_conf = False diff --git a/tests/components/tcp_uart/test_flush_host.cpp b/tests/components/tcp_uart/test_flush_host.cpp new file mode 100644 index 0000000000..9c6c63110c --- /dev/null +++ b/tests/components/tcp_uart/test_flush_host.cpp @@ -0,0 +1,69 @@ +#include + +#include +#include +#include +#include + +#include "esphome/components/tcp_uart/tcp_uart.h" + +#ifdef USE_HOST + +namespace esphome::tcp_uart::testing { + +class TcpUartUnderTest : public TcpUart { + public: + TcpUartUnderTest() : TcpUart("peer", 1) { this->link_.begin("flush_test"); } + socket::TcpClientLink &link() { return this->link_; } +}; + +class TcpUartFlush : public ::testing::Test { + protected: + void SetUp() override { + // EPIPE must come back as an errno, not a signal. + signal(SIGPIPE, SIG_IGN); + int fds[2]; + ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + this->peer_fd_ = fds[1]; + this->uart_.link().adopt(std::make_unique(fds[0])); + } + void TearDown() override { + if (this->peer_fd_ >= 0) { + ::close(this->peer_fd_); + } + this->uart_.link().close(); + } + + TcpUartUnderTest uart_; + int peer_fd_{-1}; +}; + +TEST_F(TcpUartFlush, SuccessWhenTheByteGoesOut) { + uint8_t b = 'x'; + this->uart_.write_array(&b, 1); + EXPECT_EQ(this->uart_.flush(), uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS); + char got; + EXPECT_EQ(::read(this->peer_fd_, &got, 1), 1); + EXPECT_EQ(got, 'x'); +} + +TEST_F(TcpUartFlush, FailedWhenTheFlushDropsTheLink) { + uint8_t b = 'x'; + this->uart_.write_array(&b, 1); + ::close(this->peer_fd_); + this->peer_fd_ = -1; + // The drop happens inside this flush; checking connected() first would + // wrongly report success. + ASSERT_TRUE(this->uart_.is_connected()); + EXPECT_EQ(this->uart_.flush(), uart::UARTFlushResult::UART_FLUSH_RESULT_FAILED); + EXPECT_FALSE(this->uart_.is_connected()); +} + +TEST_F(TcpUartFlush, FailedWhileTheLinkIsDown) { + this->uart_.link().close(); + EXPECT_EQ(this->uart_.flush(), uart::UARTFlushResult::UART_FLUSH_RESULT_FAILED); +} + +} // namespace esphome::tcp_uart::testing + +#endif diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp index bd45e60a00..0f6de0ee9f 100644 --- a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp @@ -17,9 +17,8 @@ void TcpClientLinkTestComponent::loop() { if (!up) { return; } - // Retries a partial echo; inline no-op when nothing is queued. - this->link_.flush_tx(); - if (!this->link_.ready()) { + // Echo only once the previous echo is fully sent, so it never drops bytes. + if (!this->link_.flush_tx() || !this->link_.ready()) { return; } uint8_t buf[64]; diff --git a/tests/integration/fixtures/tcp_uart_bridge.yaml b/tests/integration/fixtures/tcp_uart_bridge.yaml new file mode 100644 index 0000000000..12791e7dd4 --- /dev/null +++ b/tests/integration/fixtures/tcp_uart_bridge.yaml @@ -0,0 +1,33 @@ +esphome: + name: tcp-uart-bridge-test + +host: + +api: + services: + - service: send_byte + then: + - lambda: |- + id(tcp_uart_1).write_byte('X'); + auto r = id(tcp_uart_1).flush(); + ESP_LOGI("bridge_test", "Flush result %d", (int) r); + +logger: + level: INFO + +tcp_uart: + - id: tcp_uart_1 + host: 127.0.0.1 + port: 18125 + reconnect_interval: 3s + connected: + name: Bridge Connected + +interval: + - interval: 50ms + then: + - lambda: |- + uint8_t b; + while (id(tcp_uart_1).read_byte(&b)) { + id(tcp_uart_1).write_byte(b); + } diff --git a/tests/integration/test_socket_tcp_client_link.py b/tests/integration/test_socket_tcp_client_link.py index 31cc344f67..16b5f544f7 100644 --- a/tests/integration/test_socket_tcp_client_link.py +++ b/tests/integration/test_socket_tcp_client_link.py @@ -1,7 +1,8 @@ """Integration test for socket::TcpClientLink on host. Pytest runs a real TCP server; the device echoes through the link. -Covers connect, read, write, a server-initiated drop and the reconnect. +Covers connect, read, write, a server-initiated drop, the reconnect and +that no bytes from the first session leak into the second. """ from __future__ import annotations @@ -14,6 +15,7 @@ import pytest from .types import APIClientConnectedFactory, RunCompiledFunction PAYLOAD = b"hello link" +SECOND_PAYLOAD = b"second session" @pytest.mark.asyncio @@ -27,6 +29,7 @@ async def test_socket_tcp_client_link( yaml_config = yaml_config.replace("port: 18123", f"port: {server_port}") echoed: list[bytes] = [] + second_echoed: list[bytes] = [] echo_done = asyncio.Event() reconnected = asyncio.Event() link_down = asyncio.Event() @@ -56,6 +59,14 @@ async def test_socket_tcp_client_link( # Drop the connection so the link has to reconnect. writer.close() return + # Second session: the first bytes back must be this session's echo; + # anything left over from the first session would arrive ahead of it. + writer.write(SECOND_PAYLOAD) + await writer.drain() + with contextlib.suppress(TimeoutError, asyncio.IncompleteReadError): + second_echoed.append( + await asyncio.wait_for(reader.readexactly(len(SECOND_PAYLOAD)), 10) + ) reconnected.set() server = await asyncio.start_server(handle, "127.0.0.1", server_port) @@ -83,6 +94,9 @@ async def test_socket_tcp_client_link( await asyncio.wait_for(second_link_up.wait(), timeout=15.0) except TimeoutError: pytest.fail("Link did not reconnect after the server dropped it") + assert second_echoed == [SECOND_PAYLOAD], ( + "Second session echo wrong; stale bytes from the first session?" + ) finally: server.close() await server.wait_closed() diff --git a/tests/integration/test_tcp_uart_bridge.py b/tests/integration/test_tcp_uart_bridge.py new file mode 100644 index 0000000000..fb22a26b55 --- /dev/null +++ b/tests/integration/test_tcp_uart_bridge.py @@ -0,0 +1,102 @@ +"""Integration test for the tcp_uart bridge on host. + +Pytest runs a real TCP server; a lambda echoes UART bytes back to the socket. +Covers the UART read and write paths, flush results for a live and a dropped +link, the offline drop warning and the automatic reconnect. +""" + +from __future__ import annotations + +import asyncio +import contextlib + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + +GREETING = b"hi" +RECONNECT_GREETING = b"yo" + + +async def _wait(event: asyncio.Event, timeout: float, message: str) -> None: + try: + await asyncio.wait_for(event.wait(), timeout) + except TimeoutError: + pytest.fail(message) + + +@pytest.mark.asyncio +async def test_tcp_uart_bridge( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + unused_tcp_port_factory, +) -> None: + server_port = unused_tcp_port_factory() + yaml_config = yaml_config.replace("port: 18125", f"port: {server_port}") + + lines = LineWaiter() + echo_ok = asyncio.Event() + service_byte_ok = asyncio.Event() + reconnect_echo_ok = asyncio.Event() + sessions = 0 + + async def handle( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + nonlocal sessions + sessions += 1 + if sessions == 1: + writer.write(GREETING) + await writer.drain() + with contextlib.suppress(TimeoutError, asyncio.IncompleteReadError): + if ( + await asyncio.wait_for(reader.readexactly(len(GREETING)), 10) + == GREETING + ): + echo_ok.set() + if await asyncio.wait_for(reader.readexactly(1), 10) == b"X": + service_byte_ok.set() + writer.close() + return + writer.write(RECONNECT_GREETING) + await writer.drain() + with contextlib.suppress(TimeoutError, asyncio.IncompleteReadError): + if ( + await asyncio.wait_for(reader.readexactly(len(RECONNECT_GREETING)), 10) + == RECONNECT_GREETING + ): + reconnect_echo_ok.set() + + server = await asyncio.start_server(handle, "127.0.0.1", server_port) + try: + async with ( + run_compiled(yaml_config, line_callback=lines.callback), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "tcp-uart-bridge-test" + _, services = await client.list_entities_services() + send_byte = next(s for s in services if s.name == "send_byte") + + await _wait(echo_ok, 15.0, "UART echo through the bridge never arrived") + + await client.execute_service(send_byte, {}) + await _wait(service_byte_ok, 10.0, "Service byte never reached the server") + await lines.wait_for("Flush result 0") + + # The server closed session one; a write while down must warn and + # a flush on the down link must report FAILED (2). + await lines.wait_for("Connection lost") + await client.execute_service(send_byte, {}) + await lines.wait_for("Not connected, dropped") + await lines.wait_for("Flush result 2") + + await _wait( + reconnect_echo_ok, 15.0, "Bridge did not reconnect and echo again" + ) + finally: + server.close() + await server.wait_closed() From 26398747a1121dd641270d1836f96c12a5813fa2 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:23:29 +0200 Subject: [PATCH 069/221] [uart_tcp] Add a hardware UART copied to one TCP socket (#19887) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston --- CODEOWNERS | 1 + esphome/components/uart_tcp/__init__.py | 71 ++++++++ esphome/components/uart_tcp/uart_tcp.cpp | 170 ++++++++++++++++++ esphome/components/uart_tcp/uart_tcp.h | 48 +++++ tests/components/uart_tcp/common.yaml | 11 ++ .../uart_tcp/test-client.esp32-idf.yaml | 16 ++ .../components/uart_tcp/test.bk72xx-ard.yaml | 3 + tests/components/uart_tcp/test.esp32-idf.yaml | 3 + .../components/uart_tcp/test.esp8266-ard.yaml | 3 + tests/components/uart_tcp/test.host.yaml | 12 ++ .../components/uart_tcp/test.rp2040-ard.yaml | 3 + .../integration/fixtures/uart_tcp_bridge.yaml | 22 +++ tests/integration/test_uart_tcp_bridge.py | 110 ++++++++++++ 13 files changed, 473 insertions(+) create mode 100644 esphome/components/uart_tcp/__init__.py create mode 100644 esphome/components/uart_tcp/uart_tcp.cpp create mode 100644 esphome/components/uart_tcp/uart_tcp.h create mode 100644 tests/components/uart_tcp/common.yaml create mode 100644 tests/components/uart_tcp/test-client.esp32-idf.yaml create mode 100644 tests/components/uart_tcp/test.bk72xx-ard.yaml create mode 100644 tests/components/uart_tcp/test.esp32-idf.yaml create mode 100644 tests/components/uart_tcp/test.esp8266-ard.yaml create mode 100644 tests/components/uart_tcp/test.host.yaml create mode 100644 tests/components/uart_tcp/test.rp2040-ard.yaml create mode 100644 tests/integration/fixtures/uart_tcp_bridge.yaml create mode 100644 tests/integration/test_uart_tcp_bridge.py diff --git a/CODEOWNERS b/CODEOWNERS index 5f89315d23..b453cf4538 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -602,6 +602,7 @@ esphome/components/uart/button/* @ssieb esphome/components/uart/event/* @eoasmxd esphome/components/uart/packet_transport/* @clydebarrow esphome/components/uart_mux/* @kbx81 +esphome/components/uart_tcp/* @Bascht74 esphome/components/udp/* @clydebarrow esphome/components/ufire_ec/* @pvizeli esphome/components/ufire_ise/* @pvizeli diff --git a/esphome/components/uart_tcp/__init__.py b/esphome/components/uart_tcp/__init__.py new file mode 100644 index 0000000000..e7a85a0f2d --- /dev/null +++ b/esphome/components/uart_tcp/__init__.py @@ -0,0 +1,71 @@ +import esphome.codegen as cg +from esphome.components import binary_sensor, socket, uart +from esphome.components.const import CONF_HOST, CONF_RECONNECT_INTERVAL, CONF_ROLE +import esphome.config_validation as cv +from esphome.const import ( + CONF_ID, + CONF_PORT, + CONF_UART_ID, + DEVICE_CLASS_CONNECTIVITY, + ENTITY_CATEGORY_DIAGNOSTIC, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@Bascht74"] +DEPENDENCIES = ["network", "uart"] +AUTO_LOAD = ["binary_sensor", "socket"] +MULTI_CONF = True + +uart_tcp_ns = cg.esphome_ns.namespace("uart_tcp") +UartTcp = uart_tcp_ns.class_("UartTcp", cg.Component, uart.UARTDevice) + +CONF_CONNECTED = "connected" + + +def _consume_sockets(config: ConfigType) -> ConfigType: + if config[CONF_ROLE] == "server": + socket.consume_sockets(1, "uart_tcp", socket.SocketType.TCP_LISTEN)(config) + return socket.consume_sockets(1, "uart_tcp")(config) + + +BASE_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(UartTcp), + cv.Required(CONF_UART_ID): cv.use_id(uart.UARTComponent), + cv.Required(CONF_PORT): cv.port, + cv.Optional( + CONF_RECONNECT_INTERVAL, default="5s" + ): cv.positive_time_period_milliseconds, + cv.Optional(CONF_CONNECTED): binary_sensor.binary_sensor_schema( + device_class=DEVICE_CLASS_CONNECTIVITY, + entity_category=ENTITY_CATEGORY_DIAGNOSTIC, + ), + } +).extend(cv.COMPONENT_SCHEMA) + +CONFIG_SCHEMA = cv.All( + cv.typed_schema( + { + "client": BASE_SCHEMA.extend({cv.Required(CONF_HOST): cv.string}), + "server": BASE_SCHEMA, + }, + key=CONF_ROLE, + default_type="client", + lower=True, + ), + _consume_sockets, +) + + +async def to_code(config: ConfigType) -> None: + socket.require_tcp_client_link() + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) + cg.add(var.set_server(config[CONF_ROLE] == "server")) + cg.add(var.set_port(config[CONF_PORT])) + cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL])) + if (host := config.get(CONF_HOST)) is not None: + cg.add(var.set_host(host)) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_CONNECTED, var.set_connected_sensor) diff --git a/esphome/components/uart_tcp/uart_tcp.cpp b/esphome/components/uart_tcp/uart_tcp.cpp new file mode 100644 index 0000000000..2531b73b20 --- /dev/null +++ b/esphome/components/uart_tcp/uart_tcp.cpp @@ -0,0 +1,170 @@ +#include "uart_tcp.h" + +#include "esphome/core/log.h" + +#include +#include +#include + +namespace esphome::uart_tcp { + +static const char *const TAG = "uart_tcp"; + +// One client at a time; a second connection waits in the stack until the first drops. +static constexpr int LISTEN_BACKLOG = 1; +// Bytes per 16 ms loop pass at 10 bits per byte: baud / 10 / 62.5. +static constexpr uint32_t BAUD_PACE_DIVISOR = 625; + +void UartTcp::setup() { + this->link_.begin(TAG); + if (this->connected_sensor_ != nullptr) { + this->connected_sensor_->publish_state(false); + } +} + +void UartTcp::dump_config() { + ESP_LOGCONFIG(TAG, + "UART TCP:\n" + " %s: %s:%u\n" + " Reconnect Interval: %" PRIu32 "ms", + this->server_ ? LOG_STR_LITERAL("Listen") : LOG_STR_LITERAL("Host"), + this->server_ ? LOG_STR_LITERAL("*") : this->link_.host(), this->link_.port(), + this->link_.reconnect_interval()); + LOG_BINARY_SENSOR(" ", "Connected", this->connected_sensor_); +} + +void UartTcp::on_shutdown() { + this->link_.close(); + this->listen_.reset(); +} + +void UartTcp::sync_link_() { + bool up = this->link_.connected(); + this->link_was_up_ = up; + if (up) { + // The driver kept whatever arrived while the link was down. + this->discard_uart_(); + } + if (this->connected_sensor_ != nullptr) { + this->connected_sensor_->publish_state(up); + } +} + +void UartTcp::try_listen_() { + this->listen_ = socket::socket_ip_loop_monitored(SOCK_STREAM, IPPROTO_TCP); + int err = errno; + if (this->listen_ != nullptr) { + int yes = 1; + this->listen_->setsockopt(SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + struct sockaddr_storage local; + socklen_t local_len = + socket::set_sockaddr_any(reinterpret_cast(&local), sizeof(local), this->link_.port()); + // A blocking listener would stall loop() inside accept(), so its + // setblocking result is part of the success condition. + if (this->listen_->setblocking(false) == 0 && local_len != 0 && + this->listen_->bind(reinterpret_cast(&local), local_len) == 0 && + this->listen_->listen(LISTEN_BACKLOG) == 0) { + ESP_LOGI(TAG, "Listening on %u", this->link_.port()); + return; + } + // Captured before reset(); the close inside can overwrite errno. + err = errno; + this->listen_.reset(); + } + ESP_LOGW(TAG, "Listen on %u failed: %d", this->link_.port(), err); + this->link_.note_attempt(); +} + +void UartTcp::accept_client_() { + auto client = this->listen_->accept_loop_monitored(nullptr, nullptr); + if (client == nullptr) { + // A reset during the handshake or a signal only affects that connection. + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED || errno == EINTR) { + return; + } + // Rebuild the listener after the backoff instead of spinning on it. + int err = errno; + this->listen_.reset(); + ESP_LOGW(TAG, "Accept failed: %d", err); + this->link_.note_attempt(); + return; + } + this->link_.adopt(std::move(client)); + ESP_LOGI(TAG, "Client connected"); +} + +void UartTcp::read_socket_() { + // A hardware write blocks until the driver takes every byte. Leave what does + // not fit in the socket, so TCP flow control throttles the peer. + size_t room = this->parent_->available_for_write(); + if (room == SIZE_MAX) { + // Capacity unknown on this platform; pace to one loop pass of UART time + // (16 ms at 10 bits per byte) so a blocking write stays short. + room = std::max(1, this->parent_->get_baud_rate() / BAUD_PACE_DIVISOR); + } + if (room == 0) { + this->rx_pending_ = true; + return; + } + uint8_t tmp[READ_CHUNK]; + size_t want = std::min(room, sizeof(tmp)); + ssize_t count = this->link_.read(tmp, want); + if (count <= 0) { + // A dropped link (-1) is cleaned up by sync_link_() on the next loop. + if (count == 0) { + this->rx_pending_ = false; + } + return; + } + this->rx_pending_ = static_cast(count) == want; + this->write_array(tmp, static_cast(count)); +} + +void UartTcp::discard_uart_() { + // Drain exactly what was buffered while the link was down; later bytes are live. + uint8_t dump[32]; + size_t left = this->available(); + while (left != 0) { + size_t n = std::min(left, sizeof(dump)); + if (!this->read_array(dump, n)) { + return; + } + left -= n; + } +} + +void UartTcp::read_uart_() { + size_t want = std::min(this->available(), this->link_.tx_free()); + if (want != 0 && this->read_array(this->link_.tx_tail(), want)) { + this->link_.tx_commit(want); + } +} + +void UartTcp::loop() { + if (this->server_) { + if (this->listen_ == nullptr && !this->link_.in_backoff()) { + this->try_listen_(); + } + // link_was_up_ holds the accept until the previous drop's edge has run, + // so the sensor and the stale UART discard always see the disconnect. + if (this->listen_ != nullptr && !this->link_.connected() && !this->link_was_up_ && this->listen_->ready()) { + this->accept_client_(); + } + } else { + this->link_.poll(); + } + if (this->link_.connected() != this->link_was_up_) { + this->sync_link_(); + } + if (!this->link_was_up_) { + return; + } + if (this->rx_pending_ || this->link_.ready()) { + this->read_socket_(); + } + // UART bytes picked up here go out in the same pass. + this->read_uart_(); + this->link_.flush_tx(); +} + +} // namespace esphome::uart_tcp diff --git a/esphome/components/uart_tcp/uart_tcp.h b/esphome/components/uart_tcp/uart_tcp.h new file mode 100644 index 0000000000..2a59bf968f --- /dev/null +++ b/esphome/components/uart_tcp/uart_tcp.h @@ -0,0 +1,48 @@ +#pragma once + +#include "esphome/components/binary_sensor/binary_sensor.h" +#include "esphome/components/socket/tcp_client_link.h" +#include "esphome/components/uart/uart.h" +#include "esphome/core/component.h" + +#include +#include + +namespace esphome::uart_tcp { + +/// Copies raw bytes between one hardware UART and one TCP socket. +class UartTcp : public Component, public uart::UARTDevice { + public: + void set_host(const char *host) { this->link_.set_host(host); } + void set_port(uint16_t port) { this->link_.set_port(port); } + void set_server(bool server) { this->server_ = server; } + void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); } + void set_connected_sensor(binary_sensor::BinarySensor *sensor) { this->connected_sensor_ = sensor; } + + void setup() override; + void loop() override; + void dump_config() override; + void on_shutdown() override; + float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } + + protected: + void sync_link_(); + void try_listen_(); + void accept_client_(); + void read_socket_(); + void read_uart_(); + void discard_uart_(); + + static constexpr size_t READ_CHUNK = 128; + + socket::TcpClientLink link_; + std::unique_ptr listen_; + binary_sensor::BinarySensor *connected_sensor_{nullptr}; + bool server_{false}; + // The link state loop() saw last; edges clear the buffer and publish the sensor. + bool link_was_up_{false}; + // A read stopped before EAGAIN. ready() stays false until new data arrives. + bool rx_pending_{false}; +}; + +} // namespace esphome::uart_tcp diff --git a/tests/components/uart_tcp/common.yaml b/tests/components/uart_tcp/common.yaml new file mode 100644 index 0000000000..8010878e89 --- /dev/null +++ b/tests/components/uart_tcp/common.yaml @@ -0,0 +1,11 @@ +wifi: + ssid: MySSID + password: password1 + +uart_tcp: + - id: bridge + uart_id: uart_bus + role: server + port: 502 + connected: + name: UART TCP Connected diff --git a/tests/components/uart_tcp/test-client.esp32-idf.yaml b/tests/components/uart_tcp/test-client.esp32-idf.yaml new file mode 100644 index 0000000000..92b0213f74 --- /dev/null +++ b/tests/components/uart_tcp/test-client.esp32-idf.yaml @@ -0,0 +1,16 @@ +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + +wifi: + ssid: MySSID + password: password1 + +uart_tcp: + - id: bridge + uart_id: uart_bus + role: client + host: 192.0.2.20 + port: 502 + reconnect_interval: 10s + connected: + name: UART TCP Connected diff --git a/tests/components/uart_tcp/test.bk72xx-ard.yaml b/tests/components/uart_tcp/test.bk72xx-ard.yaml new file mode 100644 index 0000000000..719d654f14 --- /dev/null +++ b/tests/components/uart_tcp/test.bk72xx-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart: !include ../../test_build_components/common/uart/bk72xx-ard.yaml + uart_tcp: !include common.yaml diff --git a/tests/components/uart_tcp/test.esp32-idf.yaml b/tests/components/uart_tcp/test.esp32-idf.yaml new file mode 100644 index 0000000000..8e7422f24b --- /dev/null +++ b/tests/components/uart_tcp/test.esp32-idf.yaml @@ -0,0 +1,3 @@ +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + uart_tcp: !include common.yaml diff --git a/tests/components/uart_tcp/test.esp8266-ard.yaml b/tests/components/uart_tcp/test.esp8266-ard.yaml new file mode 100644 index 0000000000..18ae2ef804 --- /dev/null +++ b/tests/components/uart_tcp/test.esp8266-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + uart_tcp: !include common.yaml diff --git a/tests/components/uart_tcp/test.host.yaml b/tests/components/uart_tcp/test.host.yaml new file mode 100644 index 0000000000..96a636fa98 --- /dev/null +++ b/tests/components/uart_tcp/test.host.yaml @@ -0,0 +1,12 @@ +uart: + - id: uart_bus + baud_rate: 9600 + port: /dev/ttyS0 + +uart_tcp: + - id: bridge + uart_id: uart_bus + host: 127.0.0.1 + port: 44502 + connected: + name: UART TCP Connected diff --git a/tests/components/uart_tcp/test.rp2040-ard.yaml b/tests/components/uart_tcp/test.rp2040-ard.yaml new file mode 100644 index 0000000000..5ece3bc3f4 --- /dev/null +++ b/tests/components/uart_tcp/test.rp2040-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + uart_tcp: !include common.yaml diff --git a/tests/integration/fixtures/uart_tcp_bridge.yaml b/tests/integration/fixtures/uart_tcp_bridge.yaml new file mode 100644 index 0000000000..c61add801f --- /dev/null +++ b/tests/integration/fixtures/uart_tcp_bridge.yaml @@ -0,0 +1,22 @@ +esphome: + name: uart-tcp-bridge-test + +host: + +api: + +logger: + level: INFO + +uart: + - id: uart_bus + baud_rate: 115200 + port: PTY_PATH + +uart_tcp: + - id: bridge + uart_id: uart_bus + role: server + port: 18126 + connected: + name: Bridge Connected diff --git a/tests/integration/test_uart_tcp_bridge.py b/tests/integration/test_uart_tcp_bridge.py new file mode 100644 index 0000000000..b98d3ffac9 --- /dev/null +++ b/tests/integration/test_uart_tcp_bridge.py @@ -0,0 +1,110 @@ +"""Integration test for the uart_tcp bridge on host. + +The UART bus is backed by a pty; pytest holds the controller side and connects +as the TCP client. Covers both transfer directions, the stale-byte discard +at every accept, and the drop plus client replacement path. +""" + +from __future__ import annotations + +import asyncio +import os +import pathlib + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_uart_tcp_bridge( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + unused_tcp_port_factory, +) -> None: + server_port = unused_tcp_port_factory() + controller_fd, device_fd = os.openpty() + os.set_blocking(controller_fd, False) + # uart's validate_port wants a two segment device path; Linux ptys live at + # /dev/pts/N, so hand the config a /tmp symlink instead. + pty_link = f"/tmp/uart-tcp-pty-{os.getpid()}" + pathlib.Path(pty_link).symlink_to(os.ttyname(device_fd)) + yaml_config = yaml_config.replace("port: 18126", f"port: {server_port}") + yaml_config = yaml_config.replace("PTY_PATH", pty_link) + + lines = LineWaiter() + loop = asyncio.get_running_loop() + uart_rx = bytearray() + uart_rx_event = asyncio.Event() + + def on_controller_readable() -> None: + try: + chunk = os.read(controller_fd, 256) + except BlockingIOError: + return + if chunk: + uart_rx.extend(chunk) + uart_rx_event.set() + + async def read_uart(count: int, timeout: float = 10.0) -> bytes: + while len(uart_rx) < count: + uart_rx_event.clear() + await asyncio.wait_for(uart_rx_event.wait(), timeout) + data = bytes(uart_rx[:count]) + del uart_rx[:count] + return data + + async def wait_log_count(needle: str, count: int, timeout: float = 15.0) -> None: + async with asyncio.timeout(timeout): + while sum(needle in line for line in lines.lines) < count: + await asyncio.sleep(0.05) + + loop.add_reader(controller_fd, on_controller_readable) + try: + async with ( + run_compiled(yaml_config, line_callback=lines.callback), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "uart-tcp-bridge-test" + await lines.wait_for("Listening on") + + # Bytes written before any client connects must never reach one. + os.write(controller_fd, b"STALE") + await asyncio.sleep(0.2) + + reader, writer = await asyncio.open_connection("127.0.0.1", server_port) + await wait_log_count("Client connected", 1) + os.write(controller_fd, b"live!") + assert await asyncio.wait_for(reader.readexactly(5), 10) == b"live!", ( + "First bytes to the client were not the live payload" + ) + writer.write(b"down1") + await writer.drain() + assert await read_uart(5) == b"down1" + + # Drop the client; bytes while no client is connected are discarded + # when the next one is accepted. + writer.close() + await lines.wait_for("Connection lost") + os.write(controller_fd, b"gap") + await asyncio.sleep(0.2) + + reader, writer = await asyncio.open_connection("127.0.0.1", server_port) + await wait_log_count("Client connected", 2) + os.write(controller_fd, b"live2") + assert await asyncio.wait_for(reader.readexactly(5), 10) == b"live2", ( + "Second client received stale bytes from the gap" + ) + writer.write(b"down2") + await writer.drain() + assert await read_uart(5) == b"down2" + writer.close() + finally: + loop.remove_reader(controller_fd) + os.close(controller_fd) + os.close(device_fd) + pathlib.Path(pty_link).unlink() From bf176633b2bcde0bc44668cc2bbc4fdd8a03d2f8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:09:28 -0500 Subject: [PATCH 070/221] [xiaomi_mhoc303] Use the sub_* helpers for optional child entities (#20024) --- esphome/components/xiaomi_mhoc303/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_mhoc303/sensor.py b/esphome/components/xiaomi_mhoc303/sensor.py index 9ce544fc59..44fb9c7654 100644 --- a/esphome/components/xiaomi_mhoc303/sensor.py +++ b/esphome/components/xiaomi_mhoc303/sensor.py @@ -63,12 +63,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From d0eddc8fc8d722103329e693eee8db596788a9f3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:09:50 -0500 Subject: [PATCH 071/221] [xiaomi_cgpr1] Use the sub_* helpers for optional child entities (#20023) --- esphome/components/xiaomi_cgpr1/binary_sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_cgpr1/binary_sensor.py b/esphome/components/xiaomi_cgpr1/binary_sensor.py index 219432c1fc..b742f1eaec 100644 --- a/esphome/components/xiaomi_cgpr1/binary_sensor.py +++ b/esphome/components/xiaomi_cgpr1/binary_sensor.py @@ -71,12 +71,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_IDLE_TIME in config: - sens = await sensor.new_sensor(config[CONF_IDLE_TIME]) - cg.add(var.set_idle_time(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) - if CONF_ILLUMINANCE in config: - sens = await sensor.new_sensor(config[CONF_ILLUMINANCE]) - cg.add(var.set_illuminance(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_IDLE_TIME, var.set_idle_time) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) + await sensors(CONF_ILLUMINANCE, var.set_illuminance) From 09ba060fd57d49e6995b9af1f94af2902caca5cb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:10:15 -0500 Subject: [PATCH 072/221] [xiaomi_xmwsdj04mmc] Use the sub_* helpers for optional child entities (#20022) --- esphome/components/xiaomi_xmwsdj04mmc/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_xmwsdj04mmc/sensor.py b/esphome/components/xiaomi_xmwsdj04mmc/sensor.py index 8282523129..8c47b526b1 100644 --- a/esphome/components/xiaomi_xmwsdj04mmc/sensor.py +++ b/esphome/components/xiaomi_xmwsdj04mmc/sensor.py @@ -67,12 +67,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature(sens)) - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity(sens)) - if battery_level_config := config.get(CONF_BATTERY_LEVEL): - sens = await sensor.new_sensor(battery_level_config) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 984de588533e993b4aaa1589ba10f1149d08bc5d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:10:41 -0500 Subject: [PATCH 073/221] [xiaomi_mhoc401] Use the sub_* helpers for optional child entities (#20021) --- esphome/components/xiaomi_mhoc401/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_mhoc401/sensor.py b/esphome/components/xiaomi_mhoc401/sensor.py index 52cae36fcd..e74c656e51 100644 --- a/esphome/components/xiaomi_mhoc401/sensor.py +++ b/esphome/components/xiaomi_mhoc401/sensor.py @@ -67,12 +67,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From a531ea07a2861d2edc7d56bba26728629bec8226 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:11:02 -0500 Subject: [PATCH 074/221] [xiaomi_cgdk2] Use the sub_* helpers for optional child entities (#20020) --- esphome/components/xiaomi_cgdk2/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_cgdk2/sensor.py b/esphome/components/xiaomi_cgdk2/sensor.py index 83de32f411..e1d44d6bac 100644 --- a/esphome/components/xiaomi_cgdk2/sensor.py +++ b/esphome/components/xiaomi_cgdk2/sensor.py @@ -66,12 +66,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 5def4d46ca7ec68535e4f939fad1791c255eed79 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:22:21 +0200 Subject: [PATCH 075/221] [socket] Add an IPv4 allow list (#20025) Co-authored-by: J. Nick Koston --- esphome/components/socket/__init__.py | 37 ++++++++- esphome/components/socket/ipv4_allow.h | 56 ++++++++++++++ esphome/components/socket/socket.cpp | 44 +++++------ esphome/components/socket/socket.h | 4 + .../socket/test_ipv4_allow_host.cpp | 75 +++++++++++++++++++ .../socket/test_socket_ipv4_allow.py | 46 ++++++++++++ 6 files changed, 239 insertions(+), 23 deletions(-) create mode 100644 esphome/components/socket/ipv4_allow.h create mode 100644 tests/components/socket/test_ipv4_allow_host.cpp create mode 100644 tests/unit_tests/components/socket/test_socket_ipv4_allow.py diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index a814f46e3b..850f7ebcbc 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -1,18 +1,22 @@ from collections.abc import Callable, MutableMapping from dataclasses import dataclass from enum import StrEnum +from ipaddress import IPv4Address, IPv4Network import logging import esphome.codegen as cg from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv -from esphome.core import CORE +from esphome.core import CORE, ID from esphome.types import ConfigType _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@esphome/core"] +socket_ns = cg.esphome_ns.namespace("socket") +Ipv4AllowEntry = socket_ns.struct("Ipv4AllowEntry") + CONF_IMPLEMENTATION = "implementation" IMPLEMENTATION_LWIP_TCP = "lwip_tcp" IMPLEMENTATION_LWIP_SOCKETS = "lwip_sockets" @@ -140,6 +144,37 @@ def require_wake_loop_threadsafe() -> None: cg.add_define("USE_SOCKET_SELECT_SUPPORT") +# For an Ipv4Allow config option; a sanity cap on the list length. +IPV4_ALLOW_SCHEMA = cv.All(cv.ensure_list(cv.ipv4network), cv.Length(max=255)) + + +def _network_order(addr: IPv4Address) -> int: + """The s_addr value for addr on the little endian targets.""" + return int.from_bytes(addr.packed, "little") + + +def add_ipv4_allow( + setter: cg.MockObj, networks: list[IPv4Network], owner_id: ID | str +) -> None: + """Emit a flash array for validated IPV4_ALLOW_SCHEMA entries and wire it to setter. + + PROGMEM on esp8266. Emits nothing for an empty list. + """ + if not networks: + return + entries = [ + cg.StructInitializer( + Ipv4AllowEntry, + ("addr", _network_order(net.network_address)), + ("mask", _network_order(net.netmask)), + ) + for net in networks + ] + arr_id = ID(f"{owner_id}_ipv4_allow", is_declaration=True, type=Ipv4AllowEntry) + arr = cg.progmem_array(arr_id, cg.ArrayInitializer(*entries)) + cg.add(setter(arr, len(entries))) + + def require_ipv4_resolve() -> None: """Compile the shared IPv4 lookup; call from a consumer's to_code.""" cg.add_define("USE_SOCKET_IPV4_RESOLVE") diff --git a/esphome/components/socket/ipv4_allow.h b/esphome/components/socket/ipv4_allow.h new file mode 100644 index 0000000000..adb00b5449 --- /dev/null +++ b/esphome/components/socket/ipv4_allow.h @@ -0,0 +1,56 @@ +#pragma once + +#include "headers.h" +#include "socket.h" +#include "esphome/core/hal.h" + +#include +#include + +namespace esphome::socket { + +/// One allowed IPv4 network, network byte order, host bits cleared. +/// Lives in flash; read via progmem_memcpy. +struct Ipv4AllowEntry { + uint32_t addr; + uint32_t mask; +}; + +/// IPv4 peers that may connect. An empty list allows every peer. +class Ipv4Allow { + public: + void set(const Ipv4AllowEntry *entries, size_t count) { + this->entries_ = entries; + this->count_ = count; + } + + /// A v4 mapped IPv6 peer is unwrapped; any other family fails a non empty list. + bool allows(const struct sockaddr *peer) const { + if (this->count_ == 0) { + return true; + } + uint32_t addr; + return sockaddr_to_ipv4(peer, &addr) && this->allows(addr); + } + + /// addr is network byte order, as it sits in a sockaddr_in. + bool allows(uint32_t addr) const { + if (this->count_ == 0) { + return true; + } + for (size_t i = 0; i != this->count_; i++) { + Ipv4AllowEntry entry; + progmem_memcpy(&entry, &this->entries_[i], sizeof(entry)); + if ((addr & entry.mask) == entry.addr) { + return true; + } + } + return false; + } + + private: + const Ipv4AllowEntry *entries_{nullptr}; + size_t count_{0}; +}; + +} // namespace esphome::socket diff --git a/esphome/components/socket/socket.cpp b/esphome/components/socket/socket.cpp index f2d428af77..f6d1daa124 100644 --- a/esphome/components/socket/socket.cpp +++ b/esphome/components/socket/socket.cpp @@ -78,6 +78,25 @@ static inline const char *esphome_inet_ntop6(const void *addr, char *buf, size_t #endif #endif +bool sockaddr_to_ipv4(const struct sockaddr *addr, uint32_t *out) { + if (addr->sa_family == AF_INET) { + *out = reinterpret_cast(addr)->sin_addr.s_addr; + return true; + } +#if USE_NETWORK_IPV6 + if (addr->sa_family == AF_INET6) { + // ::ffff:a.b.c.d; s6_addr is the portable byte view on every stack. + static constexpr uint8_t V4_MAPPED_PREFIX[12] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF}; + const uint8_t *bytes = reinterpret_cast(addr)->sin6_addr.s6_addr; + if (memcmp(bytes, V4_MAPPED_PREFIX, sizeof(V4_MAPPED_PREFIX)) == 0) { + memcpy(out, bytes + sizeof(V4_MAPPED_PREFIX), sizeof(*out)); + return true; + } + } +#endif + return false; +} + // Format sockaddr into caller-provided buffer, returns length written (excluding null) size_t format_sockaddr_to(const struct sockaddr *addr_ptr, socklen_t len, std::span buf) { if (addr_ptr->sa_family == AF_INET && len >= sizeof(const struct sockaddr_in)) { @@ -88,29 +107,10 @@ size_t format_sockaddr_to(const struct sockaddr *addr_ptr, socklen_t len, std::s #if USE_NETWORK_IPV6 else if (addr_ptr->sa_family == AF_INET6 && len >= sizeof(sockaddr_in6)) { const auto *addr = reinterpret_cast(addr_ptr); -#ifdef USE_HOST - // Format IPv4-mapped IPv6 addresses as regular IPv4 (POSIX layout, no LWIP union) - if (IN6_IS_ADDR_V4MAPPED(&addr->sin6_addr) && - esphome_inet_ntop4(&addr->sin6_addr.s6_addr[12], buf.data(), buf.size()) != nullptr) { + uint32_t v4; + // Format a v4 mapped peer as plain IPv4. + if (sockaddr_to_ipv4(addr_ptr, &v4) && esphome_inet_ntop4(&v4, buf.data(), buf.size()) != nullptr) return strlen(buf.data()); - } -#elif defined(USE_ZEPHYR) - // Format IPv4-mapped IPv6 addresses as regular IPv4. Zephyr uses the standard POSIX - // s6_addr layout (not the LWIP union) but provides no IN6_IS_ADDR_V4MAPPED macro, so - // detect the ::ffff:0:0/96 prefix directly on the address words. - if (addr->sin6_addr.s6_addr32[0] == 0 && addr->sin6_addr.s6_addr32[1] == 0 && - addr->sin6_addr.s6_addr32[2] == htonl(0xFFFF) && - esphome_inet_ntop4(&addr->sin6_addr.s6_addr32[3], buf.data(), buf.size()) != nullptr) { - return strlen(buf.data()); - } -#elif !defined(USE_SOCKET_IMPL_LWIP_TCP) - // Format IPv4-mapped IPv6 addresses as regular IPv4 (LWIP layout) - if (addr->sin6_addr.un.u32_addr[0] == 0 && addr->sin6_addr.un.u32_addr[1] == 0 && - addr->sin6_addr.un.u32_addr[2] == htonl(0xFFFF) && - esphome_inet_ntop4(&addr->sin6_addr.un.u32_addr[3], buf.data(), buf.size()) != nullptr) { - return strlen(buf.data()); - } -#endif if (esphome_inet_ntop6(&addr->sin6_addr, buf.data(), buf.size()) != nullptr) return strlen(buf.data()); } diff --git a/esphome/components/socket/socket.h b/esphome/components/socket/socket.h index 8ce392ff1d..9d7dedbbbf 100644 --- a/esphome/components/socket/socket.h +++ b/esphome/components/socket/socket.h @@ -137,6 +137,10 @@ inline std::unique_ptr socket_ip_loop_monitored(int type, int prot /// @return Size of the sockaddr structure used, or 0 on error socklen_t set_sockaddr(struct sockaddr *addr, socklen_t addrlen, const char *ip_address, uint16_t port); +/// Network order IPv4 of an AF_INET or v4 mapped AF_INET6 peer; false for anything else. +/// addr must hold the family's full struct. +bool sockaddr_to_ipv4(const struct sockaddr *addr, uint32_t *out); + /// Convenience overload for std::string (backward compatible). inline socklen_t set_sockaddr(struct sockaddr *addr, socklen_t addrlen, const std::string &ip_address, uint16_t port) { return set_sockaddr(addr, addrlen, ip_address.c_str(), port); diff --git a/tests/components/socket/test_ipv4_allow_host.cpp b/tests/components/socket/test_ipv4_allow_host.cpp new file mode 100644 index 0000000000..e4257b41c4 --- /dev/null +++ b/tests/components/socket/test_ipv4_allow_host.cpp @@ -0,0 +1,75 @@ +#include + +#include + +#include "esphome/components/socket/ipv4_allow.h" +#include "esphome/components/socket/socket.h" + +#ifdef USE_HOST + +namespace esphome::socket::testing { + +// The size_t count packs into the pointer's padding; no RAM over a uint8_t. +static_assert(sizeof(Ipv4Allow) == 2 * sizeof(void *), "unexpected padding in Ipv4Allow"); + +// 192.168.175.20/32 and 192.168.175.0/24, network order, host bits cleared, +// mirroring what add_ipv4_allow emits. +static const Ipv4AllowEntry ENTRIES[] = { + {htonl(0xC0A8AF14), htonl(0xFFFFFFFF)}, + {htonl(0xC0A8AF00), htonl(0xFFFFFF00)}, +}; + +// Runs the peer through the same parser production addresses go through. +static bool allows_peer(const Ipv4Allow &list, const char *ip) { + struct sockaddr_storage peer {}; + EXPECT_NE(set_sockaddr(reinterpret_cast(&peer), sizeof(peer), ip, 0), 0); + return list.allows(reinterpret_cast(&peer)); +} + +TEST(Ipv4Allow, EmptyAllowsEveryPeer) { + Ipv4Allow list; + EXPECT_TRUE(list.allows(htonl(0xC0A8AF01))); + EXPECT_TRUE(allows_peer(list, "10.0.0.1")); + EXPECT_TRUE(allows_peer(list, "fe80::1")); +} + +TEST(Ipv4Allow, MatchesHostAndNetworkEntries) { + Ipv4Allow list; + list.set(ENTRIES, std::size(ENTRIES)); + EXPECT_TRUE(list.allows(htonl(0xC0A8AF14))); + EXPECT_TRUE(list.allows(htonl(0xC0A8AF01))); + EXPECT_TRUE(list.allows(htonl(0xC0A8AFFF))); + EXPECT_FALSE(list.allows(htonl(0xC0A8B001))); +} + +TEST(Ipv4Allow, ChecksTheV4PeerInsideASockaddr) { + Ipv4Allow list; + list.set(ENTRIES, std::size(ENTRIES)); + EXPECT_TRUE(allows_peer(list, "192.168.175.66")); + EXPECT_FALSE(allows_peer(list, "10.0.0.1")); +} + +TEST(Ipv4Allow, UnwrapsAV4MappedIpv6Peer) { + Ipv4Allow list; + list.set(ENTRIES, std::size(ENTRIES)); + EXPECT_TRUE(allows_peer(list, "::ffff:192.168.175.66")); + // A native IPv6 peer cannot match an IPv4 list. + EXPECT_FALSE(allows_peer(list, "fe80::1")); +} + +TEST(Ipv4Allow, InstancesKeepIndependentLists) { + // One bridge per allow list; each instance points at its own entries. + static const Ipv4AllowEntry OTHER[] = {{htonl(0x0A000000), htonl(0xFF000000)}}; + Ipv4Allow first; + Ipv4Allow second; + first.set(ENTRIES, std::size(ENTRIES)); + second.set(OTHER, std::size(OTHER)); + EXPECT_TRUE(first.allows(htonl(0xC0A8AF14))); + EXPECT_FALSE(second.allows(htonl(0xC0A8AF14))); + EXPECT_TRUE(second.allows(htonl(0x0A00002A))); + EXPECT_FALSE(first.allows(htonl(0x0A00002A))); +} + +} // namespace esphome::socket::testing + +#endif diff --git a/tests/unit_tests/components/socket/test_socket_ipv4_allow.py b/tests/unit_tests/components/socket/test_socket_ipv4_allow.py new file mode 100644 index 0000000000..bc1185c26f --- /dev/null +++ b/tests/unit_tests/components/socket/test_socket_ipv4_allow.py @@ -0,0 +1,46 @@ +"""Tests for the socket component's IPv4 allow list codegen helper.""" + +from ipaddress import IPv4Address, IPv4Network +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.components import socket +import esphome.config_validation as cv + + +def test_network_order_swaps_to_sockaddr_value() -> None: + """The emitted uint32 must equal s_addr on the little endian targets.""" + assert socket._network_order(IPv4Address("192.168.175.20")) == 0x14AFA8C0 + assert socket._network_order(IPv4Address("255.255.255.0")) == 0x00FFFFFF + assert socket._network_order(IPv4Address("0.0.0.0")) == 0 + + +def test_add_ipv4_allow_emits_nothing_for_an_empty_list() -> None: + setter = MagicMock() + with patch.object(socket.cg, "add") as add: + socket.add_ipv4_allow(setter, [], "bridge") + add.assert_not_called() + setter.assert_not_called() + + +def test_add_ipv4_allow_wires_the_setter_with_cleared_host_bits() -> None: + setter = MagicMock() + networks = [IPv4Network("192.168.175.33/24", strict=False)] + with ( + patch.object(socket.cg, "add") as add, + patch.object(socket.cg, "progmem_array") as array, + ): + socket.add_ipv4_allow(setter, networks, "bridge") + rendered = str(array.call_args.args[1]) + assert str(socket._network_order(IPv4Address("192.168.175.0"))) in rendered + assert str(socket._network_order(IPv4Address("255.255.255.0"))) in rendered + setter.assert_called_once_with(array.return_value, 1) + add.assert_called_once() + + +def test_schema_caps_the_list_length() -> None: + """The sanity cap rejects a list past 255 entries.""" + assert len(socket.IPV4_ALLOW_SCHEMA([f"10.0.{i}.0/24" for i in range(255)])) == 255 + with pytest.raises(cv.Invalid): + socket.IPV4_ALLOW_SCHEMA([f"10.0.{i}.0/24" for i in range(256)]) From 995109aa4f361f5ccce1857a5f0c4148013e1a7a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:28:41 -0500 Subject: [PATCH 076/221] [es8388] Use the sub_* helpers for optional child entities (#20030) --- esphome/components/es8388/select/__init__.py | 26 ++++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/esphome/components/es8388/select/__init__.py b/esphome/components/es8388/select/__init__.py index b81bcd13cf..62ddd6cb38 100644 --- a/esphome/components/es8388/select/__init__.py +++ b/esphome/components/es8388/select/__init__.py @@ -30,19 +30,13 @@ CONFIG_SCHEMA = cv.All( async def to_code(config: ConfigType) -> None: - parent = await cg.get_variable(config[CONF_ES8388_ID]) - if dac_output_config := config.get(CONF_DAC_OUTPUT): - s = await select.new_select( - dac_output_config, - options=["LINE1", "LINE2", "BOTH"], - ) - await cg.register_parented(s, parent) - cg.add(parent.set_dac_output_select(s)) - - if adc_input_mic_config := config.get(CONF_ADC_INPUT_MIC): - s = await select.new_select( - adc_input_mic_config, - options=["LINE1", "LINE2", "DIFFERENCE"], - ) - await cg.register_parented(s, parent) - cg.add(parent.set_adc_input_mic_select(s)) + hub = await cg.get_variable(config[CONF_ES8388_ID]) + selects = select.sub_selects(config, parent=hub) + await selects( + CONF_DAC_OUTPUT, hub.set_dac_output_select, options=["LINE1", "LINE2", "BOTH"] + ) + await selects( + CONF_ADC_INPUT_MIC, + hub.set_adc_input_mic_select, + options=["LINE1", "LINE2", "DIFFERENCE"], + ) From 2d286031a4fd18a27e7a69b8ebde58773a44f77c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:29:09 -0500 Subject: [PATCH 077/221] [sht3xd] Use the sub_* helpers for optional child entities (#20031) --- esphome/components/sht3xd/sensor.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/esphome/components/sht3xd/sensor.py b/esphome/components/sht3xd/sensor.py index eadca15050..baee146a22 100644 --- a/esphome/components/sht3xd/sensor.py +++ b/esphome/components/sht3xd/sensor.py @@ -56,10 +56,6 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_heater_enabled(config[CONF_HEATER_ENABLED])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) From 25a30fc472a085a729eb5da906948165c493150a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:29:37 -0500 Subject: [PATCH 078/221] [mhz19] Use the sub_* helpers for optional child entities (#20032) --- esphome/components/mhz19/sensor.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/esphome/components/mhz19/sensor.py b/esphome/components/mhz19/sensor.py index 9b0eeeb28e..053f17765e 100644 --- a/esphome/components/mhz19/sensor.py +++ b/esphome/components/mhz19/sensor.py @@ -77,13 +77,9 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await uart.register_uart_device(var, config) - if co2 := config.get(CONF_CO2): - sens = await sensor.new_sensor(co2) - cg.add(var.set_co2_sensor(sens)) - - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) if ( automatic_baseline_calibration := config.get( From 4f1be0fcb24a08e4c7427323d8caa8c6b4db1f50 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:29:58 -0500 Subject: [PATCH 079/221] [shtcx] Use the sub_* helpers for optional child entities (#20033) --- esphome/components/shtcx/sensor.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/esphome/components/shtcx/sensor.py b/esphome/components/shtcx/sensor.py index bda48c07ce..d6ca2b131b 100644 --- a/esphome/components/shtcx/sensor.py +++ b/esphome/components/shtcx/sensor.py @@ -51,10 +51,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature_sensor(sens)) - - if humidity := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) From cbe89c940d4629371b584f0bb498a904a954ab73 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 11:30:24 -0500 Subject: [PATCH 080/221] [atm90e32] Use the sub_* helpers for optional child entities (#20034) --- esphome/components/atm90e32/sensor.py | 9 +++------ esphome/components/atm90e32/text_sensor/__init__.py | 9 ++++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/esphome/components/atm90e32/sensor.py b/esphome/components/atm90e32/sensor.py index 38b24c7cf6..e3d5fa2de2 100644 --- a/esphome/components/atm90e32/sensor.py +++ b/esphome/components/atm90e32/sensor.py @@ -241,12 +241,9 @@ async def to_code(config: ConfigType) -> None: if peak_current_config := conf.get(CONF_PEAK_CURRENT): sens = await sensor.new_sensor(peak_current_config) cg.add(var.set_peak_current_sensor(i, sens)) - if frequency_config := config.get(CONF_FREQUENCY): - sens = await sensor.new_sensor(frequency_config) - cg.add(var.set_freq_sensor(sens)) - if chip_temperature_config := config.get(CONF_CHIP_TEMPERATURE): - sens = await sensor.new_sensor(chip_temperature_config) - cg.add(var.set_chip_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_FREQUENCY, var.set_freq_sensor) + await sensors(CONF_CHIP_TEMPERATURE, var.set_chip_temperature_sensor) cg.add(var.set_line_freq(config[CONF_LINE_FREQUENCY])) cg.add(var.set_current_phases(config[CONF_CURRENT_PHASES])) cg.add(var.set_pga_gain(config[CONF_GAIN_PGA])) diff --git a/esphome/components/atm90e32/text_sensor/__init__.py b/esphome/components/atm90e32/text_sensor/__init__.py index 30585cb873..6b959c19fa 100644 --- a/esphome/components/atm90e32/text_sensor/__init__.py +++ b/esphome/components/atm90e32/text_sensor/__init__.py @@ -36,14 +36,13 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: - parent = await cg.get_variable(config[CONF_ID]) + hub = await cg.get_variable(config[CONF_ID]) if phase_cfg := config.get(CONF_PHASE_STATUS): for i, key in enumerate(PHASE_KEYS): if sub_phase_cfg := phase_cfg.get(key): sens = await text_sensor.new_text_sensor(sub_phase_cfg) - cg.add(parent.set_phase_status_text_sensor(i, sens)) + cg.add(hub.set_phase_status_text_sensor(i, sens)) - if freq_status_config := config.get(CONF_FREQUENCY_STATUS): - sens = await text_sensor.new_text_sensor(freq_status_config) - cg.add(parent.set_freq_status_text_sensor(sens)) + text_sensors = text_sensor.sub_text_sensors(config) + await text_sensors(CONF_FREQUENCY_STATUS, hub.set_freq_status_text_sensor) From b6c79fdc9ae41ec6d6920827190ea23d38e8835e Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:07:00 +0200 Subject: [PATCH 081/221] [socket] Share the server side as TcpListener (#20037) Co-authored-by: J. Nick Koston --- esphome/components/socket/__init__.py | 20 +++++ esphome/components/socket/ipv4_allow.h | 13 ++- esphome/components/socket/tcp_listener.cpp | 90 +++++++++++++++++++ esphome/components/socket/tcp_listener.h | 60 +++++++++++++ esphome/components/uart_tcp/__init__.py | 23 ++--- esphome/components/uart_tcp/uart_tcp.cpp | 66 +++----------- esphome/components/uart_tcp/uart_tcp.h | 16 +++- esphome/core/defines.h | 2 + tests/components/uart_tcp/common.yaml | 3 + .../integration/fixtures/uart_tcp_bridge.yaml | 8 ++ tests/integration/test_uart_tcp_bridge.py | 10 +++ .../socket/test_socket_ipv4_allow.py | 8 +- .../socket/test_socket_source_filter.py | 13 +++ 13 files changed, 262 insertions(+), 70 deletions(-) create mode 100644 esphome/components/socket/tcp_listener.cpp create mode 100644 esphome/components/socket/tcp_listener.h diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index 850f7ebcbc..269791f5e7 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -5,6 +5,7 @@ from ipaddress import IPv4Address, IPv4Network import logging import esphome.codegen as cg +from esphome.components.const import CONF_ROLE from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv from esphome.core import CORE, ID @@ -162,6 +163,7 @@ def add_ipv4_allow( """ if not networks: return + cg.add_define("USE_SOCKET_IPV4_ALLOW") entries = [ cg.StructInitializer( Ipv4AllowEntry, @@ -186,6 +188,23 @@ def require_tcp_client_link() -> None: cg.add_define("USE_SOCKET_TCP_CLIENT_LINK") +def require_tcp_listener() -> None: + """Compile the TCP listener; call from a server role's to_code.""" + require_tcp_client_link() + cg.add_define("USE_SOCKET_TCP_LISTENER") + + +def consume_role_sockets(component: str) -> Callable[[ConfigType], ConfigType]: + """Socket accounting for a role keyed client or server schema.""" + + def validator(config: ConfigType) -> ConfigType: + if config[CONF_ROLE] == "server": + consume_sockets(1, component, SocketType.TCP_LISTEN)(config) + return consume_sockets(1, component)(config) + + return validator + + CONFIG_SCHEMA = cv.Schema( { cv.SplitDefault( @@ -239,5 +258,6 @@ FILTER_SOURCE_FILES = filter_source_files_from_defines( "lwip_sockets_impl.cpp": "USE_SOCKET_IMPL_LWIP_SOCKETS", "ipv4_resolve.cpp": "USE_SOCKET_IPV4_RESOLVE", "tcp_client_link.cpp": "USE_SOCKET_TCP_CLIENT_LINK", + "tcp_listener.cpp": "USE_SOCKET_TCP_LISTENER", } ) diff --git a/esphome/components/socket/ipv4_allow.h b/esphome/components/socket/ipv4_allow.h index adb00b5449..f2f66edc05 100644 --- a/esphome/components/socket/ipv4_allow.h +++ b/esphome/components/socket/ipv4_allow.h @@ -39,15 +39,22 @@ class Ipv4Allow { return true; } for (size_t i = 0; i != this->count_; i++) { - Ipv4AllowEntry entry; - progmem_memcpy(&entry, &this->entries_[i], sizeof(entry)); - if ((addr & entry.mask) == entry.addr) { + Ipv4AllowEntry e = this->entry(i); + if ((addr & e.mask) == e.addr) { return true; } } return false; } + size_t size() const { return this->count_; } + /// A copy of entry i, read from flash. + Ipv4AllowEntry entry(size_t i) const { + Ipv4AllowEntry e; + progmem_memcpy(&e, &this->entries_[i], sizeof(e)); + return e; + } + private: const Ipv4AllowEntry *entries_{nullptr}; size_t count_{0}; diff --git a/esphome/components/socket/tcp_listener.cpp b/esphome/components/socket/tcp_listener.cpp new file mode 100644 index 0000000000..e1f00e2927 --- /dev/null +++ b/esphome/components/socket/tcp_listener.cpp @@ -0,0 +1,90 @@ +#include "tcp_listener.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +#include +#include + +namespace esphome::socket { + +// One client at a time; a second connection waits in the stack until the first drops. +static constexpr int LISTEN_BACKLOG = 1; +#ifdef USE_SOCKET_IPV4_ALLOW +static constexpr uint32_t REJECT_LOG_INTERVAL_MS = 5000; +#endif + +void TcpListener::try_listen_(TcpClientLink &link) { + this->listen_ = socket_ip_loop_monitored(SOCK_STREAM, IPPROTO_TCP); + int err = errno; + if (this->listen_ != nullptr) { + int yes = 1; + this->listen_->setsockopt(SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + struct sockaddr_storage local; + socklen_t local_len = set_sockaddr_any(reinterpret_cast(&local), sizeof(local), link.port()); + // A blocking listener would stall loop() inside accept(), so its + // setblocking result is part of the success condition. + if (this->listen_->setblocking(false) == 0 && local_len != 0 && + this->listen_->bind(reinterpret_cast(&local), local_len) == 0 && + this->listen_->listen(LISTEN_BACKLOG) == 0) { + ESP_LOGI(this->tag_, "Listening on %u", link.port()); + return; + } + // Captured before reset(); the close inside can overwrite errno. + err = errno; + this->listen_.reset(); + } + ESP_LOGW(this->tag_, "Listen on %u failed: %d", link.port(), err); + link.note_attempt(); +} + +void TcpListener::accept_(TcpClientLink &link) { + struct sockaddr_storage peer {}; + socklen_t peer_len = sizeof(peer); + auto client = this->listen_->accept_loop_monitored(reinterpret_cast(&peer), &peer_len); + if (client == nullptr) { + // A reset during the handshake or a signal only affects that connection. + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED || errno == EINTR) { + return; + } + // Rebuild the listener after the backoff instead of spinning on it. + int err = errno; + this->listen_.reset(); + ESP_LOGW(this->tag_, "Accept failed: %d", err); + link.note_attempt(); + return; + } + const auto *sa = reinterpret_cast(&peer); + char text[SOCKADDR_STR_LEN]; + format_sockaddr_to(sa, peer_len, std::span(text)); +#ifdef USE_SOCKET_IPV4_ALLOW + if (!this->allow_.allows(sa)) { + uint32_t now = App.get_loop_component_start_time(); + if (this->last_reject_log_ms_ == 0 || now - this->last_reject_log_ms_ >= REJECT_LOG_INTERVAL_MS) { + this->last_reject_log_ms_ = now; + ESP_LOGW(this->tag_, "Rejected %s", text); + } + return; + } +#endif + link.adopt(std::move(client)); + ESP_LOGI(this->tag_, "Client connected from %s", text); +} + +void TcpListener::dump_config() const { +#ifdef USE_SOCKET_IPV4_ALLOW + for (size_t i = 0; i < this->allow_.size(); i++) { + Ipv4AllowEntry e = this->allow_.entry(i); + // Network order is dotted order, and the contiguous mask's popcount is the prefix. + const auto *b = reinterpret_cast(&e.addr); + ESP_LOGCONFIG(this->tag_, " Allowed IP: %u.%u.%u.%u/%u", b[0], b[1], b[2], b[3], + static_cast(__builtin_popcount(e.mask))); + } +#endif +} + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/socket/tcp_listener.h b/esphome/components/socket/tcp_listener.h new file mode 100644 index 0000000000..a42080e641 --- /dev/null +++ b/esphome/components/socket/tcp_listener.h @@ -0,0 +1,60 @@ +#pragma once + +#include "headers.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#ifdef USE_SOCKET_IPV4_ALLOW +#include "ipv4_allow.h" +#endif +#include "socket.h" +#include "tcp_client_link.h" + +#include +#include + +namespace esphome::socket { + +/// The server side of a bridged TCP link: owns the listen socket and the +/// allow list, accepts one peer at a time and adopts it into a TcpClientLink, +/// sharing that link's retry clock and connect port. +class TcpListener { + public: +#ifdef USE_SOCKET_IPV4_ALLOW + void set_allow(const Ipv4AllowEntry *entries, size_t count) { this->allow_.set(entries, count); } +#endif + + /// Call from setup(); tag names the log lines. + void begin(const char *tag) { this->tag_ = tag; } + /// Server state machine; call every loop. may_accept lets the caller hold + /// accepts until its own disconnect edge has run. + void poll(TcpClientLink &link, bool may_accept) { + if (this->listen_ == nullptr) { + if (!link.in_backoff()) { + this->try_listen_(link); + } + return; + } + if (may_accept && !link.connected() && this->listen_->ready()) { + this->accept_(link); + } + } + void close() { this->listen_.reset(); } + /// One config line per allowed network. + void dump_config() const; + + protected: + void try_listen_(TcpClientLink &link); + void accept_(TcpClientLink &link); + + std::unique_ptr listen_; + const char *tag_{nullptr}; +#ifdef USE_SOCKET_IPV4_ALLOW + uint32_t last_reject_log_ms_{0}; + Ipv4Allow allow_; +#endif +}; + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/uart_tcp/__init__.py b/esphome/components/uart_tcp/__init__.py index e7a85a0f2d..cd56e2c8bc 100644 --- a/esphome/components/uart_tcp/__init__.py +++ b/esphome/components/uart_tcp/__init__.py @@ -19,15 +19,10 @@ MULTI_CONF = True uart_tcp_ns = cg.esphome_ns.namespace("uart_tcp") UartTcp = uart_tcp_ns.class_("UartTcp", cg.Component, uart.UARTDevice) +CONF_ALLOWED_IPS = "allowed_ips" CONF_CONNECTED = "connected" -def _consume_sockets(config: ConfigType) -> ConfigType: - if config[CONF_ROLE] == "server": - socket.consume_sockets(1, "uart_tcp", socket.SocketType.TCP_LISTEN)(config) - return socket.consume_sockets(1, "uart_tcp")(config) - - BASE_SCHEMA = cv.Schema( { cv.GenerateID(): cv.declare_id(UartTcp), @@ -47,22 +42,30 @@ CONFIG_SCHEMA = cv.All( cv.typed_schema( { "client": BASE_SCHEMA.extend({cv.Required(CONF_HOST): cv.string}), - "server": BASE_SCHEMA, + "server": BASE_SCHEMA.extend( + {cv.Optional(CONF_ALLOWED_IPS): socket.IPV4_ALLOW_SCHEMA} + ), }, key=CONF_ROLE, default_type="client", lower=True, ), - _consume_sockets, + socket.consume_role_sockets("uart_tcp"), ) async def to_code(config: ConfigType) -> None: - socket.require_tcp_client_link() var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) await uart.register_uart_device(var, config) - cg.add(var.set_server(config[CONF_ROLE] == "server")) + if config[CONF_ROLE] == "server": + socket.require_tcp_listener() + cg.add(var.set_server(True)) + socket.add_ipv4_allow( + var.set_allow, config.get(CONF_ALLOWED_IPS), config[CONF_ID] + ) + else: + socket.require_tcp_client_link() cg.add(var.set_port(config[CONF_PORT])) cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL])) if (host := config.get(CONF_HOST)) is not None: diff --git a/esphome/components/uart_tcp/uart_tcp.cpp b/esphome/components/uart_tcp/uart_tcp.cpp index 2531b73b20..cf15802b55 100644 --- a/esphome/components/uart_tcp/uart_tcp.cpp +++ b/esphome/components/uart_tcp/uart_tcp.cpp @@ -10,13 +10,14 @@ namespace esphome::uart_tcp { static const char *const TAG = "uart_tcp"; -// One client at a time; a second connection waits in the stack until the first drops. -static constexpr int LISTEN_BACKLOG = 1; // Bytes per 16 ms loop pass at 10 bits per byte: baud / 10 / 62.5. static constexpr uint32_t BAUD_PACE_DIVISOR = 625; void UartTcp::setup() { this->link_.begin(TAG); +#ifdef USE_SOCKET_TCP_LISTENER + this->listener_.begin(TAG); +#endif if (this->connected_sensor_ != nullptr) { this->connected_sensor_->publish_state(false); } @@ -30,12 +31,17 @@ void UartTcp::dump_config() { this->server_ ? LOG_STR_LITERAL("Listen") : LOG_STR_LITERAL("Host"), this->server_ ? LOG_STR_LITERAL("*") : this->link_.host(), this->link_.port(), this->link_.reconnect_interval()); +#ifdef USE_SOCKET_TCP_LISTENER + this->listener_.dump_config(); +#endif LOG_BINARY_SENSOR(" ", "Connected", this->connected_sensor_); } void UartTcp::on_shutdown() { this->link_.close(); - this->listen_.reset(); +#ifdef USE_SOCKET_TCP_LISTENER + this->listener_.close(); +#endif } void UartTcp::sync_link_() { @@ -50,49 +56,6 @@ void UartTcp::sync_link_() { } } -void UartTcp::try_listen_() { - this->listen_ = socket::socket_ip_loop_monitored(SOCK_STREAM, IPPROTO_TCP); - int err = errno; - if (this->listen_ != nullptr) { - int yes = 1; - this->listen_->setsockopt(SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); - struct sockaddr_storage local; - socklen_t local_len = - socket::set_sockaddr_any(reinterpret_cast(&local), sizeof(local), this->link_.port()); - // A blocking listener would stall loop() inside accept(), so its - // setblocking result is part of the success condition. - if (this->listen_->setblocking(false) == 0 && local_len != 0 && - this->listen_->bind(reinterpret_cast(&local), local_len) == 0 && - this->listen_->listen(LISTEN_BACKLOG) == 0) { - ESP_LOGI(TAG, "Listening on %u", this->link_.port()); - return; - } - // Captured before reset(); the close inside can overwrite errno. - err = errno; - this->listen_.reset(); - } - ESP_LOGW(TAG, "Listen on %u failed: %d", this->link_.port(), err); - this->link_.note_attempt(); -} - -void UartTcp::accept_client_() { - auto client = this->listen_->accept_loop_monitored(nullptr, nullptr); - if (client == nullptr) { - // A reset during the handshake or a signal only affects that connection. - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED || errno == EINTR) { - return; - } - // Rebuild the listener after the backoff instead of spinning on it. - int err = errno; - this->listen_.reset(); - ESP_LOGW(TAG, "Accept failed: %d", err); - this->link_.note_attempt(); - return; - } - this->link_.adopt(std::move(client)); - ESP_LOGI(TAG, "Client connected"); -} - void UartTcp::read_socket_() { // A hardware write blocks until the driver takes every byte. Leave what does // not fit in the socket, so TCP flow control throttles the peer. @@ -141,18 +104,17 @@ void UartTcp::read_uart_() { } void UartTcp::loop() { +#ifdef USE_SOCKET_TCP_LISTENER if (this->server_) { - if (this->listen_ == nullptr && !this->link_.in_backoff()) { - this->try_listen_(); - } // link_was_up_ holds the accept until the previous drop's edge has run, // so the sensor and the stale UART discard always see the disconnect. - if (this->listen_ != nullptr && !this->link_.connected() && !this->link_was_up_ && this->listen_->ready()) { - this->accept_client_(); - } + this->listener_.poll(this->link_, !this->link_was_up_); } else { this->link_.poll(); } +#else + this->link_.poll(); +#endif if (this->link_.connected() != this->link_was_up_) { this->sync_link_(); } diff --git a/esphome/components/uart_tcp/uart_tcp.h b/esphome/components/uart_tcp/uart_tcp.h index 2a59bf968f..1e19166f87 100644 --- a/esphome/components/uart_tcp/uart_tcp.h +++ b/esphome/components/uart_tcp/uart_tcp.h @@ -2,6 +2,9 @@ #include "esphome/components/binary_sensor/binary_sensor.h" #include "esphome/components/socket/tcp_client_link.h" +#ifdef USE_SOCKET_TCP_LISTENER +#include "esphome/components/socket/tcp_listener.h" +#endif #include "esphome/components/uart/uart.h" #include "esphome/core/component.h" @@ -15,9 +18,14 @@ class UartTcp : public Component, public uart::UARTDevice { public: void set_host(const char *host) { this->link_.set_host(host); } void set_port(uint16_t port) { this->link_.set_port(port); } - void set_server(bool server) { this->server_ = server; } void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); } void set_connected_sensor(binary_sensor::BinarySensor *sensor) { this->connected_sensor_ = sensor; } +#ifdef USE_SOCKET_TCP_LISTENER + void set_server(bool server) { this->server_ = server; } +#ifdef USE_SOCKET_IPV4_ALLOW + void set_allow(const socket::Ipv4AllowEntry *entries, size_t count) { this->listener_.set_allow(entries, count); } +#endif +#endif void setup() override; void loop() override; @@ -27,8 +35,6 @@ class UartTcp : public Component, public uart::UARTDevice { protected: void sync_link_(); - void try_listen_(); - void accept_client_(); void read_socket_(); void read_uart_(); void discard_uart_(); @@ -36,7 +42,9 @@ class UartTcp : public Component, public uart::UARTDevice { static constexpr size_t READ_CHUNK = 128; socket::TcpClientLink link_; - std::unique_ptr listen_; +#ifdef USE_SOCKET_TCP_LISTENER + socket::TcpListener listener_; +#endif binary_sensor::BinarySensor *connected_sensor_{nullptr}; bool server_{false}; // The link state loop() saw last; edges clear the buffer and publish the sensor. diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 21c4e3d996..97b1395553 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -419,8 +419,10 @@ #define USE_SENDSPIN_VISUALIZER #define USE_SENDSPIN_PORT 8928 // NOLINT #define USE_SOCKET_IMPL_BSD_SOCKETS +#define USE_SOCKET_IPV4_ALLOW #define USE_SOCKET_IPV4_RESOLVE #define USE_SOCKET_TCP_CLIENT_LINK +#define USE_SOCKET_TCP_LISTENER #define USE_LWIP_FAST_SELECT #define USE_SPEAKER diff --git a/tests/components/uart_tcp/common.yaml b/tests/components/uart_tcp/common.yaml index 8010878e89..eacad3dd6d 100644 --- a/tests/components/uart_tcp/common.yaml +++ b/tests/components/uart_tcp/common.yaml @@ -7,5 +7,8 @@ uart_tcp: uart_id: uart_bus role: server port: 502 + allowed_ips: + - 192.168.1.10 + - 192.168.1.0/24 connected: name: UART TCP Connected diff --git a/tests/integration/fixtures/uart_tcp_bridge.yaml b/tests/integration/fixtures/uart_tcp_bridge.yaml index c61add801f..2ab70a03b6 100644 --- a/tests/integration/fixtures/uart_tcp_bridge.yaml +++ b/tests/integration/fixtures/uart_tcp_bridge.yaml @@ -18,5 +18,13 @@ uart_tcp: uart_id: uart_bus role: server port: 18126 + allowed_ips: + - 127.0.0.1 connected: name: Bridge Connected + - id: denied_bridge + uart_id: uart_bus + role: server + port: 18127 + allowed_ips: + - 192.0.2.1 diff --git a/tests/integration/test_uart_tcp_bridge.py b/tests/integration/test_uart_tcp_bridge.py index b98d3ffac9..909df32410 100644 --- a/tests/integration/test_uart_tcp_bridge.py +++ b/tests/integration/test_uart_tcp_bridge.py @@ -25,6 +25,7 @@ async def test_uart_tcp_bridge( unused_tcp_port_factory, ) -> None: server_port = unused_tcp_port_factory() + denied_port = unused_tcp_port_factory() controller_fd, device_fd = os.openpty() os.set_blocking(controller_fd, False) # uart's validate_port wants a two segment device path; Linux ptys live at @@ -32,6 +33,7 @@ async def test_uart_tcp_bridge( pty_link = f"/tmp/uart-tcp-pty-{os.getpid()}" pathlib.Path(pty_link).symlink_to(os.ttyname(device_fd)) yaml_config = yaml_config.replace("port: 18126", f"port: {server_port}") + yaml_config = yaml_config.replace("port: 18127", f"port: {denied_port}") yaml_config = yaml_config.replace("PTY_PATH", pty_link) lines = LineWaiter() @@ -103,6 +105,14 @@ async def test_uart_tcp_bridge( await writer.drain() assert await read_uart(5) == b"down2" writer.close() + + # A peer outside the allow list is rejected and closed. + denied_reader, denied_writer = await asyncio.open_connection( + "127.0.0.1", denied_port + ) + await lines.wait_for("Rejected 127.0.0.1") + assert await asyncio.wait_for(denied_reader.read(8), 10) == b"" + denied_writer.close() finally: loop.remove_reader(controller_fd) os.close(controller_fd) diff --git a/tests/unit_tests/components/socket/test_socket_ipv4_allow.py b/tests/unit_tests/components/socket/test_socket_ipv4_allow.py index bc1185c26f..8a2f60a4b9 100644 --- a/tests/unit_tests/components/socket/test_socket_ipv4_allow.py +++ b/tests/unit_tests/components/socket/test_socket_ipv4_allow.py @@ -18,9 +18,13 @@ def test_network_order_swaps_to_sockaddr_value() -> None: def test_add_ipv4_allow_emits_nothing_for_an_empty_list() -> None: setter = MagicMock() - with patch.object(socket.cg, "add") as add: + with ( + patch.object(socket.cg, "add") as add, + patch.object(socket.cg, "add_define") as add_define, + ): socket.add_ipv4_allow(setter, [], "bridge") add.assert_not_called() + add_define.assert_not_called() setter.assert_not_called() @@ -29,6 +33,7 @@ def test_add_ipv4_allow_wires_the_setter_with_cleared_host_bits() -> None: networks = [IPv4Network("192.168.175.33/24", strict=False)] with ( patch.object(socket.cg, "add") as add, + patch.object(socket.cg, "add_define") as add_define, patch.object(socket.cg, "progmem_array") as array, ): socket.add_ipv4_allow(setter, networks, "bridge") @@ -37,6 +42,7 @@ def test_add_ipv4_allow_wires_the_setter_with_cleared_host_bits() -> None: assert str(socket._network_order(IPv4Address("255.255.255.0"))) in rendered setter.assert_called_once_with(array.return_value, 1) add.assert_called_once() + add_define.assert_called_once_with("USE_SOCKET_IPV4_ALLOW") def test_schema_caps_the_list_length() -> None: diff --git a/tests/unit_tests/components/socket/test_socket_source_filter.py b/tests/unit_tests/components/socket/test_socket_source_filter.py index 3967c9568d..c46a3f1e8f 100644 --- a/tests/unit_tests/components/socket/test_socket_source_filter.py +++ b/tests/unit_tests/components/socket/test_socket_source_filter.py @@ -13,6 +13,7 @@ def test_helper_files_filtered_until_required() -> None: filtered = socket.FILTER_SOURCE_FILES() assert "ipv4_resolve.cpp" in filtered assert "tcp_client_link.cpp" in filtered + assert "tcp_listener.cpp" in filtered mock_core.defines = {Define("USE_SOCKET_IPV4_RESOLVE")} filtered = socket.FILTER_SOURCE_FILES() @@ -22,10 +23,12 @@ def test_helper_files_filtered_until_required() -> None: mock_core.defines = { Define("USE_SOCKET_IPV4_RESOLVE"), Define("USE_SOCKET_TCP_CLIENT_LINK"), + Define("USE_SOCKET_TCP_LISTENER"), } filtered = socket.FILTER_SOURCE_FILES() assert "ipv4_resolve.cpp" not in filtered assert "tcp_client_link.cpp" not in filtered + assert "tcp_listener.cpp" not in filtered def test_require_tcp_client_link_pulls_in_the_resolver() -> None: @@ -36,3 +39,13 @@ def test_require_tcp_client_link_pulls_in_the_resolver() -> None: "USE_SOCKET_IPV4_RESOLVE", "USE_SOCKET_TCP_CLIENT_LINK", } + + +def test_require_tcp_listener_pulls_in_the_link() -> None: + with patch.object(socket.cg, "add_define") as add_define: + socket.require_tcp_listener() + assert {call.args[0] for call in add_define.call_args_list} == { + "USE_SOCKET_IPV4_RESOLVE", + "USE_SOCKET_TCP_CLIENT_LINK", + "USE_SOCKET_TCP_LISTENER", + } From 79ad67b56ffa9b35a25884d7864c9739a51dd36b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:28:18 -0500 Subject: [PATCH 082/221] [aht10][am2315c][am2320][dht][dht12] Use the sub_* helpers for optional child entities (#20038) --- esphome/components/aht10/sensor.py | 10 +++------- esphome/components/am2315c/sensor.py | 10 +++------- esphome/components/am2320/sensor.py | 10 +++------- esphome/components/dht/sensor.py | 9 +++------ esphome/components/dht12/sensor.py | 10 +++------- 5 files changed, 15 insertions(+), 34 deletions(-) diff --git a/esphome/components/aht10/sensor.py b/esphome/components/aht10/sensor.py index ae669d0000..add86e16a6 100644 --- a/esphome/components/aht10/sensor.py +++ b/esphome/components/aht10/sensor.py @@ -57,10 +57,6 @@ async def to_code(config: ConfigType) -> None: await i2c.register_i2c_device(var, config) cg.add(var.set_variant(config[CONF_VARIANT])) - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature_sensor(sens)) - - if humidity := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) diff --git a/esphome/components/am2315c/sensor.py b/esphome/components/am2315c/sensor.py index febb11409c..39ded956c3 100644 --- a/esphome/components/am2315c/sensor.py +++ b/esphome/components/am2315c/sensor.py @@ -46,10 +46,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) diff --git a/esphome/components/am2320/sensor.py b/esphome/components/am2320/sensor.py index ffac0e6407..30a1b1d8a3 100644 --- a/esphome/components/am2320/sensor.py +++ b/esphome/components/am2320/sensor.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) diff --git a/esphome/components/dht/sensor.py b/esphome/components/dht/sensor.py index 7376adb287..c7ae70b834 100644 --- a/esphome/components/dht/sensor.py +++ b/esphome/components/dht/sensor.py @@ -61,11 +61,8 @@ async def to_code(config: ConfigType) -> None: pin = await gpio_pin_expression(config[CONF_PIN]) cg.add(var.set_pin(pin)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) cg.add(var.set_dht_model(config[CONF_MODEL])) diff --git a/esphome/components/dht12/sensor.py b/esphome/components/dht12/sensor.py index 2bc6e94515..88877b2dc2 100644 --- a/esphome/components/dht12/sensor.py +++ b/esphome/components/dht12/sensor.py @@ -46,10 +46,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) From 78f39c7feed6a72a716f4402de18428c20af3dac Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:28:31 -0500 Subject: [PATCH 083/221] [hdc1080][hdc2010][hdc2080][hdc302x][hte501] Use the sub_* helpers for optional child entities (#20039) --- esphome/components/hdc1080/sensor.py | 10 +++------- esphome/components/hdc2010/sensor.py | 10 +++------- esphome/components/hdc2080/sensor.py | 10 +++------- esphome/components/hdc302x/sensor.py | 10 +++------- esphome/components/hte501/sensor.py | 10 +++------- 5 files changed, 15 insertions(+), 35 deletions(-) diff --git a/esphome/components/hdc1080/sensor.py b/esphome/components/hdc1080/sensor.py index b2b6dc533a..c35488c96c 100644 --- a/esphome/components/hdc1080/sensor.py +++ b/esphome/components/hdc1080/sensor.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) diff --git a/esphome/components/hdc2010/sensor.py b/esphome/components/hdc2010/sensor.py index ad0311fb4f..9837b9af33 100644 --- a/esphome/components/hdc2010/sensor.py +++ b/esphome/components/hdc2010/sensor.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) diff --git a/esphome/components/hdc2080/sensor.py b/esphome/components/hdc2080/sensor.py index b5388b4c2b..63625bbf94 100644 --- a/esphome/components/hdc2080/sensor.py +++ b/esphome/components/hdc2080/sensor.py @@ -49,10 +49,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) diff --git a/esphome/components/hdc302x/sensor.py b/esphome/components/hdc302x/sensor.py index 2686ba5322..46e1a95cb1 100644 --- a/esphome/components/hdc302x/sensor.py +++ b/esphome/components/hdc302x/sensor.py @@ -66,13 +66,9 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temp_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temp_config) - cg.add(var.set_temp_sensor(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temp_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) cg.add(var.set_power_mode(config[CONF_POWER_MODE])) diff --git a/esphome/components/hte501/sensor.py b/esphome/components/hte501/sensor.py index bf9fe4000e..80d5986295 100644 --- a/esphome/components/hte501/sensor.py +++ b/esphome/components/hte501/sensor.py @@ -50,10 +50,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature_sensor(sens)) - - if humidity := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) From 6d33b0401df057b09d3f0ad4dbe8cd1ddfd80c10 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:28:40 -0500 Subject: [PATCH 084/221] [nrf52] Use the shared managed ccache settings for the native build (#20027) --- esphome/components/nrf52/__init__.py | 15 ++-- esphome/components/nrf52/clang_tidy.py | 2 +- esphome/components/nrf52/framework.py | 16 ++++- tests/unit_tests/components/nrf52/test_pch.py | 71 ++++++++++++++----- tests/unit_tests/test_nrf52_framework.py | 29 +++++++- 5 files changed, 105 insertions(+), 28 deletions(-) diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 07507901ba..1e841fffcf 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -9,6 +9,7 @@ import subprocess from esphome import pins from esphome.build_helpers import pch +from esphome.build_helpers.ccache import resolve_ccache_path import esphome.codegen as cg from esphome.components.zephyr import ( add_extra_script, @@ -584,7 +585,7 @@ def upload_program(config: ConfigType, args, host: str) -> bool: raise EsphomeError("Not implemented yet") check_and_install() paths = get_build_paths() - env = get_build_env() + env = get_build_env(None) # no compile, just nrfutil build_dir = CORE.relative_pioenvs_path(CORE.name) dfu_package = build_dir / "firmware.zip" if not dfu_package.is_file(): @@ -666,7 +667,7 @@ def upload_program(config: ConfigType, args, host: str) -> bool: if not CORE.using_toolchain_platformio: check_and_install() paths = get_build_paths() - env = get_build_env() + env = get_build_env(resolve_ccache_path()) # west flash may rebuild build_dir = CORE.relative_pioenvs_path(CORE.name) west_cmd = [ str(paths["python_executable"]), @@ -949,7 +950,10 @@ def run_compile(args, config: ConfigType) -> bool: check_and_install() paths = get_build_paths() - env = get_build_env() + # Depend mode in the shared ccache settings keeps the .gch sound + # across Kconfig flips. + ccache = resolve_ccache_path() + env = get_build_env(ccache) pch_on = _pch_usable() cmake_lists_changed = _generate_cmake_lists(pch_on) @@ -976,11 +980,6 @@ def run_compile(args, config: ConfigType) -> bool: if pch_on: pch.log_pch_in_use() - # Zephyr turns ccache on by itself when it is installed - env.update(pch.ccache_pch_env()) - # Depend mode, or a Kconfig flip reuses a stale .gch: autoconf.h is - # all #defines, which vanish from the preprocessed creation hash. - env.setdefault("CCACHE_DEPEND", "1") _write_pch_checksum(build_dir, source_dir) west_cmd = _west_build_command( diff --git a/esphome/components/nrf52/clang_tidy.py b/esphome/components/nrf52/clang_tidy.py index 89272cd406..d1f9597976 100644 --- a/esphome/components/nrf52/clang_tidy.py +++ b/esphome/components/nrf52/clang_tidy.py @@ -277,7 +277,7 @@ def generate_compile_commands(work_dir: Path, platformio_ini: Path) -> Path: ] if not run_command_ok( west_cmd, - env=get_build_env(), + env=get_build_env(None), # configure only, nothing compiles stream_output=True, cwd=str(paths["framework_path"]), ): diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 58c87619bf..e6ccde6a2d 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -10,6 +10,7 @@ import platform import shutil import sys +from esphome.build_helpers.ccache import ccache_env from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path from esphome.components.zephyr.const import KEY_SYSBUILD, KEY_ZEPHYR import esphome.config_validation as cv @@ -194,7 +195,14 @@ def get_build_paths() -> dict: } -def get_build_env() -> dict: +def get_build_env(ccache: str | None) -> dict: + """Build the west/sdk-nrf process environment. + + ``ccache`` is the resolved binary (resolve_ccache_path), or None when + ccache is disabled or the caller never compiles; it brings the shared + managed-ccache settings and the pch sloppiness, so every caller that + may compile gets the same cache. + """ version = _get_version_str() venv_bin_dir = get_python_env_executable_path( _get_python_env_path(version), "python" @@ -211,6 +219,12 @@ def get_build_env() -> dict: # non-root builds failed to locate the SDK with it, while # ZEPHYR_SDK_INSTALL_DIR fixed the same invocation. env["ZEPHYR_SDK_INSTALL_DIR"] = str(_get_toolchain_path(_get_toolchain_version())) + if ccache is None: + # Zephyr wraps compiles with any ccache it finds; unmanaged it + # must not cache (a sysbuild image never sees USE_CCACHE=0). + env.setdefault("CCACHE_DISABLE", "1") + else: + env.update(ccache_env(ccache, SDK_NRF_TOOLS_CACHE)) return env diff --git a/tests/unit_tests/components/nrf52/test_pch.py b/tests/unit_tests/components/nrf52/test_pch.py index 04932e76cc..41635c3580 100644 --- a/tests/unit_tests/components/nrf52/test_pch.py +++ b/tests/unit_tests/components/nrf52/test_pch.py @@ -6,6 +6,7 @@ from unittest.mock import Mock, patch import pytest from esphome.components import nrf52 +from esphome.components.nrf52 import framework from esphome.components.zephyr.const import KEY_BOARD import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain @@ -105,6 +106,23 @@ def test_pch_checksum_waits_for_the_first_configure(tmp_path: Path) -> None: assert not build_dir.exists() +def _fake_build_env(ccache: str | None) -> dict[str, str]: + """The real get_build_env with only the install path lookups stubbed.""" + with ( + patch.object(framework, "_get_version_str", return_value="v1"), + patch.object(framework, "_get_python_env_path", return_value=Path("/penv")), + patch.object( + framework, + "get_python_env_executable_path", + return_value=Path("/penv/bin/python"), + ), + patch.object(framework, "_get_framework_path", return_value=Path("/fw")), + patch.object(framework, "_get_toolchain_version", return_value="t1"), + patch.object(framework, "_get_toolchain_path", return_value=Path("/tc")), + ): + return framework.get_build_env(ccache) + + @pytest.fixture def run_cmd(tmp_path: Path) -> Mock: CORE.config_path = tmp_path / "test.yaml" @@ -113,6 +131,7 @@ def run_cmd(tmp_path: Path) -> Mock: CORE.toolchain = Toolchain.SDK_NRF CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 2, 0)} with ( + patch.dict("os.environ", {}, clear=True), patch.object(nrf52, "check_and_install"), patch.object(nrf52, "_generate_cmake_lists", return_value=False), patch.object( @@ -120,18 +139,18 @@ def run_cmd(tmp_path: Path) -> Mock: "get_build_paths", return_value={"python_executable": "python3", "framework_path": tmp_path}, ), - patch.object(nrf52, "get_build_env", return_value={}), + patch.object(nrf52, "get_build_env", side_effect=_fake_build_env), + patch.object(nrf52, "resolve_ccache_path", return_value="/usr/bin/ccache"), patch.object(nrf52, "zephyr_data", return_value={KEY_BOARD: "board"}), patch.object(nrf52, "run_command_ok", return_value=False) as run, ): yield run -def test_ccache_pch_settings_reach_west(run_cmd: Mock) -> None: - # clear=True also drops ambient CCACHE_* overrides; the header is on - # explicitly since Windows hosts start with it off +def test_shared_ccache_settings_reach_west(run_cmd: Mock, tmp_path: Path) -> None: + # The header is on explicitly since Windows hosts start with it off with ( - patch.dict("os.environ", {"ESPHOME_PCH_ENABLE": "1"}, clear=True), + patch.dict("os.environ", {"ESPHOME_PCH_ENABLE": "1"}), pytest.raises(EsphomeError, match="nRF52 native build failed"), ): nrf52.run_compile(None, {}) @@ -140,23 +159,43 @@ def test_ccache_pch_settings_reach_west(run_cmd: Mock) -> None: assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros" # Without depend mode a Kconfig flip reuses a stale .gch assert env["CCACHE_DEPEND"] == "1" + # The full managed set, not a bespoke subset + assert env["CCACHE_DIR"].endswith("ccache") + assert env["CCACHE_NOHASHDIR"] == "true" + assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve()) + assert "CCACHE_DISABLE" not in env -def test_ccache_depend_respects_a_user_override(run_cmd: Mock) -> None: +def test_user_exported_ccache_values_win(run_cmd: Mock) -> None: + user = {"ESPHOME_PCH_ENABLE": "1", "CCACHE_DEPEND": "0", "CCACHE_DIR": "/mine"} with ( - patch.object(nrf52, "get_build_env", return_value={"CCACHE_DEPEND": "0"}), - patch.dict("os.environ", {"ESPHOME_PCH_ENABLE": "1"}, clear=True), + patch.dict("os.environ", user), pytest.raises(EsphomeError, match="nRF52 native build failed"), ): nrf52.run_compile(None, {}) - assert run_cmd.call_args.kwargs["env"]["CCACHE_DEPEND"] == "0" + env = run_cmd.call_args.kwargs["env"] + assert env["CCACHE_DEPEND"] == "0" + assert env["CCACHE_DIR"] == "/mine" -def test_disabled_leaves_the_west_environment_alone( - run_cmd: Mock, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") - with pytest.raises(EsphomeError, match="nRF52 native build failed"): +def test_no_ccache_disables_the_zephyr_launcher(run_cmd: Mock) -> None: + """ESPHOME_CCACHE_ENABLE=0 must also stop Zephyr's self-enabled ccache.""" + with ( + patch.object(nrf52, "resolve_ccache_path", return_value=None), + pytest.raises(EsphomeError, match="nRF52 native build failed"), + ): nrf52.run_compile(None, {}) - assert "CCACHE_PCH_EXTSUM" not in run_cmd.call_args.kwargs["env"] - assert "CCACHE_DEPEND" not in run_cmd.call_args.kwargs["env"] + env = run_cmd.call_args.kwargs["env"] + assert env["CCACHE_DISABLE"] == "1" + assert "CCACHE_DEPEND" not in env + + +def test_disabled_pch_still_gets_the_shared_settings(run_cmd: Mock) -> None: + with ( + patch.dict("os.environ", {"ESPHOME_PCH_ENABLE": "0"}), + pytest.raises(EsphomeError, match="nRF52 native build failed"), + ): + nrf52.run_compile(None, {}) + env = run_cmd.call_args.kwargs["env"] + assert "CCACHE_PCH_EXTSUM" not in env + assert env["CCACHE_DEPEND"] == "1" diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index 17d583d139..d9cf2d37b3 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -1078,8 +1078,9 @@ def test_get_build_env( containerized non-root builds and was removed. """ monkeypatch.setenv("SOME_PREEXISTING_VAR", "kept") + monkeypatch.delenv("CCACHE_DISABLE", raising=False) - env = get_build_env() + env = get_build_env(None) tools = get_sdk_nrf_tools_path() venv_bin_dir = get_python_env_executable_path( @@ -1096,6 +1097,30 @@ def test_get_build_env( assert "Zephyr-sdk_DIR" not in env # The rest of the process environment is inherited assert env["SOME_PREEXISTING_VAR"] == "kept" + # No managed settings without a resolved binary; the self-enabled + # Zephyr ccache must not cache + assert "CCACHE_DIR" not in env or "CCACHE_DIR" in os.environ + assert env["CCACHE_DISABLE"] == "1" + + +def test_get_build_env_with_ccache( + nrf52_dirs: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A resolved ccache brings the shared managed settings.""" + for key in ( + "CCACHE_DIR", + "CCACHE_DEPEND", + "CCACHE_NOHASHDIR", + "CCACHE_BASEDIR", + "CCACHE_DISABLE", + ): + monkeypatch.delenv(key, raising=False) + CORE.build_path = tmp_path / "build" + env = get_build_env("/usr/bin/ccache") + assert env["CCACHE_DIR"] == str(get_sdk_nrf_tools_path() / "ccache") + assert env["CCACHE_DEPEND"] == "1" + assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve()) + assert "CCACHE_DISABLE" not in env def test_get_build_env_sdk_3_4_0_uses_toolchain_root( @@ -1104,7 +1129,7 @@ def test_get_build_env_sdk_3_4_0_uses_toolchain_root( """For NCS >= 3.4.0, ZEPHYR_SDK_INSTALL_DIR still points at the toolchain root.""" CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} - env = get_build_env() + env = get_build_env(None) tools = get_sdk_nrf_tools_path() assert env["ZEPHYR_SDK_INSTALL_DIR"] == str(tools / "toolchains" / "1.0.1") From e3fcf173c2f47c9eceb1ce15ad373cffaf6af18d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:28:44 -0500 Subject: [PATCH 085/221] [bmp085][dps310][lps22][mpl3115a2][ms5611] Use the sub_* helpers for optional child entities (#20040) --- esphome/components/bmp085/sensor.py | 10 +++------- esphome/components/dps310/sensor.py | 10 +++------- esphome/components/lps22/sensor.py | 10 +++------- esphome/components/mpl3115a2/sensor.py | 14 ++++---------- esphome/components/ms5611/sensor.py | 10 +++------- 5 files changed, 16 insertions(+), 38 deletions(-) diff --git a/esphome/components/bmp085/sensor.py b/esphome/components/bmp085/sensor.py index e4e559844e..5eb8dae731 100644 --- a/esphome/components/bmp085/sensor.py +++ b/esphome/components/bmp085/sensor.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature(sens)) - - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_PRESSURE, var.set_pressure) diff --git a/esphome/components/dps310/sensor.py b/esphome/components/dps310/sensor.py index 8b8fd8373b..e3827b484c 100644 --- a/esphome/components/dps310/sensor.py +++ b/esphome/components/dps310/sensor.py @@ -54,10 +54,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature_sensor(sens)) - - if pressure := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure) - cg.add(var.set_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_PRESSURE, var.set_pressure_sensor) diff --git a/esphome/components/lps22/sensor.py b/esphome/components/lps22/sensor.py index 2eec2c586c..f219639eee 100644 --- a/esphome/components/lps22/sensor.py +++ b/esphome/components/lps22/sensor.py @@ -50,10 +50,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_PRESSURE, var.set_pressure_sensor) diff --git a/esphome/components/mpl3115a2/sensor.py b/esphome/components/mpl3115a2/sensor.py index 4fbc353644..8235e2d618 100644 --- a/esphome/components/mpl3115a2/sensor.py +++ b/esphome/components/mpl3115a2/sensor.py @@ -64,13 +64,7 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_PRESSURE in config: - sens = await sensor.new_sensor(config[CONF_PRESSURE]) - cg.add(var.set_pressure(sens)) - elif CONF_ALTITUDE in config: - sens = await sensor.new_sensor(config[CONF_ALTITUDE]) - cg.add(var.set_altitude(sens)) - - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_PRESSURE, var.set_pressure) + await sensors(CONF_ALTITUDE, var.set_altitude) + await sensors(CONF_TEMPERATURE, var.set_temperature) diff --git a/esphome/components/ms5611/sensor.py b/esphome/components/ms5611/sensor.py index e42e0824dd..bf6d4528e6 100644 --- a/esphome/components/ms5611/sensor.py +++ b/esphome/components/ms5611/sensor.py @@ -50,10 +50,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature_sensor(sens)) - - if pressure := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure) - cg.add(var.set_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_PRESSURE, var.set_pressure_sensor) From 7169c0c9303983e71561a31d084584e9b495716e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:28:59 -0500 Subject: [PATCH 086/221] [ags10][cm1106][sds011][pm1006][senseair] Use the sub_* helpers for optional child entities (#20041) --- esphome/components/ags10/sensor.py | 14 ++++---------- esphome/components/cm1106/sensor.py | 5 ++--- esphome/components/pm1006/sensor.py | 5 ++--- esphome/components/sds011/sensor.py | 10 +++------- esphome/components/senseair/sensor.py | 5 ++--- 5 files changed, 13 insertions(+), 26 deletions(-) diff --git a/esphome/components/ags10/sensor.py b/esphome/components/ags10/sensor.py index fbd4bf590e..d20d80b2fc 100644 --- a/esphome/components/ags10/sensor.py +++ b/esphome/components/ags10/sensor.py @@ -62,16 +62,10 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - sens = await sensor.new_sensor(config[CONF_TVOC]) - cg.add(var.set_tvoc(sens)) - - if version_config := config.get(CONF_VERSION): - sens = await sensor.new_sensor(version_config) - cg.add(var.set_version(sens)) - - if resistance_config := config.get(CONF_RESISTANCE): - sens = await sensor.new_sensor(resistance_config) - cg.add(var.set_resistance(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TVOC, var.set_tvoc) + await sensors(CONF_VERSION, var.set_version) + await sensors(CONF_RESISTANCE, var.set_resistance) AGS10_NEW_I2C_ADDRESS_SCHEMA = cv.maybe_simple_value( diff --git a/esphome/components/cm1106/sensor.py b/esphome/components/cm1106/sensor.py index bf12bc38d9..c3939735a4 100644 --- a/esphome/components/cm1106/sensor.py +++ b/esphome/components/cm1106/sensor.py @@ -54,9 +54,8 @@ async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) await uart.register_uart_device(var, config) - if co2_config := config.get(CONF_CO2): - sens = await sensor.new_sensor(co2_config) - cg.add(var.set_co2_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) CALIBRATION_ACTION_SCHEMA = maybe_simple_id( diff --git a/esphome/components/pm1006/sensor.py b/esphome/components/pm1006/sensor.py index 447671ebb3..98c6406676 100644 --- a/esphome/components/pm1006/sensor.py +++ b/esphome/components/pm1006/sensor.py @@ -62,6 +62,5 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await uart.register_uart_device(var, config) - if CONF_PM_2_5 in config: - sens = await sensor.new_sensor(config[CONF_PM_2_5]) - cg.add(var.set_pm_2_5_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_PM_2_5, var.set_pm_2_5_sensor) diff --git a/esphome/components/sds011/sensor.py b/esphome/components/sds011/sensor.py index e05d3b02e6..fe1481df9a 100644 --- a/esphome/components/sds011/sensor.py +++ b/esphome/components/sds011/sensor.py @@ -93,10 +93,6 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_update_interval_min(update_interval)) cg.add(var.set_rx_mode_only(config[CONF_RX_ONLY])) - if CONF_PM_2_5 in config: - sens = await sensor.new_sensor(config[CONF_PM_2_5]) - cg.add(var.set_pm_2_5_sensor(sens)) - - if CONF_PM_10_0 in config: - sens = await sensor.new_sensor(config[CONF_PM_10_0]) - cg.add(var.set_pm_10_0_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_PM_2_5, var.set_pm_2_5_sensor) + await sensors(CONF_PM_10_0, var.set_pm_10_0_sensor) diff --git a/esphome/components/senseair/sensor.py b/esphome/components/senseair/sensor.py index 9445e5867c..2494e9a6d1 100644 --- a/esphome/components/senseair/sensor.py +++ b/esphome/components/senseair/sensor.py @@ -53,9 +53,8 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await uart.register_uart_device(var, config) - if co2 := config.get(CONF_CO2): - sens = await sensor.new_sensor(co2) - cg.add(var.set_co2_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) CALIBRATION_ACTION_SCHEMA = maybe_simple_id( From ba9d54b8ad2a20b601d1ee300b4c4d40cc3f2a5c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:29:14 -0500 Subject: [PATCH 087/221] [bmi160][mpu6050][mpu6886][mlx90614][max31855] Use the sub_* helpers for optional child entities (#20042) --- esphome/components/bmi160/sensor.py | 21 ++++++++------------- esphome/components/max31855/sensor.py | 5 ++--- esphome/components/mlx90614/sensor.py | 11 +++-------- esphome/components/mpu6050/sensor.py | 21 ++++++++------------- esphome/components/mpu6886/sensor.py | 21 ++++++++------------- 5 files changed, 29 insertions(+), 50 deletions(-) diff --git a/esphome/components/bmi160/sensor.py b/esphome/components/bmi160/sensor.py index 4309f0a79f..9ad1b19fa0 100644 --- a/esphome/components/bmi160/sensor.py +++ b/esphome/components/bmi160/sensor.py @@ -88,16 +88,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - for d in ["x", "y", "z"]: - accel_key = f"acceleration_{d}" - if accel_key in config: - sens = await sensor.new_sensor(config[accel_key]) - cg.add(getattr(var, f"set_accel_{d}_sensor")(sens)) - accel_key = f"gyroscope_{d}" - if accel_key in config: - sens = await sensor.new_sensor(config[accel_key]) - cg.add(getattr(var, f"set_gyro_{d}_sensor")(sens)) - - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_ACCELERATION_X, var.set_accel_x_sensor) + await sensors(CONF_GYROSCOPE_X, var.set_gyro_x_sensor) + await sensors(CONF_ACCELERATION_Y, var.set_accel_y_sensor) + await sensors(CONF_GYROSCOPE_Y, var.set_gyro_y_sensor) + await sensors(CONF_ACCELERATION_Z, var.set_accel_z_sensor) + await sensors(CONF_GYROSCOPE_Z, var.set_gyro_z_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) diff --git a/esphome/components/max31855/sensor.py b/esphome/components/max31855/sensor.py index a52f45a18f..3d2a83f8cd 100644 --- a/esphome/components/max31855/sensor.py +++ b/esphome/components/max31855/sensor.py @@ -41,6 +41,5 @@ async def to_code(config: ConfigType) -> None: var = await sensor.new_sensor(config) await cg.register_component(var, config) await spi.register_spi_device(var, config) - if CONF_REFERENCE_TEMPERATURE in config: - tc_ref = await sensor.new_sensor(config[CONF_REFERENCE_TEMPERATURE]) - cg.add(var.set_reference_sensor(tc_ref)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_REFERENCE_TEMPERATURE, var.set_reference_sensor) diff --git a/esphome/components/mlx90614/sensor.py b/esphome/components/mlx90614/sensor.py index 0cf9b95dde..e818ea708d 100644 --- a/esphome/components/mlx90614/sensor.py +++ b/esphome/components/mlx90614/sensor.py @@ -53,12 +53,7 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_AMBIENT in config: - sens = await sensor.new_sensor(config[CONF_AMBIENT]) - cg.add(var.set_ambient_sensor(sens)) - - if CONF_OBJECT in config: - sens = await sensor.new_sensor(config[CONF_OBJECT]) - cg.add(var.set_object_sensor(sens)) - + sensors = sensor.sub_sensors(config) + await sensors(CONF_AMBIENT, var.set_ambient_sensor) + if await sensors(CONF_OBJECT, var.set_object_sensor): cg.add(var.set_emissivity(config[CONF_OBJECT][CONF_EMISSIVITY])) diff --git a/esphome/components/mpu6050/sensor.py b/esphome/components/mpu6050/sensor.py index a8370ced7f..bd3de2d32c 100644 --- a/esphome/components/mpu6050/sensor.py +++ b/esphome/components/mpu6050/sensor.py @@ -70,16 +70,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - for d in ["x", "y", "z"]: - accel_key = f"accel_{d}" - if accel_key in config: - sens = await sensor.new_sensor(config[accel_key]) - cg.add(getattr(var, f"set_accel_{d}_sensor")(sens)) - accel_key = f"gyro_{d}" - if accel_key in config: - sens = await sensor.new_sensor(config[accel_key]) - cg.add(getattr(var, f"set_gyro_{d}_sensor")(sens)) - - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_ACCEL_X, var.set_accel_x_sensor) + await sensors(CONF_GYRO_X, var.set_gyro_x_sensor) + await sensors(CONF_ACCEL_Y, var.set_accel_y_sensor) + await sensors(CONF_GYRO_Y, var.set_gyro_y_sensor) + await sensors(CONF_ACCEL_Z, var.set_accel_z_sensor) + await sensors(CONF_GYRO_Z, var.set_gyro_z_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) diff --git a/esphome/components/mpu6886/sensor.py b/esphome/components/mpu6886/sensor.py index 5bdb836128..878540d516 100644 --- a/esphome/components/mpu6886/sensor.py +++ b/esphome/components/mpu6886/sensor.py @@ -71,16 +71,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - for d in ["x", "y", "z"]: - accel_key = f"accel_{d}" - if accel_key in config: - sens = await sensor.new_sensor(config[accel_key]) - cg.add(getattr(var, f"set_accel_{d}_sensor")(sens)) - accel_key = f"gyro_{d}" - if accel_key in config: - sens = await sensor.new_sensor(config[accel_key]) - cg.add(getattr(var, f"set_gyro_{d}_sensor")(sens)) - - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_ACCEL_X, var.set_accel_x_sensor) + await sensors(CONF_GYRO_X, var.set_gyro_x_sensor) + await sensors(CONF_ACCEL_Y, var.set_accel_y_sensor) + await sensors(CONF_GYRO_Y, var.set_gyro_y_sensor) + await sensors(CONF_ACCEL_Z, var.set_accel_z_sensor) + await sensors(CONF_GYRO_Z, var.set_gyro_z_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) From 46b12b43b2fc44977235437b4f2d1113ff377f6d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:29:27 -0500 Subject: [PATCH 088/221] [ens210][htu31d][hyt271][honeywell_hih_i2c][ms8607] Use the sub_* helpers for optional child entities (#20043) --- esphome/components/ens210/sensor.py | 10 +++------- esphome/components/honeywell_hih_i2c/sensor.py | 10 +++------- esphome/components/htu31d/sensor.py | 10 +++------- esphome/components/hyt271/sensor.py | 10 +++------- esphome/components/ms8607/sensor.py | 15 +++++---------- 5 files changed, 17 insertions(+), 38 deletions(-) diff --git a/esphome/components/ens210/sensor.py b/esphome/components/ens210/sensor.py index bfd758f92f..cb49986f7c 100644 --- a/esphome/components/ens210/sensor.py +++ b/esphome/components/ens210/sensor.py @@ -50,10 +50,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) diff --git a/esphome/components/honeywell_hih_i2c/sensor.py b/esphome/components/honeywell_hih_i2c/sensor.py index 5250e1c1c7..8ab2cd16b8 100644 --- a/esphome/components/honeywell_hih_i2c/sensor.py +++ b/esphome/components/honeywell_hih_i2c/sensor.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) diff --git a/esphome/components/htu31d/sensor.py b/esphome/components/htu31d/sensor.py index 8960759d9b..41cc54686a 100644 --- a/esphome/components/htu31d/sensor.py +++ b/esphome/components/htu31d/sensor.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) diff --git a/esphome/components/hyt271/sensor.py b/esphome/components/hyt271/sensor.py index 3f006a65fe..1fe5d9e14a 100644 --- a/esphome/components/hyt271/sensor.py +++ b/esphome/components/hyt271/sensor.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature) - cg.add(var.set_temperature(sens)) - - if humidity := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity) - cg.add(var.set_humidity(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) diff --git a/esphome/components/ms8607/sensor.py b/esphome/components/ms8607/sensor.py index e5d90432d8..13377394be 100644 --- a/esphome/components/ms8607/sensor.py +++ b/esphome/components/ms8607/sensor.py @@ -68,17 +68,12 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_PRESSURE, var.set_pressure_sensor) - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) + if await sensors(CONF_HUMIDITY, var.set_humidity_sensor): + humidity_config = config[CONF_HUMIDITY] humidity_device = cg.new_Pvariable(humidity_config[CONF_HUMIDITY_I2C_ID]) await i2c.register_i2c_device(humidity_device, humidity_config) cg.add(var.set_humidity_device(humidity_device)) From 690227f6d89d56c3dfbe613c429f58af3dc500d4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:29:49 -0500 Subject: [PATCH 089/221] [honeywellabp][honeywellabp2_i2c][npi19][xgzp68xx][xdb401] Use the sub_* helpers for optional child entities (#20044) --- esphome/components/honeywellabp/sensor.py | 11 +++-------- esphome/components/honeywellabp2_i2c/sensor.py | 10 ++++------ esphome/components/npi19/sensor.py | 10 +++------- esphome/components/xdb401/sensor.py | 10 +++------- esphome/components/xgzp68xx/sensor.py | 12 +++++------- 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/esphome/components/honeywellabp/sensor.py b/esphome/components/honeywellabp/sensor.py index 4b116f0f16..d7e7cf29a4 100644 --- a/esphome/components/honeywellabp/sensor.py +++ b/esphome/components/honeywellabp/sensor.py @@ -56,14 +56,9 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await spi.register_spi_device(var, config) - if CONF_PRESSURE in config: + sensors = sensor.sub_sensors(config) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): conf = config[CONF_PRESSURE] - sens = await sensor.new_sensor(conf) - cg.add(var.set_pressure_sensor(sens)) cg.add(var.set_honeywellabp_min_pressure(conf[CONF_MIN_PRESSURE])) cg.add(var.set_honeywellabp_max_pressure(conf[CONF_MAX_PRESSURE])) - - if CONF_TEMPERATURE in config: - conf = config[CONF_TEMPERATURE] - sens = await sensor.new_sensor(conf) - cg.add(var.set_temperature_sensor(sens)) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) diff --git a/esphome/components/honeywellabp2_i2c/sensor.py b/esphome/components/honeywellabp2_i2c/sensor.py index 299acd4b52..d4c4e44a3b 100644 --- a/esphome/components/honeywellabp2_i2c/sensor.py +++ b/esphome/components/honeywellabp2_i2c/sensor.py @@ -63,13 +63,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): + pressure_config = config[CONF_PRESSURE] cg.add(var.set_min_pressure(pressure_config[CONF_MIN_PRESSURE])) cg.add(var.set_max_pressure(pressure_config[CONF_MAX_PRESSURE])) cg.add(var.set_transfer_function(pressure_config[TRANSFER_FUNCTION])) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) diff --git a/esphome/components/npi19/sensor.py b/esphome/components/npi19/sensor.py index 4c7db10ce9..14dcd72384 100644 --- a/esphome/components/npi19/sensor.py +++ b/esphome/components/npi19/sensor.py @@ -44,10 +44,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if raw_pressure_config := config.get(CONF_RAW_PRESSURE): - sens = await sensor.new_sensor(raw_pressure_config) - cg.add(var.set_raw_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_RAW_PRESSURE, var.set_raw_pressure_sensor) diff --git a/esphome/components/xdb401/sensor.py b/esphome/components/xdb401/sensor.py index c629e46650..df274ac09b 100644 --- a/esphome/components/xdb401/sensor.py +++ b/esphome/components/xdb401/sensor.py @@ -56,10 +56,6 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_pressure_range_bar(config[CONF_PRESSURE_RANGE_BAR])) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_PRESSURE, var.set_pressure_sensor) diff --git a/esphome/components/xgzp68xx/sensor.py b/esphome/components/xgzp68xx/sensor.py index 83c20dfbd8..eb3b042cb7 100644 --- a/esphome/components/xgzp68xx/sensor.py +++ b/esphome/components/xgzp68xx/sensor.py @@ -70,13 +70,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) - cg.add(var.set_pressure_oversampling(pressure_config[CONF_OVERSAMPLING])) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): + oversampling = config[CONF_PRESSURE][CONF_OVERSAMPLING] + cg.add(var.set_pressure_oversampling(oversampling)) cg.add(var.set_k_value(config[CONF_K_VALUE])) From b15bc083f63192f10156ed539c67fd6974b1ecab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:30:01 -0500 Subject: [PATCH 090/221] [mcp9600][tem3200][kmeteriso][ufire_ec][ufire_ise] Use the sub_* helpers for optional child entities (#20045) --- esphome/components/kmeteriso/sensor.py | 9 +++------ esphome/components/mcp9600/sensor.py | 12 +++--------- esphome/components/tem3200/sensor.py | 10 +++------- esphome/components/ufire_ec/sensor.py | 10 +++------- esphome/components/ufire_ise/sensor.py | 10 +++------- 5 files changed, 15 insertions(+), 36 deletions(-) diff --git a/esphome/components/kmeteriso/sensor.py b/esphome/components/kmeteriso/sensor.py index 3e007d1310..5b7087362f 100644 --- a/esphome/components/kmeteriso/sensor.py +++ b/esphome/components/kmeteriso/sensor.py @@ -48,9 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - if internal_temperature_config := config.get(CONF_INTERNAL_TEMPERATURE): - sens = await sensor.new_sensor(internal_temperature_config) - cg.add(var.set_internal_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_INTERNAL_TEMPERATURE, var.set_internal_temperature_sensor) diff --git a/esphome/components/mcp9600/sensor.py b/esphome/components/mcp9600/sensor.py index 5542ffaa6c..75f2e025c7 100644 --- a/esphome/components/mcp9600/sensor.py +++ b/esphome/components/mcp9600/sensor.py @@ -71,12 +71,6 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_thermocouple_type(config[CONF_THERMOCOUPLE_TYPE])) - if CONF_HOT_JUNCTION in config: - conf = config[CONF_HOT_JUNCTION] - sens = await sensor.new_sensor(conf) - cg.add(var.set_hot_junction(sens)) - - if CONF_COLD_JUNCTION in config: - conf = config[CONF_COLD_JUNCTION] - sens = await sensor.new_sensor(conf) - cg.add(var.set_cold_junction(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_HOT_JUNCTION, var.set_hot_junction) + await sensors(CONF_COLD_JUNCTION, var.set_cold_junction) diff --git a/esphome/components/tem3200/sensor.py b/esphome/components/tem3200/sensor.py index 289641f951..2b1839c529 100644 --- a/esphome/components/tem3200/sensor.py +++ b/esphome/components/tem3200/sensor.py @@ -46,10 +46,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - - if raw_pressure_config := config.get(CONF_RAW_PRESSURE): - sens = await sensor.new_sensor(raw_pressure_config) - cg.add(var.set_raw_pressure_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_RAW_PRESSURE, var.set_raw_pressure_sensor) diff --git a/esphome/components/ufire_ec/sensor.py b/esphome/components/ufire_ec/sensor.py index 5bfa959c04..6a632e05cd 100644 --- a/esphome/components/ufire_ec/sensor.py +++ b/esphome/components/ufire_ec/sensor.py @@ -64,13 +64,9 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_temperature_compensation(config[CONF_TEMPERATURE_COMPENSATION])) cg.add(var.set_temperature_coefficient(config[CONF_TEMPERATURE_COEFFICIENT])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - - if CONF_EC in config: - sens = await sensor.new_sensor(config[CONF_EC]) - cg.add(var.set_ec_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_EC, var.set_ec_sensor) if CONF_TEMPERATURE_SENSOR in config: sens = await cg.get_variable(config[CONF_TEMPERATURE_SENSOR]) diff --git a/esphome/components/ufire_ise/sensor.py b/esphome/components/ufire_ise/sensor.py index 5399692e3c..6f205af784 100644 --- a/esphome/components/ufire_ise/sensor.py +++ b/esphome/components/ufire_ise/sensor.py @@ -56,13 +56,9 @@ async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - - if CONF_PH in config: - sens = await sensor.new_sensor(config[CONF_PH]) - cg.add(var.set_ph_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_PH, var.set_ph_sensor) if CONF_TEMPERATURE_SENSOR in config: sens = await cg.get_variable(config[CONF_TEMPERATURE_SENSOR]) From b0d5681022226eb5aa10f54c5d11f1e4ad420a8f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:30:17 -0500 Subject: [PATCH 091/221] [lc709203f][max17043][pulse_counter][pulse_meter][duty_time] Use the sub_* helpers for optional child entities (#20046) --- esphome/components/duty_time/sensor.py | 5 ++--- esphome/components/lc709203f/sensor.py | 17 ++++++----------- esphome/components/max17043/sensor.py | 10 +++------- esphome/components/pulse_counter/sensor.py | 5 ++--- esphome/components/pulse_meter/sensor.py | 5 ++--- 5 files changed, 15 insertions(+), 27 deletions(-) diff --git a/esphome/components/duty_time/sensor.py b/esphome/components/duty_time/sensor.py index 7fd01db836..6b035d5c98 100644 --- a/esphome/components/duty_time/sensor.py +++ b/esphome/components/duty_time/sensor.py @@ -63,9 +63,8 @@ async def to_code(config: ConfigType) -> None: if CONF_LAMBDA in config: lambda_ = await cg.process_lambda(config[CONF_LAMBDA], [], return_type=cg.bool_) cg.add(var.set_lambda(lambda_)) - if CONF_LAST_TIME in config: - sens = await sensor.new_sensor(config[CONF_LAST_TIME]) - cg.add(var.set_last_duty_time_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_LAST_TIME, var.set_last_duty_time_sensor) # AUTOMATIONS diff --git a/esphome/components/lc709203f/sensor.py b/esphome/components/lc709203f/sensor.py index 3319c9be4b..0281609edf 100644 --- a/esphome/components/lc709203f/sensor.py +++ b/esphome/components/lc709203f/sensor.py @@ -79,15 +79,10 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_pack_size(config.get(CONF_SIZE))) cg.add(var.set_pack_voltage(BATTERY_VOLTAGE_OPTIONS[config[CONF_VOLTAGE]])) - if voltage_config := config.get(CONF_BATTERY_VOLTAGE): - sens = await sensor.new_sensor(voltage_config) - cg.add(var.set_voltage_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_BATTERY_VOLTAGE, var.set_voltage_sensor) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_remaining_sensor) - if level_config := config.get(CONF_BATTERY_LEVEL): - sens = await sensor.new_sensor(level_config) - cg.add(var.set_battery_remaining_sensor(sens)) - - if temp_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temp_config) - cg.add(var.set_temperature_sensor(sens)) - cg.add(var.set_thermistor_b_constant(temp_config[CONF_B_CONSTANT])) + if await sensors(CONF_TEMPERATURE, var.set_temperature_sensor): + b_constant = config[CONF_TEMPERATURE][CONF_B_CONSTANT] + cg.add(var.set_thermistor_b_constant(b_constant)) diff --git a/esphome/components/max17043/sensor.py b/esphome/components/max17043/sensor.py index 0b72955c74..1b621057c6 100644 --- a/esphome/components/max17043/sensor.py +++ b/esphome/components/max17043/sensor.py @@ -53,13 +53,9 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if voltage_config := config.get(CONF_BATTERY_VOLTAGE): - sens = await sensor.new_sensor(voltage_config) - cg.add(var.set_voltage_sensor(sens)) - - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_remaining_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_BATTERY_VOLTAGE, var.set_voltage_sensor) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_remaining_sensor) MAX17043_ACTION_SCHEMA = maybe_simple_id( diff --git a/esphome/components/pulse_counter/sensor.py b/esphome/components/pulse_counter/sensor.py index 677cfb6a5e..4cf88bafe9 100644 --- a/esphome/components/pulse_counter/sensor.py +++ b/esphome/components/pulse_counter/sensor.py @@ -140,9 +140,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_falling_edge_mode(count[CONF_FALLING_EDGE])) cg.add(var.set_filter_us(config[CONF_INTERNAL_FILTER])) - if CONF_TOTAL in config: - sens = await sensor.new_sensor(config[CONF_TOTAL]) - cg.add(var.set_total_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TOTAL, var.set_total_sensor) automation.register_apply_action( diff --git a/esphome/components/pulse_meter/sensor.py b/esphome/components/pulse_meter/sensor.py index 2df1ba96e9..e523ebe756 100644 --- a/esphome/components/pulse_meter/sensor.py +++ b/esphome/components/pulse_meter/sensor.py @@ -92,9 +92,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_timeout_us(config[CONF_TIMEOUT])) cg.add(var.set_filter_mode(config[CONF_INTERNAL_FILTER_MODE])) - if CONF_TOTAL in config: - sens = await sensor.new_sensor(config[CONF_TOTAL]) - cg.add(var.set_total_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TOTAL, var.set_total_sensor) automation.register_apply_action( From b130c79267711a3be16825ae3ef79926a74c8504 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:32:00 -0500 Subject: [PATCH 092/221] [xiaomi_hhccpot002][xiaomi_miscale][xiaomi_wx08zm][radon_eye_rd200][am43] Use the sub_* helpers for optional child entities (#20047) --- esphome/components/am43/sensor/__init__.py | 10 +++------- esphome/components/radon_eye_rd200/sensor.py | 9 +++------ esphome/components/xiaomi_hhccpot002/sensor.py | 9 +++------ esphome/components/xiaomi_miscale/sensor.py | 9 +++------ esphome/components/xiaomi_wx08zm/binary_sensor.py | 9 +++------ 5 files changed, 15 insertions(+), 31 deletions(-) diff --git a/esphome/components/am43/sensor/__init__.py b/esphome/components/am43/sensor/__init__.py index 80341972a9..ef746f3e52 100644 --- a/esphome/components/am43/sensor/__init__.py +++ b/esphome/components/am43/sensor/__init__.py @@ -48,10 +48,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await ble_client.register_ble_node(var, config) - if battery_level_config := config.get(CONF_BATTERY_LEVEL): - sens = await sensor.new_sensor(battery_level_config) - cg.add(var.set_battery(sens)) - - if illuminance_config := config.get(CONF_ILLUMINANCE): - sens = await sensor.new_sensor(illuminance_config) - cg.add(var.set_illuminance(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_BATTERY_LEVEL, var.set_battery) + await sensors(CONF_ILLUMINANCE, var.set_illuminance) diff --git a/esphome/components/radon_eye_rd200/sensor.py b/esphome/components/radon_eye_rd200/sensor.py index f38a035226..0e691e382d 100644 --- a/esphome/components/radon_eye_rd200/sensor.py +++ b/esphome/components/radon_eye_rd200/sensor.py @@ -47,9 +47,6 @@ async def to_code(config: ConfigType) -> None: await ble_client.register_ble_node(var, config) - if CONF_RADON in config: - sens = await sensor.new_sensor(config[CONF_RADON]) - cg.add(var.set_radon(sens)) - if CONF_RADON_LONG_TERM in config: - sens = await sensor.new_sensor(config[CONF_RADON_LONG_TERM]) - cg.add(var.set_radon_long_term(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_RADON, var.set_radon) + await sensors(CONF_RADON_LONG_TERM, var.set_radon_long_term) diff --git a/esphome/components/xiaomi_hhccpot002/sensor.py b/esphome/components/xiaomi_hhccpot002/sensor.py index 061753c44c..73d5f3ff19 100644 --- a/esphome/components/xiaomi_hhccpot002/sensor.py +++ b/esphome/components/xiaomi_hhccpot002/sensor.py @@ -53,9 +53,6 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_MOISTURE in config: - sens = await sensor.new_sensor(config[CONF_MOISTURE]) - cg.add(var.set_moisture(sens)) - if CONF_CONDUCTIVITY in config: - sens = await sensor.new_sensor(config[CONF_CONDUCTIVITY]) - cg.add(var.set_conductivity(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_MOISTURE, var.set_moisture) + await sensors(CONF_CONDUCTIVITY, var.set_conductivity) diff --git a/esphome/components/xiaomi_miscale/sensor.py b/esphome/components/xiaomi_miscale/sensor.py index fa4296d7a9..c6277a515d 100644 --- a/esphome/components/xiaomi_miscale/sensor.py +++ b/esphome/components/xiaomi_miscale/sensor.py @@ -58,9 +58,6 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_clear_impedance(config[CONF_CLEAR_IMPEDANCE])) - if CONF_WEIGHT in config: - sens = await sensor.new_sensor(config[CONF_WEIGHT]) - cg.add(var.set_weight(sens)) - if CONF_IMPEDANCE in config: - sens = await sensor.new_sensor(config[CONF_IMPEDANCE]) - cg.add(var.set_impedance(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_WEIGHT, var.set_weight) + await sensors(CONF_IMPEDANCE, var.set_impedance) diff --git a/esphome/components/xiaomi_wx08zm/binary_sensor.py b/esphome/components/xiaomi_wx08zm/binary_sensor.py index 4a6267f8b9..8e9b32b797 100644 --- a/esphome/components/xiaomi_wx08zm/binary_sensor.py +++ b/esphome/components/xiaomi_wx08zm/binary_sensor.py @@ -56,9 +56,6 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TABLET in config: - sens = await sensor.new_sensor(config[CONF_TABLET]) - cg.add(var.set_tablet(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TABLET, var.set_tablet) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 8a49ba7e9851e2a0f45e59b0297c96931c06679a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:32:09 -0500 Subject: [PATCH 093/221] [iaqcore][t6615][hc8][as3935][tx20] Use the sub_* helpers for optional child entities (#20048) --- esphome/components/as3935/sensor.py | 10 +++------- esphome/components/hc8/sensor.py | 5 ++--- esphome/components/iaqcore/sensor.py | 10 +++------- esphome/components/t6615/sensor.py | 5 ++--- esphome/components/tx20/sensor.py | 12 +++--------- 5 files changed, 13 insertions(+), 29 deletions(-) diff --git a/esphome/components/as3935/sensor.py b/esphome/components/as3935/sensor.py index b727b8fdb9..f03213c8ca 100644 --- a/esphome/components/as3935/sensor.py +++ b/esphome/components/as3935/sensor.py @@ -35,10 +35,6 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_AS3935_ID]) - if distance_config := config.get(CONF_DISTANCE): - sens = await sensor.new_sensor(distance_config) - cg.add(hub.set_distance_sensor(sens)) - - if lightning_energy_config := config.get(CONF_LIGHTNING_ENERGY): - sens = await sensor.new_sensor(lightning_energy_config) - cg.add(hub.set_energy_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_DISTANCE, hub.set_distance_sensor) + await sensors(CONF_LIGHTNING_ENERGY, hub.set_energy_sensor) diff --git a/esphome/components/hc8/sensor.py b/esphome/components/hc8/sensor.py index 05d8e233ef..10074fbf10 100644 --- a/esphome/components/hc8/sensor.py +++ b/esphome/components/hc8/sensor.py @@ -55,9 +55,8 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await uart.register_uart_device(var, config) - if co2 := config.get(CONF_CO2): - sens = await sensor.new_sensor(co2) - cg.add(var.set_co2_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) cg.add(var.set_warmup_seconds(config[CONF_WARMUP_TIME])) diff --git a/esphome/components/iaqcore/sensor.py b/esphome/components/iaqcore/sensor.py index 1b905e4c63..daee46f083 100644 --- a/esphome/components/iaqcore/sensor.py +++ b/esphome/components/iaqcore/sensor.py @@ -47,12 +47,8 @@ async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) - if co2_config := config.get(CONF_CO2): - sens = await sensor.new_sensor(co2_config) - cg.add(var.set_co2(sens)) - - if tvoc_config := config.get(CONF_TVOC): - sens = await sensor.new_sensor(tvoc_config) - cg.add(var.set_tvoc(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2) + await sensors(CONF_TVOC, var.set_tvoc) await i2c.register_i2c_device(var, config) diff --git a/esphome/components/t6615/sensor.py b/esphome/components/t6615/sensor.py index 44dba52ae8..4b055516e7 100644 --- a/esphome/components/t6615/sensor.py +++ b/esphome/components/t6615/sensor.py @@ -48,6 +48,5 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await uart.register_uart_device(var, config) - if co2 := config.get(CONF_CO2): - sens = await sensor.new_sensor(co2) - cg.add(var.set_co2_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) diff --git a/esphome/components/tx20/sensor.py b/esphome/components/tx20/sensor.py index 3a24b8dc0b..77418e5284 100644 --- a/esphome/components/tx20/sensor.py +++ b/esphome/components/tx20/sensor.py @@ -44,15 +44,9 @@ async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) - if CONF_WIND_SPEED in config: - conf = config[CONF_WIND_SPEED] - sens = await sensor.new_sensor(conf) - cg.add(var.set_wind_speed_sensor(sens)) - - if CONF_WIND_DIRECTION_DEGREES in config: - conf = config[CONF_WIND_DIRECTION_DEGREES] - sens = await sensor.new_sensor(conf) - cg.add(var.set_wind_direction_degrees_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_WIND_SPEED, var.set_wind_speed_sensor) + await sensors(CONF_WIND_DIRECTION_DEGREES, var.set_wind_direction_degrees_sensor) pin = await cg.gpio_pin_expression(config[CONF_PIN]) cg.add(var.set_pin(pin)) From 3cc4df4b3a5ce55efe26237ee99d71fff90b8e50 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:32:26 -0500 Subject: [PATCH 094/221] [rd03d][sim800l][tas58xx][bedjet][hoermann_hcp] Use the sub_* helpers for optional child entities (#20049) --- esphome/components/bedjet/sensor/__init__.py | 10 +++------- .../components/hoermann_hcp/text_sensor/__init__.py | 13 +++++-------- esphome/components/rd03d/binary_sensor.py | 5 ++--- esphome/components/rd03d/sensor.py | 5 ++--- esphome/components/sim800l/binary_sensor.py | 7 +++---- esphome/components/sim800l/sensor.py | 7 +++---- esphome/components/tas58xx/binary_sensor.py | 9 ++++----- 7 files changed, 22 insertions(+), 34 deletions(-) diff --git a/esphome/components/bedjet/sensor/__init__.py b/esphome/components/bedjet/sensor/__init__.py index 595e798e49..9319af664f 100644 --- a/esphome/components/bedjet/sensor/__init__.py +++ b/esphome/components/bedjet/sensor/__init__.py @@ -44,10 +44,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await register_bedjet_child(var, config) - if outlet_temperature_sensor := config.get(CONF_OUTLET_TEMPERATURE): - sensor_var = await sensor.new_sensor(outlet_temperature_sensor) - cg.add(var.set_outlet_temperature_sensor(sensor_var)) - - if ambient_temperature_sensor := config.get(CONF_AMBIENT_TEMPERATURE): - sensor_var = await sensor.new_sensor(ambient_temperature_sensor) - cg.add(var.set_ambient_temperature_sensor(sensor_var)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_OUTLET_TEMPERATURE, var.set_outlet_temperature_sensor) + await sensors(CONF_AMBIENT_TEMPERATURE, var.set_ambient_temperature_sensor) diff --git a/esphome/components/hoermann_hcp/text_sensor/__init__.py b/esphome/components/hoermann_hcp/text_sensor/__init__.py index 2f038025b9..da460905c0 100644 --- a/esphome/components/hoermann_hcp/text_sensor/__init__.py +++ b/esphome/components/hoermann_hcp/text_sensor/__init__.py @@ -35,16 +35,13 @@ CONFIG_SCHEMA = cv.All( async def to_code(config: ConfigType) -> None: - parent = await cg.get_variable(config[CONF_HOERMANN_HCP_ID]) + hub = await cg.get_variable(config[CONF_HOERMANN_HCP_ID]) if (conf := config.get(CONF_DOOR_STATE)) is not None: - var = await text_sensor.new_text_sensor(conf, parent) + var = await text_sensor.new_text_sensor(conf, hub) await cg.register_component(var, conf) # Only the identity sensors need the exchange with the motor compiled in. if CONF_SERIAL_NUMBER in config or CONF_VERSION in config: cg.add_define("USE_HOERMANN_HCP_IDENTITY") - if (conf := config.get(CONF_SERIAL_NUMBER)) is not None: - sens = await text_sensor.new_text_sensor(conf) - cg.add(parent.set_serial_number_text_sensor(sens)) - if (conf := config.get(CONF_VERSION)) is not None: - sens = await text_sensor.new_text_sensor(conf) - cg.add(parent.set_version_text_sensor(sens)) + text_sensors = text_sensor.sub_text_sensors(config) + await text_sensors(CONF_SERIAL_NUMBER, hub.set_serial_number_text_sensor) + await text_sensors(CONF_VERSION, hub.set_version_text_sensor) diff --git a/esphome/components/rd03d/binary_sensor.py b/esphome/components/rd03d/binary_sensor.py index 2c040d0560..9971c413bd 100644 --- a/esphome/components/rd03d/binary_sensor.py +++ b/esphome/components/rd03d/binary_sensor.py @@ -30,9 +30,8 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_RD03D_ID]) - if target_config := config.get(CONF_TARGET): - sens = await binary_sensor.new_binary_sensor(target_config) - cg.add(hub.set_target_binary_sensor(sens)) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_TARGET, hub.set_target_binary_sensor) for i in range(MAX_TARGETS): if target_config := config.get(f"target_{i + 1}"): diff --git a/esphome/components/rd03d/sensor.py b/esphome/components/rd03d/sensor.py index d29656bab0..fe47e5ae32 100644 --- a/esphome/components/rd03d/sensor.py +++ b/esphome/components/rd03d/sensor.py @@ -79,9 +79,8 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_RD03D_ID]) - if target_count_config := config.get(CONF_TARGET_COUNT): - sens = await sensor.new_sensor(target_count_config) - cg.add(hub.set_target_count_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TARGET_COUNT, hub.set_target_count_sensor) for i in range(MAX_TARGETS): if target_config := config.get(f"target_{i + 1}"): diff --git a/esphome/components/sim800l/binary_sensor.py b/esphome/components/sim800l/binary_sensor.py index 1cf0b4e1b8..c2bfa94fd5 100644 --- a/esphome/components/sim800l/binary_sensor.py +++ b/esphome/components/sim800l/binary_sensor.py @@ -19,8 +19,7 @@ CONFIG_SCHEMA = { async def to_code(config): - sim800l_component = await cg.get_variable(config[CONF_SIM800L_ID]) + hub = await cg.get_variable(config[CONF_SIM800L_ID]) - if CONF_REGISTERED in config: - sens = await binary_sensor.new_binary_sensor(config[CONF_REGISTERED]) - cg.add(sim800l_component.set_registered_binary_sensor(sens)) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_REGISTERED, hub.set_registered_binary_sensor) diff --git a/esphome/components/sim800l/sensor.py b/esphome/components/sim800l/sensor.py index 010e325df4..cc2ebb93a7 100644 --- a/esphome/components/sim800l/sensor.py +++ b/esphome/components/sim800l/sensor.py @@ -27,8 +27,7 @@ CONFIG_SCHEMA = { async def to_code(config): - sim800l_component = await cg.get_variable(config[CONF_SIM800L_ID]) + hub = await cg.get_variable(config[CONF_SIM800L_ID]) - if CONF_RSSI in config: - sens = await sensor.new_sensor(config[CONF_RSSI]) - cg.add(sim800l_component.set_rssi_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_RSSI, hub.set_rssi_sensor) diff --git a/esphome/components/tas58xx/binary_sensor.py b/esphome/components/tas58xx/binary_sensor.py index 8a5bb03dd7..398cfa2440 100644 --- a/esphome/components/tas58xx/binary_sensor.py +++ b/esphome/components/tas58xx/binary_sensor.py @@ -40,12 +40,11 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: - parent = await cg.get_variable(config[CONF_TAS58XX_ID]) - if sensor_config := config.get(CONF_HAVE_FAULT): - sens = await binary_sensor.new_binary_sensor(sensor_config) - cg.add(parent.set_have_fault_binary_sensor(sens)) + hub = await cg.get_variable(config[CONF_TAS58XX_ID]) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_HAVE_FAULT, hub.set_have_fault_binary_sensor) for key in FAULT_SENSORS: if sensor_config := config.get(key): sens = await binary_sensor.new_binary_sensor(sensor_config) fault = getattr(FaultSensor, f"FAULT_SENSOR_{key.upper()}") - cg.add(parent.set_fault_binary_sensor(fault, sens)) + cg.add(hub.set_fault_binary_sensor(fault, sens)) From 483b9ebe2ce7f925a2fd3cdbe481fa5ed8caeff3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:32:45 -0500 Subject: [PATCH 095/221] [combination][libretiny][sound_level] Use the sub_* helpers for optional child entities (#20050) --- esphome/components/combination/sensor.py | 5 ++--- esphome/components/libretiny/text_sensor.py | 7 +++---- esphome/components/sound_level/sensor.py | 9 +++------ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/esphome/components/combination/sensor.py b/esphome/components/combination/sensor.py index ccc5a03964..34e9b99ce8 100644 --- a/esphome/components/combination/sensor.py +++ b/esphome/components/combination/sensor.py @@ -203,6 +203,5 @@ async def to_code(config: ConfigType) -> None: else: cg.add(var.add_source(source)) - if CONF_STD_DEV in config: - sens = await sensor.new_sensor(config[CONF_STD_DEV]) - cg.add(var.set_std_dev_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_STD_DEV, var.set_std_dev_sensor) diff --git a/esphome/components/libretiny/text_sensor.py b/esphome/components/libretiny/text_sensor.py index c1012774c8..18b412c460 100644 --- a/esphome/components/libretiny/text_sensor.py +++ b/esphome/components/libretiny/text_sensor.py @@ -25,8 +25,7 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config): - lt_component = await cg.get_variable(config[CONF_LIBRETINY]) + hub = await cg.get_variable(config[CONF_LIBRETINY]) - if CONF_VERSION in config: - sens = await text_sensor.new_text_sensor(config[CONF_VERSION]) - cg.add(lt_component.set_version_sensor(sens)) + text_sensors = text_sensor.sub_text_sensors(config) + await text_sensors(CONF_VERSION, hub.set_version_sensor) diff --git a/esphome/components/sound_level/sensor.py b/esphome/components/sound_level/sensor.py index a28e91dfd5..c8e31226ef 100644 --- a/esphome/components/sound_level/sensor.py +++ b/esphome/components/sound_level/sensor.py @@ -73,12 +73,9 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_measurement_duration(config[CONF_MEASUREMENT_DURATION])) - if peak_config := config.get(CONF_PEAK): - sens = await sensor.new_sensor(peak_config) - cg.add(var.set_peak_sensor(sens)) - if rms_config := config.get(CONF_RMS): - sens = await sensor.new_sensor(rms_config) - cg.add(var.set_rms_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_PEAK, var.set_peak_sensor) + await sensors(CONF_RMS, var.set_rms_sensor) SOUND_LEVEL_ACTION_SCHEMA = automation.maybe_simple_id( From a5178a0f3ddf2d22dfe123690774a386c897ea9d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 13:33:03 -0500 Subject: [PATCH 096/221] [bmp280_base][bmp3xx_base][bmp581_base][spa06_base][mlx90393] Use the sub_* helpers for optional child entities (#20051) --- esphome/components/bmp280_base/__init__.py | 11 +++++------ esphome/components/bmp3xx_base/__init__.py | 11 +++++------ esphome/components/bmp581_base/__init__.py | 11 +++++------ esphome/components/mlx90393/sensor.py | 17 +++++------------ esphome/components/spa06_base/__init__.py | 11 +++++------ 5 files changed, 25 insertions(+), 36 deletions(-) diff --git a/esphome/components/bmp280_base/__init__.py b/esphome/components/bmp280_base/__init__.py index c0f0ae90bf..a956ed34f6 100644 --- a/esphome/components/bmp280_base/__init__.py +++ b/esphome/components/bmp280_base/__init__.py @@ -75,14 +75,13 @@ async def to_code_base(config: ConfigType) -> MockObj: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + if await sensors(CONF_TEMPERATURE, var.set_temperature_sensor): + temperature_config = config[CONF_TEMPERATURE] cg.add(var.set_temperature_oversampling(temperature_config[CONF_OVERSAMPLING])) - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): + pressure_config = config[CONF_PRESSURE] cg.add(var.set_pressure_oversampling(pressure_config[CONF_OVERSAMPLING])) cg.add(var.set_iir_filter(config[CONF_IIR_FILTER])) diff --git a/esphome/components/bmp3xx_base/__init__.py b/esphome/components/bmp3xx_base/__init__.py index 75e168378e..6d0618e136 100644 --- a/esphome/components/bmp3xx_base/__init__.py +++ b/esphome/components/bmp3xx_base/__init__.py @@ -80,18 +80,17 @@ async def to_code_base(config: ConfigType) -> MockObj: await cg.register_component(var, config) cg.add(var.set_iir_filter_config(config[CONF_IIR_FILTER])) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + if await sensors(CONF_TEMPERATURE, var.set_temperature_sensor): + temperature_config = config[CONF_TEMPERATURE] cg.add( var.set_temperature_oversampling_config( temperature_config[CONF_OVERSAMPLING] ) ) - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): + pressure_config = config[CONF_PRESSURE] cg.add(var.set_pressure_oversampling_config(pressure_config[CONF_OVERSAMPLING])) return var diff --git a/esphome/components/bmp581_base/__init__.py b/esphome/components/bmp581_base/__init__.py index 1c2c5c37d4..5d031a8208 100644 --- a/esphome/components/bmp581_base/__init__.py +++ b/esphome/components/bmp581_base/__init__.py @@ -137,9 +137,9 @@ CONFIG_SCHEMA_BASE = cv.Schema( async def to_code_base(config: ConfigType) -> MockObj: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + if await sensors(CONF_TEMPERATURE, var.set_temperature_sensor): + temperature_config = config[CONF_TEMPERATURE] cg.add( var.set_temperature_oversampling_config( temperature_config[CONF_OVERSAMPLING] @@ -149,9 +149,8 @@ async def to_code_base(config: ConfigType) -> MockObj: var.set_temperature_iir_filter_config(temperature_config[CONF_IIR_FILTER]) ) - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): + pressure_config = config[CONF_PRESSURE] cg.add(var.set_pressure_oversampling_config(pressure_config[CONF_OVERSAMPLING])) cg.add(var.set_pressure_iir_filter_config(pressure_config[CONF_IIR_FILTER])) diff --git a/esphome/components/mlx90393/sensor.py b/esphome/components/mlx90393/sensor.py index 59bdffc114..4ff42c7d9c 100644 --- a/esphome/components/mlx90393/sensor.py +++ b/esphome/components/mlx90393/sensor.py @@ -139,21 +139,14 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_temperature_compensation(config[CONF_TEMPERATURE_COMPENSATION])) cg.add(var.set_hallconf(config[CONF_HALLCONF])) - if CONF_X_AXIS in config: - sens = await sensor.new_sensor(config[CONF_X_AXIS]) - cg.add(var.set_x_sensor(sens)) + sensors = sensor.sub_sensors(config) + if await sensors(CONF_X_AXIS, var.set_x_sensor): cg.add(var.set_resolution(0, RESOLUTION[config[CONF_X_AXIS][CONF_RESOLUTION]])) - if CONF_Y_AXIS in config: - sens = await sensor.new_sensor(config[CONF_Y_AXIS]) - cg.add(var.set_y_sensor(sens)) + if await sensors(CONF_Y_AXIS, var.set_y_sensor): cg.add(var.set_resolution(1, RESOLUTION[config[CONF_Y_AXIS][CONF_RESOLUTION]])) - if CONF_Z_AXIS in config: - sens = await sensor.new_sensor(config[CONF_Z_AXIS]) - cg.add(var.set_z_sensor(sens)) + if await sensors(CONF_Z_AXIS, var.set_z_sensor): cg.add(var.set_resolution(2, RESOLUTION[config[CONF_Z_AXIS][CONF_RESOLUTION]])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_t_sensor(sens)) + if await sensors(CONF_TEMPERATURE, var.set_t_sensor): cg.add(var.set_t_oversampling(config[CONF_TEMPERATURE][CONF_OVERSAMPLING])) if CONF_DRDY_PIN in config: pin = await cg.gpio_pin_expression(config[CONF_DRDY_PIN]) diff --git a/esphome/components/spa06_base/__init__.py b/esphome/components/spa06_base/__init__.py index c995c2c087..e084020a14 100644 --- a/esphome/components/spa06_base/__init__.py +++ b/esphome/components/spa06_base/__init__.py @@ -181,9 +181,9 @@ CONFIG_SCHEMA_BASE.add_extra(measurement_timing_check) async def to_code_base(config: ConfigType) -> MockObj: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + if await sensors(CONF_TEMPERATURE, var.set_temperature_sensor): + temperature_config = config[CONF_TEMPERATURE] cg.add( var.set_temperature_oversampling_config( temperature_config[CONF_OVERSAMPLING] @@ -193,9 +193,8 @@ async def to_code_base(config: ConfigType) -> MockObj: var.set_temperature_sample_rate_config(temperature_config[CONF_SAMPLE_RATE]) ) - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): + pressure_config = config[CONF_PRESSURE] cg.add(var.set_pressure_oversampling_config(pressure_config[CONF_OVERSAMPLING])) cg.add(var.set_pressure_sample_rate_config(pressure_config[CONF_SAMPLE_RATE])) From de0a0b2af4ffd080ef41f1642fae6487ab51aba0 Mon Sep 17 00:00:00 2001 From: David Coulson <23066302+davidcoulson@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:46:16 -0400 Subject: [PATCH 097/221] [ethernet] Fix IPv6 SLAAC: link-local on link-up, pass all-nodes multicast (#19752) Co-authored-by: Claude Opus 5.5 Co-authored-by: J. Nick Koston --- .../components/ethernet/ethernet_component.h | 3 + .../ethernet/ethernet_component_esp32.cpp | 62 ++++++++++++++++--- .../ethernet/test-ipv6.esp32-idf.yaml | 12 ++++ 3 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 tests/components/ethernet/test-ipv6.esp32-idf.yaml diff --git a/esphome/components/ethernet/ethernet_component.h b/esphome/components/ethernet/ethernet_component.h index 75dcfb45f6..71fa9d40b6 100644 --- a/esphome/components/ethernet/ethernet_component.h +++ b/esphome/components/ethernet/ethernet_component.h @@ -220,6 +220,9 @@ class EthernetComponent final : public Component { protected: void start_connect_(); void finish_connect_(); +#if LWIP_IPV6 + esp_err_t ensure_ip6_linklocal_(); +#endif void dump_connect_params_(); #ifdef USE_ESP32 diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index d15ef5a34c..2c3c5f1ef9 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -10,6 +10,10 @@ #include #include #include "esp_event.h" +#if USE_NETWORK_IPV6 +#include +#include +#endif #ifdef USE_PSRAM #include #endif @@ -509,6 +513,18 @@ void EthernetComponent::ethernet_lazy_init_() { } } #endif +#if USE_NETWORK_IPV6 && ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0) + // Since IDF 5.5 the internal EMAC drops multicast groups that were never added (before, + // it passed all multicast), and lwIP never adds all-nodes, so router advertisements + // were lost and SLAAC never ran. + { + uint8_t all_nodes[6] = {0x33, 0x33, 0x00, 0x00, 0x00, 0x01}; + if (esp_err_t filter_err = esp_eth_ioctl(this->eth_handle_, ETH_CMD_ADD_MAC_FILTER, all_nodes); + filter_err != ESP_OK) { + ESP_LOGD(TAG, "IPv6 all-nodes multicast filter not added: %s", esp_err_to_name(filter_err)); + } + } +#endif // Register user defined event handers err = esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, &EthernetComponent::eth_event_handler, nullptr); @@ -755,6 +771,13 @@ void EthernetComponent::eth_event_handler(void *arg, esp_event_base_t event_base global_eth_component->notify_ip_state_listeners_(); } #endif +#if USE_NETWORK_IPV6 + // Start SLAAC on link-up, not after the DHCPv4 lease. This also restores the + // link-local after a link flap, which clears the IPv6 addresses. + if (esp_err_t ll_err = esp_netif_create_ip6_linklocal(global_eth_component->eth_netif_); ll_err != ESP_OK) { + ESP_LOGW(TAG, "esp_netif_create_ip6_linklocal failed on link-up: %s", esp_err_to_name(ll_err)); + } +#endif /* USE_NETWORK_IPV6 */ break; case ETHERNET_EVENT_DISCONNECTED: event_name = "ETH disconnected"; @@ -791,7 +814,10 @@ void EthernetComponent::got_ip6_event_handler(void *arg, esp_event_base_t event_ void *event_data) { ip_event_got_ip6_t *event = (ip_event_got_ip6_t *) event_data; ESP_LOGV(TAG, "[Ethernet event] ETH Got IPv6: " IPV6STR, IPV62STR(event->ip6_info.ip)); - global_eth_component->ipv6_count_ += 1; + // Count the addresses on the interface, not the events: recreating the link-local + // after a link flap fires another event for the same address. + struct esp_ip6_addr if_ip6s[CONFIG_LWIP_IPV6_NUM_ADDRESSES]; + global_eth_component->ipv6_count_ = esp_netif_get_all_ip6(global_eth_component->eth_netif_, if_ip6s); #if (USE_NETWORK_MIN_IPV6_ADDR_COUNT > 0) global_eth_component->connected_ = global_eth_component->got_ipv4_address_ && (global_eth_component->ipv6_count_ >= USE_NETWORK_MIN_IPV6_ADDR_COUNT); @@ -806,6 +832,29 @@ void EthernetComponent::got_ip6_event_handler(void *arg, esp_event_base_t event_ } #endif /* USE_NETWORK_IPV6 */ +#if USE_NETWORK_IPV6 +// Create the link-local address unless the interface already has one, including one still in +// duplicate address detection: recreating it would restart DAD. esp_netif_get_ip6_linklocal() +// only reports a preferred address, so ask lwIP for the slot state instead. +esp_err_t EthernetComponent::ensure_ip6_linklocal_() { + if (auto *netif = static_cast(esp_netif_get_netif_impl(this->eth_netif_)); netif != nullptr) { + u8_t state; + { + LwIPLock lock; + state = netif_ip6_addr_state(netif, 0); + } + if (ip6_addr_istentative(state) || ip6_addr_isvalid(state)) { + return ESP_OK; + } + } + esp_err_t err = esp_netif_create_ip6_linklocal(this->eth_netif_); + if (err == ESP_OK) { + ESP_LOGD(TAG, "IPv6 link-local address created"); + } + return err; +} +#endif /* USE_NETWORK_IPV6 */ + void EthernetComponent::finish_connect_() { #if USE_NETWORK_IPV6 // Retry IPv6 link-local setup if it failed during initial connect @@ -816,10 +865,7 @@ void EthernetComponent::finish_connect_() { // - Cable unplugged/network interruption (#10705) // We can now retry since we're in CONNECTED state and the interface is definitely up. if (!this->ipv6_setup_done_) { - esp_err_t err = esp_netif_create_ip6_linklocal(this->eth_netif_); - if (err == ESP_OK) { - ESP_LOGD(TAG, "IPv6 link-local address created (retry succeeded)"); - } + this->ensure_ip6_linklocal_(); // Always set the flag to prevent continuous retries // If IPv6 setup fails here with the interface up and stable, it's // likely a persistent issue (IPv6 disabled at router, hardware @@ -833,7 +879,9 @@ void EthernetComponent::finish_connect_() { void EthernetComponent::start_connect_() { global_eth_component->got_ipv4_address_ = false; #if USE_NETWORK_IPV6 - global_eth_component->ipv6_count_ = 0; + // Recount rather than zero: addresses that survive a reconnect are not announced again. + struct esp_ip6_addr if_ip6s[CONFIG_LWIP_IPV6_NUM_ADDRESSES]; + global_eth_component->ipv6_count_ = esp_netif_get_all_ip6(this->eth_netif_, if_ip6s); this->ipv6_setup_done_ = false; #endif /* USE_NETWORK_IPV6 */ this->connect_begin_ = millis(); @@ -912,7 +960,7 @@ void EthernetComponent::start_connect_() { // - At bootup when link isn't ready (#10281) // - After disconnection/cable unplugged (#10705) // We'll retry in finish_connect_() if it fails here. - err = esp_netif_create_ip6_linklocal(this->eth_netif_); + err = this->ensure_ip6_linklocal_(); if (err != ESP_OK) { if (err == ESP_ERR_ESP_NETIF_INVALID_PARAMS) { // This is a programming error, not a transient failure diff --git a/tests/components/ethernet/test-ipv6.esp32-idf.yaml b/tests/components/ethernet/test-ipv6.esp32-idf.yaml new file mode 100644 index 0000000000..607e82dfa9 --- /dev/null +++ b/tests/components/ethernet/test-ipv6.esp32-idf.yaml @@ -0,0 +1,12 @@ +network: + enable_ipv6: true + +ethernet: + type: IP101 + mdc_pin: 23 + mdio_pin: 18 + clk: + pin: 0 + mode: CLK_EXT_IN + phy_addr: 1 + power_pin: 5 From 5328e5f5e0905b499f6c2a2b7912b182bed094f1 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:47:10 +0200 Subject: [PATCH 098/221] [tcp_uart] Add a server role with an IPv4 allow list (#20026) --- esphome/components/tcp_uart/__init__.py | 69 +++++++++++++------ esphome/components/tcp_uart/tcp_uart.cpp | 29 +++++++- esphome/components/tcp_uart/tcp_uart.h | 25 +++++-- tests/components/tcp_uart/common.yaml | 15 ++-- .../tcp_uart/test-client.esp32-idf.yaml | 11 +++ tests/components/tcp_uart/test_flush_host.cpp | 6 +- .../integration/fixtures/tcp_uart_server.yaml | 30 ++++++++ tests/integration/test_tcp_uart_server.py | 56 +++++++++++++++ 8 files changed, 205 insertions(+), 36 deletions(-) create mode 100644 tests/components/tcp_uart/test-client.esp32-idf.yaml create mode 100644 tests/integration/fixtures/tcp_uart_server.yaml create mode 100644 tests/integration/test_tcp_uart_server.py diff --git a/esphome/components/tcp_uart/__init__.py b/esphome/components/tcp_uart/__init__.py index e46e4dbfd0..dbe805c281 100644 --- a/esphome/components/tcp_uart/__init__.py +++ b/esphome/components/tcp_uart/__init__.py @@ -5,6 +5,7 @@ from esphome.components.const import ( CONF_HOST, CONF_PARITY, CONF_RECONNECT_INTERVAL, + CONF_ROLE, CONF_STOP_BITS, ) import esphome.config_validation as cv @@ -25,43 +26,71 @@ MULTI_CONF = True tcp_uart_ns = cg.esphome_ns.namespace("tcp_uart") TcpUart = tcp_uart_ns.class_("TcpUart", uart.UARTComponent, cg.Component) +CONF_ALLOWED_IPS = "allowed_ips" CONF_CONNECTED = "connected" +BASE_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(TcpUart), + cv.Required(CONF_PORT): cv.port, + cv.Optional(CONF_BAUD_RATE, default=9600): cv.int_range(min=1), + cv.Optional(CONF_DATA_BITS, default=8): cv.int_range(min=5, max=8), + cv.Optional(CONF_PARITY, default="NONE"): cv.enum( + uart.UART_PARITY_OPTIONS, upper=True + ), + cv.Optional(CONF_STOP_BITS, default=1): cv.one_of(1, 2, int=True), + cv.Optional( + CONF_RECONNECT_INTERVAL, default="5s" + ): cv.positive_time_period_milliseconds, + cv.Optional(CONF_CONNECTED): binary_sensor.binary_sensor_schema( + device_class=DEVICE_CLASS_CONNECTIVITY, + entity_category=ENTITY_CATEGORY_DIAGNOSTIC, + ), + } +).extend(cv.COMPONENT_SCHEMA) + CONFIG_SCHEMA = cv.All( - cv.Schema( + cv.typed_schema( { - cv.GenerateID(): cv.declare_id(TcpUart), - cv.Required(CONF_HOST): cv.string, - cv.Required(CONF_PORT): cv.port, - cv.Optional(CONF_BAUD_RATE, default=9600): cv.int_range(min=1), - cv.Optional(CONF_DATA_BITS, default=8): cv.int_range(min=5, max=8), - cv.Optional(CONF_PARITY, default="NONE"): cv.enum( - uart.UART_PARITY_OPTIONS, upper=True + "client": BASE_SCHEMA.extend( + { + cv.Required(CONF_HOST): cv.string, + } ), - cv.Optional(CONF_STOP_BITS, default=1): cv.one_of(1, 2, int=True), - cv.Optional( - CONF_RECONNECT_INTERVAL, default="5s" - ): cv.positive_time_period_milliseconds, - cv.Optional(CONF_CONNECTED): binary_sensor.binary_sensor_schema( - device_class=DEVICE_CLASS_CONNECTIVITY, - entity_category=ENTITY_CATEGORY_DIAGNOSTIC, + "server": BASE_SCHEMA.extend( + { + cv.Optional(CONF_ALLOWED_IPS): socket.IPV4_ALLOW_SCHEMA, + } ), - } - ).extend(cv.COMPONENT_SCHEMA), - socket.consume_sockets(1, "tcp_uart"), + }, + key=CONF_ROLE, + default_type="client", + lower=True, + ), + socket.consume_role_sockets("tcp_uart"), ) async def to_code(config: ConfigType) -> None: - socket.require_tcp_client_link() - var = cg.new_Pvariable(config[CONF_ID], config[CONF_HOST], config[CONF_PORT]) + var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) + if config[CONF_ROLE] == "server": + socket.require_tcp_listener() + cg.add(var.set_server(True)) + socket.add_ipv4_allow( + var.set_allow, config.get(CONF_ALLOWED_IPS), config[CONF_ID] + ) + else: + socket.require_tcp_client_link() + cg.add(var.set_port(config[CONF_PORT])) cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL])) # The socket is not clocked. These only satisfy UARTComponent and a consumer check. cg.add(var.set_baud_rate(config[CONF_BAUD_RATE])) cg.add(var.set_data_bits(config[CONF_DATA_BITS])) cg.add(var.set_stop_bits(config[CONF_STOP_BITS])) cg.add(var.set_parity(config[CONF_PARITY])) + if (host := config.get(CONF_HOST)) is not None: + cg.add(var.set_host(host)) binary_sensors = binary_sensor.sub_binary_sensors(config) await binary_sensors(CONF_CONNECTED, var.set_connected_sensor) diff --git a/esphome/components/tcp_uart/tcp_uart.cpp b/esphome/components/tcp_uart/tcp_uart.cpp index b24280c8b1..5913610ee8 100644 --- a/esphome/components/tcp_uart/tcp_uart.cpp +++ b/esphome/components/tcp_uart/tcp_uart.cpp @@ -14,6 +14,9 @@ static constexpr uint32_t DROP_LOG_INTERVAL_MS = 5000; void TcpUart::setup() { this->link_.begin(TAG); +#ifdef USE_SOCKET_TCP_LISTENER + this->listener_.begin(TAG); +#endif if (this->connected_sensor_ != nullptr) { this->connected_sensor_->publish_state(false); } @@ -22,12 +25,24 @@ void TcpUart::setup() { void TcpUart::dump_config() { ESP_LOGCONFIG(TAG, "TCP UART:\n" - " Host: %s:%u\n" + " %s: %s:%u\n" " Reconnect Interval: %" PRIu32 "ms", - this->link_.host(), this->link_.port(), this->link_.reconnect_interval()); + this->server_ ? LOG_STR_LITERAL("Listen") : LOG_STR_LITERAL("Host"), + this->server_ ? LOG_STR_LITERAL("*") : this->link_.host(), this->link_.port(), + this->link_.reconnect_interval()); +#ifdef USE_SOCKET_TCP_LISTENER + this->listener_.dump_config(); +#endif LOG_BINARY_SENSOR(" ", "Connected", this->connected_sensor_); } +void TcpUart::on_shutdown() { + this->link_.close(); +#ifdef USE_SOCKET_TCP_LISTENER + this->listener_.close(); +#endif +} + void TcpUart::sync_link_() { bool up = this->link_.connected(); this->link_was_up_ = up; @@ -63,7 +78,17 @@ void TcpUart::read_socket_() { } void TcpUart::loop() { +#ifdef USE_SOCKET_TCP_LISTENER + if (this->server_) { + // link_was_up_ holds the accept until the previous drop's edge has run, + // so the sensor and the cleared RX buffer always see the disconnect. + this->listener_.poll(this->link_, !this->link_was_up_); + } else { + this->link_.poll(); + } +#else this->link_.poll(); +#endif if (this->link_.connected() != this->link_was_up_) { this->sync_link_(); } diff --git a/esphome/components/tcp_uart/tcp_uart.h b/esphome/components/tcp_uart/tcp_uart.h index 7ce822c2fb..29f7af564a 100644 --- a/esphome/components/tcp_uart/tcp_uart.h +++ b/esphome/components/tcp_uart/tcp_uart.h @@ -2,6 +2,9 @@ #include "esphome/components/binary_sensor/binary_sensor.h" #include "esphome/components/socket/tcp_client_link.h" +#ifdef USE_SOCKET_TCP_LISTENER +#include "esphome/components/socket/tcp_listener.h" +#endif #include "esphome/components/uart/uart_component.h" #include "esphome/core/component.h" @@ -9,22 +12,26 @@ namespace esphome::tcp_uart { -/// TCP client presented as a UART. Bytes are copied unchanged. +/// TCP client or server presented as a UART. Bytes are copied unchanged. class TcpUart : public uart::UARTComponent, public Component { public: - TcpUart(const char *host, uint16_t port) { - this->link_.set_host(host); - this->link_.set_port(port); - this->rx_buffer_size_ = RX_BUFFER_SIZE; - } + TcpUart() { this->rx_buffer_size_ = RX_BUFFER_SIZE; } + void set_host(const char *host) { this->link_.set_host(host); } + void set_port(uint16_t port) { this->link_.set_port(port); } void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); } void set_connected_sensor(binary_sensor::BinarySensor *sensor) { this->connected_sensor_ = sensor; } +#ifdef USE_SOCKET_TCP_LISTENER + void set_server(bool server) { this->server_ = server; } +#ifdef USE_SOCKET_IPV4_ALLOW + void set_allow(const socket::Ipv4AllowEntry *entries, size_t count) { this->listener_.set_allow(entries, count); } +#endif +#endif void setup() override; void loop() override; void dump_config() override; - void on_shutdown() override { this->link_.close(); } + void on_shutdown() override; float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } void write_array(const uint8_t *data, size_t len) override; @@ -47,11 +54,15 @@ class TcpUart : public uart::UARTComponent, public Component { static constexpr size_t RX_BUFFER_SIZE = 1024; socket::TcpClientLink link_; +#ifdef USE_SOCKET_TCP_LISTENER + socket::TcpListener listener_; +#endif binary_sensor::BinarySensor *connected_sensor_{nullptr}; uint32_t last_drop_log_ms_{0}; // rx_[rx_start_, rx_end_) holds unread bytes; read_socket_() compacts to the front. uint16_t rx_start_{0}; uint16_t rx_end_{0}; + bool server_{false}; // The link state loop() saw last; edges clear rx_ and publish the sensor. bool link_was_up_{false}; // A read stopped before EAGAIN. ready() stays false until new data arrives. diff --git a/tests/components/tcp_uart/common.yaml b/tests/components/tcp_uart/common.yaml index f3c1d061aa..2c2d264f7f 100644 --- a/tests/components/tcp_uart/common.yaml +++ b/tests/components/tcp_uart/common.yaml @@ -3,18 +3,21 @@ wifi: password: password1 tcp_uart: - - id: tcp_uart_1 - host: 192.0.2.10 - port: 502 + - id: tcp_uart_server + role: server + port: 5020 reconnect_interval: 10s + allowed_ips: + - 192.0.2.20 + - 192.0.2.0/24 connected: - name: TCP UART Connected + name: TCP UART Server Connected interval: - interval: 60s then: - lambda: |- uint8_t byte; - if (id(tcp_uart_1).available() && id(tcp_uart_1).read_byte(&byte)) { - id(tcp_uart_1).write_byte(byte); + if (id(tcp_uart_server).available() && id(tcp_uart_server).read_byte(&byte)) { + id(tcp_uart_server).write_byte(byte); } diff --git a/tests/components/tcp_uart/test-client.esp32-idf.yaml b/tests/components/tcp_uart/test-client.esp32-idf.yaml new file mode 100644 index 0000000000..c63c6a7c7d --- /dev/null +++ b/tests/components/tcp_uart/test-client.esp32-idf.yaml @@ -0,0 +1,11 @@ +wifi: + ssid: MySSID + password: password1 + +tcp_uart: + - id: tcp_uart_1 + host: 192.0.2.10 + port: 502 + reconnect_interval: 10s + connected: + name: TCP UART Connected diff --git a/tests/components/tcp_uart/test_flush_host.cpp b/tests/components/tcp_uart/test_flush_host.cpp index 9c6c63110c..5573d587ce 100644 --- a/tests/components/tcp_uart/test_flush_host.cpp +++ b/tests/components/tcp_uart/test_flush_host.cpp @@ -13,7 +13,11 @@ namespace esphome::tcp_uart::testing { class TcpUartUnderTest : public TcpUart { public: - TcpUartUnderTest() : TcpUart("peer", 1) { this->link_.begin("flush_test"); } + TcpUartUnderTest() { + this->set_host("peer"); + this->set_port(1); + this->link_.begin("flush_test"); + } socket::TcpClientLink &link() { return this->link_; } }; diff --git a/tests/integration/fixtures/tcp_uart_server.yaml b/tests/integration/fixtures/tcp_uart_server.yaml new file mode 100644 index 0000000000..20bc221d9d --- /dev/null +++ b/tests/integration/fixtures/tcp_uart_server.yaml @@ -0,0 +1,30 @@ +esphome: + name: tcp-uart-server-test + +host: + +api: + +logger: + level: INFO + +tcp_uart: + - id: allowed_bus + role: server + port: 18126 + allowed_ips: + - 127.0.0.1 + - id: denied_bus + role: server + port: 18127 + allowed_ips: + - 192.0.2.1 + +interval: + - interval: 50ms + then: + - lambda: |- + uint8_t b; + while (id(allowed_bus).read_byte(&b)) { + id(allowed_bus).write_byte(b); + } diff --git a/tests/integration/test_tcp_uart_server.py b/tests/integration/test_tcp_uart_server.py new file mode 100644 index 0000000000..9a69ec42fd --- /dev/null +++ b/tests/integration/test_tcp_uart_server.py @@ -0,0 +1,56 @@ +"""Integration test for a tcp_uart server on host. + +Pytest connects as the TCP client. One server allows 127.0.0.1 and echoes. +The other allows only 192.0.2.1, so the same client is closed. +""" + +from __future__ import annotations + +import asyncio +import contextlib + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + +PAYLOAD = b"ping!" + + +@pytest.mark.asyncio +async def test_tcp_uart_server( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + unused_tcp_port_factory, +) -> None: + allowed_port = unused_tcp_port_factory() + denied_port = unused_tcp_port_factory() + yaml_config = yaml_config.replace("port: 18126", f"port: {allowed_port}") + yaml_config = yaml_config.replace("port: 18127", f"port: {denied_port}") + + lines = LineWaiter() + async with ( + run_compiled(yaml_config, line_callback=lines.callback), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "tcp-uart-server-test" + await lines.wait_for(f"Listening on {allowed_port}") + await lines.wait_for(f"Listening on {denied_port}") + + reader, writer = await asyncio.open_connection("127.0.0.1", allowed_port) + await lines.wait_for("Client connected from 127.0.0.1") + writer.write(PAYLOAD) + await writer.drain() + assert await asyncio.wait_for(reader.readexactly(len(PAYLOAD)), 10) == PAYLOAD + writer.close() + + denied_reader, denied_writer = await asyncio.open_connection( + "127.0.0.1", denied_port + ) + await lines.wait_for("Rejected 127.0.0.1") + with contextlib.suppress(ConnectionResetError): + assert await asyncio.wait_for(denied_reader.read(8), 10) == b"" + denied_writer.close() From 25eaf1d02551fca169d5a2a8aece37c3702513cb Mon Sep 17 00:00:00 2001 From: David Coulson <23066302+davidcoulson@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:07:58 -0400 Subject: [PATCH 099/221] [wifi] Start IPv6 link-local on connect, not on DHCPv4 lease (#19749) Co-authored-by: Claude Opus 5.5 Co-authored-by: J. Nick Koston --- esphome/components/wifi/wifi_component_esp_idf.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 24bf64a99c..c44ee5b1de 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -829,6 +829,12 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { (const char *) it.ssid, bssid_buf, it.channel, get_auth_mode_str(it.authmode)); #endif s_sta_connected = true; +#if USE_NETWORK_IPV6 + // Start SLAAC on association, not after a DHCPv4 lease that may never arrive. + if (esp_err_t ll_err = esp_netif_create_ip6_linklocal(s_sta_netif); ll_err != ESP_OK) { + ESP_LOGW(TAG, "esp_netif_create_ip6_linklocal failed: %s", esp_err_to_name(ll_err)); + } +#endif /* USE_NETWORK_IPV6 */ if (this->state_ == WIFI_COMPONENT_STATE_STA_CONNECTED) { // Driver-initiated roam: the WIFI_REASON_ROAMING disconnect was ignored, // so the state machine never left STA_CONNECTED. @@ -881,9 +887,6 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { } else if (data->event_base == IP_EVENT && data->event_id == IP_EVENT_STA_GOT_IP) { const auto &it = data->data.ip_got_ip; -#if USE_NETWORK_IPV6 - esp_netif_create_ip6_linklocal(s_sta_netif); -#endif /* USE_NETWORK_IPV6 */ ESP_LOGV(TAG, "static_ip=" IPSTR " gateway=" IPSTR, IP2STR(&it.ip_info.ip), IP2STR(&it.ip_info.gw)); this->got_ipv4_address_ = true; #ifdef USE_WIFI_IP_STATE_LISTENERS From fe5a155c13872f7966fa0aa455bc16977b0f57b4 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:14:51 +0200 Subject: [PATCH 100/221] [const] Hoist the allowed ips key to the shared constants (#20053) Co-authored-by: J. Nick Koston --- esphome/components/const/__init__.py | 2 ++ esphome/components/tcp_uart/__init__.py | 5 ++--- esphome/components/uart_tcp/__init__.py | 11 +++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index b67964d47f..6e98162baa 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -7,6 +7,7 @@ BYTE_ORDER_BIG = "big_endian" CONF_ACCELEROMETER_ODR = "accelerometer_odr" CONF_ACCELEROMETER_RANGE = "accelerometer_range" +CONF_ALLOWED_IPS = "allowed_ips" CONF_B_CONSTANT = "b_constant" CONF_BREATH_VOC_EQUIVALENT = "breath_voc_equivalent" CONF_BYTE_ORDER = "byte_order" @@ -15,6 +16,7 @@ CONF_CLIMATE_ID = "climate_id" CONF_CO2_EQUIVALENT = "co2_equivalent" CONF_COLOR_DEPTH = "color_depth" CONF_COLUMNS = "columns" +CONF_CONNECTED = "connected" CONF_CONVERSION_RATE = "conversion_rate" CONF_CRC_ENABLE = "crc_enable" CONF_DATA_BITS = "data_bits" diff --git a/esphome/components/tcp_uart/__init__.py b/esphome/components/tcp_uart/__init__.py index dbe805c281..8e14e690f9 100644 --- a/esphome/components/tcp_uart/__init__.py +++ b/esphome/components/tcp_uart/__init__.py @@ -1,6 +1,8 @@ import esphome.codegen as cg from esphome.components import binary_sensor, socket, uart from esphome.components.const import ( + CONF_ALLOWED_IPS, + CONF_CONNECTED, CONF_DATA_BITS, CONF_HOST, CONF_PARITY, @@ -26,9 +28,6 @@ MULTI_CONF = True tcp_uart_ns = cg.esphome_ns.namespace("tcp_uart") TcpUart = tcp_uart_ns.class_("TcpUart", uart.UARTComponent, cg.Component) -CONF_ALLOWED_IPS = "allowed_ips" -CONF_CONNECTED = "connected" - BASE_SCHEMA = cv.Schema( { diff --git a/esphome/components/uart_tcp/__init__.py b/esphome/components/uart_tcp/__init__.py index cd56e2c8bc..3829f046ca 100644 --- a/esphome/components/uart_tcp/__init__.py +++ b/esphome/components/uart_tcp/__init__.py @@ -1,6 +1,12 @@ import esphome.codegen as cg from esphome.components import binary_sensor, socket, uart -from esphome.components.const import CONF_HOST, CONF_RECONNECT_INTERVAL, CONF_ROLE +from esphome.components.const import ( + CONF_ALLOWED_IPS, + CONF_CONNECTED, + CONF_HOST, + CONF_RECONNECT_INTERVAL, + CONF_ROLE, +) import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -19,9 +25,6 @@ MULTI_CONF = True uart_tcp_ns = cg.esphome_ns.namespace("uart_tcp") UartTcp = uart_tcp_ns.class_("UartTcp", cg.Component, uart.UARTDevice) -CONF_ALLOWED_IPS = "allowed_ips" -CONF_CONNECTED = "connected" - BASE_SCHEMA = cv.Schema( { From 03f264775f09003eaf3898fa167477df0d0cdc8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:37:16 -0500 Subject: [PATCH 101/221] Bump filelock from 4.0.5 to 4.0.6 (#20055) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 32931b650e..21bd6f1e1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.12.1 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver -filelock==4.0.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==4.0.6 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.3 From 911bd3dadb86f4608cfde56eeb265322336f8439 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:37:34 -0500 Subject: [PATCH 102/221] Bump pylint from 4.0.9 to 4.0.10 (#20056) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index a66e05ba74..fa56852637 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,4 @@ -pylint==4.0.9 +pylint==4.0.10 flake8==7.4.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.9 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py From 4447f8a065b4b88167d397d229e3f298fb841d6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:37:45 -0500 Subject: [PATCH 103/221] Bump platformdirs from 4.12.1 to 4.12.2 (#20054) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 21bd6f1e1e..1f21fd8160 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.12.1 # native esp-idf toolchain global cache dir +platformdirs==4.12.2 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver filelock==4.0.6 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From fcf8fe2c11f83db03d4e69f22418ef62d22cc5e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:38:02 -0500 Subject: [PATCH 104/221] Bump pre-commit-ci/lite-action from 1.1.0 to 1.2.0 (#20057) Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7ed035063..4ebbe52d20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,7 +299,7 @@ jobs: # Pushes any fixes the hooks made back to the pull request. This step # must keep its default name: the GitHub App that performs the push # locates the workflow run by that name. - - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 # v1.1.0 + - uses: pre-commit-ci/lite-action@062bca0919bc9d6e66755cc05074b70c77e111fc # v1.2.0 if: always() with: msg: apply automatic formatting fixes From f5c79eeec0f961b0108f0a89ba158f00183cf693 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 2 Oct 2026 19:29:34 -0500 Subject: [PATCH 105/221] [qmi8658] Pin motion_id in test config to fix grouped component test conflict (#20075) --- tests/components/qmi8658/common-actions.yaml | 2 ++ tests/components/qmi8658/common.yaml | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/tests/components/qmi8658/common-actions.yaml b/tests/components/qmi8658/common-actions.yaml index b8ab845ef1..62371223cc 100644 --- a/tests/components/qmi8658/common-actions.yaml +++ b/tests/components/qmi8658/common-actions.yaml @@ -3,5 +3,7 @@ button: name: Clear motion calibration on_press: - motion.clear_calibration: + id: qmi8658_motion - motion.clear_calibration: + id: qmi8658_motion save: true diff --git a/tests/components/qmi8658/common.yaml b/tests/components/qmi8658/common.yaml index 7d4de0f97e..bfb9caff02 100644 --- a/tests/components/qmi8658/common.yaml +++ b/tests/components/qmi8658/common.yaml @@ -3,6 +3,7 @@ sensor: name: "QMI8658 Temperature" - platform: motion + motion_id: qmi8658_motion type: acceleration_x name: "Accel X" accuracy_decimals: 4 @@ -11,44 +12,55 @@ sensor: window_size: 4 send_every: 1 - platform: motion + motion_id: qmi8658_motion type: acceleration_y name: "Accel Y" accuracy_decimals: 4 - platform: motion + motion_id: qmi8658_motion type: acceleration_z name: "Accel Z" accuracy_decimals: 4 # Gyroscope axes (unit: °/s) - platform: motion + motion_id: qmi8658_motion type: gyroscope_x name: "Gyro X" - platform: motion + motion_id: qmi8658_motion type: gyroscope_y name: "Gyro Y" - platform: motion + motion_id: qmi8658_motion type: gyroscope_z name: "Gyro Z" - platform: motion + motion_id: qmi8658_motion type: angular_rate_x name: "Angular Rate X" - platform: motion + motion_id: qmi8658_motion type: angular_rate_y name: "Angular Rate Y" - platform: motion + motion_id: qmi8658_motion type: angular_rate_z name: "Angular Rate Z" - platform: motion + motion_id: qmi8658_motion type: pitch name: "Pitch" - platform: motion + motion_id: qmi8658_motion type: roll name: "Roll" motion: - platform: qmi8658 + id: qmi8658_motion i2c_id: i2c_bus # Accelerometer full-scale range: 2G | 4G | 8G | 16G accelerometer_range: 4G From 8691a43879ffc00ebcd363910967dca941a16b61 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 3 Oct 2026 06:29:06 -0500 Subject: [PATCH 106/221] [font] Keep the glyph table in flash (#20079) --- esphome/components/font/__init__.py | 3 +- esphome/components/font/font.cpp | 3 +- esphome/components/font/font.h | 14 +++++++++ .../font/config/glyph_table.yaml | 11 +++++++ tests/component_tests/font/test_font.py | 30 ++++++++++++------- 5 files changed, 49 insertions(+), 12 deletions(-) create mode 100644 tests/component_tests/font/config/glyph_table.yaml diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index 918fde5dbd..a2fad4bc25 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -726,7 +726,8 @@ async def to_code(config): ) ] - glyphs = cg.static_const_array(config[CONF_RAW_GLYPH_ID], glyph_initializer) + # constexpr Glyph table in flash; on ESP8266 static_const_array would place it in RAM. + glyphs = cg.progmem_array(config[CONF_RAW_GLYPH_ID], glyph_initializer) font_height = pt_to_px(base_font.size.height) ascender = pt_to_px(base_font.size.ascender) diff --git a/esphome/components/font/font.cpp b/esphome/components/font/font.cpp index fda9c269e5..1ed946dadb 100644 --- a/esphome/components/font/font.cpp +++ b/esphome/components/font/font.cpp @@ -309,7 +309,8 @@ void Font::print(int x_start, int y_start, display::Display *display, Color colo // Unknown char, skip ESP_LOGW(TAG, "Codepoint 0x%08" PRIx32 " not found in font", code_point); if (!this->glyphs_.empty()) { - uint8_t glyph_width = this->glyphs_[0].advance; + // Full-width read: a narrowing byte load would fault on a PROGMEM table on ESP8266. + int glyph_width = this->glyphs_[0].advance; display->rectangle(x_at, y_start, glyph_width, this->height_, color); x_at += glyph_width; } diff --git a/esphome/components/font/font.h b/esphome/components/font/font.h index fa24181bd0..bab7b449ba 100644 --- a/esphome/components/font/font.h +++ b/esphome/components/font/font.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "esphome/core/color.h" #include "esphome/core/datatypes.h" #include "esphome/core/defines.h" @@ -36,6 +38,18 @@ class Glyph final { int width; int height; }; +// The glyph table lives in flash, read with plain loads, which ESP8266 only allows for whole words. +static_assert(std::is_same_v, "Glyph fields must stay word sized"); +static_assert(std::is_same_v, "Glyph fields must stay word sized"); +static_assert(std::is_same_v, "Glyph fields must stay word sized"); +static_assert(std::is_same_v, "Glyph fields must stay word sized"); +static_assert(std::is_same_v, "Glyph fields must stay word sized"); +static_assert(std::is_same_v, "Glyph fields must stay word sized"); +static_assert(std::is_same_v, "Glyph fields must stay word sized"); +#ifdef USE_ESP8266 +static_assert(alignof(Glyph) == sizeof(uint32_t), "Glyph fields must stay word sized"); +static_assert(sizeof(Glyph) == 7 * sizeof(uint32_t), "Glyph is read from flash with word loads"); +#endif class Font final #ifdef USE_DISPLAY diff --git a/tests/component_tests/font/config/glyph_table.yaml b/tests/component_tests/font/config/glyph_table.yaml new file mode 100644 index 0000000000..54776e6b6e --- /dev/null +++ b/tests/component_tests/font/config/glyph_table.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp8266: + board: esp01_1m + +font: + - file: ../NotoSans-Regular.ttf + id: noto + size: 12 + glyphs: "AB" diff --git a/tests/component_tests/font/test_font.py b/tests/component_tests/font/test_font.py index 55e27ae84c..8de0c0a22c 100644 --- a/tests/component_tests/font/test_font.py +++ b/tests/component_tests/font/test_font.py @@ -4,8 +4,10 @@ Focuses on verifying that long multi-byte (Chinese/CJK) glyph strings are correctly processed through the font configuration pipeline. """ +from collections.abc import Callable import functools from pathlib import Path +import re from unittest.mock import MagicMock, patch import pytest @@ -260,15 +262,12 @@ def mock_cg(): with ( patch("esphome.components.font.cg.add_define") as mock_define, patch("esphome.components.font.cg.progmem_array") as mock_progmem, - patch("esphome.components.font.cg.static_const_array") as mock_static, patch("esphome.components.font.cg.new_Pvariable") as mock_new_pvar, ): mock_progmem.return_value = MagicMock() - mock_static.return_value = MagicMock() yield { "add_define": mock_define, "progmem_array": mock_progmem, - "static_const_array": mock_static, "new_Pvariable": mock_new_pvar, } @@ -288,13 +287,13 @@ async def test_to_code_long_latin_generates_all_glyphs(mock_cg): mock_cg["add_define"].assert_any_call("USE_FONT") # progmem_array receives the combined bitmap data (non-empty) - mock_cg["progmem_array"].assert_called_once() - bitmap_data = mock_cg["progmem_array"].call_args.args[1] + # bitmap data first, then the glyph table + assert mock_cg["progmem_array"].call_count == 2 + bitmap_data = mock_cg["progmem_array"].call_args_list[0].args[1] assert len(bitmap_data) > 0 - # static_const_array receives one entry per unique glyph - mock_cg["static_const_array"].assert_called_once() - glyph_initializer = mock_cg["static_const_array"].call_args.args[1] + # the glyph table receives one entry per unique glyph + glyph_initializer = mock_cg["progmem_array"].call_args_list[1].args[1] assert len(glyph_initializer) == glyph_count # new_Pvariable is called with the correct glyph count @@ -314,7 +313,7 @@ async def test_to_code_glyph_entries_contain_expected_fields(mock_cg): await to_code(config) - glyph_initializer = mock_cg["static_const_array"].call_args.args[1] + glyph_initializer = mock_cg["progmem_array"].call_args_list[1].args[1] for entry in glyph_initializer: assert len(entry) == 7, f"Glyph entry should have 7 fields, got {len(entry)}" codepoint = entry[0] @@ -332,6 +331,17 @@ async def test_to_code_glyphs_sorted_by_utf8(mock_cg): await to_code(config) - glyph_initializer = mock_cg["static_const_array"].call_args.args[1] + glyph_initializer = mock_cg["progmem_array"].call_args_list[1].args[1] codepoints = [entry[0] for entry in glyph_initializer] assert codepoints == sorted(codepoints) + + +def test_glyph_table_is_progmem( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The constexpr glyph table goes to flash, not a RAM static const array.""" + main_cpp = generate_main(component_config_path("glyph_table.yaml")) + + assert re.search(r"static constexpr font::Glyph \w+\[\] PROGMEM = ", main_cpp) + assert "static const font::Glyph" not in main_cpp From 2d56559f04d43095467c4a37c46ad5a97e471f18 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 3 Oct 2026 06:36:33 -0500 Subject: [PATCH 107/221] [lvgl] Reject ESP8266 at validation (#20099) --- esphome/components/lvgl/__init__.py | 9 +++++++++ esphome/components/lvgl/lvgl_esphome.cpp | 4 ++-- tests/component_tests/lvgl/test_platforms.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 tests/component_tests/lvgl/test_platforms.py diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index ca9a317fec..66aaac5301 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -48,6 +48,7 @@ from esphome.cpp_generator import MockObj from esphome.final_validate import full_config from esphome.helpers import write_file_if_changed from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor +from esphome.types import ConfigType from esphome.writer import clean_build from esphome.yaml_util import load_yaml @@ -641,7 +642,15 @@ LVGL_TOP_LEVEL_SCHEMA = ( ) +def _not_on_esp8266(config: ConfigType) -> ConfigType: + # ESP8266 does not have enough RAM for LVGL to be practical. + if CORE.is_esp8266: + raise cv.Invalid("LVGL is not supported on ESP8266") + return config + + LVGL_SCHEMA = cv.All( + _not_on_esp8266, container_schema(obj_spec, LVGL_TOP_LEVEL_SCHEMA), cv.has_at_most_one_key(CONF_PAGES, df.CONF_LAYOUT), add_hello_world, diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index 90a770ac3b..466d400023 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -1037,7 +1037,7 @@ void lv_mem_init() {} void lv_mem_deinit() {} -#if defined(USE_HOST) || defined(USE_RP2) || defined(USE_ESP8266) +#ifndef USE_ESP32 void *lv_malloc_core(size_t size) { auto *ptr = malloc(size); // NOLINT if (ptr == nullptr) { @@ -1053,7 +1053,7 @@ static void *lv_alloc_draw_buf(size_t size, bool internal) { return malloc(size); // NOLINT } -#elif defined(USE_ESP32) +#else static unsigned cap_bits = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; // NOLINT static void *lv_alloc_draw_buf(size_t size, bool internal) { diff --git a/tests/component_tests/lvgl/test_platforms.py b/tests/component_tests/lvgl/test_platforms.py new file mode 100644 index 0000000000..ac4da769ee --- /dev/null +++ b/tests/component_tests/lvgl/test_platforms.py @@ -0,0 +1,14 @@ +"""LVGL rejects ESP8266 at validation time.""" + +import pytest + +from esphome.components import lvgl +from esphome.config_validation import Invalid +from esphome.const import PlatformFramework +from tests.component_tests.types import SetCoreConfigCallable + + +def test_esp8266_is_rejected(set_core_config: SetCoreConfigCallable) -> None: + set_core_config(PlatformFramework.ESP8266_ARDUINO) + with pytest.raises(Invalid, match="not supported on ESP8266"): + lvgl.CONFIG_SCHEMA({}) From b070c828b3a61ecd33b6eeea8457fa10bd9e2427 Mon Sep 17 00:00:00 2001 From: abderrahmen bejaoui <115119179+abderbejaoui@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:37:03 +0100 Subject: [PATCH 108/221] [es8311] Encode the REG02 MCLK pre-multiplier instead of writing the factor (#20121) --- esphome/components/es8311/es8311.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/esphome/components/es8311/es8311.cpp b/esphome/components/es8311/es8311.cpp index 0386d84200..9a4da3744a 100644 --- a/esphome/components/es8311/es8311.cpp +++ b/esphome/components/es8311/es8311.cpp @@ -92,6 +92,21 @@ uint8_t ES8311::calculate_resolution_value(ES8311Resolution resolution) { } } +/// Encodes the MCLK pre-multiplier for REG02 bits 3-4. The coefficient table stores the multiplier as the factor +/// itself (1, 2, 4 or 8), while the register takes 0, 1, 2 or 3 (ES8311 datasheet, register 0x02 MULT_PRE). +static uint8_t encode_pre_mult(uint8_t pre_mult) { + switch (pre_mult) { + case 2: + return 1; + case 4: + return 2; + case 8: + return 3; + default: + return 0; + } +} + const ES8311Coefficient *ES8311::get_coefficient(uint32_t mclk, uint32_t rate) { for (const auto &coefficient : ES8311_COEFFICIENTS) { if (coefficient.mclk == mclk && coefficient.rate == rate) @@ -127,7 +142,7 @@ bool ES8311::configure_clock_() { ES8311_ERROR_CHECK(this->read_byte(ES8311_REG02_CLK_MANAGER, ®02)); reg02 &= 0x07; reg02 |= (coefficient->pre_div - 1) << 5; - reg02 |= coefficient->pre_mult << 3; + reg02 |= encode_pre_mult(coefficient->pre_mult) << 3; ES8311_ERROR_CHECK(this->write_byte(ES8311_REG02_CLK_MANAGER, reg02)); // Register 0x03 From 63cb75726b60365d8b68d9e0e22164e9bc94af9c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 13:06:05 -0500 Subject: [PATCH 109/221] [noise] Use the shared PROGMEM table helper for the PSK (#20101) --- esphome/codegen.py | 1 + esphome/components/api/__init__.py | 2 +- esphome/components/esphome/ota/__init__.py | 2 +- esphome/components/noise/__init__.py | 16 +++----- esphome/cpp_generator.py | 37 ++++++++++++++++++- tests/component_tests/ota/test_esphome_ota.py | 2 +- tests/unit_tests/test_cpp_generator.py | 37 +++++++++++++++++++ 7 files changed, 82 insertions(+), 15 deletions(-) diff --git a/esphome/codegen.py b/esphome/codegen.py index 4de1d8d5c1..daf53db43f 100644 --- a/esphome/codegen.py +++ b/esphome/codegen.py @@ -40,6 +40,7 @@ from esphome.cpp_generator import ( # noqa: F401 progmem_array, safe_exp, set_cpp_standard, + shared_progmem_array, statement, static_const_array, static_function, diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 40f520a057..25ebc93458 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -601,7 +601,7 @@ async def to_code(config: ConfigType) -> None: if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None: if key := encryption_config.get(CONF_KEY): - cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) + cg.add(var.set_noise_psk(new_psk_progmem(key))) cg.add_define("USE_API_NOISE_PSK_FROM_YAML") else: # No key provided, but encryption desired diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 6ecae24e29..ffd7c50949 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -360,7 +360,7 @@ async def to_code(config: ConfigType) -> None: # Build time key: the ota keeps its own pointer so safe mode, which # has no api server, still has it cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) + cg.add(var.set_noise_psk(new_psk_progmem(key))) elif CONF_ENCRYPTION in api_conf: # Runtime key: found in the api server, or in preferences in safe mode cg.add_define("USE_OTA_ENCRYPTION") diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 19ad36612d..d9880fe6f9 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -5,12 +5,11 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import CONF_ENCRYPTION, CONF_KEY -from esphome.core import CORE, ID +from esphome.core import CORE from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] -DOMAIN = "noise" # Keep in sync with platformio.ini and esphome/idf_component.yml. # LIBSODIUM_VERSION must match the version noise-c pins in its manifests. @@ -74,18 +73,13 @@ def static_encryption_key(conf: ConfigType) -> str | None: return (conf.get(CONF_ENCRYPTION) or {}).get(CONF_KEY) or None -def new_psk_progmem(parent_id: ID, key: str) -> MockObj: +def new_psk_progmem(key: str) -> MockObj: """Emit the decoded key as a PROGMEM array; the component keeps a pointer so the key never occupies RAM. Components sharing one key (api and ota) share the array.""" - decoded = decode_encryption_key(key) - arrays: dict[bytes, MockObj] = CORE.data.setdefault(DOMAIN, {}) - if (array := arrays.get(decoded)) is None: - array = arrays[decoded] = cg.progmem_array( - ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), - list(decoded), - ) - return array + return cg.shared_progmem_array( + "noise_psk", cg.uint8, list(decode_encryption_key(key)) + ) def encryption_schema(config: ConfigType | None) -> ConfigType: diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index b0c3533e04..a5ffaefe0b 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -19,7 +19,12 @@ from esphome.core import ( TimePeriodNanoseconds, TimePeriodSeconds, ) -from esphome.helpers import cpp_string_escape, indent, indent_all_but_first_and_last +from esphome.helpers import ( + cpp_string_escape, + ensure_unique_string, + indent, + indent_all_but_first_and_last, +) from esphome.types import Expression, SafeExpType, TemplateArgsType from esphome.util import OrderedDict from esphome.yaml_util import ESPHomeDataBase @@ -463,6 +468,36 @@ def progmem_array(id_, rhs) -> "MockObj": return obj +def shared_progmem_array( + name: str, type_: "MockObjClass", rhs: SafeExpType, *, share: bool = True +) -> "MockObj": + """Emit a global PROGMEM array once per distinct type and contents; later calls reuse it. + + The array is ``static constexpr``, so elements must be constant expressions and lambdas + must be captureless. Its name is made unique against every config id and variable. + ``share=False`` always emits a new array, e.g. for lambdas that may keep static state. + """ + from esphome.config import iter_ids + from esphome.config_validation import RESERVED_IDS + + arrays: dict[str, MockObj] = CORE.data.setdefault("shared_progmem_array", {}) + rhs = safe_exp(rhs) + key = f"{type_} {rhs}" + if share and (array := arrays.get(key)) is not None: + return array + used = {str(i) for i, _ in iter_ids(CORE.config)} + used |= {str(i) for i in CORE.variables} + used |= set(RESERVED_IDS) | CORE.loaded_integrations + id_ = ID(ensure_unique_string(name, used), is_declaration=True, type=type_) + # Global, so any scope can use it; anything a lambda references is already declared. + CORE.add_global(ProgmemAssignmentExpression(type_, id_, rhs)) + array = MockObj(id_, ".") + CORE.register_variable(id_, array) + if share: + arrays[key] = array + return array + + def static_const_array(id_, rhs) -> "MockObj": rhs = safe_exp(rhs) obj = MockObj(id_, ".") diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index cd66503c1b..77db149923 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -560,7 +560,7 @@ def test_encryption_offer_codegen( own_key = encrypted and "USE_OTA_ENCRYPTION_PROVISIONED" not in defines_present assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key # The api shares the ota's array instead of emitting the same key twice - assert main_cpp.count("_psk[] PROGMEM") == (1 if own_key else 0) + assert main_cpp.count("noise_psk[] PROGMEM") == (1 if own_key else 0) assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) # The noise transport source compiles only when the define is set assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) diff --git a/tests/unit_tests/test_cpp_generator.py b/tests/unit_tests/test_cpp_generator.py index fa6255b8c9..06b7c56bae 100644 --- a/tests/unit_tests/test_cpp_generator.py +++ b/tests/unit_tests/test_cpp_generator.py @@ -4,6 +4,7 @@ import math import pytest from esphome import cpp_generator as cg, cpp_types as ct +from esphome.core import CORE, ID class TestExpressions: @@ -808,3 +809,39 @@ async def test_templatable__lambda_with_std_string() -> None: result = await cg.templatable(lambda_obj, [], ct.std_string) assert isinstance(result, cg.LambdaExpression) + + +class TestSharedProgmemArray: + def test_identical_contents_share_one_array(self) -> None: + CORE.config = {} + a = cg.shared_progmem_array("table", ct.uint8, [1, 2, 3]) + b = cg.shared_progmem_array("table", ct.uint8, [1, 2, 3]) + c = cg.shared_progmem_array("table", ct.uint8, [4]) + assert a is b + assert str(a) != str(c) + assert sum("PROGMEM" in str(st) for st in CORE.global_statements) == 2 + assert not any("PROGMEM" in str(st) for st in CORE.main_statements) + + def test_share_false_always_emits_a_new_array(self) -> None: + CORE.config = {} + a = cg.shared_progmem_array("table", ct.uint8, [1], share=False) + b = cg.shared_progmem_array("table", ct.uint8, [1], share=False) + assert str(a) != str(b) + assert sum("PROGMEM" in str(st) for st in CORE.global_statements) == 2 + + def test_same_contents_different_type_are_separate(self) -> None: + CORE.config = {} + a = cg.shared_progmem_array("table", ct.uint8, [1]) + b = cg.shared_progmem_array("table", ct.uint16, [1]) + assert str(a) != str(b) + + def test_name_avoids_config_ids(self) -> None: + CORE.config = {"sensor": [{"id": ID("table", is_declaration=True)}]} + array = cg.shared_progmem_array("table", ct.uint8, [1]) + assert str(array) == "table_2" + + def test_name_avoids_registered_variables(self) -> None: + CORE.config = {} + CORE.register_variable(ID("table", is_declaration=True), cg.MockObj("table")) + array = cg.shared_progmem_array("table", ct.uint8, [1]) + assert str(array) == "table_2" From c6e3a5a16560d5ca2c291b4047cb66ace4bdca55 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 14:37:12 -0500 Subject: [PATCH 110/221] [voice_assistant] Group the one byte members at the end (#20147) --- .../voice_assistant/voice_assistant.h | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index e99ee07099..ae5eee3e22 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -44,7 +44,7 @@ enum VoiceAssistantFeature : uint32_t { FEATURE_MULTI_CHANNEL_AUDIO = 1 << 6, }; -enum class State { +enum class State : uint8_t { IDLE, START_MICROPHONE, STARTING_MICROPHONE, @@ -96,7 +96,7 @@ struct Configuration { }; #ifdef USE_MEDIA_PLAYER -enum class MediaPlayerResponseState { +enum class MediaPlayerResponseState : uint8_t { IDLE, URL_SENT, PLAYING, @@ -237,6 +237,7 @@ class VoiceAssistant final : public Component { void set_state_(State state); void set_state_(State state, State desired_state); + bool start_udp_socket_(); void signal_stop_(); void start_playback_timeout_(); @@ -280,8 +281,6 @@ class VoiceAssistant final : public Component { Trigger timer_updated_trigger_; Trigger timer_cancelled_trigger_; Trigger &> timer_tick_trigger_; - bool has_timers_{false}; - bool timer_tick_running_{false}; microphone::MicrophoneSource *mic_source_{nullptr}; microphone::MicrophoneSource *mic_source2_{nullptr}; @@ -292,19 +291,12 @@ class VoiceAssistant final : public Component { size_t speaker_buffer_index_{0}; size_t speaker_buffer_size_{0}; size_t speaker_bytes_received_{0}; - bool wait_for_stream_end_{false}; - bool stream_ended_{false}; #endif #ifdef USE_MEDIA_PLAYER media_player::MediaPlayer *media_player_{nullptr}; std::string tts_response_url_; - bool started_streaming_tts_{false}; - - MediaPlayerResponseState media_player_response_state_{MediaPlayerResponseState::IDLE}; #endif - bool local_output_{false}; - std::string conversation_id_; std::string wake_word_; @@ -324,29 +316,37 @@ class VoiceAssistant final : public Component { // prolonged one can be detected and stopped; 0 means no imbalance is currently being timed. uint32_t audio_channel_stall_start_{0}; - bool use_wake_word_; - uint8_t noise_suppression_level_; - uint8_t auto_gain_; float volume_multiplier_; uint32_t conversation_timeout_; - bool continuous_{false}; - bool silence_detection_; - - bool continue_conversation_{false}; - - State state_{State::IDLE}; - State desired_state_{State::IDLE}; - - AudioMode audio_mode_{AUDIO_MODE_UDP}; - bool udp_socket_running_{false}; - bool start_udp_socket_(); - Configuration config_{}; #ifdef USE_MICRO_WAKE_WORD micro_wake_word::MicroWakeWord *micro_wake_word_{nullptr}; #endif + + // 1 byte members grouped at the end so they share padding instead of each taking a word + State state_{State::IDLE}; + State desired_state_{State::IDLE}; + AudioMode audio_mode_{AUDIO_MODE_UDP}; + bool has_timers_{false}; + bool timer_tick_running_{false}; +#ifdef USE_SPEAKER + bool wait_for_stream_end_{false}; + bool stream_ended_{false}; +#endif +#ifdef USE_MEDIA_PLAYER + MediaPlayerResponseState media_player_response_state_{MediaPlayerResponseState::IDLE}; + bool started_streaming_tts_{false}; +#endif + bool local_output_{false}; + bool use_wake_word_; + uint8_t noise_suppression_level_; + uint8_t auto_gain_; + bool continuous_{false}; + bool silence_detection_; + bool continue_conversation_{false}; + bool udp_socket_running_{false}; }; extern VoiceAssistant *global_voice_assistant; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) From 4b5375145ae725010defedad1f435d6bffb1ca5f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 14:38:05 -0500 Subject: [PATCH 111/221] [microphone] Move bits_per_sample_ to the end of MicrophoneSource (#20146) --- esphome/components/microphone/microphone_source.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/microphone/microphone_source.h b/esphome/components/microphone/microphone_source.h index d7a3352432..770399fef5 100644 --- a/esphome/components/microphone/microphone_source.h +++ b/esphome/components/microphone/microphone_source.h @@ -34,7 +34,7 @@ class MicrophoneSource final { */ public: MicrophoneSource(Microphone *mic, uint8_t bits_per_sample, int32_t gain_factor, bool passive) - : mic_(mic), bits_per_sample_(bits_per_sample), gain_factor_(gain_factor), passive_(passive) {} + : mic_(mic), gain_factor_(gain_factor), passive_(passive), bits_per_sample_(bits_per_sample) {} /// @brief Enables a channel to be processed through the callback. /// @@ -81,11 +81,11 @@ class MicrophoneSource final { std::shared_ptr> processed_samples_; Microphone *mic_; - uint8_t bits_per_sample_; std::bitset<8> channels_; int32_t gain_factor_; bool enabled_{false}; bool passive_; // Only pass audio if ``mic_`` is already running + uint8_t bits_per_sample_; }; } // namespace esphome::microphone From bccdbc23c4149f0f56d54d5f5e7d866b5d9987df Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 14:38:12 -0500 Subject: [PATCH 112/221] [mixer] Pack has_contributed_ next to pause_state_ (#20144) --- esphome/components/mixer/speaker/mixer_speaker.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/mixer/speaker/mixer_speaker.h b/esphome/components/mixer/speaker/mixer_speaker.h index 494443d695..04b0cb94f4 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.h +++ b/esphome/components/mixer/speaker/mixer_speaker.h @@ -107,12 +107,12 @@ class SourceSpeaker final : public speaker::Speaker, public Component { bool stop_gracefully_{false}; bool pause_state_{false}; + std::atomic has_contributed_{false}; // Tracks if source has contributed during this session esp_audio_libs::gain::GainRamp ducking_ramp_; std::atomic pending_playback_frames_{0}; std::atomic playback_delay_frames_{0}; // Frames in output pipeline when this source started contributing - std::atomic has_contributed_{false}; // Tracks if source has contributed during this session EventGroupHandle_t event_group_{nullptr}; uint32_t stopping_start_ms_{0}; From 2ee13d59acaac3cf301218e018e28afca36a1634 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 14:38:32 -0500 Subject: [PATCH 113/221] [speaker] Pack mute_state_ next to state_ (#20143) --- esphome/components/speaker/speaker.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/speaker/speaker.h b/esphome/components/speaker/speaker.h index 01e9ca042e..277b36a171 100644 --- a/esphome/components/speaker/speaker.h +++ b/esphome/components/speaker/speaker.h @@ -127,9 +127,9 @@ class Speaker { #endif State state_{STATE_STOPPED}; + bool mute_state_{false}; audio::AudioStreamInfo audio_stream_info_; float volume_{1.0f}; - bool mute_state_{false}; #ifdef USE_AUDIO_DAC audio_dac::AudioDac *audio_dac_{nullptr}; From d0473438d973a4cf4c45876b8183321fab66c007 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 14:38:54 -0500 Subject: [PATCH 114/221] [i2s_audio] Pack pdm_ with the other microphone bools (#20145) --- esphome/components/i2s_audio/microphone/i2s_audio_microphone.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h index 37895ac4e7..1b17380ed8 100644 --- a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h +++ b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.h @@ -60,11 +60,11 @@ class I2SAudioMicrophone final : public I2SAudioIn, public microphone::Microphon gpio_num_t din_pin_{I2S_GPIO_UNUSED}; i2s_chan_handle_t rx_handle_; - bool pdm_{false}; #if SOC_I2S_SUPPORTS_PDM_RX i2s_pdm_dsr_t pdm_dsr_{I2S_PDM_DSR_8S}; #endif + bool pdm_{false}; bool correct_dc_offset_; bool locked_driver_{false}; int32_t dc_offset_prev_input_{0}; From 84435e7d81007ab26580ccd6cd83e0c3a082e624 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 14:39:20 -0500 Subject: [PATCH 115/221] [sendspin] Pack mdns_advertised_ next to task_stack_in_psram_ (#20148) --- esphome/components/sendspin/sendspin_hub.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esphome/components/sendspin/sendspin_hub.h b/esphome/components/sendspin/sendspin_hub.h index 8ab70d61ef..ea09dafe68 100644 --- a/esphome/components/sendspin/sendspin_hub.h +++ b/esphome/components/sendspin/sendspin_hub.h @@ -308,6 +308,9 @@ class SendspinHub final : public Component, CallbackManager group_update_callbacks_{}; bool task_stack_in_psram_{false}; +#ifdef USE_MDNS_SUPPORTS_ENABLE_DISABLE + bool mdns_advertised_{false}; // Last state requested from mdns +#endif // Requested client state, applied from loop(). Empty until the switch restores its state. std::optional enabled_; @@ -320,7 +323,6 @@ class SendspinHub final : public Component, #ifdef USE_MDNS_SUPPORTS_ENABLE_DISABLE mdns::MDNSComponent *mdns_{nullptr}; - bool mdns_advertised_{false}; // Last state requested from mdns #endif }; From fb8f679be96fa3915a28ab213d26dc1ba5fd2848 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 14:40:57 -0500 Subject: [PATCH 116/221] [ring_buffer] Use the static ring buffer structure as its handle (#20134) --- .../components/ring_buffer/ring_buffer.cpp | 40 +++++++++++-------- esphome/components/ring_buffer/ring_buffer.h | 4 +- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/esphome/components/ring_buffer/ring_buffer.cpp b/esphome/components/ring_buffer/ring_buffer.cpp index 9604290cf0..dfcc53ebb2 100644 --- a/esphome/components/ring_buffer/ring_buffer.cpp +++ b/esphome/components/ring_buffer/ring_buffer.cpp @@ -10,8 +10,9 @@ namespace esphome::ring_buffer { static const char *const TAG = "ring_buffer"; RingBuffer::~RingBuffer() { - if (this->handle_ != nullptr) { - vRingbufferDelete(this->handle_); + // create() only builds the ring buffer once storage_ is allocated + if (this->storage_ != nullptr) { + vRingbufferDelete(this->handle_()); RAMAllocator allocator; allocator.deallocate(this->storage_, this->size_); } @@ -31,7 +32,12 @@ std::unique_ptr RingBuffer::create(size_t len, MemoryPreference pref return nullptr; } - rb->handle_ = xRingbufferCreateStatic(rb->size_, RINGBUF_TYPE_BYTEBUF, rb->storage_, &rb->structure_); + // handle_() assumes the returned handle is &structure_; NULL means the arguments were rejected + if (xRingbufferCreateStatic(rb->size_, RINGBUF_TYPE_BYTEBUF, rb->storage_, &rb->structure_) != rb->handle_()) { + allocator.deallocate(rb->storage_, rb->size_); + rb->storage_ = nullptr; + return nullptr; + } ESP_LOGD(TAG, "Created ring buffer with size %u", len); return rb; @@ -39,16 +45,16 @@ std::unique_ptr RingBuffer::create(size_t len, MemoryPreference pref void *RingBuffer::receive_acquire(size_t &length, size_t max_length, TickType_t ticks_to_wait) { length = 0; - void *buffer_data = xRingbufferReceiveUpTo(this->handle_, &length, ticks_to_wait, max_length); + void *buffer_data = xRingbufferReceiveUpTo(this->handle_(), &length, ticks_to_wait, max_length); return buffer_data; } -void RingBuffer::receive_release(void *item) { vRingbufferReturnItem(this->handle_, item); } +void RingBuffer::receive_release(void *item) { vRingbufferReturnItem(this->handle_(), item); } size_t RingBuffer::read(void *data, size_t len, TickType_t ticks_to_wait) { size_t bytes_read = 0; - void *buffer_data = xRingbufferReceiveUpTo(this->handle_, &bytes_read, ticks_to_wait, len); + void *buffer_data = xRingbufferReceiveUpTo(this->handle_(), &bytes_read, ticks_to_wait, len); if (buffer_data == nullptr) { return 0; @@ -56,14 +62,14 @@ size_t RingBuffer::read(void *data, size_t len, TickType_t ticks_to_wait) { std::memcpy(data, buffer_data, bytes_read); - vRingbufferReturnItem(this->handle_, buffer_data); + vRingbufferReturnItem(this->handle_(), buffer_data); if (bytes_read < len) { // Data may have wrapped around, so read a second time to receive the remainder size_t follow_up_bytes_read = 0; size_t bytes_remaining = len - bytes_read; - buffer_data = xRingbufferReceiveUpTo(this->handle_, &follow_up_bytes_read, 0, bytes_remaining); + buffer_data = xRingbufferReceiveUpTo(this->handle_(), &follow_up_bytes_read, 0, bytes_remaining); if (buffer_data == nullptr) { return bytes_read; @@ -71,7 +77,7 @@ size_t RingBuffer::read(void *data, size_t len, TickType_t ticks_to_wait) { std::memcpy((void *) ((uint8_t *) (data) + bytes_read), buffer_data, follow_up_bytes_read); - vRingbufferReturnItem(this->handle_, buffer_data); + vRingbufferReturnItem(this->handle_(), buffer_data); bytes_read += follow_up_bytes_read; } @@ -89,13 +95,13 @@ size_t RingBuffer::write(const void *data, size_t len) { size_t RingBuffer::write_without_replacement(const void *data, size_t len, TickType_t ticks_to_wait, bool write_partial) { - if (!xRingbufferSend(this->handle_, data, len, ticks_to_wait)) { + if (!xRingbufferSend(this->handle_(), data, len, ticks_to_wait)) { if (!write_partial) { return 0; // Not enough space available and not allowed to write partial data } // Couldn't fit all the data, write what will fit size_t free = std::min(this->free(), len); - if (xRingbufferSend(this->handle_, data, free, 0)) { + if (xRingbufferSend(this->handle_(), data, free, 0)) { return free; } return 0; @@ -105,11 +111,11 @@ size_t RingBuffer::write_without_replacement(const void *data, size_t len, TickT size_t RingBuffer::available() const { UBaseType_t ux_items_waiting = 0; - vRingbufferGetInfo(this->handle_, nullptr, nullptr, nullptr, nullptr, &ux_items_waiting); + vRingbufferGetInfo(this->handle_(), nullptr, nullptr, nullptr, nullptr, &ux_items_waiting); return ux_items_waiting; } -size_t RingBuffer::free() const { return xRingbufferGetCurFreeSize(this->handle_); } +size_t RingBuffer::free() const { return xRingbufferGetCurFreeSize(this->handle_()); } BaseType_t RingBuffer::reset() { // Discards all the available data @@ -119,15 +125,15 @@ BaseType_t RingBuffer::reset() { bool RingBuffer::discard_bytes_(size_t discard_bytes) { size_t bytes_read = 0; - void *buffer_data = xRingbufferReceiveUpTo(this->handle_, &bytes_read, 0, discard_bytes); + void *buffer_data = xRingbufferReceiveUpTo(this->handle_(), &bytes_read, 0, discard_bytes); if (buffer_data != nullptr) - vRingbufferReturnItem(this->handle_, buffer_data); + vRingbufferReturnItem(this->handle_(), buffer_data); if (bytes_read < discard_bytes) { size_t wrapped_bytes_read = 0; - buffer_data = xRingbufferReceiveUpTo(this->handle_, &wrapped_bytes_read, 0, discard_bytes - bytes_read); + buffer_data = xRingbufferReceiveUpTo(this->handle_(), &wrapped_bytes_read, 0, discard_bytes - bytes_read); if (buffer_data != nullptr) { - vRingbufferReturnItem(this->handle_, buffer_data); + vRingbufferReturnItem(this->handle_(), buffer_data); bytes_read += wrapped_bytes_read; } } diff --git a/esphome/components/ring_buffer/ring_buffer.h b/esphome/components/ring_buffer/ring_buffer.h index 62094899d7..ed2d4b4345 100644 --- a/esphome/components/ring_buffer/ring_buffer.h +++ b/esphome/components/ring_buffer/ring_buffer.h @@ -115,7 +115,9 @@ class RingBuffer { /// @return True if all bytes were successfully discarded, false otherwise bool discard_bytes_(size_t discard_bytes); - RingbufHandle_t handle_{nullptr}; + // xRingbufferCreateStatic() returns the static structure itself as the handle + RingbufHandle_t handle_() const { return const_cast(&this->structure_); } + StaticRingbuffer_t structure_; uint8_t *storage_{nullptr}; size_t size_{0}; From aad7907c37d03b10186d109b9ff653713481053f Mon Sep 17 00:00:00 2001 From: guillempages Date: Sun, 4 Oct 2026 21:55:55 +0200 Subject: [PATCH 117/221] [http_request] Reduce log level for missing header (#18932) --- esphome/components/http_request/http_request.cpp | 3 +-- esphome/components/http_request/http_request.h | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/esphome/components/http_request/http_request.cpp b/esphome/components/http_request/http_request.cpp index d45208ed5d..37e0de5d6a 100644 --- a/esphome/components/http_request/http_request.cpp +++ b/esphome/components/http_request/http_request.cpp @@ -25,11 +25,10 @@ std::string HttpContainer::get_response_header(const std::string &header_name) { auto lower = str_lower_case(header_name); // NOLINT for (const auto &entry : this->response_headers_) { if (entry.name == lower) { - ESP_LOGD(TAG, "Header with name %s found with value %s", lower.c_str(), entry.value.c_str()); + ESP_LOGV(TAG, "Header with name %s found with value %s", lower.c_str(), entry.value.c_str()); return entry.value; } } - ESP_LOGW(TAG, "No header with name %s found", lower.c_str()); return ""; } diff --git a/esphome/components/http_request/http_request.h b/esphome/components/http_request/http_request.h index 71668b8556..68696118ea 100644 --- a/esphome/components/http_request/http_request.h +++ b/esphome/components/http_request/http_request.h @@ -268,6 +268,11 @@ class HttpContainer : public Parented { return !this->is_chunked_ && this->bytes_read_ >= this->content_length; } + /** + * @brief Get the value of a response header by name. + * @param header_name The name of the header to retrieve. + * @return The value of the header if found, otherwise an empty string. + */ std::string get_response_header(const std::string &header_name); protected: From 4f539d33916c2f2bf5eb41a7848a5db88f796583 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 5 Oct 2026 08:56:34 +1300 Subject: [PATCH 118/221] [homeassistant] Use input_number.set_value for input_number entities (#19965) --- .../homeassistant/number/__init__.py | 8 +++- .../number/homeassistant_number.cpp | 13 ++++++- .../component_tests/homeassistant/__init__.py | 0 .../homeassistant/test_number.py | 32 ++++++++++++++++ .../homeassistant/test_number.yaml | 17 +++++++++ .../test_number_invalid_domain.yaml | 17 +++++++++ tests/components/homeassistant/common.yaml | 3 ++ .../fixtures/api_homeassistant.yaml | 13 +++++++ tests/integration/test_api_homeassistant.py | 38 +++++++++++++++++++ 9 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/homeassistant/__init__.py create mode 100644 tests/component_tests/homeassistant/test_number.py create mode 100644 tests/component_tests/homeassistant/test_number.yaml create mode 100644 tests/component_tests/homeassistant/test_number_invalid_domain.yaml diff --git a/esphome/components/homeassistant/number/__init__.py b/esphome/components/homeassistant/number/__init__.py index ab1389e13a..9658d3dc84 100644 --- a/esphome/components/homeassistant/number/__init__.py +++ b/esphome/components/homeassistant/number/__init__.py @@ -7,19 +7,23 @@ from .. import ( HOME_ASSISTANT_IMPORT_CONTROL_SCHEMA, homeassistant_ns, setup_home_assistant_entity, + validate_entity_domain, ) CODEOWNERS = ["@landonr"] DEPENDENCIES = ["api"] +SUPPORTED_DOMAINS = ["input_number", "number"] + HomeassistantNumber = homeassistant_ns.class_( "HomeassistantNumber", number.Number, cg.Component ) -CONFIG_SCHEMA = ( +CONFIG_SCHEMA = cv.All( number.number_schema(HomeassistantNumber) .extend(HOME_ASSISTANT_IMPORT_CONTROL_SCHEMA) - .extend(cv.COMPONENT_SCHEMA) + .extend(cv.COMPONENT_SCHEMA), + validate_entity_domain("number", SUPPORTED_DOMAINS), ) diff --git a/esphome/components/homeassistant/number/homeassistant_number.cpp b/esphome/components/homeassistant/number/homeassistant_number.cpp index 965f91d202..553c2466f8 100644 --- a/esphome/components/homeassistant/number/homeassistant_number.cpp +++ b/esphome/components/homeassistant/number/homeassistant_number.cpp @@ -1,5 +1,7 @@ #include "homeassistant_number.h" +#include + #include "esphome/components/api/api_pb2.h" #include "esphome/components/api/api_server.h" #include "esphome/core/log.h" @@ -80,12 +82,19 @@ void HomeassistantNumber::control(float value) { this->publish_state(value); - static constexpr auto SERVICE_NAME = StringRef::from_lit("number.set_value"); + static constexpr auto NUMBER_SERVICE_NAME = StringRef::from_lit("number.set_value"); + static constexpr auto INPUT_NUMBER_SERVICE_NAME = StringRef::from_lit("input_number.set_value"); + static constexpr char INPUT_PREFIX[] = "input_"; static constexpr auto ENTITY_ID_KEY = StringRef::from_lit("entity_id"); static constexpr auto VALUE_KEY = StringRef::from_lit("value"); api::HomeassistantActionRequest resp; - resp.service = SERVICE_NAME; + // input_number entities have their own set_value action + if (strncmp(this->entity_id_, INPUT_PREFIX, sizeof(INPUT_PREFIX) - 1) == 0) { + resp.service = INPUT_NUMBER_SERVICE_NAME; + } else { + resp.service = NUMBER_SERVICE_NAME; + } resp.data.init(2); auto &entity_id = resp.data.emplace_back(); diff --git a/tests/component_tests/homeassistant/__init__.py b/tests/component_tests/homeassistant/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/homeassistant/test_number.py b/tests/component_tests/homeassistant/test_number.py new file mode 100644 index 0000000000..d17842307d --- /dev/null +++ b/tests/component_tests/homeassistant/test_number.py @@ -0,0 +1,32 @@ +"""Tests for the homeassistant number platform.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.config import read_config +from esphome.core import CORE + +VALID_CONFIG = "tests/component_tests/homeassistant/test_number.yaml" +INVALID_CONFIG = "tests/component_tests/homeassistant/test_number_invalid_domain.yaml" + + +def test_number_and_input_number_entities( + generate_main: Callable[[str | Path], str], +) -> None: + """Both number and input_number entity IDs are accepted and passed to the C++ class.""" + main_cpp = generate_main(VALID_CONFIG) + + assert 'ha_number->set_entity_id("number.some_number");' in main_cpp + assert 'ha_input_number->set_entity_id("input_number.some_input");' in main_cpp + + +def test_number_rejects_unsupported_domain( + capsys: pytest.CaptureFixture[str], +) -> None: + """An entity ID from another domain fails config validation.""" + CORE.config_path = Path(INVALID_CONFIG) + + assert read_config({}) is None + assert "is not supported by the number platform" in capsys.readouterr().out diff --git a/tests/component_tests/homeassistant/test_number.yaml b/tests/component_tests/homeassistant/test_number.yaml new file mode 100644 index 0000000000..bcd8365495 --- /dev/null +++ b/tests/component_tests/homeassistant/test_number.yaml @@ -0,0 +1,17 @@ +--- +esphome: + name: test + +host: + +logger: + +api: + +number: + - platform: homeassistant + entity_id: number.some_number + id: ha_number + - platform: homeassistant + entity_id: input_number.some_input + id: ha_input_number diff --git a/tests/component_tests/homeassistant/test_number_invalid_domain.yaml b/tests/component_tests/homeassistant/test_number_invalid_domain.yaml new file mode 100644 index 0000000000..684de60620 --- /dev/null +++ b/tests/component_tests/homeassistant/test_number_invalid_domain.yaml @@ -0,0 +1,17 @@ +--- +esphome: + name: test + +host: + +logger: + +api: + +number: + - platform: homeassistant + entity_id: number.some_number + id: ha_number + - platform: homeassistant + entity_id: sensor.foo + id: ha_other_number diff --git a/tests/components/homeassistant/common.yaml b/tests/components/homeassistant/common.yaml index 1099f7ea85..9bc0defae2 100644 --- a/tests/components/homeassistant/common.yaml +++ b/tests/components/homeassistant/common.yaml @@ -71,6 +71,9 @@ number: - platform: homeassistant entity_id: number.hello_world id: ha_hello_world_number + - platform: homeassistant + entity_id: input_number.hello_world + id: ha_hello_world_input_number sensor: - platform: homeassistant diff --git a/tests/integration/fixtures/api_homeassistant.yaml b/tests/integration/fixtures/api_homeassistant.yaml index 2d77821ff3..deea19bdeb 100644 --- a/tests/integration/fixtures/api_homeassistant.yaml +++ b/tests/integration/fixtures/api_homeassistant.yaml @@ -21,6 +21,12 @@ api: - number.set: id: ha_number value: 42.5 + - number.set: + id: ha_real_number + value: 12.5 + - number.set: + id: ha_real_input_number + value: 7 - switch.turn_on: ha_switch - switch.turn_off: ha_switch - logger.log: "=== All tests completed ===" @@ -129,6 +135,13 @@ text_sensor: # Number component for testing HA number control number: + # Real homeassistant numbers: the action used depends on the entity domain + - platform: homeassistant + id: ha_real_number + entity_id: number.test_real_number + - platform: homeassistant + id: ha_real_input_number + entity_id: input_number.test_real_input_number - platform: template name: "HA Controlled Number" id: ha_number diff --git a/tests/integration/test_api_homeassistant.py b/tests/integration/test_api_homeassistant.py index b4adedf873..bb99b41813 100644 --- a/tests/integration/test_api_homeassistant.py +++ b/tests/integration/test_api_homeassistant.py @@ -86,6 +86,15 @@ async def test_api_homeassistant( "nonexistent.action_for_error_test": loop.create_future(), # error_test_call } + # Futures keyed by (service, entity_id) for the real homeassistant numbers + ha_real_number_futures = { + ("number.set_value", "number.test_real_number"): loop.create_future(), + ( + "input_number.set_value", + "input_number.test_real_input_number", + ): loop.create_future(), + } + # Future for error message test action_error_received_future = loop.create_future() @@ -102,6 +111,10 @@ async def test_api_homeassistant( if not future.done(): future.set_result(service_call) + key = (service_call.service, service_call.data.get("entity_id")) + if key in ha_real_number_futures and not ha_real_number_futures[key].done(): + ha_real_number_futures[key].set_result(service_call) + # Immediately respond to the error test call so the test can proceed # This needs to happen synchronously so ESPHome receives the response # before logging "=== All tests completed ===" @@ -201,6 +214,16 @@ async def test_api_homeassistant( # Empty state with valid entity should work (use different entity to not interfere with test) client.send_home_assistant_state("sensor.edge_case_empty_state", "", "") + # The real homeassistant numbers reject values outside min/max, which + # default to 0 until Home Assistant provides them + for entity in ( + "number.test_real_number", + "input_number.test_real_input_number", + ): + client.send_home_assistant_state(entity, "min", "0") + client.send_home_assistant_state(entity, "max", "100") + client.send_home_assistant_state(entity, "step", "0.5") + # List entities and services _, services = await client.list_entities_services() @@ -334,6 +357,21 @@ async def test_api_homeassistant( # The value might be formatted with trailing zeros assert float(number_call.data["value"]) == 42.5 + # 7b. Real homeassistant numbers pick the action from the entity domain + real_number_call = await asyncio.wait_for( + ha_real_number_futures[("number.set_value", "number.test_real_number")], + timeout=2.0, + ) + assert float(real_number_call.data["value"]) == 12.5 + + real_input_number_call = await asyncio.wait_for( + ha_real_number_futures[ + ("input_number.set_value", "input_number.test_real_input_number") + ], + timeout=2.0, + ) + assert float(real_input_number_call.data["value"]) == 7 + # 8. HA Switch service calls switch_on_call = await asyncio.wait_for( service_call_futures["switch.turn_on"], timeout=2.0 From 52fc5f2c75dc4e0534d05ed7d3cbcb5c756adbaf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 15:25:25 -0500 Subject: [PATCH 119/221] [ld2450] Pack direction_dedup_ after zone_config_ (#20152) --- esphome/components/ld2450/ld2450.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esphome/components/ld2450/ld2450.h b/esphome/components/ld2450/ld2450.h index c4f06ad224..d6285cce8a 100644 --- a/esphome/components/ld2450/ld2450.h +++ b/esphome/components/ld2450/ld2450.h @@ -176,6 +176,10 @@ class LD2450Component : public Component, public uart::UARTDevice { bool bluetooth_on_{false}; Target target_info_[MAX_TARGETS]; Zone zone_config_[MAX_ZONES]; +#ifdef USE_TEXT_SENSOR + // Byte aligned, so it packs after zone_config_ instead of padding before data_callback_ + std::array, MAX_TARGETS> direction_dedup_{}; +#endif #ifdef USE_NUMBER ESPPreferenceObject pref_; // only used when numbers are in use @@ -194,7 +198,6 @@ class LD2450Component : public Component, public uart::UARTDevice { #endif #ifdef USE_TEXT_SENSOR std::array direction_text_sensors_{}; - std::array, MAX_TARGETS> direction_dedup_{}; #endif LazyCallbackManager data_callback_; From 7421b165e831e081488af93b87d234d2f4811d23 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 15:25:28 -0500 Subject: [PATCH 120/221] [rtttl] Pack state_ after wholenote_duration_ (#20151) --- esphome/components/rtttl/rtttl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/rtttl/rtttl.h b/esphome/components/rtttl/rtttl.h index 7a32b79b17..055204571e 100644 --- a/esphome/components/rtttl/rtttl.h +++ b/esphome/components/rtttl/rtttl.h @@ -82,14 +82,14 @@ class Rtttl final : public Component { uint16_t note_duration_{0}; /// The duration of a whole note in milliseconds. uint16_t wholenote_duration_; + /// The current state of the RTTTL player. + State state_{State::STOPPED}; /// The time in milliseconds since microcontroller boot when the last note was started. uint32_t last_note_start_time_; /// The frequency of the current note in Hz. uint32_t output_freq_{0}; /// The gain of the output. float gain_{0.6f}; - /// The current state of the RTTTL player. - State state_{State::STOPPED}; #ifdef USE_OUTPUT /// The output to write the sound to. From 474c2791fe757414b534f3b94cd037115a040137 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 15:25:35 -0500 Subject: [PATCH 121/221] [ledc] Pack initialized_ next to bit_depth_ (#20150) --- esphome/components/ledc/ledc_output.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/ledc/ledc_output.h b/esphome/components/ledc/ledc_output.h index de63325455..1df0f67572 100644 --- a/esphome/components/ledc/ledc_output.h +++ b/esphome/components/ledc/ledc_output.h @@ -35,11 +35,11 @@ class LEDCOutput final : public output::FloatOutput, public Component { InternalGPIOPin *pin_; uint8_t channel_{}; uint8_t bit_depth_{}; + bool initialized_ = false; float phase_angle_{0.0f}; float frequency_{}; float duty_{0.0f}; uint32_t last_duty_{UINT32_MAX}; - bool initialized_ = false; }; } // namespace esphome::ledc From fe2e541dc80b8cad4f5a4bfc9d4dcf22a46b1c50 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 15:26:21 -0500 Subject: [PATCH 122/221] [template] Pack restore_value_ next to optimistic_ in TemplateNumber (#20149) --- esphome/components/template/number/template_number.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/template/number/template_number.h b/esphome/components/template/number/template_number.h index e51e858ccf..2c4932f7e8 100644 --- a/esphome/components/template/number/template_number.h +++ b/esphome/components/template/number/template_number.h @@ -25,8 +25,8 @@ class TemplateNumber final : public number::Number, public PollingComponent { protected: void control(float value) override; bool optimistic_{false}; - float initial_value_{NAN}; bool restore_value_{false}; + float initial_value_{NAN}; Trigger set_trigger_; TemplateLambda f_; From 2f9e37d6a973dc3e65cfe5ae58cff5a1bfce7abf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:00:23 -0500 Subject: [PATCH 123/221] [wifi] Let the empty pending_ flags struct take no space (#20142) --- esphome/components/wifi/wifi_component.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 6791379649..6f03bbe594 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -983,7 +983,8 @@ class WiFiComponent final : public Component { // Bools and bitfields // Pending listener callbacks deferred from platform callbacks to main loop. - struct { + // Empty when no listener needs deferring (e.g. ESP32 without connect state listeners) + [[no_unique_address]] struct { #ifdef USE_WIFI_CONNECT_STATE_LISTENERS // Deferred until state machine reaches STA_CONNECTED so wifi.connected // condition returns true in listener automations. From 3a06318b6585321bd479e15edd7cdce91ca55c97 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:02:18 -0500 Subject: [PATCH 124/221] [template] Use distinct empty types so both unused select members take no space (#20141) --- esphome/components/template/select/template_select.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/template/select/template_select.h b/esphome/components/template/select/template_select.h index 1cc28a36d3..c6a1b8bd08 100644 --- a/esphome/components/template/select/template_select.h +++ b/esphome/components/template/select/template_select.h @@ -9,7 +9,8 @@ namespace esphome::template_ { -struct Empty {}; +// Distinct per member: two empty members of the same type cannot share an address +template struct Empty {}; class BaseTemplateSelect : public select::Select, public PollingComponent {}; void dump_config_helper(BaseTemplateSelect *sel_comp, bool optimistic, bool has_lambda, size_t initial_option_index, @@ -59,8 +60,8 @@ class TemplateSelect : public BaseTemplateSelect { if constexpr (RESTORE_VALUE) this->pref_.save(&index); } - [[no_unique_address]] std::conditional_t, Empty> f_{}; - [[no_unique_address]] std::conditional_t pref_{}; + [[no_unique_address]] std::conditional_t, Empty<0>> f_{}; + [[no_unique_address]] std::conditional_t> pref_{}; }; /// Template select with set_action trigger - only instantiated when set_action is configured From a1a4474a019ea2712f3a8660ac67de9f4654a9bc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:02:24 -0500 Subject: [PATCH 125/221] [script] Let the empty argument tuple take no space for scripts without parameters (#20140) --- esphome/components/script/script.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/script/script.h b/esphome/components/script/script.h index b9ea75e0bc..97b27af498 100644 --- a/esphome/components/script/script.h +++ b/esphome/components/script/script.h @@ -254,7 +254,8 @@ template class ScriptExecuteAction, T } Script *script_; - Args args_; + // Empty tuple for scripts without parameters; takes no space then + [[no_unique_address]] Args args_; }; /** Wait for a script to finish before continuing. From 9b83f37b786b7136516c4df80bd9d0e7ee28167d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:02:27 -0500 Subject: [PATCH 126/221] [core] Let the empty Scheduler mutex take no space on single-threaded platforms (#20139) --- esphome/core/scheduler.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 56fc83f12f..4911bdde74 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -518,7 +518,8 @@ class Scheduler { uint32_t hash_or_id, SchedulerItem::Type type, bool find_first); - Mutex lock_; + // Empty on single-threaded platforms (ESP8266, RP2), so it takes no space there + [[no_unique_address]] Mutex lock_; std::vector items_; std::vector to_add_; From c4aa8fa9c5e58423a94deb41090df0d8072bba9f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:02:40 -0500 Subject: [PATCH 127/221] [wifi] Place ap_ before the byte fields so they pack after it (#20138) --- esphome/components/wifi/wifi_component.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 6f03bbe594..5a0a4d2c7e 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -874,9 +874,6 @@ class WiFiComponent final : public Component { #ifdef WIFI_SCAN_RESULTS_LOCK_ENABLED Mutex scan_result_lock_; #endif -#ifdef USE_WIFI_AP - WiFiAP ap_; -#endif #ifdef USE_WIFI_IP_STATE_LISTENERS StaticVector ip_state_listeners_; #endif @@ -924,6 +921,9 @@ class WiFiComponent final : public Component { uint32_t roaming_scan_end_{0}; // Timestamp when last roaming scan completed #ifdef USE_WIFI_AP uint32_t ap_timeout_{90000}; // Keep in sync with DEFAULT_AP_TIMEOUT in __init__.py + // WiFiAP is byte aligned unless manual IP or EAP is enabled; placed before the + // 1-byte members so they pack into its trailing bytes instead of padding after it + WiFiAP ap_; #endif // 1-byte enums and integers From 0dc10d9f30a351dc65a91761a9d02bb90c621e0e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:04:44 -0500 Subject: [PATCH 128/221] [improv_ble] Pack the byte fields into the padding after connecting_sta_ (#20137) --- .../components/improv_ble/improv_ble_component.h | 16 ++++++++-------- esphome/components/wifi/wifi_component.h | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/esphome/components/improv_ble/improv_ble_component.h b/esphome/components/improv_ble/improv_ble_component.h index 2552bed69b..126663ed00 100644 --- a/esphome/components/improv_ble/improv_ble_component.h +++ b/esphome/components/improv_ble/improv_ble_component.h @@ -72,9 +72,6 @@ class ImprovBLEComponent final : public Component, public improv_base::ImprovBas improv::Error get_improv_error_state() const { return this->error_state_; } protected: - bool should_start_{false}; - bool setup_complete_{false}; - uint32_t identify_start_{0}; uint32_t identify_duration_; uint32_t authorized_start_{0}; @@ -84,6 +81,14 @@ class ImprovBLEComponent final : public Component, public improv_base::ImprovBas std::vector incoming_data_; wifi::WiFiAP connecting_sta_; + // Single byte fields here fill the alignment gap between connecting_sta_ and the next pointer + improv::State state_{improv::STATE_STOPPED}; + improv::Error error_state_{improv::ERROR_NONE}; + bool should_start_{false}; + bool setup_complete_{false}; + bool status_indicator_state_{false}; + bool advertising_device_name_{false}; + bool advertising_requested_{false}; BLEService *service_{nullptr}; BLECharacteristic *status_{nullptr}; @@ -99,16 +104,11 @@ class ImprovBLEComponent final : public Component, public improv_base::ImprovBas output::BinaryOutput *status_indicator_{nullptr}; #endif - improv::State state_{improv::STATE_STOPPED}; - improv::Error error_state_{improv::ERROR_NONE}; #ifdef USE_IMPROV_BLE_STATE_CALLBACK CallbackManager state_callback_{}; #endif - bool status_indicator_state_{false}; uint32_t last_name_adv_time_{0}; - bool advertising_device_name_{false}; - bool advertising_requested_{false}; void set_status_indicator_state_(bool state); void update_advertising_type_(); void request_advertising_(); diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 5a0a4d2c7e..082c059a14 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -303,7 +303,7 @@ class WiFiAP { bssid_t bssid_{}; // 6 bytes, all zeros = any/not set uint8_t channel_{0}; // 1 byte, 0 = auto/not set int8_t priority_{0}; // 1 byte - bool hidden_{false}; // 1 byte (+ 3 bytes end padding to 4-byte align) + bool hidden_{false}; // 1 byte; WiFiAP is byte aligned unless manual IP or EAP adds a 4 byte aligned member }; class WiFiScanResult { From 8f0e032b5ea048f5e4cb676acf8d97c41e7f16c3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:05:15 -0500 Subject: [PATCH 129/221] [esp32_ble] Group the one byte io_cap_ with the other byte fields (#20136) --- esphome/components/esp32_ble/ble.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index fd4fb15ff6..950ae6e0bc 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -235,14 +235,14 @@ class ESP32BLE final : public Component { #ifdef USE_ESP32_BLE_ADVERTISING BLEAdvertising *advertising_{}; // 4 bytes (pointer) #endif - const char *name_{nullptr}; // 4 bytes (pointer to string literal in flash) - esp_ble_io_cap_t io_cap_{ESP_IO_CAP_NONE}; // 4 bytes (enum) - uint32_t advertising_cycle_time_{}; // 4 bytes + const char *name_{nullptr}; // 4 bytes (pointer to string literal in flash) + uint32_t advertising_cycle_time_{}; // 4 bytes // 2-byte aligned members uint16_t appearance_{0}; // 2 bytes // 1-byte aligned members (grouped together to minimize padding) + esp_ble_io_cap_t io_cap_{ESP_IO_CAP_NONE}; // 1 byte (uint8_t typedef) BLEComponentState state_{BLE_COMPONENT_STATE_OFF}; // 1 byte (uint8_t enum) bool enable_on_boot_{}; // 1 byte #ifdef USE_ESP32_BLE_ADVERTISING From cfbbd13b86df52017ecb4ba537b7c799ae6fede7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:05:29 -0500 Subject: [PATCH 130/221] [remote_receiver] Pack carrier_duty_percent_ next to with_dma_ (#20135) --- esphome/components/remote_receiver/remote_receiver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/remote_receiver/remote_receiver.h b/esphome/components/remote_receiver/remote_receiver.h index 6f93979b18..1c1c473f47 100644 --- a/esphome/components/remote_receiver/remote_receiver.h +++ b/esphome/components/remote_receiver/remote_receiver.h @@ -88,8 +88,8 @@ class RemoteReceiverComponent final : public remote_base::RemoteReceiverBase, rmt_channel_handle_t channel_{NULL}; uint32_t filter_symbols_{0}; uint32_t receive_symbols_{0}; - bool with_dma_{false}; uint32_t carrier_frequency_{0}; + bool with_dma_{false}; uint8_t carrier_duty_percent_{100}; #endif From e708c85100f1b2342a2d3b0c60e70d8aa5abde42 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:12:34 -0500 Subject: [PATCH 131/221] [uart] Skip the parity setter when it matches the default (#20069) --- esphome/components/uart/__init__.py | 4 +++- esphome/components/uart/uart_component.h | 2 +- .../uart/config/parity_default.yaml | 12 ++++++++++ .../uart/config/parity_even.yaml | 13 ++++++++++ tests/component_tests/uart/test_parity.py | 24 +++++++++++++++++++ 5 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/component_tests/uart/config/parity_default.yaml create mode 100644 tests/component_tests/uart/config/parity_even.yaml create mode 100644 tests/component_tests/uart/test_parity.py diff --git a/esphome/components/uart/__init__.py b/esphome/components/uart/__init__.py index 598e3df168..24a854ec39 100644 --- a/esphome/components/uart/__init__.py +++ b/esphome/components/uart/__init__.py @@ -402,7 +402,9 @@ async def to_code(config): cg.add(var.set_clock_source(UART_CLOCK_SOURCES[clock_source])) cg.add(var.set_stop_bits(config[CONF_STOP_BITS])) cg.add(var.set_data_bits(config[CONF_DATA_BITS])) - cg.add(var.set_parity(config[CONF_PARITY])) + # Skip the setter when the config matches the C++ initializer (UART_CONFIG_PARITY_NONE). + if (parity := config[CONF_PARITY]) != "NONE": + cg.add(var.set_parity(parity)) if CONF_DEBUG in config: await debug_to_code(config[CONF_DEBUG], var) diff --git a/esphome/components/uart/uart_component.h b/esphome/components/uart/uart_component.h index 4269ef0b16..8f143b7977 100644 --- a/esphome/components/uart/uart_component.h +++ b/esphome/components/uart/uart_component.h @@ -217,7 +217,7 @@ class UARTComponent { uint32_t baud_rate_{0}; uint8_t stop_bits_{0}; uint8_t data_bits_{0}; - UARTParityOptions parity_{UART_CONFIG_PARITY_NONE}; + UARTParityOptions parity_{UART_CONFIG_PARITY_NONE}; // Must match the parity check in __init__.py #ifdef USE_UART_DEBUGGER CallbackManager debug_callback_{}; #endif diff --git a/tests/component_tests/uart/config/parity_default.yaml b/tests/component_tests/uart/config/parity_default.yaml new file mode 100644 index 0000000000..0f073aaac8 --- /dev/null +++ b/tests/component_tests/uart/config/parity_default.yaml @@ -0,0 +1,12 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +uart: + - id: default_uart + tx_pin: GPIO17 + rx_pin: GPIO16 + baud_rate: 9600 diff --git a/tests/component_tests/uart/config/parity_even.yaml b/tests/component_tests/uart/config/parity_even.yaml new file mode 100644 index 0000000000..1ae742d57a --- /dev/null +++ b/tests/component_tests/uart/config/parity_even.yaml @@ -0,0 +1,13 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +uart: + - id: even_uart + tx_pin: GPIO17 + rx_pin: GPIO16 + baud_rate: 9600 + parity: EVEN diff --git a/tests/component_tests/uart/test_parity.py b/tests/component_tests/uart/test_parity.py new file mode 100644 index 0000000000..c18869a8c4 --- /dev/null +++ b/tests/component_tests/uart/test_parity.py @@ -0,0 +1,24 @@ +"""Tests for UART parity code generation.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_default_parity_is_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Parity NONE is the C++ initializer, so the setter is skipped.""" + main_cpp = generate_main(component_config_path("parity_default.yaml")) + + assert "default_uart->set_parity(" not in main_cpp + + +def test_custom_parity_is_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A non default parity still reaches the setter.""" + main_cpp = generate_main(component_config_path("parity_even.yaml")) + + assert "even_uart->set_parity(uart::UART_CONFIG_PARITY_EVEN);" in main_cpp From e09cab94d459fb874b8e956f9c724a1ce6accd92 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:13:01 -0500 Subject: [PATCH 132/221] [version] Skip the hide hash and hide timestamp setters when they match the defaults (#20067) --- esphome/components/version/text_sensor.py | 7 +++++-- .../components/version/version_text_sensor.h | 4 ++-- tests/component_tests/version/__init__.py | 0 .../component_tests/version/config/version.yaml | 16 ++++++++++++++++ tests/component_tests/version/test_version.py | 17 +++++++++++++++++ 5 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/version/__init__.py create mode 100644 tests/component_tests/version/config/version.yaml create mode 100644 tests/component_tests/version/test_version.py diff --git a/esphome/components/version/text_sensor.py b/esphome/components/version/text_sensor.py index cad239dc41..189d18256f 100644 --- a/esphome/components/version/text_sensor.py +++ b/esphome/components/version/text_sensor.py @@ -35,5 +35,8 @@ CONFIG_SCHEMA = ( async def to_code(config: ConfigType) -> None: var = await text_sensor.new_text_sensor(config) await cg.register_component(var, config) - cg.add(var.set_hide_hash(config[CONF_HIDE_HASH])) - cg.add(var.set_hide_timestamp(config[CONF_HIDE_TIMESTAMP])) + # Both flags default to false in C++, so only true needs a setter. + if config[CONF_HIDE_HASH]: + cg.add(var.set_hide_hash(True)) + if config[CONF_HIDE_TIMESTAMP]: + cg.add(var.set_hide_timestamp(True)) diff --git a/esphome/components/version/version_text_sensor.h b/esphome/components/version/version_text_sensor.h index 7ff6ac4d35..8537435079 100644 --- a/esphome/components/version/version_text_sensor.h +++ b/esphome/components/version/version_text_sensor.h @@ -15,8 +15,8 @@ class VersionTextSensor final : public text_sensor::TextSensor, public Component void dump_config() override; protected: - bool hide_hash_{false}; - bool hide_timestamp_{false}; + bool hide_hash_{false}; // Must match the setter check in text_sensor.py + bool hide_timestamp_{false}; // Must match the setter check in text_sensor.py }; } // namespace esphome::version diff --git a/tests/component_tests/version/__init__.py b/tests/component_tests/version/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/version/config/version.yaml b/tests/component_tests/version/config/version.yaml new file mode 100644 index 0000000000..c602f88183 --- /dev/null +++ b/tests/component_tests/version/config/version.yaml @@ -0,0 +1,16 @@ +--- +esphome: + name: test + +esp8266: + board: d1_mini + +text_sensor: + - platform: version + id: default_version + name: Default Version + - platform: version + id: hidden_version + name: Hidden Version + hide_hash: true + hide_timestamp: true diff --git a/tests/component_tests/version/test_version.py b/tests/component_tests/version/test_version.py new file mode 100644 index 0000000000..14886264a2 --- /dev/null +++ b/tests/component_tests/version/test_version.py @@ -0,0 +1,17 @@ +"""Tests for the version text sensor codegen.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_default_hide_flags_are_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Both flags are false in C++, so only true reaches the setters.""" + main_cpp = generate_main(component_config_path("version.yaml")) + + assert "default_version->set_hide_hash(" not in main_cpp + assert "default_version->set_hide_timestamp(" not in main_cpp + assert "hidden_version->set_hide_hash(true);" in main_cpp + assert "hidden_version->set_hide_timestamp(true);" in main_cpp From 31e0a31c651d7ce7ba59353760dd9b36cb498e39 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:13:41 -0500 Subject: [PATCH 133/221] [api] Skip the listen backlog setter when it matches the default (#20066) --- esphome/components/api/__init__.py | 7 +++++-- esphome/components/api/api_server.h | 4 +--- tests/component_tests/api/config/custom.yaml | 1 + tests/component_tests/api/config/defaults.yaml | 1 + tests/component_tests/api/config/esp8266.yaml | 12 ++++++++++++ tests/component_tests/api/test_default_setters.py | 14 +++++++++++++- 6 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 tests/component_tests/api/config/esp8266.yaml diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 25ebc93458..764a3d0f1e 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -142,6 +142,7 @@ CONF_STATE_SUBSCRIPTION_ONLY = "state_subscription_only" DEFAULT_PORT = 6053 DEFAULT_REBOOT_TIMEOUT = "15min" DEFAULT_BATCH_DELAY = "100ms" +DEFAULT_LISTEN_BACKLOG = 4 def _register_provisioning_source(config: ConfigType) -> ConfigType: @@ -477,8 +478,10 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_reboot_timeout(reboot_timeout)) if (batch_delay := config[CONF_BATCH_DELAY]) != cv.time_period(DEFAULT_BATCH_DELAY): cg.add(var.set_batch_delay(batch_delay)) - if CONF_LISTEN_BACKLOG in config: - cg.add(var.set_listen_backlog(config[CONF_LISTEN_BACKLOG])) + if ( + listen_backlog := config.get(CONF_LISTEN_BACKLOG) + ) is not None and listen_backlog != DEFAULT_LISTEN_BACKLOG: + cg.add(var.set_listen_backlog(listen_backlog)) cg.add_define("MAX_API_CONNECTIONS", config[CONF_MAX_CONNECTIONS]) cg.add_define("API_MAX_SEND_QUEUE", config[CONF_MAX_SEND_QUEUE]) diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 865cd17156..491ab38342 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -357,9 +357,7 @@ class APIServer final : public Component // Group smaller types together uint16_t port_{6053}; // Keep in sync with DEFAULT_PORT in __init__.py uint16_t batch_delay_{100}; // Keep in sync with DEFAULT_BATCH_DELAY in __init__.py - // Connection limits - these defaults will be overridden by config values - // from cv.SplitDefault in __init__.py which sets platform-specific defaults. - uint8_t listen_backlog_{4}; + uint8_t listen_backlog_{4}; // Keep in sync with DEFAULT_LISTEN_BACKLOG in __init__.py bool shutting_down_ = false; uint8_t api_connection_count_{0}; #if defined(USE_PROVISIONING) && defined(USE_API_NOISE) diff --git a/tests/component_tests/api/config/custom.yaml b/tests/component_tests/api/config/custom.yaml index cdf4038d5d..476a46ebff 100644 --- a/tests/component_tests/api/config/custom.yaml +++ b/tests/component_tests/api/config/custom.yaml @@ -13,3 +13,4 @@ api: port: 6054 reboot_timeout: 0s batch_delay: 0ms + listen_backlog: 2 diff --git a/tests/component_tests/api/config/defaults.yaml b/tests/component_tests/api/config/defaults.yaml index b20fd9b884..49d7b0abcc 100644 --- a/tests/component_tests/api/config/defaults.yaml +++ b/tests/component_tests/api/config/defaults.yaml @@ -13,3 +13,4 @@ api: port: 6053 reboot_timeout: 15min batch_delay: 100ms + listen_backlog: 4 diff --git a/tests/component_tests/api/config/esp8266.yaml b/tests/component_tests/api/config/esp8266.yaml new file mode 100644 index 0000000000..0ba6ccbd33 --- /dev/null +++ b/tests/component_tests/api/config/esp8266.yaml @@ -0,0 +1,12 @@ +--- +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: test + password: testtest + +api: diff --git a/tests/component_tests/api/test_default_setters.py b/tests/component_tests/api/test_default_setters.py index 32d35cacb7..2c7f843c68 100644 --- a/tests/component_tests/api/test_default_setters.py +++ b/tests/component_tests/api/test_default_setters.py @@ -12,7 +12,7 @@ def test_default_values_are_not_emitted( component_config_path: Callable[[str], Path], config_file: str, ) -> None: - """Port 6053, a 15 min reboot timeout and 100 ms batch delay are C++ initializers. + """Port 6053, a 15 min reboot timeout, 100 ms batch delay and backlog 4 are C++ initializers. Both the schema defaults and the same values written explicitly take the skip path. """ @@ -21,6 +21,7 @@ def test_default_values_are_not_emitted( assert "api_apiserver_id->set_port(" not in main_cpp assert "api_apiserver_id->set_reboot_timeout(" not in main_cpp assert "api_apiserver_id->set_batch_delay(" not in main_cpp + assert "api_apiserver_id->set_listen_backlog(" not in main_cpp def test_custom_values_are_emitted( @@ -33,3 +34,14 @@ def test_custom_values_are_emitted( assert "api_apiserver_id->set_port(6054);" in main_cpp assert "api_apiserver_id->set_reboot_timeout(0);" in main_cpp assert "api_apiserver_id->set_batch_delay(0);" in main_cpp + assert "api_apiserver_id->set_listen_backlog(2);" in main_cpp + + +def test_esp8266_listen_backlog_is_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The ESP8266 schema default of 1 differs from the C++ initializer, so it is set.""" + main_cpp = generate_main(component_config_path("esp8266.yaml")) + + assert "api_apiserver_id->set_listen_backlog(1);" in main_cpp From 4f9cd7821a7433ddf9fe9edf6bc52ccc3b89576d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:14:42 -0500 Subject: [PATCH 134/221] [core] Log ignored --skip-bootloader at debug level (#20059) --- esphome/__main__.py | 6 +++--- tests/unit_tests/test_main.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 40457dbcc2..6c984050f4 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -829,9 +829,9 @@ def write_cpp_file() -> int: def compile_program(args: ArgsProtocol, config: ConfigType) -> int: if CORE.skip_bootloader and not (CORE.is_esp32 and CORE.using_toolchain_esp_idf): - # Info, not a warning: an orchestrator cannot see YAML toolchain - # overrides, this is its expected no-op, and a full build is safe. - _LOGGER.info( + # Debug only: an orchestrator cannot see YAML toolchain overrides, + # so this is its expected no-op, and a full build is safe. + _LOGGER.debug( "--skip-bootloader ignored: only supported on ESP32 with the " "esp-idf toolchain" ) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index e46b61cfd5..4edf6d5fc3 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -5358,7 +5358,7 @@ def _setup_build_info_test( return build_info_path, firmware_path -def test_compile_program_warns_and_ignores_skip_bootloader_elsewhere( +def test_compile_program_quietly_ignores_skip_bootloader_elsewhere( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: """A full build is always safe; orchestrators cannot see YAML overrides.""" @@ -5372,11 +5372,13 @@ def test_compile_program_warns_and_ignores_skip_bootloader_elsewhere( side_effect=EsphomeError("stop here"), ), pytest.raises(EsphomeError, match="stop here"), - caplog.at_level("INFO"), + caplog.at_level("DEBUG"), ): compile_program(MockArgs(), config) - assert "--skip-bootloader ignored" in caplog.text + assert [ + r.levelno for r in caplog.records if "--skip-bootloader ignored" in r.message + ] == [logging.DEBUG] assert CORE.skip_bootloader is False From feedea4954a342091c9751ccfd1ad68db59ef6e3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:15:58 -0500 Subject: [PATCH 135/221] [web_server] Remove dead ESP-IDF event source connect callback and pack expose_log_ (#20130) --- esphome/components/web_server/web_server.h | 4 +++- esphome/components/web_server_idf/web_server_idf.cpp | 7 +------ esphome/components/web_server_idf/web_server_idf.h | 7 ------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/esphome/components/web_server/web_server.h b/esphome/components/web_server/web_server.h index 143e9a15ea..6f08a8dc8f 100644 --- a/esphome/components/web_server/web_server.h +++ b/esphome/components/web_server/web_server.h @@ -531,6 +531,9 @@ class WebServer final : public Component, public AsyncWebHandler { bool include_internal_{false}; protected: + // Declared here so it packs into the padding after include_internal_ + bool expose_log_{true}; + void add_sorting_info_(JsonObject &root, EntityBase *entity); #ifdef USE_LIGHT @@ -614,7 +617,6 @@ class WebServer final : public Component, public AsyncWebHandler { #ifdef USE_WEBSERVER_JS_INCLUDE const char *js_include_{nullptr}; #endif - bool expose_log_{true}; #ifdef USE_WEBSERVER_ALLOWED_ORIGINS // Extra origins allowed to make cross-origin browser requests ("*" means any origin). // Only compiled when allowed_origins is configured; same-origin is always allowed regardless. diff --git a/esphome/components/web_server_idf/web_server_idf.cpp b/esphome/components/web_server_idf/web_server_idf.cpp index 510b4e50f1..9798410e23 100644 --- a/esphome/components/web_server_idf/web_server_idf.cpp +++ b/esphome/components/web_server_idf/web_server_idf.cpp @@ -761,7 +761,7 @@ void AsyncEventSource::adopt_pending_sessions_main_loop_() { this->has_pending_sessions_.store(false, std::memory_order_relaxed); } for (auto *rsp : incoming) { - // Already disconnected? Drop it; skip on_connect_/session start on a dead session. + // Already disconnected? Drop it; skip session start on a dead session. if (rsp->safe_to_delete_()) { delete rsp; // NOLINT(cppcoreguidelines-owning-memory) continue; @@ -775,12 +775,7 @@ void AsyncEventSource::adopt_pending_sessions_main_loop_() { continue; } this->sessions_.push_back(rsp); - // Prime first so on_connect_ observes a session that has already sent its - // initial ping/config/sorting_groups, matching the pre-refactor ordering. rsp->start_session_main_loop_(); - if (this->on_connect_) { - this->on_connect_(rsp); - } } } // NOLINTEND(clang-analyzer-cplusplus.NewDeleteLeaks) diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h index 2b62d68790..581ba8323b 100644 --- a/esphome/components/web_server_idf/web_server_idf.h +++ b/esphome/components/web_server_idf/web_server_idf.h @@ -379,7 +379,6 @@ using AsyncEventSourceClient = AsyncEventSourceResponse; class AsyncEventSource : public AsyncWebHandler { friend class AsyncEventSourceResponse; - using connect_handler_t = std::function; public: AsyncEventSource(StringRef url, esphome::web_server::WebServer *ws) : url_(url), web_server_(ws) {} @@ -394,11 +393,6 @@ class AsyncEventSource : public AsyncWebHandler { } // NOLINTNEXTLINE(readability-identifier-naming) void handleRequest(AsyncWebServerRequest *request) override; - // Callback runs on the main loop (not the httpd task) after the session's - // initial ping/config/sorting_groups have been sent. - // NOLINTNEXTLINE(readability-identifier-naming) - void onConnect(connect_handler_t &&cb) { this->on_connect_ = std::move(cb); } - void try_send_nodefer(const char *message, size_t message_len, const char *event = nullptr, uint32_t id = 0, uint32_t reconnect = 0); void deferrable_send_state(void *source, const char *event_type, message_generator_t *message_generator); @@ -418,7 +412,6 @@ class AsyncEventSource : public AsyncWebHandler { // Httpd-task intake; guarded by pending_mutex_, gated by has_pending_sessions_. std::vector pending_sessions_; Mutex pending_mutex_; - connect_handler_t on_connect_{}; esphome::web_server::WebServer *web_server_; std::atomic has_pending_sessions_{false}; }; From b36c5441cb2d997bc528677624f6a351eef7a5ef Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:16:16 -0500 Subject: [PATCH 136/221] [esp32] Remove the unused loop_task_handle global (#20132) --- esphome/components/esp32/core.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/esphome/components/esp32/core.cpp b/esphome/components/esp32/core.cpp index a6916fe739..f718d16df4 100644 --- a/esphome/components/esp32/core.cpp +++ b/esphome/components/esp32/core.cpp @@ -17,8 +17,7 @@ extern "C" __attribute__((weak)) void initArduino() {} namespace esphome { // HAL functions live in hal.cpp. This file keeps only the loop task setup. -TaskHandle_t loop_task_handle = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static StaticTask_t loop_task_tcb; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static StaticTask_t loop_task_tcb; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) static StackType_t loop_task_stack[ESPHOME_LOOP_TASK_STACK_SIZE]; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) @@ -40,11 +39,10 @@ extern "C" void app_main() { initArduino(); esp32::setup_preferences(); #if CONFIG_FREERTOS_UNICORE - loop_task_handle = xTaskCreateStatic(loop_task, "loopTask", ESPHOME_LOOP_TASK_STACK_SIZE, nullptr, 1, loop_task_stack, - &loop_task_tcb); + xTaskCreateStatic(loop_task, "loopTask", ESPHOME_LOOP_TASK_STACK_SIZE, nullptr, 1, loop_task_stack, &loop_task_tcb); #else - loop_task_handle = xTaskCreateStaticPinnedToCore(loop_task, "loopTask", ESPHOME_LOOP_TASK_STACK_SIZE, nullptr, 1, - loop_task_stack, &loop_task_tcb, 1); + xTaskCreateStaticPinnedToCore(loop_task, "loopTask", ESPHOME_LOOP_TASK_STACK_SIZE, nullptr, 1, loop_task_stack, + &loop_task_tcb, 1); #endif } From acf7257293fd6f04182f0f5a7d91d8e0c535c18b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:17:22 -0500 Subject: [PATCH 137/221] [core] Use the static queue structure as the FreeRTOSQueue handle (#20133) --- esphome/core/freertos_queue.h | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/esphome/core/freertos_queue.h b/esphome/core/freertos_queue.h index 2f3faf818a..1067ae07cc 100644 --- a/esphome/core/freertos_queue.h +++ b/esphome/core/freertos_queue.h @@ -27,9 +27,7 @@ namespace esphome { template class FreeRTOSQueue { public: - FreeRTOSQueue() : dropped_count_(0) { - this->handle_ = xQueueCreateStatic(SIZE, sizeof(T *), this->storage_, &this->queue_buf_); - } + FreeRTOSQueue() : dropped_count_(0) { xQueueCreateStatic(SIZE, sizeof(T *), this->storage_, &this->queue_buf_); } // No destructor — ESPHome components are never destroyed. Intentionally // omitted to avoid pulling in vQueueDelete code on resource-constrained targets. @@ -44,7 +42,7 @@ template class FreeRTOSQueue { if (element == nullptr) return false; - if (xQueueSend(this->handle_, &element, 0) != pdPASS) { + if (xQueueSend(this->handle_(), &element, 0) != pdPASS) { this->increment_dropped_count(); return false; } @@ -53,7 +51,7 @@ template class FreeRTOSQueue { T *pop() { T *element; - if (xQueueReceive(this->handle_, &element, 0) != pdTRUE) { + if (xQueueReceive(this->handle_(), &element, 0) != pdTRUE) { return nullptr; } return element; @@ -80,18 +78,24 @@ template class FreeRTOSQueue { portEXIT_CRITICAL(); } - bool empty() const { return uxQueueMessagesWaiting(this->handle_) == 0; } + bool empty() const { return uxQueueMessagesWaiting(this->handle_()) == 0; } - bool full() const { return uxQueueSpacesAvailable(this->handle_) == 0; } + bool full() const { return uxQueueSpacesAvailable(this->handle_()) == 0; } - size_t size() const { return uxQueueMessagesWaiting(this->handle_); } + size_t size() const { return uxQueueMessagesWaiting(this->handle_()); } protected: // Static storage for the queue — lives in BSS, no heap allocation uint8_t storage_[SIZE * sizeof(T *)]; StaticQueue_t queue_buf_; - QueueHandle_t handle_; uint16_t dropped_count_; + + // xQueueCreateStatic() returns the static queue structure itself as the handle; + // its only other checks are asserts, and a zero length is the one we could hit + static_assert(SIZE > 0, "FreeRTOSQueue needs at least one slot"); + QueueHandle_t handle_() const { + return reinterpret_cast(const_cast(&this->queue_buf_)); + } }; } // namespace esphome From ad1b6ab37e501db1b5d02d7830518da70cdac0af Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:23:36 -0500 Subject: [PATCH 138/221] [web_server] Keep allowed origins in a shared flash table (#20086) --- esphome/components/web_server/__init__.py | 11 ++++++++++- esphome/components/web_server/web_server.cpp | 3 ++- esphome/components/web_server/web_server.h | 6 +++--- .../web_server/config/allowed_origins.yaml | 15 +++++++++++++++ .../web_server/test_allowed_origins.py | 18 ++++++++++++++++++ 5 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 tests/component_tests/web_server/config/allowed_origins.yaml create mode 100644 tests/component_tests/web_server/test_allowed_origins.py diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 2b00a6f8de..6bc4d7ab5c 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -413,7 +413,16 @@ async def to_code(config: ConfigType) -> None: cg.add_define("USE_WEBSERVER_PRIVATE_NETWORK_ACCESS") if (allowed_origins := config.get(CONF_ALLOWED_ORIGINS)) is not None: cg.add_define("USE_WEBSERVER_ALLOWED_ORIGINS") - cg.add(var.set_allowed_origins(allowed_origins)) + # Shared flash table ended by nullptr, so the server stores only a pointer. + cg.add( + var.set_allowed_origins( + cg.shared_progmem_array( + "web_server_allowed_origins", + cg.const_char_ptr, + [*allowed_origins, cg.nullptr], + ) + ) + ) if (auth := config.get(CONF_AUTH)) is not None: cg.add_define("USE_WEBSERVER_AUTH") # The scheme is fixed at build time so the unused Basic/Digest code path is compiled diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp index 74641738ec..90c6b167f6 100644 --- a/esphome/components/web_server/web_server.cpp +++ b/esphome/components/web_server/web_server.cpp @@ -495,7 +495,8 @@ bool WebServer::is_request_origin_allowed_(AsyncWebServerRequest *request, const #ifdef USE_WEBSERVER_ALLOWED_ORIGINS // Otherwise the origin must be explicitly allowed via configuration. - for (const char *allowed_origin : this->allowed_origins_) { + for (const char *const *it = this->allowed_origins_; *it != nullptr; it++) { + const char *allowed_origin = *it; // A single "*" entry allows any origin. if (allowed_origin[0] == '*' && allowed_origin[1] == '\0') return true; diff --git a/esphome/components/web_server/web_server.h b/esphome/components/web_server/web_server.h index 6f08a8dc8f..1f9b6cb934 100644 --- a/esphome/components/web_server/web_server.h +++ b/esphome/components/web_server/web_server.h @@ -254,9 +254,9 @@ class WebServer final : public Component, public AsyncWebHandler { * * This list is also used to authorize Private Network Access requests when that feature is enabled. * - * @param origins The list of allowed origins. + * @param origins nullptr-terminated table of allowed origins; must outlive the server. */ - void set_allowed_origins(std::initializer_list origins) { this->allowed_origins_ = origins; } + void set_allowed_origins(const char *const *origins) { this->allowed_origins_ = origins; } #endif // ========== INTERNAL METHODS ========== @@ -620,7 +620,7 @@ class WebServer final : public Component, public AsyncWebHandler { #ifdef USE_WEBSERVER_ALLOWED_ORIGINS // Extra origins allowed to make cross-origin browser requests ("*" means any origin). // Only compiled when allowed_origins is configured; same-origin is always allowed regardless. - FixedVector allowed_origins_; + const char *const *allowed_origins_{nullptr}; #endif /// Check whether the given request Origin is permitted. Same-origin (matching the Host the diff --git a/tests/component_tests/web_server/config/allowed_origins.yaml b/tests/component_tests/web_server/config/allowed_origins.yaml new file mode 100644 index 0000000000..2ced49d6d8 --- /dev/null +++ b/tests/component_tests/web_server/config/allowed_origins.yaml @@ -0,0 +1,15 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +wifi: + ssid: test + password: testtest + +web_server: + allowed_origins: + - https://app.esphome.io + - "*" diff --git a/tests/component_tests/web_server/test_allowed_origins.py b/tests/component_tests/web_server/test_allowed_origins.py new file mode 100644 index 0000000000..7a6c602c12 --- /dev/null +++ b/tests/component_tests/web_server/test_allowed_origins.py @@ -0,0 +1,18 @@ +"""Tests for the web_server allowed_origins codegen.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_allowed_origins_are_a_progmem_table( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Allowed origins live in a nullptr-terminated flash table instead of a heap vector.""" + main_cpp = generate_main(component_config_path("allowed_origins.yaml")) + + assert ( + "static constexpr const char * web_server_allowed_origins[] PROGMEM = " + '{"https://app.esphome.io", "*", nullptr};' in main_cpp + ) + assert "->set_allowed_origins(web_server_allowed_origins);" in main_cpp From 0017b9c439db5c78988f7a0e1595d4b20e6acd8f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:26:42 -0500 Subject: [PATCH 139/221] [matrix_keypad] Keep the keys in flash (#20110) --- esphome/components/matrix_keypad/__init__.py | 11 ++++++-- .../matrix_keypad/matrix_keypad.cpp | 8 +++--- .../components/matrix_keypad/matrix_keypad.h | 5 ++-- .../component_tests/matrix_keypad/__init__.py | 0 .../matrix_keypad/config/keys.yaml | 23 ++++++++++++++++ .../matrix_keypad/test_keys.py | 27 +++++++++++++++++++ 6 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 tests/component_tests/matrix_keypad/__init__.py create mode 100644 tests/component_tests/matrix_keypad/config/keys.yaml create mode 100644 tests/component_tests/matrix_keypad/test_keys.py diff --git a/esphome/components/matrix_keypad/__init__.py b/esphome/components/matrix_keypad/__init__.py index 2e43eaf7e2..f9b744daf0 100644 --- a/esphome/components/matrix_keypad/__init__.py +++ b/esphome/components/matrix_keypad/__init__.py @@ -27,6 +27,10 @@ CONF_HAS_PULLDOWNS = "has_pulldowns" def check_keys(obj: ConfigType) -> ConfigType: + for ch in obj.get(CONF_KEYS, ""): + if not ch.isascii(): + # Each key is reported as one byte, so only ASCII characters can be key codes + raise cv.Invalid(f"Key code {ch!r} is not an ASCII character") if CONF_KEYS in obj and len(obj[CONF_KEYS]) != len(obj[CONF_ROWS]) * len( obj[CONF_COLUMNS] ): @@ -74,8 +78,11 @@ async def to_code(config: ConfigType) -> None: pin = await cg.gpio_pin_expression(conf[CONF_PIN]) col_pins.append(pin) cg.add(var.set_columns(col_pins)) - if CONF_KEYS in config: - cg.add(var.set_keys(config[CONF_KEYS])) + if (keys := config.get(CONF_KEYS)) is not None: + table = cg.shared_progmem_array( + "matrix_keypad_keys", cg.uint8, list(keys.encode()) + ) + cg.add(var.set_keys(table)) cg.add(var.set_debounce_time(config[CONF_DEBOUNCE_TIME])) if CONF_HAS_DIODES in config: cg.add(var.set_has_diodes(config[CONF_HAS_DIODES])) diff --git a/esphome/components/matrix_keypad/matrix_keypad.cpp b/esphome/components/matrix_keypad/matrix_keypad.cpp index 3b71b50fd8..97641ac2d3 100644 --- a/esphome/components/matrix_keypad/matrix_keypad.cpp +++ b/esphome/components/matrix_keypad/matrix_keypad.cpp @@ -58,8 +58,8 @@ void MatrixKeypad::loop() { ESP_LOGD(TAG, "key @ row %d, col %d released", row, col); for (auto &listener : this->listeners_) listener->button_released(row, col); - if (this->pressed_key_ < (int) this->keys_.size()) { - uint8_t keycode = this->keys_[this->pressed_key_]; + if (this->keys_ != nullptr) { + uint8_t keycode = progmem_read_byte(&this->keys_[this->pressed_key_]); ESP_LOGD(TAG, "key '%c' released", keycode); for (auto &listener : this->listeners_) listener->key_released(keycode); @@ -81,8 +81,8 @@ void MatrixKeypad::loop() { ESP_LOGD(TAG, "key @ row %d, col %d pressed", row, col); for (auto &listener : this->listeners_) listener->button_pressed(row, col); - if (key < (int) this->keys_.size()) { - uint8_t keycode = this->keys_[key]; + if (this->keys_ != nullptr) { + uint8_t keycode = progmem_read_byte(&this->keys_[key]); ESP_LOGD(TAG, "key '%c' pressed", keycode); for (auto &trigger : this->key_triggers_) trigger->trigger(keycode); diff --git a/esphome/components/matrix_keypad/matrix_keypad.h b/esphome/components/matrix_keypad/matrix_keypad.h index 8c9acc8e0c..2384691e32 100644 --- a/esphome/components/matrix_keypad/matrix_keypad.h +++ b/esphome/components/matrix_keypad/matrix_keypad.h @@ -27,7 +27,8 @@ class MatrixKeypad final : public key_provider::KeyProvider, public Component { void dump_config() override; void set_columns(std::vector pins) { columns_ = std::move(pins); }; void set_rows(std::vector pins) { rows_ = std::move(pins); }; - void set_keys(std::string keys) { keys_ = std::move(keys); }; + /// `keys` is a codegen PROGMEM table with one key code per button. + void set_keys(const uint8_t *keys) { this->keys_ = keys; } void set_debounce_time(uint32_t debounce_time) { debounce_time_ = debounce_time; }; void set_has_diodes(bool has_diodes) { has_diodes_ = has_diodes; }; void set_has_pulldowns(bool has_pulldowns) { has_pulldowns_ = has_pulldowns; }; @@ -38,7 +39,7 @@ class MatrixKeypad final : public key_provider::KeyProvider, public Component { protected: std::vector rows_; std::vector columns_; - std::string keys_; + const uint8_t *keys_{nullptr}; uint32_t debounce_time_ = 0; bool has_diodes_{false}; bool has_pulldowns_{false}; diff --git a/tests/component_tests/matrix_keypad/__init__.py b/tests/component_tests/matrix_keypad/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/matrix_keypad/config/keys.yaml b/tests/component_tests/matrix_keypad/config/keys.yaml new file mode 100644 index 0000000000..99d32ffd14 --- /dev/null +++ b/tests/component_tests/matrix_keypad/config/keys.yaml @@ -0,0 +1,23 @@ +esphome: + name: test + +esp8266: + board: esp01_1m + +matrix_keypad: + - id: keypad_a + rows: + - pin: 12 + - pin: 13 + columns: + - pin: 14 + - pin: 15 + keys: "1234" + - id: keypad_b + rows: + - pin: 4 + - pin: 5 + columns: + - pin: 0 + - pin: 2 + keys: "1234" diff --git a/tests/component_tests/matrix_keypad/test_keys.py b/tests/component_tests/matrix_keypad/test_keys.py new file mode 100644 index 0000000000..7993a20b70 --- /dev/null +++ b/tests/component_tests/matrix_keypad/test_keys.py @@ -0,0 +1,27 @@ +"""Matrix keypad key codes live in a shared PROGMEM table instead of a std::string.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.matrix_keypad import check_keys +import esphome.config_validation as cv + + +def test_keys_use_shared_progmem_table( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("keys.yaml")) + + assert ( + "static constexpr uint8_t matrix_keypad_keys[] PROGMEM = {49, 50, 51, 52};" + in main_cpp + ) + assert main_cpp.count("->set_keys(matrix_keypad_keys);") == 2 + + +def test_non_ascii_keys_are_rejected() -> None: + with pytest.raises(cv.Invalid, match="'é' is not an ASCII"): + check_keys({"keys": "1é34"}) From 99bf0800928176a61e70cfa26272abb2c635dca9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:30:44 -0500 Subject: [PATCH 140/221] [select] Cover select.is in the component test (#20112) --- tests/components/select/common.yaml | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/components/select/common.yaml b/tests/components/select/common.yaml index f20c024cb8..ff41b48ff4 100644 --- a/tests/components/select/common.yaml +++ b/tests/components/select/common.yaml @@ -33,6 +33,20 @@ button: cycle: false - select.first: select_test_select - select.last: select_test_select + - if: + condition: + select.is: + id: select_test_select + options: [one, two] + then: + - select.next: select_test_select + - if: + condition: + select.is: + id: select_test_select + lambda: return current == "three"; + then: + - select.first: select_test_select - select.operation: id: select_test_select operation: next @@ -41,3 +55,47 @@ button: id: select_test_select operation: !lambda return SELECT_OP_PREVIOUS; cycle: !lambda return true; + - platform: template + name: "Test Select Options Lambdas" + on_press: + - lambda: |- + auto &options = id(select_test_select).traits.get_options(); + for (size_t i = 0; i < options.size(); i++) + ESP_LOGD("test", "%s", options[i]); + - lambda: |- + std::string wanted = "two"; + const auto &opts = id(select_test_select).traits.get_options(); + if (opts.empty()) + return; + for (int i = 0; i < (int) opts.size(); i++) { + if (opts[i] == wanted) + ESP_LOGD("test", "found at %d", i); + } + if (id(select_test_select).traits.get_options()[0] == wanted) + ESP_LOGD("test", "first"); + auto it = std::find(opts.begin(), opts.end(), std::string("three")); + ESP_LOGD("test", "index %d", (int) (it - opts.begin())); + - lambda: |- + static std::vector names{"red", "green"}; + FixedVector fresh; + fresh.init(names.size()); + for (auto &name : names) + fresh.push_back(name.c_str()); + if (id(select_test_select).traits.get_options().size() != fresh.size()) + id(select_test_select).traits.set_options(fresh); + id(select_test_select).traits.set_options({"one", "two", "three"}); + - lambda: |- + const char *value = "two"; + const auto &options = id(select_test_select).traits.get_options(); + const auto *it = std::find_if(options.begin(), options.end(), + [value](const char *option) { return strcmp(option, value) == 0; }); + if (it == options.end()) + return; + uint32_t index = (uint32_t) (it - options.begin()); + ESP_LOGD("test", "index %u", (unsigned) index); + const auto *it2 = std::find(options.begin(), options.end(), options[0]); + ESP_LOGD("test", "first at %d", (int) (it2 - options.begin())); + - lambda: |- + auto &options = id(select_test_select).traits.get_options(); + for (size_t i = 0; i < options.size(); i++) + ESP_LOGD("test", "%u: %s", (unsigned) i, options.at(i)); From e4be4c7fcbda0e8f1dfcf848e030dd79fc8aeecb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 18:30:45 -0500 Subject: [PATCH 141/221] [select] Keep select.is option lists in flash (#20108) --- esphome/components/select/__init__.py | 10 +--- tests/component_tests/select/__init__.py | 0 .../select/config/select_is.yaml | 52 +++++++++++++++++++ .../component_tests/select/test_select_is.py | 22 ++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 tests/component_tests/select/__init__.py create mode 100644 tests/component_tests/select/config/select_is.yaml create mode 100644 tests/component_tests/select/test_select_is.py diff --git a/esphome/components/select/__init__.py b/esphome/components/select/__init__.py index ae78e729e6..fec88d2bfd 100644 --- a/esphome/components/select/__init__.py +++ b/esphome/components/select/__init__.py @@ -187,14 +187,8 @@ automation.register_apply_action( async def select_is_to_code(config, condition_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) if options := config.get(CONF_OPTIONS): - # List of constant options - # Create a constexpr and pass that with a template length - arr_id = ID( - f"{condition_id}_data", - is_declaration=True, - type=global_ns.namespace("constexpr char * const"), - ) - arg = cg.static_const_array(arr_id, cg.ArrayInitializer(*options)) + # Shared flash table of option pointers, length passed as a template argument + arg = cg.shared_progmem_array("select_is_options", cg.const_char_ptr, options) template_arg = TemplateArguments(len(options), *template_arg) else: # Lambda diff --git a/tests/component_tests/select/__init__.py b/tests/component_tests/select/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/select/config/select_is.yaml b/tests/component_tests/select/config/select_is.yaml new file mode 100644 index 0000000000..3bc3bdcc02 --- /dev/null +++ b/tests/component_tests/select/config/select_is.yaml @@ -0,0 +1,52 @@ +esphome: + name: test + +esp8266: + board: esp01_1m + +select: + - platform: template + id: mode + optimistic: true + options: [one, two, three] + +button: + - platform: template + name: A + on_press: + - if: + condition: + select.is: + id: mode + options: [one, two] + then: + - logger.log: a + - platform: template + name: B + on_press: + - if: + condition: + select.is: + id: mode + options: [one, two] + then: + - logger.log: b + - platform: template + name: C + on_press: + - if: + condition: + select.is: + id: mode + options: three + then: + - logger.log: c + - if: + condition: + select.is: + id: mode + lambda: return current == "two"; + then: + - logger.log: d + +logger: diff --git a/tests/component_tests/select/test_select_is.py b/tests/component_tests/select/test_select_is.py new file mode 100644 index 0000000000..6c179c3c85 --- /dev/null +++ b/tests/component_tests/select/test_select_is.py @@ -0,0 +1,22 @@ +"""select.is option lists live in shared PROGMEM tables.""" + +from collections.abc import Callable +from pathlib import Path +import re + + +def test_select_is_options_use_shared_progmem_tables( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("select_is.yaml")) + + tables = re.findall( + r"static constexpr const char \* (select_is_options\w*)\[\] PROGMEM = \{([^;]*)\};", + main_cpp, + ) + # Two conditions with [one, two] share one table; [three] gets its own + assert sorted(body for _, body in tables) == ['"one", "two"', '"three"'] + one_two = next(name for name, body in tables if body == '"one", "two"') + assert main_cpp.count(f", {one_two});") == 2 + assert "SelectIsCondition<0" in main_cpp From 60e511282ac5e0464f98aaf0562e1084e95de952 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 19:45:18 -0500 Subject: [PATCH 142/221] [udp] Keep addresses in a shared flash table (#20085) --- esphome/components/udp/__init__.py | 10 ++++++- esphome/components/udp/udp_component.cpp | 9 ++++-- esphome/components/udp/udp_component.h | 7 ++--- tests/component_tests/udp/__init__.py | 0 .../udp/config/udp_addresses.yaml | 14 +++++++++ .../udp/config/udp_no_addresses.yaml | 12 ++++++++ .../component_tests/udp/test_udp_addresses.py | 29 +++++++++++++++++++ 7 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 tests/component_tests/udp/__init__.py create mode 100644 tests/component_tests/udp/config/udp_addresses.yaml create mode 100644 tests/component_tests/udp/config/udp_no_addresses.yaml create mode 100644 tests/component_tests/udp/test_udp_addresses.py diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index d96a731e9c..316d4fc311 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -124,7 +124,15 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_broadcast_port(conf_port[CONF_BROADCAST_PORT])) if (listen_address := str(config[CONF_LISTEN_ADDRESS])) != "255.255.255.255": cg.add(var.set_listen_address(listen_address)) - cg.add(var.set_addresses([str(addr) for addr in config[CONF_ADDRESSES]])) + # Shared flash table ended by nullptr, so the component stores only a pointer. + if addresses := [str(addr) for addr in config[CONF_ADDRESSES]]: + cg.add( + var.set_addresses( + cg.shared_progmem_array( + "udp_addresses", cg.const_char_ptr, [*addresses, cg.nullptr] + ) + ) + ) for conf in config.get(CONF_ON_RECEIVE, []): trigger_id = cg.new_Pvariable(conf[CONF_TRIGGER_ID]) trigger = await automation.build_automation(trigger_id, trigger_argtype, conf) diff --git a/esphome/components/udp/udp_component.cpp b/esphome/components/udp/udp_component.cpp index f662cba4cf..904ed97dc0 100644 --- a/esphome/components/udp/udp_component.cpp +++ b/esphome/components/udp/udp_component.cpp @@ -11,7 +11,8 @@ static const char *const TAG = "udp"; void UDPComponent::setup() { #if defined(USE_SOCKET_IMPL_BSD_SOCKETS) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) - for (const auto &address : this->addresses_) { + for (const char *const *it = this->addresses_; it != nullptr && *it != nullptr; it++) { + const char *address = *it; struct sockaddr saddr {}; if (socket::set_sockaddr(&saddr, sizeof(saddr), address, this->broadcast_port_) == 0) { ESP_LOGW(TAG, "Invalid address %s", address); @@ -97,7 +98,8 @@ void UDPComponent::setup() { #endif #ifdef USE_SOCKET_IMPL_LWIP_TCP // 8266 and RP2040 `Duino - for (const auto &address : this->addresses_) { + for (const char *const *it = this->addresses_; it != nullptr && *it != nullptr; it++) { + const char *address = *it; auto ipaddr = IPAddress(); if (!ipaddr.fromString(address)) { ESP_LOGW(TAG, "Invalid address %s", address); @@ -138,7 +140,8 @@ void UDPComponent::dump_config() { " Listen Port: %u\n" " Broadcast Port: %u", this->listen_port_, this->broadcast_port_); - for (const char *address : this->addresses_) { + for (const char *const *it = this->addresses_; it != nullptr && *it != nullptr; it++) { + const char *address = *it; ESP_LOGCONFIG(TAG, " Address: %s", address); } if (this->listen_address_.has_value()) { diff --git a/esphome/components/udp/udp_component.h b/esphome/components/udp/udp_component.h index 274e0119ee..17e27fb770 100644 --- a/esphome/components/udp/udp_component.h +++ b/esphome/components/udp/udp_component.h @@ -20,9 +20,8 @@ namespace esphome::udp { static const size_t MAX_PACKET_SIZE = 508; class UDPComponent final : public Component { public: - void set_addresses(std::initializer_list addresses) { this->addresses_ = addresses; } - /// Prevent accidental use of std::string which would dangle - void set_addresses(std::initializer_list addresses) = delete; + /// nullptr-terminated table of addresses; must outlive the component. + void set_addresses(const char *const *addresses) { this->addresses_ = addresses; } void set_listen_address(const char *listen_addr) { this->listen_address_ = network::IPAddress(listen_addr); } void set_listen_port(uint16_t port) { this->listen_port_ = port; } void set_broadcast_port(uint16_t port) { this->broadcast_port_ = port; } @@ -52,7 +51,7 @@ class UDPComponent final : public Component { std::vector ipaddrs_{}; WiFiUDP udp_client_{}; #endif - FixedVector addresses_{}; + const char *const *addresses_{nullptr}; optional listen_address_{}; }; diff --git a/tests/component_tests/udp/__init__.py b/tests/component_tests/udp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/udp/config/udp_addresses.yaml b/tests/component_tests/udp/config/udp_addresses.yaml new file mode 100644 index 0000000000..405c863e10 --- /dev/null +++ b/tests/component_tests/udp/config/udp_addresses.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: test + password: password1 + +udp: + addresses: + - 10.0.0.1 + - 10.0.0.2 diff --git a/tests/component_tests/udp/config/udp_no_addresses.yaml b/tests/component_tests/udp/config/udp_no_addresses.yaml new file mode 100644 index 0000000000..d9d3a53a65 --- /dev/null +++ b/tests/component_tests/udp/config/udp_no_addresses.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: test + password: password1 + +udp: + addresses: [] diff --git a/tests/component_tests/udp/test_udp_addresses.py b/tests/component_tests/udp/test_udp_addresses.py new file mode 100644 index 0000000000..434ef02dcf --- /dev/null +++ b/tests/component_tests/udp/test_udp_addresses.py @@ -0,0 +1,29 @@ +"""Tests for the udp component codegen.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_addresses_are_a_progmem_table( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Addresses live in a nullptr-terminated flash table instead of a heap vector.""" + main_cpp = generate_main(component_config_path("udp_addresses.yaml")) + + assert ( + "static constexpr const char * udp_addresses[] PROGMEM = " + '{"10.0.0.1", "10.0.0.2", nullptr};' in main_cpp + ) + assert "->set_addresses(udp_addresses);" in main_cpp + + +def test_empty_addresses_emit_no_table( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """An empty address list leaves the pointer null instead of emitting a table.""" + main_cpp = generate_main(component_config_path("udp_no_addresses.yaml")) + + assert "udp_addresses" not in main_cpp + assert "set_addresses(" not in main_cpp From 66eec7af15eba41697eca4eb33c0508ab818e295 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 19:58:35 -0500 Subject: [PATCH 143/221] [wifi] Keep a second fallback AP start from resetting the LN882x (#20168) --- esphome/components/wifi/wifi_component_libretiny.cpp | 12 ++++++++++++ tests/components/wifi/test-ap.ln882x-ard.yaml | 9 +++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/components/wifi/test-ap.ln882x-ard.yaml diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index 940f2a0783..1d9fda6d2f 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -28,6 +28,11 @@ extern "C" { } #endif +#ifdef USE_LN882X +#include +#undef DATA // clashes with setup_priority::DATA +#endif + #include "esphome/core/application.h" #include "esphome/core/hal.h" #include "esphome/core/helpers.h" @@ -770,6 +775,13 @@ bool WiFiComponent::wifi_start_ap_(const WiFiAP &ap) { yield(); +#ifdef USE_LN882X + // LibreTiny's LN882H softAP() points ap.bssid at a static array and frees it + // on the next call, tripping FreeRTOS's configASSERT until the watchdog + // resets the chip. Clear it so that a second AP start frees nothing. + static_cast(WiFi.data)->ap.bssid = nullptr; +#endif + return WiFi.softAP(ap.ssid_.c_str(), ap.password_.empty() ? NULL : ap.password_.c_str(), ap.has_channel() ? ap.get_channel() : 1, ap.get_hidden()); } diff --git a/tests/components/wifi/test-ap.ln882x-ard.yaml b/tests/components/wifi/test-ap.ln882x-ard.yaml new file mode 100644 index 0000000000..6a38eb5f27 --- /dev/null +++ b/tests/components/wifi/test-ap.ln882x-ard.yaml @@ -0,0 +1,9 @@ +wifi: + networks: + - ssid: MySSID + password: password1 + ap: + ssid: Fallback Hotspot + password: password2 + +captive_portal: From 8b9d51cf93972a9a94f2abf0010fcbcc983a1bb8 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:21:46 -0500 Subject: [PATCH 144/221] Bump bundled esphome-device-builder to 1.20.0 (#20172) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8d3a5d2cc5..50cb8e137d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.19.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.20.0 RUN \ platformio settings set enable_telemetry No \ From 6f2d2a2001556ad25c6b824851eac85f16207893 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 20:53:47 -0500 Subject: [PATCH 145/221] [libretiny] Fix LN882x boot crash with the logger or a uart on UART0 (#20162) --- esphome/components/libretiny/core.cpp | 24 +++++++++++++++++-- esphome/components/libretiny/core.h | 17 ++++++++++++- .../components/logger/logger_libretiny.cpp | 5 ++++ .../uart/uart_component_libretiny.cpp | 4 ++++ .../logger/test-uart0.ln882x-ard.yaml | 2 ++ tests/components/uart/test.ln882x-ard.yaml | 14 +++++++++++ 6 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 tests/components/logger/test-uart0.ln882x-ard.yaml create mode 100644 tests/components/uart/test.ln882x-ard.yaml diff --git a/esphome/components/libretiny/core.cpp b/esphome/components/libretiny/core.cpp index 8686a41e64..1f5de35d2c 100644 --- a/esphome/components/libretiny/core.cpp +++ b/esphome/components/libretiny/core.cpp @@ -1,6 +1,26 @@ #ifdef USE_LIBRETINY -// HAL functions live in hal.cpp. core.cpp is intentionally empty for -// libretiny — there is no extra component bootstrap to keep here. +// HAL functions live in hal.cpp; this file only holds the serial port +// workaround below. + +#include "core.h" + +#ifdef USE_LN882X +#include + +// The SDK's open port handles (components/serial/serial.c); not in its header. +extern "C" Serial_t *serial_handles[SER_PORT_NUM]; + +namespace esphome::libretiny { + +void ensure_serial_handle(uint8_t port) { + if (port >= SER_PORT_NUM || serial_handles[port] != nullptr) + return; + // serial_init() zeroes and opens it; the port stays open for good. + serial_handles[port] = new Serial_t(); +} + +} // namespace esphome::libretiny +#endif // USE_LN882X #endif // USE_LIBRETINY diff --git a/esphome/components/libretiny/core.h b/esphome/components/libretiny/core.h index f909db4f0f..8f6760c0fc 100644 --- a/esphome/components/libretiny/core.h +++ b/esphome/components/libretiny/core.h @@ -4,6 +4,21 @@ #include -namespace esphome::libretiny {} // namespace esphome::libretiny +namespace esphome::libretiny { + +#ifdef USE_LN882X +/// Give a hardware UART its SDK handle before its Serial begin()s. +/// +/// LibreTiny's LN882H Serial passes the SDK's per port handle to +/// serial_init(), which fills it in, but only the default log port (UART1) +/// has one at boot. Any other port hands it NULL and the chip faults before +/// setup, so a logger or uart on UART0 never boots. Drop this once LibreTiny +/// gives each port storage of its own. +void ensure_serial_handle(uint8_t port); +#else +inline void ensure_serial_handle(uint8_t /*port*/) {} +#endif + +} // namespace esphome::libretiny #endif // USE_LIBRETINY diff --git a/esphome/components/logger/logger_libretiny.cpp b/esphome/components/logger/logger_libretiny.cpp index bc3922c436..9a322f751d 100644 --- a/esphome/components/logger/logger_libretiny.cpp +++ b/esphome/components/logger/logger_libretiny.cpp @@ -1,6 +1,8 @@ #ifdef USE_LIBRETINY #include "logger.h" +#include "esphome/components/libretiny/core.h" + namespace esphome::logger { static const char *const TAG = "logger"; @@ -11,18 +13,21 @@ void Logger::pre_setup() { #if LT_HW_UART0 case UART_SELECTION_UART0: this->hw_serial_ = &Serial0; + libretiny::ensure_serial_handle(0); Serial0.begin(this->baud_rate_); break; #endif #if LT_HW_UART1 case UART_SELECTION_UART1: this->hw_serial_ = &Serial1; + libretiny::ensure_serial_handle(1); Serial1.begin(this->baud_rate_); break; #endif #if LT_HW_UART2 case UART_SELECTION_UART2: this->hw_serial_ = &Serial2; + libretiny::ensure_serial_handle(2); Serial2.begin(this->baud_rate_); break; #endif diff --git a/esphome/components/uart/uart_component_libretiny.cpp b/esphome/components/uart/uart_component_libretiny.cpp index 4eacd980db..293cb9e786 100644 --- a/esphome/components/uart/uart_component_libretiny.cpp +++ b/esphome/components/uart/uart_component_libretiny.cpp @@ -6,6 +6,8 @@ #include "esphome/core/log.h" #include "uart_component_libretiny.h" +#include "esphome/components/libretiny/core.h" + #ifdef USE_LOGGER #include "esphome/components/logger/logger.h" #endif @@ -118,6 +120,8 @@ void LibreTinyUARTComponent::setup() { #endif } + if (this->hardware_idx_ >= 0) + libretiny::ensure_serial_handle(this->hardware_idx_); this->serial_->begin(this->baud_rate_, get_config()); } diff --git a/tests/components/logger/test-uart0.ln882x-ard.yaml b/tests/components/logger/test-uart0.ln882x-ard.yaml new file mode 100644 index 0000000000..864d2b96de --- /dev/null +++ b/tests/components/logger/test-uart0.ln882x-ard.yaml @@ -0,0 +1,2 @@ +logger: + hardware_uart: UART0 diff --git a/tests/components/uart/test.ln882x-ard.yaml b/tests/components/uart/test.ln882x-ard.yaml new file mode 100644 index 0000000000..ccb4edfe40 --- /dev/null +++ b/tests/components/uart/test.ln882x-ard.yaml @@ -0,0 +1,14 @@ +uart: + - id: uart_id + tx_pin: PA2 + rx_pin: PA3 + baud_rate: 9600 + data_bits: 8 + parity: NONE + stop_bits: 1 + +switch: + - platform: uart + name: "UART Switch" + uart_id: uart_id + data: [0x01, 0x02, 0x03] From d9b3f299e0c893e281ec646a2c5361e8a781756d Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:19:12 +1100 Subject: [PATCH 146/221] [exponential_moving_average] Add EMA preserved across reboots (#19882) Co-authored-by: J. Nick Koston --- CODEOWNERS | 1 + .../exponential_moving_average/__init__.py | 1 + .../exponential_moving_average_sensor.cpp | 116 ++++++ .../exponential_moving_average_sensor.h | 61 +++ .../exponential_moving_average/sensor.py | 103 +++++ .../exponential_moving_average/__init__.py | 0 .../exponential_moving_average_test.yaml | 40 ++ .../test_exponential_moving_average.py | 166 ++++++++ .../exponential_moving_average/common.h | 52 +++ .../exponential_moving_average/common.yaml | 25 ++ .../test_exponential_moving_average.cpp | 356 ++++++++++++++++++ .../test.esp32-idf.yaml | 2 + 12 files changed, 923 insertions(+) create mode 100644 esphome/components/exponential_moving_average/__init__.py create mode 100644 esphome/components/exponential_moving_average/exponential_moving_average_sensor.cpp create mode 100644 esphome/components/exponential_moving_average/exponential_moving_average_sensor.h create mode 100644 esphome/components/exponential_moving_average/sensor.py create mode 100644 tests/component_tests/exponential_moving_average/__init__.py create mode 100644 tests/component_tests/exponential_moving_average/config/exponential_moving_average_test.yaml create mode 100644 tests/component_tests/exponential_moving_average/test_exponential_moving_average.py create mode 100644 tests/components/exponential_moving_average/common.h create mode 100644 tests/components/exponential_moving_average/common.yaml create mode 100644 tests/components/exponential_moving_average/sensor/test_exponential_moving_average.cpp create mode 100644 tests/components/exponential_moving_average/test.esp32-idf.yaml diff --git a/CODEOWNERS b/CODEOWNERS index b453cf4538..80f243e82e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -191,6 +191,7 @@ esphome/components/espnow/* @jesserockz esphome/components/espnow/packet_transport/* @EasilyBoredEngineer esphome/components/ethernet_info/* @gtjadsonsantos esphome/components/event/* @nohat +esphome/components/exponential_moving_average/* @clydebarrow esphome/components/exposure_notifications/* @OttoWinter esphome/components/ezo/* @ssieb esphome/components/ezo_pmp/* @carlos-sarmiento diff --git a/esphome/components/exponential_moving_average/__init__.py b/esphome/components/exponential_moving_average/__init__.py new file mode 100644 index 0000000000..c58ce8a01e --- /dev/null +++ b/esphome/components/exponential_moving_average/__init__.py @@ -0,0 +1 @@ +CODEOWNERS = ["@clydebarrow"] diff --git a/esphome/components/exponential_moving_average/exponential_moving_average_sensor.cpp b/esphome/components/exponential_moving_average/exponential_moving_average_sensor.cpp new file mode 100644 index 0000000000..243e2cf31e --- /dev/null +++ b/esphome/components/exponential_moving_average/exponential_moving_average_sensor.cpp @@ -0,0 +1,116 @@ +#include "exponential_moving_average_sensor.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +#include + +namespace esphome::exponential_moving_average { + +static const char *const TAG = "exponential_moving_average"; + +const LogString *time_weighting_to_string(TimeWeighting weighting) { + switch (weighting) { + case TIME_WEIGHTING_PREVIOUS: + return LOG_STR("previous"); + case TIME_WEIGHTING_LINEAR: + return LOG_STR("linear"); + default: + return LOG_STR("new"); + } +} + +ScaledDuration scale_duration(uint32_t ms) { + if (ms < 1000) + return {static_cast(ms), LOG_STR("ms"), 0}; + if (ms < 60 * 1000) + return {ms / 1000.0f, LOG_STR("s"), 1}; + if (ms < 60 * 60 * 1000) + return {ms / (60 * 1000.0f), LOG_STR("min"), 1}; + return {ms / (60 * 60 * 1000.0f), LOG_STR("h"), 1}; +} + +void ExponentialMovingAverageSensor::setup() { + if (this->restore_) { + this->pref_ = this->make_entity_preference(); + float restored; + if (this->pref_.load(&restored) && std::isfinite(restored)) { + this->accumulator_ = restored; + this->publish_state(restored); + } + } + const uint32_t now = App.get_loop_component_start_time(); + this->last_update_ = now; + this->source_->add_on_state_callback( + [this](float value) { this->process_(value, App.get_loop_component_start_time()); }); + // The source may have published during its own setup(), before the callback was added. + if (this->source_->has_state()) + this->process_(this->source_->state, now); +} + +void ExponentialMovingAverageSensor::dump_config() { + LOG_SENSOR("", "Exponential Moving Average Sensor", this); + if (this->time_constant_ms_ != 0) { + const ScaledDuration time_constant = scale_duration(this->time_constant_ms_); + ESP_LOGCONFIG(TAG, + " Time Constant: %.*f %s\n" + " Time Weighting: %s", + time_constant.decimals, time_constant.value, LOG_STR_ARG(time_constant.unit), + LOG_STR_ARG(time_weighting_to_string(this->time_weighting_))); + } else { + ESP_LOGCONFIG(TAG, " Alpha: %.3f", this->alpha_); + } + ESP_LOGCONFIG(TAG, " Restore: %s", YESNO(this->restore_)); +} + +void ExponentialMovingAverageSensor::reset() { this->publish_and_save_(NAN); } + +void ExponentialMovingAverageSensor::process_(float value, uint32_t now) { + if (std::isnan(value)) + return; + // After a reboot the downtime is unknown, so the first interval is measured from setup(). + const uint32_t dt = now - this->last_update_; + this->last_update_ = now; + const float previous = this->previous_value_; + this->previous_value_ = value; + if (std::isnan(this->accumulator_)) { + this->publish_and_save_(value); + return; + } + if (this->time_constant_ms_ == 0) { + this->publish_and_save_(this->alpha_ * value + (1.0f - this->alpha_) * this->accumulator_); + return; + } + // Computed in double with expm1(): when the interval is short compared to the time constant, the weights are + // tiny and float rounding of exp() would swamp them. + const double x = static_cast(dt) / this->time_constant_ms_; + // The share of the old average replaced during this interval. + const double gain = -std::expm1(-x); + const double average = this->accumulator_; + // After a reboot there is no previous reading, so only the new value can be used. + const TimeWeighting weighting = std::isnan(previous) ? TIME_WEIGHTING_NEW : this->time_weighting_; + double result; + switch (weighting) { + case TIME_WEIGHTING_PREVIOUS: + result = average + gain * (previous - average); + break; + case TIME_WEIGHTING_LINEAR: { + // Exact result for a value moving in a straight line from the previous reading to the new one. + const double weight_new = x > 0.0 ? (x + std::expm1(-x)) / x : 0.0; + result = average + (gain - weight_new) * (previous - average) + weight_new * (value - average); + break; + } + default: + result = average + gain * (value - average); + break; + } + this->publish_and_save_(static_cast(result)); +} + +void ExponentialMovingAverageSensor::publish_and_save_(float value) { + this->accumulator_ = value; + this->publish_state(value); + if (this->restore_) + this->pref_.save(&value); +} + +} // namespace esphome::exponential_moving_average diff --git a/esphome/components/exponential_moving_average/exponential_moving_average_sensor.h b/esphome/components/exponential_moving_average/exponential_moving_average_sensor.h new file mode 100644 index 0000000000..98bd88553b --- /dev/null +++ b/esphome/components/exponential_moving_average/exponential_moving_average_sensor.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include + +#include "esphome/core/component.h" +#include "esphome/core/log.h" +#include "esphome/core/preferences.h" +#include "esphome/components/sensor/sensor.h" + +namespace esphome::exponential_moving_average { + +/// Which value is assumed to apply during the time between two readings, when a time constant is used. +enum TimeWeighting : uint8_t { + TIME_WEIGHTING_NEW = 0, + TIME_WEIGHTING_PREVIOUS, + TIME_WEIGHTING_LINEAR, +}; + +const LogString *time_weighting_to_string(TimeWeighting weighting); + +/// A duration in the largest of ms, s, min or h that keeps the value at 1 or more. +struct ScaledDuration { + float value; + const LogString *unit; + uint8_t decimals; +}; + +ScaledDuration scale_duration(uint32_t ms); + +class ExponentialMovingAverageSensor : public sensor::Sensor, public Component { + public: + explicit ExponentialMovingAverageSensor(sensor::Sensor *source) : source_(source) {} + + void setup() override; + void dump_config() override; + + void set_alpha(float alpha) { this->alpha_ = alpha; } + /// When non-zero, each sample is weighted by the time since the previous one instead of by a fixed alpha. + void set_time_constant(uint32_t time_constant_ms) { this->time_constant_ms_ = time_constant_ms; } + void set_time_weighting(TimeWeighting weighting) { this->time_weighting_ = weighting; } + void set_restore(bool restore) { this->restore_ = restore; } + /// Clear the average; the next sample starts it again. + void reset(); + + protected: + void process_(float value, uint32_t now); + void publish_and_save_(float value); + + sensor::Sensor *source_; + ESPPreferenceObject pref_; + float alpha_{0.1f}; + float accumulator_{NAN}; + float previous_value_{NAN}; + uint32_t time_constant_ms_{0}; + uint32_t last_update_{0}; + TimeWeighting time_weighting_{TIME_WEIGHTING_NEW}; + bool restore_{true}; +}; + +} // namespace esphome::exponential_moving_average diff --git a/esphome/components/exponential_moving_average/sensor.py b/esphome/components/exponential_moving_average/sensor.py new file mode 100644 index 0000000000..20c8135636 --- /dev/null +++ b/esphome/components/exponential_moving_average/sensor.py @@ -0,0 +1,103 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import sensor +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACCURACY_DECIMALS, + CONF_ALPHA, + CONF_DEVICE_CLASS, + CONF_ICON, + CONF_ID, + CONF_RESTORE, + CONF_SENSOR, + CONF_STATE_CLASS, + CONF_TIME_CONSTANT, + CONF_UNIT_OF_MEASUREMENT, +) +from esphome.core.entity_helpers import inherit_property_from +from esphome.types import ConfigType + +exponential_moving_average_ns = cg.esphome_ns.namespace("exponential_moving_average") +ExponentialMovingAverageSensor = exponential_moving_average_ns.class_( + "ExponentialMovingAverageSensor", sensor.Sensor, cg.Component +) + +TimeWeighting = exponential_moving_average_ns.enum("TimeWeighting") +TIME_WEIGHTINGS: dict[str, cg.MockObj] = { + "new": TimeWeighting.TIME_WEIGHTING_NEW, + "previous": TimeWeighting.TIME_WEIGHTING_PREVIOUS, + "linear": TimeWeighting.TIME_WEIGHTING_LINEAR, +} + +CONF_TIME_WEIGHTING: str = "time_weighting" + +DEFAULT_ALPHA: float = 0.1 + + +def inherit_accuracy_decimals(decimals: int, config: ConfigType) -> int: + # An average carries more precision than the individual readings. + return decimals + 1 + + +def validate_time_weighting(config: ConfigType) -> ConfigType: + if CONF_TIME_WEIGHTING in config and CONF_TIME_CONSTANT not in config: + raise cv.Invalid( + f"'{CONF_TIME_WEIGHTING}' can only be used with '{CONF_TIME_CONSTANT}'", + path=[CONF_TIME_WEIGHTING], + ) + return config + + +CONFIG_SCHEMA = cv.All( + sensor.sensor_schema(ExponentialMovingAverageSensor) + .extend( + { + cv.Required(CONF_SENSOR): cv.use_id(sensor.Sensor), + cv.Optional(CONF_ALPHA): cv.All( + cv.float_, cv.Range(min=0, min_included=False, max=1) + ), + cv.Optional(CONF_TIME_CONSTANT): cv.positive_time_period_milliseconds, + cv.Optional(CONF_TIME_WEIGHTING): cv.enum(TIME_WEIGHTINGS, lower=True), + cv.Optional(CONF_RESTORE, default=True): cv.boolean, + } + ) + .extend(cv.COMPONENT_SCHEMA), + cv.has_at_most_one_key(CONF_ALPHA, CONF_TIME_CONSTANT), + validate_time_weighting, +) + +FINAL_VALIDATE_SCHEMA = cv.All( + inherit_property_from(CONF_ICON, CONF_SENSOR), + inherit_property_from(CONF_UNIT_OF_MEASUREMENT, CONF_SENSOR), + inherit_property_from( + CONF_ACCURACY_DECIMALS, CONF_SENSOR, transform=inherit_accuracy_decimals + ), + inherit_property_from(CONF_DEVICE_CLASS, CONF_SENSOR), + inherit_property_from(CONF_STATE_CLASS, CONF_SENSOR), +) + + +async def to_code(config: ConfigType) -> None: + source = await cg.get_variable(config[CONF_SENSOR]) + var = cg.new_Pvariable(config[CONF_ID], source) + await cg.register_component(var, config) + await sensor.register_sensor(var, config) + + if (time_constant := config.get(CONF_TIME_CONSTANT)) is not None: + cg.add(var.set_time_constant(time_constant)) + if (weighting := config.get(CONF_TIME_WEIGHTING)) is not None: + cg.add(var.set_time_weighting(weighting)) + else: + cg.add(var.set_alpha(config.get(CONF_ALPHA, DEFAULT_ALPHA))) + cg.add(var.set_restore(config[CONF_RESTORE])) + + +automation.register_apply_action( + "sensor.exponential_moving_average.reset", + automation.maybe_simple_id( + { + cv.Required(CONF_ID): cv.use_id(ExponentialMovingAverageSensor), + } + ), + automation.ApplyCall("reset()"), +) diff --git a/tests/component_tests/exponential_moving_average/__init__.py b/tests/component_tests/exponential_moving_average/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/exponential_moving_average/config/exponential_moving_average_test.yaml b/tests/component_tests/exponential_moving_average/config/exponential_moving_average_test.yaml new file mode 100644 index 0000000000..88735777d2 --- /dev/null +++ b/tests/component_tests/exponential_moving_average/config/exponential_moving_average_test.yaml @@ -0,0 +1,40 @@ +esphome: + name: test + on_boot: + then: + - sensor.exponential_moving_average.reset: ema_default + +esp32: + board: esp32dev + +sensor: + - platform: template + id: source_sensor + unit_of_measurement: "°C" + accuracy_decimals: 1 + device_class: temperature + state_class: measurement + lambda: return 1.0; + - platform: exponential_moving_average + id: ema_default + name: EMA Default + sensor: source_sensor + - platform: exponential_moving_average + id: ema_alpha + name: EMA Alpha + sensor: source_sensor + alpha: 0.25 + unit_of_measurement: "K" + accuracy_decimals: 3 + - platform: exponential_moving_average + id: ema_time_constant + name: EMA Time Constant + sensor: source_sensor + time_constant: 5min + restore: false + - platform: exponential_moving_average + id: ema_linear + name: EMA Linear + sensor: source_sensor + time_constant: 30s + time_weighting: linear diff --git a/tests/component_tests/exponential_moving_average/test_exponential_moving_average.py b/tests/component_tests/exponential_moving_average/test_exponential_moving_average.py new file mode 100644 index 0000000000..fea5dddb00 --- /dev/null +++ b/tests/component_tests/exponential_moving_average/test_exponential_moving_average.py @@ -0,0 +1,166 @@ +"""Tests for the exponential_moving_average sensor.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome import config_validation as cv +from esphome.components.exponential_moving_average.sensor import CONFIG_SCHEMA + + +def test_default_alpha_and_restore( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Without alpha or time_constant, alpha defaults to 0.1 and restore is on.""" + main_cpp = generate_main( + component_config_path("exponential_moving_average_test.yaml") + ) + + assert ( + "new(ema_default) exponential_moving_average::ExponentialMovingAverageSensor(source_sensor);" + in main_cpp + ) + assert "ema_default->set_alpha(0.1f);" in main_cpp + assert "ema_default->set_restore(true);" in main_cpp + + +def test_alpha( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main( + component_config_path("exponential_moving_average_test.yaml") + ) + + assert "ema_alpha->set_alpha(0.25f);" in main_cpp + assert "ema_alpha->set_time_constant" not in main_cpp + + +def test_time_constant_replaces_alpha( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main( + component_config_path("exponential_moving_average_test.yaml") + ) + + assert "ema_time_constant->set_time_constant(300000);" in main_cpp + assert "ema_time_constant->set_alpha" not in main_cpp + assert "ema_time_constant->set_restore(false);" in main_cpp + assert "ema_time_constant->set_time_weighting" not in main_cpp + + +def test_time_weighting( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main( + component_config_path("exponential_moving_average_test.yaml") + ) + + assert "ema_linear->set_time_constant(30000);" in main_cpp + assert ( + "ema_linear->set_time_weighting(exponential_moving_average::TIME_WEIGHTING_LINEAR);" + in main_cpp + ) + + +def test_properties_inherited_from_source( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Unset properties come from the source sensor, with one extra decimal; set ones are kept.""" + main_cpp = generate_main( + component_config_path("exponential_moving_average_test.yaml") + ) + + assert "ema_default->set_accuracy_decimals(2);" in main_cpp + assert "ema_alpha->set_accuracy_decimals(3);" in main_cpp + default_line = next( + line for line in main_cpp.splitlines() if '"EMA Default"' in line + ) + alpha_line = next(line for line in main_cpp.splitlines() if '"EMA Alpha"' in line) + assert "°C" in default_line + assert "temperature" in default_line + assert "K" in alpha_line + + +def test_reset_action( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main( + component_config_path("exponential_moving_average_test.yaml") + ) + + assert "::ema_default->reset();" in main_cpp + + +def test_alpha_and_time_constant_are_exclusive() -> None: + with pytest.raises(cv.Invalid, match="Cannot specify more than one of"): + CONFIG_SCHEMA( + { + "id": "ema", + "name": "EMA", + "sensor": "source", + "alpha": 0.5, + "time_constant": "1min", + } + ) + + +@pytest.mark.parametrize("alpha", [0, -0.1, 1.5]) +def test_alpha_out_of_range(alpha: float) -> None: + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA({"id": "ema", "name": "EMA", "sensor": "source", "alpha": alpha}) + + +@pytest.mark.parametrize("alpha", [0.01, 1]) +def test_alpha_in_range(alpha: float) -> None: + config = CONFIG_SCHEMA( + {"id": "ema", "name": "EMA", "sensor": "source", "alpha": alpha} + ) + + assert config["alpha"] == alpha + + +def test_time_weighting_requires_time_constant() -> None: + with pytest.raises(cv.Invalid, match="can only be used with 'time_constant'"): + CONFIG_SCHEMA( + { + "id": "ema", + "name": "EMA", + "sensor": "source", + "time_weighting": "previous", + } + ) + + +@pytest.mark.parametrize("weighting", ["new", "previous", "linear", "LINEAR"]) +def test_time_weighting_values(weighting: str) -> None: + config = CONFIG_SCHEMA( + { + "id": "ema", + "name": "EMA", + "sensor": "source", + "time_constant": "1min", + "time_weighting": weighting, + } + ) + + assert config["time_weighting"] == weighting.lower() + + +def test_time_weighting_rejects_unknown_value() -> None: + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA( + { + "id": "ema", + "name": "EMA", + "sensor": "source", + "time_constant": "1min", + "time_weighting": "trapezoid", + } + ) diff --git a/tests/components/exponential_moving_average/common.h b/tests/components/exponential_moving_average/common.h new file mode 100644 index 0000000000..599c36bb0d --- /dev/null +++ b/tests/components/exponential_moving_average/common.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include "esphome/components/exponential_moving_average/exponential_moving_average_sensor.h" +#include "esphome/core/preferences.h" +#ifdef USE_HOST +#include "esphome/components/host/preferences.h" +#endif + +namespace esphome::exponential_moving_average::testing { + +class TestableExponentialMovingAverageSensor : public ExponentialMovingAverageSensor { + public: + using ExponentialMovingAverageSensor::ExponentialMovingAverageSensor; + using ExponentialMovingAverageSensor::process_; +}; + +// Unnamed sensors share one preference key, so a second instance created after +// the first one behaves like the same sensor after a reboot. +class ExponentialMovingAverageTest : public ::testing::Test { + protected: + void SetUp() override { + if (const char *prefdir = getenv("ESPHOME_PREFDIR"); prefdir != nullptr) + this->saved_prefdir_ = prefdir; + // Keep preferences away from the user's home directory. + setenv("ESPHOME_PREFDIR", std::filesystem::temp_directory_path().c_str(), 1); +#ifdef USE_HOST + host::setup_preferences(); +#endif + global_preferences->reset(); + } + + void TearDown() override { + global_preferences->reset(); + if (this->saved_prefdir_.has_value()) { + setenv("ESPHOME_PREFDIR", this->saved_prefdir_->c_str(), 1); + } else { + unsetenv("ESPHOME_PREFDIR"); + } + } + + std::optional saved_prefdir_; + sensor::Sensor source_; +}; + +} // namespace esphome::exponential_moving_average::testing diff --git a/tests/components/exponential_moving_average/common.yaml b/tests/components/exponential_moving_average/common.yaml new file mode 100644 index 0000000000..c3cf9da5f9 --- /dev/null +++ b/tests/components/exponential_moving_average/common.yaml @@ -0,0 +1,25 @@ +sensor: + - platform: template + id: ema_source + name: EMA Source + unit_of_measurement: "°C" + accuracy_decimals: 1 + lambda: return 21.5; + update_interval: 10s + - platform: exponential_moving_average + id: ema_alpha + name: EMA Alpha + sensor: ema_source + alpha: 0.2 + - platform: exponential_moving_average + name: EMA Time Constant + sensor: ema_source + time_constant: 5min + time_weighting: previous + restore: false + +button: + - platform: template + name: EMA Reset + on_press: + - sensor.exponential_moving_average.reset: ema_alpha diff --git a/tests/components/exponential_moving_average/sensor/test_exponential_moving_average.cpp b/tests/components/exponential_moving_average/sensor/test_exponential_moving_average.cpp new file mode 100644 index 0000000000..d601374990 --- /dev/null +++ b/tests/components/exponential_moving_average/sensor/test_exponential_moving_average.cpp @@ -0,0 +1,356 @@ +#include + +#include "../common.h" + +namespace esphome::exponential_moving_average::testing { + +TEST_F(ExponentialMovingAverageTest, FirstValueStartsTheAverage) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.setup(); + EXPECT_FALSE(ema.has_state()); + + ema.process_(10.0f, 0); + EXPECT_FLOAT_EQ(ema.state, 10.0f); +} + +TEST_F(ExponentialMovingAverageTest, AlphaWeightsEachValue) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_alpha(0.5f); + ema.setup(); + + ema.process_(10.0f, 0); + ema.process_(20.0f, 0); + EXPECT_FLOAT_EQ(ema.state, 15.0f); + ema.process_(20.0f, 0); + EXPECT_FLOAT_EQ(ema.state, 17.5f); +} + +TEST_F(ExponentialMovingAverageTest, NanValuesAreIgnored) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_alpha(0.5f); + ema.setup(); + + ema.process_(10.0f, 0); + ema.process_(NAN, 0); + EXPECT_FLOAT_EQ(ema.state, 10.0f); + ema.process_(20.0f, 0); + EXPECT_FLOAT_EQ(ema.state, 15.0f); +} + +TEST_F(ExponentialMovingAverageTest, FollowsSourceSensor) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_alpha(0.25f); + ema.setup(); + + this->source_.publish_state(8.0f); + this->source_.publish_state(0.0f); + EXPECT_FLOAT_EQ(ema.state, 6.0f); +} + +TEST_F(ExponentialMovingAverageTest, TimeConstantWeightsByElapsedTime) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(1000); + ema.setup(); + + ema.process_(0.0f, 0); + ema.process_(1.0f, 1000); + EXPECT_NEAR(ema.state, 1.0f - std::exp(-1.0f), 1e-5f); +} + +TEST_F(ExponentialMovingAverageTest, TimeConstantResultDoesNotDependOnSampleRate) { + TestableExponentialMovingAverageSensor fast(&this->source_); + fast.set_time_constant(1000); + fast.set_restore(false); + fast.setup(); + fast.process_(0.0f, 0); + for (uint32_t t = 100; t <= 1000; t += 100) + fast.process_(1.0f, t); + + TestableExponentialMovingAverageSensor slow(&this->source_); + slow.set_time_constant(1000); + slow.set_restore(false); + slow.setup(); + slow.process_(0.0f, 0); + slow.process_(1.0f, 1000); + + EXPECT_NEAR(fast.state, slow.state, 1e-5f); +} + +TEST_F(ExponentialMovingAverageTest, TimeConstantIgnoresRepeatAtSameTime) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(1000); + ema.setup(); + + ema.process_(5.0f, 0); + ema.process_(100.0f, 0); + EXPECT_FLOAT_EQ(ema.state, 5.0f); +} + +TEST_F(ExponentialMovingAverageTest, TimeConstantHandlesTimerWraparound) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(1000); + ema.setup(); + + ema.process_(0.0f, UINT32_MAX - 499); + ema.process_(1.0f, 500); + EXPECT_NEAR(ema.state, 1.0f - std::exp(-1.0f), 1e-5f); +} + +TEST_F(ExponentialMovingAverageTest, ResetStartsANewAverage) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_alpha(0.5f); + ema.setup(); + + ema.process_(10.0f, 0); + ema.reset(); + EXPECT_TRUE(std::isnan(ema.state)); + ema.process_(40.0f, 0); + EXPECT_FLOAT_EQ(ema.state, 40.0f); +} + +TEST_F(ExponentialMovingAverageTest, AverageIsRestoredAfterReboot) { + { + TestableExponentialMovingAverageSensor before(&this->source_); + before.set_alpha(0.5f); + before.setup(); + before.process_(10.0f, 0); + before.process_(20.0f, 0); + } + + TestableExponentialMovingAverageSensor after(&this->source_); + after.set_alpha(0.5f); + after.setup(); + ASSERT_TRUE(after.has_state()); + EXPECT_FLOAT_EQ(after.state, 15.0f); + + // Continues from the restored value rather than starting again. + after.process_(25.0f, 0); + EXPECT_FLOAT_EQ(after.state, 20.0f); +} + +TEST_F(ExponentialMovingAverageTest, NothingRestoredWhenRestoreIsOff) { + { + TestableExponentialMovingAverageSensor before(&this->source_); + before.setup(); + before.process_(10.0f, 0); + } + + TestableExponentialMovingAverageSensor after(&this->source_); + after.set_restore(false); + after.setup(); + EXPECT_FALSE(after.has_state()); + after.process_(30.0f, 0); + EXPECT_FLOAT_EQ(after.state, 30.0f); +} + +TEST_F(ExponentialMovingAverageTest, ResetClearsTheSavedAverage) { + { + TestableExponentialMovingAverageSensor before(&this->source_); + before.setup(); + before.process_(10.0f, 0); + before.reset(); + } + + TestableExponentialMovingAverageSensor after(&this->source_); + after.setup(); + EXPECT_FALSE(after.has_state()); +} + +TEST_F(ExponentialMovingAverageTest, PreviousWeightingCountsGapAtPreviousValue) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(1000); + ema.set_time_weighting(TIME_WEIGHTING_PREVIOUS); + ema.setup(); + + // The value stayed at 20 for an hour before changing to 25. + ema.process_(20.0f, 0); + ema.process_(25.0f, 3600000); + EXPECT_FLOAT_EQ(ema.state, 20.0f); + + // The 25 is counted over the following interval. + ema.process_(25.0f, 3601000); + EXPECT_NEAR(ema.state, 20.0f + 5.0f * (1.0f - std::exp(-1.0f)), 1e-4f); +} + +TEST_F(ExponentialMovingAverageTest, NewWeightingCountsGapAtNewValue) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(1000); + ema.setup(); + + ema.process_(20.0f, 0); + ema.process_(25.0f, 3600000); + EXPECT_FLOAT_EQ(ema.state, 25.0f); +} + +TEST_F(ExponentialMovingAverageTest, LinearWeightingFollowsStraightLine) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(1000); + ema.set_time_weighting(TIME_WEIGHTING_LINEAR); + ema.setup(); + + // An average of a value rising steadily from 0 to 1 over one time constant ends at exp(-1). + ema.process_(0.0f, 0); + ema.process_(1.0f, 1000); + EXPECT_NEAR(ema.state, std::exp(-1.0f), 1e-5f); +} + +TEST_F(ExponentialMovingAverageTest, LinearWeightingMatchesManySmallSteps) { + TestableExponentialMovingAverageSensor coarse(&this->source_); + coarse.set_time_constant(1000); + coarse.set_time_weighting(TIME_WEIGHTING_LINEAR); + coarse.set_restore(false); + coarse.setup(); + coarse.process_(0.0f, 0); + coarse.process_(10.0f, 2000); + + TestableExponentialMovingAverageSensor fine(&this->source_); + fine.set_time_constant(1000); + fine.set_time_weighting(TIME_WEIGHTING_LINEAR); + fine.set_restore(false); + fine.setup(); + fine.process_(0.0f, 0); + for (uint32_t t = 10; t <= 2000; t += 10) + fine.process_(t / 200.0f, t); + + EXPECT_NEAR(coarse.state, fine.state, 1e-3f); +} + +TEST_F(ExponentialMovingAverageTest, LinearWeightingIgnoresRepeatAtSameTime) { + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(1000); + ema.set_time_weighting(TIME_WEIGHTING_LINEAR); + ema.setup(); + + ema.process_(5.0f, 0); + ema.process_(100.0f, 0); + EXPECT_FLOAT_EQ(ema.state, 5.0f); +} + +TEST_F(ExponentialMovingAverageTest, FirstValueAfterRebootUsesNewValue) { + { + TestableExponentialMovingAverageSensor before(&this->source_); + before.setup(); + before.process_(10.0f, 0); + } + + // No reading from before the reboot is known, so the new value is used for the first interval. + TestableExponentialMovingAverageSensor after(&this->source_); + after.set_time_constant(1000); + after.set_time_weighting(TIME_WEIGHTING_PREVIOUS); + after.setup(); + after.process_(20.0f, 1000); + EXPECT_NEAR(after.state, 10.0f + 10.0f * (1.0f - std::exp(-1.0f)), 1e-4f); +} + +TEST_F(ExponentialMovingAverageTest, StartsFromSourceThatAlreadyHasAValue) { + sensor::Sensor source; + source.publish_state(12.0f); + + TestableExponentialMovingAverageSensor ema(&source); + ema.set_alpha(0.5f); + ema.set_restore(false); + ema.setup(); + ASSERT_TRUE(ema.has_state()); + EXPECT_FLOAT_EQ(ema.state, 12.0f); + + // The value read at setup is only counted once. + source.publish_state(20.0f); + EXPECT_FLOAT_EQ(ema.state, 16.0f); +} + +TEST_F(ExponentialMovingAverageTest, SourceValueAtSetupBlendsWithRestoredAverage) { + { + TestableExponentialMovingAverageSensor before(&this->source_); + before.setup(); + before.process_(10.0f, 0); + } + + sensor::Sensor source; + source.publish_state(20.0f); + TestableExponentialMovingAverageSensor after(&source); + after.set_alpha(0.5f); + after.setup(); + EXPECT_FLOAT_EQ(after.state, 15.0f); +} + +TEST_F(ExponentialMovingAverageTest, SourceNanAtSetupIsIgnored) { + sensor::Sensor source; + source.publish_state(NAN); + + TestableExponentialMovingAverageSensor ema(&source); + ema.set_restore(false); + ema.setup(); + EXPECT_FALSE(ema.has_state()); +} + +// Reference weights from the Taylor series, accurate for the small ratios used below. +static double series_gain(double x) { return x - x * x / 2 + x * x * x / 6; } +static double series_weight_new(double x) { return x / 2 - x * x / 6 + x * x * x / 24; } + +TEST_F(ExponentialMovingAverageTest, LinearWeightingAccurateWithLongTimeConstant) { + constexpr uint32_t time_constant = 43200000; // 12 hours + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(time_constant); + ema.set_time_weighting(TIME_WEIGHTING_LINEAR); + ema.set_restore(false); + ema.setup(); + + const double x = 1000.0 / time_constant; + const double weight_new = series_weight_new(x); + const double weight_previous = series_gain(x) - weight_new; + + ema.process_(0.0f, 0); + ema.process_(10.0f, 1000); + const double expected = weight_new * 10.0; + EXPECT_NEAR(ema.state, expected, expected * 1e-4); + + ema.process_(20.0f, 2000); + const double expected2 = expected + weight_previous * (10.0 - expected) + weight_new * (20.0 - expected); + EXPECT_NEAR(ema.state, expected2, expected2 * 1e-4); +} + +TEST_F(ExponentialMovingAverageTest, VeryShortIntervalStillMovesAverage) { + constexpr uint32_t time_constant = 4 * 24 * 3600000; // 4 days, with a reading on every 16 ms loop + TestableExponentialMovingAverageSensor ema(&this->source_); + ema.set_time_constant(time_constant); + ema.set_restore(false); + ema.setup(); + + ema.process_(0.0f, 0); + ema.process_(1000.0f, 16); + const double expected = series_gain(16.0 / time_constant) * 1000.0; + EXPECT_NEAR(ema.state, expected, expected * 1e-4); +} + +TEST(TimeWeightingTest, Names) { + EXPECT_STREQ(LOG_STR_ARG(time_weighting_to_string(TIME_WEIGHTING_NEW)), "new"); + EXPECT_STREQ(LOG_STR_ARG(time_weighting_to_string(TIME_WEIGHTING_PREVIOUS)), "previous"); + EXPECT_STREQ(LOG_STR_ARG(time_weighting_to_string(TIME_WEIGHTING_LINEAR)), "linear"); +} + +struct ScaleDurationCase { + uint32_t ms; + float value; + const char *unit; + uint8_t decimals; +}; + +class ScaleDurationTest : public ::testing::TestWithParam {}; + +TEST_P(ScaleDurationTest, PicksLargestUnitOfAtLeastOne) { + const ScaleDurationCase &c = GetParam(); + const ScaledDuration scaled = scale_duration(c.ms); + EXPECT_FLOAT_EQ(scaled.value, c.value); + EXPECT_STREQ(LOG_STR_ARG(scaled.unit), c.unit); + EXPECT_EQ(scaled.decimals, c.decimals); +} + +INSTANTIATE_TEST_SUITE_P( + Units, ScaleDurationTest, + ::testing::Values(ScaleDurationCase{1, 1.0f, "ms", 0}, ScaleDurationCase{999, 999.0f, "ms", 0}, + ScaleDurationCase{1000, 1.0f, "s", 1}, ScaleDurationCase{95000, 1.5833334f, "min", 1}, + ScaleDurationCase{59999, 59.999f, "s", 1}, ScaleDurationCase{60000, 1.0f, "min", 1}, + ScaleDurationCase{300000, 5.0f, "min", 1}, ScaleDurationCase{3599999, 59.999983f, "min", 1}, + ScaleDurationCase{3600000, 1.0f, "h", 1}, ScaleDurationCase{86400000, 24.0f, "h", 1})); + +} // namespace esphome::exponential_moving_average::testing diff --git a/tests/components/exponential_moving_average/test.esp32-idf.yaml b/tests/components/exponential_moving_average/test.esp32-idf.yaml new file mode 100644 index 0000000000..2f4b6a0f57 --- /dev/null +++ b/tests/components/exponential_moving_average/test.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + exponential_moving_average: !include common.yaml From 3b319240c6d12afca15e4bedf5bfb2cb128c15a0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 21:20:32 -0500 Subject: [PATCH 147/221] [mipi_rgb] Keep the init sequence in flash (#20090) --- esphome/components/mipi_rgb/display.py | 3 ++- esphome/components/mipi_rgb/mipi_rgb.cpp | 16 ++++++++-------- esphome/components/mipi_rgb/mipi_rgb.h | 9 +++++++-- .../mipi_rgb/test_reset_sequence.py | 18 +++++++++++++++--- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/esphome/components/mipi_rgb/display.py b/esphome/components/mipi_rgb/display.py index 9cba56a976..71234c2f25 100644 --- a/esphome/components/mipi_rgb/display.py +++ b/esphome/components/mipi_rgb/display.py @@ -286,7 +286,8 @@ async def to_code(config: ConfigType) -> None: if CONF_SPI_ID in config: await spi.register_spi_device(var, config, write_only=True) sequence = model.get_sequence(config, add_reset=True) - cg.add(var.set_init_sequence(sequence)) + table = cg.shared_progmem_array("mipi_rgb_init_sequence", cg.uint8, sequence) + cg.add(var.set_init_sequence(table, len(sequence))) cg.add(var.set_color_mode(COLOR_ORDERS[config[CONF_COLOR_ORDER]])) cg.add(var.set_invert_colors(config[CONF_INVERT_COLORS])) diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index 034efb9c92..4928b6f667 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -78,27 +78,28 @@ void MipiRgbSpi::write_data_(uint8_t value) { void MipiRgbSpi::write_init_sequence_() { size_t index = 0; - auto &vec = this->init_sequence_; - while (index != vec.size()) { - if (vec.size() - index < 2) { + const uint8_t *seq = this->init_sequence_; + const size_t len = this->init_sequence_len_; + while (index != len) { + if (len - index < 2) { this->mark_failed(LOG_STR("Malformed init sequence")); return; } - uint8_t cmd = vec[index++]; - uint8_t x = vec[index++]; + uint8_t cmd = seq[index++]; + uint8_t x = seq[index++]; if (x == DELAY_FLAG) { ESP_LOGD(TAG, "Delay %dms", cmd); delay(cmd); } else { uint8_t num_args = x & 0x7F; - if (vec.size() - index < num_args) { + if (len - index < num_args) { this->mark_failed(LOG_STR("Malformed init sequence")); return; } if (cmd == SLEEP_OUT) { delay(120); // NOLINT } - const auto *ptr = vec.data() + index; + const auto *ptr = seq + index; char hex_buf[format_hex_pretty_size(MIPI_RGB_MAX_CMD_LOG_BYTES)]; ESP_LOGD(TAG, "Write command %02X, length %d, byte(s) %s", cmd, num_args, format_hex_pretty_to(hex_buf, ptr, num_args, '.')); @@ -111,7 +112,6 @@ void MipiRgbSpi::write_init_sequence_() { } } // this->spi_teardown(); // SPI not needed after this - this->init_sequence_.clear(); delay(10); } diff --git a/esphome/components/mipi_rgb/mipi_rgb.h b/esphome/components/mipi_rgb/mipi_rgb.h index f528943c1b..8b29e273ab 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.h +++ b/esphome/components/mipi_rgb/mipi_rgb.h @@ -109,7 +109,10 @@ class MipiRgbSpi final : public MipiRgb, public: MipiRgbSpi(int width, int height) : MipiRgb(width, height) {} - void set_init_sequence(const std::vector &init_sequence) { this->init_sequence_ = init_sequence; } + void set_init_sequence(const uint8_t *init_sequence, size_t len) { + this->init_sequence_ = init_sequence; + this->init_sequence_len_ = len; + } void set_dc_pin(GPIOPin *dc_pin) { this->dc_pin_ = dc_pin; } void setup() override; @@ -120,7 +123,9 @@ class MipiRgbSpi final : public MipiRgb, void dump_config() override; GPIOPin *dc_pin_{nullptr}; - std::vector init_sequence_; + // Shared PROGMEM table + const uint8_t *init_sequence_{nullptr}; + size_t init_sequence_len_{0}; }; #endif diff --git a/tests/component_tests/mipi_rgb/test_reset_sequence.py b/tests/component_tests/mipi_rgb/test_reset_sequence.py index 50478361b1..b0fd3ed573 100644 --- a/tests/component_tests/mipi_rgb/test_reset_sequence.py +++ b/tests/component_tests/mipi_rgb/test_reset_sequence.py @@ -9,6 +9,7 @@ here. from collections.abc import Callable from pathlib import Path +import re # A model with no reset_pin default: SWRESET ({1, 0}) is prepended ahead of the # inherited ST7701S reset_delay ({50, 255}). @@ -54,6 +55,18 @@ display: """ +def _init_sequence(main_cpp: str, display_id: str) -> str: + """Return the shared PROGMEM init sequence table passed to a display, as rendered.""" + match = re.search(rf"{display_id}->set_init_sequence\((\w+), \d+\);", main_cpp) + assert match is not None + table = re.search( + rf"static constexpr uint8_t {match.group(1)}\[\] PROGMEM = (\{{[^;]*\}});", + main_cpp, + ) + assert table is not None + return table.group(1) + + def test_swreset_and_reset_delay_without_reset_pin( generate_main: Callable[[str | Path], str], tmp_path: Path, @@ -64,7 +77,7 @@ def test_swreset_and_reset_delay_without_reset_pin( main_cpp = generate_main(yaml_file) - assert "no_reset_display->set_init_sequence({1, 0, 50, 255," in main_cpp + assert _init_sequence(main_cpp, "no_reset_display").startswith("{1, 0, 50, 255,") def test_reset_delay_only_with_reset_pin( @@ -77,5 +90,4 @@ def test_reset_delay_only_with_reset_pin( main_cpp = generate_main(yaml_file) - assert "has_reset_display->set_init_sequence({50, 255," in main_cpp - assert "has_reset_display->set_init_sequence({1, 0," not in main_cpp + assert _init_sequence(main_cpp, "has_reset_display").startswith("{50, 255,") From f91ac5ed1d3226113957d912b265ca0915ae5c15 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 21:20:56 -0500 Subject: [PATCH 148/221] [mipi_dsi] Keep the init sequence in flash (#20091) --- esphome/components/mipi_dsi/display.py | 3 ++- esphome/components/mipi_dsi/mipi_dsi.cpp | 15 ++++++++------- esphome/components/mipi_dsi/mipi_dsi.h | 9 +++++++-- .../mipi_dsi/test_mipi_dsi_config.py | 8 +++++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/esphome/components/mipi_dsi/display.py b/esphome/components/mipi_dsi/display.py index b23982655a..6bf94bf364 100644 --- a/esphome/components/mipi_dsi/display.py +++ b/esphome/components/mipi_dsi/display.py @@ -201,7 +201,8 @@ async def to_code(config: ConfigType) -> None: sequence = model.get_sequence(config) cg.add(var.set_model(config[CONF_MODEL])) - cg.add(var.set_init_sequence(sequence)) + table = cg.shared_progmem_array("mipi_dsi_init_sequence", cg.uint8, sequence) + cg.add(var.set_init_sequence(table, len(sequence))) cg.add(var.set_invert_colors(config[CONF_INVERT_COLORS])) cg.add(var.set_hsync_pulse_width(config[CONF_HSYNC_PULSE_WIDTH])) cg.add(var.set_hsync_back_porch(config[CONF_HSYNC_BACK_PORCH])) diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index b6612038b6..a27be568f2 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -121,20 +121,21 @@ void MipiDsi::setup() { return; } size_t index = 0; - auto &vec = this->init_sequence_; - while (index != vec.size()) { - if (vec.size() - index < 2) { + const uint8_t *seq = this->init_sequence_; + const size_t len = this->init_sequence_len_; + while (index != len) { + if (len - index < 2) { this->mark_failed(LOG_STR("Malformed init sequence")); return; } - uint8_t cmd = vec[index++]; - uint8_t x = vec[index++]; + uint8_t cmd = seq[index++]; + uint8_t x = seq[index++]; if (x == DELAY_FLAG) { ESP_LOGD(TAG, "Delay %dms", cmd); delay(cmd); } else { uint8_t num_args = x & 0x7F; - if (vec.size() - index < num_args) { + if (len - index < num_args) { this->mark_failed(LOG_STR("Malformed init sequence")); return; } @@ -145,7 +146,7 @@ void MipiDsi::setup() { delay(duration); } } - const auto *ptr = vec.data() + index; + const auto *ptr = seq + index; #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERY_VERBOSE char hex_buf[format_hex_pretty_size(MIPI_DSI_MAX_CMD_LOG_BYTES)]; #endif diff --git a/esphome/components/mipi_dsi/mipi_dsi.h b/esphome/components/mipi_dsi/mipi_dsi.h index 7bf2feb73c..4243d3116c 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.h +++ b/esphome/components/mipi_dsi/mipi_dsi.h @@ -55,7 +55,10 @@ class MipiDsi final : public display::Display { void set_vsync_pulse_width(uint16_t vsync_pulse_width) { this->vsync_pulse_width_ = vsync_pulse_width; } void set_vsync_back_porch(uint16_t vsync_back_porch) { this->vsync_back_porch_ = vsync_back_porch; } void set_vsync_front_porch(uint16_t vsync_front_porch) { this->vsync_front_porch_ = vsync_front_porch; } - void set_init_sequence(const std::vector &init_sequence) { this->init_sequence_ = init_sequence; } + void set_init_sequence(const uint8_t *init_sequence, size_t len) { + this->init_sequence_ = init_sequence; + this->init_sequence_len_ = len; + } void set_model(const char *model) { this->model_ = model; } void set_lane_bit_rate(float lane_bit_rate) { this->lane_bit_rate_ = lane_bit_rate; } void set_lanes(uint8_t lanes) { this->lanes_ = lanes; } @@ -91,7 +94,9 @@ class MipiDsi final : public display::Display { uint16_t vsync_back_porch_ = 10; uint16_t vsync_front_porch_ = 10; const char *model_{"Unknown"}; - std::vector init_sequence_{}; + // Shared PROGMEM table + const uint8_t *init_sequence_{nullptr}; + size_t init_sequence_len_{0}; float pclk_frequency_ = 16; // in MHz float lane_bit_rate_{1500}; // in Mbps uint8_t lanes_{2}; // 1, 2, 3 or 4 lanes diff --git a/tests/component_tests/mipi_dsi/test_mipi_dsi_config.py b/tests/component_tests/mipi_dsi/test_mipi_dsi_config.py index 6259d85184..ac385cc2d8 100644 --- a/tests/component_tests/mipi_dsi/test_mipi_dsi_config.py +++ b/tests/component_tests/mipi_dsi/test_mipi_dsi_config.py @@ -3,6 +3,7 @@ from collections.abc import Callable import logging from pathlib import Path +import re import pytest @@ -201,7 +202,12 @@ def test_code_generation( "new(p4_nano) mipi_dsi::MipiDsi(800, 1280, display::COLOR_BITNESS_565, 16);" in main_cpp ) - assert "set_init_sequence({224, 1, 0, 225, 1, 147, 226, 1," in main_cpp + seq = re.search(r"p4_nano->set_init_sequence\((\w+), \d+\);", main_cpp) + assert seq is not None + assert ( + f"static constexpr uint8_t {seq.group(1)}[] PROGMEM = " + "{224, 1, 0, 225, 1, 147, 226, 1," in main_cpp + ) assert "p4_nano->set_lane_bit_rate(1500.0f);" in main_cpp assert "p4_nano->set_rotation(display::DISPLAY_ROTATION_90_DEGREES);" in main_cpp assert "p4_86->set_rotation(display::DISPLAY_ROTATION_0_DEGREES);" not in main_cpp From 88378ab354301687c0aa5a93a0ed72e442bacec5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 21:21:28 -0500 Subject: [PATCH 149/221] [mipi_spi] Keep the init sequence in flash (#20089) --- esphome/components/mipi_spi/display.py | 3 +- esphome/components/mipi_spi/mipi_spi.h | 35 +++++++++------ tests/component_tests/mipi_spi/test_init.py | 49 ++++++++++++++++++--- 3 files changed, 68 insertions(+), 19 deletions(-) diff --git a/esphome/components/mipi_spi/display.py b/esphome/components/mipi_spi/display.py index e8b54da5c7..b49675ca4c 100644 --- a/esphome/components/mipi_spi/display.py +++ b/esphome/components/mipi_spi/display.py @@ -401,7 +401,8 @@ async def to_code(config: ConfigType) -> None: init_sequence = model.get_sequence(config, add_madctl=False, add_reset=True) var_id.type, templateargs = get_instance(config) var = cg.new_Pvariable(var_id, TemplateArguments(*templateargs)) - cg.add(var.set_init_sequence(init_sequence)) + table = cg.shared_progmem_array("mipi_spi_init_sequence", cg.uint8, init_sequence) + cg.add(var.set_init_sequence(table, len(init_sequence))) cg.add(var.set_model(config[CONF_MODEL])) if enable_pin := config.get(CONF_ENABLE_PIN): enable = [await cg.gpio_pin_expression(pin) for pin in enable_pin] diff --git a/esphome/components/mipi_spi/mipi_spi.h b/esphome/components/mipi_spi/mipi_spi.h index e97b0e3a31..e428a1f96e 100644 --- a/esphome/components/mipi_spi/mipi_spi.h +++ b/esphome/components/mipi_spi/mipi_spi.h @@ -5,6 +5,7 @@ #include "esphome/components/spi/spi.h" #include "esphome/components/display/display.h" #include "esphome/components/display/display_color_utils.h" +#include "esphome/core/hal.h" #include "esphome/core/helpers.h" namespace esphome::mipi_spi { @@ -133,7 +134,10 @@ class MipiSpi : public display::Display, return HEIGHT; } - void set_init_sequence(const std::vector &sequence) { this->init_sequence_ = sequence; } + void set_init_sequence(const uint8_t *sequence, size_t len) { + this->init_sequence_ = sequence; + this->init_sequence_len_ = len; + } // reset the display, and write the init sequence void setup() override { @@ -159,15 +163,16 @@ class MipiSpi : public display::Display, // need to know when the display is ready for SLPOUT command - will be 120ms after reset auto when = millis() + 120; size_t index = 0; - auto &vec = this->init_sequence_; - while (index != vec.size()) { - if (vec.size() - index < 2) { + const uint8_t *seq = this->init_sequence_; + const size_t len = this->init_sequence_len_; + while (index != len) { + if (len - index < 2) { esph_log_e(TAG, "Malformed init sequence"); this->mark_failed(); return; } - uint8_t cmd = vec[index++]; - uint8_t x = vec[index++]; + uint8_t cmd = progmem_read_byte(seq + index++); + uint8_t x = progmem_read_byte(seq + index++); if (x == DELAY_FLAG) { if (cmd == 0) { cmd = clamp_at_least((int) (when - millis()), 0); @@ -176,19 +181,21 @@ class MipiSpi : public display::Display, delay(cmd); } else { uint8_t num_args = x & 0x7F; - if (vec.size() - index < num_args) { + if (len - index < num_args) { esph_log_e(TAG, "Malformed init sequence"); this->mark_failed(); return; } - const auto *ptr = vec.data() + index; - this->write_command_(cmd, ptr, num_args); + // The sequence is in flash, which SPI DMA (and ESP8266 byte loads) cannot read + uint8_t args[0x80]; + progmem_memcpy(args, seq + index, num_args); + this->write_command_(cmd, args, num_args); index += num_args; } } this->reset_params_(); - // init sequence no longer needed - this->init_sequence_.clear(); + // Marks init as done, so later commands log at verbose level instead of debug + this->init_sequence_len_ = 0; } // Drawing operations @@ -236,7 +243,7 @@ class MipiSpi : public display::Display, void write_command_(uint8_t cmd, const uint8_t *bytes, size_t len) { char hex_buf[format_hex_pretty_size(MIPI_SPI_MAX_CMD_LOG_BYTES)]; // Don't spam the log after setup - if (this->init_sequence_.empty()) { + if (this->init_sequence_len_ == 0) { esph_log_v(TAG, "Command %02X, length %d, bytes %s", cmd, len, format_hex_pretty_to(hex_buf, bytes, len)); } else { esph_log_d(TAG, "Command %02X, length %d, bytes %s", cmd, len, format_hex_pretty_to(hex_buf, bytes, len)); @@ -486,7 +493,9 @@ class MipiSpi : public display::Display, bool invert_colors_{}; optional brightness_{}; const char *model_{"Unknown"}; - std::vector init_sequence_{}; + // Shared PROGMEM table + const uint8_t *init_sequence_{nullptr}; + size_t init_sequence_len_{0}; }; /** diff --git a/tests/component_tests/mipi_spi/test_init.py b/tests/component_tests/mipi_spi/test_init.py index f29883684c..cacdc0c106 100644 --- a/tests/component_tests/mipi_spi/test_init.py +++ b/tests/component_tests/mipi_spi/test_init.py @@ -2,6 +2,7 @@ from collections.abc import Callable from pathlib import Path +import re from typing import Any import pytest @@ -33,6 +34,18 @@ from esphome.types import ConfigType from tests.component_tests.types import SetCoreConfigCallable +def _init_sequence(main_cpp: str, display_id: str = r"\w+") -> str: + """Return the shared PROGMEM init sequence table passed to a display, as rendered.""" + match = re.search(rf"{display_id}->set_init_sequence\((\w+), \d+\);", main_cpp) + assert match is not None + table = re.search( + rf"static constexpr uint8_t {match.group(1)}\[\] PROGMEM = (\{{[^;]*\}});", + main_cpp, + ) + assert table is not None + return table.group(1) + + def run_schema_validation(config: ConfigType) -> None: """Run schema validation on a configuration.""" FINAL_VALIDATE_SCHEMA(CONFIG_SCHEMA(config)) @@ -362,7 +375,7 @@ def test_native_generation( in main_cpp ) # A 10ms post-reset delay ({10, 255}) is prepended ahead of the model commands. - assert "set_init_sequence({10, 255, 240, 1, 8, 242" in main_cpp + assert _init_sequence(main_cpp).startswith("{10, 255, 240, 1, 8, 242") assert "show_test_card();" in main_cpp assert "set_write_only(true);" in main_cpp @@ -379,7 +392,7 @@ def test_lvgl_generation( in main_cpp ) # A 10ms post-reset delay ({10, 255}) is prepended ahead of the model commands. - assert "set_init_sequence({10, 255, 177, 3, 1, 44, 45, 178" in main_cpp + assert _init_sequence(main_cpp).startswith("{10, 255, 177, 3, 1, 44, 45, 178") assert "show_test_card();" not in main_cpp assert "set_auto_clear(false);" in main_cpp @@ -428,7 +441,9 @@ def test_swreset_prepended_without_reset_pin( # SWRESET ({1, 0}) followed by a 10ms delay ({10, 255}) is inserted ahead of # the model's own commands. - assert "swreset_display->set_init_sequence({1, 0, 10, 255, 160, 1, 1," in main_cpp + assert _init_sequence(main_cpp, "swreset_display").startswith( + "{1, 0, 10, 255, 160, 1, 1," + ) def test_swreset_not_prepended_with_reset_pin( @@ -449,5 +464,29 @@ def test_swreset_not_prepended_with_reset_pin( main_cpp = generate_main(yaml_file) # The delay ({10, 255}) is still present, but no leading SWRESET ({1, 0}). - assert "hwreset_display->set_init_sequence({10, 255, 160, 1, 1," in main_cpp - assert "hwreset_display->set_init_sequence({1, 0," not in main_cpp + assert _init_sequence(main_cpp, "hwreset_display").startswith( + "{10, 255, 160, 1, 1," + ) + + +def test_identical_init_sequences_share_one_table( + generate_main: Callable[[str | Path], str], + tmp_path: Path, +) -> None: + """Two displays with the same init sequence point at one PROGMEM table.""" + yaml_file = tmp_path / "shared.yaml" + display = _SWRESET_YAML.split("display:\n", 1)[1] + yaml_file.write_text( + _SWRESET_YAML.format(display_id="first", reset_line=" reset_pin: 5") + + display.format(display_id="second", reset_line=" reset_pin: 6") + .replace("cs_pin: 8", "cs_pin: 9") + .replace("dc_pin: 4", "dc_pin: 7") + ) + + main_cpp = generate_main(yaml_file) + + first = re.search(r"first->set_init_sequence\((\w+), \d+\);", main_cpp) + second = re.search(r"second->set_init_sequence\((\w+), \d+\);", main_cpp) + assert first is not None and second is not None + assert first.group(1) == second.group(1) + assert main_cpp.count("mipi_spi_init_sequence[] PROGMEM") == 1 From 645144ad4d4a6ca4c1b1b2c9292eb8a759fc9200 Mon Sep 17 00:00:00 2001 From: Max Slotov Date: Mon, 5 Oct 2026 05:21:48 +0300 Subject: [PATCH 150/221] [uart] Validate parity and stop bits correctly (#20161) --- esphome/components/uart/__init__.py | 3 +- esphome/components/usb_uart/usb_uart.h | 17 ++++++- .../uart/test_final_validate.py | 46 +++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 tests/component_tests/uart/test_final_validate.py diff --git a/esphome/components/uart/__init__.py b/esphome/components/uart/__init__.py index 24a854ec39..25fbfb7599 100644 --- a/esphome/components/uart/__init__.py +++ b/esphome/components/uart/__init__.py @@ -489,7 +489,8 @@ def final_validate_device_schema( return value def validate_stop_bits(value): - if value != stop_bits: + # usb_uart channels store stop bits as strings ("1", "1.5", "2"). + if float(value) != stop_bits: raise cv.Invalid( f"Component {name} requires {stop_bits} stop bits for the uart referenced by {uart_bus}" ) diff --git a/esphome/components/usb_uart/usb_uart.h b/esphome/components/usb_uart/usb_uart.h index 22563209da..c99992d878 100644 --- a/esphome/components/usb_uart/usb_uart.h +++ b/esphome/components/usb_uart/usb_uart.h @@ -155,7 +155,22 @@ class USBUartChannelBase : public uart::UARTComponent, public Parentedparity_ = parity; } + void set_parity(UARTParityOptions parity) { + this->parity_ = parity; + // Keep the base-class parity in sync so uart::UARTComponent::get_parity() reports the configured value. + // MARK/SPACE have no uart:: equivalent and report as NONE. + switch (parity) { + case UART_CONFIG_PARITY_EVEN: + uart::UARTComponent::set_parity(uart::UART_CONFIG_PARITY_EVEN); + break; + case UART_CONFIG_PARITY_ODD: + uart::UARTComponent::set_parity(uart::UART_CONFIG_PARITY_ODD); + break; + default: + uart::UARTComponent::set_parity(uart::UART_CONFIG_PARITY_NONE); + break; + } + } void set_debug(bool debug) { this->debug_ = debug; } void set_dummy_receiver(bool dummy_receiver) { this->dummy_receiver_ = dummy_receiver; } void set_debug_prefix(const char *prefix) { this->debug_prefix_ = StringRef(prefix); } diff --git a/tests/component_tests/uart/test_final_validate.py b/tests/component_tests/uart/test_final_validate.py new file mode 100644 index 0000000000..b2ad5f4390 --- /dev/null +++ b/tests/component_tests/uart/test_final_validate.py @@ -0,0 +1,46 @@ +"""Tests for final validation of UART device requirements against usb_uart channels.""" + +from pathlib import Path + +import pytest + +from esphome import config, yaml_util +from esphome.core import CORE + +USB_UART_CONFIG = """ +esphome: + name: usb-uart-final-validate +esp32: + variant: esp32s3 + framework: + type: esp-idf +usb_uart: + - type: ch340 + channels: + - id: meter_uart + baud_rate: 1200 + stop_bits: {stop_bits} +sensor: + - platform: kamstrup_kmp + uart_id: meter_uart + heat_energy: + name: Heat energy +""" + + +@pytest.mark.parametrize( + ("stop_bits", "valid"), [("2", True), ("1", False), ("1.5", False)] +) +def test_stop_bits_requirement_on_usb_uart_channel( + stop_bits: str, valid: bool, tmp_path: Path +) -> None: + """usb_uart stores stop bits as strings; they must still satisfy a numeric requirement.""" + path = tmp_path / "usb_uart.yaml" + path.write_text(USB_UART_CONFIG.format(stop_bits=stop_bits), encoding="utf-8") + CORE.config_path = path + result = config.validate_config(yaml_util.load_yaml(path), {}) + errors = [str(err) for err in result.errors] + if valid: + assert errors == [] + else: + assert any("requires 2 stop bits" in err for err in errors), errors From 2d73ea24ca2877307d0359f04cd7f76f1d6125a8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 4 Oct 2026 21:23:09 -0500 Subject: [PATCH 151/221] [libretiny] Keep a scan that finds no networks from resetting the LN882x (#20169) --- esphome/components/libretiny/__init__.py | 5 +++++ .../components/libretiny/ln882x_zero_malloc.c | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 esphome/components/libretiny/ln882x_zero_malloc.c diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index 8cd7f3e7b6..5d87dee07a 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -32,6 +32,7 @@ from esphome.storage_json import StorageJSON from . import gpio # noqa: F401 from .const import ( COMPONENT_BK72XX, + COMPONENT_LN882X, CONF_GPIO_RECOVER, CONF_LOGLEVEL, CONF_SDK_SILENT, @@ -490,6 +491,10 @@ async def component_to_code(config): # Not enabled on RTL87xx/LN882x — costs more heap than it saves there. cg.add_build_flag("-DconfigSUPPORT_STATIC_ALLOCATION=1") + # LN882x: a zero-size allocation must not trip the SDK's assert (see ln882x_zero_malloc.c). + if config[CONF_COMPONENT_ID] == COMPONENT_LN882X: + cg.add_build_flag("-Wl,--wrap=pvPortMalloc") + # RTL8710B needs FreeRTOS 8.2.3+ for xTaskNotifyGive/ulTaskNotifyTake # required by AsyncTCP 3.4.3+ (https://github.com/esphome/esphome/issues/10220) # RTL8720C (ambz2) requires FreeRTOS 10.x so this only applies to RTL8710B diff --git a/esphome/components/libretiny/ln882x_zero_malloc.c b/esphome/components/libretiny/ln882x_zero_malloc.c new file mode 100644 index 0000000000..77e30b3a7d --- /dev/null +++ b/esphome/components/libretiny/ln882x_zero_malloc.c @@ -0,0 +1,19 @@ +/* + * LN882x: heap_5 returns NULL for a zero-size request, and the SDK's + * malloc-failed hook ends in LN_ASSERT(0), which spins until the watchdog + * resets the chip. LibreTiny's scan handler asks for zero entries when a scan + * finds no networks. Linked with -Wl,--wrap=pvPortMalloc so every allocation + * comes through here; drop it once LibreTiny no longer allocates zero entries. + */ + +#ifdef USE_LN882X + +#include + +void *__real_pvPortMalloc(size_t size); // NOLINT(readability-identifier-naming) + +void *__wrap_pvPortMalloc(size_t size) { // NOLINT(readability-identifier-naming) + return __real_pvPortMalloc(size == 0 ? 1 : size); +} + +#endif // USE_LN882X From 99fd1bca3e0f68ae052949dfe4fb8522e8074f87 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 05:58:43 +0000 Subject: [PATCH 152/221] [ci] Refresh integration test durations --- .../integration_test_durations.json | 329 +++++++++--------- 1 file changed, 167 insertions(+), 162 deletions(-) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json index 4f2e245e12..060799862f 100644 --- a/tests/integration/integration_test_durations.json +++ b/tests/integration/integration_test_durations.json @@ -1,164 +1,169 @@ { - "tests/integration/test_action_concurrent_reentry.py": 14.22, - "tests/integration/test_addressable_light_transition.py": 12.87, - "tests/integration/test_alarm_control_panel_state_transitions.py": 10.89, - "tests/integration/test_api_action_metadata.py": 8.48, - "tests/integration/test_api_action_responses.py": 14.99, - "tests/integration/test_api_action_timeout.py": 17.11, - "tests/integration/test_api_conditional_memory.py": 12.13, - "tests/integration/test_api_custom_services.py": 13.35, - "tests/integration/test_api_decode_wire_types.py": 11.09, - "tests/integration/test_api_empty_message_roundtrip.py": 11.44, - "tests/integration/test_api_encode_boundaries.py": 14.78, - "tests/integration/test_api_get_time_response_timezone.py": 13.95, - "tests/integration/test_api_homeassistant.py": 16.39, - "tests/integration/test_api_homeassistant_action_no_subscriber.py": 10.59, - "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 7.64, - "tests/integration/test_api_list_entities_backpressure.py": 13.25, - "tests/integration/test_api_message_size_batching.py": 9.9, - "tests/integration/test_api_reboot_timeout.py": 12.0, - "tests/integration/test_api_string_lambda.py": 9.04, - "tests/integration/test_api_vv_logging.py": 12.89, - "tests/integration/test_api_zero_psk_provisioning.py": 17.56, - "tests/integration/test_areas_and_devices.py": 9.95, - "tests/integration/test_automation_wait_actions.py": 11.31, - "tests/integration/test_automations.py": 26.82, - "tests/integration/test_batch_delay_zero_rapid_transitions.py": 10.76, - "tests/integration/test_binary_sensor_autorepeat_filter.py": 11.71, - "tests/integration/test_binary_sensor_invalidate_state.py": 9.11, - "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 12.55, - "tests/integration/test_build_info.py": 8.14, - "tests/integration/test_camera_mock.py": 9.88, - "tests/integration/test_climate_control_action.py": 12.5, - "tests/integration/test_climate_custom_modes.py": 10.59, - "tests/integration/test_continuation_actions.py": 10.74, - "tests/integration/test_cover_control_action.py": 11.06, - "tests/integration/test_crc8_helper.py": 9.21, - "tests/integration/test_device_id_in_state.py": 20.43, - "tests/integration/test_duplicate_entities.py": 12.01, - "tests/integration/test_entity_icon.py": 14.29, - "tests/integration/test_entity_missing_state.py": 11.84, - "tests/integration/test_fan_turn_on_action.py": 13.41, - "tests/integration/test_fnv1_hash_object_id.py": 6.02, - "tests/integration/test_fnv1a_hash.py": 9.59, - "tests/integration/test_gpio_expander_cache.py": 7.68, - "tests/integration/test_host_logger_thread_safety.py": 12.31, - "tests/integration/test_host_mode_basic.py": 9.92, - "tests/integration/test_host_mode_batch_delay.py": 8.99, - "tests/integration/test_host_mode_climate_basic_state.py": 13.73, - "tests/integration/test_host_mode_climate_control.py": 10.2, - "tests/integration/test_host_mode_empty_string_options.py": 11.81, - "tests/integration/test_host_mode_entity_fields.py": 14.54, - "tests/integration/test_host_mode_fan_preset.py": 12.02, - "tests/integration/test_host_mode_many_entities.py": 22.44, - "tests/integration/test_host_mode_many_entities_multiple_connections.py": 12.55, - "tests/integration/test_host_mode_noise_encryption.py": 25.74, - "tests/integration/test_host_mode_reconnect.py": 6.67, - "tests/integration/test_host_mode_sensor.py": 11.49, - "tests/integration/test_host_ota.py": 112.15, - "tests/integration/test_host_preferences.py": 12.61, - "tests/integration/test_host_preferences_suspend_resume.py": 11.62, - "tests/integration/test_improv_serial_uart.py": 8.08, - "tests/integration/test_large_message_batching.py": 10.44, - "tests/integration/test_legacy_area.py": 7.85, - "tests/integration/test_legacy_climate_compat.py": 13.36, - "tests/integration/test_legacy_fan_compat.py": 10.6, - "tests/integration/test_light_automations.py": 10.63, - "tests/integration/test_light_binary_effect_off_phase.py": 20.09, - "tests/integration/test_light_calls.py": 15.97, - "tests/integration/test_light_constant_brightness.py": 15.27, - "tests/integration/test_light_control_action.py": 13.14, - "tests/integration/test_light_dim_relative_action.py": 10.89, - "tests/integration/test_light_effect_zero_brightness.py": 11.44, - "tests/integration/test_light_initial_state.py": 8.19, - "tests/integration/test_light_restore_state.py": 10.78, - "tests/integration/test_light_resume_effect.py": 12.26, - "tests/integration/test_light_toggle_action.py": 10.53, - "tests/integration/test_light_transition_state_publish_interval.py": 43.37, - "tests/integration/test_lock_automations.py": 10.88, - "tests/integration/test_logger_buffered_recursion_guard.py": 8.93, - "tests/integration/test_loop_disable_enable.py": 11.81, - "tests/integration/test_loop_interval_decoupling.py": 14.96, - "tests/integration/test_loop_interval_default_not_pulled_forward.py": 15.07, - "tests/integration/test_lvgl_headless_render.py": 55.09, - "tests/integration/test_micros_to_millis.py": 7.96, - "tests/integration/test_multi_click_trigger.py": 14.25, - "tests/integration/test_multi_device_preferences.py": 11.2, - "tests/integration/test_noise_encryption_key_protection.py": 20.23, - "tests/integration/test_object_id_api_verification.py": 15.03, - "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 9.16, - "tests/integration/test_object_id_no_friendly_name.py": 13.66, - "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 22.28, - "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 28.84, - "tests/integration/test_online_image_bmp.py": 14.85, - "tests/integration/test_oversized_payloads.py": 31.19, - "tests/integration/test_preference_key_stability.py": 8.3, - "tests/integration/test_runtime_stats.py": 14.29, - "tests/integration/test_safe_mode_loop_runs.py": 7.53, - "tests/integration/test_scheduler_blocking_warning.py": 26.59, - "tests/integration/test_scheduler_bulk_cleanup.py": 9.11, - "tests/integration/test_scheduler_defer_cancel.py": 7.84, - "tests/integration/test_scheduler_defer_cancel_regular.py": 8.05, - "tests/integration/test_scheduler_defer_fifo_simple.py": 12.32, - "tests/integration/test_scheduler_defer_stress.py": 12.6, - "tests/integration/test_scheduler_heap_stress.py": 7.94, - "tests/integration/test_scheduler_internal_id_no_collision.py": 13.8, - "tests/integration/test_scheduler_interval_reschedule.py": 8.2, - "tests/integration/test_scheduler_interval_zero_coerced.py": 8.34, - "tests/integration/test_scheduler_null_name.py": 11.88, - "tests/integration/test_scheduler_numeric_id_test.py": 13.83, - "tests/integration/test_scheduler_pool.py": 13.71, - "tests/integration/test_scheduler_rapid_cancellation.py": 4.5, - "tests/integration/test_scheduler_recursive_timeout.py": 2.26, - "tests/integration/test_scheduler_removed_item_race.py": 10.73, - "tests/integration/test_scheduler_self_keyed.py": 9.9, - "tests/integration/test_scheduler_simultaneous_callbacks.py": 11.04, - "tests/integration/test_scheduler_string_test.py": 13.88, - "tests/integration/test_script_array_params.py": 12.34, - "tests/integration/test_script_delay_params.py": 13.26, - "tests/integration/test_script_queued.py": 15.09, - "tests/integration/test_script_queued_idle_loop.py": 9.25, - "tests/integration/test_script_wait_on_boot.py": 12.55, - "tests/integration/test_sdl_headless_screenshot.py": 15.02, - "tests/integration/test_select_stringref_trigger.py": 13.35, - "tests/integration/test_sensor_filters_delta.py": 10.95, - "tests/integration/test_sensor_filters_ring_buffer.py": 12.24, - "tests/integration/test_sensor_filters_sliding_window.py": 36.89, - "tests/integration/test_sensor_filters_value_list.py": 13.19, - "tests/integration/test_sensor_raw_state.py": 22.36, - "tests/integration/test_sensor_timeout_filter.py": 7.08, - "tests/integration/test_set_internal_at_boot.py": 10.04, - "tests/integration/test_snapshot_display.py": 14.16, - "tests/integration/test_socket_wake_gate_tcp.py": 11.87, - "tests/integration/test_status_flags.py": 16.85, - "tests/integration/test_strftime_to.py": 9.78, - "tests/integration/test_syslog.py": 19.32, - "tests/integration/test_template_alarm_control_panel_many_sensors.py": 10.29, - "tests/integration/test_template_climate_basic.py": 8.97, - "tests/integration/test_template_climate_custom_modes.py": 12.47, - "tests/integration/test_template_climate_nonoptimistic.py": 13.76, - "tests/integration/test_template_climate_on_control_ordering.py": 4.05, - "tests/integration/test_template_climate_publish_all_fields.py": 11.22, - "tests/integration/test_template_climate_sensor_push.py": 9.25, - "tests/integration/test_template_climate_set_actions.py": 9.75, - "tests/integration/test_template_climate_two_point_temperature.py": 8.42, - "tests/integration/test_template_text_save.py": 13.08, - "tests/integration/test_template_time.py": 11.6, - "tests/integration/test_text_command.py": 8.74, - "tests/integration/test_text_sensor_raw_state.py": 8.84, - "tests/integration/test_uart_mock_ld2410.py": 24.25, - "tests/integration/test_uart_mock_ld2412.py": 25.28, - "tests/integration/test_uart_mock_ld2420.py": 19.0, - "tests/integration/test_uart_mock_ld2450.py": 13.48, - "tests/integration/test_uart_mock_modbus.py": 207.43, - "tests/integration/test_udp.py": 12.28, - "tests/integration/test_use_address_runtime.py": 15.24, - "tests/integration/test_valve_control_action.py": 13.21, - "tests/integration/test_varint_five_byte_device_id.py": 11.19, - "tests/integration/test_wait_until_mid_loop_timing.py": 6.22, - "tests/integration/test_wait_until_on_boot.py": 8.33, - "tests/integration/test_wait_until_ordering.py": 10.08, - "tests/integration/test_wait_until_reentrant_restart.py": 12.43, - "tests/integration/test_wake_loop_forces_phase_b.py": 10.22, - "tests/integration/test_water_heater_template.py": 24.1 + "tests/integration/test_action_concurrent_reentry.py": 12.49, + "tests/integration/test_addressable_light_transition.py": 17.27, + "tests/integration/test_alarm_control_panel_state_transitions.py": 15.05, + "tests/integration/test_api_action_metadata.py": 10.78, + "tests/integration/test_api_action_responses.py": 17.68, + "tests/integration/test_api_action_timeout.py": 17.48, + "tests/integration/test_api_conditional_memory.py": 11.82, + "tests/integration/test_api_custom_services.py": 8.17, + "tests/integration/test_api_decode_wire_types.py": 17.82, + "tests/integration/test_api_empty_message_roundtrip.py": 12.6, + "tests/integration/test_api_encode_boundaries.py": 14.83, + "tests/integration/test_api_get_time_response_timezone.py": 12.93, + "tests/integration/test_api_homeassistant.py": 19.79, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 12.45, + "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 7.67, + "tests/integration/test_api_list_entities_backpressure.py": 12.69, + "tests/integration/test_api_message_size_batching.py": 15.39, + "tests/integration/test_api_reboot_timeout.py": 12.25, + "tests/integration/test_api_string_lambda.py": 13.84, + "tests/integration/test_api_vv_logging.py": 12.98, + "tests/integration/test_api_zero_psk_provisioning.py": 19.36, + "tests/integration/test_areas_and_devices.py": 8.09, + "tests/integration/test_automation_wait_actions.py": 14.26, + "tests/integration/test_automations.py": 20.27, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 15.02, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 14.18, + "tests/integration/test_binary_sensor_invalidate_state.py": 13.04, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 10.44, + "tests/integration/test_build_info.py": 12.48, + "tests/integration/test_camera_mock.py": 12.53, + "tests/integration/test_climate_control_action.py": 9.7, + "tests/integration/test_climate_custom_modes.py": 14.68, + "tests/integration/test_continuation_actions.py": 12.16, + "tests/integration/test_cover_control_action.py": 11.76, + "tests/integration/test_crc8_helper.py": 8.21, + "tests/integration/test_device_id_in_state.py": 26.48, + "tests/integration/test_duplicate_entities.py": 14.89, + "tests/integration/test_entity_icon.py": 15.75, + "tests/integration/test_entity_missing_state.py": 9.8, + "tests/integration/test_fan_turn_on_action.py": 12.96, + "tests/integration/test_fnv1_hash_object_id.py": 2.55, + "tests/integration/test_fnv1a_hash.py": 9.37, + "tests/integration/test_gpio_expander_cache.py": 12.31, + "tests/integration/test_host_logger_thread_safety.py": 7.8, + "tests/integration/test_host_mode_basic.py": 11.59, + "tests/integration/test_host_mode_batch_delay.py": 8.18, + "tests/integration/test_host_mode_climate_basic_state.py": 12.53, + "tests/integration/test_host_mode_climate_control.py": 14.3, + "tests/integration/test_host_mode_empty_string_options.py": 13.74, + "tests/integration/test_host_mode_entity_fields.py": 15.51, + "tests/integration/test_host_mode_fan_preset.py": 9.63, + "tests/integration/test_host_mode_many_entities.py": 16.22, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 13.26, + "tests/integration/test_host_mode_noise_encryption.py": 29.86, + "tests/integration/test_host_mode_reconnect.py": 11.7, + "tests/integration/test_host_mode_sensor.py": 11.74, + "tests/integration/test_host_ota.py": 129.75, + "tests/integration/test_host_preferences.py": 8.46, + "tests/integration/test_host_preferences_suspend_resume.py": 14.3, + "tests/integration/test_improv_serial_uart.py": 15.6, + "tests/integration/test_large_message_batching.py": 13.6, + "tests/integration/test_legacy_area.py": 11.44, + "tests/integration/test_legacy_climate_compat.py": 12.99, + "tests/integration/test_legacy_fan_compat.py": 8.75, + "tests/integration/test_light_automations.py": 15.38, + "tests/integration/test_light_binary_effect_off_phase.py": 23.85, + "tests/integration/test_light_calls.py": 16.45, + "tests/integration/test_light_constant_brightness.py": 16.87, + "tests/integration/test_light_control_action.py": 16.25, + "tests/integration/test_light_dim_relative_action.py": 14.03, + "tests/integration/test_light_effect_zero_brightness.py": 16.53, + "tests/integration/test_light_initial_state.py": 15.16, + "tests/integration/test_light_repeat_effect.py": 22.35, + "tests/integration/test_light_restore_state.py": 11.68, + "tests/integration/test_light_resume_effect.py": 17.07, + "tests/integration/test_light_toggle_action.py": 14.35, + "tests/integration/test_light_transition_state_publish_interval.py": 47.18, + "tests/integration/test_lock_automations.py": 12.65, + "tests/integration/test_logger_buffered_recursion_guard.py": 12.2, + "tests/integration/test_loop_disable_enable.py": 11.31, + "tests/integration/test_loop_interval_decoupling.py": 13.71, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 11.78, + "tests/integration/test_lvgl_headless_render.py": 49.4, + "tests/integration/test_micros_to_millis.py": 6.71, + "tests/integration/test_multi_click_trigger.py": 10.38, + "tests/integration/test_multi_device_preferences.py": 16.06, + "tests/integration/test_noise_encryption_key_protection.py": 13.43, + "tests/integration/test_object_id_api_verification.py": 14.32, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 12.57, + "tests/integration/test_object_id_no_friendly_name.py": 15.59, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 24.62, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 29.88, + "tests/integration/test_online_image_bmp.py": 23.67, + "tests/integration/test_oversized_payloads.py": 51.21, + "tests/integration/test_preference_key_stability.py": 14.85, + "tests/integration/test_runtime_stats.py": 11.2, + "tests/integration/test_safe_mode_loop_runs.py": 8.22, + "tests/integration/test_scheduler_blocking_warning.py": 28.18, + "tests/integration/test_scheduler_bulk_cleanup.py": 12.49, + "tests/integration/test_scheduler_defer_cancel.py": 9.95, + "tests/integration/test_scheduler_defer_cancel_regular.py": 13.32, + "tests/integration/test_scheduler_defer_fifo_simple.py": 13.18, + "tests/integration/test_scheduler_defer_stress.py": 14.29, + "tests/integration/test_scheduler_heap_stress.py": 4.73, + "tests/integration/test_scheduler_internal_id_no_collision.py": 14.05, + "tests/integration/test_scheduler_interval_reschedule.py": 9.0, + "tests/integration/test_scheduler_interval_zero_coerced.py": 3.27, + "tests/integration/test_scheduler_null_name.py": 12.94, + "tests/integration/test_scheduler_numeric_id_test.py": 11.83, + "tests/integration/test_scheduler_pool.py": 9.66, + "tests/integration/test_scheduler_rapid_cancellation.py": 3.95, + "tests/integration/test_scheduler_recursive_timeout.py": 12.91, + "tests/integration/test_scheduler_removed_item_race.py": 11.9, + "tests/integration/test_scheduler_self_keyed.py": 13.42, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 8.01, + "tests/integration/test_scheduler_string_test.py": 10.2, + "tests/integration/test_script_array_params.py": 12.99, + "tests/integration/test_script_delay_params.py": 14.73, + "tests/integration/test_script_queued.py": 16.62, + "tests/integration/test_script_queued_idle_loop.py": 11.13, + "tests/integration/test_script_wait_on_boot.py": 13.31, + "tests/integration/test_sdl_headless_screenshot.py": 14.76, + "tests/integration/test_select_stringref_trigger.py": 10.98, + "tests/integration/test_sensor_filters_delta.py": 12.63, + "tests/integration/test_sensor_filters_ring_buffer.py": 11.13, + "tests/integration/test_sensor_filters_sliding_window.py": 40.2, + "tests/integration/test_sensor_filters_value_list.py": 10.25, + "tests/integration/test_sensor_raw_state.py": 26.04, + "tests/integration/test_sensor_timeout_filter.py": 13.81, + "tests/integration/test_set_internal_at_boot.py": 14.13, + "tests/integration/test_snapshot_display.py": 16.36, + "tests/integration/test_socket_ipv4_resolve.py": 11.75, + "tests/integration/test_socket_tcp_client_link.py": 9.79, + "tests/integration/test_socket_wake_gate_tcp.py": 15.29, + "tests/integration/test_status_flags.py": 20.87, + "tests/integration/test_strftime_to.py": 10.06, + "tests/integration/test_syslog.py": 18.51, + "tests/integration/test_tcp_uart_bridge.py": 14.71, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 8.4, + "tests/integration/test_template_climate_basic.py": 14.22, + "tests/integration/test_template_climate_custom_modes.py": 11.61, + "tests/integration/test_template_climate_nonoptimistic.py": 13.47, + "tests/integration/test_template_climate_on_control_ordering.py": 13.99, + "tests/integration/test_template_climate_publish_all_fields.py": 10.98, + "tests/integration/test_template_climate_sensor_push.py": 14.86, + "tests/integration/test_template_climate_set_actions.py": 11.79, + "tests/integration/test_template_climate_two_point_temperature.py": 14.92, + "tests/integration/test_template_text_save.py": 13.35, + "tests/integration/test_template_time.py": 10.07, + "tests/integration/test_text_command.py": 12.11, + "tests/integration/test_text_sensor_raw_state.py": 13.96, + "tests/integration/test_uart_mock_ld2410.py": 30.03, + "tests/integration/test_uart_mock_ld2412.py": 35.57, + "tests/integration/test_uart_mock_ld2420.py": 26.6, + "tests/integration/test_uart_mock_ld2450.py": 13.03, + "tests/integration/test_uart_mock_modbus.py": 309.49, + "tests/integration/test_uart_tcp_bridge.py": 12.67, + "tests/integration/test_udp.py": 11.64, + "tests/integration/test_use_address_runtime.py": 16.79, + "tests/integration/test_valve_control_action.py": 12.76, + "tests/integration/test_varint_five_byte_device_id.py": 14.05, + "tests/integration/test_wait_until_mid_loop_timing.py": 12.48, + "tests/integration/test_wait_until_on_boot.py": 12.54, + "tests/integration/test_wait_until_ordering.py": 13.32, + "tests/integration/test_wait_until_reentrant_restart.py": 5.89, + "tests/integration/test_wake_loop_forces_phase_b.py": 13.8, + "tests/integration/test_water_heater_template.py": 24.77 } From bbd05e4811d6cb2b44f37f1de8f971669b728a75 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:50:18 +1100 Subject: [PATCH 153/221] [esp32] Work around ESP-IDF stack bug on P4 (#20181) --- esphome/components/esp32/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 6a693eff89..1166afe4ec 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -2873,6 +2873,8 @@ async def to_code(config): "CONFIG_ESP32P4_SELECTS_REV_LESS_V3", config.get(CONF_ENGINEERING_SAMPLE, False), ) + # Work around ESP-IDF bug: see https://github.com/espressif/esp-idf/issues/19020 + add_idf_sdkconfig_option("CONFIG_ESP_MAIN_TASK_STACK_SIZE", 8192) # ESP32-C2 defaults to the ROM's newlib "nano" printf, which does not # understand %zu or %lld and crashes on any %s that follows one. From 862047386c1b691972bbee0c334d8a4d870be4ef Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:46:36 +0200 Subject: [PATCH 154/221] [modbus] Reject send_wait_time and turnaround_time above 65535 ms (#20160) --- esphome/components/modbus/__init__.py | 14 ++++++++------ tests/component_tests/modbus/test_modbus.py | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/esphome/components/modbus/__init__.py b/esphome/components/modbus/__init__.py index db7484e306..cad2781cc1 100644 --- a/esphome/components/modbus/__init__.py +++ b/esphome/components/modbus/__init__.py @@ -52,6 +52,12 @@ CONF_TURNAROUND_TIME = "turnaround_time" MODBUS_ROLES = ["client", "server"] +# The client hub takes these times as 16-bit milliseconds; a Modbus timeout is far shorter anyway. +_HUB_TIME_PERIOD = cv.All( + cv.positive_time_period_milliseconds, + cv.Range(max=cv.TimePeriod(milliseconds=65535)), +) + # The write (mutating) function codes, matching modbus::helpers::is_function_code_write(). 0x17 # (read/write multiple) is included: it mutates, so the hub treats it as a write despite its read half. @@ -273,12 +279,8 @@ CONFIG_SCHEMA = cv.typed_schema( { cv.GenerateID(): cv.declare_id(ModbusClient), cv.Optional(CONF_FLOW_CONTROL_PIN): pins.gpio_output_pin_schema, - cv.Optional( - CONF_SEND_WAIT_TIME, default="2000ms" - ): cv.positive_time_period_milliseconds, - cv.Optional( - CONF_TURNAROUND_TIME, default="600ms" - ): cv.positive_time_period_milliseconds, + cv.Optional(CONF_SEND_WAIT_TIME, default="2000ms"): _HUB_TIME_PERIOD, + cv.Optional(CONF_TURNAROUND_TIME, default="600ms"): _HUB_TIME_PERIOD, } ) .extend(cv.COMPONENT_SCHEMA) diff --git a/tests/component_tests/modbus/test_modbus.py b/tests/component_tests/modbus/test_modbus.py index 1eafb13166..d7e2c9ba2f 100644 --- a/tests/component_tests/modbus/test_modbus.py +++ b/tests/component_tests/modbus/test_modbus.py @@ -4,7 +4,11 @@ import pytest from esphome import config_validation as cv from esphome.components import modbus -from esphome.components.modbus import CONF_MODBUS_ID, _validate_server_address +from esphome.components.modbus import ( + _HUB_TIME_PERIOD, + CONF_MODBUS_ID, + _validate_server_address, +) from esphome.const import CONF_ADDRESS @@ -36,3 +40,13 @@ def test_client_schema_still_accepts_address_zero() -> None: # A client may address 0: writes are broadcast, and reads are allowed with allow_broadcast_read. schema = modbus.modbus_device_schema(0x01) assert schema({CONF_MODBUS_ID: "hub", CONF_ADDRESS: 0})[CONF_ADDRESS] == 0 + + +def test_hub_time_accepts_up_to_65535_ms() -> None: + assert _HUB_TIME_PERIOD("65535ms").total_milliseconds == 65535 + + +def test_hub_time_rejects_values_the_hub_would_truncate() -> None: + # The setters take 16-bit milliseconds: 70 s would silently become 4464 ms. + with pytest.raises(cv.Invalid): + _HUB_TIME_PERIOD("70s") From 59b49d34cc963ff97e5c92ed8ed92a2387b2502a Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:47:18 +0200 Subject: [PATCH 155/221] [tcp_uart][uart_tcp] Expose the no-arg load_settings() and mark UartTcp final (#20163) --- esphome/components/tcp_uart/tcp_uart.h | 1 + esphome/components/uart_tcp/uart_tcp.h | 2 +- tests/components/tcp_uart/test.esp32-idf.yaml | 8 ++++++++ tests/components/tcp_uart/test.esp8266-ard.yaml | 8 ++++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/esphome/components/tcp_uart/tcp_uart.h b/esphome/components/tcp_uart/tcp_uart.h index 29f7af564a..e973c7a189 100644 --- a/esphome/components/tcp_uart/tcp_uart.h +++ b/esphome/components/tcp_uart/tcp_uart.h @@ -44,6 +44,7 @@ class TcpUart : public uart::UARTComponent, public Component { bool is_connected() override { return this->link_.connected(); } #if defined(USE_ESP8266) || defined(USE_ESP32) void load_settings(bool dump_config) override {} + using UARTComponent::load_settings; // also bring in the no-arg overload for convenience #endif protected: diff --git a/esphome/components/uart_tcp/uart_tcp.h b/esphome/components/uart_tcp/uart_tcp.h index 1e19166f87..f557cca33f 100644 --- a/esphome/components/uart_tcp/uart_tcp.h +++ b/esphome/components/uart_tcp/uart_tcp.h @@ -14,7 +14,7 @@ namespace esphome::uart_tcp { /// Copies raw bytes between one hardware UART and one TCP socket. -class UartTcp : public Component, public uart::UARTDevice { +class UartTcp final : public Component, public uart::UARTDevice { public: void set_host(const char *host) { this->link_.set_host(host); } void set_port(uint16_t port) { this->link_.set_port(port); } diff --git a/tests/components/tcp_uart/test.esp32-idf.yaml b/tests/components/tcp_uart/test.esp32-idf.yaml index 2d42c282f7..54bdfbb390 100644 --- a/tests/components/tcp_uart/test.esp32-idf.yaml +++ b/tests/components/tcp_uart/test.esp32-idf.yaml @@ -1,2 +1,10 @@ packages: tcp_uart: !include common.yaml + +# load_settings() exists on ESP8266 and ESP32 only; both overloads must resolve. +interval: + - interval: 60s + then: + - lambda: |- + id(tcp_uart_server).load_settings(); + id(tcp_uart_server).load_settings(false); diff --git a/tests/components/tcp_uart/test.esp8266-ard.yaml b/tests/components/tcp_uart/test.esp8266-ard.yaml index 2d42c282f7..54bdfbb390 100644 --- a/tests/components/tcp_uart/test.esp8266-ard.yaml +++ b/tests/components/tcp_uart/test.esp8266-ard.yaml @@ -1,2 +1,10 @@ packages: tcp_uart: !include common.yaml + +# load_settings() exists on ESP8266 and ESP32 only; both overloads must resolve. +interval: + - interval: 60s + then: + - lambda: |- + id(tcp_uart_server).load_settings(); + id(tcp_uart_server).load_settings(false); From aa02a0996ab6fbdcddd8f19a6e8fd76e96484c89 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:48:33 +0200 Subject: [PATCH 156/221] [socket][tcp_uart][uart_tcp] Reject an IPv6 host at config time (#20165) --- esphome/components/socket/__init__.py | 15 ++++++++++++ esphome/components/tcp_uart/__init__.py | 2 +- esphome/components/uart_tcp/__init__.py | 2 +- .../socket/test_socket_ipv4_host.py | 23 +++++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 tests/unit_tests/components/socket/test_socket_ipv4_host.py diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index 269791f5e7..3d2cd8ef50 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -149,6 +149,21 @@ def require_wake_loop_threadsafe() -> None: IPV4_ALLOW_SCHEMA = cv.All(cv.ensure_list(cv.ipv4network), cv.Length(max=255)) +_HOST = cv.Any(cv.domain, cv.hostname) + + +def ipv4_host(value: object) -> str: + """Validate an IPv4 address or a hostname; the resolver behind it is IPv4 only.""" + value = cv.string(value) + try: + cv.ipv6address(value) + except cv.Invalid: + return _HOST(value) + raise cv.Invalid( + "IPv6 addresses are not supported, use an IPv4 address or a hostname" + ) + + def _network_order(addr: IPv4Address) -> int: """The s_addr value for addr on the little endian targets.""" return int.from_bytes(addr.packed, "little") diff --git a/esphome/components/tcp_uart/__init__.py b/esphome/components/tcp_uart/__init__.py index 8e14e690f9..5481c8b315 100644 --- a/esphome/components/tcp_uart/__init__.py +++ b/esphome/components/tcp_uart/__init__.py @@ -54,7 +54,7 @@ CONFIG_SCHEMA = cv.All( { "client": BASE_SCHEMA.extend( { - cv.Required(CONF_HOST): cv.string, + cv.Required(CONF_HOST): socket.ipv4_host, } ), "server": BASE_SCHEMA.extend( diff --git a/esphome/components/uart_tcp/__init__.py b/esphome/components/uart_tcp/__init__.py index 3829f046ca..6e09ecac23 100644 --- a/esphome/components/uart_tcp/__init__.py +++ b/esphome/components/uart_tcp/__init__.py @@ -44,7 +44,7 @@ BASE_SCHEMA = cv.Schema( CONFIG_SCHEMA = cv.All( cv.typed_schema( { - "client": BASE_SCHEMA.extend({cv.Required(CONF_HOST): cv.string}), + "client": BASE_SCHEMA.extend({cv.Required(CONF_HOST): socket.ipv4_host}), "server": BASE_SCHEMA.extend( {cv.Optional(CONF_ALLOWED_IPS): socket.IPV4_ALLOW_SCHEMA} ), diff --git a/tests/unit_tests/components/socket/test_socket_ipv4_host.py b/tests/unit_tests/components/socket/test_socket_ipv4_host.py new file mode 100644 index 0000000000..7953447b95 --- /dev/null +++ b/tests/unit_tests/components/socket/test_socket_ipv4_host.py @@ -0,0 +1,23 @@ +"""Tests for the socket component's IPv4 host validator.""" + +import pytest + +from esphome.components import socket +import esphome.config_validation as cv + + +@pytest.mark.parametrize("value", ["192.0.2.10", "bridge.local", "bridge"]) +def test_ipv4_host_accepts_ipv4_address_and_hostname(value: str) -> None: + assert socket.ipv4_host(value) == value + + +@pytest.mark.parametrize("value", ["2001:db8::10", "::1", "::ffff:192.0.2.10"]) +def test_ipv4_host_rejects_ipv6_address(value: str) -> None: + with pytest.raises(cv.Invalid, match="IPv6 addresses are not supported"): + socket.ipv4_host(value) + + +@pytest.mark.parametrize("value", ["bridge host", ""]) +def test_ipv4_host_rejects_invalid_host(value: str) -> None: + with pytest.raises(cv.Invalid): + socket.ipv4_host(value) From 00303fb72b47aded066914e997cc767e63cc99dc Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:55:18 +0100 Subject: [PATCH 157/221] [esp32][api][noise] Resolve arduino's libsodium requirement to the converted esphome/libsodium (#20185) --- esphome/components/esp32/__init__.py | 61 ++++++++++++------- esphome/espidf/clang_tidy.py | 31 ++++++++-- tests/components/api/test.esp32-ard.yaml | 6 ++ tests/components/noise/test.esp32-ard.yaml | 2 + .../test_esp32_write_idf_component_yml.py | 46 ++++++++++++++ tests/unit_tests/test_espidf_clang_tidy.py | 48 +++++++++++++++ 6 files changed, 166 insertions(+), 28 deletions(-) create mode 100644 tests/components/api/test.esp32-ard.yaml create mode 100644 tests/components/noise/test.esp32-ard.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 1166afe4ec..be6b2c3266 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -57,7 +57,7 @@ from esphome.const import ( from esphome.core import CORE, EsphomeError, HexInt from esphome.core.config import BOARD_MAX_LENGTH from esphome.coroutine import CoroPriority, coroutine_with_priority -from esphome.espidf.component import generate_idf_components +from esphome.espidf.component import IDFComponent, generate_idf_components import esphome.final_validate as fv from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor @@ -3539,6 +3539,30 @@ def _write_idf_component_yml(): yml_path = CORE.relative_build_path("src/idf_component.yml") dependencies: dict[str, dict] = {} + converted: list[IDFComponent] = [] + if CORE.using_toolchain_esp_idf: + # Convert the PlatformIO libraries to ESP-IDF components as a batch so + # PlatformIO resolves the whole dependency tree at once -- deduplicating + # shared transitive deps (e.g. esphome/libsodium pulled by both noise-c + # and esp_wireguard) to a single version instead of clashing + # override_path entries. + libraries = [ + library + for name, library in CORE.platformio_libraries.items() + # Don't process arduino libraries + if name not in ARDUINO_DISABLED_LIBRARIES + ] + # A library also declared as a managed component is not converted too, or + # IDF sees the same requirement twice; converted components reach it through + # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}. + managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {})) + converted = generate_idf_components(libraries, managed=managed) + # IDF names a component after its directory and a later registration of the + # same name replaces the earlier one, so a stub beside a converted library of + # the same name (espressif/libsodium vs esphome/libsodium) would win or lose + # on path order. Such a stub points at the converted library instead. + converted_by_name = {component.path.name: component for component in converted} + # For Arduino builds, override unused managed components from the Arduino framework # by pointing them to empty stub directories using override_path # This prevents the IDF component manager from downloading the real components @@ -3562,8 +3586,17 @@ def _write_idf_component_yml(): # always writes, and ninja keeps triggering CMake re-runs on # otherwise-cached rebuilds. for component_name in sorted(components_to_stub): + stub_name = _idf_component_stub_name(component_name) + stub_path = stubs_dir / stub_name + if (component := converted_by_name.get(stub_name)) is not None: + if stub_path.exists(): + rmtree(stub_path) + dependencies[_idf_component_dep_name(component_name)] = { + "version": "*", + "override_path": str(component.path), + } + continue # Create stub directory with minimal CMakeLists.txt - stub_path = stubs_dir / _idf_component_stub_name(component_name) stub_path.mkdir(exist_ok=True) stub_cmake = stub_path / "CMakeLists.txt" if not stub_cmake.exists(): @@ -3605,26 +3638,10 @@ def _write_idf_component_yml(): ref=str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]), ) - if CORE.using_toolchain_esp_idf: - # Convert the PlatformIO libraries to ESP-IDF components as a batch so - # PlatformIO resolves the whole dependency tree at once -- deduplicating - # shared transitive deps (e.g. esphome/libsodium pulled by both noise-c - # and esp_wireguard) to a single version instead of clashing - # override_path entries. - libraries = [ - library - for name, library in CORE.platformio_libraries.items() - # Don't process arduino libraries - if name not in ARDUINO_DISABLED_LIBRARIES - ] - # A library also declared as a managed component is not converted too, or - # IDF sees the same requirement twice; converted components reach it through - # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}. - managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {})) - for component in generate_idf_components(libraries, managed=managed): - dependencies[component.get_sanitized_name()] = { - "override_path": str(component.path) - } + for component in converted: + dependencies[component.get_sanitized_name()] = { + "override_path": str(component.path) + } if CORE.data[KEY_ESP32][KEY_COMPONENTS]: components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS] diff --git a/esphome/espidf/clang_tidy.py b/esphome/espidf/clang_tidy.py index 623b352e97..e109363d21 100644 --- a/esphome/espidf/clang_tidy.py +++ b/esphome/espidf/clang_tidy.py @@ -278,18 +278,25 @@ def _convert_pio_libs( return deps -def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: +def _arduino_excluded_stubs( + work_dir: Path, converted: dict[str, dict[str, str]] | None = None +) -> dict[str, dict]: """Stub the arduino-bundled IDF components ESPHome doesn't use. arduino-esp32 declares deps (libsodium, RainMaker, modbus, ...) that ESPHome replaces with its own library (noise-c) or doesn't use; point each at an empty override_path component so the IDF manager doesn't resolve/download - them -- notably so ``espressif/libsodium`` doesn't clash with the converted - noise-c's ``libsodium``. Mirrors esp32's ``_write_idf_component_yml``. + them. Mirrors esp32's ``_write_idf_component_yml``. Components ESPHome's own idf_component.yml provides (e.g. lan867x for ethernet) are NOT stubbed -- those are real deps we need, and arduino-esp32 resolves to the same component rather than conflicting. + + ``converted`` is the manifest block from ``_convert_pio_libs``. IDF names a + component after its directory and a later registration of the same name + replaces the earlier one, so a stub beside a converted library of the same + name (``espressif/libsodium`` vs ``esphome/libsodium``) would win or lose on + path order; such a stub points at the converted library instead. """ from esphome.components.esp32 import ( _idf_component_dep_name, @@ -298,6 +305,10 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: ) esphome_deps = _esphome_manifest_deps() + converted_paths = { + Path(dep["override_path"]).name: dep["override_path"] + for dep in (converted or {}).values() + } stubs_dir = work_dir / "component_stubs" stubs_dir.mkdir(parents=True, exist_ok=True) @@ -305,7 +316,14 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: for component in sorted(arduino_excluded_idf_components()): if _idf_component_dep_name(component) in esphome_deps: continue # ESPHome needs this one for real (don't stub it away) - stub_path = stubs_dir / _idf_component_stub_name(component) + stub_name = _idf_component_stub_name(component) + if (path := converted_paths.get(stub_name)) is not None: + deps[_idf_component_dep_name(component)] = { + "version": "*", + "override_path": path, + } + continue + stub_path = stubs_dir / stub_name stub_path.mkdir(exist_ok=True) (stub_path / "CMakeLists.txt").write_text( "idf_component_register()\n", encoding="utf-8" @@ -401,11 +419,12 @@ def _generate_compile_commands( # Framework deps (e.g. arduino-esp32) + PlatformIO libs converted to local # IDF components, all added to the manifest as deps. extra_deps = dict(settings.framework_deps) - extra_deps.update(_convert_pio_libs(platformio_ini, settings.target_framework)) + converted = _convert_pio_libs(platformio_ini, settings.target_framework) + extra_deps.update(converted) if settings.target_framework == "arduino": # Stub the arduino-bundled components ESPHome doesn't use (avoids the # libsodium clash with noise-c and ~26 unused heavy downloads). - extra_deps.update(_arduino_excluded_stubs(work_dir)) + extra_deps.update(_arduino_excluded_stubs(work_dir, converted)) # Phase 1: discover the components available for this target. _write_tidy_project(work_dir, [], extra_deps, settings) diff --git a/tests/components/api/test.esp32-ard.yaml b/tests/components/api/test.esp32-ard.yaml new file mode 100644 index 0000000000..bd0915f0ef --- /dev/null +++ b/tests/components/api/test.esp32-ard.yaml @@ -0,0 +1,6 @@ +packages: + api: !include common.yaml + +wifi: + ssid: MySSID + password: password1 diff --git a/tests/components/noise/test.esp32-ard.yaml b/tests/components/noise/test.esp32-ard.yaml new file mode 100644 index 0000000000..550ffd1f88 --- /dev/null +++ b/tests/components/noise/test.esp32-ard.yaml @@ -0,0 +1,2 @@ +packages: + noise: !include common.yaml diff --git a/tests/unit_tests/components/test_esp32_write_idf_component_yml.py b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py index 931de627a1..878780cba8 100644 --- a/tests/unit_tests/components/test_esp32_write_idf_component_yml.py +++ b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py @@ -10,11 +10,13 @@ from pathlib import Path from unittest.mock import MagicMock import pytest +import yaml from esphome.components import esp32 import esphome.config_validation as cv from esphome.const import ( KEY_CORE, + KEY_FRAMEWORK_VERSION, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, Framework, @@ -173,3 +175,47 @@ def test_write_idf_component_yml_arduino_stubs_follow_idf_version( / esp32._idf_component_stub_name("espressif__libsodium") ) assert stub_dir.is_dir() is libsodium_stubbed + + +def test_write_idf_component_yml_stub_yields_to_converted_library_of_same_name( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """IDF names a component after its directory and the later registration wins, + so the espressif/libsodium stub and the converted esphome/libsodium raced on + path order (#20102). The stub entry points at the converted library instead.""" + _setup_core(tmp_path) + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = str(Framework.ARDUINO) + CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version(3, 3, 11) + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_COMPONENTS: {}, + esp32.KEY_IDF_VERSION: cv.Version(5, 5, 5), + esp32.KEY_ARDUINO_LIBRARIES: set(), + } + # A stale stub from an earlier build must not linger beside the real library + stale_stub = tmp_path / "component_stubs" / "libsodium" + stale_stub.mkdir(parents=True) + + converted = MagicMock() + converted.get_sanitized_name.return_value = "esphome/libsodium" + converted.path = tmp_path / "pio_components" / "esphome" / "libsodium" + monkeypatch.setattr( + esp32, "generate_idf_components", lambda libraries, managed=None: [converted] + ) + + esp32._write_idf_component_yml() + + manifest = yaml.safe_load( + (tmp_path / "src" / "idf_component.yml").read_text(encoding="utf-8") + )["dependencies"] + assert manifest["espressif/libsodium"] == { + "version": "*", + "override_path": str(converted.path), + } + assert manifest["esphome/libsodium"] == {"override_path": str(converted.path)} + assert not stale_stub.exists() + # Stubs without a converted namesake are unaffected + assert (tmp_path / "component_stubs" / "cbor").is_dir() + assert manifest["espressif/cbor"]["override_path"] == str( + tmp_path / "component_stubs" / "cbor" + ) diff --git a/tests/unit_tests/test_espidf_clang_tidy.py b/tests/unit_tests/test_espidf_clang_tidy.py index b0d016b1b4..2516a9012f 100644 --- a/tests/unit_tests/test_espidf_clang_tidy.py +++ b/tests/unit_tests/test_espidf_clang_tidy.py @@ -153,6 +153,33 @@ def test_generate_compile_commands_configures_twice( assert mock_write.call_args_list[1].args[1] == ["esp_timer", "lwip"] +def test_generate_compile_commands_hands_converted_libs_to_arduino_stubs( + tmp_path: Path, +) -> None: + """On Arduino the stub generator sees the converted libraries, so a stub + sharing a converted library's directory name can point at it instead.""" + converted = {"esphome/libsodium": {"override_path": str(tmp_path / "libsodium")}} + stubs = {"espressif/libsodium": {"version": "*", "override_path": "x"}} + with ( + patch.object(clang_tidy, "_setup_core"), + patch.object(clang_tidy, "_convert_pio_libs", return_value=converted), + patch.object( + clang_tidy, "_arduino_excluded_stubs", return_value=stubs + ) as mock_stubs, + patch.object(clang_tidy, "_write_tidy_project") as mock_write, + patch("esphome.espidf.toolchain.run_reconfigure", return_value=0), + patch("esphome.build_gen.espidf.get_available_components", return_value=[]), + ): + clang_tidy._generate_compile_commands( + tmp_path, _settings(target_framework="arduino"), tmp_path / "platformio.ini" + ) + + mock_stubs.assert_called_once_with(tmp_path, converted) + extra_deps = mock_write.call_args_list[0].args[2] + assert extra_deps["esphome/libsodium"] == converted["esphome/libsodium"] + assert extra_deps["espressif/libsodium"] == stubs["espressif/libsodium"] + + def test_esphome_manifest_deps_reads_repo_manifest() -> None: """Top-level dependency names, independent of the per-dependency rules.""" manifest = yaml.safe_load( @@ -284,6 +311,27 @@ def test_arduino_excluded_stubs_skips_components_esphome_manifest_provides( assert (stub_path / "CMakeLists.txt").is_file() +def test_arduino_excluded_stubs_points_libsodium_at_converted_library( + tmp_path: Path, +) -> None: + """Below IDF 6.0 the converted esphome/libsodium shares its directory name + with the espressif/libsodium stub; IDF would keep whichever registers last, + so the stub entry points at the converted library instead (#20102).""" + _set_idf_version(cv.Version(5, 5, 4)) + converted_path = str(tmp_path / "pio" / "esphome" / "libsodium") + converted = {"esphome/libsodium": {"override_path": converted_path}} + + deps = _arduino_excluded_stubs(tmp_path, converted) + + assert deps["espressif/libsodium"] == { + "version": "*", + "override_path": converted_path, + } + assert not (tmp_path / "component_stubs" / "libsodium").exists() + # Stubs without a converted namesake are unaffected. + assert (tmp_path / "component_stubs" / "cbor" / "CMakeLists.txt").is_file() + + def test_arduino_excluded_stubs_skips_libsodium_from_idf_6(tmp_path: Path) -> None: """From IDF 6.0 arduino-esp32 drops espressif/libsodium; a stub would clash with esphome/libsodium.""" From ca74e71348eec007e2ed820fa7848f3e14a53722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Metrich?= <45318189+FredM67@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:16:21 +0200 Subject: [PATCH 158/221] [emontx] Use total state class for energy sensors (#20128) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/emontx/sensor/__init__.py | 3 ++- tests/component_tests/emontx/test_sensor_defaults.py | 11 +++++------ tests/components/emontx/validate.esp32-idf.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/esphome/components/emontx/sensor/__init__.py b/esphome/components/emontx/sensor/__init__.py index 56a7fb8b55..eadba1ea3f 100644 --- a/esphome/components/emontx/sensor/__init__.py +++ b/esphome/components/emontx/sensor/__init__.py @@ -16,6 +16,7 @@ from esphome.const import ( DEVICE_CLASS_TEMPERATURE, DEVICE_CLASS_VOLTAGE, STATE_CLASS_MEASUREMENT, + STATE_CLASS_TOTAL, STATE_CLASS_TOTAL_INCREASING, UNIT_AMPERE, UNIT_CELSIUS, @@ -70,7 +71,7 @@ SENSOR_CONFIGS = { "E": { CONF_UNIT_OF_MEASUREMENT: UNIT_WATT_HOURS, CONF_DEVICE_CLASS: DEVICE_CLASS_ENERGY, - CONF_STATE_CLASS: STATE_CLASS_TOTAL_INCREASING, + CONF_STATE_CLASS: STATE_CLASS_TOTAL, CONF_ACCURACY_DECIMALS: 0, }, "V": { diff --git a/tests/component_tests/emontx/test_sensor_defaults.py b/tests/component_tests/emontx/test_sensor_defaults.py index 5c6a8e4108..2d64f92be6 100644 --- a/tests/component_tests/emontx/test_sensor_defaults.py +++ b/tests/component_tests/emontx/test_sensor_defaults.py @@ -18,6 +18,7 @@ from esphome.const import ( DEVICE_CLASS_TEMPERATURE, DEVICE_CLASS_VOLTAGE, STATE_CLASS_MEASUREMENT, + STATE_CLASS_TOTAL, STATE_CLASS_TOTAL_INCREASING, UNIT_AMPERE, UNIT_CELSIUS, @@ -42,14 +43,12 @@ def _resolve_via_config_schema(tag: str) -> dict: def test_config_schema_applies_tag_default_state_class(): """If sensor_schema(state_class=...) is reintroduced, the schema-level default wins over apply_tag_defaults' per-prefix value, and E1 would - resolve to measurement instead of total_increasing. Driving the real + resolve to measurement instead of total. Driving the real CONFIG_SCHEMA (not just apply_tag_defaults) catches that, since sensor_schema() runs before apply_tag_defaults in the cv.All() chain. """ result = _resolve_via_config_schema("E1") - assert result[CONF_STATE_CLASS] == sensor.validate_state_class( - STATE_CLASS_TOTAL_INCREASING - ) + assert result[CONF_STATE_CLASS] == sensor.validate_state_class(STATE_CLASS_TOTAL) def test_config_schema_applies_tag_default_accuracy_decimals(): @@ -70,8 +69,8 @@ def _make_config(tag: str) -> dict: ("tag", "expected_state_class", "expected_decimals"), [ # Known numeric-index prefixes - ("E1", STATE_CLASS_TOTAL_INCREASING, 0), - ("E12", STATE_CLASS_TOTAL_INCREASING, 0), + ("E1", STATE_CLASS_TOTAL, 0), + ("E12", STATE_CLASS_TOTAL, 0), ("P1", STATE_CLASS_MEASUREMENT, 0), ("V1", STATE_CLASS_MEASUREMENT, 2), ("I1", STATE_CLASS_MEASUREMENT, 2), diff --git a/tests/components/emontx/validate.esp32-idf.yaml b/tests/components/emontx/validate.esp32-idf.yaml index 882ee26fcb..548a3c9c41 100644 --- a/tests/components/emontx/validate.esp32-idf.yaml +++ b/tests/components/emontx/validate.esp32-idf.yaml @@ -8,7 +8,7 @@ packages: # exercised, not the user-override path. sensor: - # Energy sensor (E prefix): expects state_class=total_increasing, unit=Wh, + # Energy sensor (E prefix): expects state_class=total, unit=Wh, # device_class=energy, accuracy_decimals=0 - platform: emontx tag_name: E1 From 9a97419cfc06265e636a65f76944125adc4276d2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:21:23 -0500 Subject: [PATCH 159/221] [ethernet] Store PHY registers in a StaticVector sized from the config (#20131) --- esphome/components/ethernet/__init__.py | 29 ++++++++++--------- .../components/ethernet/ethernet_component.h | 10 +++++-- .../ethernet/ethernet_component_esp32.cpp | 8 ++--- esphome/core/defines.h | 1 + tests/components/ethernet/common-rtl8201.yaml | 6 ++++ 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/esphome/components/ethernet/__init__.py b/esphome/components/ethernet/__init__.py index ce7206bf57..ff05caca83 100644 --- a/esphome/components/ethernet/__init__.py +++ b/esphome/components/ethernet/__init__.py @@ -636,6 +636,19 @@ def phy_register(address: int, value: int, page: int) -> cg.StructInitializer: ) +def _add_phy_registers(var: cg.MockObj, config: ConfigType) -> None: + if not (registers := config.get(CONF_PHY_REGISTERS)): + return + cg.add_define("ESPHOME_ETHERNET_PHY_REGISTER_COUNT", len(registers)) + for register_value in registers: + reg = phy_register( + register_value.get(CONF_ADDRESS), + register_value.get(CONF_VALUE), + register_value.get(CONF_PAGE_ID), + ) + cg.add(var.add_phy_register(reg)) + + @coroutine_with_priority(CoroPriority.COMMUNICATION) async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) @@ -741,13 +754,7 @@ async def _to_code_esp32(var: cg.MockObj, config: ConfigType) -> None: cg.add(var.set_mdio_pin(config[CONF_MDIO_PIN])) if CONF_POWER_PIN in config: cg.add(var.set_power_pin(config[CONF_POWER_PIN])) - for register_value in config.get(CONF_PHY_REGISTERS, []): - reg = phy_register( - register_value.get(CONF_ADDRESS), - register_value.get(CONF_VALUE), - register_value.get(CONF_PAGE_ID), - ) - cg.add(var.add_phy_register(reg)) + _add_phy_registers(var, config) else: cg.add(var.set_phy_addr(config[CONF_PHY_ADDR])) cg.add(var.set_mdc_pin(config[CONF_MDC_PIN])) @@ -756,13 +763,7 @@ async def _to_code_esp32(var: cg.MockObj, config: ConfigType) -> None: cg.add(var.set_clk_pin(config[CONF_CLK][CONF_PIN])) if CONF_POWER_PIN in config: cg.add(var.set_power_pin(config[CONF_POWER_PIN])) - for register_value in config.get(CONF_PHY_REGISTERS, []): - reg = phy_register( - register_value.get(CONF_ADDRESS), - register_value.get(CONF_VALUE), - register_value.get(CONF_PAGE_ID), - ) - cg.add(var.add_phy_register(reg)) + _add_phy_registers(var, config) # Register Ethernet with the esp32 sdkconfig reconciler. It disables the # WiFi stack and WiFi/BT coexistence only when Ethernet runs without WiFi, diff --git a/esphome/components/ethernet/ethernet_component.h b/esphome/components/ethernet/ethernet_component.h index 71fa9d40b6..44c4d6733d 100644 --- a/esphome/components/ethernet/ethernet_component.h +++ b/esphome/components/ethernet/ethernet_component.h @@ -193,7 +193,9 @@ class EthernetComponent final : public Component { void set_mdio_pin(uint8_t mdio_pin) { this->mdio_pin_ = mdio_pin; } void set_clk_pin(uint8_t clk_pin) { this->clk_pin_ = clk_pin; } void set_clk_mode(emac_rmii_clock_mode_t clk_mode) { this->clk_mode_ = clk_mode; } - void add_phy_register(PHYRegister register_value); +#ifdef ESPHOME_ETHERNET_PHY_REGISTER_COUNT + void add_phy_register(PHYRegister register_value) { this->phy_registers_.push_back(register_value); } +#endif #endif // USE_ETHERNET_SPI #endif // USE_ESP32 @@ -255,8 +257,10 @@ class EthernetComponent final : public Component { /// reset) and set the RGMII Tx/Rx clock delays needed for reliable data sampling. void yt8531_phy_init_(); #endif +#ifdef ESPHOME_ETHERNET_PHY_REGISTER_COUNT /// @brief Set arbitratry PHY registers from config. void write_phy_register_(esp_eth_mac_t *mac, PHYRegister register_data); +#endif #ifdef USE_ETHERNET_SPI uint8_t clk_pin_; @@ -280,7 +284,9 @@ class EthernetComponent final : public Component { // Group all 32-bit members first int power_pin_{-1}; emac_rmii_clock_mode_t clk_mode_{EMAC_CLK_EXT_IN}; - std::vector phy_registers_{}; +#ifdef ESPHOME_ETHERNET_PHY_REGISTER_COUNT + StaticVector phy_registers_{}; +#endif // Group all 8-bit members together uint8_t clk_pin_{0}; diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index 2c3c5f1ef9..e9281453c9 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -462,9 +462,11 @@ void EthernetComponent::ethernet_lazy_init_() { } #endif // USE_ETHERNET_KSZ8081 +#ifdef ESPHOME_ETHERNET_PHY_REGISTER_COUNT for (const auto &phy_register : this->phy_registers_) { this->write_phy_register_(mac, phy_register); } +#endif #if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) #ifdef USE_ETHERNET_GENERIC @@ -1043,10 +1045,6 @@ void EthernetComponent::dump_connect_params_() { #endif /* USE_NETWORK_IPV6 */ } -#ifndef USE_ETHERNET_SPI -void EthernetComponent::add_phy_register(PHYRegister register_value) { this->phy_registers_.push_back(register_value); } -#endif - void EthernetComponent::get_eth_mac_address_raw(uint8_t *mac) { if (!this->ethernet_initialized_) { // External callers (mdns, ethernet_info, etc.) may ask for the MAC before/regardless @@ -1144,6 +1142,7 @@ void EthernetComponent::ksz8081_set_clock_reference_(esp_eth_mac_t *mac) { } #endif // USE_ETHERNET_KSZ8081 +#ifdef ESPHOME_ETHERNET_PHY_REGISTER_COUNT void EthernetComponent::write_phy_register_(esp_eth_mac_t *mac, PHYRegister register_data) { esp_err_t err; @@ -1168,6 +1167,7 @@ void EthernetComponent::write_phy_register_(esp_eth_mac_t *mac, PHYRegister regi } #endif } +#endif // ESPHOME_ETHERNET_PHY_REGISTER_COUNT #ifdef USE_ETHERNET_YT8531 void EthernetComponent::yt8531_phy_init_() { diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 97b1395553..be8d89c557 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -491,6 +491,7 @@ #define USE_ETHERNET_CONNECT_TRIGGER #define USE_ETHERNET_DISCONNECT_TRIGGER #define ESPHOME_ETHERNET_IP_STATE_LISTENERS 2 +#define ESPHOME_ETHERNET_PHY_REGISTER_COUNT 2 #endif #ifdef USE_ESP32 diff --git a/tests/components/ethernet/common-rtl8201.yaml b/tests/components/ethernet/common-rtl8201.yaml index d5a60f6e98..5e24017ff3 100644 --- a/tests/components/ethernet/common-rtl8201.yaml +++ b/tests/components/ethernet/common-rtl8201.yaml @@ -7,6 +7,12 @@ ethernet: mode: CLK_EXT_IN phy_addr: 0 power_pin: 33 + phy_registers: + - address: 0x10 + value: 0x1FFA + page_id: 0x07 + - address: 0x19 + value: 0x0006 manual_ip: static_ip: 192.168.178.56 gateway: 192.168.178.1 From 1110857ffaa6123b3135d57e9936c9a47dbec653 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:21:52 -0500 Subject: [PATCH 160/221] [wifi] Keep ESP8266's report of a replaced attempt from failing the next one (#20183) --- esphome/components/wifi/wifi_component.cpp | 7 ++++--- esphome/components/wifi/wifi_component_esp8266.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 125139ad16..86e13ef84b 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -762,7 +762,7 @@ void WiFiComponent::restart_adapter() { // and check_connecting_finished() is called after cooldown without going // through start_connecting() first. Without this clear, stale errors would // trigger spurious "failed (callback)" logs. The canonical clear location - // is in start_connecting(); this is the only exception to that pattern. + // is in start_connecting() (ESP8266 also clears after wifi_station_connect()). this->error_from_callback_ = false; } @@ -1218,8 +1218,9 @@ void WiFiComponent::start_connecting(const WiFiAP &ap) { // Clear any stale error from previous connection attempt. // This is the canonical location for clearing the flag since all connection - // attempts go through start_connecting(). The only other clear is in - // restart_adapter() which enters COOLDOWN without calling start_connecting(). + // attempts go through start_connecting(). restart_adapter() also clears it, as it + // enters COOLDOWN without calling start_connecting(), and ESP8266 clears it again + // after wifi_station_connect(), whose callbacks fire synchronously. this->error_from_callback_ = false; if (!this->wifi_sta_connect_(ap)) { diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 60ec3f9a4d..aa1fa67b7b 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -376,17 +376,17 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { this->wifi_apply_hostname_(); - // Reset flags, do this _before_ wifi_station_connect as the callback method - // may be called from wifi_station_connect - this->sta_state_ = static_cast(ESP8266WiFiSTAState::CONNECTING); - ETS_UART_INTR_DISABLE(); ret = wifi_station_connect(); ETS_UART_INTR_ENABLE(); if (!ret) { ESP_LOGV(TAG, "wifi_station_connect failed"); + this->sta_state_ = static_cast(ESP8266WiFiSTAState::ERROR_FAILED); return false; } + // Reset after, not before: wifi_station_connect() synchronously reports the attempt it replaces as failed + this->sta_state_ = static_cast(ESP8266WiFiSTAState::CONNECTING); + this->error_from_callback_ = false; #if USE_NETWORK_IPV6 bool connected = false; From 5be0aca6d2396a708a191b75536770b8e4f2b0d2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:24:19 -0500 Subject: [PATCH 161/221] [libretiny] Build LN882x with 1-byte enums to match the SDK's WiFi library (#20173) --- esphome/components/libretiny/__init__.py | 4 +++ esphome/components/libretiny/hal.cpp | 6 +++++ tests/component_tests/libretiny/__init__.py | 0 .../libretiny/config/bk72xx.yaml | 5 ++++ .../libretiny/config/ln882x.yaml | 5 ++++ .../libretiny/test_short_enums.py | 27 +++++++++++++++++++ 6 files changed, 47 insertions(+) create mode 100644 tests/component_tests/libretiny/__init__.py create mode 100644 tests/component_tests/libretiny/config/bk72xx.yaml create mode 100644 tests/component_tests/libretiny/config/ln882x.yaml create mode 100644 tests/component_tests/libretiny/test_short_enums.py diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index 5d87dee07a..5d630fec6c 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -526,6 +526,10 @@ async def component_to_code(config): # RAM-executable output section and prints a post-link placement summary. if FAMILY_COMPONENT[config[CONF_FAMILY]] != COMPONENT_BK72XX: cg.add_platformio_option("extra_scripts", ["pre:patch_linker.py"]) + # Match the 1-byte enums of the LN882H SDK's prebuilt WiFi library. + if FAMILY_COMPONENT[config[CONF_FAMILY]] == COMPONENT_LN882X: + cg.add_build_unflag("-fno-short-enums") + cg.add_build_flag("-fshort-enums") # dummy version code cg.add_define("USE_ARDUINO_VERSION_CODE", cg.RawExpression("VERSION_CODE(0, 0, 0)")) # decrease web server stack size (16k words -> 4k words) diff --git a/esphome/components/libretiny/hal.cpp b/esphome/components/libretiny/hal.cpp index 01b276005d..f97a5a19a7 100644 --- a/esphome/components/libretiny/hal.cpp +++ b/esphome/components/libretiny/hal.cpp @@ -12,6 +12,12 @@ // libretiny component's API. namespace esphome::libretiny {} // namespace esphome::libretiny +#if defined(USE_LN882X) && !defined(CLANG_TIDY) // clang-tidy runs without the PlatformIO flags +// Fail the build if the -fshort-enums swap in __init__.py stops working. +enum Ln882xEnumSizeCheck { LN882X_ENUM_SIZE_CHECK }; +static_assert(sizeof(Ln882xEnumSizeCheck) == 1, "LN882x must build with -fshort-enums to match the SDK"); +#endif + namespace esphome { // yield(), delay(), micros(), millis(), millis_64(), delayMicroseconds(), diff --git a/tests/component_tests/libretiny/__init__.py b/tests/component_tests/libretiny/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/libretiny/config/bk72xx.yaml b/tests/component_tests/libretiny/config/bk72xx.yaml new file mode 100644 index 0000000000..35450f1039 --- /dev/null +++ b/tests/component_tests/libretiny/config/bk72xx.yaml @@ -0,0 +1,5 @@ +esphome: + name: test + +bk72xx: + board: generic-bk7231n-qfn32-tuya diff --git a/tests/component_tests/libretiny/config/ln882x.yaml b/tests/component_tests/libretiny/config/ln882x.yaml new file mode 100644 index 0000000000..4d0381737a --- /dev/null +++ b/tests/component_tests/libretiny/config/ln882x.yaml @@ -0,0 +1,5 @@ +esphome: + name: test + +ln882x: + board: generic-ln882h diff --git a/tests/component_tests/libretiny/test_short_enums.py b/tests/component_tests/libretiny/test_short_enums.py new file mode 100644 index 0000000000..185c47c506 --- /dev/null +++ b/tests/component_tests/libretiny/test_short_enums.py @@ -0,0 +1,27 @@ +"""LN882x builds with 1-byte enums to match the SDK's prebuilt libraries; +the other LibreTiny families keep LibreTiny's own enum size.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from esphome.core import CORE + + +def test_ln882x_swaps_in_short_enums( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("ln882x.yaml")) + assert "-fno-short-enums" in CORE.build_unflags + assert "-fshort-enums" in CORE.build_flags + + +def test_bk72xx_keeps_libretiny_enums( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("bk72xx.yaml")) + assert "-fno-short-enums" not in CORE.build_unflags + assert "-fshort-enums" not in CORE.build_flags From 4258b226a429f60c1bce39e3af9da89dd12ac7f8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:26:22 -0500 Subject: [PATCH 162/221] [wifi] Default LN882x power save to none, as on BK72xx and RTL87xx (#20170) --- esphome/components/wifi/__init__.py | 2 +- tests/component_tests/wifi/config/bare_ln882x.yaml | 12 ++++++++++++ tests/component_tests/wifi/test_default_setters.py | 4 +++- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/component_tests/wifi/config/bare_ln882x.yaml diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index fa552da969..553c737dbc 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -514,7 +514,7 @@ CONFIG_SCHEMA = cv.All( rp2="light", bk72xx="none", rtl87xx="none", - ln882x="light", + ln882x="none", ): cv.enum(WIFI_POWER_SAVE_MODES, upper=True), cv.Optional(CONF_FAST_CONNECT, default=False): _fast_connect_schema, cv.Optional(CONF_USE_ADDRESS): cv.string_strict, diff --git a/tests/component_tests/wifi/config/bare_ln882x.yaml b/tests/component_tests/wifi/config/bare_ln882x.yaml new file mode 100644 index 0000000000..885a469b8d --- /dev/null +++ b/tests/component_tests/wifi/config/bare_ln882x.yaml @@ -0,0 +1,12 @@ +--- +esphome: + name: test + +ln882x: + board: generic-ln882h + +wifi: + ssid: test + password: testtest + ap: + ssid: fallback diff --git a/tests/component_tests/wifi/test_default_setters.py b/tests/component_tests/wifi/test_default_setters.py index b326f3eaee..dfac443060 100644 --- a/tests/component_tests/wifi/test_default_setters.py +++ b/tests/component_tests/wifi/test_default_setters.py @@ -6,7 +6,9 @@ from pathlib import Path import pytest -@pytest.mark.parametrize("config_file", ["bare.yaml", "defaults.yaml"]) +@pytest.mark.parametrize( + "config_file", ["bare.yaml", "defaults.yaml", "bare_ln882x.yaml"] +) def test_default_values_are_not_emitted( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], From 9820896430a7814af8725405446011cb09c184f1 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:34:07 +0100 Subject: [PATCH 163/221] [wireguard] Bump the library to esphome/wireguard 0.4.7 (#20188) --- esphome/components/wireguard/__init__.py | 4 ++-- esphome/idf_component.yml | 2 +- platformio.ini | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/esphome/components/wireguard/__init__.py b/esphome/components/wireguard/__init__.py index ac62812fcf..94af559175 100644 --- a/esphome/components/wireguard/__init__.py +++ b/esphome/components/wireguard/__init__.py @@ -143,12 +143,12 @@ async def to_code(config): if CORE.is_esp32: add_idf_sdkconfig_option("CONFIG_LWIP_PPP_SUPPORT", True) - # This flag is added here because the esp_wireguard library statically + # This flag is added here because the wireguard library statically # set the size of its allowed_ips list at compile time using this value; # the '+1' modifier is relative to the device's own address that will # be automatically added to the provided list. cg.add_build_flag(f"-DCONFIG_WIREGUARD_MAX_SRC_IPS={len(allowed_ips) + 1}") - cg.add_library("droscy/esp_wireguard", "0.4.5") + cg.add_library("esphome/wireguard", "0.4.7") await cg.register_component(var, config) diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index fc8090a322..58027cb33f 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -112,7 +112,7 @@ dependencies: version: 0.1.30 rules: - if: "$ESPHOME_ARDUINO_COMPONENT == 0 || idf_version >= 6.0.0" - # Declared alongside noise-c so the library converter skips esp_wireguard's copy + # Declared alongside noise-c so the library converter skips wireguard's copy esphome/libsodium: version: 1.10021.11 rules: diff --git a/platformio.ini b/platformio.ini index 00fad1ff66..deb4bf8ea0 100644 --- a/platformio.ini +++ b/platformio.ini @@ -130,7 +130,7 @@ lib_deps = ESP8266HTTPClient ; http_request (Arduino built-in) ESP8266mDNS ; mdns (Arduino built-in) DNSServer ; captive_portal (Arduino built-in) - droscy/esp_wireguard@0.4.5 ; wireguard + esphome/wireguard@0.4.7 ; wireguard build_flags = ${common:arduino.build_flags} @@ -165,7 +165,7 @@ lib_deps = DNSServer ; captive_portal (Arduino built-in) makuna/NeoPixelBus@2.8.0 ; neopixelbus esphome/ESP32-audioI2S@2.3.0 ; i2s_audio - droscy/esp_wireguard@0.4.5 ; wireguard + esphome/wireguard@0.4.7 ; wireguard build_flags = ${common:arduino.build_flags} @@ -188,7 +188,7 @@ platform_packages = framework = espidf lib_deps = ${common:idf.lib_deps} - droscy/esp_wireguard@0.4.5 ; wireguard + esphome/wireguard@0.4.7 ; wireguard tonia/HeatpumpIR@1.0.42 ; heatpumpir build_flags = ${common:idf.build_flags} @@ -246,7 +246,7 @@ lib_deps = ${common.lib_deps_base} ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base - droscy/esp_wireguard@0.4.5 ; wireguard + esphome/wireguard@0.4.7 ; wireguard esphome/noise-c@0.1.30 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal From bc388b1de529e96909d947faf56a5cdd8562b570 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Mon, 5 Oct 2026 11:51:49 -0400 Subject: [PATCH 164/221] [epaper_spi] Add SSD1681 and the Waveshare 1.54in V2 panel (#18073) --- .../components/epaper_spi/models/ssd1683.py | 12 ++++++++++++ .../epaper_spi/test.esp32-s3-idf.yaml | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/esphome/components/epaper_spi/models/ssd1683.py b/esphome/components/epaper_spi/models/ssd1683.py index b43168a336..eb51ba1f2a 100644 --- a/esphome/components/epaper_spi/models/ssd1683.py +++ b/esphome/components/epaper_spi/models/ssd1683.py @@ -26,3 +26,15 @@ goodisplay_gdey042t81 = ssd1683.extend( width=400, height=300, ) + +# The SSD1681 shares the SSD1683 command set for everything this driver uses: +# the same 0x01/0x11/0x18 initialisation, the same 0x44/0x45 window and +# 0x4E/0x4F cursor registers with a byte-addressed X and a 16-bit Y, and the +# same 0x22/0x20 update. It only differs in the panel sizes it drives. +ssd1681 = SSD1683("ssd1681") + +waveshare_1_54in_v2 = ssd1681.extend( + "waveshare-1.54in-v2", + width=200, + height=200, +) diff --git a/tests/components/epaper_spi/test.esp32-s3-idf.yaml b/tests/components/epaper_spi/test.esp32-s3-idf.yaml index 1678f46331..7a4ce3a5ff 100644 --- a/tests/components/epaper_spi/test.esp32-s3-idf.yaml +++ b/tests/components/epaper_spi/test.esp32-s3-idf.yaml @@ -190,6 +190,24 @@ display: allow_other_uses: true number: GPIO4 + # Waveshare 1.54" V2 mono e-paper (200x200, SSD1681) + - platform: epaper_spi + spi_id: spi_bus + model: waveshare-1.54in-v2 + full_update_every: 30 + cs_pin: + allow_other_uses: true + number: GPIO5 + dc_pin: + allow_other_uses: true + number: GPIO17 + reset_pin: + allow_other_uses: true + number: GPIO16 + busy_pin: + allow_other_uses: true + number: GPIO4 + # Waveshare 2.13" V4 B series 3-color e-paper (122x250, BWR, SSD1680) - platform: epaper_spi spi_id: spi_bus From 417fdd9dc76ef0a5a29e151cbf32a43e10d068c1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:53:35 -0500 Subject: [PATCH 165/221] [nrf52] Share the ccache between devices (#20028) --- esphome/components/nrf52/__init__.py | 19 +++-- esphome/components/nrf52/framework.py | 78 ++++++++++++++++--- tests/unit_tests/components/nrf52/test_pch.py | 75 ++++++++++++++---- tests/unit_tests/test_nrf52_framework.py | 49 +++++++++++- 4 files changed, 183 insertions(+), 38 deletions(-) diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 1e841fffcf..86d485777c 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -27,6 +27,7 @@ from esphome.components.zephyr.const import ( CONF_CDC_ACM, KEY_BOARD, KEY_BOOTLOADER, + KEY_SYSBUILD, KEY_ZEPHYR, CdcAcm, ) @@ -77,6 +78,7 @@ from .framework import ( get_build_paths, setup_platformio_python_env, toolchain_tool, + wanted_west_projects, ) # force import gpio to register pin schema @@ -822,19 +824,22 @@ _PCH_SUM_PATH = "CMakeFiles/app.dir/cmake_pch.hxx.gch.sum" def _write_pch_checksum(build_dir: Path, source_dir: Path) -> None: - """Write the checksum ccache reads in place of the .gch. The app binary - dir only exists after the first configure; sysbuild nests it.""" - app_dir = build_dir / "zephyr" - if not (app_dir / "CMakeCache.txt").is_file(): - app_dir = build_dir - if not (app_dir / "CMakeCache.txt").is_file(): - return + """Write the checksum ccache reads in place of the .gch; before the + first build too, or its compiles hash the path laden .gch instead. + The app image dir follows the SDK version, like get_elf_path; + 2.9.2+ always wraps the build in sysbuild.""" + app_dir = build_dir + if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(2, 9, 2): + app_dir = build_dir / "zephyr" checksum = pch.pch_checksum( CORE.relative_src_path(), pch.PCH_DEFAULT_HEADERS, ( str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]), zephyr_data()[KEY_BOARD], + # Kconfig inputs that reach autoconf.h without a .conf line + ",".join(sorted(wanted_west_projects())), + str(zephyr_data().get(KEY_SYSBUILD)), # What the Zephyr configuration is generated from *( path.read_text(encoding="utf-8") diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index e6ccde6a2d..00796562f4 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -225,6 +225,19 @@ def get_build_env(ccache: str | None) -> dict: env.setdefault("CCACHE_DISABLE", "1") else: env.update(ccache_env(ccache, SDK_NRF_TOOLS_CACHE)) + # Drop only the per build map entry (posix, CMake's spelling); + # its from side covers no compiled sources. A spaced path cannot + # survive ccache's space split list, so it stays hashed. + source_dir = CORE.relative_build_path("zephyr").as_posix() + if any(ch.isspace() for ch in source_dir): + _LOGGER.debug( + "Whitespace in %s; the per build map stays hashed", source_dir + ) + else: + device_map = f"-fmacro-prefix-map={source_dir}=CMAKE_SOURCE_DIR" + env["CCACHE_IGNOREOPTIONS"] = ( + f"{env.get('CCACHE_IGNOREOPTIONS', '')} {device_map}".strip() + ) return env @@ -301,21 +314,62 @@ def setup_platformio_python_env() -> None: _prepend_env_path("PATH", str(env_python_path.parent)) +def _patch_framework_file(path: Path, old: str, new: str) -> bool: + """Replace ``old`` with ``new`` in a framework script, atomically and + keeping the file mode (helpers.write_file would flatten it to 0o644). + Returns False when nothing matched.""" + import tempfile + + content = path.read_text(encoding="utf-8") + patched = content.replace(old, new) + if patched == content: + return False + # Unique sibling tmp: the install lock is best effort, so two builds + # may patch at once and a shared tmp name could rename a half + # written file into place. + fd, tmp_name = tempfile.mkstemp(dir=path.parent, suffix=".tmp") + tmp = Path(tmp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(patched) + shutil.copymode(path, tmp) + tmp.replace(path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + return True + + def _patch_uf2conv_escape_sequences(framework_path: Path) -> None: # SDK v2.6.1 ships uf2conv.py with '\s+' — an unrecognised escape that # Python 3.12+ flags with SyntaxWarning (a future version will reject it). uf2conv = framework_path / "zephyr" / "scripts" / "build" / "uf2conv.py" - if not uf2conv.exists(): + if uf2conv.exists(): + _patch_framework_file( + uf2conv, "re.split('\\s+', line)", "re.split('\\\\s+', line)" + ) + + +def _patch_gen_defines_dts_path(framework_path: Path) -> None: + # The absolute zephyr.dts.pre path in the header's top comment is + # its only per device byte and blocks ccache sharing; emit the + # basename. Upstream candidate. + gen_defines = framework_path / "zephyr" / "scripts" / "dts" / "gen_defines.py" + if not gen_defines.exists(): return - content = uf2conv.read_text(encoding="utf-8") - patched = content.replace("re.split('\\s+', line)", "re.split('\\\\s+', line)") - if patched == content: + if _patch_framework_file( + gen_defines, " {edt.dts_path}", " {os.path.basename(edt.dts_path)}" + ): return - # Write atomically so a concurrent build never sees a truncated file - tmp = uf2conv.with_suffix(".py.tmp") - tmp.write_text(patched, encoding="utf-8") - shutil.copymode(uf2conv, tmp) - tmp.replace(uf2conv) + if "{os.path.basename(edt.dts_path)}" not in gen_defines.read_text( + encoding="utf-8" + ): + # Upstream reformatted the comment; sharing silently degrading + # would be invisible, so say it out loud. + _LOGGER.warning( + "gen_defines.py no longer matches; the devicetree header " + "stays per device and ccache sharing between devices degrades" + ) # West projects every build needs; components add others with include_west_project() @@ -349,7 +403,7 @@ def bluetooth_west_projects() -> tuple[str, ...]: return ("tinycrypt",) -def _wanted_west_projects() -> set[str]: +def wanted_west_projects() -> set[str]: projects = set(_get_data().west_projects) # Zephyr 4.1 moved the Cortex-M core headers to cmsis_6 if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(3, 1, 0): @@ -587,7 +641,7 @@ def _check_and_install(version: str) -> None: framework_path = _get_framework_path(version) sentinel = framework_path / ".ready" zephyr_reqs = framework_path / "zephyr" / "scripts" / "requirements.txt" - projects = _wanted_west_projects() + projects = wanted_west_projects() if not sentinel.exists() or not zephyr_reqs.exists(): _install_framework(env_python_path, framework_path, version, projects) framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] @@ -596,6 +650,8 @@ def _check_and_install(version: str) -> None: sentinel.touch() else: _fetch_missing_west_projects(env_python_path, framework_path, version, projects) + # Every run: existing installs need it too, and it is a no-op once applied + _patch_gen_defines_dts_path(framework_path) zephyr_sentinel = python_env_path / ".zephyr_reqs_ready" if ( diff --git a/tests/unit_tests/components/nrf52/test_pch.py b/tests/unit_tests/components/nrf52/test_pch.py index 41635c3580..7eb2fe10f3 100644 --- a/tests/unit_tests/components/nrf52/test_pch.py +++ b/tests/unit_tests/components/nrf52/test_pch.py @@ -5,9 +5,11 @@ from unittest.mock import Mock, patch import pytest +from esphome.build_helpers import pch from esphome.components import nrf52 from esphome.components.nrf52 import framework -from esphome.components.zephyr.const import KEY_BOARD +from esphome.components.nrf52.toolchain import get_elf_path +from esphome.components.zephyr.const import KEY_BOARD, KEY_SYSBUILD import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain from esphome.core import CORE, EsphomeError @@ -55,8 +57,6 @@ def test_cmake_lists_pch_block_disabled(tmp_path: Path) -> None: def test_the_zephyr_compiler_decides_on_windows( windows_gcc_rule: None, version: tuple[int, ...], on: bool ) -> None: - from esphome.build_helpers import pch - # platformdirs would pick its Windows backend from the patched sys.platform with ( patch.object(nrf52, "toolchain_tool", lambda name: Path(f"/sdk/{name}.exe")), @@ -66,8 +66,16 @@ def test_the_zephyr_compiler_decides_on_windows( assert asked.call_args.args[0] == (Path("/sdk/g++.exe"),) -def _write_checksum(tmp_path: Path, app: str, conf: str = "CONFIG_X=y\n") -> Path: +def _write_checksum( + tmp_path: Path, + app: str, + conf: str = "CONFIG_X=y\n", + configured: bool = True, + sysbuild: bool = False, + version: cv.Version | None = None, +) -> Path: """Write the checksum for a build dir whose app image sits in ``app``.""" + version = version or cv.Version(2, 9, 2) CORE.build_path = tmp_path header = tmp_path / "src" / "esphome" / "core" / "pch_prefix.h" header.parent.mkdir(parents=True, exist_ok=True) @@ -77,33 +85,65 @@ def _write_checksum(tmp_path: Path, app: str, conf: str = "CONFIG_X=y\n") -> Pat (source_dir / "prj.conf").write_text(conf) (source_dir / "CMakeLists.txt").write_text("not part of the checksum\n") build_dir = tmp_path / ".pioenvs" / "livingroom" - (build_dir / app).mkdir(parents=True, exist_ok=True) - (build_dir / app / "CMakeCache.txt").write_text("") + if configured: + (build_dir / app).mkdir(parents=True, exist_ok=True) + (build_dir / app / "CMakeCache.txt").write_text("") with ( - patch.dict(CORE.data, {KEY_CORE: {KEY_FRAMEWORK_VERSION: "2.9.2"}}), - patch.object(nrf52, "zephyr_data", return_value={KEY_BOARD: "board"}), + patch.dict(CORE.data, {KEY_CORE: {KEY_FRAMEWORK_VERSION: version}}), + patch.object( + nrf52, + "zephyr_data", + return_value={KEY_BOARD: "board", KEY_SYSBUILD: sysbuild}, + ), ): nrf52._write_pch_checksum(build_dir, source_dir) return build_dir / app / SUM -@pytest.mark.parametrize("app", ["zephyr", "."]) -def test_pch_checksum_is_written_next_to_the_gch(tmp_path: Path, app: str) -> None: - """Sysbuild nests the app image; without it the build dir is the app.""" - sum_path = _write_checksum(tmp_path, app) +@pytest.mark.parametrize( + ("app", "version"), [("zephyr", cv.Version(2, 9, 2)), (".", cv.Version(2, 9, 1))] +) +def test_pch_checksum_is_written_next_to_the_gch( + tmp_path: Path, app: str, version: cv.Version +) -> None: + """The SDK version decides the layout, like get_elf_path.""" + sum_path = _write_checksum(tmp_path, app, version=version) assert len(sum_path.read_text().strip()) == 64 +def test_pch_checksum_tracks_the_kconfig_side_inputs(tmp_path: Path) -> None: + """West projects and the sysbuild flag reach autoconf without a .conf + line; the sum must move with them or stale objects get served.""" + first = _write_checksum(tmp_path, "zephyr").read_text() + with patch.object(nrf52, "wanted_west_projects", return_value={"extra"}): + second = _write_checksum(tmp_path, "zephyr").read_text() + assert first != second + third = _write_checksum(tmp_path, "zephyr", sysbuild=True).read_text() + assert first != third + + def test_pch_checksum_tracks_the_zephyr_configuration(tmp_path: Path) -> None: first = _write_checksum(tmp_path, "zephyr").read_text() assert _write_checksum(tmp_path, "zephyr", "CONFIG_X=n\n").read_text() != first -def test_pch_checksum_waits_for_the_first_configure(tmp_path: Path) -> None: - CORE.build_path = tmp_path - build_dir = tmp_path / ".pioenvs" / "livingroom" - nrf52._write_pch_checksum(build_dir, tmp_path / "zephyr") - assert not build_dir.exists() +@pytest.mark.parametrize("version", [cv.Version(2, 9, 2), cv.Version(2, 9, 1)]) +def test_pch_checksum_lands_where_the_build_writes_the_image( + tmp_path: Path, version: cv.Version +) -> None: + """One layout rule: the sum must sit in get_elf_path's app dir, or a + layout drift silently costs the first build's sharing.""" + app = "zephyr" if version >= cv.Version(2, 9, 2) else "." + sum_path = _write_checksum(tmp_path, app, version=version) + CORE.name = "livingroom" + with patch.dict(CORE.data, {KEY_CORE: {KEY_FRAMEWORK_VERSION: version}}): + expected = get_elf_path().parent.parent / SUM + assert sum_path.resolve() == expected.resolve() + + +def test_pch_checksum_written_before_the_first_configure(tmp_path: Path) -> None: + """The first build's compiles hash the sum in place of the .gch.""" + assert _write_checksum(tmp_path, "zephyr", configured=False).is_file() def _fake_build_env(ccache: str | None) -> dict[str, str]: @@ -130,6 +170,7 @@ def run_cmd(tmp_path: Path) -> Mock: CORE.name = "livingroom" CORE.toolchain = Toolchain.SDK_NRF CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 2, 0)} + (tmp_path / "build" / "zephyr").mkdir(parents=True) with ( patch.dict("os.environ", {}, clear=True), patch.object(nrf52, "check_and_install"), diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index d9cf2d37b3..522061dec2 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -11,7 +11,7 @@ from unittest.mock import ANY, call, patch import platformdirs import pytest -from esphome.components.nrf52 import _resolve_toolchain +from esphome.components.nrf52 import _resolve_toolchain, framework from esphome.components.nrf52.framework import ( _PLATFORMIO_PENV_REQUIREMENTS, _REQUIREMENTS, @@ -22,12 +22,12 @@ from esphome.components.nrf52.framework import ( _get_toolchain_platform_info, _install_toolchain, _needs_venv_rebuild, - _wanted_west_projects, check_and_install, get_build_env, get_sdk_nrf_tools_path, include_west_project, setup_platformio_python_env, + wanted_west_projects, ) from esphome.components.zephyr.const import KEY_SYSBUILD, KEY_ZEPHYR import esphome.config_validation as cv @@ -539,7 +539,7 @@ class TestCheckAndInstall: """Zephyr 4.1 moved the Cortex-M core headers to the cmsis_6 module.""" CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse(sdk_version)} - assert ("cmsis_6" in _wanted_west_projects()) is has_cmsis_6 + assert ("cmsis_6" in wanted_west_projects()) is has_cmsis_6 def test_default_projects_never_read_the_stamp( self, @@ -1121,6 +1121,22 @@ def test_get_build_env_with_ccache( assert env["CCACHE_DEPEND"] == "1" assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve()) assert "CCACHE_DISABLE" not in env + # Only the per build map entry leaves the hash; user maps stay in + assert env["CCACHE_IGNOREOPTIONS"] == ( + f"-fmacro-prefix-map={(tmp_path / 'build' / 'zephyr').as_posix()}" + "=CMAKE_SOURCE_DIR" + ) + + +def test_get_build_env_skips_the_map_entry_on_whitespace( + nrf52_dirs: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The ignore list splits on spaces; a spaced path cannot be + expressed, so the entry is left hashed rather than emitted broken.""" + monkeypatch.delenv("CCACHE_IGNOREOPTIONS", raising=False) + CORE.build_path = tmp_path / "with space" / "build" + env = get_build_env("/usr/bin/ccache") + assert "CCACHE_IGNOREOPTIONS" not in env def test_get_build_env_sdk_3_4_0_uses_toolchain_root( @@ -1250,3 +1266,30 @@ def test_resolve_toolchain_rejects_unsupported() -> None: CORE.toolchain = Toolchain.ARDUINO with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): _resolve_toolchain({}) + + +def test_patch_gen_defines_relativizes_the_dts_path(tmp_path: Path) -> None: + """The absolute dts.pre path is the only per device byte in the + devicetree header; the patch makes gen_defines emit the basename.""" + gen = tmp_path / "zephyr" / "scripts" / "dts" / "gen_defines.py" + gen.parent.mkdir(parents=True) + gen.write_text("s = f'DTS input file:\\n {edt.dts_path}\\n'\n") + framework._patch_gen_defines_dts_path(tmp_path) + assert "{os.path.basename(edt.dts_path)}" in gen.read_text() + assert not list(gen.parent.glob("*.tmp")) # no leftovers + before = gen.read_text() + framework._patch_gen_defines_dts_path(tmp_path) # idempotent + assert gen.read_text() == before + framework._patch_gen_defines_dts_path(tmp_path / "absent") # tolerant + + +def test_patch_gen_defines_warns_when_the_anchor_is_gone( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A reformatted upstream must not silently cost the sharing.""" + gen = tmp_path / "zephyr" / "scripts" / "dts" / "gen_defines.py" + gen.parent.mkdir(parents=True) + gen.write_text("s = 'something else entirely'\n") + with caplog.at_level("WARNING"): + framework._patch_gen_defines_dts_path(tmp_path) + assert "gen_defines.py no longer matches" in caplog.text From a058937fe002be00176e98c4bfccf3c8519b84fe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:53:47 -0500 Subject: [PATCH 166/221] [api] Count packed varints a machine word at a time (#20122) --- esphome/components/api/proto.h | 31 +++---- .../api/test_count_packed_varints.cpp | 84 +++++++++++++++++++ 2 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 tests/components/api/test_count_packed_varints.cpp diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index c112b0f415..214d9f417f 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -56,22 +56,25 @@ inline constexpr int64_t decode_zigzag64(uint64_t value) { return (value & 1) ? static_cast(~(value >> 1)) : static_cast(value >> 1); } -/// Count number of varints in a packed buffer -inline uint16_t count_packed_varints(const uint8_t *data, size_t len) { - uint16_t count = 0; - while (len > 0) { - // Skip varint bytes until we find one without continuation bit - while (len > 0 && (*data & 0x80)) { - data++; - len--; - } - if (len > 0) { - data++; - len--; - count++; +/// Count varints in a packed buffer: len minus bytes with the continuation bit, summed a word at a time. +/// Word is a template parameter so tests can cover the 32-bit path on a 64-bit host. +template inline uint16_t count_packed_varints(const uint8_t *data, size_t len) { + constexpr size_t word_size = sizeof(Word); + constexpr Word lane_ones = ~Word{0} / 0xFF; // 0x01..01 + const uint8_t *end = data + len; + size_t continuations = 0; + while (data != end) { + // Unaligned word loads fault on Xtensa + if ((reinterpret_cast(data) & (word_size - 1)) == 0 && static_cast(end - data) >= word_size) { + Word word; + memcpy(&word, __builtin_assume_aligned(data, word_size), word_size); + continuations += (((word >> 7) & lane_ones) * lane_ones) >> (word_size * 8 - 8); + data += word_size; + } else { + continuations += *data++ >> 7; } } - return count; + return static_cast(len - continuations); } /// Encode a varint directly into a pre-allocated buffer. diff --git a/tests/components/api/test_count_packed_varints.cpp b/tests/components/api/test_count_packed_varints.cpp new file mode 100644 index 0000000000..5745403a9f --- /dev/null +++ b/tests/components/api/test_count_packed_varints.cpp @@ -0,0 +1,84 @@ +#include + +#include +#include +#include + +#include "esphome/components/api/proto.h" + +namespace esphome::api::testing { + +// The original byte at a time implementation. +static uint16_t reference_count(const uint8_t *data, size_t len) { + uint16_t count = 0; + while (len > 0) { + while (len > 0 && (*data & 0x80)) { + data++; + len--; + } + if (len > 0) { + data++; + len--; + count++; + } + } + return count; +} + +TEST(CountPackedVarints, EmptyBuffer) { + const uint8_t data[1] = {0x00}; + EXPECT_EQ(count_packed_varints(data, 0), 0); +} + +TEST(CountPackedVarints, SingleByteVarints) { + const uint8_t data[] = {0x00, 0x01, 0x7F}; + EXPECT_EQ(count_packed_varints(data, sizeof(data)), 3); +} + +TEST(CountPackedVarints, MultiByteVarints) { + // 3 varints: 2 bytes, 3 bytes, 1 byte + const uint8_t data[] = {0x80, 0x01, 0x80, 0x80, 0x01, 0x05}; + EXPECT_EQ(count_packed_varints(data, sizeof(data)), 3); +} + +TEST(CountPackedVarints, TruncatedTrailingVarintIsNotCounted) { + const uint8_t data[] = {0x05, 0x80, 0x80, 0x80}; + EXPECT_EQ(count_packed_varints(data, sizeof(data)), 1); +} + +TEST(CountPackedVarints, AllContinuationBytes) { + std::vector data(5000, 0x80); + EXPECT_EQ(count_packed_varints(data.data(), data.size()), 0); +} + +TEST(CountPackedVarints, EveryStartOffsetAndLength) { + // Cover every alignment and length around word boundaries. + std::mt19937 rng(42); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) reproducible + std::vector buf(300); + for (auto &byte : buf) + byte = static_cast(rng() & 0xFF); + for (size_t offset = 0; offset < 16; offset++) { + for (size_t len = 0; len + offset <= buf.size(); len++) { + const uint16_t expected = reference_count(buf.data() + offset, len); + EXPECT_EQ(count_packed_varints(buf.data() + offset, len), expected) + << "offset=" << offset << " len=" << len; + EXPECT_EQ(count_packed_varints(buf.data() + offset, len), expected) + << "offset=" << offset << " len=" << len; + } + } +} + +TEST(CountPackedVarints, LongBuffer) { + std::mt19937 rng(7); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) reproducible + std::vector buf(5000); + for (auto &byte : buf) + byte = static_cast((rng() % 4 == 0) ? (0x80 | (rng() & 0x7F)) : (rng() & 0x7F)); + for (size_t offset = 0; offset < 8; offset++) { + const size_t len = buf.size() - offset; + const uint16_t expected = reference_count(buf.data() + offset, len); + EXPECT_EQ(count_packed_varints(buf.data() + offset, len), expected) << "offset=" << offset; + EXPECT_EQ(count_packed_varints(buf.data() + offset, len), expected) << "offset=" << offset; + } +} + +} // namespace esphome::api::testing From 666cf169b9ebb6266476af0b14a2853f54409c1d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:54:03 -0500 Subject: [PATCH 167/221] [remote_base] Keep RC switch protocols in flash (#20081) --- esphome/components/remote_base/__init__.py | 44 +++++++++----- .../remote_base/rc_switch_protocol.cpp | 12 ++-- .../remote_base/rc_switch_protocol.h | 13 ++++- .../config/rc_switch_protocol.yaml | 58 +++++++++++++++++++ .../test_rc_switch_protocol.py | 25 ++++++++ 5 files changed, 133 insertions(+), 19 deletions(-) create mode 100644 tests/component_tests/remote_receiver/config/rc_switch_protocol.yaml create mode 100644 tests/component_tests/remote_receiver/test_rc_switch_protocol.py diff --git a/esphome/components/remote_base/__init__.py b/esphome/components/remote_base/__init__.py index befef872f0..5d615f0edf 100644 --- a/esphome/components/remote_base/__init__.py +++ b/esphome/components/remote_base/__init__.py @@ -1538,9 +1538,7 @@ def validate_rc_switch_raw_code(value): return value -def build_rc_switch_protocol(config): - if isinstance(config, int): - return rc_switch_protocol(config) +def build_custom_rc_switch_protocol(config: ConfigType) -> MockObj: pl = config[CONF_PULSE_LENGTH] return RCSwitchBase( config[CONF_SYNC][0] * pl, @@ -1553,6 +1551,24 @@ def build_rc_switch_protocol(config): ) +def rc_switch_protocol_in_flash(config: int | ConfigType) -> MockObj: + """Pointer to the protocol in flash: a built-in table entry or a shared custom table.""" + if isinstance(config, int): + return cg.RawExpression(f"&{RC_SWITCH_PROTOCOLS}[{config}]") + return cg.shared_progmem_array( + "rc_switch_custom_protocol", + RCSwitchBase, + [build_custom_rc_switch_protocol(config)], + ) + + +def rc_switch_protocol_value(config: int | ConfigType) -> MockObj: + """RAM copy of a constant protocol for the transmit actions, read from its flash table.""" + if isinstance(config, int): + return rc_switch_protocol(config) + return rc_switch_protocol_copy(rc_switch_protocol_in_flash(config)) + + RC_SWITCH_RAW_SCHEMA = cv.Schema( { cv.Required(CONF_CODE): validate_rc_switch_raw_code, @@ -1628,6 +1644,8 @@ RC_SWITCH_TRANSMITTER = cv.Schema( ) rc_switch_protocol = ns.rc_switch_protocol +rc_switch_protocol_copy = ns.rc_switch_protocol_copy +RC_SWITCH_PROTOCOLS = ns.RC_SWITCH_PROTOCOLS RCSwitchData = ns.struct("RCSwitchData") RCSwitchBase = ns.class_("RCSwitchBase") RCSwitchTrigger = ns.class_("RCSwitchTrigger", RemoteReceiverTrigger) @@ -1642,7 +1660,7 @@ RCSwitchRawReceiver = ns.class_("RCSwitchRawReceiver", RemoteReceiverBinarySenso @register_binary_sensor("rc_switch_raw", RCSwitchRawReceiver, RC_SWITCH_RAW_SCHEMA) def rc_switch_raw_binary_sensor(var, config): - cg.add(var.set_protocol(build_rc_switch_protocol(config[CONF_PROTOCOL]))) + cg.add(var.set_protocol(rc_switch_protocol_in_flash(config[CONF_PROTOCOL]))) cg.add(var.set_code(config[CONF_CODE])) @@ -1653,7 +1671,7 @@ def rc_switch_raw_binary_sensor(var, config): ) async def rc_switch_raw_action(var, config, args): proto = await cg.templatable( - config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=build_rc_switch_protocol + config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=rc_switch_protocol_value ) cg.add(var.set_protocol(proto)) cg.add(var.set_code(await cg.templatable(config[CONF_CODE], args, cg.std_string))) @@ -1663,7 +1681,7 @@ async def rc_switch_raw_action(var, config, args): "rc_switch_type_a", RCSwitchRawReceiver, RC_SWITCH_TYPE_A_SCHEMA ) def rc_switch_type_a_binary_sensor(var, config): - cg.add(var.set_protocol(build_rc_switch_protocol(config[CONF_PROTOCOL]))) + cg.add(var.set_protocol(rc_switch_protocol_in_flash(config[CONF_PROTOCOL]))) cg.add(var.set_type_a(config[CONF_GROUP], config[CONF_DEVICE], config[CONF_STATE])) @@ -1674,7 +1692,7 @@ def rc_switch_type_a_binary_sensor(var, config): ) async def rc_switch_type_a_action(var, config, args): proto = await cg.templatable( - config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=build_rc_switch_protocol + config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=rc_switch_protocol_value ) cg.add(var.set_protocol(proto)) cg.add(var.set_group(await cg.templatable(config[CONF_GROUP], args, cg.std_string))) @@ -1688,7 +1706,7 @@ async def rc_switch_type_a_action(var, config, args): "rc_switch_type_b", RCSwitchRawReceiver, RC_SWITCH_TYPE_B_SCHEMA ) def rc_switch_type_b_binary_sensor(var, config): - cg.add(var.set_protocol(build_rc_switch_protocol(config[CONF_PROTOCOL]))) + cg.add(var.set_protocol(rc_switch_protocol_in_flash(config[CONF_PROTOCOL]))) cg.add( var.set_type_b(config[CONF_ADDRESS], config[CONF_CHANNEL], config[CONF_STATE]) ) @@ -1701,7 +1719,7 @@ def rc_switch_type_b_binary_sensor(var, config): ) async def rc_switch_type_b_action(var, config, args): proto = await cg.templatable( - config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=build_rc_switch_protocol + config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=rc_switch_protocol_value ) cg.add(var.set_protocol(proto)) cg.add(var.set_address(await cg.templatable(config[CONF_ADDRESS], args, cg.uint8))) @@ -1713,7 +1731,7 @@ async def rc_switch_type_b_action(var, config, args): "rc_switch_type_c", RCSwitchRawReceiver, RC_SWITCH_TYPE_C_SCHEMA ) def rc_switch_type_c_binary_sensor(var, config): - cg.add(var.set_protocol(build_rc_switch_protocol(config[CONF_PROTOCOL]))) + cg.add(var.set_protocol(rc_switch_protocol_in_flash(config[CONF_PROTOCOL]))) cg.add( var.set_type_c( config[CONF_FAMILY], @@ -1731,7 +1749,7 @@ def rc_switch_type_c_binary_sensor(var, config): ) async def rc_switch_type_c_action(var, config, args): proto = await cg.templatable( - config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=build_rc_switch_protocol + config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=rc_switch_protocol_value ) cg.add(var.set_protocol(proto)) cg.add( @@ -1748,7 +1766,7 @@ async def rc_switch_type_c_action(var, config, args): RC_SWITCH_TYPE_D_SCHEMA.extend(RC_SWITCH_TRANSMITTER), ) def rc_switch_type_d_binary_sensor(var, config): - cg.add(var.set_protocol(build_rc_switch_protocol(config[CONF_PROTOCOL]))) + cg.add(var.set_protocol(rc_switch_protocol_in_flash(config[CONF_PROTOCOL]))) cg.add(var.set_type_d(config[CONF_GROUP], config[CONF_DEVICE], config[CONF_STATE])) @@ -1759,7 +1777,7 @@ def rc_switch_type_d_binary_sensor(var, config): ) async def rc_switch_type_d_action(var, config, args): proto = await cg.templatable( - config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=build_rc_switch_protocol + config[CONF_PROTOCOL], args, RCSwitchBase, to_exp=rc_switch_protocol_value ) cg.add(var.set_protocol(proto)) cg.add(var.set_group(await cg.templatable(config[CONF_GROUP], args, cg.std_string))) diff --git a/esphome/components/remote_base/rc_switch_protocol.cpp b/esphome/components/remote_base/rc_switch_protocol.cpp index de16c55cb0..bdd6615d43 100644 --- a/esphome/components/remote_base/rc_switch_protocol.cpp +++ b/esphome/components/remote_base/rc_switch_protocol.cpp @@ -9,12 +9,16 @@ namespace esphome::remote_base { static const char *const TAG = "remote.rc_switch"; RCSwitchBase rc_switch_protocol(uint8_t index) { - RCSwitchBase protocol; // entry 0 is the all-zero protocol, so an out of range index from a lambda transmits nothing if (index >= std::size(RC_SWITCH_PROTOCOLS)) index = 0; - progmem_memcpy(&protocol, &RC_SWITCH_PROTOCOLS[index], sizeof(protocol)); - return protocol; + return rc_switch_protocol_copy(&RC_SWITCH_PROTOCOLS[index]); +} + +RCSwitchBase rc_switch_protocol_copy(const RCSwitchBase *protocol) { + RCSwitchBase copy; + progmem_memcpy(©, protocol, sizeof(copy)); + return copy; } void RCSwitchBase::one(RemoteTransmitData *dst) const { @@ -232,7 +236,7 @@ uint64_t decode_binary_string_mask(const std::string &data) { bool RCSwitchRawReceiver::matches(RemoteReceiveData src) { uint64_t decoded_code; uint8_t decoded_nbits; - if (!this->protocol_.decode(src, &decoded_code, &decoded_nbits)) + if (!this->protocol_->decode(src, &decoded_code, &decoded_nbits)) return false; return decoded_nbits == this->nbits_ && (decoded_code & this->mask_) == (this->code_ & this->mask_); diff --git a/esphome/components/remote_base/rc_switch_protocol.h b/esphome/components/remote_base/rc_switch_protocol.h index 9ccea4d15a..a102da3ff6 100644 --- a/esphome/components/remote_base/rc_switch_protocol.h +++ b/esphome/components/remote_base/rc_switch_protocol.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "esphome/core/component.h" #include "remote_base.h" @@ -66,6 +68,9 @@ class RCSwitchBase { uint32_t one_high_{}; uint32_t one_low_{}; uint32_t inverted_{}; // bool widened so every field is a word: the table is read from flash + + // A bool here would still pad to 28 bytes, so the size check below alone would not catch it. + static_assert(std::is_same_v, "inverted_ must stay a word for flash reads"); }; // Constant-initialized and kept in flash on every platform. The decoder reads entries in place @@ -87,6 +92,9 @@ inline constexpr RCSwitchBase RC_SWITCH_PROTOCOLS[] PROGMEM = { /// RAM copy of RC_SWITCH_PROTOCOLS[index] (0 when out of range) for the transmit actions and the dumper, made with /// progmem_memcpy so no byte load ever touches the flash table on ESP8266 RCSwitchBase rc_switch_protocol(uint8_t index); +/// RAM copy of a protocol stored in flash, made with progmem_memcpy (own name: `rc_switch_protocol(0)` stays +/// unambiguous) +RCSwitchBase rc_switch_protocol_copy(const RCSwitchBase *protocol); uint64_t decode_binary_string(const std::string &data); @@ -200,7 +208,8 @@ template class RCSwitchTypeDAction : public RemoteTransmitterAct class RCSwitchRawReceiver : public RemoteReceiverBinarySensorBase { public: - void set_protocol(const RCSwitchBase &a_protocol) { this->protocol_ = a_protocol; } + /// `protocol` must outlive the receiver: a RC_SWITCH_PROTOCOLS entry or a codegen flash table. + void set_protocol(const RCSwitchBase *protocol) { this->protocol_ = protocol; } void set_code(uint64_t code) { this->code_ = code; } void set_code(const std::string &code) { this->code_ = decode_binary_string(code); @@ -228,7 +237,7 @@ class RCSwitchRawReceiver : public RemoteReceiverBinarySensorBase { protected: bool matches(RemoteReceiveData src) override; - RCSwitchBase protocol_; + const RCSwitchBase *protocol_{nullptr}; // in flash; decoded in place (word-only fields) uint64_t code_; uint64_t mask_{0xFFFFFFFFFFFFFFFF}; uint8_t nbits_; diff --git a/tests/component_tests/remote_receiver/config/rc_switch_protocol.yaml b/tests/component_tests/remote_receiver/config/rc_switch_protocol.yaml new file mode 100644 index 0000000000..d79080cedf --- /dev/null +++ b/tests/component_tests/remote_receiver/config/rc_switch_protocol.yaml @@ -0,0 +1,58 @@ +--- +esphome: + name: test + +esp8266: + board: esp01_1m + +remote_transmitter: + pin: GPIO4 + carrier_duty_percent: 100% + +remote_receiver: + pin: GPIO5 + +button: + - platform: template + name: Numbered + on_press: + - remote_transmitter.transmit_rc_switch_raw: + code: '1000100000000000' + protocol: 1 + - platform: template + name: Custom A + on_press: + - remote_transmitter.transmit_rc_switch_raw: + code: '1010101010101010' + protocol: + pulse_length: 320 + - platform: template + name: Custom B + on_press: + - remote_transmitter.transmit_rc_switch_type_b: + address: 4 + channel: 2 + state: true + protocol: + pulse_length: 320 + - platform: template + name: Lambda + on_press: + - remote_transmitter.transmit_rc_switch_raw: + code: '1010101010101011' + protocol: !lambda return remote_base::rc_switch_protocol(2); + +binary_sensor: + - platform: remote_receiver + name: Numbered RX + rc_switch_raw: + code: '1000100000000000' + protocol: 1 + - platform: remote_receiver + name: Custom RX + rc_switch_raw: + code: '1010101010101010' + protocol: + pulse_length: 320 + - platform: template + id: rc_switch_custom_protocol diff --git a/tests/component_tests/remote_receiver/test_rc_switch_protocol.py b/tests/component_tests/remote_receiver/test_rc_switch_protocol.py new file mode 100644 index 0000000000..5c0d14c794 --- /dev/null +++ b/tests/component_tests/remote_receiver/test_rc_switch_protocol.py @@ -0,0 +1,25 @@ +"""RC switch protocols are read from flash tables instead of being copied per entity.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_rc_switch_protocols_share_flash_tables( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("rc_switch_protocol.yaml")) + + # One shared table for the identical custom protocols; the user id keeps its name. + assert main_cpp.count("static constexpr remote_base::RCSwitchBase") == 1 + assert ( + "remote_base::RCSwitchBase rc_switch_custom_protocol_2[] PROGMEM = " + "{remote_base::RCSwitchBase(320, 9920, 320, 960, 960, 320, false)};" + ) in main_cpp + assert main_cpp.count("rc_switch_protocol_copy(rc_switch_custom_protocol_2)") == 2 + # Receivers point into flash instead of holding a copy. + assert "->set_protocol(&remote_base::RC_SWITCH_PROTOCOLS[1]);" in main_cpp + assert "->set_protocol(rc_switch_custom_protocol_2);" in main_cpp + # Numbered and lambda protocols keep their existing paths. + assert "return remote_base::rc_switch_protocol(1);" in main_cpp + assert "return remote_base::rc_switch_protocol(2);" in main_cpp From f334b9ea1106a289b44439dc1c139db24e4e21fa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:54:17 -0500 Subject: [PATCH 168/221] [core] Decode base64 in a single pass (#20123) --- esphome/core/helpers.cpp | 84 +++++++------------------- tests/components/core/test_helpers.cpp | 21 +++++++ 2 files changed, 43 insertions(+), 62 deletions(-) diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index 313daacbd0..a11b07808d 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -621,15 +620,11 @@ int8_t step_to_accuracy_decimals(float step) { return decimals; } -// Map a base64/base64url character to its 6-bit value (0-63) arithmetically. -// No lookup table: a table would occupy RAM on ESP8266 (.rodata lives in DRAM there). -// Supports both standard base64 (+/) and base64url (-_) alphabets. -// NOTE: This returns 0 for both 'A' (valid base64 char at index 0) and invalid characters. -// This is safe because is_base64() is ALWAYS checked before calling this function, -// preventing invalid characters from ever reaching here. The base64_decode function -// stops processing at the first invalid character due to the is_base64() check in its -// while loop condition, making this edge case harmless in practice. -static inline uint8_t base64_find_char(char c) { +static constexpr uint8_t INVALID_BASE64_CHAR = 0xFF; + +// 6-bit value of a base64 or base64url char, or INVALID_BASE64_CHAR. +// No lookup table: .rodata lives in DRAM on ESP8266. +static constexpr uint8_t base64_char_value(uint8_t c) { if (c >= 'A' && c <= 'Z') return c - 'A'; if (c >= 'a' && c <= 'z') @@ -641,70 +636,35 @@ static inline uint8_t base64_find_char(char c) { return 62; if (c == '/' || c == '_') return 63; - return 0; + return INVALID_BASE64_CHAR; } -// Check if character is valid base64 or base64url -static inline bool is_base64(char c) { return (isalnum(c) || (c == '+') || (c == '/') || (c == '-') || (c == '_')); } - // base64_encode (both overloads) moved to alloc_helpers.cpp size_t base64_decode(const std::string &encoded_string, uint8_t *buf, size_t buf_len) { return base64_decode(reinterpret_cast(encoded_string.data()), encoded_string.size(), buf, buf_len); } -// Decode 4 base64 characters to up to 'count' output bytes, returns true if truncated. -static inline bool base64_decode_quad(uint8_t *char_array_4, int count, uint8_t *buf, size_t buf_len, size_t &out) { - for (int i = 0; i < 4; i++) - char_array_4[i] = base64_find_char(char_array_4[i]); - - uint8_t char_array_3[3]; - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - bool truncated = false; - for (int j = 0; j < count; j++) { - if (out < buf_len) { - buf[out++] = char_array_3[j]; - } else { - truncated = true; - } - } - return truncated; -} - size_t base64_decode(const uint8_t *encoded_data, size_t encoded_len, uint8_t *buf, size_t buf_len) { - size_t in_len = encoded_len; - int i = 0; - size_t in = 0; size_t out = 0; - uint8_t char_array_4[4]; - bool truncated = false; - - // SAFETY: The loop condition checks is_base64() before processing each character. - // This ensures base64_find_char() is only called on valid base64 characters, - // preventing the edge case where invalid chars would return 0 (same as 'A'). - while (in_len-- && (encoded_data[in] != '=') && is_base64(encoded_data[in])) { - char_array_4[i++] = encoded_data[in]; - in++; - if (i == 4) { - truncated |= base64_decode_quad(char_array_4, 3, buf, buf_len, out); - i = 0; + uint32_t accum = 0; + uint32_t bits = 0; + // Stops at '=' or any non-alphabet char; leftover bits of a partial group are dropped. + for (size_t in = 0; in < encoded_len; in++) { + uint8_t value = base64_char_value(encoded_data[in]); + if (value == INVALID_BASE64_CHAR) + break; + accum = (accum << 6) | value; + bits += 6; + if (bits >= 8) { + bits -= 8; + if (out == buf_len) { + ESP_LOGW(TAG, "Base64 decode: buffer too small, truncating"); + return out; + } + buf[out++] = static_cast(accum >> bits); } } - - if (i) { - for (int j = i; j < 4; j++) - char_array_4[j] = 0; - - truncated |= base64_decode_quad(char_array_4, i - 1, buf, buf_len, out); - } - - if (truncated) { - ESP_LOGW(TAG, "Base64 decode: buffer too small, truncating"); - } - return out; } diff --git a/tests/components/core/test_helpers.cpp b/tests/components/core/test_helpers.cpp index 72af605d61..9fee0c723b 100644 --- a/tests/components/core/test_helpers.cpp +++ b/tests/components/core/test_helpers.cpp @@ -281,6 +281,27 @@ TEST(Base64, Rfc4648Vectors) { } } +TEST(Base64, DecodeTruncatesToBuffer) { + uint8_t buf[4]; + size_t len = base64_decode(std::string("Zm9vYmFy"), buf, sizeof(buf)); + EXPECT_EQ(len, 4u); + EXPECT_EQ(memcmp(buf, "foob", 4), 0); +} + +TEST(Base64, DecodeStopsAtNonAlphabetChar) { + uint8_t buf[8]; + EXPECT_EQ(base64_decode(std::string("Zm9v!Zm9v"), buf, sizeof(buf)), 3u); + EXPECT_EQ(memcmp(buf, "foo", 3), 0); + EXPECT_EQ(base64_decode(std::string("Zm9v Zm9v"), buf, sizeof(buf)), 3u); + EXPECT_EQ(base64_decode(std::string("Zm9v\xC3Zm9v"), buf, sizeof(buf)), 3u); +} + +TEST(Base64, DecodeDropsPartialGroup) { + uint8_t buf[8]; + EXPECT_EQ(base64_decode(std::string("Z"), buf, sizeof(buf)), 0u); + EXPECT_EQ(base64_decode(std::string("Zm9vY"), buf, sizeof(buf)), 3u); +} + // --- step_to_accuracy_decimals() --- TEST(StepToAccuracyDecimals, TypicalSteps) { From bb8c23805a23c2fa96b9de2c664a646b545bb60d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:55:11 -0500 Subject: [PATCH 169/221] [wifi] Connect LN882x to the BSSID asked for (#20174) --- esphome/components/wifi/__init__.py | 4 ++++ esphome/components/wifi/wifi_component_libretiny.cpp | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 553c737dbc..676aa0d294 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -734,6 +734,10 @@ async def to_code(config): if not config[CONF_ENABLE_ON_BOOT]: cg.add(var.set_enable_on_boot(False)) + # LN882x: hand the SDK the BSSID LibreTiny 1.13 drops (see wifi_component_libretiny.cpp); remove once fixed upstream. + if CORE.is_ln882x: + cg.add_build_flag("-Wl,--wrap=wifi_sta_connect") + # post_connect_roaming defaults to true in C++ - disable if user disabled it # or if 802.11k/v is enabled (driver handles roaming natively) if ( diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index 1d9fda6d2f..1d615e0a0d 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -31,6 +31,17 @@ extern "C" { #ifdef USE_LN882X #include #undef DATA // clashes with setup_priority::DATA +// LibreTiny 1.13's LN882H reconnect() gives the SDK a NULL bssid; pass on the requested one. Remove once it passes +// info.bssid. Safe without a BSSID: begin() frees sta.bssid (resetNetworkInfo), so it stays NULL. +// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) +// Names are mandated by the --wrap linker mechanism. +extern "C" int __real_wifi_sta_connect(wifi_sta_connect_t *connect, wifi_scan_cfg_t *scan_cfg); +extern "C" int __wrap_wifi_sta_connect(wifi_sta_connect_t *connect, wifi_scan_cfg_t *scan_cfg) { + if (connect->bssid == nullptr) + connect->bssid = static_cast(WiFi.data)->sta.bssid; + return __real_wifi_sta_connect(connect, scan_cfg); +} +// NOLINTEND(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) #endif #include "esphome/core/application.h" From c8eaf518baadc9a15843cacc793bc0dab24edcab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:56:02 -0500 Subject: [PATCH 170/221] [libretiny] Flag the LN882H SDK default MAC (#20175) --- esphome/components/libretiny/lt_component.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/esphome/components/libretiny/lt_component.cpp b/esphome/components/libretiny/lt_component.cpp index 0ab064e3e1..d7d93249c5 100644 --- a/esphome/components/libretiny/lt_component.cpp +++ b/esphome/components/libretiny/lt_component.cpp @@ -2,6 +2,9 @@ #ifdef USE_LIBRETINY +#include + +#include "esphome/core/helpers.h" #include "esphome/core/log.h" namespace esphome::libretiny { @@ -18,6 +21,17 @@ void LTComponent::dump_config() { ESP_LOGCONFIG(TAG, " Optimization: -Os, SDK: -O" STRINGIFY_MACRO(__OPTIMIZE_LEVEL__)); #endif +#if defined(USE_LN882X) && defined(USE_WIFI) + // The SDK falls back to this MAC when the board layout's KV store holds none. + static constexpr uint8_t SDK_DEFAULT_MAC[MAC_ADDRESS_SIZE] = {0x00, 0x50, 0xC2, 0x5E, 0x10, 0x88}; + uint8_t mac[MAC_ADDRESS_SIZE]; + get_mac_address_raw(mac); + if (memcmp(mac, SDK_DEFAULT_MAC, MAC_ADDRESS_SIZE) == 0) { + ESP_LOGE(TAG, "MAC is the LN882H SDK default 00:50:C2:5E:10:88; only one device with it can be on a network. " + "On a Tuya module, set board: to its Tuya board (e.g. wl2s, wl2h-u) and flash over serial"); + } +#endif + #ifdef USE_TEXT_SENSOR if (this->version_ != nullptr) { this->version_->publish_state(<_BANNER_STR[10]); From 8735aee847cda3c0cde1b688d46d6e656c0ac662 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 10:56:24 -0500 Subject: [PATCH 171/221] [wifi] Leave the AP before an LN882x reboots (#20176) --- esphome/components/wifi/wifi_component.h | 3 +++ esphome/components/wifi/wifi_component_libretiny.cpp | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 082c059a14..9418165501 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -506,6 +506,9 @@ class WiFiComponent final : public Component { // (In most use cases you won't need these) /// Setup WiFi interface. void setup() override; +#ifdef USE_LN882X + void on_powerdown() override; +#endif void start(); void dump_config() override; void restart_adapter(); diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index 1d615e0a0d..1f43f37c0e 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -800,6 +800,17 @@ bool WiFiComponent::wifi_start_ap_(const WiFiAP &ap) { network::IPAddress WiFiComponent::wifi_soft_ap_ip() { return {WiFi.softAPIP()}; } #endif // USE_WIFI_AP +#ifdef USE_LN882X +void WiFiComponent::on_powerdown() { + // Leave the AP so it no longer holds the association when we come back. + if (WiFi.status() != WL_CONNECTED) // associated, even before DHCP has finished + return; + ESP_LOGD(TAG, "Disconnecting before powerdown"); + wifi_sta_disconnect(); // not WiFi.disconnect(): it frees the SSID its event handlers read + delay(100); // NOLINT: once per reboot or deep sleep, lets the deauth go out +} +#endif + bool WiFiComponent::wifi_disconnect_() { // Reset state first so disconnect events aren't ignored // and wifi_sta_connect_status_() returns IDLE instead of CONNECTING From 4a47a984a0f6bd240ad451974ec96be469ad05b2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 11:41:33 -0500 Subject: [PATCH 172/221] [audio_file] Keep audio files in flash instead of building them in setup() (#20072) --- esphome/components/audio/audio_reader.cpp | 2 +- esphome/components/audio/audio_reader.h | 4 +- esphome/components/audio_file/__init__.py | 41 ++++++++++++++----- esphome/components/audio_file/audio_file.h | 4 +- .../media_source/audio_file_media_source.h | 2 +- .../speaker/media_player/__init__.py | 2 +- .../speaker/media_player/audio_pipeline.cpp | 2 +- .../speaker/media_player/audio_pipeline.h | 4 +- .../media_player/speaker_media_player.cpp | 2 +- .../media_player/speaker_media_player.h | 6 +-- tests/component_tests/audio_file/__init__.py | 0 .../audio_file/config/audio_file.yaml | 14 +++++++ .../audio_file/test_audio_file.py | 22 ++++++++++ 13 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 tests/component_tests/audio_file/__init__.py create mode 100644 tests/component_tests/audio_file/config/audio_file.yaml create mode 100644 tests/component_tests/audio_file/test_audio_file.py diff --git a/esphome/components/audio/audio_reader.cpp b/esphome/components/audio/audio_reader.cpp index e69f33ac2d..9f2960620e 100644 --- a/esphome/components/audio/audio_reader.cpp +++ b/esphome/components/audio/audio_reader.cpp @@ -72,7 +72,7 @@ esp_err_t AudioReader::add_sink(const std::weak_ptr &ou return ESP_ERR_INVALID_STATE; } -esp_err_t AudioReader::start(AudioFile *audio_file, AudioFileType &file_type) { +esp_err_t AudioReader::start(const AudioFile *audio_file, AudioFileType &file_type) { file_type = AudioFileType::NONE; this->current_audio_file_ = audio_file; diff --git a/esphome/components/audio/audio_reader.h b/esphome/components/audio/audio_reader.h index b1f76172b0..db85185ca6 100644 --- a/esphome/components/audio/audio_reader.h +++ b/esphome/components/audio/audio_reader.h @@ -47,7 +47,7 @@ class AudioReader { /// @param audio_file AudioFile struct containing the file. /// @param file_type AudioFileType variable passed-by-reference indicating the type of file being read. /// @return ESP_OK - esp_err_t start(AudioFile *audio_file, AudioFileType &file_type); + esp_err_t start(const AudioFile *audio_file, AudioFileType &file_type); /// @brief Reads new file data from the source and sends to the ring buffer sink. /// @return AudioReaderState @@ -69,7 +69,7 @@ class AudioReader { esp_http_client_handle_t client_{nullptr}; - AudioFile *current_audio_file_{nullptr}; + const AudioFile *current_audio_file_{nullptr}; AudioFileType audio_file_type_{AudioFileType::NONE}; const uint8_t *file_current_{nullptr}; }; diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py index d59ed7411a..7726ef5837 100644 --- a/esphome/components/audio_file/__init__.py +++ b/esphome/components/audio_file/__init__.py @@ -19,7 +19,7 @@ from esphome.const import ( CONF_URL, ) from esphome.core import CORE, ID, HexInt -from esphome.cpp_generator import MockObj +from esphome.cpp_generator import MockObj, ProgmemAssignmentExpression from esphome.external_files import download_web_files_in_config from esphome.types import ConfigType @@ -151,11 +151,14 @@ TYPED_FILE_SCHEMA = cv.typed_schema( ) +CONF_FILE_DATA_ID = "file_data_id" + MEDIA_FILE_TYPE_SCHEMA = cv.Schema( { cv.Required(CONF_ID): cv.declare_id(audio.AudioFile), cv.Required(CONF_FILE): _file_schema, cv.GenerateID(CONF_RAW_DATA_ID): cv.declare_id(cg.uint8), + cv.GenerateID(CONF_FILE_DATA_ID): cv.declare_id(audio.AudioFile), } ) @@ -217,9 +220,9 @@ def audio_files_schema() -> cv.All: def generate_audio_file_code(file_config: ConfigType) -> MockObj: - """Generate the progmem data, AudioFile struct, and Pvariable for one file. + """Generate the progmem data and a flash AudioFile for one file. - Returns the created Pvariable. Caller is responsible for any further + Returns a const pointer to the AudioFile. Caller is responsible for any further registration (the audio_file component additionally registers each file in its named C++ registry; other consumers may skip that). """ @@ -230,17 +233,33 @@ def generate_audio_file_code(file_config: ConfigType) -> MockObj: else: data, media_file_type = read_audio_file_and_type(file_config) - rhs = [HexInt(x) for x in data] - prog_arr = cg.progmem_array(file_config[CONF_RAW_DATA_ID], rhs) - - media_files_struct = cg.StructInitializer( + # Global constants so the AudioFile lives in flash; the id stays a plain pointer + # because actions render id arguments as ``::``. + data_id = file_config[CONF_RAW_DATA_ID] + cg.add_global( + ProgmemAssignmentExpression( + data_id.type, data_id, cg.safe_exp([HexInt(x) for x in data]) + ) + ) + media_file = cg.StructInitializer( audio.AudioFile, - ("data", prog_arr), - ("length", len(rhs)), + ("data", MockObj(data_id, ".")), + ("length", len(data)), ("file_type", media_file_type), ) - - return cg.new_Pvariable(file_config[CONF_ID], media_files_struct) + file_var_id = file_config[CONF_ID] + storage = file_config[CONF_FILE_DATA_ID] + cg.add_global( + cg.RawStatement(f"static constexpr {storage.type} {storage} = {media_file};") + ) + cg.add_global( + cg.RawStatement( + f"static const {storage.type} *const {file_var_id} = &{storage};" + ) + ) + var = MockObj(file_var_id, "->") + CORE.register_variable(file_var_id, var) + return var CONFIG_SCHEMA = cv.All( diff --git a/esphome/components/audio_file/audio_file.h b/esphome/components/audio_file/audio_file.h index 537e19fb3c..177374cf91 100644 --- a/esphome/components/audio_file/audio_file.h +++ b/esphome/components/audio_file/audio_file.h @@ -10,14 +10,14 @@ namespace esphome::audio_file { struct NamedAudioFile { - audio::AudioFile *file; + const audio::AudioFile *file; const char *file_id; }; inline StaticVector named_audio_files; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -inline void add_named_audio_file(audio::AudioFile *file, const char *file_id) { +inline void add_named_audio_file(const audio::AudioFile *file, const char *file_id) { named_audio_files.push_back({file, file_id}); } diff --git a/esphome/components/audio_file/media_source/audio_file_media_source.h b/esphome/components/audio_file/media_source/audio_file_media_source.h index d269f77c35..12735eaca1 100644 --- a/esphome/components/audio_file/media_source/audio_file_media_source.h +++ b/esphome/components/audio_file/media_source/audio_file_media_source.h @@ -46,7 +46,7 @@ class AudioFileMediaSource final : public Component, protected: std::unique_ptr decoder_; audio::AudioStreamInfo stream_info_; - audio::AudioFile *current_file_{nullptr}; + const audio::AudioFile *current_file_{nullptr}; // Written from the main loop in handle_command(), read from the decoder task in // on_audio_write(). Must be atomic to avoid a data race. diff --git a/esphome/components/speaker/media_player/__init__.py b/esphome/components/speaker/media_player/__init__.py index 78892bff22..159af15808 100644 --- a/esphome/components/speaker/media_player/__init__.py +++ b/esphome/components/speaker/media_player/__init__.py @@ -251,7 +251,7 @@ automation.register_apply_action( # so a trigger arg with the same name cannot shadow it. ( CONF_MEDIA_FILE, - audio.AudioFile.operator("ptr"), + audio.AudioFile.operator("const").operator("ptr"), lambda _, value: f"::{value}", ), (CONF_ANNOUNCEMENT, cg.bool_), diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index 509984cfa2..f150a48573 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -66,7 +66,7 @@ void AudioPipeline::start_url(const std::string &uri) { this->pending_url_ = true; } -void AudioPipeline::start_file(audio::AudioFile *audio_file) { +void AudioPipeline::start_file(const audio::AudioFile *audio_file) { if (this->is_playing_) { xEventGroupSetBits(this->event_group_, PIPELINE_COMMAND_STOP); } diff --git a/esphome/components/speaker/media_player/audio_pipeline.h b/esphome/components/speaker/media_player/audio_pipeline.h index 02dad15de9..b114a7b7a5 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.h +++ b/esphome/components/speaker/media_player/audio_pipeline.h @@ -74,7 +74,7 @@ class AudioPipeline final { /// @brief Starts an audio pipeline given a AudioFile pointer /// @param audio_file pointer to an AudioFile object /// @return ESP_OK if successful or an appropriate error if not - void start_file(audio::AudioFile *audio_file); + void start_file(const audio::AudioFile *audio_file); /// @brief Stops the pipeline. Sends a stop signal to each task (if running) and clears the ring buffers. /// @return ESP_OK if successful or ESP_ERR_TIMEOUT if the tasks did not indicate they stopped @@ -121,7 +121,7 @@ class AudioPipeline final { speaker::Speaker *speaker_{nullptr}; std::string current_uri_{}; - audio::AudioFile *current_audio_file_{nullptr}; + const audio::AudioFile *current_audio_file_{nullptr}; audio::AudioFileType current_audio_file_type_; audio::AudioStreamInfo current_audio_stream_info_; diff --git a/esphome/components/speaker/media_player/speaker_media_player.cpp b/esphome/components/speaker/media_player/speaker_media_player.cpp index cac2f6649c..ab98935bda 100644 --- a/esphome/components/speaker/media_player/speaker_media_player.cpp +++ b/esphome/components/speaker/media_player/speaker_media_player.cpp @@ -493,7 +493,7 @@ void SpeakerMediaPlayer::loop() { #endif } -void SpeakerMediaPlayer::play_file(audio::AudioFile *media_file, bool announcement, bool enqueue) { +void SpeakerMediaPlayer::play_file(const audio::AudioFile *media_file, bool announcement, bool enqueue) { if (!this->is_ready()) { // Ignore any commands sent before the media player is setup return; diff --git a/esphome/components/speaker/media_player/speaker_media_player.h b/esphome/components/speaker/media_player/speaker_media_player.h index 889aedbc9a..e98196785c 100644 --- a/esphome/components/speaker/media_player/speaker_media_player.h +++ b/esphome/components/speaker/media_player/speaker_media_player.h @@ -28,13 +28,13 @@ struct MediaCallCommand { optional volume; optional announce; optional url; // Must be manually deleted after receiving this struct from a queue - optional file; + optional file; optional enqueue; }; struct PlaylistItem { optional url; - optional file; + optional file; }; struct VolumeRestoreState { @@ -87,7 +87,7 @@ class SpeakerMediaPlayer final : public Component, Trigger<> *get_unmute_trigger() { return &this->unmute_trigger_; } Trigger *get_volume_trigger() { return &this->volume_trigger_; } - void play_file(audio::AudioFile *media_file, bool announcement, bool enqueue); + void play_file(const audio::AudioFile *media_file, bool announcement, bool enqueue); void set_playlist_delay_ms(AudioPipelineType pipeline_type, uint32_t delay_ms); diff --git a/tests/component_tests/audio_file/__init__.py b/tests/component_tests/audio_file/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/audio_file/config/audio_file.yaml b/tests/component_tests/audio_file/config/audio_file.yaml new file mode 100644 index 0000000000..a25feedc36 --- /dev/null +++ b/tests/component_tests/audio_file/config/audio_file.yaml @@ -0,0 +1,14 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +audio_file: + - id: chime + file: + type: local + path: ../../../components/audio_file/test.wav diff --git a/tests/component_tests/audio_file/test_audio_file.py b/tests/component_tests/audio_file/test_audio_file.py new file mode 100644 index 0000000000..32323c2816 --- /dev/null +++ b/tests/component_tests/audio_file/test_audio_file.py @@ -0,0 +1,22 @@ +"""Tests for the audio_file component codegen.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_audio_file_is_constant_in_flash( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The AudioFile is a global constant behind a const pointer, not a placement new.""" + main_cpp = generate_main(component_config_path("audio_file.yaml")) + + assert ( + "static constexpr audio::AudioFile audio_audiofile_id = audio::AudioFile{" + in main_cpp + ) + assert ( + "static const audio::AudioFile *const chime = &audio_audiofile_id;" in main_cpp + ) + assert "new(chime)" not in main_cpp + assert 'audio_file::add_named_audio_file(chime, "chime");' in main_cpp From fc441646f459b73fa1d9e5764d3ff55833c01485 Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Mon, 5 Oct 2026 18:40:06 +0100 Subject: [PATCH 173/221] [unicode][font] move extract_unicode_codepoint to new component (#19969) --- CODEOWNERS | 1 + esphome/components/font/__init__.py | 1 + esphome/components/font/font.cpp | 107 +------------------------ esphome/components/unicode/__init__.py | 1 + esphome/components/unicode/unicode.cpp | 97 ++++++++++++++++++++++ esphome/components/unicode/unicode.h | 20 +++++ 6 files changed, 123 insertions(+), 104 deletions(-) create mode 100644 esphome/components/unicode/__init__.py create mode 100644 esphome/components/unicode/unicode.cpp create mode 100644 esphome/components/unicode/unicode.h diff --git a/CODEOWNERS b/CODEOWNERS index 80f243e82e..43034ce308 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -609,6 +609,7 @@ esphome/components/ufire_ec/* @pvizeli esphome/components/ufire_ise/* @pvizeli esphome/components/ufm01/* @ljungqvist esphome/components/ultrasonic/* @ssieb @swoboda1337 +esphome/components/unicode/* @esphome/core esphome/components/update/* @jesserockz esphome/components/uponor_smatrix/* @kroimon esphome/components/usb_cdc_acm/* @kbx81 diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index a2fad4bc25..732933ba48 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -41,6 +41,7 @@ _LOGGER = logging.getLogger(__name__) DOMAIN = "font" MULTI_CONF = True +AUTO_LOAD = ["unicode"] CODEOWNERS = ["@esphome/core", "@clydebarrow"] diff --git a/esphome/components/font/font.cpp b/esphome/components/font/font.cpp index 1ed946dadb..2592370b6a 100644 --- a/esphome/components/font/font.cpp +++ b/esphome/components/font/font.cpp @@ -3,6 +3,7 @@ #include "esphome/core/color.h" #include "esphome/core/hal.h" #include "esphome/core/log.h" +#include "esphome/components/unicode/unicode.h" namespace esphome::font { static const char *const TAG = "font"; @@ -121,108 +122,6 @@ const Glyph *Font::get_glyph_data_(uint32_t unicode_letter) { } #endif -/** - * Attempt to extract a 32 bit Unicode codepoint from a UTF-8 string. - * If successful, return the codepoint and set the length to the number of bytes read. - * If the end of the string has been reached and a valid codepoint has not been found, return 0 and set the length to - * 0. - * - * @param utf8_str The input string - * @param length Pointer to length storage - * @return The extracted code point - */ -static uint32_t extract_unicode_codepoint(const char *utf8_str, size_t *length) { - // Safely cast to uint8_t* for correct bitwise operations on bytes - const uint8_t *current = reinterpret_cast(utf8_str); - uint32_t code_point = 0; - uint8_t c1 = *current++; - - // check for end of string - if (c1 == 0) { - *length = 0; - return 0; - } - - // --- 1-Byte Sequence: 0xxxxxxx (ASCII) --- - if (c1 < 0x80) { - // Valid ASCII byte. - code_point = c1; - // Optimization: No need to check for continuation bytes. - } - // --- 2-Byte Sequence: 110xxxxx 10xxxxxx --- - else if ((c1 & 0xE0) == 0xC0) { - uint8_t c2 = *current++; - - // Error Check 1: Check if c2 is a valid continuation byte (10xxxxxx) - if ((c2 & 0xC0) != 0x80) { - *length = 0; - return 0; - } - - code_point = (c1 & 0x1F) << 6; - code_point |= (c2 & 0x3F); - - // Error Check 2: Overlong check (2-byte must be > 0x7F) - if (code_point <= 0x7F) { - *length = 0; - return 0; - } - } - // --- 3-Byte Sequence: 1110xxxx 10xxxxxx 10xxxxxx --- - else if ((c1 & 0xF0) == 0xE0) { - uint8_t c2 = *current++; - uint8_t c3 = *current++; - - // Error Check 1: Check continuation bytes - if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80)) { - *length = 0; - return 0; - } - - code_point = (c1 & 0x0F) << 12; - code_point |= (c2 & 0x3F) << 6; - code_point |= (c3 & 0x3F); - - // Error Check 2: Overlong check (3-byte must be > 0x7FF) - // Also check for surrogates (0xD800-0xDFFF) - if (code_point <= 0x7FF || (code_point >= 0xD800 && code_point <= 0xDFFF)) { - *length = 0; - return 0; - } - } - // --- 4-Byte Sequence: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx --- - else if ((c1 & 0xF8) == 0xF0) { - uint8_t c2 = *current++; - uint8_t c3 = *current++; - uint8_t c4 = *current++; - - // Error Check 1: Check continuation bytes - if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80) || ((c4 & 0xC0) != 0x80)) { - *length = 0; - return 0; - } - - code_point = (c1 & 0x07) << 18; - code_point |= (c2 & 0x3F) << 12; - code_point |= (c3 & 0x3F) << 6; - code_point |= (c4 & 0x3F); - - // Error Check 2: Overlong check (4-byte must be > 0xFFFF) - // Also check for valid Unicode range (must be <= 0x10FFFF) - if (code_point <= 0xFFFF || code_point > 0x10FFFF) { - *length = 0; - return 0; - } - } - // --- Invalid leading byte (e.g., 10xxxxxx or 11111xxx) --- - else { - *length = 0; - return 0; - } - *length = current - reinterpret_cast(utf8_str); - return code_point; -} - Font::Font(const Glyph *data, int data_nr, int baseline, int height, int descender, int xheight, int capheight, uint8_t bpp) : glyphs_(ConstVector(data, data_nr)), @@ -271,7 +170,7 @@ void Font::measure(const char *str, int *width, int *x_offset, int *baseline, in int x = 0; for (;;) { size_t length; - auto code_point = extract_unicode_codepoint(str, &length); + auto code_point = unicode::extract_unicode_codepoint(str, &length); if (length == 0) break; str += length; @@ -300,7 +199,7 @@ void Font::print(int x_start, int y_start, display::Display *display, Color colo int x_at = x_start; for (;;) { size_t length; - auto code_point = extract_unicode_codepoint(text, &length); + auto code_point = unicode::extract_unicode_codepoint(text, &length); if (length == 0) break; text += length; diff --git a/esphome/components/unicode/__init__.py b/esphome/components/unicode/__init__.py new file mode 100644 index 0000000000..f70ffa9520 --- /dev/null +++ b/esphome/components/unicode/__init__.py @@ -0,0 +1 @@ +CODEOWNERS = ["@esphome/core"] diff --git a/esphome/components/unicode/unicode.cpp b/esphome/components/unicode/unicode.cpp new file mode 100644 index 0000000000..f5b4084fe0 --- /dev/null +++ b/esphome/components/unicode/unicode.cpp @@ -0,0 +1,97 @@ +#include "unicode.h" + +namespace esphome::unicode { + +uint32_t extract_unicode_codepoint(const char *utf8_str, size_t *length) { + // Safely cast to uint8_t* for correct bitwise operations on bytes + const uint8_t *current = reinterpret_cast(utf8_str); + uint32_t code_point = 0; + uint8_t c1 = *current++; + + // check for end of string + if (c1 == 0) { + *length = 0; + return 0; + } + + // --- 1-Byte Sequence: 0xxxxxxx (ASCII) --- + if (c1 < 0x80) { + // Valid ASCII byte. + code_point = c1; + // Optimization: No need to check for continuation bytes. + } + // --- 2-Byte Sequence: 110xxxxx 10xxxxxx --- + else if ((c1 & 0xE0) == 0xC0) { + uint8_t c2 = *current++; + + // Error Check 1: Check if c2 is a valid continuation byte (10xxxxxx) + if ((c2 & 0xC0) != 0x80) { + *length = 0; + return 0; + } + + code_point = (c1 & 0x1F) << 6; + code_point |= (c2 & 0x3F); + + // Error Check 2: Overlong check (2-byte must be > 0x7F) + if (code_point <= 0x7F) { + *length = 0; + return 0; + } + } + // --- 3-Byte Sequence: 1110xxxx 10xxxxxx 10xxxxxx --- + else if ((c1 & 0xF0) == 0xE0) { + uint8_t c2 = *current++; + uint8_t c3 = *current++; + + // Error Check 1: Check continuation bytes + if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80)) { + *length = 0; + return 0; + } + + code_point = (c1 & 0x0F) << 12; + code_point |= (c2 & 0x3F) << 6; + code_point |= (c3 & 0x3F); + + // Error Check 2: Overlong check (3-byte must be > 0x7FF) + // Also check for surrogates (0xD800-0xDFFF) + if (code_point <= 0x7FF || (code_point >= 0xD800 && code_point <= 0xDFFF)) { + *length = 0; + return 0; + } + } + // --- 4-Byte Sequence: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx --- + else if ((c1 & 0xF8) == 0xF0) { + uint8_t c2 = *current++; + uint8_t c3 = *current++; + uint8_t c4 = *current++; + + // Error Check 1: Check continuation bytes + if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80) || ((c4 & 0xC0) != 0x80)) { + *length = 0; + return 0; + } + + code_point = (c1 & 0x07) << 18; + code_point |= (c2 & 0x3F) << 12; + code_point |= (c3 & 0x3F) << 6; + code_point |= (c4 & 0x3F); + + // Error Check 2: Overlong check (4-byte must be > 0xFFFF) + // Also check for valid Unicode range (must be <= 0x10FFFF) + if (code_point <= 0xFFFF || code_point > 0x10FFFF) { + *length = 0; + return 0; + } + } + // --- Invalid leading byte (e.g., 10xxxxxx or 11111xxx) --- + else { + *length = 0; + return 0; + } + *length = current - reinterpret_cast(utf8_str); + return code_point; +} + +} // namespace esphome::unicode diff --git a/esphome/components/unicode/unicode.h b/esphome/components/unicode/unicode.h new file mode 100644 index 0000000000..9e69d036e0 --- /dev/null +++ b/esphome/components/unicode/unicode.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace esphome::unicode { + +/** + * Attempt to extract a 32 bit Unicode codepoint from a UTF-8 string. + * If successful, return the codepoint and set the length to the number of bytes read. + * If the end of the string has been reached and a valid codepoint has not been found, return 0 and set the length to + * 0. + * + * @param utf8_str The input string + * @param length Pointer to length storage + * @return The extracted code point + */ +uint32_t extract_unicode_codepoint(const char *utf8_str, size_t *length); + +} // namespace esphome::unicode From a778baddca233ab8e6fdd4b687c982bb96a53adf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 16:56:10 -0500 Subject: [PATCH 174/221] [usb_uart] Only compile the vendor drivers that are configured (#20190) --- esphome/components/usb_uart/__init__.py | 21 +++++++++++++++++++ esphome/core/defines.h | 4 ++++ tests/component_tests/usb_uart/__init__.py | 0 .../usb_uart/test_source_filter.py | 19 +++++++++++++++++ .../usb_uart/test_usb_uart_ft232.yaml | 18 ++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 tests/component_tests/usb_uart/__init__.py create mode 100644 tests/component_tests/usb_uart/test_source_filter.py create mode 100644 tests/component_tests/usb_uart/test_usb_uart_ft232.yaml diff --git a/esphome/components/usb_uart/__init__.py b/esphome/components/usb_uart/__init__.py index 5d0f8be165..af58c24d43 100644 --- a/esphome/components/usb_uart/__init__.py +++ b/esphome/components/usb_uart/__init__.py @@ -7,6 +7,7 @@ from esphome.components.usb_host import ( register_usb_client, usb_device_schema, ) +from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv from esphome.const import ( CONF_BAUD_RATE, @@ -15,6 +16,7 @@ from esphome.const import ( CONF_DEBUG, CONF_DUMMY_RECEIVER, CONF_ID, + CONF_TYPE, ) from esphome.core import CORE from esphome.cpp_types import Component @@ -64,6 +66,8 @@ class Type: self.vid = vid self.pid = pid self.cls = usb_uart_ns.class_(f"USBUartType{cls}", USBUartComponent) + # CDC ACM lives in usb_uart.cpp; each vendor driver has its own .cpp + self.driver = None if cls == "CdcAcm" else cls self._max_channels = max_channels self.baud_rate_required = baud_rate_required self.max_baud = max_baud @@ -121,6 +125,21 @@ uart_types = ( ), ) +_TYPES_BY_NAME = {it.name: it for it in uart_types} + + +def _driver_define(driver: str) -> str: + return f"USE_USB_UART_{driver}" + + +FILTER_SOURCE_FILES = filter_source_files_from_defines( + { + f"{it.driver.lower()}.cpp": _driver_define(it.driver) + for it in uart_types + if it.driver is not None + } +) + def channel_schema(type_: "Type") -> cv.Schema: schema = cv.Schema( @@ -211,6 +230,8 @@ async def to_code(config: list[ConfigType]) -> None: cg.add_define("USB_UART_OUTPUT_CHUNK_COUNT", output_chunk_count) for device in config: + if (driver := _TYPES_BY_NAME[device[CONF_TYPE]].driver) is not None: + cg.add_define(_driver_define(driver)) var = await register_usb_client(device) # The C++ default is true; only emit the override if not device.get(CONF_CLAIM_COMM_INTERFACE, True): diff --git a/esphome/core/defines.h b/esphome/core/defines.h index be8d89c557..f701cd2919 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -459,6 +459,10 @@ #define USB_HOST_MAX_REQUESTS 16 #define USB_HOST_MAX_PACKET_SIZE 64 #define USB_UART_OUTPUT_CHUNK_COUNT 5 +#define USE_USB_UART_CH34X +#define USE_USB_UART_CP210X +#define USE_USB_UART_FT23XX +#define USE_USB_UART_PL2303 #ifdef USE_ARDUINO #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 3, 7) diff --git a/tests/component_tests/usb_uart/__init__.py b/tests/component_tests/usb_uart/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/usb_uart/test_source_filter.py b/tests/component_tests/usb_uart/test_source_filter.py new file mode 100644 index 0000000000..414640f821 --- /dev/null +++ b/tests/component_tests/usb_uart/test_source_filter.py @@ -0,0 +1,19 @@ +"""Tests for the usb_uart component's driver source filtering.""" + +from collections.abc import Callable + +from esphome.core import CORE + + +def test_only_configured_driver_compiled( + generate_main: Callable[[str], str], +) -> None: + generate_main("tests/component_tests/usb_uart/test_usb_uart_ft232.yaml") + from esphome.components.usb_uart import FILTER_SOURCE_FILES + + defines = {define.name for define in CORE.defines} + # cdc_acm is built into usb_uart.cpp and adds no driver define + assert {d for d in defines if d.startswith("USE_USB_UART_")} == { + "USE_USB_UART_FT23XX" + } + assert sorted(FILTER_SOURCE_FILES()) == ["ch34x.cpp", "cp210x.cpp", "pl2303.cpp"] diff --git a/tests/component_tests/usb_uart/test_usb_uart_ft232.yaml b/tests/component_tests/usb_uart/test_usb_uart_ft232.yaml new file mode 100644 index 0000000000..e68a6246bf --- /dev/null +++ b/tests/component_tests/usb_uart/test_usb_uart_ft232.yaml @@ -0,0 +1,18 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + +usb_uart: + - type: ft232 + channels: + - id: ft_channel + baud_rate: 115200 + - type: cdc_acm + vid: 0x1234 + pid: 0x5678 + channels: + - id: cdc_channel From 7af96c8d360ef51b94051f3a48730bcbb43a8542 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:07:10 +0000 Subject: [PATCH 175/221] Bump CodSpeedHQ/action from 5.2.1 to 5.4.0 (#20197) Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ebbe52d20..5eb96af59b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -626,7 +626,7 @@ jobs: apt-get install -y libc6-dbg - name: Run CodSpeed benchmarks - uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 + uses: CodSpeedHQ/action@c4fd08a3a159bd0cc208da1e0edf32b8c47d75e5 # v5.4.0 with: run: | . venv/bin/activate From 888043762ee2bd183f26529657a68cdfbcc5afc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:07:33 +0000 Subject: [PATCH 176/221] Bump filelock from 4.0.6 to 4.0.9 (#20195) Co-authored-by: J. Nick Koston Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1f21fd8160..ce090503e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.12.2 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver -filelock==4.0.6 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==4.0.9 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.3 From cf334e875853ee7f48f6c94706e2f3bc3d5ff051 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:07:37 +0000 Subject: [PATCH 177/221] Bump pylint from 4.0.10 to 4.1.1 (#20194) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index fa56852637..b0ebf3a409 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,4 @@ -pylint==4.0.10 +pylint==4.1.1 flake8==7.4.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.9 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py From 9af9414dcc4370b48b280b4b485bc72c70f4e51a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:07:43 +0000 Subject: [PATCH 178/221] Bump cryptography from 48.0.1 to 50.0.2 (#20193) Co-authored-by: J. Nick Koston Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ce090503e0..744c5c1e66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # cryptography 49+ ships no Intel macOS wheels (arm64 only); esptool caps <49 there. # Keep 48.0.1, the last universal2 release, so esphome stays installable on Intel Macs. -cryptography==50.0.1; platform_system != "Darwin" or platform_machine != "x86_64" +cryptography==50.0.2; platform_system != "Darwin" or platform_machine != "x86_64" cryptography==48.0.1; platform_system == "Darwin" and platform_machine == "x86_64" voluptuous==0.16.0 PyYAML==6.0.3 From cdcc7bbf744d237f87357d479480ccbe6770e770 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:07:49 +0000 Subject: [PATCH 179/221] Bump ruff from 0.16.9 to 0.16.10 (#20196) Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48c0d1d6fd..727e8e2f9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.9 + rev: v0.16.10 hooks: # Run the linter. - id: ruff diff --git a/requirements_test.txt b/requirements_test.txt index b0ebf3a409..e6585f246e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.1.1 flake8==7.4.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -ruff==0.16.9 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.10 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py prek==0.5.4 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py From 039f07d3320a019b86ab05ea9286535529d8fe53 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:10:00 +0000 Subject: [PATCH 180/221] Bump bundled esphome-device-builder to 1.21.0 (#20199) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 50cb8e137d..9f29062e09 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.20.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.21.0 RUN \ platformio settings set enable_telemetry No \ From 4097f45677ba79de580344982af1d3a23f18dd59 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 19:23:39 -0500 Subject: [PATCH 181/221] [sx1509] Keep the keypad keys in flash (#20111) --- esphome/components/sx1509/__init__.py | 11 ++++++-- esphome/components/sx1509/sx1509.cpp | 6 +++-- esphome/components/sx1509/sx1509.h | 5 ++-- tests/component_tests/sx1509/__init__.py | 0 tests/component_tests/sx1509/config/keys.yaml | 23 ++++++++++++++++ tests/component_tests/sx1509/test_keys.py | 27 +++++++++++++++++++ 6 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 tests/component_tests/sx1509/__init__.py create mode 100644 tests/component_tests/sx1509/config/keys.yaml create mode 100644 tests/component_tests/sx1509/test_keys.py diff --git a/esphome/components/sx1509/__init__.py b/esphome/components/sx1509/__init__.py index 7694b8f732..e4df7b443c 100644 --- a/esphome/components/sx1509/__init__.py +++ b/esphome/components/sx1509/__init__.py @@ -43,6 +43,10 @@ SX1509KeyTrigger = sx1509_ns.class_( def check_keys(config: ConfigType) -> ConfigType: + for ch in config.get(CONF_KEYS, ""): + if not ch.isascii(): + # Each key is reported as one byte, so only ASCII characters can be key codes + raise cv.Invalid(f"Key code {ch!r} is not an ASCII character") if ( CONF_KEYS in config and len(config[CONF_KEYS]) != config[CONF_KEY_ROWS] * config[CONF_KEY_COLUMNS] @@ -98,8 +102,11 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_sleep_time(conf[CONF_SLEEP_TIME])) cg.add(var.set_scan_time(conf[CONF_SCAN_TIME])) cg.add(var.set_debounce_time(conf[CONF_DEBOUNCE_TIME])) - if keys := conf.get(CONF_KEYS): - cg.add(var.set_keys(keys)) + if (keys := conf.get(CONF_KEYS)) is not None: + table = cg.shared_progmem_array( + "sx1509_keys", cg.uint8, list(keys.encode()) + ) + cg.add(var.set_keys(table)) for tconf in conf.get(CONF_ON_KEY, []): trigger = cg.new_Pvariable(tconf[CONF_TRIGGER_ID]) cg.add(var.register_key_trigger(trigger)) diff --git a/esphome/components/sx1509/sx1509.cpp b/esphome/components/sx1509/sx1509.cpp index 2397049000..ee8606db96 100644 --- a/esphome/components/sx1509/sx1509.cpp +++ b/esphome/components/sx1509/sx1509.cpp @@ -48,7 +48,7 @@ void SX1509Component::loop() { uint16_t key_data = this->read_key_data(); for (auto *binary_sensor : this->keypad_binary_sensors_) binary_sensor->process(key_data); - if (this->keys_.empty()) + if (this->keys_ == nullptr) return; if (key_data == 0) { this->last_key_ = 0; @@ -64,7 +64,9 @@ void SX1509Component::loop() { break; } col -= 8; - uint8_t key = this->keys_[row * this->cols_ + col]; + if (row >= this->rows_ || col >= this->cols_) // a partial read can leave only a row or a column bit + return; + uint8_t key = progmem_read_byte(&this->keys_[row * this->cols_ + col]); if (key == this->last_key_) return; this->last_key_ = key; diff --git a/esphome/components/sx1509/sx1509.h b/esphome/components/sx1509/sx1509.h index c7aed2cddd..ef6ee6ec5e 100644 --- a/esphome/components/sx1509/sx1509.h +++ b/esphome/components/sx1509/sx1509.h @@ -51,7 +51,8 @@ class SX1509Component final : public Component, this->cols_ = cols; this->has_keypad_ = true; }; - void set_keys(std::string keys) { this->keys_ = std::move(keys); }; // NOLINT(performance-unnecessary-value-param) + /// `keys` is a codegen PROGMEM table with one key code per button. + void set_keys(const uint8_t *keys) { this->keys_ = keys; } void set_sleep_time(uint16_t sleep_time) { this->sleep_time_ = sleep_time; }; void set_scan_time(uint8_t scan_time) { this->scan_time_ = scan_time; }; void set_debounce_time(uint8_t debounce_time = 1) { this->debounce_time_ = debounce_time; }; @@ -76,7 +77,7 @@ class SX1509Component final : public Component, bool has_keypad_ = false; uint8_t rows_ = 0; uint8_t cols_ = 0; - std::string keys_; + const uint8_t *keys_{nullptr}; uint16_t sleep_time_ = 128; uint8_t scan_time_ = 1; uint8_t debounce_time_ = 1; diff --git a/tests/component_tests/sx1509/__init__.py b/tests/component_tests/sx1509/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/sx1509/config/keys.yaml b/tests/component_tests/sx1509/config/keys.yaml new file mode 100644 index 0000000000..e5be4554c9 --- /dev/null +++ b/tests/component_tests/sx1509/config/keys.yaml @@ -0,0 +1,23 @@ +esphome: + name: test + +esp8266: + board: esp01_1m + +i2c: + sda: 4 + scl: 5 + +sx1509: + - id: sx_a + address: 0x3E + keypad: + key_rows: 2 + key_columns: 2 + keys: "abcd" + - id: sx_b + address: 0x3F + keypad: + key_rows: 2 + key_columns: 2 + keys: "abcd" diff --git a/tests/component_tests/sx1509/test_keys.py b/tests/component_tests/sx1509/test_keys.py new file mode 100644 index 0000000000..d7d41ae525 --- /dev/null +++ b/tests/component_tests/sx1509/test_keys.py @@ -0,0 +1,27 @@ +"""SX1509 keypad key codes live in a shared PROGMEM table instead of a std::string.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.sx1509 import check_keys +import esphome.config_validation as cv + + +def test_keys_use_shared_progmem_table( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("keys.yaml")) + + assert ( + "static constexpr uint8_t sx1509_keys[] PROGMEM = {97, 98, 99, 100};" + in main_cpp + ) + assert main_cpp.count("->set_keys(sx1509_keys);") == 2 + + +def test_non_ascii_keys_are_rejected() -> None: + with pytest.raises(cv.Invalid, match="'é' is not an ASCII"): + check_keys({"keys": "1é34"}) From 40af90ad25b23b91e4f97a324375e5da2fd3de0a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 19:24:13 -0500 Subject: [PATCH 182/221] [logger] Use the static ring buffer structure as its handle on ESP32 (#20129) --- esphome/components/logger/__init__.py | 10 +++++++ .../logger/task_log_buffer_esp32.cpp | 20 +++++--------- .../components/logger/task_log_buffer_esp32.h | 8 ++++-- tests/component_tests/logger/test_logger.py | 27 +++++++++++++++++++ 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index d6b9bca38b..a966ba67ad 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -235,6 +235,15 @@ def warn_ram_log_strings(config: ConfigType) -> ConfigType: return config +def validate_task_log_buffer_alignment(value: int) -> int: + # ESP-IDF rejects a no-split ring buffer whose size is not a multiple of 4 + if CORE.is_esp32 and value % 4: + raise cv.Invalid( + f"{CONF_TASK_LOG_BUFFER_SIZE} must be a multiple of 4 on ESP32" + ) + return value + + def validate_wait_for_cdc(config: ConfigType) -> ConfigType: if config.get(CONF_WAIT_FOR_CDC) and config.get(CONF_HARDWARE_UART) != USB_CDC: raise cv.Invalid("wait_for_cdc requires hardware_uart: USB_CDC") @@ -282,6 +291,7 @@ CONFIG_SCHEMA = cv.All( max=32768, # Max: Depends on message sizes, typically ~300 messages with default size ), ), + validate_task_log_buffer_alignment, ), cv.SplitDefault( CONF_HARDWARE_UART, diff --git a/esphome/components/logger/task_log_buffer_esp32.cpp b/esphome/components/logger/task_log_buffer_esp32.cpp index cb97f5504f..75b567ff89 100644 --- a/esphome/components/logger/task_log_buffer_esp32.cpp +++ b/esphome/components/logger/task_log_buffer_esp32.cpp @@ -10,15 +10,7 @@ namespace esphome::logger { TaskLogBuffer::TaskLogBuffer() { // Create a static ring buffer with RINGBUF_TYPE_NOSPLIT for message integrity // Storage is a member array (embedded in Logger), no heap allocation needed - this->ring_buffer_ = - xRingbufferCreateStatic(sizeof(this->storage_), RINGBUF_TYPE_NOSPLIT, this->storage_, &this->structure_); -} - -TaskLogBuffer::~TaskLogBuffer() { - if (this->ring_buffer_ != nullptr) { - vRingbufferDelete(this->ring_buffer_); - this->ring_buffer_ = nullptr; - } + xRingbufferCreateStatic(sizeof(this->storage_), RINGBUF_TYPE_NOSPLIT, this->storage_, &this->structure_); } bool TaskLogBuffer::borrow_message_main_loop(LogMessage *&message, uint16_t &text_length) { @@ -27,7 +19,7 @@ bool TaskLogBuffer::borrow_message_main_loop(LogMessage *&message, uint16_t &tex } size_t item_size = 0; - void *received_item = xRingbufferReceive(ring_buffer_, &item_size, 0); + void *received_item = xRingbufferReceive(this->handle_(), &item_size, 0); if (received_item == nullptr) { return false; } @@ -44,7 +36,7 @@ void TaskLogBuffer::release_message_main_loop() { if (this->current_token_ == nullptr) { return; } - vRingbufferReturnItem(ring_buffer_, this->current_token_); + vRingbufferReturnItem(this->handle_(), this->current_token_); this->current_token_ = nullptr; // Update counter to mark all messages as processed last_processed_counter_ = message_counter_.load(std::memory_order_relaxed); @@ -71,7 +63,7 @@ bool TaskLogBuffer::send_message_thread_safe(uint8_t level, const char *tag, uin // Acquire memory directly from the ring buffer void *acquired_memory = nullptr; - BaseType_t result = xRingbufferSendAcquire(ring_buffer_, &acquired_memory, total_size, 0); + BaseType_t result = xRingbufferSendAcquire(this->handle_(), &acquired_memory, total_size, 0); if (result != pdTRUE || acquired_memory == nullptr) { return false; // Failed to acquire memory @@ -100,7 +92,7 @@ bool TaskLogBuffer::send_message_thread_safe(uint8_t level, const char *tag, uin // Handle unexpected formatting error if (ret <= 0) { - vRingbufferReturnItem(ring_buffer_, acquired_memory); + vRingbufferReturnItem(this->handle_(), acquired_memory); return false; } @@ -111,7 +103,7 @@ bool TaskLogBuffer::send_message_thread_safe(uint8_t level, const char *tag, uin msg->text_length = text_length; // Complete the send operation with the acquired memory - result = xRingbufferSendComplete(ring_buffer_, acquired_memory); + result = xRingbufferSendComplete(this->handle_(), acquired_memory); if (result != pdTRUE) { return false; // Failed to complete the message send diff --git a/esphome/components/logger/task_log_buffer_esp32.h b/esphome/components/logger/task_log_buffer_esp32.h index e819766795..0ffaa04493 100644 --- a/esphome/components/logger/task_log_buffer_esp32.h +++ b/esphome/components/logger/task_log_buffer_esp32.h @@ -47,7 +47,7 @@ class TaskLogBuffer { }; TaskLogBuffer(); - ~TaskLogBuffer(); + // No destructor: Logger is never destroyed // NOT thread-safe - borrow a message from the ring buffer, only call from main loop bool borrow_message_main_loop(LogMessage *&message, uint16_t &text_length); @@ -68,7 +68,11 @@ class TaskLogBuffer { static constexpr size_t size() { return ESPHOME_TASK_LOG_BUFFER_SIZE; } private: - RingbufHandle_t ring_buffer_{nullptr}; // FreeRTOS ring buffer handle + // xRingbufferCreateStatic() returns the static structure itself as the handle; it only + // returns NULL for a no-split size that is unaligned or under two item headers + static_assert(ESPHOME_TASK_LOG_BUFFER_SIZE % 4 == 0, "task_log_buffer_size must be a multiple of 4"); + RingbufHandle_t handle_() { return &this->structure_; } + StaticRingbuffer_t structure_; // Static structure for the ring buffer uint8_t storage_[ESPHOME_TASK_LOG_BUFFER_SIZE]; // Embedded in Logger (no separate heap allocation) diff --git a/tests/component_tests/logger/test_logger.py b/tests/component_tests/logger/test_logger.py index 199d67ff5c..20b99f718f 100644 --- a/tests/component_tests/logger/test_logger.py +++ b/tests/component_tests/logger/test_logger.py @@ -6,7 +6,11 @@ import re import pytest +from esphome.components.logger import validate_task_log_buffer_alignment +from esphome.config_validation import Invalid +from esphome.const import PlatformFramework from esphome.core import CORE +from tests.component_tests.types import SetCoreConfigCallable def test_logger_pre_setup_before_other_components(generate_main): @@ -111,3 +115,26 @@ def test_flash_log_strings_default_does_not_warn( generate_main("tests/component_tests/logger/test_logger.yaml") assert "esp8266_store_log_strings_in_flash" not in caplog.text + + +@pytest.mark.parametrize("value", [0, 640, 768, 32768]) +def test_task_log_buffer_size_aligned_on_esp32( + set_core_config: SetCoreConfigCallable, value: int +) -> None: + set_core_config(PlatformFramework.ESP32_IDF) + assert validate_task_log_buffer_alignment(value) == value + + +def test_task_log_buffer_size_unaligned_rejected_on_esp32( + set_core_config: SetCoreConfigCallable, +) -> None: + set_core_config(PlatformFramework.ESP32_IDF) + with pytest.raises(Invalid, match="multiple of 4"): + validate_task_log_buffer_alignment(641) + + +def test_task_log_buffer_size_unaligned_allowed_on_libretiny( + set_core_config: SetCoreConfigCallable, +) -> None: + set_core_config(PlatformFramework.BK72XX_ARDUINO) + assert validate_task_log_buffer_alignment(641) == 641 From 80dc93d6cea6d69317e3c4b6b7de8fad360c082b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 19:24:31 -0500 Subject: [PATCH 183/221] [core] [select] Keep select options in flash with a shared ConstVector (#20117) --- esphome/components/api/api.proto | 2 +- esphome/components/api/api_connection.cpp | 4 +- esphome/components/api/api_pb2.h | 2 +- esphome/components/api/api_pb2_includes.h | 1 + esphome/components/select/__init__.py | 4 +- esphome/components/select/select_traits.cpp | 15 ++- esphome/components/select/select_traits.h | 18 +++- esphome/core/helpers.h | 79 +++++++++++++-- .../select/config/select_options.yaml | 26 +++++ .../select/test_select_options.py | 27 ++++++ tests/components/core/const_vector_test.cpp | 96 +++++++++++++++++++ .../components/select/select_traits_test.cpp | 83 ++++++++++++++++ 12 files changed, 336 insertions(+), 21 deletions(-) create mode 100644 tests/component_tests/select/config/select_options.yaml create mode 100644 tests/component_tests/select/test_select_options.py create mode 100644 tests/components/core/const_vector_test.cpp create mode 100644 tests/components/select/select_traits_test.cpp diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 687dc1ca95..a03bf5fc1a 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1439,7 +1439,7 @@ message ListEntitiesSelectResponse { reserved 4; // Deprecated: was string unique_id string icon = 5 [(field_ifdef) = "USE_ENTITY_ICON", (max_data_length) = 63]; - repeated string options = 6 [(container_pointer_no_template) = "FixedVector"]; + repeated string options = 6 [(container_pointer_no_template) = "std::span"]; bool disabled_by_default = 7; EntityCategory entity_category = 8; uint32 device_id = 9 [(field_ifdef) = "USE_DEVICES"]; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 0207bc14b9..e50bf2f722 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -990,7 +990,9 @@ uint16_t APIConnection::try_send_select_state(EntityBase *entity, APIConnection uint16_t APIConnection::try_send_select_info(EntityBase *entity, APIConnection *conn, uint32_t remaining_size) { auto *select = static_cast(entity); ListEntitiesSelectResponse msg; - msg.options = &select->traits.get_options(); + const auto &opts = select->traits.get_options(); + const std::span options(opts.data(), opts.size()); + msg.options = &options; return fill_and_encode_entity_info(select, msg, conn, remaining_size); } void APIConnection::on_select_command_request(const SelectCommandRequest &msg) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index dbf0fb49a1..b4e3d4c55b 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1909,7 +1909,7 @@ class ListEntitiesSelectResponse final : public InfoResponseProtoMessage { #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_select_response"); } #endif - const FixedVector *options{}; + const std::span *options{}; static uint8_t *encode_msg(const void *self, ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM); uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { return encode_msg(this, buffer PROTO_ENCODE_DEBUG_ARG); diff --git a/esphome/components/api/api_pb2_includes.h b/esphome/components/api/api_pb2_includes.h index 70ba579fcc..833e6529e2 100644 --- a/esphome/components/api/api_pb2_includes.h +++ b/esphome/components/api/api_pb2_includes.h @@ -28,6 +28,7 @@ // Standard library includes that might be needed #include +#include #include #include diff --git a/esphome/components/select/__init__.py b/esphome/components/select/__init__.py index fec88d2bfd..fa266fc81f 100644 --- a/esphome/components/select/__init__.py +++ b/esphome/components/select/__init__.py @@ -95,7 +95,9 @@ def select_schema( @setup_entity("select") async def setup_select_core_(var, config, *, options: list[str]): - cg.add(var.traits.set_options(options)) + if options: + table = cg.shared_progmem_array("select_options", cg.const_char_ptr, options) + cg.add(var.traits.set_options_static(table, len(options))) for conf in config.get(CONF_ON_VALUE, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) diff --git a/esphome/components/select/select_traits.cpp b/esphome/components/select/select_traits.cpp index 67a5118646..23d1f47cce 100644 --- a/esphome/components/select/select_traits.cpp +++ b/esphome/components/select/select_traits.cpp @@ -2,13 +2,18 @@ namespace esphome::select { -void SelectTraits::set_options(const std::initializer_list &options) { this->options_ = options; } +// Runtime option lists are copied, since the argument may not outlive the select; one +// out of line copy keeps a single instance of the copy code. +void SelectTraits::set_options_copy_(const char *const *options, size_t count) { + this->options_.assign_copy(options, count); +} + +void SelectTraits::set_options(const std::initializer_list &options) { + this->set_options_copy_(options.begin(), options.size()); +} void SelectTraits::set_options(const FixedVector &options) { - this->options_.init(options.size()); - for (const auto &opt : options) { - this->options_.push_back(opt); - } + this->set_options_copy_(options.begin(), options.size()); } } // namespace esphome::select diff --git a/esphome/components/select/select_traits.h b/esphome/components/select/select_traits.h index e1b261bc96..47997ac7f2 100644 --- a/esphome/components/select/select_traits.h +++ b/esphome/components/select/select_traits.h @@ -5,14 +5,28 @@ namespace esphome::select { +/// Option strings: a shared codegen table, or a copy of a runtime list. +using SelectOptions = ConstVector; + class SelectTraits { public: + SelectTraits() = default; + SelectTraits(const SelectTraits &) = delete; + SelectTraits &operator=(const SelectTraits &) = delete; + + /// Codegen only: points at a table that outlives the select. Call before any runtime set_options; + /// it does not free a previous copy (generated setup() runs before any lambda or automation). + void set_options_static(const char *const *options, size_t count) { this->options_.assign_static(options, count); } + /// Runtime lists: the pointer list is copied, as before; the strings must still outlive the select. + void set_options(const SelectOptions &options) { this->set_options_copy_(options.data(), options.size()); } void set_options(const std::initializer_list &options); void set_options(const FixedVector &options); - const FixedVector &get_options() const { return this->options_; } + const SelectOptions &get_options() const { return this->options_; } protected: - FixedVector options_; + void set_options_copy_(const char *const *options, size_t count); + + SelectOptions options_; }; } // namespace esphome::select diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 6d00e18799..b88a9d70e9 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -129,23 +129,82 @@ template<> constexpr int64_t byteswap(int64_t n) { return __builtin_bswap64(n); /// @name Container utilities ///@{ -/// Lightweight read-only view over a const array stored in RODATA (will typically be in flash memory) -/// Avoids copying data from flash to RAM by keeping a pointer to the flash data. -/// Similar to std::span but with minimal overhead for embedded systems. - -template class ConstVector { +/// Lightweight read-only view over a const array stored in RODATA (will typically be in flash memory). +/// Iterators are raw pointers like FixedVector. With Owning = true it can also hold a heap copy it +/// owns (see the specialization below); the default view never frees and has no extra cost. +template class ConstVector { public: + using value_type = T; + + constexpr ConstVector() = default; constexpr ConstVector(const T *data, size_t size) : data_(data), size_(size) {} - const constexpr T &operator[](size_t i) const { return data_[i]; } - constexpr size_t size() const { return size_; } - constexpr bool empty() const { return size_ == 0; } + const T *begin() const { return this->data_; } + const T *end() const { return this->data_ + this->size_; } + const T *data() const { return this->data_; } + constexpr size_t size() const { return this->size_; } + constexpr bool empty() const { return this->size_ == 0; } + const constexpr T &operator[](size_t i) const { return this->data_[i]; } + const T &at(size_t i) const { return this->data_[i]; } protected: - const T *data_; - size_t size_; + const T *data_{nullptr}; + size_t size_{0}; }; +/// Owning variant: a codegen table that outlives it, or a heap copy of a runtime list it owns. +/// Ownership is the top bit of the size; it is not copyable, so a copy can never outlive the owner. +/// Elements must be whole words so ESP8266 can read a codegen table from flash. +template class ConstVector { + static_assert(std::is_trivially_copyable_v && sizeof(T) % sizeof(uint32_t) == 0, + "ConstVector elements must be whole words so ESP8266 can read them from flash"); + + public: + using value_type = T; + + constexpr ConstVector() = default; + constexpr ConstVector(const T *data, size_t size) : data_(data), size_(size) {} + ConstVector(const ConstVector &) = delete; + ConstVector &operator=(const ConstVector &) = delete; + ~ConstVector() { this->release_(); } + + const T *begin() const { return this->data_; } + const T *end() const { return this->data_ + this->size(); } + const T *data() const { return this->data_; } + size_t size() const { return this->size_ & ~OWNED_BIT; } + bool empty() const { return this->size() == 0; } + const T &operator[](size_t index) const { return this->data_[index]; } + const T &at(size_t index) const { return this->data_[index]; } + + /// Codegen only: call before any runtime copy; it does not free a previous owned copy + /// (generated setup() runs before any lambda or automation can call set_options). + void assign_static(const T *data, size_t size) { + this->data_ = data; + this->size_ = size; + } + /// Copies the list into a heap array this owns, freeing a previous owned copy. + void assign_copy(const T *data, size_t size) { + auto *table = new T[size]; // NOLINT(cppcoreguidelines-owning-memory) + std::copy(data, data + size, table); + this->release_(); + this->data_ = table; + this->size_ = size | OWNED_BIT; + } + + protected: + static constexpr size_t OWNED_BIT = size_t{1} << (sizeof(size_t) * 8 - 1); + + void release_() { + if (this->size_ & OWNED_BIT) + delete[] this->data_; // NOLINT(cppcoreguidelines-owning-memory) + } + + const T *data_{nullptr}; + size_t size_{0}; // top bit set when data_ is an owned heap copy +}; +static_assert(sizeof(ConstVector) == 2 * sizeof(void *), + "ConstVector must stay a pointer and a size"); + /// Small buffer optimization - stores data inline when small, heap-allocates for large data /// This avoids heap fragmentation for common small allocations while supporting arbitrary sizes. /// Memory management is encapsulated - callers just use set() and data(). diff --git a/tests/component_tests/select/config/select_options.yaml b/tests/component_tests/select/config/select_options.yaml new file mode 100644 index 0000000000..474fc7cec4 --- /dev/null +++ b/tests/component_tests/select/config/select_options.yaml @@ -0,0 +1,26 @@ +esphome: + name: test + +esp8266: + board: esp01_1m + +select: + - platform: template + id: fan_a + name: Fan A + optimistic: true + options: [low, medium, high] + - platform: template + id: fan_b + name: Fan B + optimistic: true + options: [low, medium, high] + - platform: template + id: mode + name: Mode + optimistic: true + options: [auto, manual] + - platform: copy + id: fan_copy + name: Fan Copy + source_id: fan_a diff --git a/tests/component_tests/select/test_select_options.py b/tests/component_tests/select/test_select_options.py new file mode 100644 index 0000000000..0fe70f7f2c --- /dev/null +++ b/tests/component_tests/select/test_select_options.py @@ -0,0 +1,27 @@ +"""Select options are shared PROGMEM tables of option pointers.""" + +from collections.abc import Callable +from pathlib import Path +import re + + +def test_select_options_use_shared_tables( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("select_options.yaml")) + + calls = { + var: table + for var, table, _ in re.findall( + r"(\w+)->traits\.set_options_static\((\w+), (\d+)\);", main_cpp + ) + } + assert set(calls) == {"fan_a", "fan_b", "mode"} + assert calls["fan_a"] == calls["fan_b"] != calls["mode"] + assert ( + f'static constexpr const char * {calls["fan_a"]}[] PROGMEM = {{"low", "medium", "high"}};' + in main_cpp + ) + # The copy select takes its options from the source at setup + assert "fan_copy->traits.set_options_static(" not in main_cpp diff --git a/tests/components/core/const_vector_test.cpp b/tests/components/core/const_vector_test.cpp new file mode 100644 index 0000000000..aae067fb0f --- /dev/null +++ b/tests/components/core/const_vector_test.cpp @@ -0,0 +1,96 @@ +#include + +#include +#include + +#include "esphome/core/helpers.h" + +namespace esphome::testing { + +static constexpr const char *const TABLE[] = {"a", "b", "c"}; + +// Exposes the owned flag, which is protected. +class ProbeVector : public ConstVector { + public: + using ConstVector::ConstVector; + bool owned() const { return (this->size_ & OWNED_BIT) != 0; } +}; + +TEST(ConstVector, StaticTableIsViewedNotOwned) { + ProbeVector list; + EXPECT_TRUE(list.empty()); + list.assign_static(TABLE, 3); + EXPECT_EQ(list.data(), TABLE); + EXPECT_EQ(list.size(), 3U); + EXPECT_FALSE(list.owned()); + EXPECT_STREQ(list[1], "b"); + EXPECT_STREQ(list.at(2), "c"); +} + +TEST(ConstVector, CopyIsOwnedAndSizeMasksTheFlag) { + ProbeVector list; + list.assign_copy(TABLE, 3); + EXPECT_NE(list.data(), TABLE); + EXPECT_TRUE(list.owned()); + EXPECT_EQ(list.size(), 3U); + EXPECT_STREQ(list[0], "a"); + + const char *const next[] = {"x", "y"}; + list.assign_copy(next, 2); // frees the previous copy + EXPECT_TRUE(list.owned()); + EXPECT_EQ(list.size(), 2U); + EXPECT_STREQ(list[1], "y"); +} + +TEST(ConstVector, StaticThenRuntimeCopiesNeverFreeTheTable) { + ProbeVector list; + list.assign_static(TABLE, 3); + EXPECT_FALSE(list.owned()); + list.assign_copy(TABLE, 2); // the static table is not owned, so nothing is freed + EXPECT_TRUE(list.owned()); + EXPECT_NE(list.data(), TABLE); + const char *const next[] = {"x"}; + list.assign_copy(next, 1); // frees the previous copy + EXPECT_EQ(list.size(), 1U); + EXPECT_STREQ(list[0], "x"); + EXPECT_STREQ(TABLE[0], "a"); +} + +TEST(ConstVector, EmptyCopyIsEmptyAndFreedOnNextSet) { + ProbeVector list; + list.assign_copy(TABLE, 3); + list.assign_copy(TABLE, 0); + EXPECT_TRUE(list.empty()); + list.assign_copy(TABLE, 2); // frees the empty copy + EXPECT_EQ(list.size(), 2U); +} + +TEST(ConstVector, OwningVariantIsNotCopyable) { + static_assert(!std::is_copy_constructible_v>); + static_assert(!std::is_copy_assignable_v>); +} + +TEST(ConstVector, CopyFromItsOwnStorage) { + ProbeVector list; + list.assign_copy(TABLE, 3); + list.assign_copy(list.data(), list.size()); + EXPECT_EQ(list.size(), 3U); + EXPECT_STREQ(list[2], "c"); +} + +TEST(ConstVector, PlainViewHasNoOwnershipCost) { + static_assert(std::is_trivially_copyable_v>); + static_assert(std::is_trivially_destructible_v>); + ConstVector list(TABLE, 3); + EXPECT_EQ(list.size(), 3U); + EXPECT_STREQ(list[2], "c"); +} + +TEST(ConstVector, IteratorsAreRawPointers) { + ConstVector list(TABLE, 3); + static_assert(std::is_same_v); + const auto *it = std::find(list.begin(), list.end(), TABLE[1]); + EXPECT_EQ(it - list.begin(), 1); +} + +} // namespace esphome::testing diff --git a/tests/components/select/select_traits_test.cpp b/tests/components/select/select_traits_test.cpp new file mode 100644 index 0000000000..2280dc203d --- /dev/null +++ b/tests/components/select/select_traits_test.cpp @@ -0,0 +1,83 @@ +#include + +#include +#include + +#include "esphome/components/select/select_traits.h" + +namespace esphome::select::testing { + +static constexpr const char *const OPTIONS[] = {"low", "medium", "high"}; + +TEST(SelectTraits, ViewsTheTableWithoutCopying) { + SelectTraits traits; + EXPECT_TRUE(traits.get_options().empty()); + traits.set_options_static(OPTIONS, 3); + const auto &options = traits.get_options(); + EXPECT_EQ(options.size(), 3U); + EXPECT_FALSE(options.empty()); + EXPECT_EQ(options.data(), OPTIONS); + EXPECT_STREQ(options[1], "medium"); + EXPECT_STREQ(options.at(2), "high"); + std::vector seen; + for (const char *option : options) + seen.emplace_back(option); + EXPECT_EQ(seen, (std::vector{"low", "medium", "high"})); +} + +TEST(SelectTraits, RuntimeListsAreCopied) { + SelectTraits traits; + traits.set_options({"a", "b"}); + EXPECT_EQ(traits.get_options().size(), 2U); + EXPECT_STREQ(traits.get_options()[1], "b"); + + FixedVector list; + list.init(3); + list.push_back("x"); + list.push_back("y"); + list.push_back("z"); + traits.set_options(list); + EXPECT_NE(traits.get_options().data(), list.begin()); + EXPECT_EQ(traits.get_options().size(), 3U); + EXPECT_STREQ(traits.get_options().at(2), "z"); + + // A later runtime list replaces the earlier copy + traits.set_options({"only"}); + EXPECT_EQ(traits.get_options().size(), 1U); + EXPECT_STREQ(traits.get_options()[0], "only"); +} + +TEST(SelectTraits, CopyingAnotherSelectSurvivesItsNextRuntimeList) { + SelectTraits source; + source.set_options({"a", "b"}); + SelectTraits copy; + copy.set_options(source.get_options()); + EXPECT_NE(copy.get_options().data(), source.get_options().data()); + + source.set_options({"c"}); + ASSERT_EQ(copy.get_options().size(), 2U); + EXPECT_STREQ(copy.get_options()[0], "a"); + EXPECT_STREQ(copy.get_options()[1], "b"); +} + +TEST(SelectTraits, StaticTablesAreNeverOwned) { + SelectTraits traits; + traits.set_options_static(OPTIONS, 3); + EXPECT_EQ(traits.get_options().data(), OPTIONS); + EXPECT_EQ(traits.get_options().size(), 3U); + // A runtime list after a static one copies and leaves the static table alone + traits.set_options({"x"}); + EXPECT_NE(traits.get_options().data(), OPTIONS); + EXPECT_EQ(traits.get_options().size(), 1U); + EXPECT_STREQ(OPTIONS[0], "low"); +} + +TEST(SelectTraits, CopyOfItsOwnOptionsStaysValid) { + SelectTraits traits; + traits.set_options({"a", "b"}); + traits.set_options(traits.get_options()); + ASSERT_EQ(traits.get_options().size(), 2U); + EXPECT_STREQ(traits.get_options()[1], "b"); +} + +} // namespace esphome::select::testing From 3a3707926dfa906672455bfd8d500c20415d0f82 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 19:25:14 -0500 Subject: [PATCH 184/221] [wifi] Run the LN882x fallback AP on its own instead of retrying AP+STA every loop (#20166) --- esphome/components/wifi/__init__.py | 4 + esphome/components/wifi/wifi_component.cpp | 125 ++++++++++++++++-- esphome/components/wifi/wifi_component.h | 12 ++ .../wifi/wifi_component_libretiny.cpp | 7 + esphome/core/defines.h | 1 + .../captive_portal/test.ln882x-ard.yaml | 1 + .../provisioning/test.ln882x-ard.yaml | 15 +++ 7 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 tests/components/captive_portal/test.ln882x-ard.yaml create mode 100644 tests/components/provisioning/test.ln882x-ard.yaml diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 676aa0d294..27ebdbc976 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -669,6 +669,10 @@ async def to_code(config): if (ap_timeout := conf[CONF_AP_TIMEOUT]) != cv.time_period(DEFAULT_AP_TIMEOUT): cg.add(var.set_ap_timeout(ap_timeout)) cg.add_define("USE_WIFI_AP") + # The LN882H radio cannot run the AP and STA together; the fallback AP + # takes turns with the networks instead. + if CORE.is_ln882x: + cg.add_define("USE_WIFI_AP_EXCLUSIVE") # ESP32: register the WiFi stack with the esp32 sdkconfig reconciler, which # drops SoftAP support / the LWIP DHCP server when AP mode is unused. diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 86e13ef84b..fd982c0d31 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -380,6 +380,25 @@ static constexpr uint8_t WIFI_RETRY_COUNT_PER_AP = 1; /// Cooldown duration in milliseconds after adapter restart or repeated failures /// Allows WiFi hardware to stabilize before next connection attempt static constexpr uint32_t WIFI_COOLDOWN_DURATION_MS = 500; +#ifdef USE_WIFI_AP +/// Whether the radio runs the fallback AP on its own, without STA alongside. +#ifdef USE_WIFI_AP_EXCLUSIVE +static constexpr bool WIFI_AP_EXCLUSIVE = true; +#else +static constexpr bool WIFI_AP_EXCLUSIVE = false; +#endif +#endif +#ifdef USE_WIFI_AP_EXCLUSIVE +/// On a radio that cannot run the AP and STA together: how long the fallback +/// AP stays up while nobody uses it before it pauses so the networks can be +/// tried again, and how long new credentials give the portal to answer +/// before the AP drops. +static constexpr uint32_t WIFI_AP_EXCLUSIVE_DWELL_MS = 300000; +/// The AP pauses after this long even with clients counted, so a lost +/// disconnect event cannot keep the networks from being tried for good. +static constexpr uint32_t WIFI_AP_EXCLUSIVE_MAX_DWELL_MS = 3 * WIFI_AP_EXCLUSIVE_DWELL_MS; +static constexpr uint32_t WIFI_AP_EXCLUSIVE_HANDOVER_MS = 1000; +#endif /// Cooldown duration when fallback AP is active and captive portal may be running /// Longer interval gives users time to configure WiFi without constant connection attempts @@ -642,6 +661,14 @@ void WiFiComponent::setup() { provisioning::global_provisioning_manager->add_on_closed_callback([this]() { if (this->ap_setup_) { ESP_LOGD(TAG, "Provisioning window closed; disabling AP"); +#ifdef USE_WIFI_AP_EXCLUSIVE + // The networks wait while the AP runs on its own; hand them the radio + // now rather than at the end of the dwell. + if (this->state_ == WIFI_COMPONENT_STATE_AP && this->has_sta()) { + this->pause_exclusive_ap_(); + return; + } +#endif this->wifi_mode_({}, false); } }); @@ -739,8 +766,11 @@ void WiFiComponent::start() { } #ifdef USE_CAPTIVE_PORTAL if (captive_portal::global_captive_portal != nullptr) { - this->wifi_sta_pre_setup_(); - this->start_scanning(); + // Where the radio scans alongside the AP, the portal can list networks. + if (!WIFI_AP_EXCLUSIVE) { + this->wifi_sta_pre_setup_(); + this->start_scanning(); + } captive_portal::global_captive_portal->start(); } #endif @@ -862,8 +892,17 @@ void WiFiComponent::loop() { } break; } - case WIFI_COMPONENT_STATE_OFF: case WIFI_COMPONENT_STATE_AP: +#ifdef USE_WIFI_AP_EXCLUSIVE + // The networks are not tried while the AP is up, so it pauses for + // them once nobody has used it for a while. Without networks the AP + // is all there is and stays. + if (this->has_sta() && now - this->ap_exclusive_changed_ > WIFI_AP_EXCLUSIVE_DWELL_MS && + (this->ap_clients_ == 0 || now - this->ap_exclusive_changed_ > WIFI_AP_EXCLUSIVE_MAX_DWELL_MS)) + this->pause_exclusive_ap_(); + break; +#endif + case WIFI_COMPONENT_STATE_OFF: break; case WIFI_COMPONENT_STATE_DISABLED: return; @@ -879,11 +918,23 @@ void WiFiComponent::loop() { provisioning::global_provisioning_manager != nullptr && provisioning::global_provisioning_manager->closed(); #endif if (this->has_ap() && !this->ap_setup_ && !provisioning_closed) { - if (this->ap_timeout_ != 0 && (now - this->last_connected_ > this->ap_timeout_)) { + if (this->ap_timeout_ != 0 && + (now - this->last_connected_ > this->ap_timeout_) +#ifdef USE_WIFI_AP_EXCLUSIVE + // After a pause, or a start that failed, the networks get a full + // ap_timeout before the AP is tried again. + && now - this->ap_exclusive_changed_ > this->ap_timeout_ +#endif + ) { ESP_LOGI(TAG, "Starting fallback AP"); +#ifdef USE_WIFI_AP_EXCLUSIVE + this->ap_exclusive_changed_ = now; +#endif this->setup_ap_config_(); #ifdef USE_CAPTIVE_PORTAL - if (captive_portal::global_captive_portal != nullptr) { + // Where the AP runs on its own, a portal with no AP behind it would + // only stretch the cooldowns. + if (captive_portal::global_captive_portal != nullptr && (!WIFI_AP_EXCLUSIVE || this->ap_setup_)) { // Reset so we force one full scan after captive portal starts // (previous scans were filtered because captive portal wasn't active yet) this->has_completed_scan_after_captive_portal_start_ = false; @@ -972,7 +1023,13 @@ network::IPAddress WiFiComponent::get_dns_address(int num) { #ifdef USE_WIFI_AP void WiFiComponent::setup_ap_config_() { +#ifdef USE_WIFI_AP_EXCLUSIVE + // The radio cannot run both, so STA goes off while the AP is up. + if (!this->wifi_mode_(false, true)) + return; +#else this->wifi_mode_({}, true); +#endif if (this->ap_setup_) return; @@ -1006,6 +1063,12 @@ void WiFiComponent::setup_ap_config_() { } } this->ap_setup_ = this->wifi_start_ap_(this->ap_); +#ifdef USE_WIFI_AP_EXCLUSIVE + // A failed start must not leave the radio in AP mode, which would refuse + // the STA attempts until the next try. + if (!this->ap_setup_) + this->wifi_mode_({}, false); +#endif char ip_buf[network::IP_ADDRESS_BUFFER_SIZE]; ESP_LOGCONFIG(TAG, @@ -1030,11 +1093,40 @@ void WiFiComponent::setup_ap_config_() { } #endif - if (!this->has_sta()) { + // Where the AP runs on its own, the STA state machine also waits in this + // state until the AP pauses. + if (!this->has_sta() || (WIFI_AP_EXCLUSIVE && this->ap_setup_)) { this->state_ = WIFI_COMPONENT_STATE_AP; } } +#ifdef USE_WIFI_AP_EXCLUSIVE +void WiFiComponent::pause_exclusive_ap_() { + ESP_LOGI(TAG, "Pausing AP to try the networks"); + // Scanning and connecting turn STA back on. + this->disable_ap_(); + this->ap_setup_ = false; + // Clients drop with the AP without a disconnect event for each. + this->ap_clients_ = 0; + this->ap_exclusive_changed_ = App.get_loop_component_start_time(); + // A fresh connection cycle, as at boot, rather than resuming the one the + // AP cut short. + this->retry_phase_ = WiFiRetryPhase::INITIAL_CONNECT; + this->num_retried_ = 0; + this->start_initial_connection_(); +} +#endif + +void WiFiComponent::disable_ap_() { +#ifdef USE_CAPTIVE_PORTAL + if (this->is_captive_portal_active_()) { + captive_portal::global_captive_portal->end(); + } +#endif + ESP_LOGD(TAG, "Disabling AP"); + this->wifi_mode_({}, false); +} + void WiFiComponent::set_ap(const WiFiAP &ap) { this->ap_ = ap; this->has_ap_ = true; @@ -1126,6 +1218,17 @@ void WiFiComponent::save_wifi_sta(const char *ssid, const char *password) { } void WiFiComponent::connect_soon_() { +#ifdef USE_WIFI_AP_EXCLUSIVE + // New credentials from the portal: the AP pauses for them once its answer + // is out, and the STA state machine picks them up from there. + if (this->ap_setup_) { + this->set_timeout("wifi-ap-handover", WIFI_AP_EXCLUSIVE_HANDOVER_MS, [this]() { + if (this->ap_setup_) + this->pause_exclusive_ap_(); + }); + return; + } +#endif // Only trigger retry if we're in cooldown - if already connecting/connected, do nothing if (this->state_ == WIFI_COMPONENT_STATE_COOLDOWN) { ESP_LOGD(TAG, "Exiting cooldown early due to new WiFi credentials"); @@ -1636,15 +1739,11 @@ void WiFiComponent::check_connecting_finished(uint32_t now) { // Reset to initial phase on successful connection (don't log transition, just reset state) this->retry_phase_ = WiFiRetryPhase::INITIAL_CONNECT; this->num_retried_ = 0; +#ifdef USE_WIFI_AP if (this->has_ap()) { -#ifdef USE_CAPTIVE_PORTAL - if (this->is_captive_portal_active_()) { - captive_portal::global_captive_portal->end(); - } -#endif - ESP_LOGD(TAG, "Disabling AP"); - this->wifi_mode_({}, false); + this->disable_ap_(); } +#endif #ifdef USE_IMPROV_BLE if (this->is_improv_ble_active_()) { improv_ble::global_improv_component->stop(); diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 9418165501..9700e36e6f 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -687,6 +687,12 @@ class WiFiComponent final : public Component { protected: #ifdef USE_WIFI_AP void setup_ap_config_(); + /// End the captive portal and turn the AP off. + void disable_ap_(); +#ifdef USE_WIFI_AP_EXCLUSIVE + /// Drop the fallback AP so the networks can be tried; it comes back after ap_timeout. + void pause_exclusive_ap_(); +#endif #endif // USE_WIFI_AP void print_connect_params_(); @@ -919,6 +925,9 @@ class WiFiComponent final : public Component { float output_power_{NAN}; uint32_t action_started_; uint32_t last_connected_{0}; +#ifdef USE_WIFI_AP_EXCLUSIVE + uint32_t ap_exclusive_changed_{0}; // When the AP was last started or paused +#endif uint32_t reboot_timeout_{900000}; // Keep in sync with DEFAULT_REBOOT_TIMEOUT in __init__.py uint32_t roaming_last_check_{0}; uint32_t roaming_scan_end_{0}; // Timestamp when last roaming scan completed @@ -1011,6 +1020,9 @@ class WiFiComponent final : public Component { bool scan_done_{false}; bool ap_setup_{false}; bool ap_started_{false}; +#ifdef USE_WIFI_AP_EXCLUSIVE + uint8_t ap_clients_{0}; // Devices joined to the AP, which keep it from pausing +#endif bool passive_scan_{false}; bool has_saved_wifi_settings_{false}; #ifdef USE_WIFI_11KV_SUPPORT diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index 1f43f37c0e..28b30628fd 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -624,6 +624,9 @@ void WiFiComponent::wifi_process_event_(LTWiFiEvent *event) { break; } case ESPHOME_EVENT_ID_WIFI_AP_STACONNECTED: { +#ifdef USE_WIFI_AP_EXCLUSIVE + this->ap_clients_++; +#endif #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE auto &it = event->data.sta_connected; char mac_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; @@ -633,6 +636,10 @@ void WiFiComponent::wifi_process_event_(LTWiFiEvent *event) { break; } case ESPHOME_EVENT_ID_WIFI_AP_STADISCONNECTED: { +#ifdef USE_WIFI_AP_EXCLUSIVE + if (this->ap_clients_ > 0) + this->ap_clients_--; +#endif #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE auto &it = event->data.sta_disconnected; char mac_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index f701cd2919..abd4a3b08d 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -602,6 +602,7 @@ // to parse against. #ifdef USE_LN882X #define USE_LN882H_BLE_TRACKER +#define USE_WIFI_AP_EXCLUSIVE #else #define USE_BK72XX_BLE_TRACKER #endif diff --git a/tests/components/captive_portal/test.ln882x-ard.yaml b/tests/components/captive_portal/test.ln882x-ard.yaml new file mode 100644 index 0000000000..dade44d145 --- /dev/null +++ b/tests/components/captive_portal/test.ln882x-ard.yaml @@ -0,0 +1 @@ +<<: !include common.yaml diff --git a/tests/components/provisioning/test.ln882x-ard.yaml b/tests/components/provisioning/test.ln882x-ard.yaml new file mode 100644 index 0000000000..39d8800961 --- /dev/null +++ b/tests/components/provisioning/test.ln882x-ard.yaml @@ -0,0 +1,15 @@ +# Provisioning window on LN882x, where the fallback AP runs on its own: closing +# the window hands the radio back to the networks. +provisioning: + timeout: 1min + +api: + encryption: + +wifi: + ssid: MySSID + password: password1 + ap: + ssid: MyAP + +captive_portal: From 8d8bd7311279bf91a8540d5c0f47edcb38cdd088 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 19:28:36 -0500 Subject: [PATCH 185/221] [ble_device_base] Keep MAC filter lists in shared flash tables (#20088) --- .../components/ble_device_base/automation.h | 21 ++++++---- .../components/ble_device_base/automation.py | 24 ++++++++++-- .../components/esp32_ble_tracker/__init__.py | 11 ++++-- .../components/esp32_ble_tracker/automation.h | 17 +++------ .../test_automations_codegen.py | 6 ++- .../esp32_ble_tracker/config/mac_filter.yaml | 27 +++++++++++++ .../esp32_ble_tracker/test_mac_filter.py | 38 +++++++++++++++++++ .../test_automations_codegen.py | 6 ++- .../test_automations_codegen.py | 8 +++- 9 files changed, 128 insertions(+), 30 deletions(-) create mode 100644 tests/component_tests/esp32_ble_tracker/config/mac_filter.yaml create mode 100644 tests/component_tests/esp32_ble_tracker/test_mac_filter.py diff --git a/esphome/components/ble_device_base/automation.h b/esphome/components/ble_device_base/automation.h index ba3128c0ee..a50813c585 100644 --- a/esphome/components/ble_device_base/automation.h +++ b/esphome/components/ble_device_base/automation.h @@ -11,29 +11,34 @@ #include "esphome/core/automation.h" #include "esphome/core/helpers.h" -#include -#include - namespace esphome::ble_device_base { +/// True if `address` is in `table`, a list of MACs ended by 0. +inline bool mac_in_table(const uint64_t *table, uint64_t address) { + for (; *table != 0; table++) { + if (*table == address) + return true; + } + return false; +} + // on_ble_advertise: fires on every BLE advertisement, optionally filtered to one or more MACs. class ESPBTAdvertiseTrigger final : public Trigger, public ESPBTDeviceListener { public: template explicit ESPBTAdvertiseTrigger(Hub *parent) { parent->register_listener(this); } - void set_addresses(std::initializer_list addresses) { this->addresses_ = addresses; } + /// Table of MACs ended by 0; must outlive the trigger. + void set_addresses(const uint64_t *addresses) { this->addresses_ = addresses; } bool parse_device(const ESPBTDevice &device) override { - if (!this->addresses_.empty() && std::find(this->addresses_.begin(), this->addresses_.end(), - device.address_uint64()) == this->addresses_.end()) { + if (this->addresses_ != nullptr && !mac_in_table(this->addresses_, device.address_uint64())) return false; - } this->trigger(device); return true; } protected: - FixedVector addresses_; + const uint64_t *addresses_{nullptr}; }; // on_ble_service_data_advertise: fires when an advertisement contains service diff --git a/esphome/components/ble_device_base/automation.py b/esphome/components/ble_device_base/automation.py index 6acc4edb92..7cc87add5a 100644 --- a/esphome/components/ble_device_base/automation.py +++ b/esphome/components/ble_device_base/automation.py @@ -69,13 +69,24 @@ def uuid_trigger_schema( ) +def _filter_mac(value: Any) -> Any: + mac = cv.mac_address(value) + if not any(mac.parts): + # 0 ends the flash MAC table, and it is never a real device address. + raise cv.Invalid("00:00:00:00:00:00 cannot be used as a MAC address filter") + return mac + + +MAC_FILTER_LIST = cv.ensure_list(_filter_mac) + + def advertise_trigger_schema(trigger_class: MockObjClass) -> Callable[[Any], Any]: """on_ble_advertise schema: multi-mac list filter, unlike the single-mac uuid_trigger_schema() — pairs with advertise_trigger_to_code().""" return automation.validate_automation( { cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(trigger_class), - cv.Optional(CONF_MAC_ADDRESS): cv.ensure_list(cv.mac_address), + cv.Optional(CONF_MAC_ADDRESS): MAC_FILTER_LIST, } ) @@ -94,11 +105,18 @@ def scan_end_trigger_schema(trigger_class: MockObjClass) -> Callable[[Any], Any] _count_listener = cg.slot_counter(LISTENER_COUNT_DEFINE) +def mac_filter_table(macs: list) -> cg.MockObj: + """Shared flash table of MACs ended by 0 (never a valid address), so triggers store a pointer.""" + return cg.shared_progmem_array( + "ble_mac_filter", cg.uint64, [*(mac.as_hex for mac in macs), 0] + ) + + async def advertise_trigger_to_code(conf: ConfigType, var: cg.MockObj) -> None: """Build an on_ble_advertise trigger (optional multi-mac filter).""" trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) - if (macs := conf.get(CONF_MAC_ADDRESS)) is not None: - cg.add(trigger.set_addresses([it.as_hex for it in macs])) + if macs := conf.get(CONF_MAC_ADDRESS): + cg.add(trigger.set_addresses(mac_filter_table(macs))) await automation.build_automation(trigger, [(ESPBTDeviceConstRef, "x")], conf) _count_listener() diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index fc52b92ee7..59ac445cc0 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -8,6 +8,10 @@ from esphome import automation import esphome.codegen as cg from esphome.components import ble_device_base, esp32_ble, ota from esphome.components.ble_device_base import CONF_CONNECTION_SCAN_WINDOW +from esphome.components.ble_device_base.automation import ( + MAC_FILTER_LIST, + mac_filter_table, +) from esphome.components.const import CONF_ON_SCAN_END, CONF_SCAN_PARAMETERS, CONF_WINDOW from esphome.components.esp32 import ( add_idf_sdkconfig_option, @@ -277,7 +281,7 @@ CONFIG_SCHEMA = cv.All( cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id( ESPBTAdvertiseTrigger ), - cv.Optional(CONF_MAC_ADDRESS): cv.ensure_list(cv.mac_address), + cv.Optional(CONF_MAC_ADDRESS): MAC_FILTER_LIST, } ), cv.Optional( @@ -381,9 +385,8 @@ async def to_code(config: ConfigType) -> None: for conf in config.get(CONF_ON_BLE_ADVERTISE, []): _request_listener_slot() trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) - if CONF_MAC_ADDRESS in conf: - addr_list = [it.as_hex for it in conf[CONF_MAC_ADDRESS]] - cg.add(trigger.set_addresses(addr_list)) + if macs := conf.get(CONF_MAC_ADDRESS): + cg.add(trigger.set_addresses(mac_filter_table(macs))) await automation.build_automation(trigger, [(ESPBTDeviceConstRef, "x")], conf) for conf in config.get(CONF_ON_BLE_SERVICE_DATA_ADVERTISE, []): _request_listener_slot() diff --git a/esphome/components/esp32_ble_tracker/automation.h b/esphome/components/esp32_ble_tracker/automation.h index 9b4eb2c3b3..f30f63a5c8 100644 --- a/esphome/components/esp32_ble_tracker/automation.h +++ b/esphome/components/esp32_ble_tracker/automation.h @@ -1,10 +1,9 @@ #pragma once #include "esphome/core/automation.h" +#include "esphome/components/ble_device_base/automation.h" #include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h" -#include - #ifdef USE_ESP32 namespace esphome::esp32_ble_tracker { @@ -12,22 +11,18 @@ namespace esphome::esp32_ble_tracker { class ESPBTAdvertiseTrigger final : public Trigger, public ESPBTDeviceListener { public: explicit ESPBTAdvertiseTrigger(ESP32BLETracker *parent) { parent->register_listener(this); } - void set_addresses(std::initializer_list addresses) { this->address_vec_ = addresses; } + /// Table of MACs ended by 0; must outlive the trigger. + void set_addresses(const uint64_t *addresses) { this->addresses_ = addresses; } bool parse_device(const ESPBTDevice &device) override { - uint64_t u64_addr = device.address_uint64(); - if (!address_vec_.empty()) { - if (std::find(address_vec_.begin(), address_vec_.end(), u64_addr) == address_vec_.end()) { - return false; - } - } - + if (this->addresses_ != nullptr && !ble_device_base::mac_in_table(this->addresses_, device.address_uint64())) + return false; this->trigger(device); return true; } protected: - std::vector address_vec_; + const uint64_t *addresses_{nullptr}; }; class BLEServiceDataAdvertiseTrigger final : public Trigger, public ESPBTDeviceListener { diff --git a/tests/component_tests/bk72xx_ble_tracker/test_automations_codegen.py b/tests/component_tests/bk72xx_ble_tracker/test_automations_codegen.py index 3f5280d965..2fb1881d1f 100644 --- a/tests/component_tests/bk72xx_ble_tracker/test_automations_codegen.py +++ b/tests/component_tests/bk72xx_ble_tracker/test_automations_codegen.py @@ -22,7 +22,11 @@ def test_trigger_codegen( main_cpp = generate_main(component_config_path("test_automations.yaml")) # on_ble_advertise: multi-mac filter (two addresses in one initializer list) - assert "set_addresses({0xAC3743775F4CULL, 0x112233445566ULL})" in main_cpp + assert ( + "static constexpr uint64_t ble_mac_filter[] PROGMEM = " + "{0xAC3743775F4CULL, 0x112233445566ULL, 0};" in main_cpp + ) + assert "set_addresses(ble_mac_filter)" in main_cpp # 128-bit service uuid goes out reversed (BLE wire order); single-mac filter assert ( "set_service_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," diff --git a/tests/component_tests/esp32_ble_tracker/config/mac_filter.yaml b/tests/component_tests/esp32_ble_tracker/config/mac_filter.yaml new file mode 100644 index 0000000000..309476a509 --- /dev/null +++ b/tests/component_tests/esp32_ble_tracker/config/mac_filter.yaml @@ -0,0 +1,27 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +logger: + +esp32_ble_tracker: + on_ble_advertise: + - mac_address: + - AC:37:43:77:5F:4C + - 11:22:33:44:55:66 + then: + - logger.log: one + - mac_address: + - AC:37:43:77:5F:4C + - 11:22:33:44:55:66 + then: + - logger.log: two + - then: + - logger.log: any + - mac_address: [] + then: + - logger.log: empty diff --git a/tests/component_tests/esp32_ble_tracker/test_mac_filter.py b/tests/component_tests/esp32_ble_tracker/test_mac_filter.py new file mode 100644 index 0000000000..45805debc8 --- /dev/null +++ b/tests/component_tests/esp32_ble_tracker/test_mac_filter.py @@ -0,0 +1,38 @@ +"""Tests for the esp32_ble_tracker on_ble_advertise MAC filter codegen.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome import config_validation as cv +from esphome.components.ble_device_base.automation import MAC_FILTER_LIST + + +def test_mac_filters_share_one_progmem_table( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Identical MAC lists share one flash table ended by 0; no or an empty filter sets none.""" + main_cpp = generate_main(component_config_path("mac_filter.yaml")) + + assert ( + "static constexpr uint64_t ble_mac_filter[] PROGMEM = " + "{0xAC3743775F4CULL, 0x112233445566ULL, 0};" in main_cpp + ) + assert main_cpp.count("set_addresses(ble_mac_filter);") == 2 + assert main_cpp.count("set_addresses(") == 2 + + +def test_zero_mac_is_rejected() -> None: + """00:00:00:00:00:00 ends the flash table, so it cannot be a filter entry.""" + assert MAC_FILTER_LIST(["11:22:33:44:55:66"])[0].parts == ( + 0x11, + 0x22, + 0x33, + 0x44, + 0x55, + 0x66, + ) + with pytest.raises(cv.Invalid): + MAC_FILTER_LIST(["00:00:00:00:00:00"]) diff --git a/tests/component_tests/ln882h_ble_tracker/test_automations_codegen.py b/tests/component_tests/ln882h_ble_tracker/test_automations_codegen.py index c1ef283ea3..3d275e479e 100644 --- a/tests/component_tests/ln882h_ble_tracker/test_automations_codegen.py +++ b/tests/component_tests/ln882h_ble_tracker/test_automations_codegen.py @@ -17,7 +17,11 @@ def test_trigger_codegen( main_cpp = generate_main(component_config_path("test_automations.yaml")) # on_ble_advertise: multi-mac filter (two addresses in one initializer list) - assert "set_addresses({0xAC3743775F4CULL, 0x112233445566ULL})" in main_cpp + assert ( + "static constexpr uint64_t ble_mac_filter[] PROGMEM = " + "{0xAC3743775F4CULL, 0x112233445566ULL, 0};" in main_cpp + ) + assert "set_addresses(ble_mac_filter)" in main_cpp # 128-bit service uuid goes out reversed (BLE wire order); single-mac filter assert ( "set_service_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," diff --git a/tests/component_tests/rp2_ble_tracker/test_automations_codegen.py b/tests/component_tests/rp2_ble_tracker/test_automations_codegen.py index 1106fd01eb..6014e9c285 100644 --- a/tests/component_tests/rp2_ble_tracker/test_automations_codegen.py +++ b/tests/component_tests/rp2_ble_tracker/test_automations_codegen.py @@ -19,8 +19,12 @@ def test_trigger_codegen( ) -> None: main_cpp = generate_main(component_config_path("test_automations.yaml")) - # on_ble_advertise: multi-mac filter (two addresses in one initializer list) - assert "set_addresses({0xAC3743775F4CULL, 0x112233445566ULL})" in main_cpp + # on_ble_advertise: multi-mac filter, a shared flash table ended by 0 + assert ( + "static constexpr uint64_t ble_mac_filter[] PROGMEM = " + "{0xAC3743775F4CULL, 0x112233445566ULL, 0};" in main_cpp + ) + assert "set_addresses(ble_mac_filter)" in main_cpp # 128-bit service uuid goes out reversed (BLE wire order); single-mac filter assert ( "set_service_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," From 082a0b746ae6d518a7ad0da73ab24bfdd4c574f4 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Tue, 6 Oct 2026 02:45:15 +0200 Subject: [PATCH 186/221] [lvgl] Remove redundant __aexit__ override (#20201) --- esphome/components/lvgl/lvcode.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/esphome/components/lvgl/lvcode.py b/esphome/components/lvgl/lvcode.py index d6d1aaa414..feffac28b3 100644 --- a/esphome/components/lvgl/lvcode.py +++ b/esphome/components/lvgl/lvcode.py @@ -209,9 +209,6 @@ class LvContext(LambdaContext): def __init__(self): super().__init__(parameters=LVGL_COMP_ARG) - async def __aexit__(self, exc_type, exc_val, exc_tb): - await super().__aexit__(exc_type, exc_val, exc_tb) - def add(self, expression: Expression | Statement): cg.add(expression) return expression From 6191b7553eed8a5e1de1d7a384927e3016d046c2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 5 Oct 2026 21:07:37 -0500 Subject: [PATCH 187/221] [light] Cover get_effects() as published configs use it in the component test (#20203) --- tests/components/light/common.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/components/light/common.yaml b/tests/components/light/common.yaml index 3cf2c59223..64406afda8 100644 --- a/tests/components/light/common.yaml +++ b/tests/components/light/common.yaml @@ -28,6 +28,21 @@ esphome: ESP_LOGI("test", "Name length: %d", (int) name.size()); } + # get_effects() shapes used by published configs and external components + - lambda: |- + const auto &effects = id(test_monochromatic_light).get_effects(); + auto &same = id(test_monochromatic_light).get_effects(); + uint32_t total = effects.size(); + for (auto *effect : effects) { + ESP_LOGD("test", "Effect %s", effect->get_name().c_str()); + } + if (total > 0) { + ESP_LOGD("test", "First %s", effects.at(0)->get_name().c_str()); + // Raw pointer iterators on purpose: external code binds std::find's result to `const auto *` + auto *it = std::find(same.begin(), same.end(), effects[0]); + ESP_LOGD("test", "Index %d", (int) (it - same.begin())); + } + # Test LightState::get_effect_name() returns StringRef - lambda: |- // Test LightState::get_effect_name() returns StringRef From 718b6651fec7607e016fb106f1e671d39f0164fb Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Tue, 6 Oct 2026 04:20:28 +0200 Subject: [PATCH 188/221] [ci] Run pylint when its version, dependencies or configuration change (#20202) --- script/determine-jobs.py | 28 ++++++++++++++++++++-------- tests/script/test_determine_jobs.py | 4 ++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 9c8f3cd3b4..9d146c94a0 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -31,7 +31,7 @@ The CI workflow uses this information to: - Skip or run integration tests - Skip or run clang-tidy (and whether to do a full scan) - Skip or run clang-format -- Skip or run Python linters (ruff, flake8, pylint, pyupgrade) +- Skip or run pylint - Skip or run downstream esphome/device-builder tests against the PR's Python code - Determine which components to test individually - Decide how to split component tests (if there are many) @@ -403,21 +403,33 @@ def should_run_clang_format(branch: str | None = None) -> bool: return _any_changed_file_endswith(branch, CPP_FILE_EXTENSIONS) +# Linter pins, dependencies and linter config can change findings without a .py change. +PYTHON_LINTERS_TRIGGER_FILES = frozenset( + { + "requirements.txt", + "requirements_test.txt", + "pyproject.toml", + } +) + + def should_run_python_linters(branch: str | None = None) -> bool: - """Determine if Python linters (ruff, flake8, pylint, pyupgrade) should run based on changed files. + """Determine if the pylint job should run based on changed files. - This function is used by the CI workflow to skip Python linting checks when no Python files - have changed, saving CI time and resources. - - Python linters will run when any Python source files have changed. + The CI workflow skips pylint unless a Python file or a file in + PYTHON_LINTERS_TRIGGER_FILES has changed. ruff, flake8 and pyupgrade run in + lint-format, which this flag does not gate. Args: branch: Branch to compare against. If None, uses default. Returns: - True if Python linters should run, False otherwise. + True if pylint should run, False otherwise. """ - return _any_changed_file_endswith(branch, PYTHON_FILE_EXTENSIONS) + return any( + file.endswith(PYTHON_FILE_EXTENSIONS) or file in PYTHON_LINTERS_TRIGGER_FILES + for file in changed_files(branch) + ) # Files outside esphome/**/*.py whose changes can affect `import esphome.__main__` diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index da688703fc..7e9b09b2b9 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -907,6 +907,10 @@ def test_should_run_clang_tidy_with_branch() -> None: (["esphome/core.py"], True), (["script/test.py"], True), (["esphome/test.pyi"], True), # .pyi files should trigger + (["requirements_test.txt"], True), + (["requirements.txt"], True), + (["pyproject.toml"], True), + (["requirements_dev.txt"], False), (["README.md"], False), ([], False), ], From 6002c28a8d0c92f22bd548c357e1c3510ad0ecaa Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:38:16 +0200 Subject: [PATCH 189/221] [host] Ignore SIGPIPE so a write to a closed peer fails with EPIPE (#20228) --- esphome/components/host/core.cpp | 2 + .../fixtures/host_write_after_peer_close.yaml | 40 +++++++++++++ .../test_host_write_after_peer_close.py | 57 +++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 tests/integration/fixtures/host_write_after_peer_close.yaml create mode 100644 tests/integration/test_host_write_after_peer_close.py diff --git a/esphome/components/host/core.cpp b/esphome/components/host/core.cpp index 9292cd77f6..f753534efd 100644 --- a/esphome/components/host/core.cpp +++ b/esphome/components/host/core.cpp @@ -90,6 +90,8 @@ int main(int argc, char **argv) { // Install signal handlers for graceful shutdown (flushes preferences to disk) std::signal(SIGINT, signal_handler); std::signal(SIGTERM, signal_handler); + // A write to a closed peer must fail with EPIPE, as on the embedded stacks, not kill the process + std::signal(SIGPIPE, SIG_IGN); esphome::host::setup_preferences(); setup(); diff --git a/tests/integration/fixtures/host_write_after_peer_close.yaml b/tests/integration/fixtures/host_write_after_peer_close.yaml new file mode 100644 index 0000000000..4fa89907e3 --- /dev/null +++ b/tests/integration/fixtures/host_write_after_peer_close.yaml @@ -0,0 +1,40 @@ +esphome: + name: host-write-after-close + +host: + +api: + actions: + - action: write_after_close + variables: + port: int + then: + - lambda: |- + // A second connection to the test server carries the go signal. + auto sync = socket::socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_storage addr; + socklen_t len = socket::set_sockaddr(reinterpret_cast(&addr), sizeof(addr), "127.0.0.1", port); + struct timeval timeout = {10, 0}; + sync->setsockopt(SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + uint8_t go; + // The loop stays blocked, so the link cannot read the close before the writes. + if (sync->connect(reinterpret_cast(&addr), len) != 0 || sync->read(&go, 1) != 1) { + ESP_LOGE("test", "No go signal"); + return; + } + // The first write draws a reset from the closed peer, a later one fails with EPIPE. + uint32_t start = millis(); + while (id(tcp_uart_1).is_connected() && millis() - start < 5000) { + id(tcp_uart_1).write_byte('b'); + id(tcp_uart_1).flush(); + delay(1); + } + +logger: + level: INFO + +tcp_uart: + - id: tcp_uart_1 + host: 127.0.0.1 + port: 18126 + reconnect_interval: 60s diff --git a/tests/integration/test_host_write_after_peer_close.py b/tests/integration/test_host_write_after_peer_close.py new file mode 100644 index 0000000000..3827459b48 --- /dev/null +++ b/tests/integration/test_host_write_after_peer_close.py @@ -0,0 +1,57 @@ +"""A host build must survive a write to a peer that has closed the connection. + +The device writes until the link drops; once the closed peer has answered with +a reset, a write must fail with EPIPE instead of SIGPIPE killing the process. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_host_write_after_peer_close( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + unused_tcp_port_factory: Callable[[], int], +) -> None: + port = unused_tcp_port_factory() + yaml_config = yaml_config.replace("port: 18126", f"port: {port}") + + lines = LineWaiter() + peers: asyncio.Queue[asyncio.StreamWriter] = asyncio.Queue() + + async def handle(_: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + peers.put_nowait(writer) + + server = await asyncio.start_server(handle, "127.0.0.1", port) + try: + async with ( + run_compiled(yaml_config, line_callback=lines.callback), + api_client_connected() as client, + ): + peer = await asyncio.wait_for(peers.get(), 15.0) + _, services = await client.list_entities_services() + action = next(s for s in services if s.name == "write_after_close") + + await client.execute_service(action, {"port": port}) + # The action opens a second connection and blocks the loop until it gets a byte. + sync = await asyncio.wait_for(peers.get(), 15.0) + peer.close() + await peer.wait_closed() + sync.write(b"g") + sync.close() + + # EPIPE (32 on Linux and macOS) must come back as an errno. + await lines.wait_for("Connection lost: 32") + assert await client.device_info() is not None + finally: + server.close() + await server.wait_closed() From b7ddd1daa2f88c6bed7a4085788e09014852a47d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 6 Oct 2026 08:03:20 -0500 Subject: [PATCH 190/221] [climate] Cover custom mode reads and runtime sets in the component test (#20208) --- tests/components/template/common-base.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index ccef9ded05..f87a4e0320 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -275,6 +275,28 @@ button: name: "Template Button" on_press: - logger.log: Button Pressed + - platform: template + name: "Climate Custom Modes Lambdas" + on_press: + - lambda: |- + // How external components and configs read and set climate custom modes + auto traits = id(template_climate_custom_modes).get_traits(); + const auto &fan_modes = traits.get_supported_custom_fan_modes(); + for (const auto *mode : fan_modes) + ESP_LOGD("test", "%s", mode); + for (const auto &preset : traits.get_supported_custom_presets()) + ESP_LOGD("test", "%s", preset); + for (const char *mode : traits.get_supported_custom_fan_modes()) + ESP_LOGD("test", "%s", mode); + std::vector presets{"eco_plus", "max"}; + id(template_climate_custom_modes).set_supported_custom_presets(presets); + id(template_climate_custom_modes).set_supported_custom_presets({"eco_plus"}); + id(template_climate_custom_modes).set_supported_custom_fan_modes({}); + std::vector new_fan_modes{"turbo", "silent"}; + id(template_climate_custom_modes).set_supported_custom_fan_modes(new_fan_modes); + static const char *const MODES[] = {"turbo", "silent"}; + id(template_climate_custom_modes).set_supported_custom_fan_modes(MODES); + id(template_climate_custom_modes).set_supported_custom_presets(MODES); cover: - platform: template From 9b1a75b0ed1a75c97c84f605c842a362ac417238 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 6 Oct 2026 08:03:58 -0500 Subject: [PATCH 191/221] [fan] Cover preset mode reads and runtime sets in the component test (#20209) --- tests/components/fan/common.yaml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/components/fan/common.yaml b/tests/components/fan/common.yaml index 12c7b6c872..4c6a6b8a63 100644 --- a/tests/components/fan/common.yaml +++ b/tests/components/fan/common.yaml @@ -66,9 +66,26 @@ binary_sensor: } return false; -# Exercise fan.turn_on with various field combinations so the -# register_apply_action codegen paths get build coverage. button: + # Read and set preset modes the way external fan components do + - platform: template + name: "Fan Preset Modes Lambdas" + on_press: + - lambda: |- + // How external components and configs read and set fan preset modes + auto traits = id(test_fan).get_traits(); + const auto &modes = traits.supported_preset_modes(); + for (const auto &mode : modes) + ESP_LOGD("test", "%s", mode); + for (const char *mode : traits.supported_preset_modes()) + ESP_LOGD("test", "%s", mode); + std::vector runtime_modes{"Eco", "Turbo"}; + id(test_fan).set_supported_preset_modes(runtime_modes); + id(test_fan).set_supported_preset_modes({"Eco"}); + id(test_fan).set_supported_preset_modes({}); + + # Exercise fan.turn_on with various field combinations so the + # register_apply_action codegen paths get build coverage. - platform: template name: "Fan Speed Only" on_press: From 29090d50a7cb988a712ad7fda9c032cf4c0be849 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 6 Oct 2026 08:04:37 -0500 Subject: [PATCH 192/221] [event] Cover how configs and components read and set event types in the component test (#20216) --- tests/components/event/common.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/components/event/common.yaml b/tests/components/event/common.yaml index 827f1ebb52..14ec46c4df 100644 --- a/tests/components/event/common.yaml +++ b/tests/components/event/common.yaml @@ -19,6 +19,12 @@ event: ESP_LOGD("test", "Event type: %.*s", (int) event_type.size(), event_type.c_str()); } + - platform: template + name: Other Event + id: other_event + event_types: + - template_event_type1 + button: - platform: template name: Trigger Event @@ -30,3 +36,21 @@ button: id: some_event event_type: !lambda |- return id(some_event).has_event() ? "template_event_type2" : "template_event_type1"; + + - platform: template + name: Event Types Lambdas + on_press: + - lambda: |- + // How external components and configs read and set event types + const auto &types = id(some_event).get_event_types(); + for (const char *type : types) + ESP_LOGD("test", "%s", type); + if (types.empty()) { + ESP_LOGD("test", "no types"); + } else if (types.size() > 1) { + ESP_LOGD("test", "%s %s", types[0], types.at(1)); + const auto *it = std::find(types.begin(), types.end(), types[1]); + ESP_LOGD("test", "Index %d", (int) (it - types.begin())); + } + id(other_event).set_event_types(id(some_event).get_event_types()); + id(other_event).set_event_types({"other_a", "other_b"}); From a69c648c6fa8ed071e9d6e902e7023799d8ee329 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 6 Oct 2026 08:04:52 -0500 Subject: [PATCH 193/221] [neopixelbus] Cover addressable_color_wipe with a random color on ESP8266 (#20221) --- tests/components/neopixelbus/test.esp8266-ard.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/components/neopixelbus/test.esp8266-ard.yaml b/tests/components/neopixelbus/test.esp8266-ard.yaml index 2c1f16a38c..2ae0477096 100644 --- a/tests/components/neopixelbus/test.esp8266-ard.yaml +++ b/tests/components/neopixelbus/test.esp8266-ard.yaml @@ -10,6 +10,16 @@ light: name: Flicker Effect With Custom Values update_interval: 16ms intensity: 5% + - addressable_color_wipe: + name: Color Wipe With Random + colors: + - red: 100% + green: 0% + blue: 0% + num_leds: 2 + gradient: true + - random: true + num_leds: 1 type: GRBW variant: SK6812 method: esp8266_uart From b63794b53c8310b42e0d2a77a7cbfd482be81727 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Tue, 6 Oct 2026 17:00:50 +0200 Subject: [PATCH 194/221] [web_server_idf] Respect explicit HTTP header limit (#18657) Co-authored-by: jeroen85 <26403565+jeroen85@users.noreply.github.com> Co-authored-by: Jeroen Jansen --- esphome/components/esp32/__init__.py | 7 ++-- esphome/components/web_server_idf/__init__.py | 4 +-- .../web_server/test_httpd_sdkconfig.py | 32 +++++++++++++++++++ .../web_server_header_limit_override.yaml | 16 ++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 tests/component_tests/web_server/test_httpd_sdkconfig.py create mode 100644 tests/component_tests/web_server/web_server_header_limit_override.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index be6b2c3266..ce1a66a4d2 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -742,9 +742,10 @@ def is_idf_sdkconfig_option_enabled(name: str) -> bool: def set_idf_sdkconfig_default(name: str, value: SdkconfigValueType) -> None: """Set an sdkconfig option unless it is already set. - For the FINAL priority reconcile jobs: they run after every to_code, - including the user's sdkconfig_options, and must not override an - existing value. + User sdkconfig_options take precedence regardless of to_code order: + esp32.to_code applies them unconditionally, and this helper preserves + values that are already set. FINAL priority reconcile jobs use the same + guard because they run after every to_code, including the user's options. """ if name not in CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]: add_idf_sdkconfig_option(name, value) diff --git a/esphome/components/web_server_idf/__init__.py b/esphome/components/web_server_idf/__init__.py index 5a400dfbf3..5eac63212f 100644 --- a/esphome/components/web_server_idf/__init__.py +++ b/esphome/components/web_server_idf/__init__.py @@ -1,6 +1,6 @@ from esphome.components.esp32 import ( - add_idf_sdkconfig_option, include_builtin_idf_component, + set_idf_sdkconfig_default, ) from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv @@ -16,7 +16,7 @@ CONFIG_SCHEMA = cv.All( async def to_code(config: ConfigType) -> None: # Increase the maximum supported size of headers section in HTTP request packet to be processed by the server - add_idf_sdkconfig_option("CONFIG_HTTPD_MAX_REQ_HDR_LEN", 1024) + set_idf_sdkconfig_default("CONFIG_HTTPD_MAX_REQ_HDR_LEN", 1024) # Re-enable esp-tls (excluded by default to save compile time); # web_server_idf.cpp includes for digest auth include_builtin_idf_component("esp-tls") diff --git a/tests/component_tests/web_server/test_httpd_sdkconfig.py b/tests/component_tests/web_server/test_httpd_sdkconfig.py new file mode 100644 index 0000000000..d3b0acb384 --- /dev/null +++ b/tests/component_tests/web_server/test_httpd_sdkconfig.py @@ -0,0 +1,32 @@ +"""Tests for ESP-IDF web server sdkconfig defaults.""" + +from collections.abc import Callable + +from esphome.components.esp32 import RawSdkconfigValue +from esphome.components.esp32.const import KEY_ESP32, KEY_SDKCONFIG_OPTIONS +from esphome.core import CORE + + +def test_web_server_idf_uses_default_header_limit( + generate_main: Callable[[str], str], +) -> None: + """The web server retains its existing 1024-byte default.""" + generate_main("tests/component_tests/web_server/web_server_no_auth.yaml") + + assert ( + CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]["CONFIG_HTTPD_MAX_REQ_HDR_LEN"] + == 1024 + ) + + +def test_web_server_idf_respects_explicit_header_limit( + generate_main: Callable[[str], str], +) -> None: + """An explicit sdkconfig option takes precedence over the component default.""" + generate_main( + "tests/component_tests/web_server/web_server_header_limit_override.yaml" + ) + + value = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]["CONFIG_HTTPD_MAX_REQ_HDR_LEN"] + assert isinstance(value, RawSdkconfigValue) + assert value.value == "4096" diff --git a/tests/component_tests/web_server/web_server_header_limit_override.yaml b/tests/component_tests/web_server/web_server_header_limit_override.yaml new file mode 100644 index 0000000000..eb1c8a87dd --- /dev/null +++ b/tests/component_tests/web_server/web_server_header_limit_override.yaml @@ -0,0 +1,16 @@ +--- +esphome: + name: test + +esp32: + board: nodemcu-32s + framework: + type: esp-idf + sdkconfig_options: + CONFIG_HTTPD_MAX_REQ_HDR_LEN: "4096" + +wifi: + ssid: MySSID + password: password1 + +web_server: From 1f53ccb5725601a4088f02ae1ee0f5410fc163b2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 6 Oct 2026 10:04:45 -0500 Subject: [PATCH 195/221] [api] Cover Home Assistant action fields in the api test config (#20236) --- tests/components/api/common-base.yaml | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index f55e7b176d..1513d537c2 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -149,6 +149,38 @@ api: args: - error.c_str() - name.c_str() + # Constant and lambda fields mixed in one action, with lambdas of several + # return types reading the trigger args, and a templated action name + - action: test_homeassistant_fields + variables: + room: string + level: int + then: + - homeassistant.action: + action: !lambda 'return std::string("light.") + (level > 0 ? "turn_on" : "turn_off");' + data: + entity_id: light.living_room + brightness: !lambda 'return level;' + data_template: + message: "{{ room }} is at {{ level }}" + variables: + room: !lambda 'return room.c_str();' + level: !lambda 'static char buf[12]; snprintf(buf, sizeof(buf), "%d", level); return buf;' + - homeassistant.action: + action: notify.html5 + data: + message: Button was pressed + # Same constant fields as above, so codegen can share them + - homeassistant.action: + action: notify.html5 + data: + message: Button was pressed + - homeassistant.event: + event: esphome.room_level + data: + room: !lambda 'return room;' + ratio: !lambda 'return level / 100.0f;' + - homeassistant.tag_scanned: !lambda 'return room;' # Test ContinuationAction (IfAction with then/else branches) - action: test_if_action variables: From e66ef45df7fd390cd7663f85d426edec96084b2b Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:08:01 +0200 Subject: [PATCH 196/221] [zigbee] add missing tests (#20234) --- tests/components/zigbee/common.yaml | 4 ++++ tests/components/zigbee/test.nrf52-mcumgr.yaml | 2 +- tests/components/zigbee/test.nrf52-xiao-ble.yaml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/components/zigbee/common.yaml b/tests/components/zigbee/common.yaml index 4518e39060..ce59f4a6ae 100644 --- a/tests/components/zigbee/common.yaml +++ b/tests/components/zigbee/common.yaml @@ -14,9 +14,12 @@ sensor: name: "Analog 1" lambda: return 10.0; accuracy_decimals: 0 + device_class: temperature + unit_of_measurement: "°C" - platform: template name: "Analog 2" lambda: return 11.0; + unit_of_measurement: "°C" - platform: template name: "Analog 3" lambda: return 12.0; @@ -41,6 +44,7 @@ number: min_value: 2 max_value: 100 step: 1 + unit_of_measurement: "°C" time: - platform: zigbee diff --git a/tests/components/zigbee/test.nrf52-mcumgr.yaml b/tests/components/zigbee/test.nrf52-mcumgr.yaml index a81feea069..cdead0db90 100644 --- a/tests/components/zigbee/test.nrf52-mcumgr.yaml +++ b/tests/components/zigbee/test.nrf52-mcumgr.yaml @@ -1,4 +1,4 @@ -<<: !include common_nrf52.yaml +<<: !include common.yaml zigbee: router: true diff --git a/tests/components/zigbee/test.nrf52-xiao-ble.yaml b/tests/components/zigbee/test.nrf52-xiao-ble.yaml index acfbc9e996..64134e536d 100644 --- a/tests/components/zigbee/test.nrf52-xiao-ble.yaml +++ b/tests/components/zigbee/test.nrf52-xiao-ble.yaml @@ -1,4 +1,4 @@ -<<: !include common_nrf52.yaml +<<: !include common.yaml zigbee: wipe_on_boot: once From fef3804aa5400a7f4ac003002141df680e0d664c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 6 Oct 2026 10:09:47 -0500 Subject: [PATCH 197/221] [espidf] Fix V1 ECDSA signed builds when skipping the bootloader (#20244) --- esphome/build_gen/espidf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 42a1de5a4b..e7a2dc1536 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -160,7 +160,7 @@ def _cmake_quote(value: str) -> str: # CONFIG_APP_BUILD_BOOTLOADER is hidden and force-selected, so it can only be # cleared at the CMake level (the same state IDF's RAM-app build type uses). -# The macro is IDF's __build_process_project_includes plus two added lines; +# The macro is IDF's __build_process_project_includes plus a few added lines; # the flag is ignored and the bootloader builds as usual if IDF changes it. IDF_BOOTLOADER_OVERRIDE = """\ # ESPHome bootloader skip switch; see esphome/espidf/toolchain.py. @@ -173,6 +173,9 @@ if(ESPHOME_SKIP_BOOTLOADER) # skipped) bootloader project_include leaks; keep it defined, or # its empty TARGET_SRC_NAME sends file(GLOB_RECURSE) across /. idf_build_get_property(idf_target IDF_TARGET) + # partition_table's V1 ECDSA signing reads this key, which the + # skipped bootloader project_include also sets. + get_filename_component(SECURE_BOOT_SIGNING_KEY "${CONFIG_SECURE_BOOT_SIGNING_KEY}" ABSOLUTE BASE_DIR "${project_dir}") idf_build_get_property(build_properties __BUILD_PROPERTIES) foreach(build_property ${build_properties}) idf_build_get_property(val ${build_property}) @@ -198,6 +201,10 @@ endif() BOOTLOADER_OVERRIDE_ADDED_LINES = ( 'set(CONFIG_APP_BUILD_BOOTLOADER "")', "idf_build_get_property(idf_target IDF_TARGET)", + ( + "get_filename_component(SECURE_BOOT_SIGNING_KEY" + ' "${CONFIG_SECURE_BOOT_SIGNING_KEY}" ABSOLUTE BASE_DIR "${project_dir}")' + ), ) _MACRO = re.compile( From 74d0a90308357f9f58a199888f2a0926ffd24341 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:11:25 +1100 Subject: [PATCH 198/221] [esphome] Revise warning text for password use (#20230) --- esphome/components/esphome/ota/__init__.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index ffd7c50949..0cb2657ea1 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -133,20 +133,12 @@ def ota_esphome_final_validate(config: ConfigType) -> None: _resolve_encryption_key(encryption_conf, api_conf) elif CONF_PASSWORD in ota_conf and static_encryption_key(api_conf) is not None: _LOGGER.warning( - "'%s' %s wastes significant flash and RAM (about 3.5 KB and 60 " - "bytes plus the password on the heap): the device already offers " - "encryption with the '%s' %s %s, which authenticates any uploader " - "that takes it, and a password only matters for uploaders without " - "encryption support; remove '%s' and add '%s' under '%s' so " - "uploads use the key and encryption is required", + "'%s' %s wastes significant flash and RAM; " + "using '%s' instead is recommended - " + "see https://esphome.io/components/ota/esphome/#configuration-variables", CONF_OTA, CONF_PASSWORD, - CONF_API, CONF_ENCRYPTION, - CONF_KEY, - CONF_PASSWORD, - CONF_ENCRYPTION, - CONF_OTA, ) elif ( CONF_PASSWORD in ota_conf From fb20966353e1b004a11cd837fc622220509990d0 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:16:57 +0100 Subject: [PATCH 199/221] [wireguard] Restart the connection while the peer is offline so the endpoint is re-resolved (#20231) --- esphome/components/wireguard/wireguard.cpp | 5 +++++ esphome/components/wireguard/wireguard.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/esphome/components/wireguard/wireguard.cpp b/esphome/components/wireguard/wireguard.cpp index fc06569fba..1fd610f4e8 100644 --- a/esphome/components/wireguard/wireguard.cpp +++ b/esphome/components/wireguard/wireguard.cpp @@ -101,8 +101,13 @@ void Wireguard::update() { if (this->wg_peer_offline_time_ == 0) { ESP_LOGW(TAG, LOGMSG_PEER_STATUS, LOGMSG_OFFLINE, latest_handshake.c_str()); this->wg_peer_offline_time_ = millis(); + this->wg_reconnect_time_ = this->wg_peer_offline_time_; } else if (this->enabled_) { ESP_LOGD(TAG, LOGMSG_PEER_STATUS, LOGMSG_OFFLINE, latest_handshake.c_str()); + if (millis() - this->wg_reconnect_time_ >= RECONNECT_INTERVAL_MS) { + this->wg_reconnect_time_ = millis(); + this->stop_connection_(); + } this->start_connection_(); } diff --git a/esphome/components/wireguard/wireguard.h b/esphome/components/wireguard/wireguard.h index 68885f3a35..faf63cd444 100644 --- a/esphome/components/wireguard/wireguard.h +++ b/esphome/components/wireguard/wireguard.h @@ -140,6 +140,7 @@ class Wireguard final : public PollingComponent { /// The last time the remote peer become offline. uint32_t wg_peer_offline_time_ = 0; + uint32_t wg_reconnect_time_ = 0; /** \brief The latest saved handshake. * @@ -159,6 +160,7 @@ void resume_wdt(); /// Size of buffer required for mask_key_to: 5 chars + "[...]=" + null = 12 static constexpr size_t MASK_KEY_BUFFER_SIZE = 12; +static constexpr uint32_t RECONNECT_INTERVAL_MS = 30000; /// Strip most part of the key only for secure printing void mask_key_to(char *buffer, size_t len, const char *key); From f670993fe92313b523afddabfadf0ba702259100 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:40:53 +0100 Subject: [PATCH 200/221] [ethernet] Pin the IDF version in the KSZ8851SNL test fixtures (#20245) --- tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml | 2 ++ tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml b/tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml index 7c8f333684..eb83960f67 100644 --- a/tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml +++ b/tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml @@ -3,6 +3,8 @@ esphome: esp32: board: esp32dev + framework: + version: 5.5.5 ethernet: id: eth_component diff --git a/tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml b/tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml index bc11c6db5b..3ca9c801b7 100644 --- a/tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml +++ b/tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml @@ -4,7 +4,7 @@ esphome: esp32: board: esp32dev framework: - version: 6.0.0 + version: 6.1.0 ethernet: id: eth_component From 980fed9e964676f563a61256f92ad8ca8c897712 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 6 Oct 2026 13:23:57 -0400 Subject: [PATCH 201/221] [sendspin] Require a type on the switch platform (#20189) --- .../components/sendspin/switch/__init__.py | 24 ++++++++++++------- tests/components/sendspin/common-switch.yaml | 1 + 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/esphome/components/sendspin/switch/__init__.py b/esphome/components/sendspin/switch/__init__.py index 63f5f7ad28..1f896640bf 100644 --- a/esphome/components/sendspin/switch/__init__.py +++ b/esphome/components/sendspin/switch/__init__.py @@ -1,7 +1,8 @@ import esphome.codegen as cg from esphome.components import switch +from esphome.components.const import CONF_ENABLED import esphome.config_validation as cv -from esphome.const import ENTITY_CATEGORY_CONFIG +from esphome.const import CONF_TYPE, ENTITY_CATEGORY_CONFIG from esphome.types import ConfigType from .. import CONF_SENDSPIN_ID, SendspinHub, sendspin_ns @@ -12,14 +13,19 @@ DEPENDENCIES = ["sendspin"] SendspinSwitch = sendspin_ns.class_("SendspinSwitch", switch.Switch, cg.Component) CONFIG_SCHEMA = cv.All( - switch.switch_schema( - SendspinSwitch, - block_inverted=True, - default_restore_mode="RESTORE_DEFAULT_ON", - entity_category=ENTITY_CATEGORY_CONFIG, - ) - .extend({cv.GenerateID(CONF_SENDSPIN_ID): cv.use_id(SendspinHub)}) - .extend(cv.COMPONENT_SCHEMA), + cv.typed_schema( + { + CONF_ENABLED: switch.switch_schema( + SendspinSwitch, + block_inverted=True, + default_restore_mode="RESTORE_DEFAULT_ON", + entity_category=ENTITY_CATEGORY_CONFIG, + ) + .extend({cv.GenerateID(CONF_SENDSPIN_ID): cv.use_id(SendspinHub)}) + .extend(cv.COMPONENT_SCHEMA), + }, + key=CONF_TYPE, + ), cv.only_on_esp32, ) diff --git a/tests/components/sendspin/common-switch.yaml b/tests/components/sendspin/common-switch.yaml index d332cb0dde..01e2f0b02b 100644 --- a/tests/components/sendspin/common-switch.yaml +++ b/tests/components/sendspin/common-switch.yaml @@ -4,3 +4,4 @@ packages: switch: - platform: sendspin name: "Sendspin Enabled" + type: enabled From cad670785b6db78db1ce7a9aea87c9b2381fd5b4 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:28:06 +0200 Subject: [PATCH 202/221] [socket] Add host tests for TcpClientLink and TcpListener (#20167) Co-authored-by: J. Nick Koston --- esphome/components/socket/tcp_client_link.cpp | 20 +-- tests/components/socket/__init__.py | 3 +- .../socket/test_ipv4_allow_host.cpp | 35 +++++ .../socket/test_tcp_client_link_host.cpp | 134 ++++++++++++++++++ .../socket/test_tcp_listener_host.cpp | 132 +++++++++++++++++ 5 files changed, 315 insertions(+), 9 deletions(-) create mode 100644 tests/components/socket/test_tcp_listener_host.cpp diff --git a/esphome/components/socket/tcp_client_link.cpp b/esphome/components/socket/tcp_client_link.cpp index 141695a4bb..9812f1835e 100644 --- a/esphome/components/socket/tcp_client_link.cpp +++ b/esphome/components/socket/tcp_client_link.cpp @@ -17,19 +17,23 @@ static constexpr uint32_t CONNECT_TIMEOUT_MS = 10000; // Non-blocking options and TCP keepalive for a bridged stream socket. // Keepalive is best-effort: the raw lwIP implementation (ESP8266, RP2040) // rejects it, so a half-open link there is only detected by a failed write. -static void set_stream_options(Socket *sock) { +static void set_stream_options(Socket *sock, const char *tag) { int yes = 1; + // Fails only on an invalid descriptor, or on raw lwIP after a peer reset that the next read() reports. sock->setblocking(false); - sock->setsockopt(IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); - sock->setsockopt(SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)); + int err = sock->setsockopt(IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); + err |= sock->setsockopt(SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)); #ifdef TCP_KEEPIDLE int idle = 30; int interval = 10; int count = 3; - sock->setsockopt(IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle)); - sock->setsockopt(IPPROTO_TCP, TCP_KEEPINTVL, &interval, sizeof(interval)); - sock->setsockopt(IPPROTO_TCP, TCP_KEEPCNT, &count, sizeof(count)); + err |= sock->setsockopt(IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle)); + err |= sock->setsockopt(IPPROTO_TCP, TCP_KEEPINTVL, &interval, sizeof(interval)); + err |= sock->setsockopt(IPPROTO_TCP, TCP_KEEPCNT, &count, sizeof(count)); #endif + if (err != 0) { + ESP_LOGV(tag, "Nodelay/keepalive not fully applied"); + } } void TcpClientLink::begin(const char *tag) { @@ -84,7 +88,7 @@ void TcpClientLink::try_connect_() { this->drop_(LOG_STR("Connect failed"), errno); return; } - set_stream_options(this->sock_.get()); + set_stream_options(this->sock_.get(), this->tag_); // Starts the pending-connect clock that poll() times out against. this->note_attempt(); // An immediate success is reported by the next poll(); poll_connect() sees it writable. @@ -95,7 +99,7 @@ void TcpClientLink::try_connect_() { void TcpClientLink::adopt(std::unique_ptr sock) { this->close(); - set_stream_options(sock.get()); + set_stream_options(sock.get(), this->tag_); this->sock_ = std::move(sock); this->connected_ = true; } diff --git a/tests/components/socket/__init__.py b/tests/components/socket/__init__.py index 6c1d2b1c08..3cd8cd839d 100644 --- a/tests/components/socket/__init__.py +++ b/tests/components/socket/__init__.py @@ -14,6 +14,7 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: await real_to_code(config) cg.add_define("USE_NETWORK_IPV6", True) # The gtests link against the filtered helper files. - socket_component.require_tcp_client_link() + socket_component.require_tcp_listener() + cg.add_define("USE_SOCKET_IPV4_ALLOW") manifest.to_code = to_code_testing diff --git a/tests/components/socket/test_ipv4_allow_host.cpp b/tests/components/socket/test_ipv4_allow_host.cpp index e4257b41c4..a8924de371 100644 --- a/tests/components/socket/test_ipv4_allow_host.cpp +++ b/tests/components/socket/test_ipv4_allow_host.cpp @@ -70,6 +70,41 @@ TEST(Ipv4Allow, InstancesKeepIndependentLists) { EXPECT_FALSE(first.allows(htonl(0x0A00002A))); } +TEST(Ipv4Allow, HostEntryMatchesOnlyThatAddress) { + static const Ipv4AllowEntry HOST[] = {{htonl(0xC0A8AF14), htonl(0xFFFFFFFF)}}; + Ipv4Allow list; + list.set(HOST, std::size(HOST)); + EXPECT_TRUE(allows_peer(list, "192.168.175.20")); + EXPECT_FALSE(allows_peer(list, "192.168.175.21")); + EXPECT_FALSE(allows_peer(list, "192.168.175.19")); +} + +TEST(Ipv4Allow, CatchAllAllowsEveryV4PeerOnly) { + static const Ipv4AllowEntry ANY[] = {{0, 0}}; + Ipv4Allow list; + list.set(ANY, std::size(ANY)); + EXPECT_TRUE(allows_peer(list, "0.0.0.0")); + EXPECT_TRUE(allows_peer(list, "255.255.255.255")); + EXPECT_TRUE(allows_peer(list, "::ffff:10.1.2.3")); + // Unlike an empty list, 0.0.0.0/0 still turns a native IPv6 peer away. + EXPECT_FALSE(allows_peer(list, "fe80::1")); +} + +TEST(Ipv4Allow, LastEntryOfAFullListMatches) { + // 255 is the schema's cap: 10.0.0.1/32 to 10.0.0.255/32. + static Ipv4AllowEntry full[255]; + for (uint32_t i = 0; i < std::size(full); i++) { + full[i] = {htonl(0x0A000001 + i), htonl(0xFFFFFFFF)}; + } + Ipv4Allow list; + list.set(full, std::size(full)); + EXPECT_EQ(list.size(), 255u); + EXPECT_TRUE(allows_peer(list, "10.0.0.1")); + EXPECT_TRUE(allows_peer(list, "10.0.0.255")); + EXPECT_FALSE(allows_peer(list, "10.0.1.0")); + EXPECT_FALSE(allows_peer(list, "10.0.0.0")); +} + } // namespace esphome::socket::testing #endif diff --git a/tests/components/socket/test_tcp_client_link_host.cpp b/tests/components/socket/test_tcp_client_link_host.cpp index 9a6d388b72..98be8190b0 100644 --- a/tests/components/socket/test_tcp_client_link_host.cpp +++ b/tests/components/socket/test_tcp_client_link_host.cpp @@ -1,15 +1,22 @@ #include +#include #include #include +#include #include +#include #include #include #include "esphome/components/socket/tcp_client_link.h" +#include "esphome/core/application.h" #ifdef USE_HOST +// Host only: ESP-IDF has no poll.h. +#include + namespace esphome::socket::testing { class LinkPeer { @@ -40,6 +47,133 @@ class LinkPeer { int peer_fd_{-1}; }; +// Sets the cached loop time the link's clock reads, as the main loop does. +static void set_loop_time(uint32_t now) { LoopBlockingGuard guard(nullptr, LOG_STR("test"), now); } + +class LinkUnderTest : public TcpClientLink { + public: + void set_socket(std::unique_ptr sock) { this->sock_ = std::move(sock); } + bool has_socket() const { return this->sock_ != nullptr; } + int fd() const { return this->sock_->get_fd(); } +}; + +class TcpClientLinkClock : public ::testing::Test { + protected: + void SetUp() override { + signal(SIGPIPE, SIG_IGN); + set_loop_time(1000); + } + void TearDown() override { + this->link_.close(); + if (this->peer_fd_ >= 0) { + ::close(this->peer_fd_); + } + set_loop_time(0); + } + // A connected socket whose send buffer is full selects as not writable, + // which poll_connect() reports as a connect still in progress. + void set_pending_socket() { + int fds[2]; + ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + this->peer_fd_ = fds[1]; + ASSERT_EQ(fcntl(fds[0], F_SETFL, fcntl(fds[0], F_GETFL, 0) | O_NONBLOCK), 0); + char fill[1024]{}; + while (::write(fds[0], fill, sizeof(fill)) > 0) { + } + ASSERT_EQ(errno, EAGAIN); + this->link_.set_socket(std::make_unique(fds[0])); + } + + LinkUnderTest link_; + int peer_fd_{-1}; +}; + +TEST_F(TcpClientLinkClock, PendingConnectTimesOutIntoBackoff) { + this->link_.begin("link_test"); + this->set_pending_socket(); + this->link_.note_attempt(); + set_loop_time(1000 + 9999); + this->link_.poll(); + ASSERT_TRUE(this->link_.has_socket()); + EXPECT_FALSE(this->link_.connected()); + set_loop_time(1000 + 10000); + this->link_.poll(); + EXPECT_FALSE(this->link_.has_socket()); + EXPECT_TRUE(this->link_.in_backoff()); +} + +TEST_F(TcpClientLinkClock, ConnectTimeoutFollowsALongerInterval) { + this->link_.set_reconnect_interval(20000); + this->link_.begin("link_test"); + this->set_pending_socket(); + this->link_.note_attempt(); + set_loop_time(1000 + 19999); + this->link_.poll(); + ASSERT_TRUE(this->link_.has_socket()); + set_loop_time(1000 + 20000); + this->link_.poll(); + EXPECT_FALSE(this->link_.has_socket()); +} + +TEST_F(TcpClientLinkClock, ResolveFailureBacksOff) { + // An IPv6 literal fails the IPv4 lookup without DNS. + this->link_.set_host("::1"); + this->link_.set_port(1); + this->link_.begin("link_test"); + this->link_.poll(); + ASSERT_FALSE(this->link_.in_backoff()); + // The failure is consumed by the next attempt and restarts the clock. + this->link_.poll(); + EXPECT_FALSE(this->link_.has_socket()); + EXPECT_TRUE(this->link_.in_backoff()); + set_loop_time(1000 + 4999); + EXPECT_TRUE(this->link_.in_backoff()); + set_loop_time(1000 + 5000); + EXPECT_FALSE(this->link_.in_backoff()); +} + +TEST_F(TcpClientLinkClock, RefusedConnectBacksOff) { + // A loopback port that was just free refuses the connect. + int probe = ::socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GE(probe, 0); + struct sockaddr_in addr {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + socklen_t len = sizeof(addr); + ASSERT_EQ(::bind(probe, reinterpret_cast(&addr), len), 0); + ASSERT_EQ(::getsockname(probe, reinterpret_cast(&addr), &len), 0); + ::close(probe); + + this->link_.set_host("127.0.0.1"); + this->link_.set_port(ntohs(addr.sin_port)); + this->link_.begin("link_test"); + // The refusal comes back from connect() itself or from a later poll. + this->link_.poll(); + if (this->link_.has_socket()) { + // Wait for the stack to finish the connect, then let poll() read the result. + struct pollfd pfd { + this->link_.fd(), POLLOUT, 0 + }; + ASSERT_EQ(::poll(&pfd, 1, 1000), 1); + this->link_.poll(); + } + EXPECT_FALSE(this->link_.connected()); + EXPECT_FALSE(this->link_.has_socket()); + EXPECT_TRUE(this->link_.in_backoff()); +} + +TEST_F(TcpClientLinkClock, BackoffSpansAMillisWrap) { + set_loop_time(UINT32_MAX - 999); + this->link_.begin("link_test"); + // begin() back-dates the clock so the first attempt is immediate. + EXPECT_FALSE(this->link_.in_backoff()); + this->link_.note_attempt(); + set_loop_time(3999); + EXPECT_TRUE(this->link_.in_backoff()); + set_loop_time(4000); + EXPECT_FALSE(this->link_.in_backoff()); +} + TEST(TcpClientLink, AdoptedSocketFlushesQueuedBytes) { LinkPeer p; ASSERT_TRUE(p.link_.connected()); diff --git a/tests/components/socket/test_tcp_listener_host.cpp b/tests/components/socket/test_tcp_listener_host.cpp new file mode 100644 index 0000000000..7c6e8392da --- /dev/null +++ b/tests/components/socket/test_tcp_listener_host.cpp @@ -0,0 +1,132 @@ +#include + +#include +#include +#include +#include +#include + +#include "esphome/components/socket/tcp_listener.h" + +#ifdef USE_HOST + +// Host only: ESP-IDF has no poll.h. +#include + +namespace esphome::socket::testing { + +class ListenerUnderTest : public TcpListener { + public: + void set_listen(std::unique_ptr sock) { this->listen_ = std::move(sock); } + bool listening() const { return this->listen_ != nullptr; } + void accept(TcpClientLink &link) { this->accept_(link); } +}; + +class TcpListenerAccept : public ::testing::Test { + protected: + void SetUp() override { + this->link_.set_port(1); + this->link_.begin("listener_test"); + this->listener_.begin("listener_test"); + } + void TearDown() override { + this->listener_.close(); + this->link_.close(); + if (this->client_fd_ >= 0) { + ::close(this->client_fd_); + } + } + // A non-blocking IPv4 listener on a free loopback port; returns the port. + uint16_t listen_on_loopback() { + int fd = ::socket(AF_INET, SOCK_STREAM, 0); + EXPECT_GE(fd, 0); + struct sockaddr_in addr {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + socklen_t len = sizeof(addr); + EXPECT_EQ(::bind(fd, reinterpret_cast(&addr), len), 0); + EXPECT_EQ(::listen(fd, 1), 0); + EXPECT_EQ(::getsockname(fd, reinterpret_cast(&addr), &len), 0); + EXPECT_EQ(fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK), 0); + this->listen_fd_ = fd; + this->listener_.set_listen(std::make_unique(fd)); + return ntohs(addr.sin_port); + } + // A blocking connect to the loopback listener; it completes into the backlog. + void connect_client(uint16_t port) { + this->client_fd_ = ::socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GE(this->client_fd_, 0); + struct sockaddr_in addr {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons(port); + ASSERT_EQ(::connect(this->client_fd_, reinterpret_cast(&addr), sizeof(addr)), 0); + // The stack may queue the connection for accept() a moment after connect() returns. + struct pollfd pfd { + this->listen_fd_, POLLIN, 0 + }; + ASSERT_EQ(::poll(&pfd, 1, 1000), 1); + } + + TcpClientLink link_; + ListenerUnderTest listener_; + int listen_fd_{-1}; + int client_fd_{-1}; +}; + +TEST_F(TcpListenerAccept, AcceptErrorRebuildsAfterTheBackoff) { + // accept() on a bad descriptor fails with EBADF, which no retry fixes. + this->listener_.set_listen(std::make_unique(-1)); + this->listener_.accept(this->link_); + EXPECT_FALSE(this->listener_.listening()); + EXPECT_TRUE(this->link_.in_backoff()); + // No new listen socket until the backoff has passed. + this->listener_.poll(this->link_, true); + EXPECT_FALSE(this->listener_.listening()); +} + +TEST_F(TcpListenerAccept, NothingPendingKeepsTheListener) { + this->listen_on_loopback(); + this->listener_.accept(this->link_); + EXPECT_TRUE(this->listener_.listening()); + EXPECT_FALSE(this->link_.connected()); + EXPECT_FALSE(this->link_.in_backoff()); +} + +TEST_F(TcpListenerAccept, AcceptedClientIsAdopted) { + this->connect_client(this->listen_on_loopback()); + this->listener_.accept(this->link_); + EXPECT_TRUE(this->link_.connected()); + EXPECT_TRUE(this->listener_.listening()); +} + +#ifdef USE_SOCKET_IPV4_ALLOW +TEST_F(TcpListenerAccept, PeerOutsideTheAllowListIsClosed) { + static const Ipv4AllowEntry ONLY_TEN[] = {{htonl(0x0A000000), htonl(0xFF000000)}}; + this->listener_.set_allow(ONLY_TEN, 1); + this->connect_client(this->listen_on_loopback()); + this->listener_.accept(this->link_); + EXPECT_FALSE(this->link_.connected()); + EXPECT_TRUE(this->listener_.listening()); + EXPECT_FALSE(this->link_.in_backoff()); + // The peer sees the close. + struct pollfd pfd { + this->client_fd_, POLLIN, 0 + }; + ASSERT_EQ(::poll(&pfd, 1, 1000), 1); + char b; + EXPECT_EQ(::read(this->client_fd_, &b, 1), 0); +} + +TEST_F(TcpListenerAccept, PeerInsideTheAllowListIsAdopted) { + static const Ipv4AllowEntry LOOPBACK_ONLY[] = {{htonl(INADDR_LOOPBACK), htonl(0xFFFFFFFF)}}; + this->listener_.set_allow(LOOPBACK_ONLY, 1); + this->connect_client(this->listen_on_loopback()); + this->listener_.accept(this->link_); + EXPECT_TRUE(this->link_.connected()); +} +#endif + +} // namespace esphome::socket::testing + +#endif From 2b02602a49515fcf06bbac86309c4afcd4382c3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:37:14 -0600 Subject: [PATCH 203/221] Bump smpclient from 7.2.0 to 7.3.0 (#17443) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 744c5c1e66..1b719701e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ resvg-py==0.5.0 freetype-py==2.5.1 jinja2==3.1.6 bleak==3.0.2 -smpclient==7.2.0 +smpclient==7.3.0 requests==2.34.2 py7zr==1.1.3 platformdirs==4.12.2 # native esp-idf toolchain global cache dir From 61ac0cb753e464dcee7bfa079a914033a5d7fc83 Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:39:02 +0200 Subject: [PATCH 204/221] [tcp_uart] Keep received bytes readable after the connection drops (#20164) --- esphome/components/tcp_uart/tcp_uart.cpp | 7 +- esphome/components/tcp_uart/tcp_uart.h | 2 +- tests/components/tcp_uart/__init__.py | 8 + .../components/tcp_uart/test_buffers_host.cpp | 234 ++++++++++++++++++ 4 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 tests/components/tcp_uart/test_buffers_host.cpp diff --git a/esphome/components/tcp_uart/tcp_uart.cpp b/esphome/components/tcp_uart/tcp_uart.cpp index 5913610ee8..3d3f8dfeba 100644 --- a/esphome/components/tcp_uart/tcp_uart.cpp +++ b/esphome/components/tcp_uart/tcp_uart.cpp @@ -46,7 +46,8 @@ void TcpUart::on_shutdown() { void TcpUart::sync_link_() { bool up = this->link_.connected(); this->link_was_up_ = up; - if (!up) { + if (up) { + // Unread bytes of the last session stay readable while down, never into the next one. this->rx_start_ = this->rx_end_ = 0; } if (this->connected_sensor_ != nullptr) { @@ -67,7 +68,7 @@ void TcpUart::read_socket_() { } ssize_t count = this->link_.read(this->rx_ + this->rx_end_, room); if (count <= 0) { - // A dropped link (-1) is cleaned up by sync_link_() on the next loop. + // A dropped link (-1) is seen by sync_link_() on the next loop. if (count == 0) { this->rx_pending_ = false; } @@ -81,7 +82,7 @@ void TcpUart::loop() { #ifdef USE_SOCKET_TCP_LISTENER if (this->server_) { // link_was_up_ holds the accept until the previous drop's edge has run, - // so the sensor and the cleared RX buffer always see the disconnect. + // so the sensor sees the disconnect and the new session's edge clears RX. this->listener_.poll(this->link_, !this->link_was_up_); } else { this->link_.poll(); diff --git a/esphome/components/tcp_uart/tcp_uart.h b/esphome/components/tcp_uart/tcp_uart.h index e973c7a189..4b2ff0aa22 100644 --- a/esphome/components/tcp_uart/tcp_uart.h +++ b/esphome/components/tcp_uart/tcp_uart.h @@ -64,7 +64,7 @@ class TcpUart : public uart::UARTComponent, public Component { uint16_t rx_start_{0}; uint16_t rx_end_{0}; bool server_{false}; - // The link state loop() saw last; edges clear rx_ and publish the sensor. + // The link state loop() saw last; edges publish the sensor, the up edge clears rx_. bool link_was_up_{false}; // A read stopped before EAGAIN. ready() stays false until new data arrives. bool rx_pending_{false}; diff --git a/tests/components/tcp_uart/__init__.py b/tests/components/tcp_uart/__init__.py index 1e4de448b7..7090910cb2 100644 --- a/tests/components/tcp_uart/__init__.py +++ b/tests/components/tcp_uart/__init__.py @@ -1,7 +1,15 @@ # This file's presence makes pytest treat this directory as a package named # "tcp_uart"; required for cpp unit testing. +from esphome.components import socket as socket_component +from esphome.types import ConfigType from tests.testing_helpers import ComponentManifestOverride def override_manifest(manifest: ComponentManifestOverride) -> None: manifest.multi_conf = False + + async def to_code_testing(config: ConfigType) -> None: + # The server-role gtest needs the listener. + socket_component.require_tcp_listener() + + manifest.to_code = to_code_testing diff --git a/tests/components/tcp_uart/test_buffers_host.cpp b/tests/components/tcp_uart/test_buffers_host.cpp new file mode 100644 index 0000000000..3ec233eba2 --- /dev/null +++ b/tests/components/tcp_uart/test_buffers_host.cpp @@ -0,0 +1,234 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "esphome/components/tcp_uart/tcp_uart.h" +#include "esphome/core/application.h" +#include "esphome/core/wake.h" + +#ifdef USE_HOST + +namespace esphome::tcp_uart::testing { + +class TcpUartLoopDriver : public TcpUart { + public: + TcpUartLoopDriver() { + this->set_host("peer"); + this->set_port(1); + this->link_.begin("buffers_test"); + } + socket::TcpClientLink &link() { return this->link_; } +}; + +class TcpUartBuffers : public ::testing::Test { + protected: + void SetUp() override { + // EPIPE must come back as an errno, not a signal. + signal(SIGPIPE, SIG_IGN); + this->connect_peer(); + } + void TearDown() override { + this->close_peer(); + this->uart_.link().close(); + } + // Hands the UART a fresh session, as an accept or a reconnect would. + void connect_peer() { + int fds[2]; + ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + this->peer_fd_ = fds[1]; + this->uart_.link().adopt(std::make_unique(fds[0])); + } + void close_peer() { + if (this->peer_fd_ >= 0) { + ::close(this->peer_fd_); + this->peer_fd_ = -1; + } + } + void send(const void *data, size_t len) { ASSERT_EQ(::write(this->peer_fd_, data, len), static_cast(len)); } + void loops(int count) { + for (int i = 0; i < count; i++) { + this->uart_.loop(); + } + } + + TcpUartLoopDriver uart_; + int peer_fd_{-1}; +}; + +TEST_F(TcpUartBuffers, BytesReceivedBeforeACloseStayReadable) { + this->loops(1); + this->send("HELLO-0123456789", 16); + this->close_peer(); + // Read the bytes, see the close, run the down edge, then idle. + this->loops(4); + ASSERT_FALSE(this->uart_.is_connected()); + ASSERT_EQ(this->uart_.available(), 16u); + uint8_t got[16]; + ASSERT_TRUE(this->uart_.read_array(got, sizeof(got))); + EXPECT_EQ(std::memcmp(got, "HELLO-0123456789", sizeof(got)), 0); + EXPECT_EQ(this->uart_.available(), 0u); +} + +TEST_F(TcpUartBuffers, UnreadBytesAreGoneWhenTheNextSessionStarts) { + this->loops(1); + this->send("OLD", 3); + this->close_peer(); + this->loops(4); + ASSERT_FALSE(this->uart_.is_connected()); + this->connect_peer(); + this->send("NEW", 3); + this->loops(1); + ASSERT_EQ(this->uart_.available(), 3u); + uint8_t got[3]; + ASSERT_TRUE(this->uart_.read_array(got, sizeof(got))); + EXPECT_EQ(std::memcmp(got, "NEW", sizeof(got)), 0); +} + +TEST_F(TcpUartBuffers, FullBufferWaitsAndCompactsAfterARead) { + static constexpr size_t TOTAL = 1500; + uint8_t data[TOTAL]; + for (size_t i = 0; i < TOTAL; i++) { + data[i] = static_cast(i % 251); + } + this->loops(1); + this->send(data, TOTAL); + this->loops(1); + ASSERT_EQ(this->uart_.available(), 1024u); + // Nothing read, no room: the rest stays in the socket. + this->loops(2); + ASSERT_EQ(this->uart_.available(), 1024u); + + uint8_t got[TOTAL]; + ASSERT_TRUE(this->uart_.read_array(got, 100)); + // The read freed the front; the next pass moves the rest down and refills. + this->loops(1); + ASSERT_EQ(this->uart_.available(), 1024u); + ASSERT_TRUE(this->uart_.read_array(got + 100, 1024)); + this->loops(1); + ASSERT_EQ(this->uart_.available(), TOTAL - 1124); + ASSERT_TRUE(this->uart_.read_array(got + 1124, TOTAL - 1124)); + EXPECT_EQ(std::memcmp(got, data, TOTAL), 0); + EXPECT_TRUE(this->uart_.is_connected()); +} + +TEST_F(TcpUartBuffers, WritesBeyondTheSendBufferAreDropped) { + static constexpr size_t TOTAL = 1500; + uint8_t data[TOTAL]; + for (size_t i = 0; i < TOTAL; i++) { + data[i] = static_cast(i % 251); + } + this->loops(1); + this->uart_.write_array(data, TOTAL); + EXPECT_EQ(this->uart_.available_for_write(), 0u); + EXPECT_EQ(this->uart_.flush(), uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS); + + uint8_t got[TOTAL]; + size_t total = 0; + while (total < 1024) { + ssize_t n = ::read(this->peer_fd_, got + total, sizeof(got) - total); + ASSERT_GT(n, 0); + total += static_cast(n); + } + EXPECT_EQ(total, 1024u); + EXPECT_EQ(std::memcmp(got, data, 1024), 0); + // The 476 bytes past the buffer never reach the peer. + EXPECT_EQ(::recv(this->peer_fd_, got, sizeof(got), MSG_DONTWAIT), -1); +} + +#ifdef USE_SOCKET_TCP_LISTENER +// Server role through the real listener on a loopback port. +class TcpUartServer : public ::testing::Test { + protected: + void SetUp() override { + signal(SIGPIPE, SIG_IGN); + // Find a free port for the listener. + int probe = ::socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GE(probe, 0); + struct sockaddr_in addr = loopback(0); + ASSERT_EQ(::bind(probe, reinterpret_cast(&addr), sizeof(addr)), 0); + socklen_t len = sizeof(addr); + ASSERT_EQ(::getsockname(probe, reinterpret_cast(&addr), &len), 0); + ::close(probe); + this->port_ = ntohs(addr.sin_port); + this->uart_.set_server(true); + this->uart_.set_port(this->port_); + this->uart_.set_reconnect_interval(0); + this->uart_.setup(); + this->pass(); + } + void TearDown() override { + this->close_peer(); + this->uart_.on_shutdown(); + } + static struct sockaddr_in loopback(uint16_t port) { + struct sockaddr_in addr {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons(port); + return addr; + } + // One main loop pass: select() marks readable sockets, then the component runs. + void pass() { + internal::wakeable_delay(5); + this->now_ += 16; + LoopBlockingGuard dispatch{nullptr, nullptr, this->now_}; + this->uart_.loop(); + } + void connect_peer() { + this->peer_fd_ = ::socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GE(this->peer_fd_, 0); + struct sockaddr_in addr = loopback(this->port_); + ASSERT_EQ(::connect(this->peer_fd_, reinterpret_cast(&addr), sizeof(addr)), 0); + for (int i = 0; i < 50 && !this->uart_.is_connected(); i++) + this->pass(); + ASSERT_TRUE(this->uart_.is_connected()); + } + void close_peer() { + if (this->peer_fd_ >= 0) { + ::close(this->peer_fd_); + this->peer_fd_ = -1; + } + } + void send(const void *data, size_t len) { ASSERT_EQ(::write(this->peer_fd_, data, len), static_cast(len)); } + void pass_until_available(size_t count) { + for (int i = 0; i < 50 && this->uart_.available() < count; i++) + this->pass(); + } + + TcpUart uart_; + uint16_t port_{0}; + int peer_fd_{-1}; + uint32_t now_{0}; +}; + +TEST_F(TcpUartServer, NextAcceptedClientStartsWithAnEmptyBuffer) { + this->connect_peer(); + this->send("OLD", 3); + this->pass_until_available(3); + ASSERT_EQ(this->uart_.available(), 3u); + this->close_peer(); + for (int i = 0; i < 50 && this->uart_.is_connected(); i++) + this->pass(); + ASSERT_FALSE(this->uart_.is_connected()); + // Unread bytes stay readable while no client is connected. + EXPECT_EQ(this->uart_.available(), 3u); + + this->connect_peer(); + this->send("NEW", 3); + this->pass_until_available(3); + ASSERT_EQ(this->uart_.available(), 3u); + uint8_t got[3]; + ASSERT_TRUE(this->uart_.read_array(got, sizeof(got))); + EXPECT_EQ(std::memcmp(got, "NEW", sizeof(got)), 0); +} +#endif + +} // namespace esphome::tcp_uart::testing + +#endif From 1520d228bc965d4ca447852f1091b3de715884e8 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 6 Oct 2026 12:43:49 -0500 Subject: [PATCH 205/221] [i2s_audio] Park the speaker data-out pin low at setup (#20226) Co-authored-by: Claude Fable 5.1 --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 14 +++++++++++--- .../i2s_audio/speaker/i2s_audio_speaker.h | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 7ae58b3e44..4b74b2e29d 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -40,6 +40,16 @@ void I2SAudioSpeakerBase::setup() { // When no audio_dac is configured, this initializes software volume control. this->set_volume(this->volume_); this->set_mute_state(this->mute_state_); + + // Until the I2S driver first starts, dout sits in its reset state (often pulled high, or a JTAG + // function on the ESP32-S3), which keeps a SPDIF optical transmitter lit. Park it low now. + this->park_dout_pin_(); +} + +void I2SAudioSpeakerBase::park_dout_pin_() { + gpio_reset_pin(this->dout_pin_); + gpio_set_direction(this->dout_pin_, GPIO_MODE_OUTPUT); + gpio_set_level(this->dout_pin_, 0); } void I2SAudioSpeakerBase::dump_config() { @@ -303,9 +313,7 @@ void I2SAudioSpeakerBase::stop_i2s_driver_() { // setup installed. If another speaker reuses this port (shared bus), its audio still reaches our // dout. Detach the pin and drive it low so a stale output stops driving downstream hardware: a // SPDIF optical transmitter would otherwise stay lit, and an analog DAC would emit noise. - gpio_reset_pin(this->dout_pin_); - gpio_set_direction(this->dout_pin_, GPIO_MODE_OUTPUT); - gpio_set_level(this->dout_pin_, 0); + this->park_dout_pin_(); } this->parent_->unlock(); } diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h index b443166ea1..965eede020 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h @@ -129,6 +129,9 @@ class I2SAudioSpeakerBase : public I2SAudioOut, public speaker::Speaker, public /// @brief Stops the I2S driver and unlocks the I2S port void stop_i2s_driver_(); + /// @brief Detaches dout from the I2S signal and drives it low + void park_dout_pin_(); + /// @brief Called in loop() when the task has stopped. Override for mode-specific cleanup. virtual void on_task_stopped() {} From 490d478e546b3a7328ceab102ea478d3eebdfd3c Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:02:34 +1100 Subject: [PATCH 206/221] [mipi_spi] Light platform for AMOLED display brightness (#20212) Co-authored-by: Claude Opus 5.5 --- esphome/components/mipi/__init__.py | 5 +- esphome/components/mipi_spi/display.py | 4 +- esphome/components/mipi_spi/light/__init__.py | 68 ++++++++++ .../mipi_spi/light/mipi_spi_light.h | 36 +++++ esphome/components/mipi_spi/mipi_spi.h | 13 +- .../mipi_spi/test_get_sequence.py | 36 ++++- tests/component_tests/mipi_spi/test_light.py | 126 ++++++++++++++++++ tests/components/mipi_spi/common.yaml | 45 +++++++ 8 files changed, 327 insertions(+), 6 deletions(-) create mode 100644 esphome/components/mipi_spi/light/__init__.py create mode 100644 esphome/components/mipi_spi/light/mipi_spi_light.h create mode 100644 tests/component_tests/mipi_spi/test_light.py diff --git a/esphome/components/mipi/__init__.py b/esphome/components/mipi/__init__.py index 50b84b630d..526afa914b 100644 --- a/esphome/components/mipi/__init__.py +++ b/esphome/components/mipi/__init__.py @@ -648,7 +648,10 @@ class DriverChip: sequence.append((INVON,)) else: sequence.append((INVOFF,)) - if brightness := config.get(CONF_BRIGHTNESS, self.get_default(CONF_BRIGHTNESS)): + brightness = config.get( + CONF_BRIGHTNESS, self.get_default(CONF_BRIGHTNESS, None) + ) + if brightness is not None: sequence.append((BRIGHTNESS, brightness)) # Add a SLPOUT command if required. if not self.skip_command("SLPOUT"): diff --git a/esphome/components/mipi_spi/display.py b/esphome/components/mipi_spi/display.py index b49675ca4c..ff49979a24 100644 --- a/esphome/components/mipi_spi/display.py +++ b/esphome/components/mipi_spi/display.py @@ -225,10 +225,10 @@ def model_schema(config: ConfigType) -> cv.All | cv.Schema: ) .extend({model.option(x): cv.boolean for x in other_options}) ) - if brightness := model.get_default(CONF_BRIGHTNESS): + if model.get_default(CONF_BRIGHTNESS, None) is not None or model.name == "CUSTOM": schema = schema.extend( { - cv.Optional(CONF_BRIGHTNESS, default=brightness): cv.int_range( + model.option(CONF_BRIGHTNESS, cv.UNDEFINED): cv.int_range( 0, 0xFF, min_included=True, max_included=True ), } diff --git a/esphome/components/mipi_spi/light/__init__.py b/esphome/components/mipi_spi/light/__init__.py new file mode 100644 index 0000000000..aec5d9b4fd --- /dev/null +++ b/esphome/components/mipi_spi/light/__init__.py @@ -0,0 +1,68 @@ +import esphome.codegen as cg +from esphome.components import light +import esphome.config_validation as cv +from esphome.const import ( + CONF_BRIGHTNESS, + CONF_DISPLAY_ID, + CONF_GAMMA_CORRECT, + CONF_MAX_BRIGHTNESS, + CONF_MIN_BRIGHTNESS, + CONF_MODEL, + CONF_OUTPUT_ID, +) +import esphome.final_validate as fv +from esphome.types import ConfigType + +from ..display import MipiSpi, mipi_spi_ns + +MipiSpiLight = mipi_spi_ns.class_("MipiSpiLight", light.LightOutput) + + +def _validate_brightness_range(config: ConfigType) -> ConfigType: + if config[CONF_MIN_BRIGHTNESS] >= config[CONF_MAX_BRIGHTNESS]: + raise cv.Invalid( + f"'{CONF_MIN_BRIGHTNESS}' must be less than '{CONF_MAX_BRIGHTNESS}'" + ) + return config + + +CONFIG_SCHEMA = cv.All( + light.BRIGHTNESS_ONLY_LIGHT_SCHEMA.extend( + { + cv.GenerateID(CONF_OUTPUT_ID): cv.declare_id(MipiSpiLight), + cv.GenerateID(CONF_DISPLAY_ID): cv.use_id(MipiSpi), + cv.Optional(CONF_GAMMA_CORRECT, default=1.0): cv.positive_float, + cv.Optional(CONF_MIN_BRIGHTNESS, default=0): cv.int_range(0, 255), + cv.Optional(CONF_MAX_BRIGHTNESS, default=255): cv.int_range(0, 255), + } + ), + _validate_brightness_range, +) + + +def _final_validate(config: ConfigType) -> None: + full_config = fv.full_config.get() + display_path = full_config.get_path_for_id(config[CONF_DISPLAY_ID])[:-1] + display_config = full_config.get_config_for_path(display_path) + if CONF_BRIGHTNESS not in display_config: + model = display_config[CONF_MODEL] + if model == "CUSTOM": + raise cv.Invalid( + f"The '{CONF_BRIGHTNESS}' option must be set in the display config" + ) + raise cv.Invalid( + f"Display model {model} does not support setting brightness with an SPI command" + ) + + +FINAL_VALIDATE_SCHEMA = _final_validate + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable( + config[CONF_OUTPUT_ID], + config[CONF_MIN_BRIGHTNESS], + config[CONF_MAX_BRIGHTNESS], + ) + await light.register_light(var, config) + await cg.register_parented(var, config[CONF_DISPLAY_ID]) diff --git a/esphome/components/mipi_spi/light/mipi_spi_light.h b/esphome/components/mipi_spi/light/mipi_spi_light.h new file mode 100644 index 0000000000..30bbdcd8ee --- /dev/null +++ b/esphome/components/mipi_spi/light/mipi_spi_light.h @@ -0,0 +1,36 @@ +#pragma once + +#include "esphome/components/light/light_output.h" +#include "esphome/components/light/light_state.h" +#include "esphome/components/mipi_spi/mipi_spi.h" +#include "esphome/core/helpers.h" + +namespace esphome::mipi_spi { + +// Brightness-only light that sets the display brightness with an SPI command. +// Brightness is scaled into [min_brightness, max_brightness]; off writes min_brightness. +class MipiSpiLight final : public light::LightOutput, public Parented { + public: + MipiSpiLight(uint8_t min_brightness, uint8_t max_brightness) + : min_brightness_(min_brightness), max_brightness_(max_brightness) {} + + light::LightTraits get_traits() override { + auto traits = light::LightTraits(); + traits.set_supported_color_modes({light::ColorMode::BRIGHTNESS}); + return traits; + } + + void write_state(light::LightState *state) override { + float brightness; + state->current_values_as_brightness(&brightness); + this->parent_->set_brightness( + this->min_brightness_ + + static_cast(roundf(brightness * (this->max_brightness_ - this->min_brightness_)))); + } + + protected: + uint8_t min_brightness_; + uint8_t max_brightness_; +}; + +} // namespace esphome::mipi_spi diff --git a/esphome/components/mipi_spi/mipi_spi.h b/esphome/components/mipi_spi/mipi_spi.h index e428a1f96e..48a2cafbdd 100644 --- a/esphome/components/mipi_spi/mipi_spi.h +++ b/esphome/components/mipi_spi/mipi_spi.h @@ -72,6 +72,12 @@ void internal_dump_config(const char *model, int width, int height, int offset_w GPIOPin *cs, GPIOPin *reset, GPIOPin *dc, int spi_mode, uint32_t data_rate, int bus_width, bool has_hardware_rotation); +// Lets a light set the display brightness without knowing the display's template parameters. +class MipiSpiBrightness { + public: + virtual void set_brightness(uint8_t brightness) = 0; +}; + /** * Base class for MIPI SPI displays. * All the methods are defined here in the header file, as it is not possible to define templated methods in a cpp file. @@ -94,6 +100,7 @@ template class MipiSpi : public display::Display, + public MipiSpiBrightness, public spi::SPIDevice { public: @@ -108,9 +115,11 @@ class MipiSpi : public display::Display, this->invert_colors_ = invert_colors; this->reset_params_(); } - void set_brightness(uint8_t brightness) { + void set_brightness(uint8_t brightness) override { this->brightness_ = brightness; - this->reset_params_(); + // Before setup the stored value is applied by reset_params_() + if (this->is_ready()) + this->write_command_(BRIGHTNESS, brightness); } void set_rotation(display::DisplayRotation rotation) override { this->rotation_ = rotation; diff --git a/tests/component_tests/mipi_spi/test_get_sequence.py b/tests/component_tests/mipi_spi/test_get_sequence.py index 263b567b9c..f8614b17a3 100644 --- a/tests/component_tests/mipi_spi/test_get_sequence.py +++ b/tests/component_tests/mipi_spi/test_get_sequence.py @@ -4,7 +4,13 @@ from collections.abc import Generator import pytest -from esphome.components.mipi import CONF_INVERT_COLORS, CONF_PIXEL_MODE, DriverChip +from esphome.components.mipi import ( + BRIGHTNESS, + CONF_INVERT_COLORS, + CONF_PIXEL_MODE, + DriverChip, +) +from esphome.const import CONF_BRIGHTNESS # A minimal config with no reset pin: enough for get_sequence(add_madctl=False) to run # without needing a full display configuration. @@ -62,3 +68,31 @@ def test_get_sequence_skips_reset_delay_validation_without_add_reset() -> None: chip = DriverChip("TEST-GET-SEQUENCE-NO-RESET", reset_delay=999) chip.get_sequence(_BASE_CONFIG, add_madctl=False, add_reset=False) + + +def _has_brightness_command(sequence: tuple[int, ...], value: int) -> bool: + """True if the flattened sequence holds a one-byte brightness (0x51) command.""" + return any( + sequence[i : i + 3] == (BRIGHTNESS, 1, value) for i in range(len(sequence) - 2) + ) + + +@pytest.mark.parametrize("brightness", [0, 0x80]) +def test_get_sequence_adds_configured_brightness(brightness: int) -> None: + """A configured brightness is sent at startup, including zero.""" + chip = DriverChip("TEST-GET-SEQUENCE-BRIGHTNESS") + + sequence = chip.get_sequence( + {**_BASE_CONFIG, CONF_BRIGHTNESS: brightness}, add_madctl=False + ) + + assert _has_brightness_command(sequence, brightness) + + +def test_get_sequence_omits_brightness_when_not_set() -> None: + """No brightness command is sent when neither config nor model sets one.""" + chip = DriverChip("TEST-GET-SEQUENCE-NO-BRIGHTNESS") + + sequence = chip.get_sequence(_BASE_CONFIG, add_madctl=False) + + assert not any(_has_brightness_command(sequence, v) for v in range(256)) diff --git a/tests/component_tests/mipi_spi/test_light.py b/tests/component_tests/mipi_spi/test_light.py new file mode 100644 index 0000000000..c42b56cf5a --- /dev/null +++ b/tests/component_tests/mipi_spi/test_light.py @@ -0,0 +1,126 @@ +"""Tests for the mipi_spi light platform validation.""" + +from __future__ import annotations + +import pytest + +from esphome import final_validate +from esphome.components.esp32 import KEY_BOARD, KEY_VARIANT, VARIANT_ESP32 +from esphome.components.mipi_spi.display import CONFIG_SCHEMA as DISPLAY_SCHEMA +from esphome.components.mipi_spi.light import ( + CONFIG_SCHEMA as LIGHT_SCHEMA, + FINAL_VALIDATE_SCHEMA as LIGHT_FINAL_VALIDATE_SCHEMA, +) +from esphome.config import Config +import esphome.config_validation as cv +from esphome.const import ( + CONF_BRIGHTNESS, + CONF_DISPLAY_ID, + CONF_GAMMA_CORRECT, + CONF_ID, + CONF_MAX_BRIGHTNESS, + CONF_MIN_BRIGHTNESS, + PlatformFramework, +) +from esphome.core import ID +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + +DISPLAY_ID = "test_display" + +_CUSTOM: ConfigType = { + "model": "custom", + "dimensions": {"width": 240, "height": 240}, + "init_sequence": [[0xA0, 0x01]], +} + + +@pytest.fixture(autouse=True) +def _esp32(set_core_config: SetCoreConfigCallable) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + + +def _light(**extra: object) -> ConfigType: + return { + "name": "Display Brightness", + CONF_DISPLAY_ID: DISPLAY_ID, + **extra, + } + + +def _final_validate_with_display(display: ConfigType) -> None: + """Validate a display, place it in the full config, then final-validate a light on it.""" + display = DISPLAY_SCHEMA( + {"id": DISPLAY_ID, "dc_pin": 18, **display}, + ) + full = Config() + full["display"] = [display] + full.declare_ids.append((display[CONF_ID], ["display", 0, CONF_ID])) + final_validate.full_config.set(full) + LIGHT_FINAL_VALIDATE_SCHEMA({CONF_DISPLAY_ID: ID(DISPLAY_ID, is_declaration=False)}) + + +def test_light_defaults() -> None: + """Gamma defaults to 1.0 and the range to the full 0-255.""" + config = LIGHT_SCHEMA(_light()) + + assert config[CONF_GAMMA_CORRECT] == 1.0 + assert config[CONF_MIN_BRIGHTNESS] == 0 + assert config[CONF_MAX_BRIGHTNESS] == 255 + + +@pytest.mark.parametrize( + ("min_brightness", "max_brightness"), + [(100, 100), (200, 100)], + ids=["equal", "inverted"], +) +def test_light_rejects_bad_range(min_brightness: int, max_brightness: int) -> None: + with pytest.raises( + cv.Invalid, match="'min_brightness' must be less than 'max_brightness'" + ): + LIGHT_SCHEMA( + _light(min_brightness=min_brightness, max_brightness=max_brightness) + ) + + +@pytest.mark.parametrize("key", [CONF_MIN_BRIGHTNESS, CONF_MAX_BRIGHTNESS]) +def test_light_rejects_out_of_range_value(key: str) -> None: + with pytest.raises(cv.Invalid): + LIGHT_SCHEMA(_light(**{key: 256})) + + +@pytest.mark.parametrize( + "display", + [ + pytest.param({"model": "rm690b0"}, id="model_with_default_brightness"), + pytest.param({**_CUSTOM, CONF_BRIGHTNESS: 0}, id="custom_with_brightness"), + ], +) +def test_light_accepts_display_with_brightness(display: ConfigType) -> None: + _final_validate_with_display(display) + + +def test_light_rejects_model_without_brightness() -> None: + with pytest.raises( + cv.Invalid, + match="Display model ILI9488 does not support setting brightness", + ): + _final_validate_with_display({"model": "ili9488"}) + + +def test_light_rejects_custom_without_brightness() -> None: + with pytest.raises( + cv.Invalid, + match="The 'brightness' option must be set in the display config", + ): + _final_validate_with_display(_CUSTOM) + + +def test_brightness_not_allowed_on_model_without_default() -> None: + with pytest.raises(cv.Invalid, match=r"extra keys not allowed.*brightness"): + DISPLAY_SCHEMA( + {"id": DISPLAY_ID, "dc_pin": 18, "model": "ili9488", CONF_BRIGHTNESS: 10} + ) diff --git a/tests/components/mipi_spi/common.yaml b/tests/components/mipi_spi/common.yaml index 6bdee9e54a..45815b1ba5 100644 --- a/tests/components/mipi_spi/common.yaml +++ b/tests/components/mipi_spi/common.yaml @@ -53,3 +53,48 @@ display: reset_pin: allow_other_uses: true number: ${reset_pin} + + - platform: mipi_spi + id: amoled_display + model: RM690B0 + dc_pin: + allow_other_uses: true + number: ${dc_pin} + cs_pin: + allow_other_uses: true + number: ${cs_pin} + reset_pin: + allow_other_uses: true + number: ${reset_pin} + + - platform: mipi_spi + id: custom_display + model: custom + brightness: 0x80 + dimensions: + width: 240 + height: 240 + init_sequence: + - [0xd0, 1] + dc_pin: + allow_other_uses: true + number: ${dc_pin} + cs_pin: + allow_other_uses: true + number: ${cs_pin} + reset_pin: + allow_other_uses: true + number: ${reset_pin} + +light: + - platform: mipi_spi + id: display_brightness + name: Display Brightness + display_id: amoled_display + + - platform: mipi_spi + id: custom_display_brightness + name: Custom Display Brightness + display_id: custom_display + min_brightness: 16 + max_brightness: 200 From 194273955c62fc2b269d9c2889654a8d9885133c Mon Sep 17 00:00:00 2001 From: mrtoy-me <118446898+mrtoy-me@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:22:00 +1000 Subject: [PATCH 207/221] [tas58xx] Add support for tas5825 dac (#19971) --- esphome/components/tas58xx/__init__.py | 9 ++ esphome/components/tas58xx/audio_dac.py | 15 ++- esphome/components/tas58xx/binary_sensor.py | 46 ++++++- esphome/components/tas58xx/model_tas5805m.cpp | 28 +++-- esphome/components/tas58xx/model_tas5825m.cpp | 119 ++++++++++++++++++ esphome/components/tas58xx/tas58xx.cpp | 7 +- esphome/components/tas58xx/tas58xx.h | 32 ++++- tests/components/tas58xx/common.yaml | 17 ++- 8 files changed, 247 insertions(+), 26 deletions(-) create mode 100644 esphome/components/tas58xx/model_tas5825m.cpp diff --git a/esphome/components/tas58xx/__init__.py b/esphome/components/tas58xx/__init__.py index b87036984d..0931d7ba1f 100644 --- a/esphome/components/tas58xx/__init__.py +++ b/esphome/components/tas58xx/__init__.py @@ -1 +1,10 @@ +from esphome.config_helpers import filter_source_files_from_defines + CODEOWNERS = ["@mrtoy-me", "@remcom"] + +FILTER_SOURCE_FILES = filter_source_files_from_defines( + { + "model_tas5805m.cpp": "USE_TAS58XX_TAS5805M", + "model_tas5825m.cpp": "USE_TAS58XX_TAS5825M", + } +) diff --git a/esphome/components/tas58xx/audio_dac.py b/esphome/components/tas58xx/audio_dac.py index 18a60a6645..7b1e3e239e 100644 --- a/esphome/components/tas58xx/audio_dac.py +++ b/esphome/components/tas58xx/audio_dac.py @@ -11,6 +11,8 @@ from esphome.const import CONF_ENABLE_PIN, CONF_ID, CONF_MODEL from esphome.cpp_generator import MockObj from esphome.types import ConfigType +from . import FILTER_SOURCE_FILES # noqa: F401 pylint: disable=unused-import + _LOGGER = logging.getLogger(__name__) DEPENDENCIES = ["i2c"] @@ -23,6 +25,9 @@ CONF_VOLUME_MIN_DB = "volume_min_db" CONF_VOLUME_MAX_DB = "volume_max_db" CONF_TAS58XX_ID = "tas58xx_id" +DAC_TAS5805M = "tas5805m" +DAC_TAS5825M = "tas5825m" + tas58xx_ns = cg.esphome_ns.namespace("tas58xx") TAS58xx = tas58xx_ns.class_("TAS58xx", AudioDac, cg.PollingComponent, i2c.I2CDevice) @@ -54,13 +59,20 @@ class Model: MODELS: dict[str, Model] = { - "tas5805m": Model( + DAC_TAS5805M: Model( model_info=tas58xx_ns.TAS5805M_MODEL, analog_gain_min_db=-15.5, volume_min_db=-103.0, volume_max_db=24.0, default_address=0x2D, ), + DAC_TAS5825M: Model( + model_info=tas58xx_ns.TAS5825M_MODEL, + analog_gain_min_db=-15.5, + volume_min_db=-103.0, + volume_max_db=24.0, + default_address=0x4C, + ), } @@ -162,6 +174,7 @@ for _name, _call in ( async def to_code(config: ConfigType) -> None: model = MODELS[config[CONF_MODEL]] + cg.add_define(f"USE_TAS58XX_{config[CONF_MODEL].upper()}") var = cg.new_Pvariable(config[CONF_ID], cg.RawExpression(f"&{model.model_info}")) await cg.register_component(var, config) await i2c.register_i2c_device(var, config) diff --git a/esphome/components/tas58xx/binary_sensor.py b/esphome/components/tas58xx/binary_sensor.py index 398cfa2440..d057739874 100644 --- a/esphome/components/tas58xx/binary_sensor.py +++ b/esphome/components/tas58xx/binary_sensor.py @@ -1,15 +1,17 @@ import esphome.codegen as cg from esphome.components import binary_sensor import esphome.config_validation as cv -from esphome.const import DEVICE_CLASS_PROBLEM, ENTITY_CATEGORY_DIAGNOSTIC +from esphome.const import CONF_MODEL, DEVICE_CLASS_PROBLEM, ENTITY_CATEGORY_DIAGNOSTIC +import esphome.final_validate as fv from esphome.types import ConfigType -from .audio_dac import CONF_TAS58XX_ID, TAS58xx, tas58xx_ns +from . import FILTER_SOURCE_FILES # noqa: F401 pylint: disable=unused-import +from .audio_dac import CONF_TAS58XX_ID, DAC_TAS5825M, TAS58xx, tas58xx_ns CONF_HAVE_FAULT = "have_fault" -# Each name matches a FaultSensor value in tas58xx.h -FAULT_SENSORS = ( +# Faults common to TAS5805M and TAS5825M +COMMON_FAULT_SENSORS = ( "left_channel_dc_fault", "right_channel_dc_fault", "left_channel_over_current", @@ -20,8 +22,22 @@ FAULT_SENSORS = ( "pvdd_over_voltage", "pvdd_under_voltage", "over_temp_shutdown", - "over_temp_warning", + "over_temp_warning", # tas582x OTW Level 3 - use tas5805 naming ) +# Additional faults available on TAS5825M +TAS5825M_ONLY_FAULT_SENSORS = ( + "load_eeprom_error", + "right_channel_cbc_over_current", + "left_channel_cbc_over_current", + "left_channel_cbc_over_current_warning", + "right_channel_cbc_over_current_warning", + "over_temp_146c_warning", # tas582x OTW Level 4 + # "over_temp_122c_warning", # tas582x OTW Level 2 - not currently included + # "over_temp_112c_warning", # tas582x OTW Level 1 - not currently included +) + +# Each name matches a FaultSensor value in tas58xx.h +FAULT_SENSORS = COMMON_FAULT_SENSORS + TAS5825M_ONLY_FAULT_SENSORS FaultSensor = tas58xx_ns.enum("FaultSensor") @@ -39,6 +55,26 @@ CONFIG_SCHEMA = cv.Schema( ) +def _final_validate(config: ConfigType) -> ConfigType: + fconf = fv.full_config.get() + hub_path = fconf.get_path_for_id(config[CONF_TAS58XX_ID]) + hub_conf = fconf.get_config_for_path(hub_path[:-1]) + + if hub_conf[CONF_MODEL] == DAC_TAS5825M: + return config + + unsupported = sorted(frozenset(TAS5825M_ONLY_FAULT_SENSORS).intersection(config)) + if unsupported: + raise cv.Invalid( + f"binary_sensor platform tas58xx: '{', '.join(unsupported)}' is only available for 'model: {DAC_TAS5825M}' " + f"- Remove from YAML for 'model: {hub_conf[CONF_MODEL]}'" + ) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_TAS58XX_ID]) binary_sensors = binary_sensor.sub_binary_sensors(config) diff --git a/esphome/components/tas58xx/model_tas5805m.cpp b/esphome/components/tas58xx/model_tas5805m.cpp index e5eb5a837e..de0a0c5d75 100644 --- a/esphome/components/tas58xx/model_tas5805m.cpp +++ b/esphome/components/tas58xx/model_tas5805m.cpp @@ -67,17 +67,23 @@ const ModelInfo TAS5805M_MODEL = { .fault_name = fault_name, .fault_sensor_bits = { - 3, // FAULT_SENSOR_LEFT_CHANNEL_DC_FAULT - 2, // FAULT_SENSOR_RIGHT_CHANNEL_DC_FAULT - 1, // FAULT_SENSOR_LEFT_CHANNEL_OVER_CURRENT - 0, // FAULT_SENSOR_RIGHT_CHANNEL_OVER_CURRENT - 15, // FAULT_SENSOR_OTP_CRC_CHECK - 14, // FAULT_SENSOR_BQ_WRITE_FAILED - 10, // FAULT_SENSOR_CLOCK_FAULT - 9, // FAULT_SENSOR_PVDD_OVER_VOLTAGE - 8, // FAULT_SENSOR_PVDD_UNDER_VOLTAGE - 16, // FAULT_SENSOR_OVER_TEMP_SHUTDOWN - 26, // FAULT_SENSOR_OVER_TEMP_WARNING + 3, // FAULT_SENSOR_LEFT_CHANNEL_DC_FAULT + 2, // FAULT_SENSOR_RIGHT_CHANNEL_DC_FAULT + 1, // FAULT_SENSOR_LEFT_CHANNEL_OVER_CURRENT + 0, // FAULT_SENSOR_RIGHT_CHANNEL_OVER_CURRENT + 15, // FAULT_SENSOR_OTP_CRC_CHECK + 14, // FAULT_SENSOR_BQ_WRITE_FAILED + 10, // FAULT_SENSOR_CLOCK_FAULT + 9, // FAULT_SENSOR_PVDD_OVER_VOLTAGE + 8, // FAULT_SENSOR_PVDD_UNDER_VOLTAGE + 16, // FAULT_SENSOR_OVER_TEMP_SHUTDOWN + 26, // FAULT_SENSOR_OVER_TEMP_WARNING + NO_BIT, // FAULT_SENSOR_LOAD_EEPROM_ERROR + NO_BIT, // FAULT_SENSOR_RIGHT_CHANNEL_CBC_OVER_CURRENT + NO_BIT, // FAULT_SENSOR_LEFT_CHANNEL_CBC_OVER_CURRENT + NO_BIT, // FAULT_SENSOR_LEFT_CHANNEL_CBC_OVER_CURRENT_WARNING + NO_BIT, // FAULT_SENSOR_RIGHT_CHANNEL_CBC_OVER_CURRENT_WARNING + NO_BIT, // FAULT_SENSOR_OVER_TEMP_146C_WARNING }, }; diff --git a/esphome/components/tas58xx/model_tas5825m.cpp b/esphome/components/tas58xx/model_tas5825m.cpp new file mode 100644 index 0000000000..3aa74c675e --- /dev/null +++ b/esphome/components/tas58xx/model_tas5825m.cpp @@ -0,0 +1,119 @@ +#include "tas58xx.h" + +#include "esphome/core/hal.h" + +namespace esphome::tas58xx { + +// Remainder of the startup sequence, run after the reset. Register 0x00 selects the page. +// All writes except starting and last three register writes are exactly the startup sequence from TI PurePath Console. +// Registers 0x46, 0x7D, 0x7E, page 1 register 0x51 and Page 2 registers 0x1D, 0x19 are not documented in the datasheet. + +// clang-format off +static const uint8_t STARTUP_SEQUENCE[][2] PROGMEM = { + {0x03, 0x00}, // Deep Sleep + {0x7D, 0x11}, + {0x7E, 0xFF}, + {0x00, 0x01}, // Page 1 + {0x51, 0x05}, + {0x00, 0x02}, // Page 2 + {0x1D, 0x00}, + {0x19, 0x80}, + {0x00, 0x00}, // Page 0 + {0x46, 0x11}, + {0x02, 0x00}, // DEVICE_CTRL_1 - BD MODE, Damp to PBL MODE + {0x53, 0x01}, // ANA_CTRL PWM Phase Control - in phase + {0x54, 0x00}, // AGAIN 0dB + {0x03, 0x02}, // Hi-Z + {0x61, 0x0B}, // GPIO0_SEL - GPIO0 as FAULTZ output + {0x60, 0x01}, // GPIO0_OE - Enable GPIO0 as output + {0x77, 0x07}, // CBC_CONTROL enabling CBC function for warnings and faults +}; +// clang-format on + +static const LogString *model_name() { return LOG_STR("TAS5825M"); } + +// An if chain rather than a switch: a switch table would land in rodata, which is RAM on ESP8266. +static const LogString *fault_name(uint8_t index) { + if (index == 0) + return LOG_STR("Right channel over current"); + if (index == 1) + return LOG_STR("Left channel over current"); + if (index == 2) + return LOG_STR("Right channel DC fault"); + if (index == 3) + return LOG_STR("Left channel DC fault"); + if (index == 8) + return LOG_STR("PVDD under voltage"); + if (index == 9) + return LOG_STR("PVDD over voltage"); + if (index == 10) + return LOG_STR("Clock fault"); + if (index == 13) + return LOG_STR("Load EEPROM error"); + if (index == 14) + return LOG_STR("BQ write failed"); + if (index == 15) + return LOG_STR("OTP CRC check error"); + if (index == 16) + return LOG_STR("Over temperature shutdown"); + if (index == 17) + return LOG_STR("Left channel CBC over current"); + if (index == 18) + return LOG_STR("Right channel CBC over current"); + // if (index == 24) + // return LOG_STR("Over temperature warning 112C"); // not currently included + // if (index == 25) + // return LOG_STR("Over temperature warning 122C"); // not currently included + if (index == 26) + return LOG_STR("Over temperature warning"); + if (index == 27) + return LOG_STR("Over temperature warning 146C"); + if (index == 28) + return LOG_STR("Right channel CBC over current warning"); + if (index == 29) + return LOG_STR("Left channel CBC over current warning"); + return LOG_STR("Unknown fault"); +} + +const ModelInfo TAS5825M_MODEL = { + .name = model_name, + .startup_sequence = STARTUP_SEQUENCE, + .startup_sequence_length = sizeof(STARTUP_SEQUENCE) / sizeof(STARTUP_SEQUENCE[0]), + .mixer_book = 0x8C, + .mixer_page = 0x0B, + .mixer_register = 0x14, + // The clock fault is left out of the log and have_fault: it is set whenever the I2S clock stops, which is normal + .fault_error_mask = 0x0007E30F, + .fault_warning_mask = + 0x3C000000, // 0x3F000000 if OVER_TEMP_122C_WARNING and OVER_TEMP_112C_WARNING included in future + // DC and over current faults keep the output off until cleared (datasheet 7.5.3.3.1, 7.5.3.3.2). They are not + // cleared automatically: a DC fault re-trips only after 570 ms, so a clear on every poll would pass DC to the + // speaker. + .fault_output_off_mask = 0x0000000F, + .fault_latched_mask = 0x3F07E70F, + .fault_name = fault_name, + .fault_sensor_bits = + { + 3, // FAULT_SENSOR_LEFT_CHANNEL_DC_FAULT + 2, // FAULT_SENSOR_RIGHT_CHANNEL_DC_FAULT + 1, // FAULT_SENSOR_LEFT_CHANNEL_OVER_CURRENT + 0, // FAULT_SENSOR_RIGHT_CHANNEL_OVER_CURRENT + 15, // FAULT_SENSOR_OTP_CRC_CHECK + 14, // FAULT_SENSOR_BQ_WRITE_FAILED + 10, // FAULT_SENSOR_CLOCK_FAULT + 9, // FAULT_SENSOR_PVDD_OVER_VOLTAGE + 8, // FAULT_SENSOR_PVDD_UNDER_VOLTAGE + 16, // FAULT_SENSOR_OVER_TEMP_SHUTDOWN + 26, // FAULT_SENSOR_OVER_TEMP_WARNING + 13, // FAULT_SENSOR_LOAD_EEPROM_ERROR + 18, // FAULT_SENSOR_RIGHT_CHANNEL_CBC_OVER_CURRENT + 17, // FAULT_SENSOR_LEFT_CHANNEL_CBC_OVER_CURRENT + 29, // FAULT_SENSOR_LEFT_CHANNEL_CBC_OVER_CURRENT_WARNING + 28, // FAULT_SENSOR_RIGHT_CHANNEL_CBC_OVER_CURRENT_WARNING + 27, // FAULT_SENSOR_OVER_TEMP_146C_WARNING + // 25, // FAULT_SENSOR_OVER_TEMP_122C_WARNING // not currently included + // 24, // FAULT_SENSOR_OVER_TEMP_112C_WARNING // not currently included + }, +}; + +} // namespace esphome::tas58xx diff --git a/esphome/components/tas58xx/tas58xx.cpp b/esphome/components/tas58xx/tas58xx.cpp index 458150a125..d042d42e24 100644 --- a/esphome/components/tas58xx/tas58xx.cpp +++ b/esphome/components/tas58xx/tas58xx.cpp @@ -196,8 +196,9 @@ bool TAS58xx::read_faults_() { if (this->have_fault_binary_sensor_ != nullptr) this->have_fault_binary_sensor_->publish_state((active & model.fault_error_mask) != 0); for (uint8_t fault = 0; fault < FAULT_SENSOR_COUNT; fault++) { - if (this->fault_binary_sensors_[fault] != nullptr) - this->fault_binary_sensors_[fault]->publish_state(faults & (uint32_t{1} << model.fault_sensor_bits[fault])); + const uint8_t bit = model.fault_sensor_bits[fault]; + if (this->fault_binary_sensors_[fault] != nullptr && bit != NO_BIT) + this->fault_binary_sensors_[fault]->publish_state(faults & (uint32_t{1} << bit)); } #endif @@ -264,7 +265,7 @@ void TAS58xx::dump_config() { " Analog Gain: %.1f dB\n" " DAC Mode: %s\n" " Mixer Mode: %s\n" - " Volume Range: %.1f dB - %.1f dB", + " Volume Range: %.1f dB to %.1f dB", this->analog_gain_db_, this->dac_mode_ == DAC_MODE_PBTL ? LOG_STR_LITERAL("PBTL") : LOG_STR_LITERAL("BTL"), LOG_STR_ARG(mixer_mode), this->volume_min_db_, this->volume_max_db_); diff --git a/esphome/components/tas58xx/tas58xx.h b/esphome/components/tas58xx/tas58xx.h index b1e9a531dd..d0f97a5cd9 100644 --- a/esphome/components/tas58xx/tas58xx.h +++ b/esphome/components/tas58xx/tas58xx.h @@ -27,9 +27,11 @@ enum MixerMode : uint8_t { MIXER_MODE_RIGHT, }; -/// Fault binary sensors that map to a single fault bit. The Python FAULT_SENSORS list uses the same names. +/// Fault binary sensors that map to a single fault bit. Note that ordering is important. +/// The Python FAULT_SENSORS list in binary_sensor.py uses the same names and ordering. enum FaultSensor : uint8_t { - FAULT_SENSOR_LEFT_CHANNEL_DC_FAULT = 0, + // Faults common to TAS5805M and TAS5825M + FAULT_SENSOR_LEFT_CHANNEL_DC_FAULT, FAULT_SENSOR_RIGHT_CHANNEL_DC_FAULT, FAULT_SENSOR_LEFT_CHANNEL_OVER_CURRENT, FAULT_SENSOR_RIGHT_CHANNEL_OVER_CURRENT, @@ -40,9 +42,26 @@ enum FaultSensor : uint8_t { FAULT_SENSOR_PVDD_UNDER_VOLTAGE, FAULT_SENSOR_OVER_TEMP_SHUTDOWN, FAULT_SENSOR_OVER_TEMP_WARNING, - FAULT_SENSOR_COUNT, + // Faults available for TAS5825M Only + FAULT_SENSOR_LOAD_EEPROM_ERROR, + FAULT_SENSOR_RIGHT_CHANNEL_CBC_OVER_CURRENT, + FAULT_SENSOR_LEFT_CHANNEL_CBC_OVER_CURRENT, + FAULT_SENSOR_LEFT_CHANNEL_CBC_OVER_CURRENT_WARNING, + FAULT_SENSOR_RIGHT_CHANNEL_CBC_OVER_CURRENT_WARNING, + FAULT_SENSOR_OVER_TEMP_146C_WARNING, + // FAULT_SENSOR_OVER_TEMP_122C_WARNING, + // FAULT_SENSOR_OVER_TEMP_112C_WARNING, + FAULT_SENSOR_COUNT, // keep last }; +// compile time check +static_assert( + FAULT_SENSOR_COUNT == 17, + "enum FaultSensor altered: Update FAULT_SENSORS in binary_sensor.py and fault_sensor_bits in each model's cpp"); + +/// ModelInfo::fault_sensor_bits value for a FaultSensor that the model does not have. +static constexpr uint8_t NO_BIT = 0xFF; + /// Everything that differs between models of the family. One constant instance exists per model, see /// model_*.cpp, and each TAS58xx instance points to the one for its model. /// @@ -67,11 +86,16 @@ struct ModelInfo { /// Faults that stay set after the condition is gone and are cleared after each read uint32_t fault_latched_mask; const LogString *(*fault_name)(uint8_t index); - /// Bit index for each FaultSensor + /// Bit index for each FaultSensor, NO_BIT used where the model does not have that fault bit uint8_t fault_sensor_bits[FAULT_SENSOR_COUNT]; }; +#ifdef USE_TAS58XX_TAS5805M extern const ModelInfo TAS5805M_MODEL; +#endif +#ifdef USE_TAS58XX_TAS5825M +extern const ModelInfo TAS5825M_MODEL; +#endif class TAS58xx : public audio_dac::AudioDac, public PollingComponent, public i2c::I2CDevice { public: diff --git a/tests/components/tas58xx/common.yaml b/tests/components/tas58xx/common.yaml index e12ddf7567..afe38fef21 100644 --- a/tests/components/tas58xx/common.yaml +++ b/tests/components/tas58xx/common.yaml @@ -12,10 +12,9 @@ audio_dac: volume_max_db: 0dB update_interval: 1s - platform: tas58xx - model: TAS5805M + model: TAS5825M id: tas58xx_amp_2 i2c_id: i2c_bus - address: 0x2C ignore_enable_pin_warning: true esphome: @@ -59,3 +58,17 @@ binary_sensor: name: Second Amp Any Fault left_channel_dc_fault: name: Second Amp Left Channel DC Fault + left_channel_cbc_over_current: + name: Left Channel CBC Current Fault + right_channel_cbc_over_current: + name: Right Channel CBC Current Fault + over_temp_shutdown: + name: Over Temperature Shutdown Fault + left_channel_cbc_over_current_warning: + name: Left Channel CBC Current Warning + right_channel_cbc_over_current_warning: + name: Right Channel CBC Current Warning + over_temp_146c_warning: + name: Over Temperature 146C Warning + over_temp_warning: + name: Over Temperature 134C Warning From 4eeb110247c089e77658a899f2e972f11f42085d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 6 Oct 2026 15:26:46 -0500 Subject: [PATCH 208/221] [esp32] Extract prefetched ESP-IDF tool archives in parallel (#18841) --- esphome/espidf/_tool_resolution.py | 67 +++ esphome/espidf/framework.py | 78 +++- esphome/espidf/get_tool_downloads.py | 79 +--- esphome/espidf/install_tool_archives.py | 141 ++++++ .../fixtures/idf_tools_stub/idf_tools.py | 50 +- tests/unit_tests/test_espidf_framework.py | 434 +++++++++++++++++- 6 files changed, 757 insertions(+), 92 deletions(-) create mode 100644 esphome/espidf/_tool_resolution.py create mode 100644 esphome/espidf/install_tool_archives.py diff --git a/esphome/espidf/_tool_resolution.py b/esphome/espidf/_tool_resolution.py new file mode 100644 index 0000000000..624a07fa25 --- /dev/null +++ b/esphome/espidf/_tool_resolution.py @@ -0,0 +1,67 @@ +"""Shared tool resolution for the sibling idf_tools-backed scripts. + +Importable because ``python