From ecbde8ddf4f12e0530ad262ef9a285b030eb8824 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:25:08 +1200 Subject: [PATCH 001/178] [uart] Migrate check_uart_settings to final validation (#18940) --- esphome/components/cm1106/cm1106.cpp | 1 - esphome/components/cm1106/sensor.py | 8 ++++++++ esphome/components/cse7761/cse7761.cpp | 1 - esphome/components/cse7761/sensor.py | 8 +++++++- esphome/components/cse7766/cse7766.cpp | 1 - esphome/components/cse7766/sensor.py | 7 ++++++- esphome/components/daly_bms/__init__.py | 8 ++++++++ esphome/components/daly_bms/daly_bms.cpp | 5 +---- esphome/components/dfplayer/__init__.py | 7 ++++++- esphome/components/dfplayer/dfplayer.cpp | 5 +---- esphome/components/hc8/hc8.cpp | 1 - esphome/components/hc8/sensor.py | 3 +++ esphome/components/he60r/he60r.cpp | 1 - .../hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp | 2 -- esphome/components/hrxl_maxsonar_wr/sensor.py | 8 ++++++++ esphome/components/hydreon_rgxx/hydreon_rgxx.cpp | 1 - esphome/components/hydreon_rgxx/sensor.py | 8 ++++++++ esphome/components/kamstrup_kmp/kamstrup_kmp.cpp | 2 -- esphome/components/kamstrup_kmp/sensor.py | 8 +++++++- esphome/components/mhz19/mhz19.cpp | 2 -- esphome/components/mhz19/sensor.py | 8 ++++++++ esphome/components/mk2pvrouter/mk2pvrouter.cpp | 5 +---- esphome/components/mk2pvrouter/mk2pvrouter.h | 1 - esphome/components/pm1006/pm1006.cpp | 1 - esphome/components/pm1006/sensor.py | 3 +++ esphome/components/pmsx003/pmsx003.cpp | 2 -- esphome/components/pmsx003/sensor.py | 8 +++++++- esphome/components/pylontech/__init__.py | 8 ++++++++ esphome/components/pylontech/pylontech.cpp | 1 - esphome/components/seeed_mr60fda2/__init__.py | 1 + .../components/seeed_mr60fda2/seeed_mr60fda2.cpp | 2 -- esphome/components/smt100/sensor.py | 8 +++++++- esphome/components/smt100/smt100.cpp | 1 - esphome/components/t6615/sensor.py | 8 +++++++- esphome/components/t6615/t6615.cpp | 1 - esphome/components/teleinfo/__init__.py | 16 ++++++++++++++++ esphome/components/teleinfo/teleinfo.cpp | 7 +------ esphome/components/teleinfo/teleinfo.h | 1 - esphome/components/tormatic/tormatic_cover.cpp | 2 -- esphome/components/uart/uart.h | 2 ++ esphome/components/ufm01/__init__.py | 1 + esphome/components/ufm01/ufm01.cpp | 1 - esphome/components/uponor_smatrix/__init__.py | 2 +- .../components/uponor_smatrix/uponor_smatrix.cpp | 2 -- esphome/components/vbus/__init__.py | 8 ++++++++ esphome/components/vbus/vbus.cpp | 5 +---- esphome/components/wl_134/text_sensor.py | 8 ++++++++ esphome/components/wl_134/wl_134.cpp | 2 -- tests/components/cse7761/test.esp32-idf.yaml | 2 +- tests/components/cse7761/test.esp8266-ard.yaml | 2 +- tests/components/cse7761/test.rp2040-ard.yaml | 2 +- .../components/kamstrup_kmp/test.esp32-idf.yaml | 2 +- .../kamstrup_kmp/test.esp8266-ard.yaml | 2 +- tests/components/pylontech/test.esp32-idf.yaml | 2 +- tests/components/pylontech/test.esp8266-ard.yaml | 2 +- tests/components/pylontech/test.rp2040-ard.yaml | 2 +- tests/components/teleinfo/test.esp32-idf.yaml | 2 +- tests/components/teleinfo/test.esp8266-ard.yaml | 2 +- tests/components/teleinfo/test.rp2040-ard.yaml | 2 +- .../teleinfo/validate-standard.esp32-idf.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/esp32-ard.yaml | 14 ++++++++++++++ .../uart_1200_even_7bits/esp32-c3-ard.yaml | 14 ++++++++++++++ .../uart_1200_even_7bits/esp32-c3-idf.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/esp32-idf.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/esp8266-ard.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/rp2040-ard.yaml | 14 ++++++++++++++ .../common/uart_38400_even/esp32-ard.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp32-c3-ard.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp32-c3-idf.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp32-idf.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp8266-ard.yaml | 12 ++++++++++++ .../common/uart_38400_even/rp2040-ard.yaml | 12 ++++++++++++ 72 files changed, 324 insertions(+), 70 deletions(-) create mode 100644 tests/components/teleinfo/validate-standard.esp32-idf.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-idf.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml diff --git a/esphome/components/cm1106/cm1106.cpp b/esphome/components/cm1106/cm1106.cpp index 7e5d25b7ae..2e3352b895 100644 --- a/esphome/components/cm1106/cm1106.cpp +++ b/esphome/components/cm1106/cm1106.cpp @@ -100,7 +100,6 @@ bool CM1106Component::cm1106_write_command_(const uint8_t *command, size_t comma void CM1106Component::dump_config() { ESP_LOGCONFIG(TAG, "CM1106:"); LOG_SENSOR(" ", "CO2", this->co2_sensor_); - this->check_uart_settings(9600); if (this->is_failed()) { ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); } diff --git a/esphome/components/cm1106/sensor.py b/esphome/components/cm1106/sensor.py index 936c5fc673..a36f0b0059 100644 --- a/esphome/components/cm1106/sensor.py +++ b/esphome/components/cm1106/sensor.py @@ -46,6 +46,14 @@ CONFIG_SCHEMA = ( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "cm1106", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: """Code generation entry point.""" diff --git a/esphome/components/cse7761/cse7761.cpp b/esphome/components/cse7761/cse7761.cpp index 4251751531..103bc84452 100644 --- a/esphome/components/cse7761/cse7761.cpp +++ b/esphome/components/cse7761/cse7761.cpp @@ -58,7 +58,6 @@ void CSE7761Component::dump_config() { ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); } LOG_UPDATE_INTERVAL(this); - this->check_uart_settings(38400, 1, uart::UART_CONFIG_PARITY_EVEN, 8); } void CSE7761Component::update() { diff --git a/esphome/components/cse7761/sensor.py b/esphome/components/cse7761/sensor.py index b53ed26ca3..5f79be0255 100644 --- a/esphome/components/cse7761/sensor.py +++ b/esphome/components/cse7761/sensor.py @@ -68,7 +68,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "cse7761", baud_rate=38400, require_rx=True, require_tx=True + "cse7761", + baud_rate=38400, + require_rx=True, + require_tx=True, + data_bits=8, + parity="EVEN", + stop_bits=1, ) diff --git a/esphome/components/cse7766/cse7766.cpp b/esphome/components/cse7766/cse7766.cpp index ce77b62b7b..30f1b7a867 100644 --- a/esphome/components/cse7766/cse7766.cpp +++ b/esphome/components/cse7766/cse7766.cpp @@ -255,7 +255,6 @@ void CSE7766Component::dump_config() { LOG_SENSOR(" ", "Apparent Power", this->apparent_power_sensor_); LOG_SENSOR(" ", "Reactive Power", this->reactive_power_sensor_); LOG_SENSOR(" ", "Power Factor", this->power_factor_sensor_); - this->check_uart_settings(4800, 1, uart::UART_CONFIG_PARITY_EVEN); } } // namespace esphome::cse7766 diff --git a/esphome/components/cse7766/sensor.py b/esphome/components/cse7766/sensor.py index a1a68e18e8..9bed0f3f59 100644 --- a/esphome/components/cse7766/sensor.py +++ b/esphome/components/cse7766/sensor.py @@ -84,7 +84,12 @@ CONFIG_SCHEMA = ( .extend(cv.COMPONENT_SCHEMA) ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "cse7766", baud_rate=4800, parity="EVEN", require_rx=True + "cse7766", + baud_rate=4800, + require_rx=True, + data_bits=8, + parity="EVEN", + stop_bits=1, ) diff --git a/esphome/components/daly_bms/__init__.py b/esphome/components/daly_bms/__init__.py index ba0be4d3a5..c0d7d0aa62 100644 --- a/esphome/components/daly_bms/__init__.py +++ b/esphome/components/daly_bms/__init__.py @@ -26,6 +26,14 @@ CONFIG_SCHEMA = ( .extend(cv.polling_component_schema("30s")) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "daly_bms", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/daly_bms/daly_bms.cpp b/esphome/components/daly_bms/daly_bms.cpp index 530d8ad541..45d4db4972 100644 --- a/esphome/components/daly_bms/daly_bms.cpp +++ b/esphome/components/daly_bms/daly_bms.cpp @@ -22,10 +22,7 @@ static const uint8_t DALY_REQUEST_TEMPERATURE = 0x96; void DalyBmsComponent::setup() { this->next_request_ = 1; } -void DalyBmsComponent::dump_config() { - ESP_LOGCONFIG(TAG, "Daly BMS:"); - this->check_uart_settings(9600); -} +void DalyBmsComponent::dump_config() { ESP_LOGCONFIG(TAG, "Daly BMS:"); } void DalyBmsComponent::update() { this->trigger_next_ = true; diff --git a/esphome/components/dfplayer/__init__.py b/esphome/components/dfplayer/__init__.py index d589381461..bb18e6ba8c 100644 --- a/esphome/components/dfplayer/__init__.py +++ b/esphome/components/dfplayer/__init__.py @@ -60,7 +60,12 @@ CONFIG_SCHEMA = cv.All( ).extend(uart.UART_DEVICE_SCHEMA) ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "dfplayer", baud_rate=9600, require_tx=True + "dfplayer", + baud_rate=9600, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/dfplayer/dfplayer.cpp b/esphome/components/dfplayer/dfplayer.cpp index 5c9d497c87..f81d1cd1b6 100644 --- a/esphome/components/dfplayer/dfplayer.cpp +++ b/esphome/components/dfplayer/dfplayer.cpp @@ -277,9 +277,6 @@ void DFPlayer::loop() { } } } -void DFPlayer::dump_config() { - ESP_LOGCONFIG(TAG, "DFPlayer:"); - this->check_uart_settings(9600); -} +void DFPlayer::dump_config() { ESP_LOGCONFIG(TAG, "DFPlayer:"); } } // namespace esphome::dfplayer diff --git a/esphome/components/hc8/hc8.cpp b/esphome/components/hc8/hc8.cpp index 900acca691..6a19f977a6 100644 --- a/esphome/components/hc8/hc8.cpp +++ b/esphome/components/hc8/hc8.cpp @@ -96,7 +96,6 @@ void HC8Component::dump_config() { " Warmup time: %" PRIu32 " s", this->warmup_seconds_); LOG_SENSOR(" ", "CO2", this->co2_sensor_); - this->check_uart_settings(9600); } } // namespace esphome::hc8 diff --git a/esphome/components/hc8/sensor.py b/esphome/components/hc8/sensor.py index 616162eb40..8a19cce8d1 100644 --- a/esphome/components/hc8/sensor.py +++ b/esphome/components/hc8/sensor.py @@ -47,6 +47,9 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( baud_rate=9600, require_rx=True, require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/he60r/he60r.cpp b/esphome/components/he60r/he60r.cpp index f49224f17c..008505e2bb 100644 --- a/esphome/components/he60r/he60r.cpp +++ b/esphome/components/he60r/he60r.cpp @@ -38,7 +38,6 @@ CoverTraits HE60rCover::get_traits() { void HE60rCover::dump_config() { LOG_COVER("", "HE60R Cover", this); - this->check_uart_settings(1200, 1, uart::UART_CONFIG_PARITY_EVEN, 8); ESP_LOGCONFIG(TAG, " Open Duration: %.1fs\n" " Close Duration: %.1fs", diff --git a/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp b/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp index 270bb2709d..b323dd0436 100644 --- a/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp +++ b/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp @@ -68,8 +68,6 @@ void HrxlMaxsonarWrComponent::check_buffer_() { void HrxlMaxsonarWrComponent::dump_config() { ESP_LOGCONFIG(TAG, "HRXL MaxSonar WR Sensor:"); LOG_SENSOR(" ", "Distance", this); - // As specified in the sensor's data sheet - this->check_uart_settings(9600, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); } } // namespace esphome::hrxl_maxsonar_wr diff --git a/esphome/components/hrxl_maxsonar_wr/sensor.py b/esphome/components/hrxl_maxsonar_wr/sensor.py index e4daacd869..b81a8b273d 100644 --- a/esphome/components/hrxl_maxsonar_wr/sensor.py +++ b/esphome/components/hrxl_maxsonar_wr/sensor.py @@ -23,6 +23,14 @@ CONFIG_SCHEMA = sensor.sensor_schema( state_class=STATE_CLASS_MEASUREMENT, ).extend(uart.UART_DEVICE_SCHEMA) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "hrxl_maxsonar_wr", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = await sensor.new_sensor(config) diff --git a/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp b/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp index 695a823cb7..05557111fc 100644 --- a/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp +++ b/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp @@ -11,7 +11,6 @@ static const char *const PROTOCOL_NAMES[] = {HYDREON_RGXX_PROTOCOL_LIST(, HYDREO static const char *const IGNORE_STRINGS[] = {HYDREON_RGXX_IGNORE_LIST(, HYDREON_RGXX_COMMA)}; void HydreonRGxxComponent::dump_config() { - this->check_uart_settings(9600, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); ESP_LOGCONFIG(TAG, "hydreon_rgxx:"); if (this->is_failed()) { ESP_LOGE(TAG, "Connection with hydreon_rgxx failed!"); diff --git a/esphome/components/hydreon_rgxx/sensor.py b/esphome/components/hydreon_rgxx/sensor.py index 58e72571ff..8e269fef9a 100644 --- a/esphome/components/hydreon_rgxx/sensor.py +++ b/esphome/components/hydreon_rgxx/sensor.py @@ -130,6 +130,14 @@ CONFIG_SCHEMA = cv.All( _validate, ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "hydreon_rgxx", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp b/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp index 70f6d4eaa7..24e5d25921 100644 --- a/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp +++ b/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp @@ -26,8 +26,6 @@ void KamstrupKMPComponent::dump_config() { LOG_SENSOR(" ", "Custom Sensor", this->custom_sensors_[i]); ESP_LOGCONFIG(TAG, " Command: 0x%04X", this->custom_commands_[i]); } - - this->check_uart_settings(1200, 2, uart::UART_CONFIG_PARITY_NONE, 8); } void KamstrupKMPComponent::update() { diff --git a/esphome/components/kamstrup_kmp/sensor.py b/esphome/components/kamstrup_kmp/sensor.py index 6465012897..f6c236b72d 100644 --- a/esphome/components/kamstrup_kmp/sensor.py +++ b/esphome/components/kamstrup_kmp/sensor.py @@ -102,7 +102,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "kamstrup_kmp", baud_rate=1200, require_rx=True, require_tx=True + "kamstrup_kmp", + baud_rate=1200, + require_rx=True, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=2, ) diff --git a/esphome/components/mhz19/mhz19.cpp b/esphome/components/mhz19/mhz19.cpp index ff518808d9..707d952f83 100644 --- a/esphome/components/mhz19/mhz19.cpp +++ b/esphome/components/mhz19/mhz19.cpp @@ -143,8 +143,6 @@ void MHZ19Component::dump_config() { ESP_LOGCONFIG(TAG, "MH-Z19:"); LOG_SENSOR(" ", "CO2", this->co2_sensor_); LOG_SENSOR(" ", "Temperature", this->temperature_sensor_); - this->check_uart_settings(9600); - if (this->abc_boot_logic_ == MHZ19_ABC_ENABLED) { ESP_LOGCONFIG(TAG, " Automatic baseline calibration enabled on boot"); } else if (this->abc_boot_logic_ == MHZ19_ABC_DISABLED) { diff --git a/esphome/components/mhz19/sensor.py b/esphome/components/mhz19/sensor.py index 33cb27080c..5852686608 100644 --- a/esphome/components/mhz19/sensor.py +++ b/esphome/components/mhz19/sensor.py @@ -80,6 +80,14 @@ CONFIG_SCHEMA = ( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "mhz19", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.cpp b/esphome/components/mk2pvrouter/mk2pvrouter.cpp index a9c922602b..0c0476fb11 100644 --- a/esphome/components/mk2pvrouter/mk2pvrouter.cpp +++ b/esphome/components/mk2pvrouter/mk2pvrouter.cpp @@ -163,10 +163,7 @@ void Mk2PVRouter::publish_value_(const char *tag, const char *val) { #endif } -void Mk2PVRouter::dump_config() { - ESP_LOGCONFIG(TAG, "Mk2PVRouter:"); - this->check_uart_settings(BAUD_RATE, 1, uart::UART_CONFIG_PARITY_EVEN, 7); -} +void Mk2PVRouter::dump_config() { ESP_LOGCONFIG(TAG, "Mk2PVRouter:"); } #ifdef MK2PVROUTER_LISTENER_COUNT void Mk2PVRouter::register_mk2pvrouter_listener(Mk2PVRouterListener *listener) { diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.h b/esphome/components/mk2pvrouter/mk2pvrouter.h index f542436f1d..fc23cf49e8 100644 --- a/esphome/components/mk2pvrouter/mk2pvrouter.h +++ b/esphome/components/mk2pvrouter/mk2pvrouter.h @@ -43,7 +43,6 @@ class Mk2PVRouter final : public Component, public uart::UARTDevice { protected: static constexpr size_t CRC_SUFFIX_LEN = 1; - static constexpr uint32_t BAUD_RATE = 9600; enum class State : uint8_t { WAITING_FOR_START, diff --git a/esphome/components/pm1006/pm1006.cpp b/esphome/components/pm1006/pm1006.cpp index 6a325c57dc..d4c6824713 100644 --- a/esphome/components/pm1006/pm1006.cpp +++ b/esphome/components/pm1006/pm1006.cpp @@ -16,7 +16,6 @@ void PM1006Component::dump_config() { ESP_LOGCONFIG(TAG, "PM1006:"); LOG_SENSOR(" ", "PM2.5", this->pm_2_5_sensor_); LOG_UPDATE_INTERVAL(this); - this->check_uart_settings(9600); } void PM1006Component::update() { diff --git a/esphome/components/pm1006/sensor.py b/esphome/components/pm1006/sensor.py index 8274726ac4..447671ebb3 100644 --- a/esphome/components/pm1006/sensor.py +++ b/esphome/components/pm1006/sensor.py @@ -48,6 +48,9 @@ def validate_interval_uart(config: ConfigType) -> None: baud_rate=9600, require_rx=True, require_tx=interval.total_milliseconds != SCHEDULER_DONT_RUN, + data_bits=8, + parity="NONE", + stop_bits=1, )(config) diff --git a/esphome/components/pmsx003/pmsx003.cpp b/esphome/components/pmsx003/pmsx003.cpp index 6275ff60c2..f8d890ac9e 100644 --- a/esphome/components/pmsx003/pmsx003.cpp +++ b/esphome/components/pmsx003/pmsx003.cpp @@ -46,8 +46,6 @@ void PMSX003Component::dump_config() { } else { ESP_LOGCONFIG(TAG, " Mode: passive with sleep/wake cycles"); } - - this->check_uart_settings(9600); } void PMSX003Component::loop() { diff --git a/esphome/components/pmsx003/sensor.py b/esphome/components/pmsx003/sensor.py index fe784c5ffe..dc85380203 100644 --- a/esphome/components/pmsx003/sensor.py +++ b/esphome/components/pmsx003/sensor.py @@ -302,7 +302,13 @@ CONFIG_SCHEMA = cv.All( def final_validate(config: ConfigType) -> None: require_tx = config[CONF_UPDATE_INTERVAL] > cv.time_period("0s") schema = uart.final_validate_device_schema( - "pmsx003", baud_rate=9600, require_rx=True, require_tx=require_tx + "pmsx003", + baud_rate=9600, + require_rx=True, + require_tx=require_tx, + data_bits=8, + parity="NONE", + stop_bits=1, ) schema(config) diff --git a/esphome/components/pylontech/__init__.py b/esphome/components/pylontech/__init__.py index 4ab606d9f9..242a613a6c 100644 --- a/esphome/components/pylontech/__init__.py +++ b/esphome/components/pylontech/__init__.py @@ -41,6 +41,14 @@ CONFIG_SCHEMA = cv.All( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "pylontech", + baud_rate=115200, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/pylontech/pylontech.cpp b/esphome/components/pylontech/pylontech.cpp index 54d9e5c654..932b71ba55 100644 --- a/esphome/components/pylontech/pylontech.cpp +++ b/esphome/components/pylontech/pylontech.cpp @@ -33,7 +33,6 @@ static const uint8_t ASCII_LF = 0x0A; PylontechComponent::PylontechComponent() {} void PylontechComponent::dump_config() { - this->check_uart_settings(115200, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); ESP_LOGCONFIG(TAG, "pylontech:"); if (this->is_failed()) { ESP_LOGE(TAG, "Connection with pylontech failed!"); diff --git a/esphome/components/seeed_mr60fda2/__init__.py b/esphome/components/seeed_mr60fda2/__init__.py index de6e8ad57b..159a1ece9c 100644 --- a/esphome/components/seeed_mr60fda2/__init__.py +++ b/esphome/components/seeed_mr60fda2/__init__.py @@ -31,6 +31,7 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( require_tx=True, require_rx=True, baud_rate=115200, + data_bits=8, parity="NONE", stop_bits=1, ) diff --git a/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp b/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp index 4875aa5cff..2d1cd0fbb4 100644 --- a/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp +++ b/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp @@ -33,8 +33,6 @@ void MR60FDA2Component::dump_config() { // Initialisation functions void MR60FDA2Component::setup() { - this->check_uart_settings(115200); - this->current_frame_locate_ = LOCATE_FRAME_HEADER; this->current_frame_id_ = 0; this->current_frame_len_ = 0; diff --git a/esphome/components/smt100/sensor.py b/esphome/components/smt100/sensor.py index 632a1e7547..7ba7da801c 100644 --- a/esphome/components/smt100/sensor.py +++ b/esphome/components/smt100/sensor.py @@ -68,7 +68,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "smt100", baud_rate=9600, require_rx=True, require_tx=True + "smt100", + baud_rate=9600, + require_rx=True, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/smt100/smt100.cpp b/esphome/components/smt100/smt100.cpp index ed33fc54c5..2889a9fb4d 100644 --- a/esphome/components/smt100/smt100.cpp +++ b/esphome/components/smt100/smt100.cpp @@ -65,7 +65,6 @@ void SMT100Component::dump_config() { LOG_SENSOR(TAG, "Temperature", this->temperature_sensor_); LOG_SENSOR(TAG, "Moisture", this->moisture_sensor_); LOG_UPDATE_INTERVAL(this); - this->check_uart_settings(9600); } int SMT100Component::readline_(int readch, char *buffer, int len) { diff --git a/esphome/components/t6615/sensor.py b/esphome/components/t6615/sensor.py index 6f3ef372bc..44dba52ae8 100644 --- a/esphome/components/t6615/sensor.py +++ b/esphome/components/t6615/sensor.py @@ -33,7 +33,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "t6615", baud_rate=19200, require_rx=True, require_tx=True + "t6615", + baud_rate=19200, + require_rx=True, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/t6615/t6615.cpp b/esphome/components/t6615/t6615.cpp index 1a98e48c14..982cc181b7 100644 --- a/esphome/components/t6615/t6615.cpp +++ b/esphome/components/t6615/t6615.cpp @@ -88,7 +88,6 @@ void T6615Component::query_ppm_() { void T6615Component::dump_config() { ESP_LOGCONFIG(TAG, "T6615:"); LOG_SENSOR(" ", "CO2", this->co2_sensor_); - this->check_uart_settings(19200); } } // namespace esphome::t6615 diff --git a/esphome/components/teleinfo/__init__.py b/esphome/components/teleinfo/__init__.py index f9233511e1..67aad11d0f 100644 --- a/esphome/components/teleinfo/__init__.py +++ b/esphome/components/teleinfo/__init__.py @@ -35,6 +35,22 @@ CONFIG_SCHEMA = ( ) +def _final_validate(config: ConfigType) -> ConfigType: + # Historical mode runs at 1200 baud, standard mode at 9600 baud. + baud_rate = 1200 if config[CONF_HISTORICAL_MODE] else 9600 + uart.final_validate_device_schema( + "teleinfo", + baud_rate=baud_rate, + data_bits=7, + parity="EVEN", + stop_bits=1, + )(config) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID], config[CONF_HISTORICAL_MODE]) await cg.register_component(var, config) diff --git a/esphome/components/teleinfo/teleinfo.cpp b/esphome/components/teleinfo/teleinfo.cpp index e00895d162..17d3d6c099 100644 --- a/esphome/components/teleinfo/teleinfo.cpp +++ b/esphome/components/teleinfo/teleinfo.cpp @@ -184,10 +184,7 @@ void TeleInfo::publish_value_(const std::string &tag, const std::string &val) { element->publish_val(val); } } -void TeleInfo::dump_config() { - ESP_LOGCONFIG(TAG, "TeleInfo:"); - this->check_uart_settings(baud_rate_, 1, uart::UART_CONFIG_PARITY_EVEN, 7); -} +void TeleInfo::dump_config() { ESP_LOGCONFIG(TAG, "TeleInfo:"); } TeleInfo::TeleInfo(bool historical_mode) { if (historical_mode) { /* @@ -195,11 +192,9 @@ TeleInfo::TeleInfo(bool historical_mode) { */ checksum_area_end_ = 2; separator_ = 0x20; - baud_rate_ = 1200; } else { checksum_area_end_ = 1; separator_ = 0x9; - baud_rate_ = 9600; } } void TeleInfo::register_teleinfo_listener(TeleInfoListener *listener) { teleinfo_listeners_.push_back(listener); } diff --git a/esphome/components/teleinfo/teleinfo.h b/esphome/components/teleinfo/teleinfo.h index 4aab3bf2cd..b1bf586e9c 100644 --- a/esphome/components/teleinfo/teleinfo.h +++ b/esphome/components/teleinfo/teleinfo.h @@ -31,7 +31,6 @@ class TeleInfo final : public PollingComponent, public uart::UARTDevice { std::vector teleinfo_listeners_{}; protected: - uint32_t baud_rate_; int checksum_area_end_; int separator_; char buf_[MAX_BUF_SIZE]; diff --git a/esphome/components/tormatic/tormatic_cover.cpp b/esphome/components/tormatic/tormatic_cover.cpp index 7004c4f836..5c8d6623b6 100644 --- a/esphome/components/tormatic/tormatic_cover.cpp +++ b/esphome/components/tormatic/tormatic_cover.cpp @@ -36,8 +36,6 @@ cover::CoverTraits Tormatic::get_traits() { void Tormatic::dump_config() { LOG_COVER("", "Tormatic Cover", this); - this->check_uart_settings(9600, 1, uart::UART_CONFIG_PARITY_NONE, 8); - ESP_LOGCONFIG(TAG, " Open Duration: %.1fs\n" " Close Duration: %.1fs", diff --git a/esphome/components/uart/uart.h b/esphome/components/uart/uart.h index 899d349e21..eda5b72ea8 100644 --- a/esphome/components/uart/uart.h +++ b/esphome/components/uart/uart.h @@ -3,6 +3,7 @@ #include #include "esphome/core/component.h" #include "esphome/core/hal.h" +#include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "uart_component.h" @@ -66,6 +67,7 @@ class UARTDevice { } /// Check that the configuration of the UART bus matches the provided values and otherwise print a warning + ESPDEPRECATED("Use uart.final_validate_device_schema() in Python instead. Removed in 2027.3.0", "2026.9.0") void check_uart_settings(uint32_t baud_rate, uint8_t stop_bits = 1, UARTParityOptions parity = UART_CONFIG_PARITY_NONE, uint8_t data_bits = 8); diff --git a/esphome/components/ufm01/__init__.py b/esphome/components/ufm01/__init__.py index ca0ea57796..85ca0eecae 100644 --- a/esphome/components/ufm01/__init__.py +++ b/esphome/components/ufm01/__init__.py @@ -30,6 +30,7 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( require_tx=True, require_rx=True, baud_rate=2400, + data_bits=8, parity="EVEN", stop_bits=1, ) diff --git a/esphome/components/ufm01/ufm01.cpp b/esphome/components/ufm01/ufm01.cpp index bafdb5d853..880132bad3 100644 --- a/esphome/components/ufm01/ufm01.cpp +++ b/esphome/components/ufm01/ufm01.cpp @@ -213,7 +213,6 @@ void UFM01Component::dump_config() { LOG_BINARY_SENSOR(" ", "Empty Tube", this->empty_tube_binary_sensor_); LOG_BINARY_SENSOR(" ", "Flow Rate Out Of Range", this->flow_rate_out_of_range_binary_sensor_); #endif - this->check_uart_settings(2400, 1, uart::UART_CONFIG_PARITY_EVEN, 8); } void UFM01Component::on_active_frame_(uint8_t data[FRAME_SIZE]) { diff --git a/esphome/components/uponor_smatrix/__init__.py b/esphome/components/uponor_smatrix/__init__.py index 093408e868..ba686dc22a 100644 --- a/esphome/components/uponor_smatrix/__init__.py +++ b/esphome/components/uponor_smatrix/__init__.py @@ -50,7 +50,7 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( require_tx=True, require_rx=True, data_bits=8, - parity=None, + parity="NONE", stop_bits=1, ) diff --git a/esphome/components/uponor_smatrix/uponor_smatrix.cpp b/esphome/components/uponor_smatrix/uponor_smatrix.cpp index c77f3468c7..74974548af 100644 --- a/esphome/components/uponor_smatrix/uponor_smatrix.cpp +++ b/esphome/components/uponor_smatrix/uponor_smatrix.cpp @@ -29,8 +29,6 @@ void UponorSmatrixComponent::dump_config() { } #endif - this->check_uart_settings(19200); - if (!this->unknown_devices_.empty()) { ESP_LOGCONFIG(TAG, " Detected unknown device addresses:"); for (auto device_address : this->unknown_devices_) { diff --git a/esphome/components/vbus/__init__.py b/esphome/components/vbus/__init__.py index 94857050f2..fd54658912 100644 --- a/esphome/components/vbus/__init__.py +++ b/esphome/components/vbus/__init__.py @@ -29,6 +29,14 @@ CONFIG_SCHEMA = uart.UART_DEVICE_SCHEMA.extend( } ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "vbus", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/vbus/vbus.cpp b/esphome/components/vbus/vbus.cpp index 81714a2049..080567e7f9 100644 --- a/esphome/components/vbus/vbus.cpp +++ b/esphome/components/vbus/vbus.cpp @@ -11,10 +11,7 @@ static const char *const TAG = "vbus"; // Maximum bytes to log in verbose hex output (16 frames * 4 bytes = 64 bytes typical) static constexpr size_t VBUS_MAX_LOG_BYTES = 64; -void VBus::dump_config() { - ESP_LOGCONFIG(TAG, "VBus:"); - check_uart_settings(9600); -} +void VBus::dump_config() { ESP_LOGCONFIG(TAG, "VBus:"); } static void septet_spread(uint8_t *data, int start, int count, uint8_t septet) { for (int i = 0; i < count; i++, septet >>= 1) { diff --git a/esphome/components/wl_134/text_sensor.py b/esphome/components/wl_134/text_sensor.py index af5e705786..2e3021504f 100644 --- a/esphome/components/wl_134/text_sensor.py +++ b/esphome/components/wl_134/text_sensor.py @@ -21,6 +21,14 @@ CONFIG_SCHEMA = ( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "wl_134", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = await text_sensor.new_text_sensor(config) diff --git a/esphome/components/wl_134/wl_134.cpp b/esphome/components/wl_134/wl_134.cpp index 5e86d5a441..858f974f2b 100644 --- a/esphome/components/wl_134/wl_134.cpp +++ b/esphome/components/wl_134/wl_134.cpp @@ -110,7 +110,5 @@ uint64_t Wl134Component::hex_lsb_ascii_to_uint64_(const uint8_t *text, uint8_t t void Wl134Component::dump_config() { ESP_LOGCONFIG(TAG, "WL-134 Sensor:"); LOG_TEXT_SENSOR("", "Tag", this); - // As specified in the sensor's data sheet - this->check_uart_settings(9600, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); } } // namespace esphome::wl_134 diff --git a/tests/components/cse7761/test.esp32-idf.yaml b/tests/components/cse7761/test.esp32-idf.yaml index a6a8fee7e9..b9ae061c25 100644 --- a/tests/components/cse7761/test.esp32-idf.yaml +++ b/tests/components/cse7761/test.esp32-idf.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO14 packages: - uart_38400: !include ../../test_build_components/common/uart_38400/esp32-idf.yaml + uart_38400_even: !include ../../test_build_components/common/uart_38400_even/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/cse7761/test.esp8266-ard.yaml b/tests/components/cse7761/test.esp8266-ard.yaml index 134274ffb8..0d57039e1c 100644 --- a/tests/components/cse7761/test.esp8266-ard.yaml +++ b/tests/components/cse7761/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO3 packages: - uart_38400: !include ../../test_build_components/common/uart_38400/esp8266-ard.yaml + uart_38400_even: !include ../../test_build_components/common/uart_38400_even/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/cse7761/test.rp2040-ard.yaml b/tests/components/cse7761/test.rp2040-ard.yaml index b813e0f7f1..65e6252c51 100644 --- a/tests/components/cse7761/test.rp2040-ard.yaml +++ b/tests/components/cse7761/test.rp2040-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart_38400: !include ../../test_build_components/common/uart_38400/rp2040-ard.yaml + uart_38400_even: !include ../../test_build_components/common/uart_38400_even/rp2040-ard.yaml <<: !include common.yaml diff --git a/tests/components/kamstrup_kmp/test.esp32-idf.yaml b/tests/components/kamstrup_kmp/test.esp32-idf.yaml index 1016905720..4e1ff86fb7 100644 --- a/tests/components/kamstrup_kmp/test.esp32-idf.yaml +++ b/tests/components/kamstrup_kmp/test.esp32-idf.yaml @@ -1,4 +1,4 @@ packages: - uart_1200: !include ../../test_build_components/common/uart_1200/esp32-idf.yaml + uart_1200_none_2stopbits: !include ../../test_build_components/common/uart_1200_none_2stopbits/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/kamstrup_kmp/test.esp8266-ard.yaml b/tests/components/kamstrup_kmp/test.esp8266-ard.yaml index f55c18eb76..631516eba9 100644 --- a/tests/components/kamstrup_kmp/test.esp8266-ard.yaml +++ b/tests/components/kamstrup_kmp/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: uart_rx_pin: GPIO3 packages: - uart_1200: !include ../../test_build_components/common/uart_1200/esp8266-ard.yaml + uart_1200_none_2stopbits: !include ../../test_build_components/common/uart_1200_none_2stopbits/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/pylontech/test.esp32-idf.yaml b/tests/components/pylontech/test.esp32-idf.yaml index b415125e84..7d5c371187 100644 --- a/tests/components/pylontech/test.esp32-idf.yaml +++ b/tests/components/pylontech/test.esp32-idf.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + uart_115200: !include ../../test_build_components/common/uart_115200/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/pylontech/test.esp8266-ard.yaml b/tests/components/pylontech/test.esp8266-ard.yaml index 96ab4ef6ac..c49b2bfee1 100644 --- a/tests/components/pylontech/test.esp8266-ard.yaml +++ b/tests/components/pylontech/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO2 packages: - uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + uart_115200: !include ../../test_build_components/common/uart_115200/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/pylontech/test.rp2040-ard.yaml b/tests/components/pylontech/test.rp2040-ard.yaml index b28f2b5e05..5b2785b792 100644 --- a/tests/components/pylontech/test.rp2040-ard.yaml +++ b/tests/components/pylontech/test.rp2040-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + uart_115200: !include ../../test_build_components/common/uart_115200/rp2040-ard.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/test.esp32-idf.yaml b/tests/components/teleinfo/test.esp32-idf.yaml index b415125e84..3071f9a67b 100644 --- a/tests/components/teleinfo/test.esp32-idf.yaml +++ b/tests/components/teleinfo/test.esp32-idf.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + uart_1200_even_7bits: !include ../../test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/test.esp8266-ard.yaml b/tests/components/teleinfo/test.esp8266-ard.yaml index 96ab4ef6ac..29490b3be3 100644 --- a/tests/components/teleinfo/test.esp8266-ard.yaml +++ b/tests/components/teleinfo/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO2 packages: - uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + uart_1200_even_7bits: !include ../../test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/test.rp2040-ard.yaml b/tests/components/teleinfo/test.rp2040-ard.yaml index b28f2b5e05..f13d5a9f8f 100644 --- a/tests/components/teleinfo/test.rp2040-ard.yaml +++ b/tests/components/teleinfo/test.rp2040-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + uart_1200_even_7bits: !include ../../test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/validate-standard.esp32-idf.yaml b/tests/components/teleinfo/validate-standard.esp32-idf.yaml new file mode 100644 index 0000000000..2ca014c8af --- /dev/null +++ b/tests/components/teleinfo/validate-standard.esp32-idf.yaml @@ -0,0 +1,14 @@ +packages: + uart_9600_even_7bits: !include ../../test_build_components/common/uart_9600_even_7bits/esp32-idf.yaml + +teleinfo: + id: test_teleinfo_standard + historical_mode: false + update_interval: 60s + +sensor: + - platform: teleinfo + name: sinsts + tag_name: SINSTS + teleinfo_id: test_teleinfo_standard + unit_of_measurement: VA diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml new file mode 100644 index 0000000000..931905032d --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml new file mode 100644 index 0000000000..a67b0b6ace --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32-C3 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml new file mode 100644 index 0000000000..135aaa68c9 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32-C3 IDF tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml new file mode 100644 index 0000000000..4cbe16dfd5 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32 IDF tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml new file mode 100644 index 0000000000..2eedcad6d3 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP8266 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml new file mode 100644 index 0000000000..d3edc1c1c9 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for RP2040 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO1 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_38400_even/esp32-ard.yaml b/tests/test_build_components/common/uart_38400_even/esp32-ard.yaml new file mode 100644 index 0000000000..4235c9c027 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml b/tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml new file mode 100644 index 0000000000..c20b7939e9 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32-C3 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml b/tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml new file mode 100644 index 0000000000..0aeb13a7c3 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32-C3 IDF tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp32-idf.yaml b/tests/test_build_components/common/uart_38400_even/esp32-idf.yaml new file mode 100644 index 0000000000..b79b91448e --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-idf.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32 IDF tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml b/tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml new file mode 100644 index 0000000000..373680e8e6 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP8266 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml b/tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml new file mode 100644 index 0000000000..950f7b4957 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for RP2040 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO1 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN From 567f7f9196425e0b8637b16b2a43373e376e2df4 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Wed, 2 Sep 2026 15:01:23 -0500 Subject: [PATCH 002/178] [serial_proxy] Skip no-op reconfigure requests (#18953) Co-authored-by: puddly <32534428+puddly@users.noreply.github.com> --- .../components/serial_proxy/serial_proxy.cpp | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index 2ab0d4ebb4..c1c1510643 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -130,17 +130,26 @@ SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uin return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; } - // Apply validated parameters - uart_comp->set_baud_rate(baudrate); - uart_comp->set_stop_bits(stop_bits); - uart_comp->set_data_bits(data_size); - - // Map parity value to UARTParityOptions + // Skip a no-op reconfigure. Clients routinely re-send identical settings on every + // port open, and on a USB UART each apply is a CDC SET_LINE_CODING control transfer. + // Some bridges watch line-coding changes as a signalling channel (a magic baud + // sequence to enter a bootloader, say), so redundant applies are not harmless. static const uart::UARTParityOptions PARITY_MAP[] = { uart::UART_CONFIG_PARITY_NONE, uart::UART_CONFIG_PARITY_EVEN, uart::UART_CONFIG_PARITY_ODD, }; + if (uart_comp->get_baud_rate() == baudrate && uart_comp->get_stop_bits() == stop_bits && + uart_comp->get_data_bits() == data_size && uart_comp->get_parity() == PARITY_MAP[parity]) { + ESP_LOGV(TAG, "Settings unchanged, skipping reconfigure [%" PRIu32 "]", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } + + // Apply validated parameters + uart_comp->set_baud_rate(baudrate); + uart_comp->set_stop_bits(stop_bits); + uart_comp->set_data_bits(data_size); + uart_comp->set_parity(PARITY_MAP[parity]); // load_settings() is available on ESP8266 and ESP32 platforms From f0e2eb96bdcc3411d42432b606d56902088f644c Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:39:32 +1000 Subject: [PATCH 003/178] [snapshot][SDL] Display headless mode and snapshots (#17917) Co-authored-by: Claude Opus 5 Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- .github/workflows/ci.yml | 15 +- .gitignore | 2 + CODEOWNERS | 1 + esphome/components/sdl/__init__.py | 253 ++++++++++++++++ esphome/components/sdl/binary_sensor.py | 253 +--------------- esphome/components/sdl/display.py | 53 +++- esphome/components/sdl/sdl_esphome.cpp | 274 +++++++++++++++--- esphome/components/sdl/sdl_esphome.h | 35 ++- .../components/sdl/touchscreen/__init__.py | 4 +- esphome/components/snapshot/__init__.py | 76 +++++ .../components/snapshot/display/__init__.py | 61 ++++ .../snapshot/display/snapshot_display.cpp | 80 +++++ .../snapshot/display/snapshot_display.h | 48 +++ esphome/components/snapshot/snapshot.cpp | 248 ++++++++++++++++ esphome/components/snapshot/snapshot.h | 72 +++++ esphome/core/defines.h | 1 + tests/component_tests/sdl/test_sdl.py | 101 +++++++ tests/components/sdl/common.yaml | 27 ++ tests/components/sdl/validate.host.yaml | 29 ++ tests/components/snapshot/common.yaml | 34 +++ tests/components/snapshot/test.host.yaml | 5 + tests/integration/artifact_utils.py | 26 ++ tests/integration/bmp_utils.py | 161 ++++++++++ .../fixtures/lvgl_headless_render.yaml | 53 ++++ .../fixtures/sdl_headless_screenshot.yaml | 29 ++ .../fixtures/snapshot_display.yaml | 28 ++ .../integration/test_lvgl_headless_render.py | 83 ++++++ .../test_sdl_headless_screenshot.py | 49 ++++ tests/integration/test_snapshot_display.py | 78 +++++ 29 files changed, 1874 insertions(+), 305 deletions(-) create mode 100644 esphome/components/snapshot/__init__.py create mode 100644 esphome/components/snapshot/display/__init__.py create mode 100644 esphome/components/snapshot/display/snapshot_display.cpp create mode 100644 esphome/components/snapshot/display/snapshot_display.h create mode 100644 esphome/components/snapshot/snapshot.cpp create mode 100644 esphome/components/snapshot/snapshot.h create mode 100644 tests/component_tests/sdl/test_sdl.py create mode 100644 tests/components/sdl/validate.host.yaml create mode 100644 tests/components/snapshot/common.yaml create mode 100644 tests/components/snapshot/test.host.yaml create mode 100644 tests/integration/artifact_utils.py create mode 100644 tests/integration/bmp_utils.py create mode 100644 tests/integration/fixtures/lvgl_headless_render.yaml create mode 100644 tests/integration/fixtures/sdl_headless_screenshot.yaml create mode 100644 tests/integration/fixtures/snapshot_display.yaml create mode 100644 tests/integration/test_lvgl_headless_render.py create mode 100644 tests/integration/test_sdl_headless_screenshot.py create mode 100644 tests/integration/test_snapshot_display.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a874a023b9..d7c93b3b86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -374,8 +374,9 @@ jobs: - name: Install apt packages (cached) # ccache speeds up the host compiles. A cache hit never touches apt # (mirror outages cannot hang the job); the timeout bounds the cold - # path. Packages and version must match seed-apt-cache exactly; - # libsdl2-dev is unused here and carried only for cache-key parity. + # path. Packages and version must match seed-apt-cache exactly. + # libsdl2-dev is needed by the headless display tests, which capture + # screenshots. timeout-minutes: 10 uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 with: @@ -438,6 +439,16 @@ jobs: echo "Bucket ${{ matrix.bucket.name }}: running ${#test_files[@]} integration tests" pytest -vv --no-cov --tb=native --durations=30 -n auto --dist worksteal \ --junitxml=junit-integration.xml "${test_files[@]}" + - name: Upload test artifacts + # Tests that compare rendered output write the image they actually got here, so a + # failure can be looked at without reproducing the whole build locally. + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-test-artifacts-${{ matrix.bucket.name }} + path: test_artifacts/ + if-no-files-found: ignore + retention-days: 7 - name: Upload junit timings # Consumed by sync-integration-durations.yml through # script/update_integration_test_durations.py; only full matrix dev diff --git a/.gitignore b/.gitignore index fdb75824fb..82b00286c7 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,8 @@ config/ !tests/component_tests/**/config/ tests/build/ tests/.esphome/ +# Output kept by failing tests for inspection; uploaded by CI +test_artifacts/ /.temp-clang-tidy.cpp /.temp/ .pio/ diff --git a/CODEOWNERS b/CODEOWNERS index 3429a93aa7..f91bc00ae5 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -496,6 +496,7 @@ esphome/components/sm2335/* @Cossid esphome/components/sml/* @alengwenus esphome/components/smt100/* @piechade esphome/components/sn74hc165/* @jesserockz +esphome/components/snapshot/* @clydebarrow esphome/components/socket/* @esphome/core esphome/components/sonoff_d1/* @anatoly-savchenkov esphome/components/sound_level/* @kahrendt diff --git a/esphome/components/sdl/__init__.py b/esphome/components/sdl/__init__.py index c58ce8a01e..872d831850 100644 --- a/esphome/components/sdl/__init__.py +++ b/esphome/components/sdl/__init__.py @@ -1 +1,254 @@ +import esphome.codegen as cg + CODEOWNERS = ["@clydebarrow"] + +SDL_KeyCode = cg.global_ns.enum("SDL_KeyCode") + +SDL_KEYS = ( + "SDLK_UNKNOWN", + "SDLK_RETURN", + "SDLK_ESCAPE", + "SDLK_BACKSPACE", + "SDLK_TAB", + "SDLK_SPACE", + "SDLK_EXCLAIM", + "SDLK_QUOTEDBL", + "SDLK_HASH", + "SDLK_PERCENT", + "SDLK_DOLLAR", + "SDLK_AMPERSAND", + "SDLK_QUOTE", + "SDLK_LEFTPAREN", + "SDLK_RIGHTPAREN", + "SDLK_ASTERISK", + "SDLK_PLUS", + "SDLK_COMMA", + "SDLK_MINUS", + "SDLK_PERIOD", + "SDLK_SLASH", + "SDLK_0", + "SDLK_1", + "SDLK_2", + "SDLK_3", + "SDLK_4", + "SDLK_5", + "SDLK_6", + "SDLK_7", + "SDLK_8", + "SDLK_9", + "SDLK_COLON", + "SDLK_SEMICOLON", + "SDLK_LESS", + "SDLK_EQUALS", + "SDLK_GREATER", + "SDLK_QUESTION", + "SDLK_AT", + "SDLK_LEFTBRACKET", + "SDLK_BACKSLASH", + "SDLK_RIGHTBRACKET", + "SDLK_CARET", + "SDLK_UNDERSCORE", + "SDLK_BACKQUOTE", + "SDLK_a", + "SDLK_b", + "SDLK_c", + "SDLK_d", + "SDLK_e", + "SDLK_f", + "SDLK_g", + "SDLK_h", + "SDLK_i", + "SDLK_j", + "SDLK_k", + "SDLK_l", + "SDLK_m", + "SDLK_n", + "SDLK_o", + "SDLK_p", + "SDLK_q", + "SDLK_r", + "SDLK_s", + "SDLK_t", + "SDLK_u", + "SDLK_v", + "SDLK_w", + "SDLK_x", + "SDLK_y", + "SDLK_z", + "SDLK_CAPSLOCK", + "SDLK_F1", + "SDLK_F2", + "SDLK_F3", + "SDLK_F4", + "SDLK_F5", + "SDLK_F6", + "SDLK_F7", + "SDLK_F8", + "SDLK_F9", + "SDLK_F10", + "SDLK_F11", + "SDLK_F12", + "SDLK_PRINTSCREEN", + "SDLK_SCROLLLOCK", + "SDLK_PAUSE", + "SDLK_INSERT", + "SDLK_HOME", + "SDLK_PAGEUP", + "SDLK_DELETE", + "SDLK_END", + "SDLK_PAGEDOWN", + "SDLK_RIGHT", + "SDLK_LEFT", + "SDLK_DOWN", + "SDLK_UP", + "SDLK_NUMLOCKCLEAR", + "SDLK_KP_DIVIDE", + "SDLK_KP_MULTIPLY", + "SDLK_KP_MINUS", + "SDLK_KP_PLUS", + "SDLK_KP_ENTER", + "SDLK_KP_1", + "SDLK_KP_2", + "SDLK_KP_3", + "SDLK_KP_4", + "SDLK_KP_5", + "SDLK_KP_6", + "SDLK_KP_7", + "SDLK_KP_8", + "SDLK_KP_9", + "SDLK_KP_0", + "SDLK_KP_PERIOD", + "SDLK_APPLICATION", + "SDLK_POWER", + "SDLK_KP_EQUALS", + "SDLK_F13", + "SDLK_F14", + "SDLK_F15", + "SDLK_F16", + "SDLK_F17", + "SDLK_F18", + "SDLK_F19", + "SDLK_F20", + "SDLK_F21", + "SDLK_F22", + "SDLK_F23", + "SDLK_F24", + "SDLK_EXECUTE", + "SDLK_HELP", + "SDLK_MENU", + "SDLK_SELECT", + "SDLK_STOP", + "SDLK_AGAIN", + "SDLK_UNDO", + "SDLK_CUT", + "SDLK_COPY", + "SDLK_PASTE", + "SDLK_FIND", + "SDLK_MUTE", + "SDLK_VOLUMEUP", + "SDLK_VOLUMEDOWN", + "SDLK_KP_COMMA", + "SDLK_KP_EQUALSAS400", + "SDLK_ALTERASE", + "SDLK_SYSREQ", + "SDLK_CANCEL", + "SDLK_CLEAR", + "SDLK_PRIOR", + "SDLK_RETURN2", + "SDLK_SEPARATOR", + "SDLK_OUT", + "SDLK_OPER", + "SDLK_CLEARAGAIN", + "SDLK_CRSEL", + "SDLK_EXSEL", + "SDLK_KP_00", + "SDLK_KP_000", + "SDLK_THOUSANDSSEPARATOR", + "SDLK_DECIMALSEPARATOR", + "SDLK_CURRENCYUNIT", + "SDLK_CURRENCYSUBUNIT", + "SDLK_KP_LEFTPAREN", + "SDLK_KP_RIGHTPAREN", + "SDLK_KP_LEFTBRACE", + "SDLK_KP_RIGHTBRACE", + "SDLK_KP_TAB", + "SDLK_KP_BACKSPACE", + "SDLK_KP_A", + "SDLK_KP_B", + "SDLK_KP_C", + "SDLK_KP_D", + "SDLK_KP_E", + "SDLK_KP_F", + "SDLK_KP_XOR", + "SDLK_KP_POWER", + "SDLK_KP_PERCENT", + "SDLK_KP_LESS", + "SDLK_KP_GREATER", + "SDLK_KP_AMPERSAND", + "SDLK_KP_DBLAMPERSAND", + "SDLK_KP_VERTICALBAR", + "SDLK_KP_DBLVERTICALBAR", + "SDLK_KP_COLON", + "SDLK_KP_HASH", + "SDLK_KP_SPACE", + "SDLK_KP_AT", + "SDLK_KP_EXCLAM", + "SDLK_KP_MEMSTORE", + "SDLK_KP_MEMRECALL", + "SDLK_KP_MEMCLEAR", + "SDLK_KP_MEMADD", + "SDLK_KP_MEMSUBTRACT", + "SDLK_KP_MEMMULTIPLY", + "SDLK_KP_MEMDIVIDE", + "SDLK_KP_PLUSMINUS", + "SDLK_KP_CLEAR", + "SDLK_KP_CLEARENTRY", + "SDLK_KP_BINARY", + "SDLK_KP_OCTAL", + "SDLK_KP_DECIMAL", + "SDLK_KP_HEXADECIMAL", + "SDLK_LCTRL", + "SDLK_LSHIFT", + "SDLK_LALT", + "SDLK_LGUI", + "SDLK_RCTRL", + "SDLK_RSHIFT", + "SDLK_RALT", + "SDLK_RGUI", + "SDLK_MODE", + "SDLK_AUDIONEXT", + "SDLK_AUDIOPREV", + "SDLK_AUDIOSTOP", + "SDLK_AUDIOPLAY", + "SDLK_AUDIOMUTE", + "SDLK_MEDIASELECT", + "SDLK_WWW", + "SDLK_MAIL", + "SDLK_CALCULATOR", + "SDLK_COMPUTER", + "SDLK_AC_SEARCH", + "SDLK_AC_HOME", + "SDLK_AC_BACK", + "SDLK_AC_FORWARD", + "SDLK_AC_STOP", + "SDLK_AC_REFRESH", + "SDLK_AC_BOOKMARKS", + "SDLK_BRIGHTNESSDOWN", + "SDLK_BRIGHTNESSUP", + "SDLK_DISPLAYSWITCH", + "SDLK_KBDILLUMTOGGLE", + "SDLK_KBDILLUMDOWN", + "SDLK_KBDILLUMUP", + "SDLK_EJECT", + "SDLK_SLEEP", + "SDLK_APP1", + "SDLK_APP2", + "SDLK_AUDIOREWIND", + "SDLK_AUDIOFASTFORWARD", + "SDLK_SOFTLEFT", + "SDLK_SOFTRIGHT", + "SDLK_CALL", + "SDLK_ENDCALL", +) + +SDL_KEYMAP = {key: getattr(SDL_KeyCode, key) for key in SDL_KEYS} diff --git a/esphome/components/sdl/binary_sensor.py b/esphome/components/sdl/binary_sensor.py index 0fdda25ed3..c978071391 100644 --- a/esphome/components/sdl/binary_sensor.py +++ b/esphome/components/sdl/binary_sensor.py @@ -7,262 +7,15 @@ from esphome.core import Lambda from esphome.cpp_generator import ExpressionStatement, RawExpression from esphome.types import ConfigType -from .display import CONF_SDL_ID, Sdl +from . import SDL_KEYMAP +from .display import CONF_SDL_ID, Sdl, headless_final_validate CODEOWNERS = ["@bdm310"] STATE_ARG = "state" -SDL_KeyCode = cg.global_ns.enum("SDL_KeyCode") +FINAL_VALIDATE_SCHEMA = headless_final_validate("binary_sensor") -SDL_KEYS = ( - "SDLK_UNKNOWN", - "SDLK_RETURN", - "SDLK_ESCAPE", - "SDLK_BACKSPACE", - "SDLK_TAB", - "SDLK_SPACE", - "SDLK_EXCLAIM", - "SDLK_QUOTEDBL", - "SDLK_HASH", - "SDLK_PERCENT", - "SDLK_DOLLAR", - "SDLK_AMPERSAND", - "SDLK_QUOTE", - "SDLK_LEFTPAREN", - "SDLK_RIGHTPAREN", - "SDLK_ASTERISK", - "SDLK_PLUS", - "SDLK_COMMA", - "SDLK_MINUS", - "SDLK_PERIOD", - "SDLK_SLASH", - "SDLK_0", - "SDLK_1", - "SDLK_2", - "SDLK_3", - "SDLK_4", - "SDLK_5", - "SDLK_6", - "SDLK_7", - "SDLK_8", - "SDLK_9", - "SDLK_COLON", - "SDLK_SEMICOLON", - "SDLK_LESS", - "SDLK_EQUALS", - "SDLK_GREATER", - "SDLK_QUESTION", - "SDLK_AT", - "SDLK_LEFTBRACKET", - "SDLK_BACKSLASH", - "SDLK_RIGHTBRACKET", - "SDLK_CARET", - "SDLK_UNDERSCORE", - "SDLK_BACKQUOTE", - "SDLK_a", - "SDLK_b", - "SDLK_c", - "SDLK_d", - "SDLK_e", - "SDLK_f", - "SDLK_g", - "SDLK_h", - "SDLK_i", - "SDLK_j", - "SDLK_k", - "SDLK_l", - "SDLK_m", - "SDLK_n", - "SDLK_o", - "SDLK_p", - "SDLK_q", - "SDLK_r", - "SDLK_s", - "SDLK_t", - "SDLK_u", - "SDLK_v", - "SDLK_w", - "SDLK_x", - "SDLK_y", - "SDLK_z", - "SDLK_CAPSLOCK", - "SDLK_F1", - "SDLK_F2", - "SDLK_F3", - "SDLK_F4", - "SDLK_F5", - "SDLK_F6", - "SDLK_F7", - "SDLK_F8", - "SDLK_F9", - "SDLK_F10", - "SDLK_F11", - "SDLK_F12", - "SDLK_PRINTSCREEN", - "SDLK_SCROLLLOCK", - "SDLK_PAUSE", - "SDLK_INSERT", - "SDLK_HOME", - "SDLK_PAGEUP", - "SDLK_DELETE", - "SDLK_END", - "SDLK_PAGEDOWN", - "SDLK_RIGHT", - "SDLK_LEFT", - "SDLK_DOWN", - "SDLK_UP", - "SDLK_NUMLOCKCLEAR", - "SDLK_KP_DIVIDE", - "SDLK_KP_MULTIPLY", - "SDLK_KP_MINUS", - "SDLK_KP_PLUS", - "SDLK_KP_ENTER", - "SDLK_KP_1", - "SDLK_KP_2", - "SDLK_KP_3", - "SDLK_KP_4", - "SDLK_KP_5", - "SDLK_KP_6", - "SDLK_KP_7", - "SDLK_KP_8", - "SDLK_KP_9", - "SDLK_KP_0", - "SDLK_KP_PERIOD", - "SDLK_APPLICATION", - "SDLK_POWER", - "SDLK_KP_EQUALS", - "SDLK_F13", - "SDLK_F14", - "SDLK_F15", - "SDLK_F16", - "SDLK_F17", - "SDLK_F18", - "SDLK_F19", - "SDLK_F20", - "SDLK_F21", - "SDLK_F22", - "SDLK_F23", - "SDLK_F24", - "SDLK_EXECUTE", - "SDLK_HELP", - "SDLK_MENU", - "SDLK_SELECT", - "SDLK_STOP", - "SDLK_AGAIN", - "SDLK_UNDO", - "SDLK_CUT", - "SDLK_COPY", - "SDLK_PASTE", - "SDLK_FIND", - "SDLK_MUTE", - "SDLK_VOLUMEUP", - "SDLK_VOLUMEDOWN", - "SDLK_KP_COMMA", - "SDLK_KP_EQUALSAS400", - "SDLK_ALTERASE", - "SDLK_SYSREQ", - "SDLK_CANCEL", - "SDLK_CLEAR", - "SDLK_PRIOR", - "SDLK_RETURN2", - "SDLK_SEPARATOR", - "SDLK_OUT", - "SDLK_OPER", - "SDLK_CLEARAGAIN", - "SDLK_CRSEL", - "SDLK_EXSEL", - "SDLK_KP_00", - "SDLK_KP_000", - "SDLK_THOUSANDSSEPARATOR", - "SDLK_DECIMALSEPARATOR", - "SDLK_CURRENCYUNIT", - "SDLK_CURRENCYSUBUNIT", - "SDLK_KP_LEFTPAREN", - "SDLK_KP_RIGHTPAREN", - "SDLK_KP_LEFTBRACE", - "SDLK_KP_RIGHTBRACE", - "SDLK_KP_TAB", - "SDLK_KP_BACKSPACE", - "SDLK_KP_A", - "SDLK_KP_B", - "SDLK_KP_C", - "SDLK_KP_D", - "SDLK_KP_E", - "SDLK_KP_F", - "SDLK_KP_XOR", - "SDLK_KP_POWER", - "SDLK_KP_PERCENT", - "SDLK_KP_LESS", - "SDLK_KP_GREATER", - "SDLK_KP_AMPERSAND", - "SDLK_KP_DBLAMPERSAND", - "SDLK_KP_VERTICALBAR", - "SDLK_KP_DBLVERTICALBAR", - "SDLK_KP_COLON", - "SDLK_KP_HASH", - "SDLK_KP_SPACE", - "SDLK_KP_AT", - "SDLK_KP_EXCLAM", - "SDLK_KP_MEMSTORE", - "SDLK_KP_MEMRECALL", - "SDLK_KP_MEMCLEAR", - "SDLK_KP_MEMADD", - "SDLK_KP_MEMSUBTRACT", - "SDLK_KP_MEMMULTIPLY", - "SDLK_KP_MEMDIVIDE", - "SDLK_KP_PLUSMINUS", - "SDLK_KP_CLEAR", - "SDLK_KP_CLEARENTRY", - "SDLK_KP_BINARY", - "SDLK_KP_OCTAL", - "SDLK_KP_DECIMAL", - "SDLK_KP_HEXADECIMAL", - "SDLK_LCTRL", - "SDLK_LSHIFT", - "SDLK_LALT", - "SDLK_LGUI", - "SDLK_RCTRL", - "SDLK_RSHIFT", - "SDLK_RALT", - "SDLK_RGUI", - "SDLK_MODE", - "SDLK_AUDIONEXT", - "SDLK_AUDIOPREV", - "SDLK_AUDIOSTOP", - "SDLK_AUDIOPLAY", - "SDLK_AUDIOMUTE", - "SDLK_MEDIASELECT", - "SDLK_WWW", - "SDLK_MAIL", - "SDLK_CALCULATOR", - "SDLK_COMPUTER", - "SDLK_AC_SEARCH", - "SDLK_AC_HOME", - "SDLK_AC_BACK", - "SDLK_AC_FORWARD", - "SDLK_AC_STOP", - "SDLK_AC_REFRESH", - "SDLK_AC_BOOKMARKS", - "SDLK_BRIGHTNESSDOWN", - "SDLK_BRIGHTNESSUP", - "SDLK_DISPLAYSWITCH", - "SDLK_KBDILLUMTOGGLE", - "SDLK_KBDILLUMDOWN", - "SDLK_KBDILLUMUP", - "SDLK_EJECT", - "SDLK_SLEEP", - "SDLK_APP1", - "SDLK_APP2", - "SDLK_AUDIOREWIND", - "SDLK_AUDIOFASTFORWARD", - "SDLK_SOFTLEFT", - "SDLK_SOFTRIGHT", - "SDLK_CALL", - "SDLK_ENDCALL", -) - -SDL_KEYMAP = {key: getattr(SDL_KeyCode, key) for key in SDL_KEYS} CONFIG_SCHEMA = ( binary_sensor.binary_sensor_schema(BinarySensor) diff --git a/esphome/components/sdl/display.py b/esphome/components/sdl/display.py index 5ced2edf5a..77b0001c55 100644 --- a/esphome/components/sdl/display.py +++ b/esphome/components/sdl/display.py @@ -4,6 +4,7 @@ from typing import Any import esphome.codegen as cg from esphome.components import display +from esphome.components.snapshot import Snapshot, register_snapshot import esphome.config_validation as cv from esphome.const import ( CONF_DIMENSIONS, @@ -16,14 +17,21 @@ from esphome.const import ( CONF_Y, PLATFORM_HOST, ) +import esphome.final_validate as fv from esphome.types import ConfigType +from . import SDL_KEYMAP + +AUTO_LOAD = ["snapshot"] + sdl_ns = cg.esphome_ns.namespace("sdl") -Sdl = sdl_ns.class_("Sdl", display.Display, cg.Component) +Sdl = sdl_ns.class_("Sdl", display.Display, cg.Component, Snapshot) sdl_window_flags = cg.global_ns.enum("SDL_WindowFlags") CONF_CENTERED_ON_DISPLAY = "centered_on_display" +CONF_HEADLESS = "headless" +CONF_SNAPSHOT_KEY = "snapshot_key" CONF_SDL_OPTIONS = "sdl_options" CONF_SDL_ID = "sdl_id" CONF_WINDOW_OPTIONS = "window_options" @@ -67,12 +75,29 @@ def _validate_position(config: dict) -> dict: raise cv.Invalid("Must specify either 'x' and 'y' or 'centered_on_display'") +def _validate_headless(config: ConfigType) -> ConfigType: + if not config[CONF_HEADLESS]: + return config + if CONF_WINDOW_OPTIONS in config: + raise cv.Invalid( + f"'{CONF_WINDOW_OPTIONS}' has no effect when '{CONF_HEADLESS}' is set - there is no window" + ) + if CONF_SNAPSHOT_KEY in config: + raise cv.Invalid( + f"'{CONF_SNAPSHOT_KEY}' cannot be used when '{CONF_HEADLESS}' is set - " + f"there is no keyboard. Use the 'snapshot.take' action instead" + ) + return config + + CONFIG_SCHEMA = cv.All( display.FULL_DISPLAY_SCHEMA.extend( cv.Schema( { cv.GenerateID(): cv.declare_id(Sdl), cv.Optional(CONF_SDL_OPTIONS, default=""): get_sdl_options, + cv.Optional(CONF_HEADLESS, default=False): cv.boolean, + cv.Optional(CONF_SNAPSHOT_KEY): cv.enum(SDL_KEYMAP), cv.Required(CONF_DIMENSIONS): cv.Any( cv.dimensions, cv.Schema( @@ -99,16 +124,42 @@ CONFIG_SCHEMA = cv.All( } ) ), + _validate_headless, cv.only_on(PLATFORM_HOST), ) +def headless_final_validate(platform: str) -> cv.Schema: + """Build a FINAL_VALIDATE_SCHEMA rejecting a platform whose sdl display is headless. + + Mouse and keyboard platforms are driven by window events, so under a headless display they + would never report anything. + """ + + def validate_display(display_config: ConfigType) -> ConfigType: + if display_config.get(CONF_HEADLESS): + raise cv.Invalid( + f"The sdl {platform} platform needs a window, but its display has " + f"'{CONF_HEADLESS}' set" + ) + return display_config + + return cv.Schema( + {cv.Required(CONF_SDL_ID): fv.id_declaration_match_schema(validate_display)}, + extra=cv.ALLOW_EXTRA, + ) + + async def to_code(config: ConfigType) -> None: for option in config[CONF_SDL_OPTIONS].split(): cg.add_build_flag(option) cg.add_build_flag("-DSDL_BYTEORDER=4321") var = cg.new_Pvariable(config[CONF_ID]) await display.register_display(var, config) + await register_snapshot(var, config) + cg.add(var.set_headless(config[CONF_HEADLESS])) + if (key := config.get(CONF_SNAPSHOT_KEY)) is not None: + cg.add(var.set_snapshot_key(key)) dimensions = config[CONF_DIMENSIONS] if isinstance(dimensions, dict): diff --git a/esphome/components/sdl/sdl_esphome.cpp b/esphome/components/sdl/sdl_esphome.cpp index c99b5081b3..03fc086021 100644 --- a/esphome/components/sdl/sdl_esphome.cpp +++ b/esphome/components/sdl/sdl_esphome.cpp @@ -2,8 +2,17 @@ #include "sdl_esphome.h" #include "esphome/components/display/display_color_utils.h" +#include + namespace esphome::sdl { +namespace { + +// Key under which each window keeps a pointer back to its Sdl instance. +constexpr const char *const WINDOW_DATA_KEY = "esphome_sdl"; + +} // namespace + int Sdl::get_width() { switch (this->rotation_) { case display::DISPLAY_ROTATION_90_DEGREES: @@ -28,17 +37,96 @@ int Sdl::get_height() { } } -void Sdl::setup() { - SDL_Init(SDL_INIT_VIDEO); - this->window_ = SDL_CreateWindow(App.get_name().c_str(), this->pos_x_, this->pos_y_, this->width_, this->height_, - this->window_options_); - this->renderer_ = SDL_CreateRenderer(this->window_, -1, SDL_RENDERER_SOFTWARE); - SDL_RenderSetLogicalSize(this->renderer_, this->width_, this->height_); +void Sdl::destroy_renderer_() { + // Reverse order of creation: the renderer refers to the window or surface it was made from. + if (this->shot_target_ != nullptr) { + SDL_DestroyTexture(this->shot_target_); + this->shot_target_ = nullptr; + } + if (this->texture_ != nullptr) { + SDL_DestroyTexture(this->texture_); + this->texture_ = nullptr; + } + if (this->renderer_ != nullptr) { + SDL_DestroyRenderer(this->renderer_); + this->renderer_ = nullptr; + } + if (this->window_ != nullptr) { + SDL_DestroyWindow(this->window_); + this->window_ = nullptr; + } + if (this->surface_ != nullptr) { + SDL_FreeSurface(this->surface_); + this->surface_ = nullptr; + } +} + +bool Sdl::setup_failed_(const char *what) { + ESP_LOGE(TAG, "%s: %s", what, SDL_GetError()); + // Give back whatever was created before the failure. Without this a half set up display leaves an + // empty window on screen for the life of the process, still registered as an event target. + this->destroy_renderer_(); + return false; +} + +bool Sdl::setup_renderer_() { + SDL_SetMainReady(); + if (this->headless_) { + // SDL_INIT_VIDEO is deliberately not requested: a software renderer bound to a surface needs no + // video device, so this works on a machine with no display server at all. + if (SDL_Init(0) != 0) + return this->setup_failed_("SDL_Init failed"); + this->surface_ = SDL_CreateRGBSurfaceWithFormat(0, this->width_, this->height_, 16, SDL_PIXELFORMAT_RGB565); + if (this->surface_ == nullptr) + return this->setup_failed_("Could not create offscreen surface"); + this->renderer_ = SDL_CreateSoftwareRenderer(this->surface_); + } else { + if (SDL_Init(SDL_INIT_VIDEO) != 0) + return this->setup_failed_("SDL_Init failed"); + this->window_ = SDL_CreateWindow(App.get_name().c_str(), this->pos_x_, this->pos_y_, this->width_, this->height_, + this->window_options_); + if (this->window_ == nullptr) + return this->setup_failed_("Could not create window"); + // Lets loop() find the display an event belongs to, so one display does not act on another's + // input when several windows are open. + SDL_SetWindowData(this->window_, WINDOW_DATA_KEY, this); + this->renderer_ = SDL_CreateRenderer(this->window_, -1, SDL_RENDERER_SOFTWARE); + } + if (this->renderer_ == nullptr) + return this->setup_failed_("Could not create renderer"); + if (SDL_RenderSetLogicalSize(this->renderer_, this->width_, this->height_) != 0) + return this->setup_failed_("Could not set renderer logical size"); this->texture_ = SDL_CreateTexture(this->renderer_, SDL_PIXELFORMAT_RGB565, SDL_TEXTUREACCESS_STATIC, this->width_, this->height_); - SDL_SetTextureBlendMode(this->texture_, SDL_BLENDMODE_BLEND); + if (this->texture_ == nullptr) + return this->setup_failed_("Could not create texture"); + // The texture has no alpha channel, so blending is pointless. Headless it would also force a + // different software blit path onto the 16 bit target surface. + if (SDL_SetTextureBlendMode(this->texture_, this->headless_ ? SDL_BLENDMODE_NONE : SDL_BLENDMODE_BLEND) != 0) + return this->setup_failed_("Could not set texture blend mode"); + return true; } + +void Sdl::setup() { + if (!this->setup_renderer_()) { + this->mark_failed(); + return; + } + if (this->headless_) { + // Nothing generates events, so there is nothing for loop() to do. + this->disable_loop(); + } else if (this->snapshot_key_ != 0) { + this->add_key_listener(this->snapshot_key_, [this](bool down) { + if (down && !this->take_snapshot(nullptr)) { + ESP_LOGW(TAG, "snapshot key did not write a file"); + } + }); + } +} + void Sdl::update() { + if (this->texture_ == nullptr) + return; this->do_update_(); if ((this->x_high_ < this->x_low_) || (this->y_high_ < this->y_low_)) return; @@ -51,12 +139,19 @@ void Sdl::update() { } void Sdl::redraw_(SDL_Rect &rect) { + // Nothing to present when headless - a snapshot blits the whole texture when it needs it, so + // doing it here as well would just burn CPU. draw_pixels_at() calls this on every partial + // update, so it is worth skipping. + if (this->headless_) + return; SDL_RenderCopy(this->renderer_, this->texture_, &rect, &rect); SDL_RenderPresent(this->renderer_); } void Sdl::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, display::ColorOrder order, display::ColorBitness bitness, bool big_endian, int x_offset, int y_offset, int x_pad) { + if (this->texture_ == nullptr) + return; SDL_Rect rect{x_start, y_start, w, h}; if (this->rotation_ != display::DISPLAY_ROTATION_0_DEGREES || bitness != display::COLOR_BITNESS_565 || big_endian) { Display::draw_pixels_at(x_start, y_start, w, h, ptr, order, bitness, big_endian, x_offset, y_offset, x_pad); @@ -69,7 +164,7 @@ void Sdl::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t * } void Sdl::draw_pixel_at(int x, int y, Color color) { - if (!this->get_clipping().inside(x, y)) + if (this->texture_ == nullptr || !this->get_clipping().inside(x, y)) return; if (this->rotation_ == display::DISPLAY_ROTATION_180_DEGREES) { @@ -104,61 +199,148 @@ void Sdl::process_key(uint32_t keycode, bool down) { callback->second(down); } +Sdl *Sdl::instance_for_window_(uint32_t window_id) { + SDL_Window *window = SDL_GetWindowFromID(window_id); + if (window == nullptr) + return nullptr; + return static_cast(SDL_GetWindowData(window, WINDOW_DATA_KEY)); +} + +void Sdl::handle_event_(const SDL_Event &event) { + switch (event.type) { + case SDL_MOUSEBUTTONDOWN: + case SDL_MOUSEBUTTONUP: + if (event.button.button == 1) { + this->mouse_x = event.button.x; + this->mouse_y = event.button.y; + this->mouse_down = event.button.state != 0; + } + break; + + case SDL_MOUSEMOTION: + if (event.motion.state & 1) { + this->mouse_x = event.motion.x; + this->mouse_y = event.motion.y; + this->mouse_down = true; + } else { + this->mouse_down = false; + } + break; + + case SDL_KEYDOWN: + // Ignore auto-repeat, otherwise holding a key floods the listeners. + if (event.key.repeat != 0) + break; + ESP_LOGD(TAG, "keydown %d", event.key.keysym.sym); + this->process_key(event.key.keysym.sym, true); + break; + + case SDL_KEYUP: + ESP_LOGD(TAG, "keyup %d", event.key.keysym.sym); + this->process_key(event.key.keysym.sym, false); + break; + + case SDL_WINDOWEVENT: + switch (event.window.event) { + case SDL_WINDOWEVENT_SIZE_CHANGED: + case SDL_WINDOWEVENT_EXPOSED: + case SDL_WINDOWEVENT_RESIZED: { + SDL_Rect rect{0, 0, this->width_, this->height_}; + this->redraw_(rect); + break; + } + default: + break; + } + break; + + default: + break; + } +} + void Sdl::loop() { SDL_Event e; - if (SDL_PollEvent(&e)) { - switch (e.type) { - case SDL_QUIT: - exit(0); + // Take everything that is waiting, not one event per loop. A touch drag produces a burst of + // motion events, and consuming them one at a time lets the queue grow without bound, so the + // pointer ends up acting on input from further and further in the past. Draining collapses a + // burst to the position it ended at, which is the one the user is asking for anyway. + while (SDL_PollEvent(&e)) { + if (e.type == SDL_QUIT) + exit(0); + // Events carry the window they happened in, so send each one to the display that owns it. + uint32_t window_id; + switch (e.type) { case SDL_MOUSEBUTTONDOWN: case SDL_MOUSEBUTTONUP: - if (e.button.button == 1) { - this->mouse_x = e.button.x; - this->mouse_y = e.button.y; - this->mouse_down = e.button.state != 0; - } + window_id = e.button.windowID; break; - case SDL_MOUSEMOTION: - if (e.motion.state & 1) { - this->mouse_x = e.button.x; - this->mouse_y = e.button.y; - this->mouse_down = true; - } else { - this->mouse_down = false; - } + window_id = e.motion.windowID; break; - case SDL_KEYDOWN: - ESP_LOGD(TAG, "keydown %d", e.key.keysym.sym); - this->process_key(e.key.keysym.sym, true); - break; - case SDL_KEYUP: - ESP_LOGD(TAG, "keyup %d", e.key.keysym.sym); - this->process_key(e.key.keysym.sym, false); + window_id = e.key.windowID; break; - case SDL_WINDOWEVENT: - switch (e.window.event) { - case SDL_WINDOWEVENT_SIZE_CHANGED: - case SDL_WINDOWEVENT_EXPOSED: - case SDL_WINDOWEVENT_RESIZED: { - SDL_Rect rect{0, 0, this->width_, this->height_}; - this->redraw_(rect); - break; - } - default: - break; - } + window_id = e.window.windowID; break; - default: + // Anything else, including the touch events SDL reports alongside the mouse events it + // synthesises from them, is not used here. ESP_LOGV(TAG, "Event %d", e.type); - break; + continue; + } + + Sdl *target = instance_for_window_(window_id); + if (target == nullptr) { + // Nothing to route this to: the window has gone, or it is not one of ours. Say so, otherwise + // input that stops working leaves no trace at all. + ESP_LOGV(TAG, "Event %d for unknown window %u", e.type, window_id); + continue; + } + target->handle_event_(e); + } +} + +bool Sdl::capture_bgr(uint8_t *dest, size_t row_stride) { + if (this->texture_ == nullptr || this->renderer_ == nullptr) { + ESP_LOGE(TAG, "Snapshot requested but SDL is not set up"); + return false; + } + if (this->shot_target_ == nullptr) { + this->shot_target_ = SDL_CreateTexture(this->renderer_, SDL_PIXELFORMAT_RGB565, SDL_TEXTUREACCESS_TARGET, + this->width_, this->height_); + if (this->shot_target_ == nullptr) { + ESP_LOGE(TAG, "Could not create capture texture: %s", SDL_GetError()); + return false; + } + SDL_SetTextureBlendMode(this->shot_target_, SDL_BLENDMODE_NONE); + } + + // Render into an offscreen target first. SDL_RenderReadPixels works in physical output pixels and + // ignores the logical size, so reading straight off a resizable window would read more pixels than + // there is room for. + // Every step is checked: a failed clear or copy would otherwise be read back as a blank or stale + // picture, written out, and reported as a snapshot that worked. + bool ok = false; + if (SDL_SetRenderTarget(this->renderer_, this->shot_target_) == 0) { + ok = SDL_SetRenderDrawColor(this->renderer_, 0, 0, 0, SDL_ALPHA_OPAQUE) == 0 && + SDL_RenderClear(this->renderer_) == 0 && + SDL_RenderCopy(this->renderer_, this->texture_, nullptr, nullptr) == 0 && + SDL_RenderReadPixels(this->renderer_, nullptr, SDL_PIXELFORMAT_BGR24, dest, static_cast(row_stride)) == 0; + if (SDL_SetRenderTarget(this->renderer_, nullptr) != 0) { + // Stuck rendering into shot_target_ from here on, so there's no point continuing. + ESP_LOGE(TAG, "Could not restore the render target: %s", SDL_GetError()); + this->mark_failed(); + return false; } } + if (!ok) { + ESP_LOGE(TAG, "Could not capture the screen: %s", SDL_GetError()); + } + return ok; } } // namespace esphome::sdl diff --git a/esphome/components/sdl/sdl_esphome.h b/esphome/components/sdl/sdl_esphome.h index 635eb1e3f8..54f0d2573f 100644 --- a/esphome/components/sdl/sdl_esphome.h +++ b/esphome/components/sdl/sdl_esphome.h @@ -1,10 +1,12 @@ #pragma once #ifdef USE_HOST +#include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/log.h" #include "esphome/core/application.h" #include "esphome/components/display/display.h" +#include "esphome/components/snapshot/snapshot.h" #define SDL_MAIN_HANDLED #include "SDL.h" #include @@ -13,7 +15,7 @@ namespace esphome::sdl { constexpr static const char *const TAG = "sdl"; -class Sdl final : public display::Display { +class Sdl final : public display::Display, public snapshot::Snapshot { public: display::DisplayType get_display_type() override { return display::DISPLAY_TYPE_COLOR; } void update() override; @@ -32,6 +34,9 @@ class Sdl final : public display::Display { this->pos_x_ = pos_x; this->pos_y_ = pos_y; } + void set_headless(bool headless) { this->headless_ = headless; } + void set_snapshot_key(int32_t keycode) { this->snapshot_key_ = keycode; } + int get_width() override; int get_height() override; float get_setup_priority() const override { return setup_priority::HARDWARE; } @@ -51,20 +56,40 @@ class Sdl final : public display::Display { int get_width_internal() override { return this->width_; } int get_height_internal() override { return this->height_; } void redraw_(SDL_Rect &rect); + bool setup_renderer_(); + /// Release the window, surface, renderer and textures, and forget them. + void destroy_renderer_(); + /// Log an SDL failure during setup, release anything already created, and return false. + bool setup_failed_(const char *what); + int snapshot_width() override { return this->width_; } + int snapshot_height() override { return this->height_; } + bool capture_bgr(uint8_t *dest, size_t row_stride) override; + void handle_event_(const SDL_Event &event); + /// The display owning the given window, or nullptr if it is not one of ours. + static Sdl *instance_for_window_(uint32_t window_id); + SDL_Renderer *renderer_{}; + SDL_Window *window_{}; + SDL_Texture *texture_{}; + // Offscreen render target used when headless. SDL_CreateSoftwareRenderer only borrows the + // surface, and the renderer goes back to using it as its output whenever the capture target is + // released, so it has to stay alive as long as the renderer does. + SDL_Surface *surface_{}; + // Capture target, created on first snapshot. + SDL_Texture *shot_target_{}; + std::map> key_callbacks_{}; int width_{}; int height_{}; uint32_t window_options_{0}; int32_t pos_x_{SDL_WINDOWPOS_UNDEFINED}; int32_t pos_y_{SDL_WINDOWPOS_UNDEFINED}; - SDL_Renderer *renderer_{}; - SDL_Window *window_{}; - SDL_Texture *texture_{}; + int32_t snapshot_key_{0}; uint16_t x_low_{0}; uint16_t y_low_{0}; uint16_t x_high_{0}; uint16_t y_high_{0}; - std::map> key_callbacks_{}; + bool headless_{false}; }; + } // namespace esphome::sdl #endif diff --git a/esphome/components/sdl/touchscreen/__init__.py b/esphome/components/sdl/touchscreen/__init__.py index d7af8da403..9b807b4585 100644 --- a/esphome/components/sdl/touchscreen/__init__.py +++ b/esphome/components/sdl/touchscreen/__init__.py @@ -4,10 +4,12 @@ import esphome.config_validation as cv from esphome.const import CONF_ID from esphome.types import ConfigType -from ..display import CONF_SDL_ID, Sdl, sdl_ns +from ..display import CONF_SDL_ID, Sdl, headless_final_validate, sdl_ns SdlTouchscreen = sdl_ns.class_("SdlTouchscreen", touchscreen.Touchscreen) +FINAL_VALIDATE_SCHEMA = headless_final_validate("touchscreen") + CONFIG_SCHEMA = touchscreen.TOUCHSCREEN_SCHEMA.extend( { diff --git a/esphome/components/snapshot/__init__.py b/esphome/components/snapshot/__init__.py new file mode 100644 index 0000000000..bf561a0e0d --- /dev/null +++ b/esphome/components/snapshot/__init__.py @@ -0,0 +1,76 @@ +"""Shared support for writing what a display is showing out to an image file. + +The component itself has no configuration. It provides the ``snapshot.take`` action and the C++ +base class behind it, so any display that can hand over its pixels - the in memory display in this +component, or an SDL window - saves files the same way, under the same directory, with the same +rules about names. +""" + +from dataclasses import dataclass + +from esphome import automation +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_ID +from esphome.core import CORE, ID +from esphome.cpp_generator import MockObj +from esphome.types import ConfigType, TemplateArgsType + +CODEOWNERS = ["@clydebarrow"] + +DOMAIN = "snapshot" + +CONF_FILENAME = "filename" + +snapshot_ns = cg.esphome_ns.namespace("snapshot") +Snapshot = snapshot_ns.class_("Snapshot") +SnapshotAction = snapshot_ns.class_("SnapshotAction", automation.Action) + + +@automation.register_action( + "snapshot.take", + SnapshotAction, + automation.maybe_simple_id( + { + cv.GenerateID(): cv.use_id(Snapshot), + cv.Optional(CONF_FILENAME): cv.templatable(cv.string), + } + ), + synchronous=True, +) +async def snapshot_take_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + if (filename := config.get(CONF_FILENAME)) is not None: + cg.add(var.set_filename(await cg.templatable(filename, args, cg.std_string))) + return var + + +@dataclass +class SnapshotData: + directory_defined: bool = False + + +def _get_data() -> SnapshotData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = SnapshotData() + return CORE.data[DOMAIN] + + +async def register_snapshot(var: MockObj, config: ConfigType) -> None: + """Set up a component so that the snapshot action can write its picture to a file.""" + data = _get_data() + # Only once, however many displays there are: two defines that say the same thing do not + # compare equal, so asking for this per display repeats the line in defines.h. + if not data.directory_defined: + data.directory_defined = True + cg.add_define( + "ESPHOME_SNAPSHOT_DIR", + (CORE.data_dir / "snapshots" / CORE.name).as_posix(), + ) + cg.add(var.set_snapshot_prefix(str(config[CONF_ID]))) diff --git a/esphome/components/snapshot/display/__init__.py b/esphome/components/snapshot/display/__init__.py new file mode 100644 index 0000000000..68429f164b --- /dev/null +++ b/esphome/components/snapshot/display/__init__.py @@ -0,0 +1,61 @@ +import esphome.codegen as cg +from esphome.components import display +import esphome.config_validation as cv +from esphome.const import ( + CONF_DIMENSIONS, + CONF_HEIGHT, + CONF_ID, + CONF_LAMBDA, + CONF_WIDTH, + PLATFORM_HOST, +) +from esphome.types import ConfigType + +from .. import Snapshot, register_snapshot, snapshot_ns + +# The base class and the file writing live in the parent component, which nothing else in a +# configuration using only this platform would pull in. +AUTO_LOAD = ["snapshot"] + +SnapshotDisplay = snapshot_ns.class_( + "SnapshotDisplay", display.DisplayBuffer, cg.Component, Snapshot +) + +CONFIG_SCHEMA = cv.All( + display.FULL_DISPLAY_SCHEMA.extend( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(SnapshotDisplay), + cv.Required(CONF_DIMENSIONS): cv.Any( + cv.dimensions, + cv.Schema( + { + cv.Required(CONF_WIDTH): cv.positive_not_null_int, + cv.Required(CONF_HEIGHT): cv.positive_not_null_int, + } + ), + ), + } + ) + ), + cv.only_on(PLATFORM_HOST), +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await display.register_display(var, config) + await register_snapshot(var, config) + + dimensions = config[CONF_DIMENSIONS] + if isinstance(dimensions, dict): + cg.add(var.set_dimensions(dimensions[CONF_WIDTH], dimensions[CONF_HEIGHT])) + else: + (width, height) = dimensions + cg.add(var.set_dimensions(width, height)) + + if lamb := config.get(CONF_LAMBDA): + lambda_ = await cg.process_lambda( + lamb, [(display.DisplayRef, "it")], return_type=cg.void + ) + cg.add(var.set_writer(lambda_)) diff --git a/esphome/components/snapshot/display/snapshot_display.cpp b/esphome/components/snapshot/display/snapshot_display.cpp new file mode 100644 index 0000000000..6297e3e18f --- /dev/null +++ b/esphome/components/snapshot/display/snapshot_display.cpp @@ -0,0 +1,80 @@ +#ifdef USE_HOST +#include "snapshot_display.h" +#include "esphome/components/display/display_color_utils.h" +#include "esphome/core/log.h" + +#include + +namespace esphome::snapshot { + +static const char *const TAG = "snapshot.display"; + +namespace { + +/// Spread a channel that only goes up to `max` over the whole 0 to 255 range, so that the +/// brightest value stays the brightest. This is the same arithmetic SDL uses, which is what makes +/// a picture taken here come out identical to the same picture taken from an SDL window. +constexpr uint8_t expand_channel(uint16_t value, uint16_t max) { return static_cast(value * 255 / max); } + +constexpr uint16_t RED_MAX = 0x1F; +constexpr uint16_t GREEN_MAX = 0x3F; +constexpr uint16_t BLUE_MAX = 0x1F; + +} // namespace + +void SnapshotDisplay::setup() { + this->init_internal_(static_cast(this->width_) * this->height_ * 2); + if (this->buffer_ == nullptr) { + this->mark_failed(LOG_STR("Could not allocate display buffer")); + } +} + +void SnapshotDisplay::dump_config() { LOG_DISPLAY("", "Snapshot", this); } + +void SnapshotDisplay::draw_absolute_pixel_internal(int x, int y, Color color) { + if (this->buffer_ == nullptr || x < 0 || x >= this->width_ || y < 0 || y >= this->height_) + return; + this->pixels_()[y * this->width_ + x] = display::ColorUtil::color_to_565(color, display::COLOR_ORDER_RGB); +} + +void SnapshotDisplay::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, + display::ColorOrder order, display::ColorBitness bitness, bool big_endian, + int x_offset, int y_offset, int x_pad) { + if (this->buffer_ == nullptr) + return; + // Anything that is not already laid out the way the buffer is, or that would reach outside it, + // goes through the base class, which turns it into one call per pixel with the bounds checked. + const bool copyable = this->rotation_ == display::DISPLAY_ROTATION_0_DEGREES && + bitness == display::COLOR_BITNESS_565 && !big_endian && x_start >= 0 && y_start >= 0 && + x_start + w <= this->width_ && y_start + h <= this->height_; + if (!copyable) { + DisplayBuffer::draw_pixels_at(x_start, y_start, w, h, ptr, order, bitness, big_endian, x_offset, y_offset, x_pad); + return; + } + const size_t stride = static_cast(x_offset) + w + x_pad; + const uint8_t *src = ptr + (stride * y_offset + x_offset) * 2; + for (int y = 0; y != h; y++) { + memcpy(&this->pixels_()[(y_start + y) * this->width_ + x_start], src + y * stride * 2, w * 2); + } +} + +bool SnapshotDisplay::capture_bgr(uint8_t *dest, size_t row_stride) { + if (this->buffer_ == nullptr) { + ESP_LOGE(TAG, "Snapshot requested but there is no buffer to read"); + return false; + } + const uint16_t *src = this->pixels_(); + for (int y = 0; y != this->height_; y++) { + uint8_t *out = dest + y * row_stride; + for (int x = 0; x != this->width_; x++) { + const uint16_t pixel = *src++; + *out++ = expand_channel(pixel & BLUE_MAX, BLUE_MAX); + *out++ = expand_channel((pixel >> 5) & GREEN_MAX, GREEN_MAX); + *out++ = expand_channel(pixel >> 11, RED_MAX); + } + } + return true; +} + +} // namespace esphome::snapshot +#endif diff --git a/esphome/components/snapshot/display/snapshot_display.h b/esphome/components/snapshot/display/snapshot_display.h new file mode 100644 index 0000000000..5317bc6058 --- /dev/null +++ b/esphome/components/snapshot/display/snapshot_display.h @@ -0,0 +1,48 @@ +#pragma once + +#ifdef USE_HOST +#include "esphome/components/display/display_buffer.h" +#include "esphome/components/snapshot/snapshot.h" +#include "esphome/core/component.h" + +namespace esphome::snapshot { + +/// A display with nowhere to show anything: it keeps the picture in memory, where the snapshot +/// action can pick it up. That makes it a way to see what a configuration draws on a machine with +/// no screen, and to check the result in a test. +class SnapshotDisplay final : public display::DisplayBuffer, public Snapshot { + public: + void setup() override; + void update() override { this->do_update_(); } + void dump_config() override; + float get_setup_priority() const override { return setup_priority::HARDWARE; } + display::DisplayType get_display_type() override { return display::DISPLAY_TYPE_COLOR; } + + void set_dimensions(uint16_t width, uint16_t height) { + this->width_ = width; + this->height_ = height; + } + + void draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, display::ColorOrder order, + display::ColorBitness bitness, bool big_endian, int x_offset, int y_offset, int x_pad) override; + + protected: + void draw_absolute_pixel_internal(int x, int y, Color color) override; + int get_width_internal() override { return this->width_; } + int get_height_internal() override { return this->height_; } + + int snapshot_width() override { return this->width_; } + int snapshot_height() override { return this->height_; } + bool capture_bgr(uint8_t *dest, size_t row_stride) override; + + /// The picture, one 16 bit RGB565 value per pixel, topmost row first. Owned by DisplayBuffer as + /// a byte pointer; this is the same memory seen as what is actually stored in it. + uint16_t *pixels_() { return reinterpret_cast(this->buffer_); } + + int width_{}; + int height_{}; +}; + +} // namespace esphome::snapshot + +#endif diff --git a/esphome/components/snapshot/snapshot.cpp b/esphome/components/snapshot/snapshot.cpp new file mode 100644 index 0000000000..995f87710e --- /dev/null +++ b/esphome/components/snapshot/snapshot.cpp @@ -0,0 +1,248 @@ +#ifdef USE_HOST +#include "snapshot.h" +#include "esphome/core/log.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace esphome::snapshot { + +namespace { + +constexpr const char *const TAG = "snapshot"; + +// Longest name we will build a path from. NAME_MAX is 255 and we may append a collision suffix. +constexpr size_t MAX_NAME_LENGTH = 200; +// Give up rather than spin forever if every candidate name is taken. +constexpr unsigned MAX_NAME_ATTEMPTS = 1000; +// A BMP file header followed by a BITMAPINFOHEADER, which is where the pixels start. +constexpr size_t BMP_HEADER_SIZE = 54; +constexpr size_t BMP_INFO_HEADER_SIZE = 40; +constexpr int BMP_BITS_PER_PIXEL = 24; + +/// True if the name already ends in ".bmp". The comparison ignores case, so "shot.BMP" is left +/// alone rather than turned into "shot.BMP.bmp". +bool has_bmp_suffix(const std::string &name) { + return name.size() >= 4 && strcasecmp(name.c_str() + name.size() - 4, ".bmp") == 0; +} + +/// Reduce a user supplied name to a single safe path component. Everything outside the allowed set +/// is replaced, so "..", "/" and absolute paths cannot escape the snapshot directory. +/// Returns an empty string if nothing usable is left. +std::string sanitise_filename(const char *const name, bool *name_changed) { + std::string result; + bool all_dots = true; + bool changed = false; + for (const char *p = name; *p != '\0'; p++) { + if (result.size() >= MAX_NAME_LENGTH) { + changed = true; + break; + } + char c = *p; + if (!(std::isalnum(static_cast(c)) || c == '.' || c == '_' || c == '-')) { + c = '_'; + changed = true; + } + if (c != '.') + all_dots = false; + result.push_back(c); + } + if (all_dots) { + *name_changed = true; + return ""; + } + if (!has_bmp_suffix(result)) + result += ".bmp"; + *name_changed = changed; + return result; +} + +/// Insert "-" before the file extension, e.g. "shot.bmp" -> "shot-1.bmp". +std::string add_suffix(const std::string &name, unsigned attempt) { + char suffix[12]; + snprintf(suffix, sizeof(suffix), "-%u", attempt); + auto dot = name.rfind('.'); + if (dot == std::string::npos) + return name + suffix; + return name.substr(0, dot) + suffix + name.substr(dot); +} + +/// Directory snapshots are written to. The environment variable lets a test redirect output +/// without rebuilding, matching how the host platform handles ESPHOME_PREFDIR. +const char *snapshot_dir() { + const char *dir = getenv("ESPHOME_SNAPSHOT_DIR"); // NOLINT(concurrency-mt-unsafe) + return dir != nullptr && dir[0] != '\0' ? dir : ESPHOME_SNAPSHOT_DIR; +} + +/// Store a value in as many bytes, least significant first, and step the pointer past it. +/// BMP is a little endian format whatever the machine writing it uses. +void put_le(uint8_t *&dest, uint32_t value, size_t bytes) { + for (size_t i = 0; i != bytes; i++) + *dest++ = static_cast(value >> (8 * i)); +} + +/// The number of bytes one row of `width` pixels takes up in the file. Rows are padded out to a +/// multiple of four bytes. +size_t bmp_row_size(int width) { return (static_cast(width) * 3 + 3) & ~size_t{3}; } + +/// Write pixels out as a 24 bit BMP. The rows given start with the topmost and are `row_stride` +/// bytes apart, which must leave room for a whole padded row; a BMP holds its rows the other way +/// up, so they go out last first. +bool write_bmp(FILE *file, const uint8_t *pixels, int width, int height, size_t row_stride) { + const size_t row_size = bmp_row_size(width); + const size_t pixel_bytes = row_size * height; + + uint8_t header[BMP_HEADER_SIZE]; + uint8_t *pos = header; + *pos++ = 'B'; + *pos++ = 'M'; + put_le(pos, static_cast(BMP_HEADER_SIZE + pixel_bytes), 4); + put_le(pos, 0, 4); // reserved + put_le(pos, BMP_HEADER_SIZE, 4); + put_le(pos, BMP_INFO_HEADER_SIZE, 4); + put_le(pos, static_cast(width), 4); + put_le(pos, static_cast(height), 4); + put_le(pos, 1, 2); // one plane + put_le(pos, BMP_BITS_PER_PIXEL, 2); + put_le(pos, 0, 4); // not compressed + put_le(pos, static_cast(pixel_bytes), 4); + put_le(pos, 0, 4); // pixels per metre across, unspecified + put_le(pos, 0, 4); // pixels per metre down, unspecified + put_le(pos, 0, 4); // no palette + put_le(pos, 0, 4); // so no palette entry matters more than another + + if (fwrite(header, 1, sizeof(header), file) != sizeof(header)) + return false; + for (int y = height - 1; y >= 0; y--) { + if (fwrite(pixels + static_cast(y) * row_stride, 1, row_size, file) != row_size) + return false; + } + return true; +} + +/// Reserve a name in the snapshot directory and write the picture to it. +/// With `exact` set the given name is the only one tried; otherwise a number is added on +/// collision. Returns true if a file was written. +bool write_snapshot_file(const uint8_t *pixels, int width, int height, size_t row_stride, const std::string &name, + bool exact) { + const std::string dir = snapshot_dir(); + std::error_code ec; + std::filesystem::create_directories(dir, ec); + if (ec) { + ESP_LOGE(TAG, "Could not create snapshot directory %s: %s", dir.c_str(), ec.message().c_str()); + return false; + } + + // O_EXCL guarantees we never write over a file that is already there. + std::string path; + int fd = -1; + for (unsigned attempt = 0; attempt < MAX_NAME_ATTEMPTS; attempt++) { + path = dir + "/" + (attempt == 0 ? name : add_suffix(name, attempt)); + fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0644); + if (fd >= 0) + break; + if (errno != EEXIST) { + ESP_LOGE(TAG, "Could not create %s: %s", path.c_str(), strerror(errno)); + return false; + } + if (exact) { + // The caller asked for this exact name, so silently writing somewhere else would be worse + // than failing - a test asserting on the path would pick up a stale file. + ESP_LOGE(TAG, "Snapshot %s already exists, not overwriting", path.c_str()); + return false; + } + } + if (fd < 0) { + ESP_LOGE(TAG, "Could not find an unused name for %s in %s", name.c_str(), dir.c_str()); + return false; + } + + FILE *file = fdopen(fd, "wb"); + if (file == nullptr) { + ESP_LOGE(TAG, "Could not open %s: %s", path.c_str(), strerror(errno)); + ::close(fd); + ::unlink(path.c_str()); + return false; + } + bool ok = write_bmp(file, pixels, width, height, row_stride); + int saved_errno = ok ? 0 : errno; + // Closing can fail in its own right - the last of the data is still on its way out. + if (fclose(file) != 0) { + if (ok) + saved_errno = errno; + ok = false; + } + if (!ok) { + ESP_LOGE(TAG, "Could not write %s: %s", path.c_str(), strerror(saved_errno)); + // Leave no truncated file behind - it would block a retry under the same name. + ::unlink(path.c_str()); + return false; + } + ESP_LOGI(TAG, "Snapshot written to %s", path.c_str()); + return true; +} + +} // namespace + +// helper function since ESP_LOGW is disallowed in a header file +void Snapshot::log_action_failed() { ESP_LOGW(TAG, "snapshot.take did not write a file"); } + +bool Snapshot::take_snapshot(const char *filename) { + const int width = this->snapshot_width(); + const int height = this->snapshot_height(); + if (width <= 0 || height <= 0) { + ESP_LOGE(TAG, "Snapshot requested but the display is %dx%d", width, height); + return false; + } + + std::string name; + bool exact = false; + if (filename != nullptr) { + bool name_changed = false; + name = sanitise_filename(filename, &name_changed); + exact = !name.empty(); + if (name_changed) { + ESP_LOGW(TAG, "Requested snapshot name '%s' is not an acceptable file name, using '%s' instead", filename, + name.empty() ? "a name made from the time" : name.c_str()); + } + } + if (name.empty()) { + struct timespec now {}; + if (clock_gettime(CLOCK_REALTIME, &now) != 0) + now = {}; + struct tm tm_buf {}; + if (localtime_r(&now.tv_sec, &tm_buf) == nullptr) + tm_buf = {}; + char stamp[32]{}; + // ::strftime to be sure of the one from ; display has an unrelated member of that name + if (::strftime(stamp, sizeof(stamp), "%Y%m%d-%H%M%S", &tm_buf) == 0) + snprintf(stamp, sizeof(stamp), "unknown-time"); + char buffer[MAX_NAME_LENGTH]; + int written = + snprintf(buffer, sizeof(buffer), "%s-%s-%03ld.bmp", this->snapshot_prefix_, stamp, now.tv_nsec / 1000000); + if (written < 0 || static_cast(written) >= sizeof(buffer)) { + ESP_LOGW(TAG, "Could not build a timestamped snapshot name, using a fallback"); + snprintf(buffer, sizeof(buffer), "snapshot.bmp"); + } + name = buffer; + } + + // Rows are padded out to a multiple of four bytes, as the file wants them, so each one can be + // written straight from the buffer. Zeroed on allocation, which is what the padding must be. + const size_t row_stride = bmp_row_size(width); + auto pixels = std::make_unique(row_stride * height); + if (!this->capture_bgr(pixels.get(), row_stride)) + return false; + return write_snapshot_file(pixels.get(), width, height, row_stride, name, exact); +} + +} // namespace esphome::snapshot +#endif diff --git a/esphome/components/snapshot/snapshot.h b/esphome/components/snapshot/snapshot.h new file mode 100644 index 0000000000..bb670e639f --- /dev/null +++ b/esphome/components/snapshot/snapshot.h @@ -0,0 +1,72 @@ +#pragma once + +#ifdef USE_HOST +#include "esphome/core/automation.h" + +#include +#include +#include + +// Directory snapshots are written to. Normally set by codegen to a folder under .esphome; the +// fallback keeps the component compiling for static analysis, where no defines.h is generated. +#ifndef ESPHOME_SNAPSHOT_DIR +#define ESPHOME_SNAPSHOT_DIR "." +#endif + +namespace esphome::snapshot { + +/// Base for anything that can hand over the picture it is showing so it can be written to a file. +/// +/// A subclass says how big the picture is and fills in the pixels. Everything else - picking a +/// name, staying inside the snapshot directory, not writing over anything, and encoding the file - +/// is done here, so every component that can take a snapshot behaves the same way. +class Snapshot { + public: + virtual ~Snapshot() = default; + + /// Set the word generated names start with. Codegen passes the component id, so with more than + /// one display in a device it is clear which one a file came from. + void set_snapshot_prefix(const char *prefix) { this->snapshot_prefix_ = prefix; } + + /// Write the current picture to a BMP file in the snapshot directory. + /// + /// Pass nullptr to have a name made up from the prefix and the current time. A file that is + /// already there is never written over. Returns true if a file was written. + bool take_snapshot(const char *filename); + + /// Log that an action-triggered snapshot did not write a file. + static void log_action_failed(); + + protected: + /// Width of the picture in pixels. + virtual int snapshot_width() = 0; + /// Height of the picture in pixels. + virtual int snapshot_height() = 0; + /// Fill in the picture: three bytes per pixel in blue, green, red order, topmost row first, with + /// `row_stride` bytes from the start of one row to the start of the next. Returns false, having + /// logged why, if the picture could not be read. + virtual bool capture_bgr(uint8_t *dest, size_t row_stride) = 0; + + const char *snapshot_prefix_{"snapshot"}; +}; + +template class SnapshotAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(std::string, filename) + + protected: + void play(const Ts &...x) override { + bool ok; + if (this->filename_.has_value()) { + ok = this->parent_->take_snapshot(this->filename_.value(x...).c_str()); + } else { + ok = this->parent_->take_snapshot(nullptr); + } + if (!ok) + this->parent_->log_action_failed(); + } +}; + +} // namespace esphome::snapshot + +#endif diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 7af41409fd..526adf74f0 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -13,6 +13,7 @@ #define ESPHOME_PROJECT_VERSION "v2" #define ESPHOME_PROJECT_VERSION_30 "v2" #define ESPHOME_VARIANT "ESP32" +#define ESPHOME_SNAPSHOT_DIR "." #define ESPHOME_NAME_ADD_MAC_SUFFIX #define ESPHOME_DEBUG_SCHEDULER #define ESPHOME_DEBUG_API diff --git a/tests/component_tests/sdl/test_sdl.py b/tests/component_tests/sdl/test_sdl.py new file mode 100644 index 0000000000..5ab5e17ee6 --- /dev/null +++ b/tests/component_tests/sdl/test_sdl.py @@ -0,0 +1,101 @@ +"""Tests for the sdl display schema, in particular the headless option.""" + +from __future__ import annotations + +import pytest + +from esphome import config_validation as cv +from esphome.components.sdl.display import ( + CONF_SDL_ID, + CONFIG_SCHEMA, + headless_final_validate, +) +from esphome.config import Config +from esphome.const import PlatformFramework +from esphome.core import ID +from esphome.final_validate import full_config +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +@pytest.fixture(autouse=True) +def _host_platform(set_core_config: SetCoreConfigCallable) -> None: + set_core_config(PlatformFramework.HOST_NATIVE) + + +def _config(**extra: object) -> ConfigType: + config: ConfigType = { + "dimensions": {"width": 320, "height": 240}, + # sdl2-config is not necessarily installed in the test environment + "sdl_options": "-lSDL2", + } + config.update(extra) + return config + + +def test_defaults_to_windowed() -> None: + """A display without the option is not headless.""" + assert CONFIG_SCHEMA(_config())["headless"] is False + + +def test_headless_accepted() -> None: + """A headless display needs nothing beyond the dimensions.""" + assert CONFIG_SCHEMA(_config(headless=True))["headless"] is True + + +def test_headless_rejects_window_options() -> None: + """Window options are meaningless without a window.""" + with pytest.raises(cv.Invalid, match="has no effect"): + CONFIG_SCHEMA( + _config(headless=True, window_options={"position": {"x": 0, "y": 0}}) + ) + + +def test_headless_rejects_snapshot_key() -> None: + """A headless display has no keyboard, so the action is the only way in.""" + with pytest.raises(cv.Invalid, match="snapshot.take"): + CONFIG_SCHEMA(_config(headless=True, snapshot_key="SDLK_F12")) + + +def test_snapshot_key_accepted_when_windowed() -> None: + """The key is only valid alongside a window.""" + config = CONFIG_SCHEMA(_config(snapshot_key="SDLK_F12")) + assert str(config["snapshot_key"]) == "SDLK_F12" + + +def _declare_sdl_display(headless: bool) -> ID: + """Register a full_config with a single sdl display declaration and return a reference to it. + + Mirrors what the real config pipeline leaves behind: a "display" domain entry plus a + declare_ids record id_declaration_match_schema uses to find it again. + """ + declared_id = ID("my_sdl", is_declaration=True) + fc = Config() + fc["display"] = [ + { + "platform": "sdl", + "id": declared_id, + "headless": headless, + "dimensions": {"width": 320, "height": 240}, + } + ] + fc.declare_ids.append((declared_id, ["display", 0, "id"])) + full_config.set(fc) + return ID("my_sdl") + + +@pytest.mark.parametrize("platform", ["binary_sensor", "touchscreen"]) +def test_headless_final_validate_rejects_headless_display(platform: str) -> None: + """binary_sensor and touchscreen both need a window, so a headless display is rejected.""" + sdl_ref = _declare_sdl_display(headless=True) + schema = headless_final_validate(platform) + with pytest.raises(cv.Invalid, match="needs a window"): + schema({CONF_SDL_ID: sdl_ref}) + + +@pytest.mark.parametrize("platform", ["binary_sensor", "touchscreen"]) +def test_headless_final_validate_accepts_windowed_display(platform: str) -> None: + """The same platforms are accepted once the display has a window.""" + sdl_ref = _declare_sdl_display(headless=False) + schema = headless_final_validate(platform) + schema({CONF_SDL_ID: sdl_ref}) # Should not raise. diff --git a/tests/components/sdl/common.yaml b/tests/components/sdl/common.yaml index 3be86cf8be..1bb0434057 100644 --- a/tests/components/sdl/common.yaml +++ b/tests/components/sdl/common.yaml @@ -14,6 +14,15 @@ display: position: x: 100 y: 100 + snapshot_key: SDLK_F12 + + - platform: sdl + id: headless_display + headless: true + show_test_card: true + dimensions: + width: 320 + height: 240 - platform: sdl id: second_display @@ -46,3 +55,21 @@ binary_sensor: sdl_id: sdl_sdl_display id: key_enter key: SDLK_RETURN + +esphome: + # A name of your own is only good for one snapshot - a second one under the same name fails + # rather than writing over the first - so these run once rather than on a repeating interval. + on_boot: + - delay: 2s + - snapshot.take: + id: headless_display + filename: test_card.bmp + - snapshot.take: + id: headless_display + filename: !lambda 'return "shot.bmp";' + +interval: + # A generated name has the time in it, so this one can repeat. + - interval: 10s + then: + - snapshot.take: sdl_sdl_display diff --git a/tests/components/sdl/validate.host.yaml b/tests/components/sdl/validate.host.yaml new file mode 100644 index 0000000000..883f34675d --- /dev/null +++ b/tests/components/sdl/validate.host.yaml @@ -0,0 +1,29 @@ +# Config-only test for the headless and screenshot options. The combinations that must be +# rejected are covered by tests/component_tests/sdl/test_sdl.py; this file checks that the +# accepted forms validate together. +host: + mac_address: "62:23:45:AF:B3:DD" + +display: + - platform: sdl + id: headless_display + headless: true + dimensions: 320x240 + + - platform: sdl + id: windowed_display + dimensions: 320x240 + snapshot_key: SDLK_F12 + +binary_sensor: + - platform: sdl + sdl_id: windowed_display + id: key_up + key: SDLK_UP + +interval: + - interval: 10s + then: + - snapshot.take: + id: headless_display + filename: periodic.bmp diff --git a/tests/components/snapshot/common.yaml b/tests/components/snapshot/common.yaml new file mode 100644 index 0000000000..9ce2d33a87 --- /dev/null +++ b/tests/components/snapshot/common.yaml @@ -0,0 +1,34 @@ +display: + - platform: snapshot + id: snapshot_display + update_interval: 1s + show_test_card: true + # An odd width exercises the row padding in the BMP writer + dimensions: + width: 101 + height: 64 + + - platform: snapshot + id: snapshot_rotated + rotation: 90 + dimensions: 320x240 + lambda: |- + it.filled_rectangle(0, 0, 40, 20, Color(0xFF, 0x80, 0x00)); + +esphome: + # A name of your own is only good for one snapshot - a second one under the same name fails + # rather than writing over the first - so these run once rather than on a repeating interval. + on_boot: + - delay: 2s + - snapshot.take: + id: snapshot_display + filename: test_card.bmp + - snapshot.take: + id: snapshot_rotated + filename: !lambda 'return "rotated.bmp";' + +interval: + # A generated name has the time in it, so this one can repeat. + - interval: 10s + then: + - snapshot.take: snapshot_display diff --git a/tests/components/snapshot/test.host.yaml b/tests/components/snapshot/test.host.yaml new file mode 100644 index 0000000000..951be2ed04 --- /dev/null +++ b/tests/components/snapshot/test.host.yaml @@ -0,0 +1,5 @@ +host: + mac_address: "62:23:45:AF:B3:DE" + +packages: + snapshot: !include common.yaml diff --git a/tests/integration/artifact_utils.py b/tests/integration/artifact_utils.py new file mode 100644 index 0000000000..cf18946512 --- /dev/null +++ b/tests/integration/artifact_utils.py @@ -0,0 +1,26 @@ +"""Shared utilities for ESPHome integration tests - keeping output from failing tests.""" + +from __future__ import annotations + +from pathlib import Path + +#: Where a failing test leaves output for someone to look at afterwards. pytest's own +#: temporary folder is no use on a CI runner, which throws the whole workspace away when +#: the job ends; the workflow uploads this folder instead when a job fails. +ARTIFACT_DIR = Path(__file__).resolve().parents[2] / "test_artifacts" + + +def keep_artifact(name: str, data: bytes) -> Path: + """Write ``data`` where it can still be read after the run, and return the path. + + Args: + name: File name to write under the artifact folder. + data: Contents to write. + + Returns: + The full path written. + """ + ARTIFACT_DIR.mkdir(parents=True, exist_ok=True) + path = ARTIFACT_DIR / name + path.write_bytes(data) + return path diff --git a/tests/integration/bmp_utils.py b/tests/integration/bmp_utils.py new file mode 100644 index 0000000000..c10aea5ade --- /dev/null +++ b/tests/integration/bmp_utils.py @@ -0,0 +1,161 @@ +"""Shared utilities for ESPHome integration tests - reading BMP snapshots.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from pathlib import Path +import struct + +# Size of the smallest BMP header pair (file header plus BITMAPINFOHEADER). +_MIN_HEADER_SIZE = 54 + +# How long capture_when_drawn() keeps asking for a picture with something on it. +DRAW_TIMEOUT = 15.0 + + +@dataclass(frozen=True) +class Bmp: + """A decoded BMP image.""" + + width: int + height: int + bits: int + #: Pixel data with the per row padding stripped, so it depends only on the image itself. + pixels: bytes + + +class NotABmpError(Exception): + """The data is not a BMP at all, as opposed to a BMP that is still being written.""" + + +def parse_bmp(data: bytes) -> Bmp | None: + """Decode a BMP, or return None if the data is not a complete image yet. + + Raises: + NotABmpError: If the data cannot become a valid BMP however much more is appended. + """ + # Writes go to the file in order, so a short read is always a prefix of what will be there. + # Anything wrong in a prefix we have already read is wrong for good, and worth saying now + # rather than reporting as a timeout later. + if len(data) >= 2 and data[:2] != b"BM": + raise NotABmpError(f"expected a BMP, got {data[:2]!r}") + if len(data) < _MIN_HEADER_SIZE: + return None + file_size = struct.unpack_from(" Bmp: + """Wait for a complete BMP file to appear at ``path`` and return it. + + The file is created before any of its contents are written, so waiting for it to exist is + not enough - a read that wins the race sees a truncated image. Keep reading until the + headers say the whole image is there. + + Args: + path: The file to wait for. + timeout: Maximum time to wait in seconds. + + Returns: + The decoded image. + + Raises: + AssertionError: If no complete image is readable within ``timeout``. + NotABmpError: If what was written is not a BMP. This is reported as soon as it is + seen, so a device that writes the wrong thing is named for what it did rather + than waiting out the timeout. + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while True: + try: + data = path.read_bytes() + except FileNotFoundError: + data = b"" + if (image := parse_bmp(data)) is not None: + return image + if loop.time() >= deadline: + break + await asyncio.sleep(0.05) + if not data: + raise AssertionError(f"no snapshot appeared at {path} within {timeout}s") + raise AssertionError( + f"{path} was still incomplete after {timeout}s ({len(data)} bytes)" + ) + + +def is_blank(image: Bmp) -> bool: + """True if every pixel of the image is the same colour. + + Whole pixels are counted rather than byte values: a plain background is usually made of more + than one distinct byte, so counting bytes would find several of them in a blank screen. + """ + return len({image.pixels[i : i + 3] for i in range(0, len(image.pixels), 3)}) <= 1 + + +async def capture_when_drawn( + take: Callable[[str], Awaitable[None]], + directory: Path, + prefix: str = "drawn", + timeout: float = DRAW_TIMEOUT, +) -> tuple[Bmp, Path]: + """Ask for snapshots until one has something drawn on it, and return it and where it went. + + A display holds one flat colour until it first draws, which is one update interval after it + starts - long enough that a test connecting over the API can easily get in first. Capturing + once and hoping would compare a blank screen against whatever the test expects, reporting a + drawing fault where the real trouble was timing. + + Args: + take: Asks the device for a snapshot under the name it is given. + directory: Where the device writes them. + prefix: Start of the names asked for. Each attempt needs its own, because a snapshot never + writes over a file that is already there. + timeout: How long to keep asking. + + Returns: + The first image that is not one flat colour, and the path it was read from. + + Raises: + AssertionError: If nothing had been drawn within ``timeout``. + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + attempt = 0 + while True: + attempt += 1 + path = directory / f"{prefix}-{attempt}.bmp" + await take(path.name) + image = await wait_for_bmp(path) + if not is_blank(image): + return image, path + if loop.time() >= deadline: + raise AssertionError( + f"the screen was still a single flat colour after {timeout}s and " + f"{attempt} captures - nothing was drawn" + ) + await asyncio.sleep(0.5) diff --git a/tests/integration/fixtures/lvgl_headless_render.yaml b/tests/integration/fixtures/lvgl_headless_render.yaml new file mode 100644 index 0000000000..670b51ab53 --- /dev/null +++ b/tests/integration/fixtures/lvgl_headless_render.yaml @@ -0,0 +1,53 @@ +esphome: + name: lvgl-headless-render-test +host: + +api: + actions: + # The name comes from the test so it can capture more than once: a snapshot never writes over + # a file that is already there, so a fixed name could only ever be captured once. + - action: take_screenshot + variables: + name: string + then: + - snapshot.take: + id: lvgl_display + filename: !lambda return name; + +logger: + level: DEBUG + +display: + # A display with no screen, so what LVGL draws depends on LVGL alone - nothing about the machine + # running the test, and no graphics library outside this repository, can move the result. + - platform: snapshot + id: lvgl_display + auto_clear_enabled: false + dimensions: + width: 300 + height: 300 + +# The widgets are spelled out here rather than left to the built in "Hello World" screen, which +# LVGL builds when nothing is configured: that screen contains a spinner, and an animation cannot +# produce the same picture twice. +# +# Everything that affects the rendered pixels is set explicitly, so the expected hash in the test +# depends only on the drawing code and the built in font. In particular the background comes from a +# full screen object rather than from the theme, so adjusting a theme default does not break this. +lvgl: + displays: lvgl_display + default_font: montserrat_14 + widgets: + - obj: + width: 100% + height: 100% + bg_color: 0x000080 + bg_opa: cover + border_width: 0 + radius: 0 + pad_all: 0 + widgets: + - label: + align: center + text: "Hello World!" + text_color: 0xFFFFFF diff --git a/tests/integration/fixtures/sdl_headless_screenshot.yaml b/tests/integration/fixtures/sdl_headless_screenshot.yaml new file mode 100644 index 0000000000..7ce2df130c --- /dev/null +++ b/tests/integration/fixtures/sdl_headless_screenshot.yaml @@ -0,0 +1,29 @@ +esphome: + name: sdl-headless-screenshot-test +host: + +api: + actions: + # The name comes from the test so it can capture more than once while it waits for the first + # frame: a snapshot never writes over a file that is already there. + - action: take_screenshot + variables: + name: string + then: + - snapshot.take: + id: sdl_display + filename: !lambda return name; + +logger: + level: DEBUG + +display: + - platform: sdl + id: sdl_display + headless: true + show_test_card: true + update_interval: 100ms + # An odd width exercises the row padding in the BMP writer + dimensions: + width: 101 + height: 64 diff --git a/tests/integration/fixtures/snapshot_display.yaml b/tests/integration/fixtures/snapshot_display.yaml new file mode 100644 index 0000000000..d10af09806 --- /dev/null +++ b/tests/integration/fixtures/snapshot_display.yaml @@ -0,0 +1,28 @@ +esphome: + name: snapshot-display-test +host: + +api: + actions: + # The name comes from the test so it can ask for several in a row and check what each one + # does with it. + - action: take_snapshot + variables: + name: string + then: + - snapshot.take: + id: snapshot_display + filename: !lambda return name; + +logger: + level: DEBUG + +display: + - platform: snapshot + id: snapshot_display + show_test_card: true + update_interval: 100ms + # An odd width exercises the row padding in the BMP writer + dimensions: + width: 101 + height: 64 diff --git a/tests/integration/test_lvgl_headless_render.py b/tests/integration/test_lvgl_headless_render.py new file mode 100644 index 0000000000..1c60e49604 --- /dev/null +++ b/tests/integration/test_lvgl_headless_render.py @@ -0,0 +1,83 @@ +"""Integration test that checks what LVGL actually draws, using a display with no screen. + +The rendered screen is compared against a hash rather than a checked in reference image, so the +repository does not have to carry a binary file. If a change to the drawing code or to the bundled +LVGL alters the output, this test fails and prints the hash it saw; update EXPECTED_SHA256 once the +new image has been looked at and found to be correct. + +The picture is drawn and encoded entirely by code in this repository, so nothing installed on the +machine running the test takes part in the result. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from .artifact_utils import keep_artifact +from .bmp_utils import capture_when_drawn +from .types import APIClientConnectedFactory, RunCompiledFunction + +WIDTH = 300 +HEIGHT = 300 + +# sha256 of the pixel data of a 300x300 screen showing "Hello World!" centred in white on a dark +# blue background, drawn with the built in montserrat_14 font. To regenerate, run this test and +# take the hash it reports. +EXPECTED_SHA256 = "a995b002dd1d183c47514da15ab9a60a3e7d788c2e24386a02fddd48655092ed" +# Bundled LVGL version (esphome/components/lvgl/__init__.py, LVGL_VERSION) the hash above was +# generated against. A version bump can shift anti-aliasing enough to change the hash even though +# nothing is actually wrong -- if this test fails, check that first before regenerating the hash. +EXPECTED_LVGL_VERSION = "9.5.0" + + +@pytest.mark.asyncio +async def test_lvgl_headless_render( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """LVGL draws the expected screen on a 300x300 display with no screen behind it.""" + snapshot_dir = tmp_path / "snapshots" + monkeypatch.setenv("ESPHOME_SNAPSHOT_DIR", str(snapshot_dir)) + + async with run_compiled(yaml_config), api_client_connected() as client: + _, services = await client.list_entities_services() + service = next(s for s in services if s.name == "take_screenshot") + + async def take(name: str) -> None: + await client.execute_service(service, {"name": name}) + + # The background is not the whole picture: LVGL must have drawn on it. Waiting for that + # rather than for a fixed time keeps a slow first frame from being reported as a hash + # mismatch, which would look like a drawing regression. + image, capture = await capture_when_drawn(take, snapshot_dir, prefix="render") + assert (image.width, image.height, image.bits) == (WIDTH, HEIGHT, 24) + + digest = hashlib.sha256(image.pixels).hexdigest() + if digest != EXPECTED_SHA256: + # Kept outside the temporary folder so CI can upload it; see artifact_utils. + kept = keep_artifact( + "lvgl_headless_render_actual.bmp", capture.read_bytes() + ) + + from esphome.components.lvgl import LVGL_VERSION + + version_hint = "" + if LVGL_VERSION != EXPECTED_LVGL_VERSION: + version_hint = ( + f"the bundled LVGL version changed ({EXPECTED_LVGL_VERSION} -> " + f"{LVGL_VERSION}), which is the likely cause\n" + ) + pytest.fail( + f"rendered screen does not match the expected hash\n" + f"{version_hint}" + f" expected: {EXPECTED_SHA256}\n" + f" actual: {digest}\n" + f"the image that was rendered has been kept at {kept}\n" + f"on CI it is in the integration-test-artifacts upload for this job" + ) diff --git a/tests/integration/test_sdl_headless_screenshot.py b/tests/integration/test_sdl_headless_screenshot.py new file mode 100644 index 0000000000..f24b21c157 --- /dev/null +++ b/tests/integration/test_sdl_headless_screenshot.py @@ -0,0 +1,49 @@ +"""Integration test for headless SDL rendering and snapshot capture. + +How a file is named and written is the same for every display that can take a snapshot and is +covered by test_snapshot_display; what is tested here is that SDL renders and can be read back +with no display server present. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from .bmp_utils import capture_when_drawn +from .types import APIClientConnectedFactory, RunCompiledFunction + +WIDTH = 101 +HEIGHT = 64 + + +@pytest.mark.asyncio +async def test_sdl_headless_screenshot( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A headless SDL display renders with no display server and can be captured.""" + snapshot_dir = tmp_path / "snapshots" + # The device reads this when it writes a file; the subprocess inherits our environment, so it + # must be set before the binary is launched. + monkeypatch.setenv("ESPHOME_SNAPSHOT_DIR", str(snapshot_dir)) + # Make sure the run really is headless even when the test machine has a display. + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + + async with run_compiled(yaml_config), api_client_connected() as client: + _, services = await client.list_entities_services() + service = next(s for s in services if s.name == "take_screenshot") + + async def take(name: str) -> None: + await client.execute_service(service, {"name": name}) + + # The test card is drawn in several colours, so once it is on the screen the picture is + # not one flat shade. Capturing until that is true waits out the first update rather than + # racing it. + image, _ = await capture_when_drawn(take, snapshot_dir) + assert (image.width, image.height, image.bits) == (WIDTH, HEIGHT, 24) diff --git a/tests/integration/test_snapshot_display.py b/tests/integration/test_snapshot_display.py new file mode 100644 index 0000000000..771cf0cf7d --- /dev/null +++ b/tests/integration/test_snapshot_display.py @@ -0,0 +1,78 @@ +"""Integration test for the snapshot display and the file writing shared with other displays.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from aioesphomeapi import LogLevel +import pytest + +from .bmp_utils import capture_when_drawn, wait_for_bmp +from .types import APIClientConnectedFactory, RunCompiledFunction + +WIDTH = 101 +HEIGHT = 64 + +# Part of the message the writer logs when it will not write over a file that is already there. +REFUSAL_MESSAGE = b"not overwriting" + + +@pytest.mark.asyncio +async def test_snapshot_display( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A display with no screen draws into memory and writes what it drew to a file.""" + snapshot_dir = tmp_path / "snapshots" + # The device reads this when it writes a file; the subprocess inherits our environment, so it + # must be set before the binary is launched. + monkeypatch.setenv("ESPHOME_SNAPSHOT_DIR", str(snapshot_dir)) + + async with run_compiled(yaml_config), api_client_connected() as client: + _, services = await client.list_entities_services() + service = next(s for s in services if s.name == "take_snapshot") + + async def take(name: str) -> None: + await client.execute_service(service, {"name": name}) + + # The test card is drawn in several colours, so once it is on the screen the picture is + # not one flat shade. Capturing until that is true waits out the first update rather than + # racing it. + image, capture = await capture_when_drawn(take, snapshot_dir) + assert (image.width, image.height, image.bits) == (WIDTH, HEIGHT, 24) + + # An extension is only added when there is not one already, whatever its case. + await take("UPPER.BMP") + await wait_for_bmp(snapshot_dir / "UPPER.BMP") + + # A name that tries to lead somewhere else is cut back to one harmless name in the + # snapshot directory. + await take("../escape") + await wait_for_bmp(snapshot_dir / ".._escape.bmp") + + # A second capture under a name already used must fail rather than write over the first. + # Wait for the device to report the refusal: on its own, an unchanged file cannot tell a + # refusal apart from a request the device has not got to yet, so a regression that wrote + # over the file could still pass on a busy machine. + refused = asyncio.Event() + + def on_log(msg) -> None: + if REFUSAL_MESSAGE in msg.message: + refused.set() + + client.subscribe_logs(on_log, log_level=LogLevel.LOG_LEVEL_DEBUG) + + before = capture.read_bytes() + await take(capture.name) + await asyncio.wait_for(refused.wait(), timeout=10.0) + assert capture.read_bytes() == before + # Nothing beyond what was asked for, leaving out however many captures it took to wait + # for the first frame. + written = sorted( + p.name for p in snapshot_dir.iterdir() if not p.name.startswith("drawn-") + ) + assert written == [".._escape.bmp", "UPPER.BMP"] From 81ecb872534532390d9ac5a2c2376d68c8fec955 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:11:56 +1200 Subject: [PATCH 004/178] Bump version to 2026.10.0-dev --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 8f6048b4d8..1619371323 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0-dev +PROJECT_NUMBER = 2026.10.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 6f83f0c937..e1d875f94b 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0-dev" +__version__ = "2026.10.0-dev" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 567a98107884152abda87bb42e3519e898e12b67 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:16:25 +1200 Subject: [PATCH 005/178] Bump version to 2026.9.0b1 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 1619371323..7b2d21027a 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.10.0-dev +PROJECT_NUMBER = 2026.9.0b1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index e1d875f94b..378da14197 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.10.0-dev" +__version__ = "2026.9.0b1" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From d1068d582fedc070cd8611b020f9e6f5188dc68c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:56:42 -0400 Subject: [PATCH 006/178] Bump ninja from 1.13.0 to 1.13.2 (#18952) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 594b44432d..4820579e61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.5 # native esp-idf toolchain global cache dir -ninja==1.13.0 # native esp8266 arduino toolchain build driver +ninja==1.13.2 # native esp8266 arduino toolchain build driver filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this From 22504309998347b1ee214107a80d627697f4cd6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:56:51 -0400 Subject: [PATCH 007/178] Bump zeroconf from 0.151.2 to 0.151.3 (#18951) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4820579e61..8731d38b7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.3.1 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.151.2 +zeroconf==0.151.3 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From f3c786c7848201fb4477233609b0e5ec11ec2010 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:11:56 +1200 Subject: [PATCH 008/178] Bump version to 2026.10.0-dev --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 8f6048b4d8..1619371323 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0-dev +PROJECT_NUMBER = 2026.10.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 6f83f0c937..e1d875f94b 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0-dev" +__version__ = "2026.10.0-dev" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 6b1163649166385b8a1dcc398349c4dbaa9f459c Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 3 Sep 2026 07:12:36 -0500 Subject: [PATCH 009/178] [remote_transmitter] Fix BK7231N build by limiting the PWM path to BK7238 (#18958) --- esphome/components/remote_transmitter/__init__.py | 14 +++++--------- .../remote_transmitter/remote_transmitter.h | 9 +++++---- .../remote_transmitter_bk72xx.cpp | 11 +++++++---- .../remote_transmitter_libretiny_isr.cpp | 10 +++++----- .../remote_transmitter/test_non_blocking_gate.py | 2 +- .../remote_transmitter/test.bk72xx-ard.yaml | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index cb2aebec91..58392c48ab 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -4,11 +4,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base from esphome.components.libretiny import get_libretiny_family -from esphome.components.libretiny.const import ( - FAMILY_BK7231N, - FAMILY_BK7238, - FAMILY_RTL8720C, -) +from esphome.components.libretiny.const import FAMILY_BK7238, FAMILY_RTL8720C from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -49,7 +45,9 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) -_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7231N, FAMILY_BK7238) +# Keep in sync with the USE_LIBRETINY_VARIANT_RTL8720C / REMOTE_TRANSMITTER_BK_PWM gates in +# remote_transmitter.h, which decide where set_non_blocking() is declared +_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7238) def _validate_non_blocking_platform(value: bool) -> bool: @@ -59,9 +57,7 @@ def _validate_non_blocking_platform(value: bool) -> bool: return cv.boolean(value) if CORE.is_libretiny and get_libretiny_family() in _NON_BLOCKING_LIBRETINY_FAMILIES: return cv.boolean(value) - raise cv.Invalid( - "non_blocking is only supported on ESP32, RTL8720C, BK7231N and BK7238" - ) + raise cv.Invalid("non_blocking is only supported on ESP32, RTL8720C and BK7238") MULTI_CONF = True diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index 313b26364d..4db4e80a60 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -12,10 +12,11 @@ #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 -// The BK7231N-style PWM block (hardware shadow-load duty updates) enables the ISR-driven -// transmitter on these families; family-level proxy for the SDK's CFG_SOC_NAME gate. -// See remote_transmitter_bk72xx.cpp. -#if defined(USE_LIBRETINY_VARIANT_BK7231N) || defined(USE_LIBRETINY_VARIANT_BK7238) +// Enables the ISR-driven transmitter on Beken. Gated on BK7238 alone: the shadow-load PWM +// block is shared with BK7231N, but LibreTiny builds that family against an older BDK whose +// PWM driver has no pwm_init_param()/pwm_start(). See remote_transmitter_bk72xx.cpp. +// Keep in sync with _NON_BLOCKING_LIBRETINY_FAMILIES in __init__.py. +#ifdef USE_LIBRETINY_VARIANT_BK7238 #define REMOTE_TRANSMITTER_BK_PWM #endif diff --git a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp index 0081ae47b3..822389ccf9 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp @@ -9,10 +9,13 @@ // with the core's fixes for type-name collisions between the two #include -// Only the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) -// supports glitch-free per-edge duty updates; older SoCs compile the generic bit-bang -// implementation (remote_transmitter.cpp) instead, and this file compiles to nothing. -// REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h. +// Needs the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) +// for glitch-free per-edge duty updates, and an SDK exposing pwm_init_param()/pwm_start(). +// BK7231N has the block but LibreTiny builds it against an older BDK offering only the +// sddev_control API (CMD_PWM_INIT_PARAM), so it stays on the generic bit-bang path until +// someone can add and validate that path on real hardware. Every other Beken SoC lacks the +// block. REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h; when it is +// unset this file compiles to nothing and remote_transmitter.cpp is used instead. namespace esphome::remote_transmitter { diff --git a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp index 003cdfa986..fad91f593f 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp @@ -3,11 +3,11 @@ #include "esphome/core/hal.h" #include "esphome/core/log.h" -// Envelope chain shared by the LibreTiny families that pace transmission from a hardware -// timer interrupt: RTL8720C (gtimer) and the BK7231N-style PWM block (BKTIMER1). Everything -// platform-specific sits behind five hooks implemented in the per-family files -- carrier -// setup, duty writes, one-shot arming and timer stop. Families without a usable timer keep -// the generic bit-bang implementation and compile none of this. +// Envelope chain shared by the LibreTiny families that pace transmission from a hardware timer +// interrupt: RTL8720C (gtimer) and BK7238 (BKTIMER1). Everything platform-specific sits behind +// five hooks implemented in the per-family files -- carrier setup, duty writes, one-shot arming +// and timer stop. Families without a usable timer keep the generic bit-bang implementation and +// compile none of this. #if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) namespace esphome::remote_transmitter { diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py index ee2769e177..525ab3329e 100644 --- a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -26,7 +26,7 @@ from ..types import SetCoreConfigCallable (PlatformFramework.ESP32_IDF, None, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), - (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, False), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7238, True), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231T, False), (PlatformFramework.ESP8266_ARDUINO, None, False), diff --git a/tests/components/remote_transmitter/test.bk72xx-ard.yaml b/tests/components/remote_transmitter/test.bk72xx-ard.yaml index ea2feafda9..f3e2da9daf 100644 --- a/tests/components/remote_transmitter/test.bk72xx-ard.yaml +++ b/tests/components/remote_transmitter/test.bk72xx-ard.yaml @@ -2,7 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO26 carrier_duty_percent: 50% - # non_blocking is bk7231n/bk7238-only; the CI board is a BK7252 + # non_blocking is bk7238-only; the CI board is a BK7252, so this builds the bit-bang path packages: buttons: !include common-buttons.yaml From b84532d2548ffe0bb6f326ee26160db423e5d936 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:15:06 +0000 Subject: [PATCH 010/178] Bump bundled esphome-device-builder to 1.14.0 (#18960) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0da8048c57..7952616496 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 RUN \ platformio settings set enable_telemetry No \ From f65ab5629e0401d34d0b0e9be1bc865c76941b9a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 3 Sep 2026 21:16:36 +0200 Subject: [PATCH 011/178] [esp8266] Drop Arduino framework versions before 3.0.0 (#18917) to Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/arduino8266/framework.py | 13 ++--- esphome/components/climate/climate.cpp | 4 +- esphome/components/debug/debug_component.cpp | 4 +- esphome/components/debug/debug_component.h | 4 +- esphome/components/debug/debug_esp8266.cpp | 2 - esphome/components/debug/sensor.py | 7 +-- esphome/components/esp8266/__init__.py | 57 ++++++------------- .../nextion/nextion_upload_arduino.cpp | 6 -- esphome/components/wifi/wifi_component.h | 5 -- .../wifi/wifi_component_esp8266.cpp | 10 +--- esphome/core/log.h | 14 ----- .../components/esp8266/test_boards.py | 17 +----- .../esp8266/test_framework_version.py | 23 ++++++++ .../unit_tests/test_arduino8266_framework.py | 17 ++---- 14 files changed, 62 insertions(+), 121 deletions(-) create mode 100644 tests/unit_tests/components/esp8266/test_framework_version.py diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 1edbe4b36f..663002b3b1 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -44,8 +44,7 @@ def get_arduino8266_tools_path() -> Path: return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) -# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the -# encoder below cannot name 3.0.0/3.0.1 either (see its docstring) +# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0 MIN_FRAMEWORK_VERSION = Version(3, 1, 1) @@ -53,20 +52,16 @@ def framework_package_version(ver: Version) -> str: """Map an Arduino core version to its registry package version (3.1.2 -> 3.30102.0; the leading 3 is the package major). - Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor - at MIN_FRAMEWORK_VERSION. + Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION. """ if ver.major > 3: raise EsphomeError( f"Arduino core {ver} is not supported yet; " "the newest known core series is 3.x" ) - if ver <= Version(2, 6, 2): - # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same - # boundary as _format_framework_arduino_version's era guard) + if ver.major < 3: raise EsphomeError( - f"Arduino core {ver} uses an older package encoding than this " - "helper implements (newer than 2.6.2)" + f"Arduino core {ver} is not supported; ESPHome requires core 3.x" ) return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 34684a87e1..f80de151b1 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -368,8 +368,8 @@ optional Climate::restore_state_() { } void Climate::save_state_(const ClimateTraits &traits) { -#if (defined(USE_ESP32) || (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0))) && \ - !defined(CLANG_TIDY) +#if (defined(USE_ESP32) || defined(USE_ESP8266)) && !defined(CLANG_TIDY) +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wclass-memaccess" #define TEMP_IGNORE_MEMACCESS #endif diff --git a/esphome/components/debug/debug_component.cpp b/esphome/components/debug/debug_component.cpp index 9020c261c2..97f4522c62 100644 --- a/esphome/components/debug/debug_component.cpp +++ b/esphome/components/debug/debug_component.cpp @@ -22,9 +22,9 @@ void DebugComponent::dump_config() { LOG_SENSOR(" ", "Free space on heap", this->free_sensor_); LOG_SENSOR(" ", "Largest free heap block", this->block_sensor_); LOG_SENSOR(" ", "CPU frequency", this->cpu_frequency_sensor_); -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#ifdef USE_ESP8266 LOG_SENSOR(" ", "Heap fragmentation", this->fragmentation_sensor_); -#endif // defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#endif // USE_ESP8266 #endif // USE_SENSOR char device_info_buffer[DEVICE_INFO_BUFFER_SIZE]; diff --git a/esphome/components/debug/debug_component.h b/esphome/components/debug/debug_component.h index 20798cf600..b05029f878 100644 --- a/esphome/components/debug/debug_component.h +++ b/esphome/components/debug/debug_component.h @@ -35,7 +35,7 @@ class DebugComponent final : public PollingComponent { #ifdef USE_SENSOR void set_free_sensor(sensor::Sensor *free_sensor) { free_sensor_ = free_sensor; } void set_block_sensor(sensor::Sensor *block_sensor) { block_sensor_ = block_sensor; } -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) void set_fragmentation_sensor(sensor::Sensor *fragmentation_sensor) { fragmentation_sensor_ = fragmentation_sensor; } #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) @@ -61,7 +61,7 @@ class DebugComponent final : public PollingComponent { sensor::Sensor *free_sensor_{nullptr}; sensor::Sensor *block_sensor_{nullptr}; -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) sensor::Sensor *fragmentation_sensor_{nullptr}; #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) diff --git a/esphome/components/debug/debug_esp8266.cpp b/esphome/components/debug/debug_esp8266.cpp index 272123dfc0..acce28818c 100644 --- a/esphome/components/debug/debug_esp8266.cpp +++ b/esphome/components/debug/debug_esp8266.cpp @@ -159,12 +159,10 @@ void DebugComponent::update_platform_() { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->block_sensor_->publish_state(ESP.getMaxFreeBlockSize()); } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) if (this->fragmentation_sensor_ != nullptr) { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->fragmentation_sensor_->publish_state(ESP.getHeapFragmentation()); } -#endif #endif } diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index 72e2efebc2..e53cb0d1e4 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -52,12 +52,9 @@ CONFIG_SCHEMA = { ), cv.Optional(CONF_FRAGMENTATION): cv.All( cv.Any( - cv.All( - cv.only_on_esp8266, - cv.require_framework_version(esp8266_arduino=cv.Version(2, 5, 2)), - ), + cv.only_on_esp8266, cv.only_on_esp32, - msg="This feature is only available on ESP8266 (Arduino 2.5.2+) and ESP32", + msg="This feature is only available on ESP8266 and ESP32", ), sensor.sensor_schema( unit_of_measurement=UNIT_PERCENT, diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 63665e7681..19dbb68f29 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -35,7 +35,7 @@ from esphome.platformio.toolchain import copy_ccache_script from esphome.storage_json import StorageJSON from esphome.types import ConfigType -from .boards import BOARDS, ESP8266_LD_SCRIPTS, board_ld_script +from .boards import BOARDS, board_ld_script from .const import ( CONF_EARLY_PIN_INIT, CONF_ENABLE_SERIAL, @@ -43,8 +43,6 @@ from .const import ( CONF_RESTORE_FROM_FLASH, KEY_BOARD, KEY_ESP8266, - KEY_FLASH_SIZE, - KEY_LDSCRIPT, KEY_PIN_INITIAL_STATES, KEY_SERIAL1_REQUIRED, KEY_SERIAL_REQUIRED, @@ -133,10 +131,6 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/esp8266/Arduino/releases) version to # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 - if ver <= cv.Version(2, 4, 1): - return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - if ver <= cv.Version(2, 6, 2): - return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" # Same encoding the native toolchain uses for its package download, so a # version bump cannot drift between the two paths. from esphome.arduino8266.framework import framework_package_version @@ -159,11 +153,9 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # - https://github.com/esp8266/Arduino/releases # - https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(3, 1, 2) -# The platformio/espressif8266 version to use for arduino 2 framework versions +# The platformio/espressif8266 version to use for arduino 3 framework versions # - https://github.com/platformio/platform-espressif8266/releases # - https://api.registry.platformio.org/v3/packages/platformio/platform/espressif8266 -ARDUINO_2_PLATFORM_VERSION = cv.Version(2, 6, 3) -# for arduino 3 framework versions ARDUINO_3_PLATFORM_VERSION = cv.Version(3, 2, 0) # for arduino 4 framework versions ARDUINO_4_PLATFORM_VERSION = cv.Version(4, 2, 1) @@ -188,6 +180,14 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: version = cv.Version.parse(cv.version_number(value[CONF_VERSION])) source = value.get(CONF_SOURCE, None) + if version < cv.Version(3, 0, 0): + raise cv.Invalid( + f"Arduino framework {version} is no longer supported; ESPHome requires " + f"C++20, which needs Arduino core 3.x. Use the recommended version " + f"({RECOMMENDED_ARDUINO_FRAMEWORK_VERSION}).", + path=[CONF_VERSION], + ) + value[CONF_VERSION] = str(version) value[CONF_SOURCE] = source or _format_framework_arduino_version(version) @@ -195,12 +195,8 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: if platform_version is None: if version >= cv.Version(3, 1, 0): platform_version = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION)) - elif version >= cv.Version(3, 0, 0): - platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) - elif version >= cv.Version(2, 5, 0): - platform_version = _parse_platform_version(str(ARDUINO_2_PLATFORM_VERSION)) else: - platform_version = _parse_platform_version(str(cv.Version(1, 8, 0))) + platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) value[CONF_PLATFORM_VERSION] = platform_version if version != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION: @@ -289,29 +285,11 @@ def check_rosetta() -> None: ) -def _choose_ld_script(board: str, ver: cv.Version) -> str | None: - """The flash ld to pin for this board and core, or None for cores - without ld-script support.""" - board_data = BOARDS[board] - ld_scripts = ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]] - if ver <= cv.Version(2, 3, 0): - # No ld script support - return None - if ver <= cv.Version(2, 4, 2): - # Old ld script path; the modern per-board override names do not - # exist in this core's SDK, so the override cannot be honored. - # Substituting the size default would move _FS_end and the - # preferences sector, wiping flash-backed state on flash. - if KEY_LDSCRIPT in board_data: - raise EsphomeError( - f"Board {board} requires its {board_data[KEY_LDSCRIPT]} " - f"flash layout, which Arduino core {ver} cannot honor; " - "use a core newer than 2.4.2" - ) - return ld_scripts[0] +def _choose_ld_script(board: str) -> str: + """The flash ld to pin for this board.""" # A per-board override preserves a layout the board shipped with # (see d1_wroom_02 in boards.py) - return board_ld_script(board_data) + return board_ld_script(BOARDS[board]) @coroutine_with_priority(CoroPriority.PLATFORM) @@ -435,10 +413,9 @@ async def to_code(config: ConfigType) -> None: ) if config[CONF_BOARD] in BOARDS: - ld_script = _choose_ld_script(config[CONF_BOARD], ver) - - if ld_script is not None: - cg.add_platformio_option("board_build.ldscript", ld_script) + cg.add_platformio_option( + "board_build.ldscript", _choose_ld_script(config[CONF_BOARD]) + ) CORE.add_job(add_pin_initial_states_array) CORE.add_job(finalize_waveform_config) diff --git a/esphome/components/nextion/nextion_upload_arduino.cpp b/esphome/components/nextion/nextion_upload_arduino.cpp index f02f32d5ca..944fa1db47 100644 --- a/esphome/components/nextion/nextion_upload_arduino.cpp +++ b/esphome/components/nextion/nextion_upload_arduino.cpp @@ -209,14 +209,8 @@ bool Nextion::upload_tft(uint32_t baud_rate, bool exit_reparse) { http_client.setTimeout(this->tft_upload_http_timeout_); bool begin_status = false; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 7, 0) http_client.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); -#elif USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) - http_client.setFollowRedirects(true); -#endif -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) http_client.setRedirectLimit(3); -#endif begin_status = http_client.begin(*this->get_wifi_client_(), this->tft_url_.c_str()); if (!begin_status) { this->connection_state_.is_updating_ = false; diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index cfdbc1a968..63df9fbfa5 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -40,11 +40,6 @@ #include #include -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(2, 4, 0) -extern "C" { -#include -}; -#endif #endif #ifdef USE_RP2 diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index b4a91fb3cd..031da1b355 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -21,7 +21,6 @@ extern "C" { #include "lwip/apps/sntp.h" #include "lwip/netif.h" // struct netif #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) #include "LwipDhcpServer.h" #if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) #include @@ -30,7 +29,6 @@ extern "C" { #define wifi_softap_set_dhcps_lease_time(time) dhcpSoftAP.set_dhcps_lease_time(time) #define wifi_softap_set_dhcps_offer_option(offer, mode) dhcpSoftAP.set_dhcps_offer_option(offer, mode) #endif -#endif } #include "esphome/core/application.h" @@ -293,7 +291,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { conf.bssid_set = 0; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) if (ap.password_.empty()) { conf.threshold.authmode = AUTH_OPEN; } else { @@ -310,7 +307,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { } } conf.threshold.rssi = -127; -#endif ETS_UART_INTR_DISABLE(); bool ret = wifi_station_set_config_current(&conf); @@ -602,7 +598,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) case EVENT_OPMODE_CHANGED: { auto it = event->event_info.opmode_changed; ESP_LOGV(TAG, "Changed Mode old=%s new=%s", LOG_STR_ARG(get_op_mode_str(it.old_opmode)), @@ -620,7 +615,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#endif default: break; } @@ -705,7 +699,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.bssid = nullptr; config.channel = 0; config.show_hidden = 1; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE; // Use shorter dwell times for roaming scans - we only need to detect strong // nearby APs, not do a thorough survey. This also reduces off-channel time @@ -724,7 +717,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.scan_time.active.min = roaming ? SCAN_ACTIVE_MIN_ROAMING_MS : SCAN_ACTIVE_MIN_DEFAULT_MS; config.scan_time.active.max = roaming ? SCAN_ACTIVE_MAX_ROAMING_MS : SCAN_ACTIVE_MAX_DEFAULT_MS; } -#endif bool ret = wifi_station_scan(&config, &WiFiComponent::s_wifi_scan_done_callback); if (!ret) { ESP_LOGV(TAG, "wifi_station_scan failed"); @@ -830,7 +822,7 @@ bool WiFiComponent::wifi_ap_ip_config_(const optional &manual_ip) { return false; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) +#if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) dhcpSoftAP.begin(&info); #endif diff --git a/esphome/core/log.h b/esphome/core/log.h index 272e516808..14d24412ef 100644 --- a/esphome/core/log.h +++ b/esphome/core/log.h @@ -18,7 +18,6 @@ #ifdef USE_STORE_LOG_STR_IN_FLASH #include "WString.h" -#include "esphome/core/defines.h" // for USE_ARDUINO_VERSION_CODE #endif // Include ESP-IDF/Arduino based logging methods here so they don't undefine ours later @@ -177,20 +176,7 @@ struct LogString; #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 0) #define LOG_STR_ARG(s) ((PGM_P) (s)) -#else -// Pre-Arduino 2.5, we can't pass a PSTR() to printf(). Emulate support by copying the message to a -// local buffer first. String length is limited to 63 characters. -// https://github.com/esp8266/Arduino/commit/6280e98b0360f85fdac2b8f10707fffb4f6e6e31 -#define LOG_STR_ARG(s) \ - ({ \ - char __buf[64]; \ - __buf[63] = '\0'; \ - strncpy_P(__buf, (PGM_P) (s), 63); \ - __buf; \ - }) -#endif #define LOG_STR(s) (reinterpret_cast(PSTR(s))) #define LOG_STR_LITERAL(s) LOG_STR_ARG(LOG_STR(s)) diff --git a/tests/unit_tests/components/esp8266/test_boards.py b/tests/unit_tests/components/esp8266/test_boards.py index df0e536d42..78213a762a 100644 --- a/tests/unit_tests/components/esp8266/test_boards.py +++ b/tests/unit_tests/components/esp8266/test_boards.py @@ -1,11 +1,7 @@ """Tests for the per-board linker-script rule.""" -import pytest - from esphome.components.esp8266 import _choose_ld_script from esphome.components.esp8266.boards import BOARDS, board_ld_script -import esphome.config_validation as cv -from esphome.core import EsphomeError def test_d1_wroom_02_keeps_its_shipped_layout() -> None: @@ -21,13 +17,6 @@ def test_default_boards_use_the_flash_size_layout() -> None: def test_choose_ld_script_paths() -> None: - """Old cores get the size default, overriding boards hard-error there - (a substituted layout would wipe flash-backed state), modern cores - honor the override.""" - assert _choose_ld_script("nodemcuv2", cv.Version(2, 3, 0)) is None - assert _choose_ld_script("nodemcuv2", cv.Version(2, 4, 2)) == "eagle.flash.4m.ld" - assert _choose_ld_script("d1_wroom_02", cv.Version(2, 7, 4)) == ( - "eagle.flash.2m64.ld" - ) - with pytest.raises(EsphomeError, match="cannot honor"): - _choose_ld_script("d1_wroom_02", cv.Version(2, 4, 2)) + """Default boards get the size layout, overriding boards keep theirs.""" + assert _choose_ld_script("nodemcuv2") == "eagle.flash.4m.ld" + assert _choose_ld_script("d1_wroom_02") == "eagle.flash.2m64.ld" diff --git a/tests/unit_tests/components/esp8266/test_framework_version.py b/tests/unit_tests/components/esp8266/test_framework_version.py new file mode 100644 index 0000000000..0107aff8dd --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_framework_version.py @@ -0,0 +1,23 @@ +"""Tests for the Arduino framework version floor.""" + +import pytest + +from esphome.components.esp8266 import _arduino_check_versions +import esphome.config_validation as cv +from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION + + +def test_versions_before_3_are_rejected() -> None: + with pytest.raises(cv.Invalid, match="no longer supported") as excinfo: + _arduino_check_versions({CONF_VERSION: "2.7.4"}) + assert excinfo.value.path == [CONF_VERSION] + + +def test_supported_versions_pass() -> None: + value = _arduino_check_versions({CONF_VERSION: "3.0.2"}) + assert value[CONF_VERSION] == "3.0.2" + assert "espressif8266@3.2.0" in value[CONF_PLATFORM_VERSION] + + value = _arduino_check_versions({CONF_VERSION: "recommended"}) + assert value[CONF_VERSION] == "3.1.2" + assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION] diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index bd0a620e10..9f415344ae 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -21,17 +21,12 @@ def _build_path(tmp_path: Path) -> None: def test_framework_package_version() -> None: assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" - # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path) - assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0" # A future major bump needs its own encoding, not a doomed registry lookup with pytest.raises(EsphomeError, match="not supported yet"): framework.framework_package_version(cv.Version(4, 0, 0)) - # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release - # keeps this encoding - with pytest.raises(EsphomeError, match="older package encoding"): - framework.framework_package_version(cv.Version(2, 6, 2)) - assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0" - assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0" + # Cores before 3.x cannot build ESPHome (C++20) and are rejected + with pytest.raises(EsphomeError, match="requires core 3"): + framework.framework_package_version(cv.Version(2, 7, 4)) def test_format_framework_arduino_version_pins_all_series() -> None: @@ -39,10 +34,10 @@ def test_format_framework_arduino_version_pins_all_series() -> None: era, including the 4.x rejection it now shares with the installer.""" from esphome.components.esp8266 import _format_framework_arduino_version as fmt - assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0" - assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0" - assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0" assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" + # Pre-3 cores are rejected with the version line anchored + with pytest.raises(cv.Invalid, match="requires core 3"): + fmt(cv.Version(2, 7, 4)) # Anchored to the framework version line, not a bare EsphomeError with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: fmt(cv.Version(4, 0, 0)) From ab800dc09dbe2d1eb1ed1ee8f091282561e0ccc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:19:29 -0400 Subject: [PATCH 012/178] Bump filelock from 3.32.4 to 3.32.5 (#18963) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8731d38b7f..8a510a2c60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.5 # native esp-idf toolchain global cache dir ninja==1.13.2 # native esp8266 arduino toolchain build driver -filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==3.32.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.2 From 51ea97deffbac5c2d1379ce20424c74d6b2259b5 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:38:55 +1200 Subject: [PATCH 013/178] [esp32_ble] Reference count BLE advertising (#18943) --- esphome/components/esp32_ble/ble.cpp | 35 +++++++++++++++---- esphome/components/esp32_ble/ble.h | 13 +++++++ .../esp32_ble_beacon/esp32_ble_beacon.cpp | 2 ++ .../components/esp32_ble_server/__init__.py | 12 +++++++ .../esp32_ble_server/ble_server.cpp | 21 +++++++++-- .../components/esp32_ble_server/ble_server.h | 11 ++++++ .../esp32_improv/esp32_improv_component.cpp | 20 ++++++++++- .../esp32_improv/esp32_improv_component.h | 3 ++ .../esp32_ble_server/config/improv_only.yaml | 13 +++++++ .../config/manufacturer_data_only.yaml | 9 +++++ .../esp32_ble_server/config/own_service.yaml | 14 ++++++++ .../esp32_ble_server/test_esp32_ble_server.py | 28 +++++++++++++++ 12 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/esp32_ble_server/config/improv_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/own_service.yaml diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 6e6fb0e30d..fc95760cf8 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -100,21 +100,38 @@ void ESP32BLE::disable() { #ifdef USE_ESP32_BLE_ADVERTISING void ESP32BLE::advertising_start() { this->advertising_init_(); - if (!this->is_active()) + this->advertising_ref_count_++; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_stop() { + if (this->advertising_ref_count_ == 0) return; - this->advertising_->start(); + this->advertising_ref_count_--; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_refresh() { + if (this->advertising_ == nullptr || !this->is_active()) + return; + // Advertise while any component still needs it, otherwise stop + if (this->advertising_ref_count_ == 0) { + this->advertising_->stop(); + } else { + this->advertising_->start(); + } } void ESP32BLE::advertising_set_service_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_service_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_manufacturer_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_manufacturer_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_service_data_and_name(std::span data, bool include_name) { @@ -136,7 +153,7 @@ void ESP32BLE::advertising_set_service_data_and_name(std::span da this->advertising_->set_service_data(data); } - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_register_raw_advertisement_callback(std::function &&callback) { @@ -147,13 +164,13 @@ void ESP32BLE::advertising_register_raw_advertisement_callback(std::functionadvertising_init_(); this->advertising_->add_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_remove_service_uuid(ESPBTUUID uuid) { this->advertising_init_(); this->advertising_->remove_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } #endif @@ -575,6 +592,10 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { } this->state_ = BLE_COMPONENT_STATE_ACTIVE; +#ifdef USE_ESP32_BLE_ADVERTISING + // Requests made before the stack was up (or before it was re-enabled) take effect now + this->advertising_refresh(); +#endif } } diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 2a355a6c8b..7d2d0438a4 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -114,7 +114,17 @@ class ESP32BLE final : public Component { void set_name(const char *name) { this->name_ = name; } #ifdef USE_ESP32_BLE_ADVERTISING + /** Request advertising on behalf of a component. + * + * Requests are reference counted: advertising runs until every component that called + * advertising_start() has released it again with advertising_stop(). Each component must + * pair its calls, so nothing advertises until something actually asks for it. + */ void advertising_start(); + /// Release a request made with advertising_start(); advertising stops at the last release. + void advertising_stop(); + /// Apply the current payload and request count: advertise while requested, otherwise stop. + void advertising_refresh(); void advertising_set_service_data(const std::vector &data); void advertising_set_manufacturer_data(const std::vector &data); void advertising_set_appearance(uint16_t appearance) { this->appearance_ = appearance; } @@ -226,6 +236,9 @@ class ESP32BLE final : public Component { // 1-byte aligned members (grouped together to minimize padding) BLEComponentState state_{BLE_COMPONENT_STATE_OFF}; // 1 byte (uint8_t enum) bool enable_on_boot_{}; // 1 byte +#ifdef USE_ESP32_BLE_ADVERTISING + uint8_t advertising_ref_count_{0}; // 1 byte, number of components requesting advertising +#endif #ifdef ESPHOME_ESP32_BLE_EXTENDED_AUTH_PARAMS optional auth_req_mode_; diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp index 9f1723430b..ab728f9f6f 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp @@ -67,6 +67,8 @@ void ESP32BLEBeacon::setup() { this->on_advertise_(); } }); + // A beacon always needs the device to advertise, and never releases the request + global_ble->advertising_start(); } void ESP32BLEBeacon::on_advertise_() { diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 855a3be29b..d8095cd702 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -596,6 +596,18 @@ async def to_code(config): cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) cg.add(var.set_max_clients(config[CONF_MAX_CLIENTS])) + # Only advertise for the server itself when the configuration gives clients something to + # find. A server that is auto-loaded purely to host a runtime service (esp32_improv) stays + # silent until that service asks for advertising. + cg.add( + var.set_advertising_required( + CONF_MANUFACTURER_DATA in config + or any( + not uuid_is(service_config[CONF_UUID], DEVICE_INFORMATION_SERVICE_UUID) + for service_config in config[CONF_SERVICES] + ) + ) + ) if CONF_MANUFACTURER_DATA in config: cg.add(var.set_manufacturer_data(config[CONF_MANUFACTURER_DATA])) for service_config in config[CONF_SERVICES]: diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp index 2dea1666bb..45679b9b98 100644 --- a/esphome/components/esp32_ble_server/ble_server.cpp +++ b/esphome/components/esp32_ble_server/ble_server.cpp @@ -81,6 +81,7 @@ void BLEServer::loop() { if (this->device_information_service_->is_running()) { this->state_ = RUNNING; this->restart_advertising_(); + this->request_advertising_(); ESP_LOGD(TAG, "BLE server setup successfully"); } else if (this->device_information_service_->is_created()) { this->device_information_service_->start(); @@ -98,6 +99,20 @@ void BLEServer::restart_advertising_() { } } +void BLEServer::request_advertising_() { + if (!this->advertising_required_ || this->advertising_requested_) + return; + this->advertising_requested_ = true; + this->parent_->advertising_start(); +} + +void BLEServer::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + this->parent_->advertising_stop(); +} + BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles) { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE char uuid_buf[esp32_ble::UUID_STR_LEN]; @@ -170,7 +185,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga this->add_client_(param->connect.conn_id); // Resume advertising so additional clients can discover and connect if (this->client_count_ < this->max_clients_) { - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); } this->dispatch_callbacks_(CallbackType::ON_CONNECT, param->connect.conn_id); break; @@ -178,7 +193,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga case ESP_GATTS_DISCONNECT_EVT: { ESP_LOGD(TAG, "BLE Client disconnected"); this->remove_client_(param->disconnect.conn_id); - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); this->dispatch_callbacks_(CallbackType::ON_DISCONNECT, param->disconnect.conn_id); break; } @@ -226,6 +241,8 @@ void BLEServer::remove_client_(uint16_t conn_id) { } void BLEServer::ble_before_disabled_event_handler() { + // Advertising is re-requested once the server is running again after BLE is re-enabled + this->release_advertising_(); // Delete all clients this->client_count_ = 0; // Delete all services diff --git a/esphome/components/esp32_ble_server/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h index fdd92812cd..7869c73cc5 100644 --- a/esphome/components/esp32_ble_server/ble_server.h +++ b/esphome/components/esp32_ble_server/ble_server.h @@ -38,6 +38,13 @@ class BLEServer final : public Component, public Parented { this->restart_advertising_(); } + /** Whether this server needs the device to advertise so clients can find and connect to it. + * + * False for a server that only hosts services created at runtime (e.g. esp32_improv), which + * request advertising themselves for as long as they need it. + */ + void set_advertising_required(bool required) { this->advertising_required_ = required; } + void set_max_clients(uint8_t max_clients) { this->max_clients_ = max_clients; } uint8_t get_max_clients() const { return this->max_clients_; } @@ -82,6 +89,8 @@ class BLEServer final : public Component, public Parented { }; void restart_advertising_(); + void request_advertising_(); + void release_advertising_(); int8_t find_client_index_(uint16_t conn_id) const; void add_client_(uint16_t conn_id); @@ -93,6 +102,8 @@ class BLEServer final : public Component, public Parented { std::vector manufacturer_data_{}; esp_gatt_if_t gatts_if_{0}; bool registered_{false}; + bool advertising_required_{true}; + bool advertising_requested_{false}; uint16_t clients_[USE_ESP32_BLE_MAX_CONNECTIONS]{}; uint8_t client_count_{0}; diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 4756fba637..9ec6eb7bab 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -112,6 +112,7 @@ void ESP32ImprovComponent::loop() { this->state_callback_.call(this->state_, this->error_state_); #endif } + this->release_advertising_(); this->incoming_data_.clear(); return; } @@ -143,8 +144,9 @@ void ESP32ImprovComponent::loop() { ESP_LOGV(TAG, "Starting with device name advertising"); this->advertising_device_name_ = true; this->last_name_adv_time_ = App.get_loop_component_start_time(); + // Set the payload before requesting, so advertising starts exactly once esp32_ble::global_ble->advertising_set_service_data_and_name(std::span{}, true); - esp32_ble::global_ble->advertising_start(); + this->request_advertising_(); // Set initial state based on whether we have an authorizer this->set_state_(this->get_initial_state_(), false); @@ -326,6 +328,8 @@ void ESP32ImprovComponent::stop() { this->set_timeout("end-service", STOP_ADVERTISING_DELAY, [this] { if (this->state_ == improv::STATE_STOPPED || this->service_ == nullptr) return; + // Release first so removing the service UUID does not restart advertising on the way out + this->release_advertising_(); this->service_->stop(); this->set_state_(improv::STATE_STOPPED); }); @@ -520,6 +524,20 @@ void ESP32ImprovComponent::update_advertising_type_() { } } +void ESP32ImprovComponent::request_advertising_() { + if (this->advertising_requested_) + return; + this->advertising_requested_ = true; + esp32_ble::global_ble->advertising_start(); +} + +void ESP32ImprovComponent::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + esp32_ble::global_ble->advertising_stop(); +} + improv::State ESP32ImprovComponent::get_initial_state_() const { #ifdef USE_BINARY_SENSOR // If we have an authorizer, start in awaiting authorization state diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index 414948c977..a40d60552a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -104,8 +104,11 @@ class ESP32ImprovComponent final : public Component, public improv_base::ImprovB bool status_indicator_state_{false}; uint32_t last_name_adv_time_{0}; bool advertising_device_name_{false}; + bool advertising_requested_{false}; void set_status_indicator_state_(bool state); void update_advertising_type_(); + void request_advertising_(); + void release_advertising_(); void set_state_(improv::State state, bool update_advertising = true); void set_error_(improv::Error error); diff --git a/tests/component_tests/esp32_ble_server/config/improv_only.yaml b/tests/component_tests/esp32_ble_server/config/improv_only.yaml new file mode 100644 index 0000000000..8a5c3ba638 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/improv_only.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32 + +wifi: + ssid: MySSID + password: password1 + +# esp32_ble_server is only auto-loaded here, so it has no services of its own. +esp32_improv: + authorizer: none diff --git a/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml new file mode 100644 index 0000000000..b7bdae4af7 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + manufacturer_data: [0x72, 0x04, 0x00, 0x23] diff --git a/tests/component_tests/esp32_ble_server/config/own_service.yaml b/tests/component_tests/esp32_ble_server/config/own_service.yaml new file mode 100644 index 0000000000..c7ef0287b0 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/own_service.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + services: + - uuid: 2a24b789-7aab-4535-af3e-ee76a35cc12d + characteristics: + - uuid: cad48e28-7fbe-41cf-bae9-d77a6c233423 + read: true + value: [1, 2, 3, 4] diff --git a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py index 88307d0dcf..4b7ab79a81 100644 --- a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py +++ b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py @@ -1,5 +1,10 @@ """Tests for esp32_ble_server configuration helpers.""" +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + import pytest from esphome.components.esp32_ble_server import ( @@ -45,3 +50,26 @@ def test_uuid_is_matches_descriptor_short_strings(uuid16) -> None: assert uuid_is(uuid16, uuid16) assert uuid_is(f"{uuid16:04X}", uuid16) assert uuid_is(f"{uuid16:08X}", uuid16) + + +@pytest.mark.parametrize( + ("config_file", "required"), + [ + # Auto-loaded by esp32_improv only: nothing to find until Improv asks for it + ("improv_only.yaml", False), + # The configuration defines a service clients are meant to connect to + ("own_service.yaml", True), + # Manufacturer data is only useful if it is actually broadcast + ("manufacturer_data_only.yaml", True), + ], +) +def test_advertising_required( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + required: bool, +) -> None: + """The server only requests advertising when the configuration needs it.""" + main_cpp = generate_main(component_config_path(config_file)) + + assert f"set_advertising_required({str(required).lower()})" in main_cpp From ce87bf9b17f5f93171e62a1f6ecbc1ade6ce1132 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:05:26 -0400 Subject: [PATCH 014/178] Bump platformdirs from 4.11.5 to 4.11.7 (#18976) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a510a2c60..cd3f7446f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.5 # native esp-idf toolchain global cache dir +platformdirs==4.11.7 # native esp-idf toolchain global cache dir ninja==1.13.2 # native esp8266 arduino toolchain build driver filelock==3.32.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From d1829c495d2c982eb2f2845406ccfe5b74bd2f64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:05:36 -0400 Subject: [PATCH 015/178] Bump prek from 0.5.0 to 0.5.1 (#18977) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index b1309ec63b..897445a4cb 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.8 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.5.0 # also change in .github/workflows/ci.yml when updating +prek==0.5.1 # also change in .github/workflows/ci.yml when updating # Unit tests pytest==9.1.1 From b66822d9bd0f741b8d40264e019264d9910fc377 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:21:47 +1000 Subject: [PATCH 016/178] [ai] Advice to agents to limit verbiage (#18980) --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index e932c50f32..15b92c4deb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -553,6 +553,7 @@ file does, and it is the authority when they disagree. The most useful starting 4. **Lint:** Run `prek` to ensure code is compliant. 5. **Commit:** Commit your changes. There is no strict format for commit messages. 6. **Pull Request:** Submit a PR against the `dev` branch. The Pull Request title must start with a `[tag]` prefix. For component work, use the component name (e.g., `[display] Fix bug`, `[abc123] Add new component`); for changes to shared/core code that isn't tied to a single component, use `[core]` (e.g., `[core] Add validator`). Update documentation, examples, and add `CODEOWNERS` entries as needed. Pull requests should always be made using the `.github/PULL_REQUEST_TEMPLATE.md` template - fill out all sections completely without removing any parts of the template. + 7. **Comments:** When commenting on GitHub PRs or issues, don't tag contributors, especially bots. Avoid referring to list items (e.g. from reviews) with the form #nn - this will be interpreted by GitHub as a reference to issue or PR nn. Keep comments short and exclude irrelevant details, backstories, restatement of previous comments and anything that is already obvious to the reader. * **Documentation Contributions:** * Documentation is hosted in the separate `esphome/esphome.io` repository. From 13dbbcaa32e94423ff5bf9fe62b6f56cb073a6c5 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sat, 5 Sep 2026 06:00:25 -0500 Subject: [PATCH 017/178] [usb_uart] Keep the comm interface number valid when its claim fails (#18968) --- esphome/components/usb_uart/usb_uart.cpp | 12 +++++++----- esphome/components/usb_uart/usb_uart.h | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/esphome/components/usb_uart/usb_uart.cpp b/esphome/components/usb_uart/usb_uart.cpp index cf66e4c369..60b7fe4e9c 100644 --- a/esphome/components/usb_uart/usb_uart.cpp +++ b/esphome/components/usb_uart/usb_uart.cpp @@ -434,11 +434,12 @@ void USBUartTypeCdcAcm::on_connected() { auto err_comm = usb_host_interface_claim(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number, 0); if (err_comm != ESP_OK) { + // Continue anyway: the interface number stays valid for CDC request addressing ESP_LOGW(TAG, "Could not claim comm interface %d: %s", channel->cdc_dev_.interrupt_interface_number, esp_err_to_name(err_comm)); - channel->cdc_dev_.interrupt_interface_number = 0xFF; // Mark as unavailable, but continue anyway } else { ESP_LOGD(TAG, "Claimed comm interface %d", channel->cdc_dev_.interrupt_interface_number); + channel->cdc_dev_.interrupt_interface_claimed = true; } } auto err = @@ -465,14 +466,15 @@ void USBUartTypeCdcAcm::on_disconnected() { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); } - if (channel->cdc_dev_.notify_ep != nullptr) { + // Only tear down the notify pipe when we claimed its interface ourselves; + // no transfer is ever submitted on it, so there is nothing else to cancel. + if (channel->cdc_dev_.notify_ep != nullptr && channel->cdc_dev_.interrupt_interface_claimed) { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); } - if (channel->cdc_dev_.interrupt_interface_number != 0xFF && - channel->cdc_dev_.interrupt_interface_number != channel->cdc_dev_.bulk_interface_number) { + if (channel->cdc_dev_.interrupt_interface_claimed) { usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number); - channel->cdc_dev_.interrupt_interface_number = 0xFF; + channel->cdc_dev_.interrupt_interface_claimed = false; } usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.bulk_interface_number); // Reset the input and output started flags to their initial state to avoid the possibility of spurious restarts diff --git a/esphome/components/usb_uart/usb_uart.h b/esphome/components/usb_uart/usb_uart.h index 00b34fb942..9d87bf964c 100644 --- a/esphome/components/usb_uart/usb_uart.h +++ b/esphome/components/usb_uart/usb_uart.h @@ -34,7 +34,10 @@ struct CdcEps { const usb_ep_desc_t *in_ep; const usb_ep_desc_t *out_ep; uint8_t bulk_interface_number; + // Also the wIndex target for CDC class requests (SET_LINE_CODING etc.), so it + // must remain valid even when the interface itself is not claimed. uint8_t interrupt_interface_number; + bool interrupt_interface_claimed{false}; }; enum CH34xChipType : uint8_t { From 84f78831f95442f124c2f652b644611b15143fbe Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:07:15 +0200 Subject: [PATCH 018/178] Bump bundled esphome-device-builder to 1.14.1 (#18981) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7952616496..2d4ddbef5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 RUN \ platformio settings set enable_telemetry No \ From ae187f81f25fcce1869a8f128c3a929ec07c7f89 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:44:37 +1000 Subject: [PATCH 019/178] [wifi] Allow a forced roam check (#17349) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Claude --- esphome/components/wifi/__init__.py | 16 ++++++++- esphome/components/wifi/automation.h | 5 +++ esphome/components/wifi/wifi_component.cpp | 38 +++++++++++++++------- esphome/components/wifi/wifi_component.h | 6 ++++ tests/components/wifi/common.yaml | 1 + 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index b8c6d774ac..1691dcc293 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -66,13 +66,14 @@ from esphome.const import ( ) from esphome.core import ( CORE, + ID, CoroPriority, EsphomeError, HexInt, coroutine_with_priority, ) import esphome.final_validate as fv -from esphome.types import ConfigType +from esphome.types import ConfigType, TemplateArgsType from . import wpa2_eap @@ -208,6 +209,7 @@ WiFiEnabledCondition = wifi_ns.class_("WiFiEnabledCondition", Condition) WiFiAPActiveCondition = wifi_ns.class_("WiFiAPActiveCondition", Condition) WiFiEnableAction = wifi_ns.class_("WiFiEnableAction", automation.Action) WiFiDisableAction = wifi_ns.class_("WiFiDisableAction", automation.Action) +WiFiRoamAction = wifi_ns.class_("WiFiRoamAction", automation.Action) WiFiConfigureAction = wifi_ns.class_( "WiFiConfigureAction", automation.Action, cg.Component ) @@ -820,6 +822,18 @@ async def wifi_disable_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg) +@automation.register_action( + "wifi.roam", WiFiRoamAction, cv.Schema({}), synchronous=True +) +async def wifi_roam_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> cg.MockObj: + return cg.new_Pvariable(action_id, template_arg) + + KEEP_SCAN_RESULTS_KEY = "wifi_keep_scan_results" RUNTIME_POWER_SAVE_KEY = "wifi_runtime_power_save" RUNTIME_ROAMING_SUPPRESSION_KEY = "wifi_runtime_roaming_suppression" diff --git a/esphome/components/wifi/automation.h b/esphome/components/wifi/automation.h index e63faa18ab..c14341330f 100644 --- a/esphome/components/wifi/automation.h +++ b/esphome/components/wifi/automation.h @@ -31,6 +31,11 @@ template class WiFiDisableAction final : public Action { void play(const Ts &...x) override { global_wifi_component->disable(); } }; +template class WiFiRoamAction final : public Action { + public: + void play(const Ts &...x) override { global_wifi_component->force_roam_check(); } +}; + template class WiFiConfigureAction final : public Action, public Component { public: TEMPLATABLE_VALUE(std::string, ssid) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 694e616476..f9e80995e1 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -846,17 +846,18 @@ void WiFiComponent::loop() { this->notify_connect_state_listeners_(); #endif - // Post-connect roaming: check for better AP - if (this->post_connect_roaming_) { - if (this->is_roaming_scan_active()) { - if (this->scan_done_) { - this->process_roaming_scan_(); - } - // else: scan in progress, wait - } else if (this->roaming_state_ == RoamingState::IDLE && this->roaming_attempts_ < ROAMING_MAX_ATTEMPTS && - now - this->roaming_last_check_ >= ROAMING_CHECK_INTERVAL && !this->roaming_suppressed_()) { - this->check_roaming_(now); + // Post-connect roaming: check for better AP. A scan may have been started by an + // explicit force_roam_check() even when post_connect_roaming_ is disabled, so the + // scan must always be consumed here to avoid leaving roaming_state_ stuck. + if (this->is_roaming_scan_active()) { + if (this->scan_done_) { + this->process_roaming_scan_(); } + // else: scan in progress, wait + } else if (this->post_connect_roaming_ && this->roaming_state_ == RoamingState::IDLE && + this->roaming_attempts_ < ROAMING_MAX_ATTEMPTS && + now - this->roaming_last_check_ >= ROAMING_CHECK_INTERVAL && !this->roaming_suppressed_()) { + this->check_roaming_(now); } } break; @@ -2463,6 +2464,17 @@ void WiFiComponent::notify_scan_results_listeners_() { } #endif // USE_WIFI_SCAN_RESULTS_LISTENERS +void WiFiComponent::force_roam_check() { + if (!this->is_connected() || this->roaming_state_ != RoamingState::IDLE || this->roaming_suppressed_()) { + ESP_LOGD(TAG, "Roam check requested, but not able to check now"); + return; + } + // Reset the attempt counter so a prior run of failed roams doesn't block this explicit request + // Note that this re-arms automatic roaming if enabled. + this->roaming_attempts_ = 0; + this->check_roaming_(millis()); +} + void WiFiComponent::check_roaming_(uint32_t now) { // Guard: not for hidden networks (may not appear in scan) const WiFiAP *selected = this->get_selected_sta_(); @@ -2484,7 +2496,11 @@ void WiFiComponent::check_roaming_(uint32_t now) { ESP_LOGD(TAG, "Roam scan (%d dBm, attempt %u/%u)", rssi, this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); this->roaming_state_ = RoamingState::SCANNING; - this->wifi_scan_start_(this->passive_scan_); + if (!this->wifi_scan_start_(this->passive_scan_)) { + // Scan failed to start (e.g. busy) - don't get stuck in SCANNING forever + ESP_LOGD(TAG, "Roam scan failed to start"); + this->roaming_state_ = RoamingState::IDLE; + } } void WiFiComponent::process_roaming_scan_() { diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 63df9fbfa5..94fdd9bc14 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -565,6 +565,12 @@ class WiFiComponent final : public Component { void set_keep_scan_results(bool keep_scan_results) { this->keep_scan_results_ = keep_scan_results; } void set_post_connect_roaming(bool enabled) { this->post_connect_roaming_ = enabled; } + /** Force an immediate post-connect roaming check, bypassing the periodic interval and the + * per-connection attempt limit. Does nothing (besides a debug log) if not connected, if a + * roam scan or connect is already in progress, or if roaming is currently suppressed. + */ + void force_roam_check(); + #ifdef USE_WIFI_CONNECT_TRIGGER Trigger<> *get_connect_trigger() { return &this->connect_trigger_; } #endif diff --git a/tests/components/wifi/common.yaml b/tests/components/wifi/common.yaml index 10b68347eb..10a8a61c66 100644 --- a/tests/components/wifi/common.yaml +++ b/tests/components/wifi/common.yaml @@ -14,6 +14,7 @@ esphome: condition: wifi.ap_active then: - logger.log: "WiFi AP is active!" + - wifi.roam wifi: networks: From 3ef7460fca9e5326b5d51e0e3bf51c1bcb8abde6 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:25:58 +0000 Subject: [PATCH 020/178] Bump bundled esphome-device-builder to 1.14.2 (#18988) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d4ddbef5d..b5170864a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 RUN \ platformio settings set enable_telemetry No \ From e3dd2f44a45bc7200566393db51fa17eb0a5edf1 Mon Sep 17 00:00:00 2001 From: elwin loomis Date: Sat, 5 Sep 2026 16:08:21 -0500 Subject: [PATCH 021/178] [mipi_dsi] Let IDF pick the DPHY PLL reference clock (#18984) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> --- esphome/components/mipi_dsi/mipi_dsi.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index 0850b50c85..0150cc2544 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -35,8 +35,8 @@ void MipiDsi::setup() { .bus_id = 0, // index from 0, specify the DSI host to use .num_data_lanes = this->lanes_, // Number of data lanes to use, can't set a value that exceeds the chip's capability - .phy_clk_src = MIPI_DSI_PHY_CLK_SRC_DEFAULT, // Clock source for the DPHY - .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps + // phy_clk_src left at 0 to enable runtime auto-select. + .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps }; auto err = esp_lcd_new_dsi_bus(&bus_config, &this->bus_handle_); if (err != ESP_OK) { From e5200db6fd6008da8a1e4b88d8b99e463aae0759 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:28:02 +0200 Subject: [PATCH 022/178] Bump bundled esphome-device-builder to 1.14.3 (#18996) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5170864a3..e875851bfb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 RUN \ platformio settings set enable_telemetry No \ From 8e1044e8ea35aa959121170d7a8000fcbf90aed2 Mon Sep 17 00:00:00 2001 From: Ricardo Sanz Date: Sun, 6 Sep 2026 23:03:07 +0200 Subject: [PATCH 023/178] [climate][template] New template climate component (#14455) --- esphome/components/climate/__init__.py | 13 + .../components/template/climate/__init__.py | 465 ++++++++++++++++++ .../components/template/climate/automation.h | 57 +++ .../template/climate/template_climate.cpp | 164 ++++++ .../template/climate/template_climate.h | 92 ++++ esphome/config_validation.py | 1 + .../template/test_template_climate.py | 145 ++++++ tests/components/climate/common.yaml | 3 +- tests/components/template/common-base.yaml | 113 +++++ .../fixtures/template_climate_basic.yaml | 72 +++ .../template_climate_custom_modes.yaml | 47 ++ .../template_climate_nonoptimistic.yaml | 56 +++ .../template_climate_on_control_ordering.yaml | 26 + .../template_climate_publish_all_fields.yaml | 63 +++ .../template_climate_sensor_push.yaml | 49 ++ .../template_climate_set_actions.yaml | 89 ++++ ...emplate_climate_two_point_temperature.yaml | 52 ++ .../test_template_climate_basic.py | 146 ++++++ .../test_template_climate_custom_modes.py | 98 ++++ .../test_template_climate_nonoptimistic.py | 107 ++++ ...st_template_climate_on_control_ordering.py | 83 ++++ ...est_template_climate_publish_all_fields.py | 96 ++++ .../test_template_climate_sensor_push.py | 88 ++++ .../test_template_climate_set_actions.py | 114 +++++ ..._template_climate_two_point_temperature.py | 118 +++++ 25 files changed, 2355 insertions(+), 2 deletions(-) create mode 100644 esphome/components/template/climate/__init__.py create mode 100644 esphome/components/template/climate/automation.h create mode 100644 esphome/components/template/climate/template_climate.cpp create mode 100644 esphome/components/template/climate/template_climate.h create mode 100644 tests/component_tests/template/test_template_climate.py create mode 100644 tests/integration/fixtures/template_climate_basic.yaml create mode 100644 tests/integration/fixtures/template_climate_custom_modes.yaml create mode 100644 tests/integration/fixtures/template_climate_nonoptimistic.yaml create mode 100644 tests/integration/fixtures/template_climate_on_control_ordering.yaml create mode 100644 tests/integration/fixtures/template_climate_publish_all_fields.yaml create mode 100644 tests/integration/fixtures/template_climate_sensor_push.yaml create mode 100644 tests/integration/fixtures/template_climate_set_actions.yaml create mode 100644 tests/integration/fixtures/template_climate_two_point_temperature.yaml create mode 100644 tests/integration/test_template_climate_basic.py create mode 100644 tests/integration/test_template_climate_custom_modes.py create mode 100644 tests/integration/test_template_climate_nonoptimistic.py create mode 100644 tests/integration/test_template_climate_on_control_ordering.py create mode 100644 tests/integration/test_template_climate_publish_all_fields.py create mode 100644 tests/integration/test_template_climate_sensor_push.py create mode 100644 tests/integration/test_template_climate_set_actions.py create mode 100644 tests/integration/test_template_climate_two_point_temperature.py diff --git a/esphome/components/climate/__init__.py b/esphome/components/climate/__init__.py index 80dd913fba..3fbca1a6d0 100644 --- a/esphome/components/climate/__init__.py +++ b/esphome/components/climate/__init__.py @@ -125,6 +125,19 @@ CLIMATE_SWING_MODES = { validate_climate_swing_mode = cv.enum(CLIMATE_SWING_MODES, upper=True) +ClimateAction = climate_ns.enum("ClimateAction") +CLIMATE_ACTIONS = { + "OFF": ClimateAction.CLIMATE_ACTION_OFF, + "COOLING": ClimateAction.CLIMATE_ACTION_COOLING, + "HEATING": ClimateAction.CLIMATE_ACTION_HEATING, + "IDLE": ClimateAction.CLIMATE_ACTION_IDLE, + "DRYING": ClimateAction.CLIMATE_ACTION_DRYING, + "FAN": ClimateAction.CLIMATE_ACTION_FAN, + "DEFROSTING": ClimateAction.CLIMATE_ACTION_DEFROSTING, +} + +validate_climate_action = cv.enum(CLIMATE_ACTIONS, upper=True) + CONF_MIN_HUMIDITY = "min_humidity" CONF_MAX_HUMIDITY = "max_humidity" CONF_TARGET_HUMIDITY = "target_humidity" diff --git a/esphome/components/template/climate/__init__.py b/esphome/components/template/climate/__init__.py new file mode 100644 index 0000000000..c39ea8f80e --- /dev/null +++ b/esphome/components/template/climate/__init__.py @@ -0,0 +1,465 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import climate, sensor +from esphome.components.climate import climate_ns +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTION, + CONF_CURRENT_TEMPERATURE, + CONF_CUSTOM_FAN_MODE, + CONF_CUSTOM_FAN_MODES, + CONF_CUSTOM_PRESET, + CONF_CUSTOM_PRESETS, + CONF_FAN_MODE, + CONF_HUMIDITY_SENSOR, + CONF_ID, + CONF_INITIAL_STATE, + CONF_MODE, + CONF_OPTIMISTIC, + CONF_PRESET, + CONF_RESTORE_MODE, + CONF_SENSOR, + CONF_SUPPORTED_FAN_MODES, + CONF_SUPPORTED_MODES, + CONF_SUPPORTED_PRESETS, + CONF_SUPPORTED_SWING_MODES, + CONF_SWING_MODE, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.types import ConfigType + +from .. import template_ns + +CONF_CURRENT_HUMIDITY = "current_humidity" +CONF_TARGET_HUMIDITY = "target_humidity" +CONF_SUPPORTS_ACTION = "supports_action" +CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE = "supports_two_point_target_temperature" +CONF_SUPPORTS_TARGET_HUMIDITY = "supports_target_humidity" +CONF_SUPPORTS_CURRENT_TEMPERATURE = "supports_current_temperature" +CONF_SUPPORTS_CURRENT_HUMIDITY = "supports_current_humidity" +CONF_SET_MODE_ACTION = "set_mode_action" +CONF_SET_TARGET_TEMPERATURE_ACTION = "set_target_temperature_action" +CONF_SET_TARGET_TEMPERATURE_LOW_ACTION = "set_target_temperature_low_action" +CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION = "set_target_temperature_high_action" +CONF_SET_TARGET_HUMIDITY_ACTION = "set_target_humidity_action" +CONF_SET_FAN_MODE_ACTION = "set_fan_mode_action" +CONF_SET_CUSTOM_FAN_MODE_ACTION = "set_custom_fan_mode_action" +CONF_SET_SWING_MODE_ACTION = "set_swing_mode_action" +CONF_SET_PRESET_ACTION = "set_preset_action" +CONF_SET_CUSTOM_PRESET_ACTION = "set_custom_preset_action" + +TemplateClimate = template_ns.class_("TemplateClimate", climate.Climate, cg.Component) +TemplateClimatePublishAction = template_ns.class_( + "TemplateClimatePublishAction", + automation.Action, + cg.Parented.template(TemplateClimate), +) + +TemplateClimateRestoreMode = template_ns.enum( + "TemplateClimateRestoreMode", is_class=True +) +CLIMATE_RESTORE_MODES = { + "NO_RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + "RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +} + +# Per-field actions that forward a requested value on. The third item is the type of `x`. +SET_ACTIONS = ( + (CONF_SET_MODE_ACTION, "get_set_mode_trigger", climate.ClimateMode), + ( + CONF_SET_TARGET_TEMPERATURE_ACTION, + "get_set_target_temperature_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + "get_set_target_temperature_low_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + "get_set_target_temperature_high_trigger", + cg.float_, + ), + (CONF_SET_TARGET_HUMIDITY_ACTION, "get_set_target_humidity_trigger", cg.float_), + (CONF_SET_FAN_MODE_ACTION, "get_set_fan_mode_trigger", climate.ClimateFanMode), + ( + CONF_SET_CUSTOM_FAN_MODE_ACTION, + "get_set_custom_fan_mode_trigger", + cg.StringRef, + ), + ( + CONF_SET_SWING_MODE_ACTION, + "get_set_swing_mode_trigger", + climate.ClimateSwingMode, + ), + (CONF_SET_PRESET_ACTION, "get_set_preset_trigger", climate.ClimatePreset), + (CONF_SET_CUSTOM_PRESET_ACTION, "get_set_custom_preset_trigger", cg.StringRef), +) + +# supports_* keys have no default so that an omitted key can mean "derive it from the sensor or +# set action that makes the trait useful", which is not expressible once a default fills it in. +DERIVED_SUPPORTS = ( + (CONF_SUPPORTS_CURRENT_TEMPERATURE, (CONF_SENSOR,)), + (CONF_SUPPORTS_CURRENT_HUMIDITY, (CONF_HUMIDITY_SENSOR,)), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + ), + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, (CONF_SET_TARGET_HUMIDITY_ACTION,)), +) + + +# Custom fan modes/presets are opaque user-defined strings with no build-time correctness check +# elsewhere (Climate::set_supported_custom_fan_modes()/set_supported_custom_presets() don't block +# empty entries), so reject empty ones here -- they could never be selected at runtime anyway. +validate_custom_climate_string = cv.All(cv.string_strict, cv.Length(min=1)) + + +def _validate_two_point(config: ConfigType) -> ConfigType: + has_low = CONF_TARGET_TEMPERATURE_LOW in config + has_high = CONF_TARGET_TEMPERATURE_HIGH in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE_LOW}' and '{CONF_TARGET_TEMPERATURE_HIGH}' must be used together" + ) + if (has_low or has_high) and CONF_TARGET_TEMPERATURE in config: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' cannot be used together with " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}'" + ) + return config + + +def _validate_set_actions(config: ConfigType) -> ConfigType: + has_low = CONF_SET_TARGET_TEMPERATURE_LOW_ACTION in config + has_high = CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}' and " + f"'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}' must be used together" + ) + if (has_low or has_high) and CONF_SET_TARGET_TEMPERATURE_ACTION in config: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_ACTION}' cannot be used together with " + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}'/'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}'" + ) + return config + + +def _resolve_supports(config: ConfigType) -> ConfigType: + # An explicit true stays valid without either, since climate.template.publish can report the + # value; an explicit false that contradicts the configuration is an error, not a silent override. + for key, sources in DERIVED_SUPPORTS: + configured = [source for source in sources if source in config] + if key not in config: + config[key] = bool(configured) + elif not config[key] and configured: + raise cv.Invalid( + f"'{key}' cannot be false while '{configured[0]}' is configured", + path=[key], + ) + return config + + +def _validate_initial_state(config: ConfigType) -> ConfigType: + # Climate keeps target_temperature and target_temperature_low in a union, so writing the wrong + # one of the pair corrupts the setpoint with no runtime complaint. + if (initial_state := config.get(CONF_INITIAL_STATE)) is None: + return config + + two_point = config[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] + if two_point and CONF_TARGET_TEMPERATURE in initial_state: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' is not available while " + f"'{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' is enabled; use " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}' instead", + path=[CONF_INITIAL_STATE, CONF_TARGET_TEMPERATURE], + ) + if not two_point: + for key in (CONF_TARGET_TEMPERATURE_LOW, CONF_TARGET_TEMPERATURE_HIGH): + if key in initial_state: + raise cv.Invalid( + f"'{key}' requires '{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' to be enabled", + path=[CONF_INITIAL_STATE, key], + ) + if ( + CONF_TARGET_HUMIDITY in initial_state + and not config[CONF_SUPPORTS_TARGET_HUMIDITY] + ): + raise cv.Invalid( + f"'{CONF_TARGET_HUMIDITY}' requires '{CONF_SUPPORTS_TARGET_HUMIDITY}' to be enabled", + path=[CONF_INITIAL_STATE, CONF_TARGET_HUMIDITY], + ) + return config + + +# Same settable fields as climate.template.publish, minus current_temperature/current_humidity/ +# action: those are reported values (from a sensor or the device), not meaningful static defaults. +INITIAL_STATE_SCHEMA = cv.All( + cv.Schema( + { + cv.Optional(CONF_MODE): climate.validate_climate_mode, + cv.Optional(CONF_TARGET_TEMPERATURE): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.temperature, + cv.Optional(CONF_TARGET_HUMIDITY): cv.percentage_int, + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): climate.validate_climate_fan_mode, + cv.Exclusive( + CONF_CUSTOM_FAN_MODE, "fan_mode" + ): validate_custom_climate_string, + cv.Optional(CONF_SWING_MODE): climate.validate_climate_swing_mode, + cv.Exclusive(CONF_PRESET, "preset"): climate.validate_climate_preset, + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): validate_custom_climate_string, + } + ), + _validate_two_point, +) + +CONFIG_SCHEMA = cv.All( + climate.climate_schema(TemplateClimate) + .extend( + { + cv.Optional(CONF_SENSOR): cv.use_id(sensor.Sensor), + cv.Optional(CONF_HUMIDITY_SENSOR): cv.use_id(sensor.Sensor), + # action only ever arrives through climate.template.publish, so unlike the other + # supports_* keys there is no set action to derive it from. + cv.Optional(CONF_SUPPORTS_ACTION, default=False): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_HUMIDITY): cv.boolean, + cv.Optional(CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_TARGET_HUMIDITY): cv.boolean, + cv.Required(CONF_SUPPORTED_MODES): cv.All( + cv.ensure_list(climate.validate_climate_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_FAN_MODES): cv.All( + cv.ensure_list(climate.validate_climate_fan_mode), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_FAN_MODES): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_SWING_MODES): cv.All( + cv.ensure_list(climate.validate_climate_swing_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_PRESETS): cv.All( + cv.ensure_list(climate.validate_climate_preset), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_PRESETS): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_OPTIMISTIC, default=True): cv.boolean, + cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum( + CLIMATE_RESTORE_MODES, upper=True + ), + cv.Optional(CONF_INITIAL_STATE): INITIAL_STATE_SCHEMA, + cv.Optional(CONF_SET_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_HUMIDITY_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_FAN_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_CUSTOM_FAN_MODE_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_SWING_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_PRESET_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_CUSTOM_PRESET_ACTION): automation.validate_automation( + single=True + ), + } + ) + .extend(cv.COMPONENT_SCHEMA), + _validate_set_actions, + _resolve_supports, + _validate_initial_state, +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await climate.register_climate(var, config) + + if (sens := config.get(CONF_SENSOR)) is not None: + cg.add(var.set_sensor(await cg.get_variable(sens))) + + if (sens := config.get(CONF_HUMIDITY_SENSOR)) is not None: + cg.add(var.set_humidity_sensor(await cg.get_variable(sens))) + + for key, flag in ( + (CONF_SUPPORTS_ACTION, climate_ns.CLIMATE_SUPPORTS_ACTION), + ( + CONF_SUPPORTS_CURRENT_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_CURRENT_TEMPERATURE, + ), + (CONF_SUPPORTS_CURRENT_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_CURRENT_HUMIDITY), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_TARGET_HUMIDITY), + ): + if config[key]: + cg.add(var.add_feature_flags(flag)) + + for mode in config[CONF_SUPPORTED_MODES]: + cg.add(var.add_supported_mode(mode)) + + for mode in config.get(CONF_SUPPORTED_FAN_MODES, []): + cg.add(var.add_supported_fan_mode(mode)) + + if CONF_CUSTOM_FAN_MODES in config: + cg.add( + var.set_supported_custom_fan_modes( + cg.ArrayInitializer(*config[CONF_CUSTOM_FAN_MODES]) + ) + ) + + for mode in config.get(CONF_SUPPORTED_SWING_MODES, []): + cg.add(var.add_supported_swing_mode(mode)) + + for preset in config.get(CONF_SUPPORTED_PRESETS, []): + cg.add(var.add_supported_preset(preset)) + + if CONF_CUSTOM_PRESETS in config: + cg.add( + var.set_supported_custom_presets( + cg.ArrayInitializer(*config[CONF_CUSTOM_PRESETS]) + ) + ) + + for key, trigger_getter, arg_type in SET_ACTIONS: + if (conf := config.get(key)) is not None: + await automation.build_automation( + getattr(var, trigger_getter)(), [(arg_type, "x")], conf + ) + + cg.add(var.set_optimistic(config[CONF_OPTIMISTIC])) + cg.add(var.set_restore_mode(config[CONF_RESTORE_MODE])) + + if (initial_state := config.get(CONF_INITIAL_STATE)) is not None: + if (v := initial_state.get(CONF_MODE)) is not None: + cg.add(var.set_mode(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add(var.set_target_temperature_high(v)) + if (v := initial_state.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(v)) + if (v := initial_state.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(v)) + if (v := initial_state.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(v)) + if (v := initial_state.get(CONF_SWING_MODE)) is not None: + cg.add(var.set_swing_mode(v)) + if (v := initial_state.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(v)) + if (v := initial_state.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(v)) + + +CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.use_id(TemplateClimate), + cv.Optional(CONF_CURRENT_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_CURRENT_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_TARGET_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_MODE): cv.templatable(climate.validate_climate_mode), + cv.Optional(CONF_ACTION): cv.templatable(climate.validate_climate_action), + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): cv.templatable( + climate.validate_climate_fan_mode + ), + cv.Exclusive(CONF_CUSTOM_FAN_MODE, "fan_mode"): cv.templatable( + validate_custom_climate_string + ), + cv.Optional(CONF_SWING_MODE): cv.templatable( + climate.validate_climate_swing_mode + ), + cv.Exclusive(CONF_PRESET, "preset"): cv.templatable( + climate.validate_climate_preset + ), + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): cv.templatable( + validate_custom_climate_string + ), + } + ), + _validate_two_point, +) + + +@automation.register_action( + "climate.template.publish", + TemplateClimatePublishAction, + CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA, + synchronous=True, +) +async def climate_template_publish_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + + if (v := config.get(CONF_CURRENT_TEMPERATURE)) is not None: + cg.add(var.set_current_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_CURRENT_HUMIDITY)) is not None: + cg.add(var.set_current_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add( + var.set_target_temperature_high(await cg.templatable(v, args, cg.float_)) + ) + if (v := config.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_MODE)) is not None: + cg.add(var.set_mode(await cg.templatable(v, args, climate.ClimateMode))) + if (v := config.get(CONF_ACTION)) is not None: + cg.add(var.set_action(await cg.templatable(v, args, climate.ClimateAction))) + if (v := config.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(await cg.templatable(v, args, climate.ClimateFanMode))) + if (v := config.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(await cg.templatable(v, args, cg.std_string))) + if (v := config.get(CONF_SWING_MODE)) is not None: + cg.add( + var.set_swing_mode(await cg.templatable(v, args, climate.ClimateSwingMode)) + ) + if (v := config.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(await cg.templatable(v, args, climate.ClimatePreset))) + if (v := config.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(await cg.templatable(v, args, cg.std_string))) + + return var diff --git a/esphome/components/template/climate/automation.h b/esphome/components/template/climate/automation.h new file mode 100644 index 0000000000..49a79ace2f --- /dev/null +++ b/esphome/components/template/climate/automation.h @@ -0,0 +1,57 @@ +#pragma once + +#include "template_climate.h" +#include "esphome/core/automation.h" + +namespace esphome::template_ { + +template +class TemplateClimatePublishAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(float, current_temperature) + TEMPLATABLE_VALUE(float, current_humidity) + TEMPLATABLE_VALUE(float, target_temperature) + TEMPLATABLE_VALUE(float, target_temperature_low) + TEMPLATABLE_VALUE(float, target_temperature_high) + TEMPLATABLE_VALUE(float, target_humidity) + TEMPLATABLE_VALUE(climate::ClimateMode, mode) + TEMPLATABLE_VALUE(climate::ClimateAction, action) + TEMPLATABLE_VALUE(climate::ClimateFanMode, fan_mode) + TEMPLATABLE_VALUE(std::string, custom_fan_mode) + TEMPLATABLE_VALUE(climate::ClimateSwingMode, swing_mode) + TEMPLATABLE_VALUE(climate::ClimatePreset, preset) + TEMPLATABLE_VALUE(std::string, custom_preset) + + void play(const Ts &...x) override { + if (this->current_temperature_.has_value()) + this->parent_->current_temperature = this->current_temperature_.value(x...); + if (this->current_humidity_.has_value()) + this->parent_->current_humidity = this->current_humidity_.value(x...); + if (this->target_temperature_.has_value()) + this->parent_->set_target_temperature(this->target_temperature_.value(x...)); + if (this->target_temperature_low_.has_value()) + this->parent_->set_target_temperature_low(this->target_temperature_low_.value(x...)); + if (this->target_temperature_high_.has_value()) + this->parent_->set_target_temperature_high(this->target_temperature_high_.value(x...)); + if (this->target_humidity_.has_value()) + this->parent_->set_target_humidity(this->target_humidity_.value(x...)); + if (this->mode_.has_value()) + this->parent_->set_mode(this->mode_.value(x...)); + if (this->action_.has_value()) + this->parent_->action = this->action_.value(x...); + if (this->fan_mode_.has_value()) + this->parent_->set_fan_mode(this->fan_mode_.value(x...)); + if (this->custom_fan_mode_.has_value()) + this->parent_->set_custom_fan_mode(StringRef(this->custom_fan_mode_.value(x...))); + if (this->swing_mode_.has_value()) + this->parent_->set_swing_mode(this->swing_mode_.value(x...)); + if (this->preset_.has_value()) + this->parent_->set_preset(this->preset_.value(x...)); + if (this->custom_preset_.has_value()) + this->parent_->set_custom_preset(StringRef(this->custom_preset_.value(x...))); + + this->parent_->publish_state(); + } +}; + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.cpp b/esphome/components/template/climate/template_climate.cpp new file mode 100644 index 0000000000..a7a4d2ccab --- /dev/null +++ b/esphome/components/template/climate/template_climate.cpp @@ -0,0 +1,164 @@ +#include "template_climate.h" +#include "esphome/core/log.h" + +namespace esphome::template_ { + +static const char *const TAG = "template.climate"; + +void TemplateClimate::setup() { + if (this->restore_mode_ == TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE) { + auto restore = this->restore_state_(); + if (restore.has_value()) { + restore->apply(this); + } + } + + // Sensors publish every reading, not just changes, so only re-publish when the value moved. + // NAN means the sensor went unavailable and is passed through rather than dropped; the second + // check stops an unavailable sensor re-publishing forever, since NAN never equals NAN. +#ifdef USE_SENSOR + if (this->sensor_ != nullptr) { + this->current_temperature = this->sensor_->state; + this->sensor_->add_on_state_callback([this](float state) { + if (state != this->current_temperature && !(std::isnan(state) && std::isnan(this->current_temperature))) { + this->current_temperature = state; + this->publish_state(); + } + }); + } + + if (this->humidity_sensor_ != nullptr) { + this->current_humidity = this->humidity_sensor_->state; + this->humidity_sensor_->add_on_state_callback([this](float state) { + if (state != this->current_humidity && !(std::isnan(state) && std::isnan(this->current_humidity))) { + this->current_humidity = state; + this->publish_state(); + } + }); + } +#endif +} + +void TemplateClimate::dump_config() { + LOG_CLIMATE("", "Template Climate", this); + ESP_LOGCONFIG(TAG, " Optimistic: %s", YESNO(this->optimistic_)); +} + +void TemplateClimate::control(const climate::ClimateCall &call) { + // Each field present fires its set_*_action; on_control sees the whole call. optimistic: true + // also applies the values right away, false waits for a climate.template.publish report. + if (auto mode = call.get_mode()) { + if (this->optimistic_) + this->mode = *mode; + this->set_mode_trigger_.trigger(*mode); + } + + if (auto target_temp = call.get_target_temperature()) { + if (this->optimistic_) + this->target_temperature = *target_temp; + this->set_target_temperature_trigger_.trigger(*target_temp); + } + + if (auto target_temp_low = call.get_target_temperature_low()) { + if (this->optimistic_) + this->target_temperature_low = *target_temp_low; + this->set_target_temperature_low_trigger_.trigger(*target_temp_low); + } + + if (auto target_temp_high = call.get_target_temperature_high()) { + if (this->optimistic_) + this->target_temperature_high = *target_temp_high; + this->set_target_temperature_high_trigger_.trigger(*target_temp_high); + } + + if (auto target_humidity = call.get_target_humidity()) { + if (this->optimistic_) + this->target_humidity = *target_humidity; + this->set_target_humidity_trigger_.trigger(*target_humidity); + } + + if (auto fan_mode = call.get_fan_mode()) { + if (this->optimistic_) + this->set_fan_mode_(*fan_mode); + this->set_fan_mode_trigger_.trigger(*fan_mode); + } + + if (call.has_custom_fan_mode()) { + if (this->optimistic_) + this->set_custom_fan_mode_(call.get_custom_fan_mode()); + this->set_custom_fan_mode_trigger_.trigger(call.get_custom_fan_mode()); + } + + if (auto swing_mode = call.get_swing_mode()) { + if (this->optimistic_) + this->swing_mode = *swing_mode; + this->set_swing_mode_trigger_.trigger(*swing_mode); + } + + if (auto preset = call.get_preset()) { + if (this->optimistic_) + this->set_preset_(*preset); + this->set_preset_trigger_.trigger(*preset); + } + + if (call.has_custom_preset()) { + if (this->optimistic_) + this->set_custom_preset_(call.get_custom_preset()); + this->set_custom_preset_trigger_.trigger(call.get_custom_preset()); + } + + if (this->optimistic_) + this->publish_state(); +} + +// A climate.template.publish report (and initial_state:) never goes through ClimateCall::validate_(), +// so check here instead -- otherwise a typo is published as state the receiving end will reject. +void TemplateClimate::set_mode(climate::ClimateMode mode) { + if (!this->traits_.supports_mode(mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported mode %u", this->get_name().c_str(), static_cast(mode)); + return; + } + this->mode = mode; +} + +void TemplateClimate::set_swing_mode(climate::ClimateSwingMode swing_mode) { + if (!this->traits_.supports_swing_mode(swing_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported swing mode %u", this->get_name().c_str(), static_cast(swing_mode)); + return; + } + this->swing_mode = swing_mode; +} + +void TemplateClimate::set_fan_mode(climate::ClimateFanMode fan_mode) { + if (!this->traits_.supports_fan_mode(fan_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported fan mode %u", this->get_name().c_str(), static_cast(fan_mode)); + return; + } + this->set_fan_mode_(fan_mode); +} + +void TemplateClimate::set_preset(climate::ClimatePreset preset) { + if (!this->traits_.supports_preset(preset)) { + ESP_LOGW(TAG, "'%s' - Unsupported preset %u", this->get_name().c_str(), static_cast(preset)); + return; + } + this->set_preset_(preset); +} + +void TemplateClimate::set_custom_fan_mode(StringRef mode) { + if (this->find_custom_fan_mode_(mode.c_str(), mode.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom fan mode '%s'", this->get_name().c_str(), mode.c_str()); + return; + } + this->set_custom_fan_mode_(mode); +} + +void TemplateClimate::set_custom_preset(StringRef preset) { + if (this->find_custom_preset_(preset.c_str(), preset.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom preset '%s'", this->get_name().c_str(), preset.c_str()); + return; + } + this->set_custom_preset_(preset); +} + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.h b/esphome/components/template/climate/template_climate.h new file mode 100644 index 0000000000..5448488c34 --- /dev/null +++ b/esphome/components/template/climate/template_climate.h @@ -0,0 +1,92 @@ +#pragma once + +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/components/climate/climate.h" +#ifdef USE_SENSOR +#include "esphome/components/sensor/sensor.h" +#endif + +namespace esphome::template_ { + +enum class TemplateClimateRestoreMode { + TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +}; + +class TemplateClimate final : public climate::Climate, public Component { + public: + void setup() override; + void dump_config() override; + + climate::ClimateTraits traits() override { return this->traits_; } + + void add_feature_flags(uint32_t flags) { this->traits_.add_feature_flags(flags); } + +#ifdef USE_SENSOR + // The matching feature flag is added from codegen, so the configuration alone decides it. + void set_sensor(sensor::Sensor *sensor) { this->sensor_ = sensor; } + void set_humidity_sensor(sensor::Sensor *sensor) { this->humidity_sensor_ = sensor; } +#endif + + void add_supported_mode(climate::ClimateMode mode) { this->traits_.add_supported_mode(mode); } + void add_supported_fan_mode(climate::ClimateFanMode mode) { this->traits_.add_supported_fan_mode(mode); } + void add_supported_swing_mode(climate::ClimateSwingMode mode) { this->traits_.add_supported_swing_mode(mode); } + void add_supported_preset(climate::ClimatePreset preset) { this->traits_.add_supported_preset(preset); } + + void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } + void set_restore_mode(TemplateClimateRestoreMode restore_mode) { this->restore_mode_ = restore_mode; } + + // Fired from control() for each field the call carries, so a device-backed config can forward + // it on. Which of these are configured also decides the two-point/target-humidity traits. + Trigger *get_set_mode_trigger() { return &this->set_mode_trigger_; } + Trigger *get_set_target_temperature_trigger() { return &this->set_target_temperature_trigger_; } + Trigger *get_set_target_temperature_low_trigger() { return &this->set_target_temperature_low_trigger_; } + Trigger *get_set_target_temperature_high_trigger() { return &this->set_target_temperature_high_trigger_; } + Trigger *get_set_target_humidity_trigger() { return &this->set_target_humidity_trigger_; } + Trigger *get_set_fan_mode_trigger() { return &this->set_fan_mode_trigger_; } + Trigger *get_set_custom_fan_mode_trigger() { return &this->set_custom_fan_mode_trigger_; } + Trigger *get_set_swing_mode_trigger() { return &this->set_swing_mode_trigger_; } + Trigger *get_set_preset_trigger() { return &this->set_preset_trigger_; } + Trigger *get_set_custom_preset_trigger() { return &this->set_custom_preset_trigger_; } + + // Used by TemplateClimatePublishAction, which is not a Climate subclass and so cannot reach the + // protected setters, and by codegen to apply `initial_state:` before setup() runs. + void set_target_temperature(float value) { this->target_temperature = value; } + void set_target_temperature_low(float value) { this->target_temperature_low = value; } + void set_target_temperature_high(float value) { this->target_temperature_high = value; } + void set_target_humidity(float value) { this->target_humidity = value; } + void set_mode(climate::ClimateMode mode); + void set_swing_mode(climate::ClimateSwingMode mode); + void set_fan_mode(climate::ClimateFanMode mode); + void set_custom_fan_mode(const char *mode) { this->set_custom_fan_mode(StringRef(mode)); } + void set_custom_fan_mode(StringRef mode); + void set_preset(climate::ClimatePreset preset); + void set_custom_preset(const char *preset) { this->set_custom_preset(StringRef(preset)); } + void set_custom_preset(StringRef preset); + + protected: + void control(const climate::ClimateCall &call) override; + + climate::ClimateTraits traits_; + bool optimistic_{false}; + TemplateClimateRestoreMode restore_mode_{TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE}; + +#ifdef USE_SENSOR + sensor::Sensor *sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; +#endif + + Trigger set_mode_trigger_; + Trigger set_target_temperature_trigger_; + Trigger set_target_temperature_low_trigger_; + Trigger set_target_temperature_high_trigger_; + Trigger set_target_humidity_trigger_; + Trigger set_fan_mode_trigger_; + Trigger set_custom_fan_mode_trigger_; + Trigger set_swing_mode_trigger_; + Trigger set_preset_trigger_; + Trigger set_custom_preset_trigger_; +}; + +} // namespace esphome::template_ diff --git a/esphome/config_validation.py b/esphome/config_validation.py index aff39201e8..685a9d04b3 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -133,6 +133,7 @@ Upper = vol.Upper Length = vol.Length Exclusive = vol.Exclusive Inclusive = vol.Inclusive +Unique = vol.Unique ALLOW_EXTRA = vol.ALLOW_EXTRA UNDEFINED = vol.UNDEFINED RequiredFieldInvalid = vol.RequiredFieldInvalid diff --git a/tests/component_tests/template/test_template_climate.py b/tests/component_tests/template/test_template_climate.py new file mode 100644 index 0000000000..304991ea64 --- /dev/null +++ b/tests/component_tests/template/test_template_climate.py @@ -0,0 +1,145 @@ +"""Tests for template climate config validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.template.climate import ( + CONF_SET_TARGET_HUMIDITY_ACTION, + CONF_SET_TARGET_TEMPERATURE_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SUPPORTS_CURRENT_HUMIDITY, + CONF_SUPPORTS_CURRENT_TEMPERATURE, + CONF_SUPPORTS_TARGET_HUMIDITY, + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + CONF_TARGET_HUMIDITY, + _resolve_supports, + _validate_initial_state, + _validate_set_actions, +) +from esphome.const import ( + CONF_HUMIDITY_SENSOR, + CONF_INITIAL_STATE, + CONF_SENSOR, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.types import ConfigType + + +def test_supports_current_temperature_derived_from_sensor() -> None: + config: ConfigType = {CONF_SENSOR: "some_sensor"} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_without_sensor() -> None: + assert _resolve_supports({})[CONF_SUPPORTS_CURRENT_TEMPERATURE] is False + + +def test_supports_current_temperature_explicit_true_without_sensor_allowed() -> None: + # The value can still be reported with climate.template.publish. + config: ConfigType = {CONF_SUPPORTS_CURRENT_TEMPERATURE: True} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_supports_current_humidity_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_HUMIDITY_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_HUMIDITY: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_two_point_derived_from_set_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + assert _resolve_supports(config)[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] is True + + +def test_two_point_false_with_set_action_rejected() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_target_humidity_derived_from_set_action() -> None: + config: ConfigType = {CONF_SET_TARGET_HUMIDITY_ACTION: [{}]} + assert _resolve_supports(config)[CONF_SUPPORTS_TARGET_HUMIDITY] is True + + +def test_set_target_temperature_low_requires_high() -> None: + config: ConfigType = {CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}]} + with pytest.raises(cv.Invalid, match="must be used together"): + _validate_set_actions(config) + + +def test_set_target_temperature_conflicts_with_two_point_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + with pytest.raises(cv.Invalid, match="cannot be used together"): + _validate_set_actions(config) + + +def test_initial_state_target_temperature_rejected_with_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_TEMPERATURE: 21.0}, + } + with pytest.raises(cv.Invalid, match="is not available"): + _validate_initial_state(config) + + +def test_initial_state_two_point_values_rejected_without_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + }, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_target_humidity_rejected_without_support() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_HUMIDITY: 50}, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_matching_two_point_accepted() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: True, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + CONF_TARGET_HUMIDITY: 50, + }, + } + assert _validate_initial_state(config) is config diff --git a/tests/components/climate/common.yaml b/tests/components/climate/common.yaml index c28fde8eeb..49386a16d5 100644 --- a/tests/components/climate/common.yaml +++ b/tests/components/climate/common.yaml @@ -30,8 +30,7 @@ climate: - switch.turn_on: climate_heater_switch - switch.turn_off: climate_cooler_switch # Thermostat-based climate so climate.control: action variants get build - # coverage (bang_bang doesn't support fan modes, presets, etc.). Climate - # has no template platform, so thermostat is the right vehicle. + # coverage (bang_bang doesn't support fan modes, presets, etc.). - platform: thermostat id: climate_test_thermostat name: Test Thermostat diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index 92a1fc8eda..02aedaf167 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -25,6 +25,27 @@ esphome: away: !lambda "return true;" is_on: !lambda "return false;" + - climate.template.publish: + id: template_climate + current_temperature: 21.0 + mode: HEAT + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE + target_temperature: 22.0 + + # Templated + - climate.template.publish: + id: template_climate + current_temperature: !lambda "return 21.5f;" + mode: !lambda "return climate::CLIMATE_MODE_COOL;" + target_temperature: !lambda "return 23.0f;" + + - climate.template.publish: + id: template_climate_custom_modes + custom_fan_mode: "turbo" + custom_preset: "eco_plus" + # Test C++ API: set_template() with stateless lambda (no captures) # NOTE: set_template() is not intended to be a public API, but we test it to ensure it doesn't break. - lambda: |- @@ -513,6 +534,98 @@ alarm_control_panel: codes: - "1234" +climate: + - platform: template + id: template_climate + name: "Template Climate" + optimistic: true + sensor: template_template_sens + supports_action: true + supports_current_humidity: true + restore_mode: NO_RESTORE + initial_state: + mode: HEAT + target_temperature: 21.0 + fan_mode: LOW + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_action: + - logger.log: + format: "set_target_temperature_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.1f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + on_control: + - logger.log: "on_control fired" + on_state: + - logger.log: "on_state fired" + + - platform: template + id: template_climate_custom_modes + name: "Template Climate Custom Modes" + optimistic: true + sensor: template_template_sens + supported_modes: + - "OFF" + - HEAT + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + initial_state: + custom_fan_mode: eco + custom_preset: max + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + water_heater: - platform: template id: template_water_heater diff --git a/tests/integration/fixtures/template_climate_basic.yaml b/tests/integration/fixtures/template_climate_basic.yaml new file mode 100644 index 0000000000..51558b4875 --- /dev/null +++ b/tests/integration/fixtures/template_climate_basic.yaml @@ -0,0 +1,72 @@ +esphome: + name: tmpl-clim-basic + on_boot: + - climate.template.publish: + id: test_climate + action: IDLE +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Basic Climate + optimistic: true + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 55.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: "OFF" + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE diff --git a/tests/integration/fixtures/template_climate_custom_modes.yaml b/tests/integration/fixtures/template_climate_custom_modes.yaml new file mode 100644 index 0000000000..9dbfe60cb9 --- /dev/null +++ b/tests/integration/fixtures/template_climate_custom_modes.yaml @@ -0,0 +1,47 @@ +esphome: + name: tmpl-clim-custom +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Custom Mode Climate + optimistic: true + sensor: test_climate_current_temperature + supported_modes: + - "OFF" + - HEAT + - COOL + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + on_control: + - lambda: |- + if (x.has_custom_fan_mode()) + ESP_LOGD("test", "on_control custom_fan_mode=%s", x.get_custom_fan_mode().c_str()); + if (x.has_custom_preset()) + ESP_LOGD("test", "on_control custom_preset=%s", x.get_custom_preset().c_str()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + custom_fan_mode: "eco" + custom_preset: "max" diff --git a/tests/integration/fixtures/template_climate_nonoptimistic.yaml b/tests/integration/fixtures/template_climate_nonoptimistic.yaml new file mode 100644 index 0000000000..2b0c7ee132 --- /dev/null +++ b/tests/integration/fixtures/template_climate_nonoptimistic.yaml @@ -0,0 +1,56 @@ +esphome: + name: tmpl-clim-nonopt +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Template Climate Nonoptimistic + optimistic: false + supported_modes: + - "OFF" + - HEAT + - COOL + - FAN_ONLY + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + - AWAY + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +button: + - platform: template + id: simulate_device_confirmation + name: Simulate Device Confirmation + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + target_temperature: 22.5 + fan_mode: HIGH + swing_mode: VERTICAL + preset: AWAY diff --git a/tests/integration/fixtures/template_climate_on_control_ordering.yaml b/tests/integration/fixtures/template_climate_on_control_ordering.yaml new file mode 100644 index 0000000000..8366a6d21e --- /dev/null +++ b/tests/integration/fixtures/template_climate_on_control_ordering.yaml @@ -0,0 +1,26 @@ +esphome: + name: tmpl-clim-oc-order +host: +api: +logger: + +# on_control fires with the full ClimateCall (arg `x`) from the base Climate component's +# ClimateCall::perform(), before validate_()/control() run -- so when the lambda action below +# runs, the entity's own .mode is still the OLD value, even though x.get_mode() already reports +# the NEW requested value. on_state fires afterward, once control() has applied it. +climate: + - platform: template + id: test_climate + name: Test On Control Ordering + optimistic: true + supported_modes: + - "OFF" + - HEAT + on_control: + - lambda: |- + ESP_LOGD("test", "on_control requested_mode=%d current_mode_before_apply=%d", + x.get_mode().has_value() ? (int) *x.get_mode() : -1, + (int) id(test_climate).mode); + on_state: + - lambda: |- + ESP_LOGD("test", "on_state mode=%d", (int) x.mode); diff --git a/tests/integration/fixtures/template_climate_publish_all_fields.yaml b/tests/integration/fixtures/template_climate_publish_all_fields.yaml new file mode 100644 index 0000000000..e57fcc4508 --- /dev/null +++ b/tests/integration/fixtures/template_climate_publish_all_fields.yaml @@ -0,0 +1,63 @@ +esphome: + name: tmpl-clim-publish-all +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Publish All Fields + optimistic: true + # current_temperature/current_humidity/action are only sent over the API at all if their + # trait is advertised: current_temperature/current_humidity because a sensor/humidity_sensor + # is referenced below, action because supports_action is set. The sensors' fixed readings + # match what climate.template.publish pushes, so the sensor callback (guarded to only publish + # on an actual change) doesn't produce an extra, unexpected state update of its own. + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + supported_fan_modes: + - AUTO + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + on_control: + # Should never fire in this test: climate.template.publish is a pure bypass and must not + # re-trigger on_control as if the entity were freshly commanded. + - logger.log: "on_control fired" + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 20.0f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 60.0f;" + update_interval: 10ms + +button: + - platform: template + id: publish_all + name: Publish All + on_press: + - climate.template.publish: + id: test_climate + current_temperature: 20.0 + current_humidity: 60.0 + target_temperature: 23.0 + mode: HEAT + action: HEATING + fan_mode: HIGH + swing_mode: VERTICAL + preset: ECO diff --git a/tests/integration/fixtures/template_climate_sensor_push.yaml b/tests/integration/fixtures/template_climate_sensor_push.yaml new file mode 100644 index 0000000000..1fc004335d --- /dev/null +++ b/tests/integration/fixtures/template_climate_sensor_push.yaml @@ -0,0 +1,49 @@ +esphome: + name: tmpl-clim-sensor-push +host: +api: +logger: + +# No lambda/update_interval: these sensors only ever report a value when a button below +# publishes one (standing in for e.g. a BLE scan callback in a real config). +sensor: + - platform: template + id: room_temperature + name: Room Temperature + - platform: template + id: room_humidity + name: Room Humidity + +climate: + - platform: template + id: test_climate + name: Test Sensor Push Climate + optimistic: true + sensor: room_temperature + humidity_sensor: room_humidity + supported_modes: + - "OFF" + - HEAT + +button: + - platform: template + id: publish_temperature + name: Publish Temperature + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_temperature_same + name: Publish Temperature Same Value + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_humidity + name: Publish Humidity + on_press: + - sensor.template.publish: + id: room_humidity + state: 65.0 diff --git a/tests/integration/fixtures/template_climate_set_actions.yaml b/tests/integration/fixtures/template_climate_set_actions.yaml new file mode 100644 index 0000000000..b247367f64 --- /dev/null +++ b/tests/integration/fixtures/template_climate_set_actions.yaml @@ -0,0 +1,89 @@ +esphome: + name: tmpl-clim-set-act +host: +api: +logger: + +# Every settable field forwards its requested value to a set_*_action. supports_two_point and +# supports_target_humidity are not declared here: they are derived from the low/high and humidity +# set actions being present. +climate: + - platform: template + id: test_climate + name: Test Set Actions + optimistic: false + restore_mode: NO_RESTORE + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + custom_fan_modes: + - turbo + custom_presets: + - eco_plus + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_low_action: + - logger.log: + format: "set_target_temperature_low_action %.1f" + args: ["x"] + set_target_temperature_high_action: + - logger.log: + format: "set_target_temperature_high_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.0f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + +button: + - platform: template + id: report_device_state + name: Report Device State + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + + - platform: template + id: report_unsupported_mode + name: Report Unsupported Mode + on_press: + - climate.template.publish: + id: test_climate + mode: DRY diff --git a/tests/integration/fixtures/template_climate_two_point_temperature.yaml b/tests/integration/fixtures/template_climate_two_point_temperature.yaml new file mode 100644 index 0000000000..ec10785ee8 --- /dev/null +++ b/tests/integration/fixtures/template_climate_two_point_temperature.yaml @@ -0,0 +1,52 @@ +esphome: + name: tmpl-clim-two-point +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Two-Point Heatpump + optimistic: true + sensor: test_climate_current_temperature + supports_two_point_target_temperature: true + supports_target_humidity: true + supported_modes: + - "OFF" + - HEAT_COOL + - HEAT + - COOL + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature_low().has_value()) + ESP_LOGD("test", "on_control target_temperature_low=%.1f", *x.get_target_temperature_low()); + if (x.get_target_temperature_high().has_value()) + ESP_LOGD("test", "on_control target_temperature_high=%.1f", *x.get_target_temperature_high()); + if (x.get_target_humidity().has_value()) + ESP_LOGD("test", "on_control target_humidity=%.1f", *x.get_target_humidity()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 21.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT_COOL + target_temperature_low: 18.0 + target_temperature_high: 24.0 + target_humidity: 50.0 diff --git a/tests/integration/test_template_climate_basic.py b/tests/integration/test_template_climate_basic.py new file mode 100644 index 0000000000..431fd4e3e8 --- /dev/null +++ b/tests/integration/test_template_climate_basic.py @@ -0,0 +1,146 @@ +"""Integration test for template climate: sensor-pushed measured values, on_control + publish +for the settable ones. + +current_temperature/current_humidity are pushed by a referenced sensor/humidity_sensor (no +polling); action is set once at boot via climate.template.publish, since it has no sensor +equivalent. mode/target_temperature/fan_mode/swing_mode/preset are plain internal state: +on_control fires exactly once per command (never before the first one), and +climate.template.publish simulates the device reporting its own state independent of any prior +command -- that report is authoritative, overriding whatever was optimistically applied earlier. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-basic" + + +@pytest.mark.asyncio +async def test_template_climate_basic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Sensor-pushed measured values, on_control + publish for settable ones.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + # Advertised capabilities come straight from the supported_*/custom_* config lists. + assert ClimateMode.OFF in test_climate.supported_modes + assert ClimateMode.HEAT in test_climate.supported_modes + assert ClimateMode.COOL in test_climate.supported_modes + + assert ClimateFanMode.AUTO in test_climate.supported_fan_modes + assert ClimateFanMode.LOW in test_climate.supported_fan_modes + assert ClimateFanMode.HIGH in test_climate.supported_fan_modes + + assert ClimateSwingMode.OFF in test_climate.supported_swing_modes + assert ClimateSwingMode.VERTICAL in test_climate.supported_swing_modes + + assert ClimatePreset.NONE in test_climate.supported_presets + assert ClimatePreset.ECO in test_climate.supported_presets + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.current_temperature == pytest.approx(22.5, abs=0.1) + assert initial.current_humidity == pytest.approx(55.0, abs=0.1) + assert initial.action == ClimateAction.IDLE + assert initial.mode == ClimateMode.OFF + # Nothing was commanded yet: on_control must not have fired. + assert not log_lines + + # Commands apply optimistically and on_control fires with the same values. + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + client.climate_command(test_climate.key, target_temperature=22.5) + state = await wait_for_climate_state() + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.HIGH) + state = await wait_for_climate_state() + assert state.fan_mode == ClimateFanMode.HIGH + + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + state = await wait_for_climate_state() + assert state.swing_mode == ClimateSwingMode.VERTICAL + + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + state = await wait_for_climate_state() + assert state.preset == ClimatePreset.ECO + + await asyncio.sleep(0.2) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + # Exactly one on_control log line per command, none extra (e.g. from a stray republish). + assert len(log_lines) == 5 + + # measured values are untouched by any of the above (no set action exists for them). + assert state.current_temperature == pytest.approx(22.5, abs=0.1) + assert state.current_humidity == pytest.approx(55.0, abs=0.1) + assert state.action == ClimateAction.IDLE + + # The device's report is authoritative and overrides everything commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.OFF + assert state.fan_mode == ClimateFanMode.AUTO + assert state.swing_mode == ClimateSwingMode.OFF + assert state.preset == ClimatePreset.NONE diff --git a/tests/integration/test_template_climate_custom_modes.py b/tests/integration/test_template_climate_custom_modes.py new file mode 100644 index 0000000000..4817fe1ddf --- /dev/null +++ b/tests/integration/test_template_climate_custom_modes.py @@ -0,0 +1,98 @@ +"""Integration test for template climate: custom fan modes and presets. + +Same on_control (forward) + climate.template.publish (device report, authoritative) pattern as +the enum-based mode/preset fields, but for the custom string variants. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-custom" + + +@pytest.mark.asyncio +async def test_template_climate_custom_modes( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Custom fan mode/preset: traits, on_control forwarding, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + assert set(test_climate.supported_custom_fan_modes) == { + "turbo", + "silent", + "eco", + } + assert set(test_climate.supported_custom_presets) == { + "eco_plus", + "power_save", + "max", + } + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.custom_fan_mode == "" + assert initial.custom_preset == "" + + client.climate_command(test_climate.key, custom_fan_mode="turbo") + state = await wait_for_climate_state() + assert state.custom_fan_mode == "turbo" + + client.climate_command(test_climate.key, custom_preset="power_save") + state = await wait_for_climate_state() + assert state.custom_preset == "power_save" + + await asyncio.sleep(0.2) + assert any("on_control custom_fan_mode=turbo" in line for line in log_lines) + assert any("on_control custom_preset=power_save" in line for line in log_lines) + + # The device's report is authoritative and overrides what was commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.custom_fan_mode == "eco" + assert state.custom_preset == "max" diff --git a/tests/integration/test_template_climate_nonoptimistic.py b/tests/integration/test_template_climate_nonoptimistic.py new file mode 100644 index 0000000000..e922ec31b9 --- /dev/null +++ b/tests/integration/test_template_climate_nonoptimistic.py @@ -0,0 +1,107 @@ +"""Integration test for template climate: optimistic: false. + +A command still fires on_control (so a real device-backed config can forward it out), but must +NOT change the entity's own state -- only an explicit climate.template.publish call (standing in +for the device confirming the command actually took effect) does that. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-nonopt" + + +@pytest.mark.asyncio +async def test_template_climate_nonoptimistic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Nonoptimistic: a command doesn't change state until explicitly published.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + state_updates: list[aioesphomeapi.ClimateState] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + confirm_button = require_entity( + entities, "simulate_device_confirmation", ButtonInfo + ) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.mode == ClimateMode.OFF + + # Send every settable field in one command. on_control must fire with all of them, but + # nothing may be applied to the entity's own state -- no ClimateState update at all. + client.climate_command( + test_climate.key, + mode=ClimateMode.HEAT, + target_temperature=22.5, + fan_mode=ClimateFanMode.HIGH, + swing_mode=ClimateSwingMode.VERTICAL, + preset=ClimatePreset.AWAY, + ) + await asyncio.sleep(0.3) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + assert not state_updates, ( + "optimistic: false must not publish a state until climate.template.publish reports it" + ) + + # The device confirms the command actually took effect. + client.button_command(confirm_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.AWAY diff --git a/tests/integration/test_template_climate_on_control_ordering.py b/tests/integration/test_template_climate_on_control_ordering.py new file mode 100644 index 0000000000..8d212b3ccb --- /dev/null +++ b/tests/integration/test_template_climate_on_control_ordering.py @@ -0,0 +1,83 @@ +"""Integration test: on_control fires before control()/on_state, with the full ClimateCall. + +on_control's lambda argument exposes get_mode()/etc. on the *requested* ClimateCall, while the +entity's own .mode field still reflects the state *before* control() applies the change -- +proving the firing order is on_control, then control(), then on_state. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-oc-order" + + +@pytest.mark.asyncio +async def test_template_climate_on_control_ordering( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """on_control sees the requested value while the entity's own state is still the old one.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line or "on_state " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + await asyncio.sleep(0.2) + + # on_control saw the new requested mode (3 == CLIMATE_MODE_HEAT) while the entity's own + # state was still the old one (0 == CLIMATE_MODE_OFF) -- proving it fired before control(). + assert any( + "on_control requested_mode=3 current_mode_before_apply=0" in line + for line in log_lines + ) + # on_state fired afterward, reporting the now-applied mode. + assert any("on_state mode=3" in line for line in log_lines) + + control_index = next( + i for i, line in enumerate(log_lines) if "on_control " in line + ) + state_index = next(i for i, line in enumerate(log_lines) if "on_state " in line) + assert control_index < state_index, "on_control must fire before on_state" diff --git a/tests/integration/test_template_climate_publish_all_fields.py b/tests/integration/test_template_climate_publish_all_fields.py new file mode 100644 index 0000000000..9c4262b311 --- /dev/null +++ b/tests/integration/test_template_climate_publish_all_fields.py @@ -0,0 +1,96 @@ +"""Integration test for template climate: climate.template.publish covering every field at once. + +A single climate.template.publish call resolves into exactly one ClimateState update, and never +triggers on_control (which would misrepresent a device state report as a fresh command). This also +exercises that a sensor/humidity_sensor whose reading matches what's about to be published doesn't +sneak in an extra state update of its own (the sensor callback only re-publishes on an actual +change). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-publish-all" + + +@pytest.mark.asyncio +async def test_template_climate_publish_all_fields( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """One climate.template.publish call setting every field resolves to one state update.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + on_control_count = 0 + + def on_log_line(line: str) -> None: + nonlocal on_control_count + if "on_control fired" in line: + on_control_count += 1 + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + publish_button = require_entity(entities, "publish_all", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.button_command(publish_button.key) + try: + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + except TimeoutError: + pytest.fail("Timeout waiting for the published climate state") + + assert state.current_temperature == pytest.approx(20.0, abs=0.1) + assert state.current_humidity == pytest.approx(60.0, abs=0.1) + assert state.target_temperature == pytest.approx(23.0, abs=0.1) + assert state.mode == ClimateMode.HEAT + assert state.action == ClimateAction.HEATING + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.ECO + + # Give any stray extra update (there shouldn't be one) a moment to arrive. + await asyncio.sleep(0.2) + assert len(state_updates) == 1, ( + f"Expected exactly one ClimateState update, got {len(state_updates)}" + ) + assert on_control_count == 0, ( + "climate.template.publish must not trigger on_control" + ) diff --git a/tests/integration/test_template_climate_sensor_push.py b/tests/integration/test_template_climate_sensor_push.py new file mode 100644 index 0000000000..1db4da81ed --- /dev/null +++ b/tests/integration/test_template_climate_sensor_push.py @@ -0,0 +1,88 @@ +"""Integration test for template climate: current_temperature/current_humidity live sensor push. + +A *later* change to a backing sensor's value -- not just its initial reading at boot -- propagates +into a new climate state via add_on_state_callback. Re-publishing the same sensor value again must +not cause a redundant climate state update. +""" + +from __future__ import annotations + +import asyncio +import math + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-sensor-push" + + +@pytest.mark.asyncio +async def test_template_climate_sensor_push( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A later change to the backing sensor pushes a new climate state; an unchanged republish does not.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + publish_temp = require_entity(entities, "publish_temperature", ButtonInfo) + publish_temp_same = require_entity( + entities, "publish_temperature_same", ButtonInfo + ) + publish_humidity = require_entity(entities, "publish_humidity", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Neither backing sensor has published anything yet. + assert math.isnan(initial.current_temperature) + assert math.isnan(initial.current_humidity) + + # A later sensor reading -- not the initial one -- pushes a new climate state. + client.button_command(publish_temp.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_temperature == pytest.approx(24.0, abs=0.1) + + client.button_command(publish_humidity.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_humidity == pytest.approx(65.0, abs=0.1) + + # Re-publishing the same temperature must not cause a redundant climate state update. + updates_before = len(state_updates) + client.button_command(publish_temp_same.key) + await asyncio.sleep(0.3) + assert len(state_updates) == updates_before, ( + "Re-publishing an unchanged sensor reading must not republish the climate state" + ) diff --git a/tests/integration/test_template_climate_set_actions.py b/tests/integration/test_template_climate_set_actions.py new file mode 100644 index 0000000000..0b1eb80874 --- /dev/null +++ b/tests/integration/test_template_climate_set_actions.py @@ -0,0 +1,114 @@ +"""Integration test: each settable field forwards its value to the matching set_*_action. + +With optimistic: false the entity state stays put until climate.template.publish reports the +device's actual state back, so the actions are the only thing that reacts to a command. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-set-act" + + +@pytest.mark.asyncio +async def test_template_climate_set_actions( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Every set_*_action fires with the requested value; state waits for a publish.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "_action " in line or "Unsupported" in line: + log_lines.append(line) + + def logged(fragment: str) -> bool: + return any(fragment in line for line in log_lines) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + report_button = require_entity(entities, "report_device_state", ButtonInfo) + unsupported_button = require_entity( + entities, "report_unsupported_mode", ButtonInfo + ) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Both traits are derived from the low/high and humidity set actions, not declared. + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + client.climate_command( + test_climate.key, target_temperature_low=18.0, target_temperature_high=24.0 + ) + client.climate_command(test_climate.key, target_humidity=55) + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.LOW) + client.climate_command(test_climate.key, custom_fan_mode="turbo") + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + client.climate_command(test_climate.key, custom_preset="eco_plus") + + for _ in range(50): + await asyncio.sleep(0.1) + if logged("set_custom_preset_action eco_plus"): + break + + assert logged("set_mode_action 3") # CLIMATE_MODE_HEAT + assert logged("set_target_temperature_low_action 18.0") + assert logged("set_target_temperature_high_action 24.0") + assert logged("set_target_humidity_action 55") + assert logged("set_fan_mode_action 3") # CLIMATE_FAN_LOW + assert logged("set_custom_fan_mode_action turbo") + assert logged("set_swing_mode_action 2") # CLIMATE_SWING_VERTICAL + assert logged("set_preset_action 5") # CLIMATE_PRESET_ECO + assert logged("set_custom_preset_action eco_plus") + + # optimistic: false, so none of the commands above touched the entity's own state -- + # a device report is what actually moves it. + client.button_command(report_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + + # A publish naming a mode outside supported_modes warns instead of publishing it. + client.button_command(unsupported_button.key) + for _ in range(50): + await asyncio.sleep(0.1) + if logged("Unsupported mode"): + break + assert logged("Unsupported mode") diff --git a/tests/integration/test_template_climate_two_point_temperature.py b/tests/integration/test_template_climate_two_point_temperature.py new file mode 100644 index 0000000000..9270b59ffc --- /dev/null +++ b/tests/integration/test_template_climate_two_point_temperature.py @@ -0,0 +1,118 @@ +"""Integration tests for template climate: two-point target temperature + humidity. + +Covers the supports_two_point_target_temperature/supports_target_humidity boolean flags plus +on_control (forwarding commands out) and climate.template.publish (the device reporting its own +authoritative state, independent of any prior command -- e.g. a device that owns its own setpoint, +changed via a physical remote). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-two-point" + + +@pytest.mark.asyncio +async def test_template_climate_two_point_temperature( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Two-point target temperature + humidity: booleans, on_control, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + test_climate = climate_infos[0] + assert test_climate.name == "Test Two-Point Heatpump" + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Nothing has been published yet: settable fields have no sensor to seed them from, so + # the entity starts at ESPHome's plain defaults. current_temperature is pushed by the + # referenced sensor, which has already settled by the time we get here. + assert initial.mode == ClimateMode.OFF + assert initial.current_temperature == pytest.approx(21.0, abs=0.1) + + # The device reports its actual state for the first time. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT_COOL + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) + + # Commands apply optimistically (settable fields are plain internal state), and on_control + # fires with the same values so a real config could forward them to the device. + client.climate_command( + test_climate.key, target_temperature_low=19.0, target_temperature_high=25.0 + ) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(19.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(25.0, abs=0.1) + await asyncio.sleep(0.2) + assert any( + "on_control target_temperature_low=19.0" in line for line in log_lines + ) + assert any( + "on_control target_temperature_high=25.0" in line for line in log_lines + ) + + client.climate_command(test_climate.key, target_humidity=45.0) + state = await wait_for_climate_state() + assert state.target_humidity == pytest.approx(45.0, abs=0.1) + await asyncio.sleep(0.2) + assert any("on_control target_humidity=45.0" in line for line in log_lines) + + # The device's next report is authoritative and overrides whatever was optimistically + # applied above -- this is the whole point of climate.template.publish: a device that owns + # its own state (e.g. changed by a physical remote) always wins. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) From 833dd0e812ecf6e413022bd23b9d4e098245d715 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 6 Sep 2026 23:59:40 +0200 Subject: [PATCH 024/178] [ota] Offer encryption with the api key so enabling it works over OTA (#18979) --- THREAT_MODEL.md | 55 ++- esphome/__main__.py | 14 +- esphome/components/api/__init__.py | 4 +- esphome/components/api/api_connection.cpp | 6 +- .../components/api/api_frame_helper_noise.cpp | 2 +- esphome/components/api/api_server.cpp | 37 +- esphome/components/api/api_server.h | 12 +- esphome/components/esphome/ota/__init__.py | 130 +++--- .../components/esphome/ota/ota_esphome.cpp | 83 ++-- esphome/components/esphome/ota/ota_esphome.h | 13 +- .../esphome/ota/ota_esphome_noise.cpp | 91 +++-- esphome/components/noise/__init__.py | 36 +- esphome/components/noise/noise.cpp | 9 + esphome/components/noise/noise.h | 16 +- esphome/components/noise/noise_handshake.cpp | 5 +- esphome/components/noise/noise_handshake.h | 6 +- esphome/core/defines.h | 3 + esphome/espota2.py | 122 +++++- esphome/wizard.py | 18 +- .../noise/test_encryption_key.py | 14 +- tests/component_tests/ota/test_esphome_ota.py | 242 +++++++++--- .../ota/test_esphome_ota_api_key_offer.yaml | 11 + ...st_esphome_ota_api_key_offer_password.yaml | 12 + .../test_esphome_ota_encryption_required.yaml | 12 + .../ota/test_esphome_ota_own_key.yaml | 11 + .../ota/test_esphome_ota_plain.yaml | 9 + .../ota/test_esphome_ota_runtime_api_key.yaml | 10 + .../components/noise/test_noise_handshake.cpp | 18 +- .../noise/test_noise_primitives.cpp | 13 +- tests/components/ota/api_key_offer.yaml | 12 + tests/components/ota/api_runtime_key.yaml | 10 + .../ota/test-api_key_offer.esp32-idf.yaml | 2 + .../ota/test-api_key_offer.esp8266-ard.yaml | 2 + .../ota/test-api_runtime_key.esp32-idf.yaml | 2 + .../ota/test-api_runtime_key.esp8266-ard.yaml | 2 + tests/integration/conftest.py | 7 + tests/integration/const.py | 7 + .../host_ota_api_key_offer_with_password.yaml | 12 + .../host_ota_provisioned_api_key.yaml | 10 + .../test_api_zero_psk_provisioning.py | 51 ++- tests/integration/test_host_ota.py | 373 ++++++++++++------ tests/unit_tests/test_espota2_noise.py | 136 ++++++- tests/unit_tests/test_main.py | 114 +++++- tests/unit_tests/test_wizard.py | 31 +- 44 files changed, 1342 insertions(+), 443 deletions(-) create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_encryption_required.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_own_key.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_plain.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml create mode 100644 tests/components/ota/api_key_offer.yaml create mode 100644 tests/components/ota/api_runtime_key.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp8266-ard.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml create mode 100644 tests/integration/fixtures/host_ota_provisioned_api_key.yaml diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index b4f557e55b..11656ff0b7 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -125,30 +125,47 @@ design is optimal or that it will not change. ## OTA update encryption The `esphome` OTA platform optionally encrypts updates with the same Noise -`NNpsk0` pattern the native API uses; one key protects the device. With an -`encryption:` block configured the guarantees are: the firmware image is -confidential in transit, the uploader is authenticated by the pre-shared key, -and the plaintext negotiation preceding the handshake is bound into the -handshake prologue, so stripping or tampering with it fails the first MAC. -Both ends fail closed with no override: a device built with a key refuses +`NNpsk0` pattern the native API uses; one key protects the device. A device +whose `api:` block has an encryption key, static in the YAML or provisioned at +runtime, compiles in the transport and offers it on every OTA connection once +it holds a key, so an uploader presenting that key gets the guarantees below +even without an `ota: encryption:` block; only that block makes the device +require encryption. The guarantees are: the firmware image is confidential in +transit, the uploader is authenticated by the pre-shared key, and the plaintext +negotiation preceding the handshake is bound into the handshake prologue, so +stripping or tampering with it fails the first MAC. With `ota: encryption:` +configured both ends fail closed with no override: the device refuses plaintext uploads, and the CLI refuses to send plaintext when a key is -configured. +configured. Without that block the CLI tries a static api key when the device +offers and, until 2027.3.0, falls back to plaintext with a warning when the +offer is missing or the handshake fails; a runtime provisioned key never +reaches the CLI, so those uploads stay plaintext. -Defeating any of that without the key is in scope: a keyed device accepting a -plaintext or downgraded upload, getting past the MAC, or recovering image -contents from captured traffic. +Defeating any of that without the key is in scope: a device that requires +encryption accepting a plaintext or downgraded upload, getting past the MAC, +or recovering image contents from captured traffic. The following are **not** vulnerabilities, by design: -- Plaintext OTA on a device with no `encryption:` block. That is the - documented default, authenticated (if at all) by the OTA password. -- The enablement window: turning encryption on takes one last upload of the - encryption-enabled firmware over the existing plaintext channel, with the - pre-existing plaintext exposure. -- The web OTA `/update` endpoint alongside encryption. The `web_server` - component keeps it always reachable, and `captive_portal:` auto-loads it - for the fallback AP window; validation warns about both combinations, and - the operator keeps the recovery path. +- Plaintext OTA on a device with no `ota: encryption:` block, including one + that offers encryption because it has an api key. That is the documented + default, authenticated (if at all) by the OTA password. An uploader that + takes the offer skips the password; the key authenticates it. With a + runtime provisioned key and no `provisioning:` window, whoever provisions + the key gains that upload path too; validation warns about the pair. +- The CLI plaintext fallback until 2027.3.0: without `ota: encryption:` an + active attacker who strips the offer or breaks the handshake can make a + keyed CLI upload plaintext, with the pre-existing plaintext exposure. A + device that requires encryption still refuses that upload. +- The enablement window: firmware built with a static api key already offers + encryption, so turning on `ota: encryption:` is itself an encrypted upload. + Older firmware needs one last plaintext upload of an offering build, with + the pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. With the `web_server` + or `prometheus` component the shared listener is always up, so the endpoint + stays reachable and validation warns about that combination; + `captive_portal:` alone brings the listener up only for the fallback AP + window, which is the intended recovery path, so that is not warned about. - CLI retry behavior on transport or MAC failures; every attempt renegotiates a fresh handshake with fresh ephemerals, so retrying does not weaken authentication. diff --git a/esphome/__main__.py b/esphome/__main__.py index b3d58ad13b..30e97f55eb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1335,12 +1335,14 @@ def _upload_via_native_api( break from esphome import espota2 + from esphome.components.noise import static_encryption_key remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) # Fail closed: an encryption block whose key did not resolve must never # fall back to a plaintext upload noise_psk = None + plaintext_fallback = False if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: noise_psk = encryption_conf.get(CONF_KEY) if not noise_psk: @@ -1351,6 +1353,10 @@ def _upload_via_native_api( # Ensure the key is a string, as required by the underlying OTA implementation. # It arrives here as a SensitiveStr which aioesphomeapi rejects. noise_psk = str(noise_psk) + elif api_key := static_encryption_key(config.get(CONF_API) or {}): + # Remove before 2027.3.0: the api key is tried, falling back to plaintext + noise_psk = str(api_key) + plaintext_fallback = True def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1382,7 +1388,13 @@ def _upload_via_native_api( _validate_bootloader_binary(binary) return espota2.run_ota( - network_devices, remote_port, password, binary, ota_type, noise_psk + network_devices, + remote_port, + password, + binary, + ota_type, + noise_psk, + plaintext_fallback=plaintext_fallback, ) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3568318dad..6202e127bf 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -14,6 +14,7 @@ from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, encryption_schema, + new_psk_progmem, validate_encryption_key, ) from esphome.config_helpers import filter_source_files_from_defines, get_logger_level @@ -589,8 +590,7 @@ async def to_code(config: ConfigType) -> None: if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None: if key := encryption_config.get(CONF_KEY): - decoded = decode_encryption_key(key) - cg.add(var.set_noise_psk(list(decoded))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) cg.add_define("USE_API_NOISE_PSK_FROM_YAML") else: # No key provided, but encryption desired diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9c609aa047..da4b7d7702 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2161,7 +2161,10 @@ void APIConnection::on_homeassistant_action_response(const HomeassistantActionRe bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg) { NoiseEncryptionSetKeyResponse resp; resp.success = false; - +#ifdef USE_API_NOISE_PSK_FROM_YAML + // A yaml key cannot be changed at runtime, so no decode or save path is built + ESP_LOGW(TAG, "Key set in YAML"); +#else #ifdef USE_PROVISIONING // Refuse to set a key once the provisioning window has closed (defense in depth; // such connections are already rejected at hello). @@ -2196,6 +2199,7 @@ bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptio } #endif } +#endif // USE_API_NOISE_PSK_FROM_YAML return this->send_message(resp); } diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 138dbdddba..29b2858aee 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -548,7 +548,7 @@ APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) { * @return 0 on success, -1 on error (check errno) */ APIError APINoiseFrameHelper::init_handshake_() { - int err = this->handshake_.init(this->ctx_.get_psk(), prologue_.data(), prologue_.size()); + int err = this->handshake_.init(this->ctx_, prologue_.data(), prologue_.size()); APIError aerr = handle_noise_error_(err, LOG_STR("noise_handshake_init"), APIError::HANDSHAKESTATE_SETUP_FAILED); if (aerr != APIError::OK) return aerr; diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 43d35363d3..78ebe5c38e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -41,13 +41,13 @@ void APIServer::setup() { ControllerRegistry::register_controller(this); #ifdef USE_API_NOISE + // Always reserve the slot: flash preferences are positional on esp8266, so + // a yaml key build must keep the layout of a runtime key build uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); - #ifndef USE_API_NOISE_PSK_FROM_YAML - // Only load saved PSK if not set from YAML - if (this->load_and_apply_noise_psk_()) { + // A cleared record loads fine but holds no key + if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { ESP_LOGD(TAG, "Loaded saved Noise PSK"); } #endif @@ -550,6 +550,7 @@ const std::vector &APIServer::get_sta #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active) { if (!this->noise_pref_.save(&new_psk)) { @@ -583,22 +584,19 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString } bool APIServer::load_and_apply_noise_psk_() { - SavedNoisePsk saved{}; - if (!this->noise_pref_.load(&saved)) + // Load into a temp so a failed read cannot disturb the key in use + SavedNoisePsk loaded{}; + if (!this->noise_pref_.load(&loaded)) return false; - this->set_noise_psk(saved.psk); + this->saved_psk_ = loaded; + // An unprovisioned device stores the reserved all-zeros key, which is no key + const bool has_key = !noise::NoiseContext::is_all_zeros(this->saved_psk_.psk); + this->noise_ctx_.set_psk(has_key ? this->saved_psk_.psk.data() : nullptr); return true; } bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else - auto &old_psk = this->noise_ctx_.get_psk(); - if (std::equal(old_psk.begin(), old_psk.end(), psk.begin())) { + if (this->saved_psk_.psk == psk) { ESP_LOGW(TAG, "New PSK matches old"); return true; } @@ -614,15 +612,8 @@ bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { } #endif return result; -#endif } bool APIServer::clear_noise_psk(bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else SavedNoisePsk empty_psk{}; bool result = this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"), make_active); @@ -634,8 +625,8 @@ bool APIServer::clear_noise_psk(bool make_active) { } #endif return result; -#endif } +#endif // USE_API_NOISE_PSK_FROM_YAML #endif #ifdef USE_HOMEASSISTANT_TIME diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 072a583901..618ea4eb11 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -76,9 +76,14 @@ class APIServer final : public Component, APIBuffer &get_shared_buffer_ref() { return shared_write_buffer_; } #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML + // Runtime key changes exist for the provisioning path only (not lambdas); + // with a yaml key they compile out bool save_noise_psk(noise::psk_t psk, bool make_active = true); bool clear_noise_psk(bool make_active = true); - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } noise::NoiseContext &get_noise_ctx() { return this->noise_ctx_; } #endif // USE_API_NOISE @@ -275,10 +280,12 @@ class APIServer final : public Component, #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active); // Load saved PSK from preferences and apply it. Returns true on success. bool load_and_apply_noise_psk_(); +#endif // USE_API_NOISE_PSK_FROM_YAML #endif // USE_API_NOISE #ifdef USE_API_HOMEASSISTANT_STATES // Helper methods to reduce code duplication @@ -358,6 +365,9 @@ class APIServer final : public Component, #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; +#ifndef USE_API_NOISE_PSK_FROM_YAML + SavedNoisePsk saved_psk_{}; // backs noise_ctx_ for a runtime provisioned key +#endif ESPPreferenceObject noise_pref_; #endif // USE_API_NOISE }; diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 1fec9e5c9b..f5eb878260 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -2,12 +2,12 @@ import logging import esphome.codegen as cg from esphome.components.noise import ( - decode_encryption_key, encryption_schema, - is_reserved_key, + new_psk_progmem, + static_encryption_key, ) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code -from esphome.config_helpers import merge_config +from esphome.config_helpers import filter_source_files_from_defines, merge_config import esphome.config_validation as cv from esphome.const import ( CONF_API, @@ -31,7 +31,6 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" -CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -41,11 +40,10 @@ DEPENDENCIES = ["network"] def AUTO_LOAD(config: ConfigType) -> list[str]: - """Auto-load noise only when encryption is configured.""" + """Auto-load noise only when encryption is configured; the api key offer + inherits it from the api component.""" base = ["sha256", "socket"] - # A falsy config is a tooling probe for the maximal set (None from - # dependency resolution, {} from the components-graph platform probe); - # a validated config always carries defaults, never empty + # A falsy config is a tooling probe for the maximal set if not config or CONF_ENCRYPTION in config: return base + ["noise"] return base @@ -132,12 +130,56 @@ def ota_esphome_final_validate(config: ConfigType) -> None: _validate_no_password_with_encryption(ota_conf) if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: _resolve_encryption_key(encryption_conf, api_conf) - if any( - conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf - ) and any( - CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + elif CONF_PASSWORD in ota_conf and static_encryption_key(api_conf) is not None: + _LOGGER.warning( + "'%s' %s wastes significant flash and RAM (about 3.5 KB and 60 " + "bytes plus the password on the heap): the device already offers " + "encryption with the '%s' %s %s, which authenticates any uploader " + "that takes it, and a password only matters for uploaders without " + "encryption support; remove '%s' and add '%s' under '%s' so " + "uploads use the key and encryption is required", + CONF_OTA, + CONF_PASSWORD, + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_PASSWORD, + CONF_ENCRYPTION, + CONF_OTA, + ) + elif ( + CONF_PASSWORD in ota_conf + and CONF_ENCRYPTION in api_conf + and not api_conf[CONF_ENCRYPTION].get(CONF_KEY) + ): + # The CLI still needs the password; whoever provisions the key skips it + _LOGGER.warning( + "The '%s' %s %s provisioned at runtime also authenticates OTA " + "uploads once provisioned; '%s' %s then only guards plaintext " + "uploads. Whoever provisions the key can upload firmware " + "without the password, so add a 'provisioning:' block to limit " + "when that is possible", + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_OTA, + CONF_PASSWORD, + ) + # web_server and prometheus keep the shared listener up; the captive + # portal's copy only exists on the fallback AP and is the recovery path + if ( + (CONF_WEB_SERVER in full_conf or "prometheus" in full_conf) + and any(conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf) + and any( + CONF_ENCRYPTION in conf + for conf in merged_ota_esphome_configs_by_port.values() + ) ): - _warn_web_server_ota(full_conf) + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -152,33 +194,11 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) -def _warn_web_server_ota(full_conf: ConfigType) -> None: - """The web_server ota platform accepts the same image over plaintext HTTP - with basic auth, bypassing the encryption; warn rather than fail so the - operator keeps the recovery path.""" - if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: - # The captive_portal auto-load: the endpoint only exists while the - # fallback AP is active - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform (auto-loaded " - "by captive_portal); the plaintext /update endpoint stays " - "reachable while the fallback AP is active", - CONF_WEB_SERVER, - ) - else: - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform; its " - "plaintext /update endpoint accepts the same image", - CONF_WEB_SERVER, - ) - - def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: """Resolve the one encryption key per device into the ota block. An explicit ota key must match the api key, a bare block inherits it, - a runtime provisioned api key cannot be inherited, and the all-zeros - provisioning sentinel is rejected (the device treats it as no key). + a runtime provisioned api key cannot be inherited. """ api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) if ota_key := encryption_conf.get(CONF_KEY): @@ -201,11 +221,6 @@ def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) - ) else: encryption_conf[CONF_KEY] = api_key - if is_reserved_key(encryption_conf[CONF_KEY]): - raise cv.Invalid( - f"The all-zeros {CONF_KEY} is reserved and provides no protection; " - f"generate a real key with: openssl rand -base64 32" - ) # Also called on merged same-port configs in final validate, where schemas @@ -267,15 +282,9 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate -def FILTER_SOURCE_FILES() -> list[str]: - """Filter out the noise transport when no ota entry configures encryption.""" - for ota_conf in CORE.config.get(CONF_OTA, []): - if ( - ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME - and ota_conf.get(CONF_ENCRYPTION) is not None - ): - return [] - return ["ota_esphome_noise.cpp"] +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} +) @coroutine_with_priority(CoroPriority.OTA_UPDATES) @@ -296,11 +305,24 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") - if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: - # A missing key was resolved from the api component in final validate. - key = encryption_conf[CONF_KEY] + # One key per device: an api encryption block supplies it (static or + # runtime) and offers; the ota block only adds the requirement + api_conf = CORE.config.get(CONF_API) or {} + encryption_conf = config.get(CONF_ENCRYPTION) + own_key = None + if encryption_conf is not None and static_encryption_key(api_conf) is None: + own_key = encryption_conf[CONF_KEY] + if own_key is not None: cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + elif CONF_ENCRYPTION in api_conf: + cg.add_define("USE_OTA_ENCRYPTION") + cg.add_define("USE_OTA_ENCRYPTION_FROM_API") + if static_encryption_key(api_conf) is None: + # The key arrives at runtime, so the offer has to look for it + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 396a47bc52..1005ed214b 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,4 +1,7 @@ #include "ota_esphome.h" +#ifdef USE_OTA_ENCRYPTION_FROM_API +#include "esphome/components/api/api_server.h" +#endif #ifdef USE_OTA #ifdef USE_OTA_PASSWORD #include "esphome/components/sha256/sha256.h" @@ -26,6 +29,16 @@ namespace esphome { static const char *const TAG = "esphome.ota"; + +#ifdef USE_OTA_ENCRYPTION +const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { +#ifdef USE_OTA_ENCRYPTION_FROM_API + return api::global_api_server->get_noise_ctx(); +#else + return this->noise_ctx_; +#endif +} +#endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -97,18 +110,30 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, "Over-The-Air updates:\n" " Address: %s:%u\n" - " Version: %d", - network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION); + " Version: %d" +#ifdef USE_OTA_ENCRYPTION + "\n Encryption: %s" +#endif + , + network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION +#ifdef USE_OTA_ENCRYPTION_REQUIRED + , + LOG_STR_LITERAL("required") +#elif defined(USE_OTA_ENCRYPTION_PROVISIONED) + // A runtime provisioned key may not exist yet + , + this->noise_context_().has_psk() ? LOG_STR_LITERAL("offered, plaintext accepted") + : LOG_STR_LITERAL("offered once the api key is provisioned") +#elif defined(USE_OTA_ENCRYPTION) + , + LOG_STR_LITERAL("offered, plaintext accepted") +#endif + ); #ifdef USE_OTA_PASSWORD if (!this->password_.empty()) { ESP_LOGCONFIG(TAG, " Password configured"); } #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { - ESP_LOGCONFIG(TAG, " Encryption configured"); - } -#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -154,10 +179,22 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +// Noise needs the extended protocol: the prologue binds the 2-byte feature ack +static constexpr uint8_t CLIENT_NOISE_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +inline bool ESPHomeOTAComponent::extended_proto_() const { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // FEATURE_READ already refused every client without the extended protocol + return true; +#else + return (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; +#endif +} + void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. /// @@ -241,12 +278,9 @@ void ESPHomeOTAComponent::handle_handshake_() { this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); -#ifdef USE_OTA_ENCRYPTION - // Fail closed: with a PSK configured the client must negotiate encryption - // (which requires the extended protocol); refuse plaintext uploads. - static constexpr uint8_t NOISE_REQUIRED_FEATURES = - CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; - if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // `ota: encryption:` requires the client to negotiate encryption + if ((this->ota_features_ & CLIENT_NOISE_FEATURES) != CLIENT_NOISE_FEATURES) { ESP_LOGW(TAG, "Client does not support encryption"); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); return; @@ -261,18 +295,21 @@ void ESPHomeOTAComponent::handle_handshake_() { // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte // legacy response. - this->extended_proto_ = (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; - if (this->extended_proto_) { + if (this->extended_proto_()) { static_assert(HANDSHAKE_BUF_SIZE >= 2, "handshake_buf_ must hold the 2-byte extended-protocol feature ack"); this->handshake_buf_[0] = ota::OTA_RESPONSE_FEATURE_FLAGS; this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // A runtime provisioned key may not exist yet + if (this->noise_context_().has_psk()) { this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; } +#elif defined(USE_OTA_ENCRYPTION) + // A yaml key always exists: validation rejects the all-zeros key + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; #endif } else { this->handshake_buf_[0] = @@ -284,15 +321,15 @@ void ESPHomeOTAComponent::handle_handshake_() { case OTAState::FEATURE_ACK: { static constexpr size_t STANDARD_PROTO_ACK_SIZE = 1; static constexpr size_t EXTENDED_PROTO_ACK_SIZE = 2; - const size_t ack_size = this->extended_proto_ ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; + const size_t ack_size = this->extended_proto_() ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } #ifdef USE_OTA_ENCRYPTION - // With a PSK configured the rest of the session runs inside the noise - // transport; the client sends the first handshake frame next, so there - // is nothing to do until data arrives. - if (this->noise_ctx_.has_psk()) { + // Latch the offer actually sent: a key activating between the two + // states must not start a session the client never expects + if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -412,7 +449,7 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge auth OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); - if (this->extended_proto_) { + if (this->extended_proto_()) { // Read ota type, 1 byte if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index fd164b8138..c6f710b3fc 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,8 +44,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#ifdef USE_OTA_ENCRYPTION - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif /// Manually set the port OTA should listen on @@ -85,9 +86,12 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; + // The api server's live context when the api has encryption, else our own + const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); + size_t noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len); bool noise_try_write_frame_(); void noise_send_reject_(const LogString *reason); ssize_t noise_decrypt_(uint8_t *buf, size_t len); @@ -144,7 +148,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION +#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#endif std::unique_ptr noise_; #endif // USE_OTA_ENCRYPTION @@ -166,6 +172,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; + // Derived from the feature byte; storing it would pad the trailing bytes + bool extended_proto_() const; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; @@ -179,7 +187,6 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint8_t auth_buf_pos_{0}; uint8_t auth_type_{0}; // Store auth type to know which hasher to use #endif // USE_OTA_PASSWORD - bool extended_proto_{false}; }; } // namespace esphome diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7f8331cf96..7401413d6d 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -3,6 +3,7 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise.h" #include "esphome/components/ota/ota_backend.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -40,24 +41,17 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // A provisioned key cleared between the offer and here is not guarded: the + // session runs on the zero key load_psk fills in and fails the client's MAC. + // Default-init: the frame buffer is written before it is read // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); - if (this->noise_ == nullptr) { - ESP_LOGW(TAG, "Session allocation failed"); - this->cleanup_connection_(); - return false; - } - + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + PROLOGUE_FEATURE_ACK_LEN]; -#ifdef USE_ESP8266 - memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#else - std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#endif + progmem_memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; // Magic bytes, already validated in MAGIC_READ std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); @@ -71,9 +65,13 @@ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; *p++ = server_feature_flags; - int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + // The caller only starts a session when the context holds a key + int err = this->noise_ == nullptr ? NOISE_ERROR_NO_MEMORY + : this->noise_->handshake.init(this->noise_context_(), prologue, sizeof(prologue)); if (err != 0) { - ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + // Raw noise codes throughout: the name table would cost flash in builds + // where only the OTA uses noise + ESP_LOGW(TAG, "Session init: %d", err); this->cleanup_connection_(); return false; } @@ -105,14 +103,16 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { s.frame_pos = 0; s.frame_len = 0; if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { - ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + ESP_LOGW(TAG, "Client rejected the handshake: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); this->cleanup_connection_(); return false; } int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); if (err != 0) { - ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); - this->noise_send_reject_(noise::reject_reason_for(err)); + // A MAC failure here almost always means the uploader has a different key + const LogString *reason = noise::reject_reason_for(err); + ESP_LOGW(TAG, "Handshake read: %s (%d)", LOG_STR_ARG(reason), err); + this->noise_send_reject_(reason); this->cleanup_connection_(); return false; } @@ -123,7 +123,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { int err = s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); if (err != 0) { - ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake write: %d", err); this->cleanup_connection_(); return false; } @@ -138,7 +138,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { int err = s.handshake.split(s.send_cipher, s.recv_cipher); if (err != 0) { - ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake split: %d", err); this->cleanup_connection_(); return false; } @@ -154,33 +154,41 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { } } +/// Payload length from a frame header, or 0 (logged) when the indicator or +/// the length is out of range. Callers pass min_len >= 1 so 0 is never valid. +size_t ESPHomeOTAComponent::noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len) { + const size_t payload_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || payload_len < min_len || payload_len > max_len) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], payload_len); + return 0; + } + return payload_len; +} + /// Non-blocking read of one handshake frame into the session buffer. bool ESPHomeOTAComponent::noise_try_read_frame_() { NoiseSession &s = *this->noise_; - while (s.frame_pos < noise::FRAME_HEADER_SIZE) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { - return false; + while (true) { + // The header first, then the body once the header says how long it is + const uint16_t want = s.frame_len == 0 ? noise::FRAME_HEADER_SIZE : s.frame_len; + if (s.frame_pos < want) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, want - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise"))) { + return false; + } + s.frame_pos += read; + continue; } - s.frame_pos += read; - } - if (s.frame_len == 0) { - const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); - if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { - ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + if (s.frame_len != 0) { + return true; + } + const size_t payload_len = this->noise_frame_payload_len_(s.frame_buf, 1, 1 + noise::MAX_HANDSHAKE_SIZE); + if (payload_len == 0) { this->cleanup_connection_(); return false; } s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; } - while (s.frame_pos < s.frame_len) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { - return false; - } - s.frame_pos += read; - } - return true; } /// Non-blocking write of the pending session-buffer frame. @@ -214,7 +222,7 @@ ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { noise_buffer_set_inout(mbuf, buf, len, len); int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Decrypt: %d", err); return -1; } return mbuf.size; @@ -229,9 +237,8 @@ ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min if (!this->readall_(header, sizeof(header))) { return -1; } - const size_t ciphertext_len = encode_uint16(header[1], header[2]); - if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { - ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + const size_t ciphertext_len = this->noise_frame_payload_len_(header, min_ciphertext, max_ciphertext); + if (ciphertext_len == 0) { return -1; } if (!this->readall_(buf, ciphertext_len)) { @@ -267,7 +274,7 @@ bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Encrypt: %d", err); return false; } noise::write_frame_header(frame, mbuf.size); diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 0f9328a482..a1d9444fc0 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -4,7 +4,9 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_KEY +from esphome.const import CONF_ENCRYPTION, CONF_KEY +from esphome.core import ID +from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] @@ -23,6 +25,14 @@ def validate_encryption_key(value: Any) -> str: if len(decoded) != 32: raise cv.Invalid("Encryption key must be base64 and 32 bytes long") + if not any(decoded): + # The device treats the all-zeros key as no key at all (it is the + # provisioning sentinel), so it must never reach a build + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"omit the {CONF_KEY} to provision it at runtime, or generate a real " + "key with: openssl rand -base64 32" + ) # Return original data for roundtrip conversion return value @@ -45,15 +55,6 @@ def decode_encryption_key(value: str) -> bytes: return decoded -def is_reserved_key(value: str) -> bool: - """Whether the key is the reserved all-zeros provisioning sentinel. - - The device treats it as no key configured, so consumers that require a - real key must reject it. - """ - return not any(decode_encryption_key(value)) - - ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), @@ -61,6 +62,21 @@ ENCRYPTION_SCHEMA = cv.Schema( ) +def static_encryption_key(conf: ConfigType) -> str | None: + """The build time key of a component config; None without one or when + the key is provisioned at runtime.""" + return (conf.get(CONF_ENCRYPTION) or {}).get(CONF_KEY) or None + + +def new_psk_progmem(parent_id: ID, key: str) -> MockObj: + """Emit the decoded key as a PROGMEM array; the component keeps a pointer + so the key never occupies RAM.""" + return cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decode_encryption_key(key)), + ) + + def encryption_schema(config: ConfigType | None) -> ConfigType: # A bare `encryption:` block is valid; a missing key means the consumer # falls back to its keyless behavior (api provisioning, ota inheriting diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 95fab322db..4806706167 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,5 +1,6 @@ #include "noise.h" #ifdef USE_NOISE +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -15,6 +16,14 @@ namespace esphome::noise { static const char *const TAG = "noise"; +void NoiseContext::load_psk(psk_t &out) const { + if (this->psk_ == nullptr) { + out.fill(0); + return; + } + progmem_memcpy(out.data(), this->psk_, out.size()); +} + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index f9da8d35b8..1033d5423c 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -23,16 +23,16 @@ class NoiseContext { } return acc == 0; } - void set_psk(psk_t psk) { - this->psk_ = psk; - this->has_psk_ = !is_all_zeros(psk); - } - const psk_t &get_psk() const { return this->psk_; } - bool has_psk() const { return this->has_psk_; } + /// psk points at 32 bytes that outlive the context (PROGMEM or caller owned + /// RAM); nullptr means no key. Runtime callers map the all-zeros key to + /// nullptr themselves; validation keeps it out of yaml. + void set_psk(const uint8_t *psk) { this->psk_ = psk; } + /// Copy the key out (flash-aware on ESP8266); all zeros when none is set. + void load_psk(psk_t &out) const; + bool has_psk() const { return this->psk_ != nullptr; } protected: - psk_t psk_{}; - bool has_psk_{false}; + const uint8_t *psk_{nullptr}; }; /// Convert a noise error code to a readable error diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 6d426de012..cc7fa603c4 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -20,7 +20,7 @@ NoiseResponderHandshake::~NoiseResponderHandshake() { } } -int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len) { +int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len) { if (this->handshake_ != nullptr) { noise_handshakestate_free(this->handshake_); this->handshake_ = nullptr; @@ -44,6 +44,9 @@ int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, siz HANDSHAKE_STEP_LOG("noise_handshakestate_new_by_id", err); return err; } + // noise-c keeps its own copy, so the key only passes through the stack here + psk_t psk; + ctx.load_psk(psk); err = noise_handshakestate_set_pre_shared_key(this->handshake_, psk.data(), psk.size()); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_set_pre_shared_key", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index 30596f35c2..bf1aa8cb7f 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -36,9 +36,9 @@ class NoiseResponderHandshake { NoiseResponderHandshake(const NoiseResponderHandshake &) = delete; NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; - /// Create and start the handshake with the given PSK and prologue. A - /// repeated call frees the previous handshake state and starts over. - [[nodiscard]] int init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len); + /// Create and start the handshake with the context's PSK and the prologue. + /// A repeated call frees the previous handshake state and starts over. + [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. [[nodiscard]] Action action() const; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 526adf74f0..9dd1e0ced6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -244,6 +244,9 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION +#define USE_OTA_ENCRYPTION_FROM_API +#define USE_OTA_ENCRYPTION_PROVISIONED +#define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ac4cbeeb7c..ce403c398d 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -202,6 +202,49 @@ class OTANetworkError(OTAError): """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" +# Remove before 2027.3.0 +class OTAEncryptionFallback(OTAError): + """The encrypted attempt failed and the caller may retry in plaintext.""" + + +# Remove before 2027.3.0 +PLAINTEXT_FALLBACK_NOTICE = ( + "A device with an api encryption key offers encryption after this " + "install; add 'encryption:' under 'ota: platform: esphome' to require it. " + "This plaintext fallback is removed in 2027.3.0." +) + + +# Remove before 2027.3.0 +class _EncryptionAttempt: + """The key an upload tries and whether it may fall back to plaintext; + a rejected handshake falls back at once, a transport fault only on repeat.""" + + def __init__(self, noise_psk: str | None, plaintext_fallback: bool) -> None: + self.noise_psk = noise_psk + self.plaintext_fallback = plaintext_fallback + self.handshake_faults = 0 + + def handshake_fault_falls_back(self) -> bool: + self.handshake_faults += 1 + return self.plaintext_fallback and self.handshake_faults >= 2 + + def downgrade(self, reason: str) -> None: + _LOGGER.warning( + "%s. Retrying in plaintext; a device that requires encryption " + "refuses it. %s", + reason, + PLAINTEXT_FALLBACK_NOTICE, + ) + self.noise_psk = None + self.plaintext_fallback = False + + +# Remove before 2027.3.0: only the fallback decision needs this distinction +class OTAHandshakeNetworkError(OTANetworkError): + """A transport failure inside the noise handshake; retrying encrypted may succeed.""" + + def _committed_error(err: OTANetworkError) -> OTAError: """Wrap a network failure that happened once the device had the full image. @@ -464,6 +507,7 @@ def perform_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> None: # Validate up front; an out-of-range value would only surface as a # ValueError deep inside send_check, bypassing OTAError handling @@ -528,19 +572,28 @@ def perform_ota( else: features = 0 - if noise_psk: - # Fail closed: never fall back to a plaintext upload when an - # encryption key is configured, an active attacker could otherwise - # strip the feature flag and capture the image (it contains the wifi - # credentials and the api encryption key). - if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if noise_psk and not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if plaintext_fallback: + # Remove before 2027.3.0: older firmware that cannot encrypt still + # gets its update on this connection + _LOGGER.warning( + "The device did not offer OTA encryption; continuing in plaintext. %s", + PLAINTEXT_FALLBACK_NOTICE, + ) + noise_psk = None + else: + # Fail closed: an attacker could otherwise strip the offer and + # capture the image (wifi credentials, api key) raise OTAError( "An OTA encryption key is configured but the device did not " "offer encryption; refusing to send the image in plaintext. " - "If the running firmware predates OTA encryption, first update " - "it without the 'ota: encryption:' block (over a trusted " - "network or via USB), then restore the block and upload again." + "The running firmware predates ESPHome 2026.9.0 or has no " + "'api: encryption: key'. With an api key, install once " + "without the 'ota: encryption:' block (that build offers " + "encryption), then restore it; otherwise flash by serial or " + "the web_server OTA platform." ) + if noise_psk: # The prologue binds every negotiation byte both sides saw, so any # tampering with the plaintext preamble breaks the handshake. prologue = ( @@ -549,8 +602,18 @@ def perform_ota( + bytes([RESPONSE_OK, version, features_to_send]) + bytes([RESPONSE_FEATURE_FLAGS, features]) ) + # Built outside the try: a local failure must never downgrade the upload sock = NoiseSocketWrapper(sock, noise_psk, prologue) - sock.do_handshake() + try: + sock.do_handshake() + except OTANetworkError as err: + # A transport fault: retry encrypted before considering plaintext + raise OTAHandshakeNetworkError(str(err)) from err + except OTAError as err: + # Remove before 2027.3.0 + if plaintext_fallback: + raise OTAEncryptionFallback(str(err)) from err + raise _LOGGER.info("Encrypted connection established") if ota_type != OTA_TYPE_UPDATE_APP: @@ -757,6 +820,7 @@ def run_ota_impl_( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -795,7 +859,9 @@ def run_ota_impl_( total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" reached_device = False - for attempt in range(total_attempts): + attempt = 0 + encryption = _EncryptionAttempt(noise_psk, plaintext_fallback) + while attempt < total_attempts: af, socktype, _, _, sa = res[attempt % len(res)] if reached_device or attempt >= len(res): _LOGGER.info( @@ -815,17 +881,40 @@ def run_ota_impl_( sock.close() _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) last_error = f"connecting to {sa[0]} failed: {err}" + attempt += 1 continue _LOGGER.info("Connected to %s", sa[0]) reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) + perform_ota( + sock, + password, + file_handle, + filename, + ota_type, + encryption.noise_psk, + encryption.plaintext_fallback, + ) + except OTAEncryptionFallback as err: + # Same address and attempt budget: not a network retry + last_error = str(err) + encryption.downgrade(last_error) + continue + except OTAHandshakeNetworkError as err: + last_error = str(err) + if encryption.handshake_fault_falls_back(): + encryption.downgrade(last_error) + continue + _LOGGER.warning("%s", last_error) + attempt += 1 + continue except OTANetworkError as err: # Transient network failure; retry last_error = str(err) _LOGGER.warning("%s", last_error) + attempt += 1 continue except OTAError as err: # Device-reported error (wrong password, wrong flash size, ...); @@ -847,10 +936,17 @@ def run_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: try: return run_ota_impl_( - remote_host, remote_port, password, filename, ota_type, noise_psk + remote_host, + remote_port, + password, + filename, + ota_type, + noise_psk, + plaintext_fallback, ) except OTAError as err: _LOGGER.error(err) diff --git a/esphome/wizard.py b/esphome/wizard.py index f7706928e9..897d5f60a1 100644 --- a/esphome/wizard.py +++ b/esphome/wizard.py @@ -148,11 +148,13 @@ def wizard_file(**kwargs: Unpack[WizardFileKwargs]) -> str: if "api_encryption_key" in kwargs: config += f' encryption:\n key: "{kwargs["api_encryption_key"]}"\n' - # Configure OTA + # The api key also secures OTA; a password only serves older uploaders config += "\nota:\n" config += " - platform: esphome\n" if "ota_password" in kwargs: config += f' password: "{kwargs["ota_password"]}"' + elif "api_encryption_key" in kwargs: + config += " encryption:" # Configuring wifi config += "\n\nwifi:\n" @@ -529,20 +531,9 @@ def wizard(path: Path) -> int: safe_print() safe_print("You'll need this key when adding the device to Home Assistant.") sleep(1) - - safe_print() - safe_print( - f"Do you want to set a {color(AnsiFore.GREEN, 'password')} for OTA updates? " - "This can be insecure if you do not trust the WiFi network." - ) - safe_print() - sleep(0.25) - safe_print("Press ENTER for no password") - ota_password = safe_input(color(AnsiFore.BOLD_WHITE, "(password): ")) else: ssid, psk = "", "" api_encryption_key = None - ota_password = "" kwargs = { "path": path, @@ -553,10 +544,9 @@ def wizard(path: Path) -> int: "psk": psk, "type": "basic", } + # The api key also secures OTA updates, so the wizard sets no OTA password if api_encryption_key: kwargs["api_encryption_key"] = api_encryption_key - if ota_password: - kwargs["ota_password"] = ota_password if not wizard_write(**kwargs): return 1 diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 10f1eb3d4c..2b79bd5464 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,11 +5,7 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import ( - decode_encryption_key, - is_reserved_key, - validate_encryption_key, -) +from esphome.components.noise import decode_encryption_key, validate_encryption_key KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -41,6 +37,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: decode_encryption_key("AAECAw==") -def test_is_reserved_key() -> None: - assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") - assert not is_reserved_key(KEY) +def test_validate_encryption_key_rejects_all_zeros() -> None: + """The all-zeros key is the provisioning sentinel the device treats as no + key, so it never reaches a build.""" + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + validate_encryption_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 873f162555..d3092294dc 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable import logging from typing import Any @@ -14,6 +15,7 @@ from esphome.components.esphome.ota import ( _validate_no_password_with_encryption, ota_esphome_final_validate, ) +from esphome.components.noise import static_encryption_key from esphome.const import ( CONF_API, CONF_ENCRYPTION, @@ -115,7 +117,6 @@ def test_non_esphome_ota_unaffected() -> None: API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" -ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" def test_encryption_key_inherited_from_api() -> None: @@ -197,36 +198,6 @@ def test_encryption_without_any_key_rejected() -> None: fv.full_config.reset(token) -def test_encryption_explicit_all_zeros_key_rejected() -> None: - """The all-zeros key is the provisioning sentinel; the device would treat - it as no PSK and accept plaintext, so it must fail validation.""" - full_conf = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) - ], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - -def test_encryption_inherited_all_zeros_key_rejected() -> None: - """An all-zeros api key must not silently disable ota encryption either.""" - full_conf = { - CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, - CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: """Same-port configs with different encryption keys raise.""" full_conf = { @@ -295,13 +266,14 @@ def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None fv.full_config.reset(token) +@pytest.mark.parametrize("component", ["web_server", "prometheus"]) def test_encryption_with_web_server_ota_warns( - caplog: pytest.LogCaptureFixture, + caplog: pytest.LogCaptureFixture, component: str ) -> None: - """With the web_server component the plaintext /update endpoint is always - on; the combination validates with a warning.""" + """web_server and prometheus keep the shared listener up, so the + plaintext /update endpoint is always on and the combination warns.""" full_conf = { - "web_server": {}, + component: {}, CONF_OTA: [ _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, @@ -316,12 +288,12 @@ def test_encryption_with_web_server_ota_warns( fv.full_config.reset(token) -def test_encryption_with_captive_portal_web_server_ota_warns( +def test_encryption_with_captive_portal_does_not_warn( caplog: pytest.LogCaptureFixture, ) -> None: """captive_portal auto-loads the web_server ota platform without the - web_server component; encryption stays usable and only warns, so the - fallback AP recovery path is not lost.""" + web_server component; its endpoint only exists while the fallback AP is + active and is the intended recovery path, so there is no warning.""" full_conf = { "captive_portal": {}, CONF_OTA: [ @@ -333,7 +305,10 @@ def test_encryption_with_captive_portal_web_server_ota_warns( try: with caplog.at_level(logging.WARNING): ota_esphome_final_validate({}) - assert any("captive_portal" in record.message for record in caplog.records) + assert not any( + "OTA encryption does not cover" in record.message + for record in caplog.records + ) esphome_conf = next( conf for conf in fv.full_config.get()[CONF_OTA] @@ -344,6 +319,100 @@ def test_encryption_with_captive_portal_web_server_ota_warns( fv.full_config.reset(token) +def test_password_with_api_key_warns(caplog: pytest.LogCaptureFixture) -> None: + """A static api key makes the device offer encryption and the CLI take + it, so the password is dead weight; the config validates with a warning.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("wastes significant flash" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_password_with_runtime_api_key_warns_differently( + caplog: pytest.LogCaptureFixture, +) -> None: + """The CLI still needs the password, but the provisioned key also + authenticates uploads; the warning says so without the flash advice.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + messages = [r.message for r in caplog.records] + assert any("provisioned at runtime also authenticates" in m for m in messages) + assert not any("wastes significant flash" in m for m in messages) + finally: + fv.full_config.reset(token) + + +def test_password_without_api_key_no_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """Without an api key there is no offer, so nothing to warn about.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("authenticates" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_web_server_component_without_ota_platform_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """The web_server component alone has no /update endpoint.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any( + "OTA encryption does not cover" in r.message for r in caplog.records + ) + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_platform_alone_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the web_server component starts the shared listener, so the ota + platform on its own never exposes /update.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("plaintext /update" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + def test_web_server_ota_without_encryption_unaffected() -> None: """web_server ota stays valid alongside an unencrypted esphome entry.""" full_conf = { @@ -370,20 +439,87 @@ def test_auto_load_pulls_noise_only_for_encryption() -> None: assert "noise" in AUTO_LOAD({}) -def test_filter_source_files_excludes_noise_without_encryption() -> None: - """The noise transport source compiles only for encrypted builds.""" - old_config = CORE.config - try: - CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} - assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] - CORE.config = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) - ] - } - assert FILTER_SOURCE_FILES() == [] - finally: - CORE.config = old_config +def test_static_encryption_key() -> None: + """Only a build-time key counts; a runtime provisioned one does not.""" + assert static_encryption_key({}) is None + assert static_encryption_key({CONF_ENCRYPTION: {}}) is None + assert static_encryption_key({CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) == API_KEY + + +@pytest.mark.parametrize( + ("yaml_name", "defines_present", "defines_absent"), + [ + # An api key alone compiles the transport in without requiring it; + # the device uses the api server's key, not a copy + ( + "api_key_offer", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A password still guards plaintext uploads on an offering device + ( + "api_key_offer_password", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # The ota encryption block is what makes the device refuse plaintext + ( + "encryption_required", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + }, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # Without api encryption the ota key is the device's own + ( + "own_key", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, + {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A key provisioned at runtime lives in the api server; the device + # offers with it once provisioned and never requires it + ( + "runtime_api_key", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + {"USE_OTA_ENCRYPTION_REQUIRED"}, + ), + # No api encryption at all keeps the noise glue out of the build + ( + "plain", + set(), + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + ), + ], +) +def test_encryption_offer_codegen( + generate_main: Callable[[str], str], + yaml_name: str, + defines_present: set[str], + defines_absent: set[str], +) -> None: + main_cpp = generate_main( + f"tests/component_tests/ota/test_esphome_ota_{yaml_name}.yaml" + ) + defines = {define.name for define in CORE.defines} + assert defines_present <= defines + assert not (defines_absent & defines) + encrypted = "USE_OTA_ENCRYPTION" in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) + # The noise transport source compiles only when the define is set + assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) def test_password_with_encryption_rejected() -> None: diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml new file mode 100644 index 0000000000..ca26eb9f46 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-offer + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml new file mode 100644 index 0000000000..1e23975690 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-offer-password + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml new file mode 100644 index 0000000000..36690038d8 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-encryption-required + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + encryption: diff --git a/tests/component_tests/ota/test_esphome_ota_own_key.yaml b/tests/component_tests/ota/test_esphome_ota_own_key.yaml new file mode 100644 index 0000000000..b6d1e4200d --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_own_key.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-own-key + +host: + +api: + +ota: + - platform: esphome + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/component_tests/ota/test_esphome_ota_plain.yaml b/tests/component_tests/ota/test_esphome_ota_plain.yaml new file mode 100644 index 0000000000..c5ca7afcf0 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_plain.yaml @@ -0,0 +1,9 @@ +esphome: + name: ota-plain + +host: + +api: + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml new file mode 100644 index 0000000000..8825335141 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: ota-runtime-key + +host: + +api: + encryption: + +ota: + - platform: esphome diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index d879a26c43..f2081f2965 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -68,6 +68,14 @@ class Initiator { static const uint8_t PROLOGUE[] = {'t', 'e', 's', 't', 'p', 'r', 'o', 'l', 'o', 'g', 'u', 'e'}; +// The context only points at the key and init() copies it before returning, +// so a temporary context over a temporary key is safe within one call +static NoiseContext ctx_for(const psk_t &psk) { + NoiseContext ctx; + ctx.set_psk(psk.data()); + return ctx; +} + static psk_t make_psk(uint8_t seed) { psk_t psk; for (size_t i = 0; i < psk.size(); i++) { @@ -102,7 +110,7 @@ TEST(NoiseResponderHandshakeTest, MessageMethodsErrorBeforeInit) { TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); @@ -155,8 +163,8 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // proves the restart took effect; the old state surviving would fail the // MAC here. NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); - ASSERT_EQ(responder.init(make_psk(9), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(9)), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(make_psk(9), PROLOGUE, sizeof(PROLOGUE)); @@ -168,7 +176,7 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { TEST(NoiseResponderHandshakeTest, WrongPskFailsWithMacFailure) { NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); Initiator initiator(make_psk(200), PROLOGUE, sizeof(PROLOGUE)); uint8_t msg[MAX_HANDSHAKE_SIZE]; @@ -185,7 +193,7 @@ TEST(NoiseResponderHandshakeTest, MismatchedPrologueFailsWithMacFailure) { // tampered preamble must fail even with the right key. const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); static const uint8_t TAMPERED[] = {'x'}; Initiator initiator(psk, TAMPERED, sizeof(TAMPERED)); diff --git a/tests/components/noise/test_noise_primitives.cpp b/tests/components/noise/test_noise_primitives.cpp index 018be9f717..8687c4b963 100644 --- a/tests/components/noise/test_noise_primitives.cpp +++ b/tests/components/noise/test_noise_primitives.cpp @@ -17,12 +17,17 @@ TEST(NoiseContextTest, AllZerosPskIsReserved) { EXPECT_FALSE(NoiseContext::is_all_zeros(psk)); NoiseContext ctx; + psk_t loaded; EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(zeros); - EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, zeros); + ctx.set_psk(psk.data()); EXPECT_TRUE(ctx.has_psk()); - EXPECT_EQ(ctx.get_psk(), psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, psk); + // Callers map the reserved key to nullptr; the context just stores what it is given + ctx.set_psk(nullptr); + EXPECT_FALSE(ctx.has_psk()); } TEST(WireFormatTest, FrameHeaderIsIndicatorPlusBigEndianLength) { diff --git a/tests/components/ota/api_key_offer.yaml b/tests/components/ota/api_key_offer.yaml new file mode 100644 index 0000000000..8d1814bf7e --- /dev/null +++ b/tests/components/ota/api_key_offer.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3290 + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/components/ota/api_runtime_key.yaml b/tests/components/ota/api_runtime_key.yaml new file mode 100644 index 0000000000..8976c92f96 --- /dev/null +++ b/tests/components/ota/api_runtime_key.yaml @@ -0,0 +1,10 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + +ota: + - platform: esphome + port: 3291 diff --git a/tests/components/ota/test-api_key_offer.esp32-idf.yaml b/tests/components/ota/test-api_key_offer.esp32-idf.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_key_offer.esp8266-ard.yaml b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp32-idf.yaml b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6777e6cabc..15c5860879 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -162,6 +162,13 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) +@pytest.fixture +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Host preferences persist per device name; give the test its own so a + provisioned key never leaks into another run.""" + monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) + + @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" diff --git a/tests/integration/const.py b/tests/integration/const.py index 6876bbd443..e35d4673af 100644 --- a/tests/integration/const.py +++ b/tests/integration/const.py @@ -9,6 +9,13 @@ API_CONNECTION_TIMEOUT = 30.0 # seconds PORT_WAIT_TIMEOUT = 30.0 # seconds PORT_POLL_INTERVAL = 0.1 # seconds +# The well-known all-zeros provisioning PSK, a key to provision over it, and +# the time the device takes to activate a newly saved key (100 ms timer plus +# margin) +ZERO_PSK = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +PROVISIONING_PSK = b"bm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm4=" +KEY_ACTIVATION_DELAY = 0.5 # seconds + # Process shutdown timeouts SIGINT_TIMEOUT = 5.0 # seconds SIGTERM_TIMEOUT = 2.0 # seconds diff --git a/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml new file mode 100644 index 0000000000..1dedcc9ee1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + password: "hunter2" +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml new file mode 100644 index 0000000000..aa0a9a66c9 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index bcea2a2471..f315335d1b 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -10,34 +10,40 @@ from __future__ import annotations import asyncio import base64 +import socket from aioesphomeapi import InvalidEncryptionKeyAPIError, RequiresEncryptionAPIError import pytest -from .types import APIClientConnectedFactory, RunCompiledFunction +from .conftest import run_binary_and_wait_for_port +from .const import KEY_ACTIVATION_DELAY, LOCALHOST, PROVISIONING_PSK, ZERO_PSK +from .types import ( + APIClientConnectedFactory, + CompileFunction, + ConfigWriter, + RunCompiledFunction, +) -# The well-known provisioning PSK: base64 of 32 zero bytes -ZERO_PSK = base64.b64encode(bytes(32)).decode() -# A real key to provision -NEW_KEY = base64.b64encode(b"n" * 32) -# Time for the device to activate a newly saved key (100ms timer plus margin) -KEY_ACTIVATION_DELAY = 0.5 - - -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so every run starts unprovisioned.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) +pytestmark = pytest.mark.usefixtures("isolated_preferences") +NEW_KEY = PROVISIONING_PSK @pytest.mark.asyncio async def test_api_zero_psk_provisioning( yaml_config: str, - run_compiled: RunCompiledFunction, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], api_client_connected: APIClientConnectedFactory, ) -> None: - """Exercise the reject paths, then provision a key over the zero-PSK channel.""" - async with run_compiled(yaml_config): + """Exercise the reject paths, provision a key over the zero-PSK channel, + and check the key comes back from preferences on the next boot.""" + port, port_socket = reserved_tcp_port + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + port_socket.close() + + async with run_binary_and_wait_for_port(binary_path, LOCALHOST, port): # --- Pre-provisioning reject paths (device state is unchanged) --- # A wrong (non-zero) PSK fails against the zero provisioning PSK @@ -97,6 +103,19 @@ async def test_api_zero_psk_provisioning( async with api_client_connected(timeout=5) as client: await client.device_info() + # The key is loaded from preferences on the next boot + lines: list[str] = [] + async with run_binary_and_wait_for_port( + binary_path, LOCALHOST, port, line_callback=lines.append + ): + async with api_client_connected(noise_psk=NEW_KEY.decode()) as client: + device_info = await client.device_info() + assert device_info.api_encryption_provisionable is False + with pytest.raises(InvalidEncryptionKeyAPIError): + async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client: + await client.device_info() + assert any("Loaded saved Noise PSK" in line for line in lines) + @pytest.mark.asyncio async def test_api_zero_psk_provisioning_plaintext( diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 4e74814534..f8c122c6e1 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -8,9 +8,12 @@ instance covers the FD_CLOEXEC path. from __future__ import annotations import asyncio +import base64 from collections.abc import Generator from contextlib import contextmanager +from dataclasses import dataclass import functools +from pathlib import Path import socket import pytest @@ -18,10 +21,18 @@ import pytest from esphome import espota2 from .conftest import run_binary, wait_and_connect_api_client -from .const import LOCALHOST, PORT_POLL_INTERVAL, PORT_WAIT_TIMEOUT -from .types import CompileFunction, ConfigWriter +from .const import ( + KEY_ACTIVATION_DELAY, + LOCALHOST, + PORT_POLL_INTERVAL, + PORT_WAIT_TIMEOUT, + PROVISIONING_PSK, + ZERO_PSK, +) +from .types import APIClientConnectedFactory, CompileFunction, ConfigWriter DEVICE_NAME = "host-ota-test" +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @contextmanager @@ -35,6 +46,14 @@ def _reserve_port() -> Generator[tuple[int, socket.socket]]: s.close() +async def _wait_for_line(lines: list[str], needle: str, timeout: float = 5.0) -> None: + """The config dump prints after every setup, a little after the api port + opens, so wait for it rather than assert on the lines seen so far.""" + async with asyncio.timeout(timeout): + while not any(needle in line for line in lines): + await asyncio.sleep(PORT_POLL_INTERVAL) + + async def _wait_for_port(host: str, port: int, timeout: float) -> None: """Poll until a TCP port accepts connections, or raise TimeoutError.""" loop = asyncio.get_running_loop() @@ -51,6 +70,102 @@ async def _wait_for_port(host: str, port: int, timeout: float) -> None: raise TimeoutError(f"Port {port} on {host} did not open within {timeout}s") +async def _build( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> tuple[int, int, Path]: + """Reserve an OTA port, compile the fixture with it, and release both + ports right before the binary is started.""" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + config_path = await write_yaml_config( + yaml_config.replace("__OTA_PORT__", str(ota_port)) + ) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + return api_port, ota_port, binary_path + + +async def _run_ota( + ota_port: int, + password: str | None, + binary_path: Path, + noise_psk: str | None, + plaintext_fallback: bool = False, +) -> int: + """espota2 is blocking; run it in the executor and return its exit code.""" + rc, _ = await asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + password, + binary_path, + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, + ), + ) + return rc + + +@dataclass +class _Device: + """A running host binary and the checks every successful OTA repeats: + a safe reboot, the api port back up, and the pid preserved by execv.""" + + api_port: int + ota_port: int + binary_path: Path + proc: asyncio.subprocess.Process | None = None + reboots: int = 0 + + def __post_init__(self) -> None: + self._rebooted = asyncio.Event() + + def on_log(self, line: str) -> None: + if "Rebooting safely" in line: + self.reboots += 1 + self._rebooted.set() + + async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: + async with asyncio.timeout(timeout): + while self.reboots < count: + self._rebooted.clear() + await self._rebooted.wait() + + async def ota( + self, + password: str | None, + noise_psk: str | None, + msg: str, + plaintext_fallback: bool = False, + ) -> None: + """Upload, then expect the re-exec with the pid preserved.""" + pid_before = self.proc.pid + expected_reboots = self.reboots + 1 + rc = await _run_ota( + self.ota_port, password, self.binary_path, noise_psk, plaintext_fallback + ) + assert rc == 0, msg + await self.wait_reboot(expected_reboots) + await _wait_for_port(LOCALHOST, self.api_port, PORT_WAIT_TIMEOUT) + assert self.proc.returncode is None, "process exited instead of execing" + assert self.proc.pid == pid_before + + async def refused_ota( + self, password: str | None, noise_psk: str | None, msg: str + ) -> None: + """Upload must fail and the device must keep running.""" + rc = await _run_ota(self.ota_port, password, self.binary_path, noise_psk) + assert rc == 1, msg + await asyncio.sleep(0.5) + assert self.proc.returncode is None, "process died on rejected OTA" + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -59,57 +174,34 @@ async def test_host_ota_self_update( reserved_tcp_port: tuple[int, socket.socket], ) -> None: """Self-OTA: upload the running binary back to itself, expect re-exec.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + staged = asyncio.Event() - loop = asyncio.get_running_loop() - ota_staged = loop.create_future() - rebooted = loop.create_future() + def on_log(line: str) -> None: + if "OTA staged at" in line: + staged.set() + dev.on_log(line) - def on_log(line: str) -> None: - if not ota_staged.done() and "OTA staged at" in line: - ota_staged.set_result(True) - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_before = await client.device_info() + assert info_before.name == DEVICE_NAME - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - async with wait_and_connect_api_client(port=api_port) as client: - info_before = await client.device_info() - assert info_before.name == DEVICE_NAME + await dev.ota(None, None, "espota2 reported failure") + assert staged.is_set() - # espota2 is blocking; run in executor. - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "espota2 reported failure" + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_after = await client.device_info() + assert info_after.name == info_before.name - await asyncio.wait_for(ota_staged, timeout=10.0) - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - - # execv preserves pid; mismatch means external respawn. - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info_after = await client.device_info() - assert info_after.name == DEVICE_NAME - assert info_after.name == info_before.name - - # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "second OTA failed -- listener leaked across execv" - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.pid == pid_before + # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). + await dev.ota(None, None, "second OTA failed -- listener leaked across execv") @pytest.mark.asyncio @@ -121,51 +213,110 @@ async def test_host_ota_encrypted( ) -> None: """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" pytest.importorskip("aioesphomeapi.noise") - noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await dev.refused_ota( + None, None, "plaintext upload to an encrypted device must fail" + ) + await dev.ota(None, API_KEY, "encrypted OTA reported failure") - loop = asyncio.get_running_loop() - rebooted = loop.create_future() - def on_log(line: str) -> None: - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) +@pytest.mark.asyncio +async def test_host_ota_api_key_offer_with_password( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + caplog: pytest.LogCaptureFixture, +) -> None: + """With only an api key the device offers encryption without requiring + it: the password still guards plaintext uploads, the key alone + authenticates an encrypted one, and until 2027.3.0 a failed encrypted + attempt falls back to plaintext.""" + pytest.importorskip("aioesphomeapi.noise") + wrong_key = base64.b64encode(b"w" * 32).decode() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "Encryption: offered") - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid + await dev.refused_ota( + None, None, "plaintext upload without the password must fail" + ) + await dev.ota( + "hunter2", None, "plaintext upload with the password must succeed" + ) + await dev.ota(None, API_KEY, "encrypted upload with the api key must succeed") - # A plaintext upload must be refused with the device unharmed - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + # Remove before 2027.3.0: a wrong key falls back to plaintext, which + # the password still guards + with caplog.at_level("WARNING", logger="esphome.espota2"): + await dev.ota( + "hunter2", + wrong_key, + "the plaintext retry with the password must succeed", + plaintext_fallback=True, ) - assert rc == 1, "plaintext upload to an encrypted device must fail" - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected plaintext OTA" + assert any("Retrying in plaintext" in r.message for r in caplog.records) + await dev.ota( + None, + API_KEY, + "the right api key encrypts without touching the fallback", + plaintext_fallback=True, + ) - # The encrypted upload goes through and the device re-execs - rc, _ = await loop.run_in_executor( - None, - functools.partial( - espota2.run_ota, - LOCALHOST, - ota_port, - None, - binary_path, - noise_psk=noise_psk, - ), - ) - assert rc == 0, "encrypted OTA reported failure" - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_preferences") +async def test_host_ota_provisioned_api_key( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """A key provisioned over the api feeds the OTA offer: plaintext works + while unprovisioned, the provisioned key encrypts, the key loaded from + preferences on the next boot keeps encrypting, and plaintext stays + accepted because only the ota block requires encryption.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "once the api key is provisioned") + + await dev.ota( + None, None, "plaintext upload to an unprovisioned device must succeed" + ) + + async with api_client_connected( + port=dev.api_port, noise_psk=ZERO_PSK + ) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + key = PROVISIONING_PSK.decode() + await dev.ota( + None, key, "encrypted upload with the provisioned key must succeed" + ) + await dev.ota(None, key, "the key loaded at boot must feed the OTA offer") + await dev.ota(None, None, "plaintext must stay accepted on an offering device") @pytest.mark.asyncio @@ -177,33 +328,25 @@ async def test_host_ota_rejects_garbage( integration_test_dir, ) -> None: """Bogus payload is rejected and the device keeps running.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # 192 bytes that are neither ELF nor Mach-O. + bogus_path = integration_test_dir / "bogus.bin" + bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) - # 192 bytes that are neither ELF nor Mach-O. - bogus_path = integration_test_dir / "bogus.bin" - bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) + async with run_binary(dev.binary_path) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + rc = await _run_ota(dev.ota_port, None, bogus_path, None) + assert rc == 1 + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected OTA" + assert proc.pid == pid_before - api_socket.close() - ota_socket.close() - - async with run_binary(binary_path) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - - loop = asyncio.get_running_loop() - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, bogus_path - ) - assert rc == 1 - - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected OTA" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info = await client.device_info() - assert info.name == DEVICE_NAME + async with wait_and_connect_api_client(port=dev.api_port) as client: + info = await client.device_info() + assert info.name == DEVICE_NAME diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py index 5b43d05530..439220f09c 100644 --- a/tests/unit_tests/test_espota2_noise.py +++ b/tests/unit_tests/test_espota2_noise.py @@ -10,12 +10,15 @@ when the installed aioesphomeapi predates the noise module. from __future__ import annotations import base64 +from collections.abc import Callable import hashlib import io +import logging from pathlib import Path import socket import sys import threading +from typing import Any from unittest.mock import Mock, patch import pytest @@ -65,8 +68,12 @@ class FakeEncryptedDevice(threading.Thread): offer_noise: bool = True, require_noise: bool = True, prologue_features_override: int | None = None, + connections: int = 1, + drop_handshakes: int = 0, ) -> None: super().__init__(daemon=True) + self.connections = connections + self.drop_handshakes = drop_handshakes # hang up mid-handshake this many times self.psk = psk self.version = version self.offer_noise = offer_noise @@ -81,10 +88,11 @@ class FakeEncryptedDevice(threading.Thread): def run(self) -> None: try: - sock, _ = self.listener.accept() - sock.settimeout(10) - with sock: - self._serve(sock) + for _ in range(self.connections): + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) except Exception as err: # noqa: BLE001 - surfaced via join_and_check self.error = err finally: @@ -109,8 +117,23 @@ class FakeEncryptedDevice(threading.Thread): return server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) - if not (self.offer_noise and noise_negotiated): - return # the client fails closed; nothing further arrives + if not (noise_negotiated and self.offer_noise): + # A device that does not require encryption continues in + # plaintext whatever the client asked for, like older firmware + try: + self._transfer( + lambda byte: sock.sendall(bytes([byte])), + lambda length: _recv_exact(sock, length), + lambda remaining: _recv_exact( + sock, min(remaining, espota2.UPLOAD_BLOCK_SIZE) + ), + ) + except ConnectionError: + # A keyed client without fallback fails closed and hangs up + if noise_negotiated and not self.offer_noise: + return + raise + return from cryptography.exceptions import InvalidTag from noise.connection import NoiseConnection @@ -134,6 +157,9 @@ class FakeEncryptedDevice(threading.Thread): msg1 = _recv_frame(sock) assert msg1[0] == 0x00 + if self.drop_handshakes > 0: + self.drop_handshakes -= 1 + return # a transport fault: the socket closes with no reply try: proto.read_message(msg1[1:]) except InvalidTag: @@ -149,6 +175,20 @@ class FakeEncryptedDevice(threading.Thread): assert len(plaintext) == length, "control units must be one per frame" return plaintext + def recv_data(_remaining: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + return plaintext + + self._transfer(send_byte, recv_unit, recv_data) + + def _transfer( + self, + send_byte: Callable[[int], None], + recv_unit: Callable[[int], bytes], + recv_data: Callable[[int], bytes], + ) -> None: + """The post-handshake exchange, identical over both transports.""" send_byte(espota2.RESPONSE_AUTH_OK) recv_unit(1) # ota type size = int.from_bytes(recv_unit(4), "big") @@ -159,9 +199,7 @@ class FakeEncryptedDevice(threading.Thread): received = b"" acked = 0 while len(received) < size: - plaintext = proto.decrypt(_recv_frame(sock)) - assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT - received += plaintext + received += recv_data(size - len(received)) if self.version >= espota2.OTA_VERSION_2_0: while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( len(received) == size and acked < size @@ -176,7 +214,10 @@ class FakeEncryptedDevice(threading.Thread): def _upload( - device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None + device: FakeEncryptedDevice, + firmware: bytes, + noise_psk: str | None, + plaintext_fallback: bool = False, ) -> None: device.start() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -184,12 +225,35 @@ def _upload( sock.connect(("127.0.0.1", device.port)) try: espota2.perform_ota( - sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + sock, + None, + io.BytesIO(firmware), + Path("firmware.bin"), + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, ) finally: sock.close() +def _run_ota( + device: FakeEncryptedDevice, firmware: bytes, tmp_path: Path, noise_psk: str +) -> int: + """Drive the retry loop, which is where the plaintext fallback reconnects.""" + path = tmp_path / "firmware.bin" + path.write_bytes(firmware) + device.start() + rc, _ = espota2.run_ota( + "127.0.0.1", + device.port, + None, + path, + noise_psk=noise_psk, + plaintext_fallback=True, + ) + return rc + + def test_encrypted_upload_success() -> None: """A full encrypted v2 upload spanning several 8192-byte blocks.""" pytest.importorskip("aioesphomeapi.noise") @@ -240,6 +304,56 @@ def test_client_fails_closed_when_device_lacks_encryption() -> None: device.join_and_check() +# Remove before 2027.3.0 +def test_fallback_when_device_does_not_offer(caplog: pytest.LogCaptureFixture) -> None: + """The api key is tried opportunistically; an older device that cannot + encrypt still gets its update, with a warning.""" + firmware = b"firmware" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + _upload(device, firmware, PSK, plaintext_fallback=True) + device.join_and_check() + assert device.received == firmware + assert any("fallback is removed in 2027.3.0" in r.message for r in caplog.records) + + +# Remove before 2027.3.0 +@pytest.mark.parametrize( + ("device_kwargs", "expected_rc", "fell_back"), + [ + # A wrong key against an offering device reconnects in plaintext + ({"psk": OTHER_PSK, "require_noise": False, "connections": 2}, 0, True), + # The plaintext retry is refused by a device that requires encryption + ({"psk": OTHER_PSK, "require_noise": True, "connections": 2}, 1, True), + # A dropped connection inside the handshake is retried encrypted + ({"require_noise": False, "connections": 2, "drop_handshakes": 1}, 0, False), + # A second transport fault inside the handshake falls back + ({"require_noise": False, "connections": 3, "drop_handshakes": 2}, 0, True), + ], + ids=["wrong_key", "wrong_key_required", "one_fault", "two_faults"], +) +def test_fallback_through_the_retry_loop( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, + device_kwargs: dict[str, Any], + expected_rc: int, + fell_back: bool, +) -> None: + pytest.importorskip("aioesphomeapi.noise") + firmware = b"firmware" + device = FakeEncryptedDevice(**device_kwargs) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + rc = _run_ota(device, firmware, tmp_path, PSK) + device.join_and_check() + assert rc == expected_rc + assert (device.received == firmware) is (expected_rc == 0) + assert ( + any("Retrying in plaintext" in r.message for r in caplog.records) is fell_back + ) + if expected_rc == 1: + assert any("requires an encrypted OTA" in r.message for r in caplog.records) + + def test_plaintext_client_gets_encryption_required_error() -> None: """A client without a key gets the device's 0x94 error message.""" device = FakeEncryptedDevice() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 5372a7203d..8fb9b7376e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -2108,7 +2108,13 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + "secret", + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2140,10 +2146,77 @@ def test_upload_program_ota_encryption_key( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=False, ) +def test_upload_program_ota_api_key_opportunistic( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Without an ota encryption block the api key is tried with a plaintext + fallback (removed in 2027.3.0).""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: key}}, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=True, + ) + + +@pytest.mark.parametrize( + "api_conf", + [{}, {CONF_ENCRYPTION: {}}], + ids=["no_encryption", "runtime_key"], +) +def test_upload_program_ota_no_usable_api_key_stays_plaintext( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + api_conf: dict[str, Any], +) -> None: + """A missing or runtime provisioned api key gives the uploader nothing + to try.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_API: api_conf, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert mock_run_ota.call_args.args[5] is None + assert mock_run_ota.call_args.kwargs == {"plaintext_fallback": False} + + def test_upload_program_ota_encryption_without_key_fails_closed( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2194,7 +2267,13 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + Path("custom.bin"), + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2250,6 +2329,7 @@ def test_upload_program_ota_partition_table_with_file_arg( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2312,6 +2392,7 @@ def test_upload_program_ota_partition_table_mqttip( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2500,6 +2581,7 @@ def test_upload_program_ota_bootloader_with_file_arg( bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, None, + plaintext_fallback=False, ) @@ -2988,7 +3070,13 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -3038,7 +3126,13 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.50"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5211,6 +5305,7 @@ def test_upload_program_ota_static_ip_with_mqttip( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5261,6 +5356,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5438,7 +5534,13 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) diff --git a/tests/unit_tests/test_wizard.py b/tests/unit_tests/test_wizard.py index 244e4eb5a1..f57ae71ae6 100644 --- a/tests/unit_tests/test_wizard.py +++ b/tests/unit_tests/test_wizard.py @@ -37,7 +37,6 @@ def wizard_answers() -> list[str]: "nodemcuv2", # board "SSID", # ssid "psk", # wifi password - "", # ota password (empty for no password) ] @@ -101,6 +100,25 @@ def test_config_file_should_include_ota(default_config: dict[str, Any]): assert "ota:" in config +def test_config_file_should_use_encryption_when_api_key_set( + default_config: dict[str, Any], +): + """ + With an API encryption key and no OTA password the OTA block reuses the key + """ + # Given + default_config["api_encryption_key"] = ( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + ) + + # When + config = wz.wizard_file(**default_config) + + # Then + assert "ota:\n - platform: esphome\n encryption:" in config + assert "password" not in config.split("ota:")[1].split("wifi:")[0] + + def test_config_file_should_include_ota_when_password_set( default_config: dict[str, Any], ): @@ -630,15 +648,15 @@ def test_wizard_write_protects_existing_config( assert config_file.read_text() == original_content -def test_wizard_accepts_ota_password( +def test_wizard_uses_the_api_key_for_ota( tmp_path: Path, monkeypatch: MonkeyPatch, wizard_answers: list[str] ): """ - The wizard should pass ota_password to wizard_write when the user provides one + The wizard generates an api key and does not ask for an OTA password; + the key secures OTA updates """ # Given - wizard_answers[5] = "my_ota_password" # Set OTA password config_file = tmp_path / "test.yaml" input_mock = MagicMock(side_effect=wizard_answers) monkeypatch.setattr("builtins.input", input_mock) @@ -653,8 +671,9 @@ def test_wizard_accepts_ota_password( # Then assert retval == 0 call_kwargs = wizard_write_mock.call_args.kwargs - assert "ota_password" in call_kwargs - assert call_kwargs["ota_password"] == "my_ota_password" + assert "api_encryption_key" in call_kwargs + assert "ota_password" not in call_kwargs + assert input_mock.call_count == len(wizard_answers) def test_wizard_accepts_rpipico_board(tmp_path: Path, monkeypatch: MonkeyPatch): From 9c00f13606886643a5e9ff8195a08621dd10e9af Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:05:16 +0000 Subject: [PATCH 025/178] Bump bundled esphome-device-builder to 1.14.4 (#19006) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e875851bfb..da76ab7b6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 RUN \ platformio settings set enable_telemetry No \ From 688af60cbfa4289fb3e13b0284622d34dbf77366 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:12:52 +0200 Subject: [PATCH 026/178] [noise] Bump noise-c to 0.1.24 and libsodium to 1.10021.6 (#18989) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index a1d9444fc0..4de706120e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.21") + cg.add_library("esphome/noise-c", "0.1.24") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.4") + cg.add_library("esphome/libsodium", "1.10021.6") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index fcf7caa7c7..779a05e7de 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.21 ; used by noise (api, ota) + esphome/noise-c@0.1.24 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index a263d7937f..4f7f5a4a4c 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.21") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.21") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.21"] + assert libs == ["esphome/noise-c @ 0.1.24"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.21", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.24", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.21"] + assert cls.calls == ["esphome/noise-c @ 0.1.24"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.21"] is None + assert compats["esphome/noise-c @ 0.1.24"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.21"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 379ef52ebd..fb79885736 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1576,7 +1576,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1596,7 +1596,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 20c7dcb1ddf6a70aaf75ff418499835c3e99228e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:26:44 +0200 Subject: [PATCH 027/178] [mdns] Guard LEAmDNS main loop calls against lwIP re-entrancy on ESP8266 (#18990) --- esphome/components/mdns/__init__.py | 2 + esphome/components/mdns/mdns_esp8266.cpp | 51 +++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index f039bb69f0..c8020104b3 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -192,6 +192,8 @@ async def to_code(config: ConfigType) -> None: if CORE.using_arduino: if CORE.is_esp8266: cg.add_library("ESP8266mDNS", None) + # No MDNS global in the build; mdns_esp8266.cpp owns a guarded MDNSResponder + cg.add_build_flag("-DNO_GLOBAL_MDNS") elif CORE.is_rp2: cg.add_library("LEAmDNS", None) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index 1f0b3c9519..0e600d3bac 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -13,8 +13,47 @@ namespace esphome::mdns { +// Main-loop calls into LEAmDNS that send (update() and close(); begin(), addService() and +// the scheduled restart never reach a send) can yield inside UdpContext::sendTimeout(); a +// packet arriving then re-enters LEAmDNS from lwIP on the same UdpContext and both sides +// free the same tx pbufs (#18760). Received packets stay queued during such a call and are +// processed from the main loop afterwards. +class GuardedMDNSResponder : public ::esp8266::MDNSImplementation::MDNSResponder { + public: + void update_guarded() { this->run_guarded_(&GuardedMDNSResponder::update); } + void close_guarded() { this->run_guarded_(&GuardedMDNSResponder::close); } + + private: + void run_guarded_(bool (GuardedMDNSResponder::*fn)()) { + UdpContext *ctx = this->m_pUDPContext; + if (ctx == nullptr) { + (this->*fn)(); + return; + } + // Set every time: a restart replaces the context together with its stock handler. Only + // begin() and the scheduled netif callback restart, never update() or close(), so the + // context cannot change underneath this call. + ctx->onRx([this]() { + if (!this->in_loop_call_) { + this->_callProcess(); + } + }); + this->in_loop_call_ = true; + (this->*fn)(); + // close() releases the context; a yield in here queues further packets for this loop too + while (this->m_pUDPContext != nullptr && this->m_pUDPContext->next()) { + this->_parseMessage(); + } + this->in_loop_call_ = false; + } + + volatile bool in_loop_call_{false}; +}; + +static GuardedMDNSResponder mdns_responder; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + static void register_esp8266(MDNSComponent *, StaticVector &services) { - MDNS.begin(App.get_name().c_str()); + mdns_responder.begin(App.get_name().c_str()); for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is @@ -30,10 +69,10 @@ static void register_esp8266(MDNSComponent *, StaticVectoris_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) return; #endif - MDNS.update(); + mdns_responder.update_guarded(); }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } @@ -81,7 +120,7 @@ void MDNSComponent::on_ip_state(const network::IPAddresses &ips, const network:: #endif void MDNSComponent::on_shutdown() { - MDNS.close(); + mdns_responder.close_guarded(); delay(10); } From 8966567be072926e211b1ca717b7b1d628be7b15 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:28:31 +0200 Subject: [PATCH 028/178] [core] Show the other downloader's progress while a prefetch job waits on its lock (#18983) --- esphome/framework_helpers.py | 61 +++++++++++++- esphome/platformio/prefetch.py | 87 ++++++++++---------- esphome/platformio/registry.py | 67 ++++++++++----- tests/unit_tests/conftest.py | 39 ++++++++- tests/unit_tests/test_framework_helpers.py | 17 ++++ tests/unit_tests/test_platformio_prefetch.py | 87 ++++++++++++++++++-- tests/unit_tests/test_platformio_registry.py | 73 ++++++++++++++-- 7 files changed, 348 insertions(+), 83 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 82bc0d3727..fc2a18a6ec 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -23,6 +23,7 @@ from esphome.net_retry import ( ) if TYPE_CHECKING: + from filelock import FileLock import requests PathType = str | os.PathLike @@ -909,6 +910,61 @@ def _part_path(dest: Path) -> Path: return dest.with_name(dest.name + ".part") +def downloaded_bytes(dest: Path, size: int | None = None) -> int: + """Bytes of ``dest`` on disk (its ``.part`` while streaming), capped at ``size``.""" + done = 0 + for candidate in (_part_path(dest), dest): + try: + done = candidate.stat().st_size + break + except FileNotFoundError: + continue + return done if size is None else min(done, size) + + +# Short lock-acquire slices so a waiting worker still observes Ctrl-C +_DOWNLOAD_LOCK_POLL = 1 + +# Waiting on another process's download; past this the caller leaves the +# file to its holder (the later sequential install waits on the same lock) +DOWNLOAD_LOCK_TIMEOUT = 60 + + +class DownloadLockUnavailable(OSError): + """The lock file cannot be used at all (a lock-less filesystem).""" + + +def wait_for_download_lock( + lock: "FileLock", + tracker: Callable[[int], None], + on_disk: Callable[[], int], + name: str, +) -> None: + """Acquire ``lock``, reporting ``on_disk()`` to ``tracker`` each poll so the + bar follows the holder's download. Raises filelock's ``Timeout`` once + ``DOWNLOAD_LOCK_TIMEOUT`` seconds pass.""" + from filelock import Timeout + + deadline = time.monotonic() + DOWNLOAD_LOCK_TIMEOUT + waiting = False + while True: + try: + lock.acquire(timeout=_DOWNLOAD_LOCK_POLL) + return + except Timeout: + pass + except OSError as err: + # Distinct from an OSError out of on_disk(), which must not + # read as "locks unsupported" + raise DownloadLockUnavailable(*err.args) from err + if not waiting: + waiting = True + _LOGGER.info("Waiting for another process downloading %s", name) + tracker(on_disk()) # raises when the batch is cancelled + if time.monotonic() >= deadline: + raise Timeout(lock.lock_file) + + def discard_partial_download(dest: Path) -> None: """Remove ``dest`` and the resume sidecars of an abandoned download.""" part = _part_path(dest) @@ -1319,10 +1375,7 @@ def download_from_mirrors( ) # Tick with the bytes already on disk so a combined bar holds # steady during the backoff instead of rewinding to zero - done = 0 - if progress is not None: - part = _part_path(path_target) - done = part.stat().st_size if part.is_file() else 0 + done = downloaded_bytes(path_target) if progress is not None else 0 _cancellable_sleep(delay, progress, done) # 3. Report every attempted URL if all mirrors failed. failures spans diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 5097239065..17a06cb9c1 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -33,11 +33,14 @@ import time from typing import Any, NamedTuple from esphome.framework_helpers import ( + DownloadLockUnavailable, content_length, discard_partial_download, + downloaded_bytes, failure_reason, resume_fetch_job, run_batch_downloads, + wait_for_download_lock, warn_prefetch_failures, ) from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree @@ -61,16 +64,10 @@ _RESOLVE_WORKERS = 8 # A hung child must not block the build; downloads resume on the next run _PREFETCH_TIMEOUT = 20 * 60 -# Waiting on another process's URL download; past this, leave it to pio -_DOWNLOAD_LOCK_TIMEOUT = 60 - # Child exit for a handled, already-warned failure; 1 would collide with # the interpreter's own import-failure exit _EXIT_HANDLED = 3 -# Short lock-acquire slices so a waiting worker still observes Ctrl-C -_URI_LOCK_POLL = 1 - # Resolution errored (vs a clean skip); suppresses the warm sentinel _RESOLVE_FAILED = object() @@ -462,51 +459,54 @@ def _uri_jobs( def _serialized_fetch_job( - dl_path: Path, lock_path: str, body: Any, unlocked_ok: bool = True + dl_path: Path, + lock_path: str, + body: Any, + size: int, + stream_dest: Path | None = None, + unlocked_ok: bool = True, ) -> Any: - """Wrap ``body`` so the shared destination is single-writer. - - Interleaved writers truncate each other's ``.part`` bytes (see - registry.py). The bounded poll observes Ctrl-C via the tracker; a - blown deadline is a clean skip (the holder's copy is what the build - needs). On a lock-less filesystem a sha256-verified body runs - unlocked with one warning; a checksum-less one - (``unlocked_ok=False``) is a counted failure instead. + """Wrap ``body`` so the shared destination is single-writer (interleaved + writers truncate each other's ``.part``, see registry.py). A blown deadline + is a clean skip. On a lock-less filesystem a sha256-verified body runs + unlocked with one warning; a checksum-less one (``unlocked_ok=False``) fails. """ + def on_disk() -> int: + # A URL job's holder streams beside the staging path until it + # promotes; after that only dl_path is left + done = downloaded_bytes(dl_path, size) + if not done and stream_dest is not None: + done = downloaded_bytes(stream_dest, size) + return done + def run(tracker: Any) -> None: from filelock import FileLock, Timeout # fallback_to_soft would leave a stale marker on lock-less # filesystems that blocks every later build (see git.py) lock = FileLock(lock_path, fallback_to_soft=False) - deadline = time.monotonic() + _DOWNLOAD_LOCK_TIMEOUT - while True: - try: - lock.acquire(timeout=_URI_LOCK_POLL) - break - except Timeout: - tracker(0) # raises when the batch is cancelled - if time.monotonic() >= deadline: - # Another process is fetching this same file; its copy - # is what the build needs (a large framework archive - # can hold the lock far longer than this deadline) - _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) - return - except OSError as err: - if not unlocked_ok: - # A body with no checksum to catch interleaved corruption - raise - lock = None - _LOGGER.warning( - "Could not lock %s (%s); downloading unlocked", - dl_path.name, - err, - ) - break + try: + wait_for_download_lock(lock, tracker, on_disk, dl_path.name) + except Timeout: + # The holder's copy is what the build needs (a large + # framework archive can outlast this deadline) + _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) + return + except DownloadLockUnavailable as err: + if not unlocked_ok: + # A body with no checksum to catch interleaved corruption + raise + lock = None + _LOGGER.warning( + "Could not lock %s (%s); downloading unlocked", + dl_path.name, + err, + ) try: if dl_path.is_file(): - return # another process finished it while we waited + tracker(size) # another process finished it while we waited + return body(tracker) finally: if lock is not None: @@ -540,6 +540,7 @@ def _registry_fetch_job( dl_path, f"{dl_path}.esphome.lock", resume_fetch_job(url, dl_path, sha256=checksum, size=size), + size, ) def run(tracker: Any) -> None: @@ -571,9 +572,9 @@ def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: tmp.replace(dl_path) def run(tracker: Any) -> None: - _serialized_fetch_job(dl_path, f"{tmp}.lock", promote, unlocked_ok=False)( - tracker - ) + _serialized_fetch_job( + dl_path, f"{tmp}.lock", promote, size, tmp, unlocked_ok=False + )(tracker) if dl_path.is_file(): # Won or lost, the race is over; staging files left behind # are dead weight PlatformIO's cache never prunes diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index 9538a28ff4..75df82da0e 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -17,8 +17,10 @@ from esphome.framework_helpers import ( archive_extract_all, download_from_mirrors, download_with_resume, + downloaded_bytes, rmdir, run_batch_downloads, + wait_for_download_lock, ) from esphome.net_retry import fetch_with_retry, http_request @@ -164,11 +166,17 @@ class _PendingArchive(NamedTuple): name: str version: str dest: Path + archive: Path url: str sha256: str size: int +def _archive_path(downloads_dir: Path, name: str, version: str) -> Path: + """The one archive path the prefetch and the sequential install share.""" + return downloads_dir / f"{name}-{version}" + + def _already_installed(dest: Path) -> bool: """Whether ``dest`` holds a completed install (extraction marker).""" return (dest / ".esphome_extracted").is_file() @@ -187,18 +195,18 @@ def prefetch_packages( lock as ``install_package``: the archive's ``.part`` file is shared, and two concurrent writers would truncate each other's bytes. """ - from filelock import FileLock + from filelock import FileLock, Timeout pending: list[_PendingArchive] = [] - seen: set[str] = set() + seen: set[Path] = set() for name, version, dest, mirrors in packages: if mirrors or (dest / ".esphome_extracted").is_file(): continue - archive_name = f"{name}-{version}" - if archive_name in seen: + archive = _archive_path(downloads_dir, name, version) + if archive in seen: # A duplicate entry would race itself between two workers continue - seen.add(archive_name) + seen.add(archive) try: url, sha256, size = registry_download(name, version) except EsphomeError as err: @@ -207,10 +215,9 @@ def prefetch_packages( continue if not size: continue - archive = downloads_dir / archive_name if archive.is_file() and archive.stat().st_size == size: continue - pending.append(_PendingArchive(name, version, dest, url, sha256, size)) + pending.append(_PendingArchive(name, version, dest, archive, url, sha256, size)) if len(pending) < 2: return downloads_dir.mkdir(parents=True, exist_ok=True) @@ -222,20 +229,36 @@ def prefetch_packages( def _fetch(entry: _PendingArchive, tracker: Callable[[int], None]) -> None: entry.dest.parent.mkdir(parents=True, exist_ok=True) - with FileLock(f"{entry.dest}.lock", fallback_to_soft=False): - # Marker re-check: a concurrent build may have installed (and - # deleted the archive of) this package while we waited; - # re-downloading would orphan a fresh copy in downloads_dir - # no branch: the thread tracer misses the skip edge; both - # arms of _already_installed are pinned directly - if not _already_installed(entry.dest): # pragma: no branch - download_with_resume( - entry.url, - downloads_dir / f"{entry.name}-{entry.version}", - sha256=entry.sha256, - size=entry.size, - progress=tracker, - ) + + def on_disk() -> int: + if done := downloaded_bytes(entry.archive, entry.size): + return done + # The holder deletes the archive once it has installed it + return entry.size if _already_installed(entry.dest) else 0 + + lock = FileLock(f"{entry.dest}.lock", fallback_to_soft=False) + try: + wait_for_download_lock(lock, tracker, on_disk, entry.name) + except Timeout: + # install_package waits on this same lock and verifies the + # holder's copy + _LOGGER.debug("Leaving %s to its current downloader", entry.name) + return + try: + if _already_installed(entry.dest): + # A concurrent build installed it while we waited; a + # re-download would orphan a fresh copy in downloads_dir + tracker(entry.size) + return + download_with_resume( + entry.url, + entry.archive, + sha256=entry.sha256, + size=entry.size, + progress=tracker, + ) + finally: + lock.release() failures = run_batch_downloads( "Downloading packages", @@ -288,7 +311,7 @@ def install_package( rmdir(dest, msg=f"Clean up incomplete {name} install") # Persistent location so an interrupted download resumes across runs. downloads_dir.mkdir(parents=True, exist_ok=True) - archive = downloads_dir / f"{name}-{version}" + archive = _archive_path(downloads_dir, name, version) _LOGGER.info("Downloading %s %s ...", name, version) if mirrors: _LOGGER.warning( diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 9de8f715ef..ad9c0bb11f 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -9,7 +9,7 @@ not be part of a unit test suite. """ -from collections.abc import Generator +from collections.abc import Callable, Generator import os from pathlib import Path import sys @@ -137,3 +137,40 @@ def mock_get_component() -> Generator[Mock, None, None]: """Mock get_component for config module.""" with patch("esphome.config.get_component") as mock: yield mock + + +@pytest.fixture +def held_lock() -> Callable[..., Callable[..., None]]: + """Factory for a ``FileLock.acquire`` fake held by another downloader. + + Each poll writes the next chunk to ``part`` (or runs it, for a callable) + and raises ``Timeout``; when the chunks run out the part is removed, + ``land()`` runs, and the acquire succeeds (also for any later job, so + ``land`` must be idempotent). + """ + from filelock import Timeout + + def make( + part: Path, + chunks: list[bytes | Callable[[], None]], + land: Callable[[], None], + ) -> Callable[..., None]: + polls = iter(chunks) + + def acquire(*args, **kwargs) -> None: + try: + chunk = next(polls) + except StopIteration: + part.unlink(missing_ok=True) + land() + return + if callable(chunk): + chunk() + else: + part.parent.mkdir(parents=True, exist_ok=True) + part.write_bytes(chunk) + raise Timeout("held") + + return acquire + + return make diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index fcc5572f51..22b34c9df5 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2353,3 +2353,20 @@ def test_discard_partial_download_logs_undeletable( ): framework_helpers.discard_partial_download(dest) assert "Could not remove" in caplog.text + + +def test_downloaded_bytes_reports_what_is_on_disk(tmp_path: Path) -> None: + """Part file first, then the landed file, both capped at size; else 0.""" + dest = tmp_path / "archive" + assert framework_helpers.downloaded_bytes(dest, 4) == 0 + part = tmp_path / "archive.part" + part.write_bytes(b"ab") + assert framework_helpers.downloaded_bytes(dest, 4) == 2 + part.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 + part.unlink() + dest.write_bytes(b"abc") + assert framework_helpers.downloaded_bytes(dest, 4) == 3 + assert framework_helpers.downloaded_bytes(dest) == 3 + dest.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index fb79885736..77490fd861 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -454,23 +454,96 @@ def test_uri_fetch_job_waits_out_a_briefly_held_lock(tmp_path: Path) -> None: assert dl_path.read_bytes() == b"data" -def test_lock_deadline_leaves_download_to_the_holder(tmp_path: Path) -> None: - """A lock held past the deadline means another process is fetching the - same file; skipping cleanly beats a misleading failure warning. The - tracker is still polled so a parked worker observes cancellation.""" +@pytest.mark.parametrize("staged", [b"", b"ab"]) +def test_lock_deadline_leaves_download_to_the_holder( + tmp_path: Path, staged: bytes +) -> None: + """A lock held past the deadline is another process's download; skip + cleanly, polling the tracker with what the holder has staged so far.""" dl_path = tmp_path / "archive" + (tmp_path / "archive.prefetch.part").write_bytes(staged) ticks: list[int] = [] with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) mock_download.assert_not_called() - assert ticks == [0] + assert ticks == [len(staged)] assert not dl_path.exists() +@pytest.mark.parametrize( + ("job", "part_name", "chunks", "expected"), + [ + ( + lambda dl_path: pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + ), + "archive.part", + [b"a", b"abc"], + [1, 3, 4], + ), + ( + lambda dl_path: pf._uri_fetch_job( + MagicMock(), "https://x/a.zip", dl_path, 4 + ), + "archive.prefetch.part", + [b"ab"], + [2, 4], + ), + ], + ids=["registry", "uri"], +) +def test_lock_wait_reports_the_holders_progress( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + held_lock, + job, + part_name: str, + chunks: list[bytes], + expected: list[int], +) -> None: + """A waiting job reports the holder's part file (the staging one for a + URL job), then the full size once the holder lands the archive.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / part_name, chunks, lambda: dl_path.write_bytes(b"abcd") + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + caplog.at_level(logging.INFO), + ): + job(dl_path)(ticks.append) + mock_download.assert_not_called() + assert ticks == expected + assert caplog.text.count("Waiting for another process downloading archive") == 1 + + +def test_uri_lock_wait_prefers_the_landed_archive(tmp_path: Path, held_lock) -> None: + """Between the holder's promotion rename and its release the staging + part is gone; the landed cache file is credited instead of 0.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / "archive.prefetch.part", + [b"ab", lambda: dl_path.write_bytes(b"abcd")], + lambda: None, + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) + mock_download.assert_not_called() + assert ticks == [2, 4, 4] + + def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: """A registry job that lost the download race to another process must not stamp a nonexistent archive into pio's usage.db.""" @@ -479,7 +552,7 @@ def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( lambda done: None diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 6ba8691c4e..9d5f6c4ce5 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -8,6 +8,7 @@ import os from pathlib import Path from unittest.mock import MagicMock, patch +from filelock import Timeout import pytest from esphome.core import EsphomeError @@ -540,16 +541,13 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: dest = tmp_path / "a" dest.mkdir() - from contextlib import contextmanager - - @contextmanager - def marker_appears_under_lock(path, **kwargs): + def marker_appears_under_lock(*args, **kwargs): # Simulates the concurrent build finishing while we waited (dest / ".esphome_extracted").touch() - yield with ( - patch("filelock.FileLock", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.acquire", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.release"), patch.object(registry, "download_with_resume") as mock_download, patch.object( registry, "registry_download", side_effect=_resolve_for({"a": 10}) @@ -559,6 +557,69 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: mock_download.assert_not_called() +def test_prefetch_packages_waits_with_the_holders_progress( + tmp_path: Path, held_lock +) -> None: + """A worker parked on another build's lock reports that build's part + file, then the full size once the marker appears.""" + dest = tmp_path / "a" + dest.mkdir() + ticks: list[int] = [] + part = tmp_path / "dl" / "a-1.0.part" + + def installed_and_pruned() -> None: + # install_package touches the marker, then unlinks the archive + (dest / ".esphome_extracted").touch() + part.unlink() + + acquire = held_lock( + part, + [lambda: None, b"abc", installed_and_pruned], + (dest / ".esphome_extracted").touch, + ) + + def fake_batch(header, jobs): + for _name, _size, fetch in jobs: + fetch(ticks.append) + return [] + + with ( + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + patch.object(registry, "run_batch_downloads", side_effect=fake_batch), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", dest, []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + assert ticks == [0, 3, 10, 10] + mock_download.assert_called_once() + + +def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( + tmp_path: Path, +) -> None: + """Past the deadline the worker skips; install_package waits on the same + lock later and verifies whatever the holder produced.""" + with ( + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", tmp_path / "a", []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + def test_already_installed_probe(tmp_path: Path) -> None: """Both arms of the marker probe the prefetch worker keys on.""" dest = tmp_path / "pkg" From d58b37faa1eff3324dd6c9389c864d5c3576eadf Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:29:20 +1200 Subject: [PATCH 029/178] [esp32_hosted] Add ESP-NOW-over-hosted shim for the ESP32-P4 (#17712) --- esphome/components/esp32_hosted/__init__.py | 36 ++ .../esp32_hosted/esp_now_hosted.cpp | 467 ++++++++++++++++++ .../esp32_hosted/esp_now_hosted_rpc.h | 128 +++++ esphome/components/espnow/__init__.py | 20 + esphome/core/defines.h | 1 + script/ci-custom.py | 17 +- .../test-espnow.esp32-p4-idf.yaml | 5 + tests/unit_tests/components/test_espnow.py | 48 ++ 8 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 esphome/components/esp32_hosted/esp_now_hosted.cpp create mode 100644 esphome/components/esp32_hosted/esp_now_hosted_rpc.h create mode 100644 tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml create mode 100644 tests/unit_tests/components/test_espnow.py diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index ab9455250c..21626e432b 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -37,6 +37,25 @@ CONF_HANDSHAKE_PIN = "handshake_pin" CONF_SDIO_FREQUENCY = "sdio_frequency" CONF_SPI_MODE = "spi_mode" +# ESP-NOW-over-hosted shim (esp_now_hosted.cpp). esp-hosted proxies esp_wifi.h +# but not esp_now.h (espressif/esp-hosted-mcu#19), and esp_wifi_remote injects +# the esp_now.h header on the ESP32-P4 host with no implementation, leaving the +# esp_now_* symbols undefined at link. On a P4 host, esp_now_hosted.cpp DEFINES +# those symbols and forwards each call to the co-processor over esp-hosted's +# CustomRpc "peer data transfer" channel, so ESPHome's `espnow` component links +# and runs unchanged (proven on a Tab5, 2026-07-20). The .cpp is guarded to +# CONFIG_IDF_TARGET_ESP32P4 so it compiles to nothing on hosts with a native +# ESP-NOW stack. CustomRpc needs these two host-side Kconfig options. Host +# registers 3 handlers (RESP, RECV, SEND); the coprocessor registers 1 (REQ); +# we ask for 8 to leave room for other CustomRpc extensions alongside. +# +# The coprocessor must run the matching custom firmware (a parallel effort in +# esphome/esp-hosted-firmware). esp_now_hosted_rpc.h here is the canonical copy +# of the wire contract and MUST stay byte-identical to the copy that coprocessor +# firmware uses — the packed structs are the on-wire layout, so any divergence +# silently corrupts every ESP-NOW frame. +_MAX_CUSTOM_MSG_HANDLERS = 8 + # Shared fields for both transport modes BASE_SCHEMA = cv.Schema( { @@ -262,6 +281,23 @@ async def to_code(config: ConfigType) -> None: else: _configure_spi(config) + # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see + # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not + # gated on the `espnow` component being present: the shim is tiny and the + # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options + # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so + # coupling the two keeps the build consistent. When `espnow` is absent the + # symbols are simply unused and never register a callback at runtime. + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + add_define("USE_ESP_NOW_HOSTED") + # esp-hosted's CustomRpc ("peer data transfer") path — off by default. + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS + ) + # Place the transport mempool in PSRAM. Required on memory-tight host # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM # mempool allocation fails at boot with `sdio_mempool_create` assert. diff --git a/esphome/components/esp32_hosted/esp_now_hosted.cpp b/esphome/components/esp32_hosted/esp_now_hosted.cpp new file mode 100644 index 0000000000..ad29b208fe --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted.cpp @@ -0,0 +1,467 @@ +/* + * esp_now_hosted — host-side shim implementing over esp-hosted + * CustomRpc, so ESPHome's `espnow` component can run on a radio-less host + * (e.g. the ESP32-P4) whose radio lives on an esp-hosted co-processor. + * + * A radio-less host has no native ESP-NOW. esp_wifi_remote INJECTS the full + * esp_now.h header (types + declarations) but ships NO implementation, so every + * esp_now_* symbol is an undefined reference at link time. This translation + * unit provides those definitions; each forwards to the co-processor over + * CustomRpc (see esphome/esp-hosted-firmware for the matching coprocessor + * handlers). No esp-hosted or esp_wifi_remote source is patched, and there is no + * duplicate-symbol clash because nothing else defines these symbols here. + * + * See esp_now_hosted_rpc.h for the wire protocol. + */ + +#include "sdkconfig.h" + +// Only build the shim on the radio-less host. On chips with a native ESP-NOW +// stack (S3, C6, …) the real symbols exist and this file must stay empty to +// avoid duplicate definitions. +#if defined(CONFIG_IDF_TARGET_ESP32P4) + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" + +#include "esp_idf_version.h" +#include "esp_log.h" +#include "esp_timer.h" + +#include // injected declarations we are now DEFINING +#include // wifi_pkt_rx_ctrl_t, wifi_tx_info_t + +// esp_hosted_misc.h (host) ships WITHOUT an extern "C" guard, so including it +// from C++ would give its declarations C++ linkage and the real C symbols in +// libesp_hosted would go unresolved at link. Wrap it. (Verified vs +// esp_hosted 2.12.9.) +extern "C" { +#include "esp_hosted_misc.h" // esp_hosted_{send_custom_data,register_custom_callback} +} + +#include "esp_now_hosted_rpc.h" + +namespace { + +const char *const TAG = "esp_now_hosted"; + +// One outstanding request at a time. ESPHome drives esp_now_* from the main +// loop; the matching response and the async RECV/SEND events all arrive on the +// single esp-hosted RPC RX thread. Serializing requests keeps the shared +// response slot race-free; a sequence number stops a late/stale response from +// being mistaken for ours. +SemaphoreHandle_t g_req_mutex = nullptr; +SemaphoreHandle_t g_resp_sem = nullptr; // given when the matching RESP lands +bool g_setup_done = false; // set only after setup fully succeeds +uint8_t g_seq = 0; +volatile uint8_t g_expect_seq = 0; +volatile int32_t g_resp_status = 0; +uint8_t g_resp_ret[16]; +volatile uint16_t g_resp_ret_len = 0; + +// Written from the main loop (register/unregister/deinit), read from the +// esp-hosted RX thread (on_recv/on_send). volatile for the same reason the +// g_resp_* globals are: force the RX thread to observe an updated pointer +// (e.g. a nulling by esp_now_deinit) rather than a cached one. +volatile esp_now_recv_cb_t g_recv_cb = nullptr; +volatile esp_now_send_cb_t g_send_cb = nullptr; + +// Local mirror of the co-processor's peer table. ESPHome's espnow component +// calls esp_now_is_peer_exist() on the main loop for every received frame +// (twice) and every send; forwarding each as a blocking RPC round-trip stalls +// the loop. The shim is the only path that mutates the co-processor peer table +// (add/del/deinit all go through here), so this mirror is authoritative and +// esp_now_is_peer_exist() can answer from it with no round-trip. +// +// esp_now_* are public C symbols: any component or user lambda may call them, +// and although ESPHome's espnow touches peers only from the main loop today +// (its RX/TX callbacks merely enqueue), the shim cannot rely on that. A short +// spinlock keeps the mirror consistent from any task/core, matching native +// esp_now_*'s own internal thread-safety. The critical sections are a bounded +// (<=20-entry) scan, so they stay tiny. ESP_NOW_MAX_TOTAL_PEER_NUM is 20. +constexpr size_t ESP_NOW_HOSTED_MAX_PEERS = 20; +uint8_t g_peer_cache[ESP_NOW_HOSTED_MAX_PEERS][6]; +size_t g_peer_count = 0; +portMUX_TYPE g_peer_lock = portMUX_INITIALIZER_UNLOCKED; + +// Caller must hold g_peer_lock. +int peer_cache_find_locked(const uint8_t *mac) { + for (size_t i = 0; i < g_peer_count; i++) { + if (memcmp(g_peer_cache[i], mac, 6) == 0) + return static_cast(i); + } + return -1; +} + +bool peer_cache_contains(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const bool found = peer_cache_find_locked(mac) >= 0; + portEXIT_CRITICAL(&g_peer_lock); + return found; +} + +void peer_cache_add(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + if (peer_cache_find_locked(mac) < 0 && g_peer_count < ESP_NOW_HOSTED_MAX_PEERS) + memcpy(g_peer_cache[g_peer_count++], mac, 6); + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_remove(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const int idx = peer_cache_find_locked(mac); + if (idx >= 0) { + g_peer_count--; + if (static_cast(idx) != g_peer_count) // move the last entry into the gap + memcpy(g_peer_cache[idx], g_peer_cache[g_peer_count], 6); + } + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_clear() { + portENTER_CRITICAL(&g_peer_lock); + g_peer_count = 0; + portEXIT_CRITICAL(&g_peer_lock); +} + +// ── CustomRpc event handlers (run on the esp-hosted RPC RX thread) ────────── +// Keep them short and non-blocking. In particular they MUST NOT call back into +// any esp_now_* shim function: that would try to take g_req_mutex / wait on the +// RX thread that delivers the response, and deadlock. + +void on_resp(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + if (len < sizeof(esp_now_hosted_resp_t)) { + ESP_LOGW(TAG, "RESP too short: %u bytes", static_cast(len)); + return; + } + const auto *r = reinterpret_cast(data); + if (r->seq != g_expect_seq) { // late response from a timed-out request (expected) + ESP_LOGV(TAG, "dropping stale RESP seq %u (want %u)", r->seq, g_expect_seq); + return; + } + g_resp_status = r->status; + uint16_t rl = r->ret_len; + if (rl > sizeof(g_resp_ret)) { + // Larger than any real opcode return — a likely wire-format drift signal. + ESP_LOGW(TAG, "RESP ret_len %u exceeds buffer, clamping (wire drift?)", rl); + rl = sizeof(g_resp_ret); + } + if (len >= sizeof(esp_now_hosted_resp_t) + rl) { + memcpy(g_resp_ret, r->ret, rl); + } else { + // Truncated frame: fail closed. Never hand the caller stale bytes left in + // g_resp_ret by a previous response, and don't let request() report a + // zeroed payload as success — override the status to an error. + ESP_LOGW(TAG, "RESP truncated: claims %u ret bytes, frame too short", rl); + rl = 0; + g_resp_status = ESP_ERR_INVALID_RESPONSE; + } + g_resp_ret_len = rl; + xSemaphoreGive(g_resp_sem); +} + +void on_recv(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once: esp_now_unregister_recv_cb()/deinit() (via + // the espnow component's disable()) can null it on the main loop between the + // guard and the call, which would otherwise turn the call into a null-deref. + const esp_now_recv_cb_t cb = g_recv_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_recv_evt_t)) { + ESP_LOGW(TAG, "RECV too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); + if (len < sizeof(esp_now_hosted_recv_evt_t) + e->data_len) { + ESP_LOGW(TAG, "RECV data_len %u exceeds frame", e->data_len); + return; + } + + // ESPHome dereferences info->rx_ctrl->{rssi,timestamp}; give it a real one. + wifi_pkt_rx_ctrl_t rx_ctrl; + memset(&rx_ctrl, 0, sizeof(rx_ctrl)); + rx_ctrl.rssi = e->rssi; + rx_ctrl.channel = e->channel; + rx_ctrl.timestamp = static_cast(esp_timer_get_time()); + + esp_now_recv_info_t info; + info.src_addr = const_cast(e->src_addr); + info.des_addr = const_cast(e->des_addr); + info.rx_ctrl = &rx_ctrl; + cb(&info, e->data, static_cast(e->data_len)); +} + +void on_send(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once (see on_recv): disable()/deinit() can null it + // on the main loop concurrently with this RX-thread callback. + const esp_now_send_cb_t cb = g_send_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_send_evt_t)) { + ESP_LOGW(TAG, "SEND evt too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0) + // IDF >= 5.5: esp_now_send_cb_t takes esp_now_send_info_t (== wifi_tx_info_t), + // whose des_addr is a POINTER (not an inline array). Point it at the event's + // MAC (valid for this callback) — do NOT memcpy into it (that writes NULL and + // faults). ESPHome reads only info->des_addr. + esp_now_send_info_t si; + memset(&si, 0, sizeof(si)); + si.des_addr = const_cast(e->des_addr); + cb(&si, static_cast(e->status)); +#else + cb(e->des_addr, static_cast(e->status)); +#endif +} + +esp_err_t ensure_setup() { + // Gate on g_setup_done, not on g_req_mutex: a failure part-way through (a + // semaphore that did not allocate, a callback that did not register) must not + // leave a later call thinking setup completed. Semaphore creation is guarded + // so a retry after a partial failure does not leak the earlier handles. + if (g_setup_done) + return ESP_OK; + if (g_req_mutex == nullptr) + g_req_mutex = xSemaphoreCreateMutex(); + if (g_resp_sem == nullptr) + g_resp_sem = xSemaphoreCreateBinary(); + if (g_req_mutex == nullptr || g_resp_sem == nullptr) + return ESP_ERR_NO_MEM; + esp_err_t err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RESP, on_resp, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RECV, on_recv, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_SEND, on_send, nullptr)) != ESP_OK) + return err; + g_setup_done = true; + return ESP_OK; +} + +// Send one request envelope. With wait=true (default) block until the matching +// response (or timeout); with wait=false return as soon as the frame is handed +// to the transport (fire-and-forget, used by esp_now_send). +// +// `tail` is an optional second chunk written straight after `payload`. Callers +// with a fixed header plus a bulk body (esp_now_send) pass the two separately +// so they never need a build buffer of their own: both chunks are laid into the +// request buffer here, under g_req_mutex, which keeps concurrent callers from +// racing and saves a full copy of the body on every transmit. +esp_err_t request(uint8_t opcode, const void *payload, uint16_t plen, void *ret, uint16_t ret_cap, uint16_t *ret_len, + bool wait = true, const void *tail = nullptr, uint16_t tail_len = 0) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + if (plen > ESP_NOW_HOSTED_MAX_PAYLOAD || tail_len > ESP_NOW_HOSTED_MAX_PAYLOAD - plen) + return ESP_ERR_INVALID_SIZE; + const uint16_t total_len = static_cast(plen + tail_len); + + if (xSemaphoreTake(g_req_mutex, portMAX_DELAY) != pdTRUE) + return ESP_FAIL; + + static uint8_t buf[sizeof(esp_now_hosted_req_t) + ESP_NOW_HOSTED_MAX_PAYLOAD]; // guarded by g_req_mutex + auto *req = reinterpret_cast(buf); + req->opcode = opcode; + req->seq = ++g_seq; + req->payload_len = total_len; + if (plen != 0) + memcpy(req->payload, payload, plen); + if (tail_len != 0) + memcpy(req->payload + plen, tail, tail_len); + g_expect_seq = req->seq; + + xSemaphoreTake(g_resp_sem, 0); // drain any stale signal before sending + err = esp_hosted_send_custom_data(ESP_NOW_HOSTED_MSG_REQ, buf, sizeof(esp_now_hosted_req_t) + total_len); + if (err != ESP_OK) { + xSemaphoreGive(g_req_mutex); + return err; + } + if (!wait) { + // Fire-and-forget (esp_now_send): the co-processor enqueues the frame and + // reports the real TX result later via the async SEND event, exactly like + // native esp_now_send. Returning here keeps the main loop off the ~100 ms+ + // RPC round-trip. The matching RESP is ignored (seq won't match the next + // waited request, so on_resp drops it). + xSemaphoreGive(g_req_mutex); + return ESP_OK; + } + if (xSemaphoreTake(g_resp_sem, pdMS_TO_TICKS(ESP_NOW_HOSTED_TIMEOUT_MS)) != pdTRUE) { + ESP_LOGW(TAG, "opcode %u timed out", opcode); + xSemaphoreGive(g_req_mutex); + return ESP_ERR_TIMEOUT; + } + + const int32_t status = g_resp_status; + if (ret != nullptr && ret_cap != 0) { + uint16_t n = g_resp_ret_len < ret_cap ? g_resp_ret_len : ret_cap; + memcpy(ret, const_cast(g_resp_ret), n); + if (ret_len != nullptr) + *ret_len = n; + } + xSemaphoreGive(g_req_mutex); + return static_cast(status); +} + +} // namespace + +// ── The surface, defined for the radio-less host ──────────────── +extern "C" { + +esp_err_t esp_now_init(void) { return request(ESP_NOW_HOSTED_OP_INIT, nullptr, 0, nullptr, 0, nullptr); } + +esp_err_t esp_now_deinit(void) { + g_recv_cb = nullptr; + g_send_cb = nullptr; + peer_cache_clear(); // the co-processor drops all peers on deinit + return request(ESP_NOW_HOSTED_OP_DEINIT, nullptr, 0, nullptr, 0, nullptr); +} + +esp_err_t esp_now_get_version(uint32_t *version) { + uint32_t v = 0; + uint16_t rl = 0; + esp_err_t err = request(ESP_NOW_HOSTED_OP_GET_VERSION, nullptr, 0, &v, sizeof(v), &rl); + if (version != nullptr) + *version = v; + return err; +} + +esp_err_t esp_now_register_recv_cb(esp_now_recv_cb_t cb) { + // Only arm the callback once the CustomRpc handlers are actually registered, + // so a failed setup leaves g_recv_cb null rather than falsely "registered". + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_recv_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_recv_cb(void) { + g_recv_cb = nullptr; + return ESP_OK; +} +esp_err_t esp_now_register_send_cb(esp_now_send_cb_t cb) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_send_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_send_cb(void) { + g_send_cb = nullptr; + return ESP_OK; +} + +static esp_err_t add_or_mod_peer(uint8_t opcode, const esp_now_peer_info_t *peer, bool wait) { + if (peer == nullptr) + return ESP_ERR_ESPNOW_ARG; + esp_now_hosted_peer_t p; + memset(&p, 0, sizeof(p)); + memcpy(p.peer_addr, peer->peer_addr, 6); + memcpy(p.lmk, peer->lmk, 16); + p.channel = peer->channel; + p.ifidx = static_cast(peer->ifidx); + p.encrypt = peer->encrypt ? 1 : 0; + return request(opcode, &p, sizeof(p), nullptr, 0, nullptr, wait); +} +esp_err_t esp_now_add_peer(const esp_now_peer_info_t *peer) { + // Fire-and-forget (wait=false): adding a peer is a blocking RPC round-trip, + // and ESPHome's espnow calls it on the main loop when a device joins the mesh + // — under co-processor load that stalls the UI (peer-churn stutter). Issue it + // without waiting and mirror it locally. Safe against a following + // esp_now_send to the same peer: both ride the same in-order CustomRpc + // channel (mutex-serialized on the host) and the co-processor processes REQs + // FIFO, so ADD_PEER is applied before the SEND. Trade-off: a co-processor-side + // failure (e.g. peer table full) is no longer reported synchronously — the + // same limitation as esp_now_send — but ESPHome only adds peers it validated. + esp_err_t err = add_or_mod_peer(ESP_NOW_HOSTED_OP_ADD_PEER, peer, /*wait=*/false); + if (err == ESP_OK) + peer_cache_add(peer->peer_addr); // keep the local mirror in sync + return err; +} +esp_err_t esp_now_mod_peer(const esp_now_peer_info_t *peer) { + // mod_peer changes a peer's parameters, not its existence, so the cache is + // unaffected. Kept synchronous — it is not on any hot path (espnow never + // calls it), so the extra round-trip does not matter and the status is useful. + return add_or_mod_peer(ESP_NOW_HOSTED_OP_MOD_PEER, peer, /*wait=*/true); +} + +esp_err_t esp_now_del_peer(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return ESP_ERR_ESPNOW_ARG; + // Fire-and-forget for the same reason as add_peer (peer churn on the main + // loop). Removal is order-independent, so this is strictly safe. + esp_err_t err = request(ESP_NOW_HOSTED_OP_DEL_PEER, peer_addr, 6, nullptr, 0, nullptr, /*wait=*/false); + if (err == ESP_OK) + peer_cache_remove(peer_addr); // keep the local mirror in sync + return err; +} + +bool esp_now_is_peer_exist(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return false; + // Answered from the local mirror — no RPC round-trip. ESPHome's espnow calls + // this on the main loop for every received frame and every send, so a + // blocking round-trip here would stall rendering under mesh traffic. + return peer_cache_contains(peer_addr); +} + +esp_err_t esp_now_send(const uint8_t *peer_addr, const uint8_t *data, size_t len) { + if (len > ESP_NOW_HOSTED_MAX_FRAME) + return ESP_ERR_ESPNOW_ARG; + if (data == nullptr && len != 0) // native esp_now_send treats this as an arg error + return ESP_ERR_ESPNOW_ARG; + // Only the small fixed header is built here; the caller's frame goes over as + // the request tail, so request() lays both into its own buffer under + // g_req_mutex. esp_now_send is a public C symbol and may be called from any + // task, and a shared build buffer here would let two callers corrupt each + // other's frame. Passing the body through also drops a full-frame copy per + // transmit, on the path this shim exists to keep quick. + uint8_t hdr[sizeof(esp_now_hosted_send_req_t)]; + auto *s = reinterpret_cast(hdr); + s->has_addr = peer_addr != nullptr ? 1 : 0; + if (peer_addr != nullptr) + memcpy(s->peer_addr, peer_addr, 6); + else + memset(s->peer_addr, 0, 6); + s->data_len = static_cast(len); + // Fire-and-forget (wait=false): native esp_now_send returns once the frame is + // queued, with the real TX result delivered later through the send callback. + // The co-processor mirrors that — it acks enqueue immediately and reports the + // outcome via the async SEND event (on_send -> on_send_report). Waiting for + // the RPC RESP here would block the main loop for the full round-trip on + // every transmit. + return request(ESP_NOW_HOSTED_OP_SEND, hdr, sizeof(hdr), nullptr, 0, nullptr, /*wait=*/false, data, + static_cast(len)); +} + +esp_err_t esp_now_set_pmk(const uint8_t *pmk) { + if (pmk == nullptr) + return ESP_ERR_ESPNOW_ARG; + return request(ESP_NOW_HOSTED_OP_SET_PMK, pmk, 16, nullptr, 0, nullptr); +} + +// Remainder of the surface. Not used by ESPHome's espnow component +// today; provided so the whole header links and future callers get a defined +// (if unimplemented) symbol rather than a link error. Wire them through +// CustomRpc if a use case appears. +esp_err_t esp_now_get_peer(const uint8_t * /*peer_addr*/, esp_now_peer_info_t * /*peer*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_now_fetch_peer(bool /*from_head*/, esp_now_peer_info_t * /*peer*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_get_peer_num(esp_now_peer_num_t * /*num*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_set_wake_window(uint16_t /*window*/) { + return ESP_ERR_NOT_SUPPORTED; // power-save wake window is not forwarded; don't claim success +} +esp_err_t esp_now_set_peer_rate_config(const uint8_t * /*peer_addr*/, esp_now_rate_config_t * /*cfg*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_wifi_config_espnow_rate(wifi_interface_t /*ifx*/, wifi_phy_rate_t /*rate*/) { + return ESP_ERR_NOT_SUPPORTED; +} + +} // extern "C" + +#endif // CONFIG_IDF_TARGET_ESP32P4 diff --git a/esphome/components/esp32_hosted/esp_now_hosted_rpc.h b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h new file mode 100644 index 0000000000..bf68c759ee --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h @@ -0,0 +1,128 @@ +/* + * esp_now_hosted — ESP-NOW-over-CustomRpc wire protocol. + * + * Shared, byte-for-byte-identical contract between: + * - the host shim (esphome/components/esp32_hosted/esp_now_hosted.cpp) + * - the coprocessor firmware (esphome/esp-hosted-firmware) + * + * It rides esp-hosted's CustomRpc channel (RPC ID 388, "peer data transfer", + * available since esp-hosted v2.8.1), teaching the radio-less host <-> radio + * co-processor link to carry esp_now.h, which esp-hosted itself does not proxy + * (Espressif issue espressif/esp-hosted-mcu#19). + * + * KEEP THE TWO COPIES IN SYNC. The canonical copy lives here; the coprocessor + * firmware uses a verbatim copy. Both sides are little-endian, so these packed + * structs are wire-compatible with no byte-swapping. + */ + +#ifndef ESP_NOW_HOSTED_RPC_H +#define ESP_NOW_HOSTED_RPC_H + +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* ── CustomRpc message IDs (any uint32_t except 0xFFFFFFFF) ────────────────── + * One REQ handler slot on the device; three event handler slots on the host. + * The bytes spell "now" + index, a private range unlikely to clash with other + * CustomRpc users (e.g. the stock peer_data_transfer example's 1..6). */ +#define ESP_NOW_HOSTED_MSG_REQ 0x6E6F7701u /* host -> device : request envelope */ +#define ESP_NOW_HOSTED_MSG_RESP 0x6E6F7702u /* device -> host : reply to a REQ */ +#define ESP_NOW_HOSTED_MSG_RECV 0x6E6F7703u /* device -> host : async RX frame */ +#define ESP_NOW_HOSTED_MSG_SEND 0x6E6F7704u /* device -> host : async TX status */ + +/* ── Request opcodes ────────────────────────────────────────────────────── */ +enum { + ESP_NOW_HOSTED_OP_INIT = 1, /* esp_now_init + register device recv/send cbs */ + ESP_NOW_HOSTED_OP_DEINIT = 2, /* unregister cbs + esp_now_deinit */ + ESP_NOW_HOSTED_OP_ADD_PEER = 3, /* payload: esp_now_hosted_peer_t */ + ESP_NOW_HOSTED_OP_DEL_PEER = 4, /* payload: 6-byte peer MAC */ + ESP_NOW_HOSTED_OP_IS_PEER_EXIST = 5, /* payload: 6-byte MAC; ret: 1 byte bool */ + ESP_NOW_HOSTED_OP_SEND = 6, /* payload: esp_now_hosted_send_req_t */ + ESP_NOW_HOSTED_OP_GET_VERSION = 7, /* ret: uint32 version */ + ESP_NOW_HOSTED_OP_SET_PMK = 8, /* payload: 16-byte PMK */ + ESP_NOW_HOSTED_OP_MOD_PEER = 9, /* payload: esp_now_hosted_peer_t */ +}; + +/* Largest ESP-NOW payload we forward. ESP-NOW v2 (IDF >= 5.4) is 1470 B; well + * under esp-hosted's 8166 B CustomRpc cap, so the shim never truncates. */ +#define ESP_NOW_HOSTED_MAX_FRAME 1470u +/* Envelope slack for the largest opcode payload (a SEND req wrapping a frame). */ +#define ESP_NOW_HOSTED_MAX_PAYLOAD (ESP_NOW_HOSTED_MAX_FRAME + 16u) +/* Host request/response round-trip timeout over the transport. Generous: + * normal RTT is sub-millisecond, but Wi-Fi/BLE contention on the co-processor + * can stall the RX thread. */ +#define ESP_NOW_HOSTED_TIMEOUT_MS 2000 + +/* ── Envelopes ──────────────────────────────────────────────────────────── */ + +/* These payloads are shared verbatim with the C co-processor firmware, so they + * use C's `typedef struct {...} name;` idiom rather than C++ `using` aliases, + * which would not compile there. Silence clang-tidy's modernize-use-using for + * the shared struct block. */ +// NOLINTBEGIN(modernize-use-using) +typedef struct { + uint8_t opcode; /* one of ESP_NOW_HOSTED_OP_* */ + uint8_t seq; /* wraps 0..255; echoed in the response for matching */ + uint16_t payload_len; /* bytes of opcode-specific payload that follow */ + uint8_t payload[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_req_t; + +typedef struct { + uint8_t opcode; /* echoes the request opcode */ + uint8_t seq; /* echoes the request seq */ + int32_t status; /* esp_err_t from the native call on the co-processor */ + uint16_t ret_len; /* bytes of return payload that follow */ + uint8_t ret[]; /* flexible (e.g. version u32, is_peer_exist bool) */ +} __attribute__((packed)) esp_now_hosted_resp_t; + +/* ── Opcode payloads ────────────────────────────────────────────────────── */ + +/* esp_now_peer_info_t minus the host-only `priv` pointer, which is meaningless + * across the transport and never set by ESPHome's espnow component. */ +typedef struct { + uint8_t peer_addr[6]; + uint8_t lmk[16]; + uint8_t channel; /* 0 = current channel */ + uint8_t ifidx; /* wifi_interface_t (0=STA, 1=AP) */ + uint8_t encrypt; /* bool */ +} __attribute__((packed)) esp_now_hosted_peer_t; + +typedef struct { + uint8_t has_addr; /* 0 => peer_addr is NULL (broadcast to all peers) */ + uint8_t peer_addr[6]; + uint16_t data_len; + uint8_t data[]; /* flexible, up to ESP_NOW_HOSTED_MAX_FRAME */ +} __attribute__((packed)) esp_now_hosted_send_req_t; + +/* ── Async events (device -> host) ──────────────────────────────────────── */ + +/* Reconstructed on the host into an esp_now_recv_info_t + a minimal + * wifi_pkt_rx_ctrl_t. ESPHome's espnow reads info->src_addr, info->des_addr, + * info->rx_ctrl->rssi and info->rx_ctrl->timestamp. */ +typedef struct { + uint8_t src_addr[6]; + uint8_t des_addr[6]; + int8_t rssi; + uint8_t channel; + uint16_t data_len; + uint8_t data[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_recv_evt_t; + +typedef struct { + uint8_t des_addr[6]; + uint8_t status; /* esp_now_send_status_t (0 = success) */ +} __attribute__((packed)) esp_now_hosted_send_evt_t; +// NOLINTEND(modernize-use-using) + +#ifdef __cplusplus +} +#endif + +#endif /* ESP_NOW_HOSTED_RPC_H */ diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 5541a6ee97..14d099ec06 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -3,6 +3,7 @@ from typing import Any from esphome import automation, core import esphome.codegen as cg from esphome.components import wifi +from esphome.components.esp32 import VARIANT_ESP32P4, get_esp32_variant from esphome.components.udp import CONF_ON_RECEIVE import esphome.config_validation as cv from esphome.const import ( @@ -17,6 +18,7 @@ from esphome.const import ( ) from esphome.core import CORE, HexInt from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] @@ -132,6 +134,24 @@ CONFIG_SCHEMA = cv.All( ) +def _validate_variant(config: ConfigType) -> ConfigType: + # ESP-NOW rides the Wi-Fi PHY. Radio-less esp32 variants have no native + # ESP-NOW; only the ESP32-P4 has a path, via the esp32_hosted shim that + # supplies the esp_now_* symbols. Fail here with a clear message instead of + # letting the build reach an "undefined reference to esp_now_*" link error. + variant = get_esp32_variant() + if wifi.variant_has_wifi(variant): + return config + if variant != VARIANT_ESP32P4: + raise cv.Invalid(f"ESP-NOW is not supported on {variant} (no Wi-Fi radio)") + if "esp32_hosted" not in fv.full_config.get(): + raise cv.Invalid(f"ESP-NOW on {variant} requires the esp32_hosted component") + return config + + +FINAL_VALIDATE_SCHEMA = _validate_variant + + async def _trigger_to_code(config: ConfigType) -> MockObj: if address := config.get(CONF_ADDRESS): address = address.parts diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9dd1e0ced6..eaece6d5ff 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_ESP32_HOSTED #define USE_ESP32_HOSTED_HTTP_UPDATE #define USE_ESP32_IMPROV_STATE_CALLBACK +#define USE_ESP_NOW_HOSTED #define USE_EVENT #define USE_FAN #define USE_GPIO_BINARY_SENSOR_INTERRUPT diff --git a/script/ci-custom.py b/script/ci-custom.py index f481fda860..e2b7cd8d37 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -294,6 +294,9 @@ def highlight(s): "esphome/components/socket/headers.h", "esphome/core/defines.h", "esphome/components/http_request/httplib.h", + # Shared C wire header (byte-identical with the co-processor firmware); + # these are protocol constants and constexpr is C++-only. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_no_defines(fname, match): @@ -816,6 +819,10 @@ def lint_relative_py_import(fname: Path, line, col, content): "esphome/components/host/helpers.cpp", "esphome/components/zephyr/helpers.cpp", "esphome/components/http_request/httplib.h", + # Global extern "C" esp_now_* linker symbols + shared C wire header; + # neither can live in a C++ namespace. + "esphome/components/esp32_hosted/esp_now_hosted.cpp", + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: @@ -841,7 +848,15 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) +@lint_content_check( + include=["*.h"], + exclude=[ + "esphome/core/entity_types.h", + # Shared C wire header; uses a classic #ifndef guard for portability + # across the co-processor firmware repo it stays byte-identical with. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + ], +) def lint_pragma_once(fname, content): if "#pragma once" not in content: return ( diff --git a/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml new file mode 100644 index 0000000000..fab0a64ab8 --- /dev/null +++ b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml @@ -0,0 +1,5 @@ +# Exercises the ESP-NOW-over-hosted shim: on the ESP32-P4 host, esp32_hosted +# supplies the esp_now_* symbols that the espnow component links against. +packages: + esp32_hosted: !include common.yaml + espnow: !include ../espnow/common.yaml diff --git a/tests/unit_tests/components/test_espnow.py b/tests/unit_tests/components/test_espnow.py new file mode 100644 index 0000000000..21305c2b33 --- /dev/null +++ b/tests/unit_tests/components/test_espnow.py @@ -0,0 +1,48 @@ +"""Tests for the espnow component's final validation.""" + +import pytest + +from esphome.components.esp32.const import ( + VARIANT_ESP32C3, + VARIANT_ESP32H2, + VARIANT_ESP32P4, +) +from esphome.components.espnow import _validate_variant +import esphome.config_validation as cv +import esphome.final_validate as fv +from esphome.types import ConfigType + + +def _run( + monkeypatch, variant: str, full_config: dict, config: ConfigType +) -> ConfigType: + monkeypatch.setattr("esphome.components.espnow.get_esp32_variant", lambda: variant) + token = fv.full_config.set(full_config) + try: + return _validate_variant(config) + finally: + fv.full_config.reset(token) + + +def test_variant_with_native_wifi_passes(monkeypatch) -> None: + """A variant with a native Wi-Fi PHY needs no shim; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32C3, {}, config) is config + + +def test_radioless_non_p4_variant_rejected(monkeypatch) -> None: + """Radio-less variants without any ESP-NOW path are rejected outright.""" + with pytest.raises(cv.Invalid, match="not supported"): + _run(monkeypatch, VARIANT_ESP32H2, {}, {}) + + +def test_p4_without_esp32_hosted_rejected(monkeypatch) -> None: + """The P4 needs the esp32_hosted shim to supply the esp_now_* symbols.""" + with pytest.raises(cv.Invalid, match="esp32_hosted"): + _run(monkeypatch, VARIANT_ESP32P4, {}, {}) + + +def test_p4_with_esp32_hosted_passes(monkeypatch) -> None: + """The P4 with esp32_hosted present validates; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32P4, {"esp32_hosted": {}}, config) is config From 3321566cc010c3a4e774a78e1d81d887c8e02879 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 3 Sep 2026 07:12:36 -0500 Subject: [PATCH 030/178] [remote_transmitter] Fix BK7231N build by limiting the PWM path to BK7238 (#18958) --- esphome/components/remote_transmitter/__init__.py | 14 +++++--------- .../remote_transmitter/remote_transmitter.h | 9 +++++---- .../remote_transmitter_bk72xx.cpp | 11 +++++++---- .../remote_transmitter_libretiny_isr.cpp | 10 +++++----- .../remote_transmitter/test_non_blocking_gate.py | 2 +- .../remote_transmitter/test.bk72xx-ard.yaml | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index cb2aebec91..58392c48ab 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -4,11 +4,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base from esphome.components.libretiny import get_libretiny_family -from esphome.components.libretiny.const import ( - FAMILY_BK7231N, - FAMILY_BK7238, - FAMILY_RTL8720C, -) +from esphome.components.libretiny.const import FAMILY_BK7238, FAMILY_RTL8720C from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -49,7 +45,9 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) -_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7231N, FAMILY_BK7238) +# Keep in sync with the USE_LIBRETINY_VARIANT_RTL8720C / REMOTE_TRANSMITTER_BK_PWM gates in +# remote_transmitter.h, which decide where set_non_blocking() is declared +_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7238) def _validate_non_blocking_platform(value: bool) -> bool: @@ -59,9 +57,7 @@ def _validate_non_blocking_platform(value: bool) -> bool: return cv.boolean(value) if CORE.is_libretiny and get_libretiny_family() in _NON_BLOCKING_LIBRETINY_FAMILIES: return cv.boolean(value) - raise cv.Invalid( - "non_blocking is only supported on ESP32, RTL8720C, BK7231N and BK7238" - ) + raise cv.Invalid("non_blocking is only supported on ESP32, RTL8720C and BK7238") MULTI_CONF = True diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index 313b26364d..4db4e80a60 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -12,10 +12,11 @@ #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 -// The BK7231N-style PWM block (hardware shadow-load duty updates) enables the ISR-driven -// transmitter on these families; family-level proxy for the SDK's CFG_SOC_NAME gate. -// See remote_transmitter_bk72xx.cpp. -#if defined(USE_LIBRETINY_VARIANT_BK7231N) || defined(USE_LIBRETINY_VARIANT_BK7238) +// Enables the ISR-driven transmitter on Beken. Gated on BK7238 alone: the shadow-load PWM +// block is shared with BK7231N, but LibreTiny builds that family against an older BDK whose +// PWM driver has no pwm_init_param()/pwm_start(). See remote_transmitter_bk72xx.cpp. +// Keep in sync with _NON_BLOCKING_LIBRETINY_FAMILIES in __init__.py. +#ifdef USE_LIBRETINY_VARIANT_BK7238 #define REMOTE_TRANSMITTER_BK_PWM #endif diff --git a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp index 0081ae47b3..822389ccf9 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp @@ -9,10 +9,13 @@ // with the core's fixes for type-name collisions between the two #include -// Only the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) -// supports glitch-free per-edge duty updates; older SoCs compile the generic bit-bang -// implementation (remote_transmitter.cpp) instead, and this file compiles to nothing. -// REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h. +// Needs the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) +// for glitch-free per-edge duty updates, and an SDK exposing pwm_init_param()/pwm_start(). +// BK7231N has the block but LibreTiny builds it against an older BDK offering only the +// sddev_control API (CMD_PWM_INIT_PARAM), so it stays on the generic bit-bang path until +// someone can add and validate that path on real hardware. Every other Beken SoC lacks the +// block. REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h; when it is +// unset this file compiles to nothing and remote_transmitter.cpp is used instead. namespace esphome::remote_transmitter { diff --git a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp index 003cdfa986..fad91f593f 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp @@ -3,11 +3,11 @@ #include "esphome/core/hal.h" #include "esphome/core/log.h" -// Envelope chain shared by the LibreTiny families that pace transmission from a hardware -// timer interrupt: RTL8720C (gtimer) and the BK7231N-style PWM block (BKTIMER1). Everything -// platform-specific sits behind five hooks implemented in the per-family files -- carrier -// setup, duty writes, one-shot arming and timer stop. Families without a usable timer keep -// the generic bit-bang implementation and compile none of this. +// Envelope chain shared by the LibreTiny families that pace transmission from a hardware timer +// interrupt: RTL8720C (gtimer) and BK7238 (BKTIMER1). Everything platform-specific sits behind +// five hooks implemented in the per-family files -- carrier setup, duty writes, one-shot arming +// and timer stop. Families without a usable timer keep the generic bit-bang implementation and +// compile none of this. #if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) namespace esphome::remote_transmitter { diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py index ee2769e177..525ab3329e 100644 --- a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -26,7 +26,7 @@ from ..types import SetCoreConfigCallable (PlatformFramework.ESP32_IDF, None, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), - (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, False), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7238, True), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231T, False), (PlatformFramework.ESP8266_ARDUINO, None, False), diff --git a/tests/components/remote_transmitter/test.bk72xx-ard.yaml b/tests/components/remote_transmitter/test.bk72xx-ard.yaml index ea2feafda9..f3e2da9daf 100644 --- a/tests/components/remote_transmitter/test.bk72xx-ard.yaml +++ b/tests/components/remote_transmitter/test.bk72xx-ard.yaml @@ -2,7 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO26 carrier_duty_percent: 50% - # non_blocking is bk7231n/bk7238-only; the CI board is a BK7252 + # non_blocking is bk7238-only; the CI board is a BK7252, so this builds the bit-bang path packages: buttons: !include common-buttons.yaml From 657116a213de452fc191772c06e20aec09d16446 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:15:06 +0000 Subject: [PATCH 031/178] Bump bundled esphome-device-builder to 1.14.0 (#18960) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0da8048c57..7952616496 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 RUN \ platformio settings set enable_telemetry No \ From e47247486ba238f16f958a3298e08c43d309e6c7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 3 Sep 2026 21:16:36 +0200 Subject: [PATCH 032/178] [esp8266] Drop Arduino framework versions before 3.0.0 (#18917) to Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/arduino8266/framework.py | 13 ++--- esphome/components/climate/climate.cpp | 4 +- esphome/components/debug/debug_component.cpp | 4 +- esphome/components/debug/debug_component.h | 4 +- esphome/components/debug/debug_esp8266.cpp | 2 - esphome/components/debug/sensor.py | 7 +-- esphome/components/esp8266/__init__.py | 57 ++++++------------- .../nextion/nextion_upload_arduino.cpp | 6 -- esphome/components/wifi/wifi_component.h | 5 -- .../wifi/wifi_component_esp8266.cpp | 10 +--- esphome/core/log.h | 14 ----- .../components/esp8266/test_boards.py | 17 +----- .../esp8266/test_framework_version.py | 23 ++++++++ .../unit_tests/test_arduino8266_framework.py | 17 ++---- 14 files changed, 62 insertions(+), 121 deletions(-) create mode 100644 tests/unit_tests/components/esp8266/test_framework_version.py diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 1edbe4b36f..663002b3b1 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -44,8 +44,7 @@ def get_arduino8266_tools_path() -> Path: return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) -# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the -# encoder below cannot name 3.0.0/3.0.1 either (see its docstring) +# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0 MIN_FRAMEWORK_VERSION = Version(3, 1, 1) @@ -53,20 +52,16 @@ def framework_package_version(ver: Version) -> str: """Map an Arduino core version to its registry package version (3.1.2 -> 3.30102.0; the leading 3 is the package major). - Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor - at MIN_FRAMEWORK_VERSION. + Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION. """ if ver.major > 3: raise EsphomeError( f"Arduino core {ver} is not supported yet; " "the newest known core series is 3.x" ) - if ver <= Version(2, 6, 2): - # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same - # boundary as _format_framework_arduino_version's era guard) + if ver.major < 3: raise EsphomeError( - f"Arduino core {ver} uses an older package encoding than this " - "helper implements (newer than 2.6.2)" + f"Arduino core {ver} is not supported; ESPHome requires core 3.x" ) return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 34684a87e1..f80de151b1 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -368,8 +368,8 @@ optional Climate::restore_state_() { } void Climate::save_state_(const ClimateTraits &traits) { -#if (defined(USE_ESP32) || (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0))) && \ - !defined(CLANG_TIDY) +#if (defined(USE_ESP32) || defined(USE_ESP8266)) && !defined(CLANG_TIDY) +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wclass-memaccess" #define TEMP_IGNORE_MEMACCESS #endif diff --git a/esphome/components/debug/debug_component.cpp b/esphome/components/debug/debug_component.cpp index 9020c261c2..97f4522c62 100644 --- a/esphome/components/debug/debug_component.cpp +++ b/esphome/components/debug/debug_component.cpp @@ -22,9 +22,9 @@ void DebugComponent::dump_config() { LOG_SENSOR(" ", "Free space on heap", this->free_sensor_); LOG_SENSOR(" ", "Largest free heap block", this->block_sensor_); LOG_SENSOR(" ", "CPU frequency", this->cpu_frequency_sensor_); -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#ifdef USE_ESP8266 LOG_SENSOR(" ", "Heap fragmentation", this->fragmentation_sensor_); -#endif // defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#endif // USE_ESP8266 #endif // USE_SENSOR char device_info_buffer[DEVICE_INFO_BUFFER_SIZE]; diff --git a/esphome/components/debug/debug_component.h b/esphome/components/debug/debug_component.h index 20798cf600..b05029f878 100644 --- a/esphome/components/debug/debug_component.h +++ b/esphome/components/debug/debug_component.h @@ -35,7 +35,7 @@ class DebugComponent final : public PollingComponent { #ifdef USE_SENSOR void set_free_sensor(sensor::Sensor *free_sensor) { free_sensor_ = free_sensor; } void set_block_sensor(sensor::Sensor *block_sensor) { block_sensor_ = block_sensor; } -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) void set_fragmentation_sensor(sensor::Sensor *fragmentation_sensor) { fragmentation_sensor_ = fragmentation_sensor; } #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) @@ -61,7 +61,7 @@ class DebugComponent final : public PollingComponent { sensor::Sensor *free_sensor_{nullptr}; sensor::Sensor *block_sensor_{nullptr}; -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) sensor::Sensor *fragmentation_sensor_{nullptr}; #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) diff --git a/esphome/components/debug/debug_esp8266.cpp b/esphome/components/debug/debug_esp8266.cpp index 272123dfc0..acce28818c 100644 --- a/esphome/components/debug/debug_esp8266.cpp +++ b/esphome/components/debug/debug_esp8266.cpp @@ -159,12 +159,10 @@ void DebugComponent::update_platform_() { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->block_sensor_->publish_state(ESP.getMaxFreeBlockSize()); } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) if (this->fragmentation_sensor_ != nullptr) { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->fragmentation_sensor_->publish_state(ESP.getHeapFragmentation()); } -#endif #endif } diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index 72e2efebc2..e53cb0d1e4 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -52,12 +52,9 @@ CONFIG_SCHEMA = { ), cv.Optional(CONF_FRAGMENTATION): cv.All( cv.Any( - cv.All( - cv.only_on_esp8266, - cv.require_framework_version(esp8266_arduino=cv.Version(2, 5, 2)), - ), + cv.only_on_esp8266, cv.only_on_esp32, - msg="This feature is only available on ESP8266 (Arduino 2.5.2+) and ESP32", + msg="This feature is only available on ESP8266 and ESP32", ), sensor.sensor_schema( unit_of_measurement=UNIT_PERCENT, diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 63665e7681..19dbb68f29 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -35,7 +35,7 @@ from esphome.platformio.toolchain import copy_ccache_script from esphome.storage_json import StorageJSON from esphome.types import ConfigType -from .boards import BOARDS, ESP8266_LD_SCRIPTS, board_ld_script +from .boards import BOARDS, board_ld_script from .const import ( CONF_EARLY_PIN_INIT, CONF_ENABLE_SERIAL, @@ -43,8 +43,6 @@ from .const import ( CONF_RESTORE_FROM_FLASH, KEY_BOARD, KEY_ESP8266, - KEY_FLASH_SIZE, - KEY_LDSCRIPT, KEY_PIN_INITIAL_STATES, KEY_SERIAL1_REQUIRED, KEY_SERIAL_REQUIRED, @@ -133,10 +131,6 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/esp8266/Arduino/releases) version to # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 - if ver <= cv.Version(2, 4, 1): - return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - if ver <= cv.Version(2, 6, 2): - return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" # Same encoding the native toolchain uses for its package download, so a # version bump cannot drift between the two paths. from esphome.arduino8266.framework import framework_package_version @@ -159,11 +153,9 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # - https://github.com/esp8266/Arduino/releases # - https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(3, 1, 2) -# The platformio/espressif8266 version to use for arduino 2 framework versions +# The platformio/espressif8266 version to use for arduino 3 framework versions # - https://github.com/platformio/platform-espressif8266/releases # - https://api.registry.platformio.org/v3/packages/platformio/platform/espressif8266 -ARDUINO_2_PLATFORM_VERSION = cv.Version(2, 6, 3) -# for arduino 3 framework versions ARDUINO_3_PLATFORM_VERSION = cv.Version(3, 2, 0) # for arduino 4 framework versions ARDUINO_4_PLATFORM_VERSION = cv.Version(4, 2, 1) @@ -188,6 +180,14 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: version = cv.Version.parse(cv.version_number(value[CONF_VERSION])) source = value.get(CONF_SOURCE, None) + if version < cv.Version(3, 0, 0): + raise cv.Invalid( + f"Arduino framework {version} is no longer supported; ESPHome requires " + f"C++20, which needs Arduino core 3.x. Use the recommended version " + f"({RECOMMENDED_ARDUINO_FRAMEWORK_VERSION}).", + path=[CONF_VERSION], + ) + value[CONF_VERSION] = str(version) value[CONF_SOURCE] = source or _format_framework_arduino_version(version) @@ -195,12 +195,8 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: if platform_version is None: if version >= cv.Version(3, 1, 0): platform_version = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION)) - elif version >= cv.Version(3, 0, 0): - platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) - elif version >= cv.Version(2, 5, 0): - platform_version = _parse_platform_version(str(ARDUINO_2_PLATFORM_VERSION)) else: - platform_version = _parse_platform_version(str(cv.Version(1, 8, 0))) + platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) value[CONF_PLATFORM_VERSION] = platform_version if version != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION: @@ -289,29 +285,11 @@ def check_rosetta() -> None: ) -def _choose_ld_script(board: str, ver: cv.Version) -> str | None: - """The flash ld to pin for this board and core, or None for cores - without ld-script support.""" - board_data = BOARDS[board] - ld_scripts = ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]] - if ver <= cv.Version(2, 3, 0): - # No ld script support - return None - if ver <= cv.Version(2, 4, 2): - # Old ld script path; the modern per-board override names do not - # exist in this core's SDK, so the override cannot be honored. - # Substituting the size default would move _FS_end and the - # preferences sector, wiping flash-backed state on flash. - if KEY_LDSCRIPT in board_data: - raise EsphomeError( - f"Board {board} requires its {board_data[KEY_LDSCRIPT]} " - f"flash layout, which Arduino core {ver} cannot honor; " - "use a core newer than 2.4.2" - ) - return ld_scripts[0] +def _choose_ld_script(board: str) -> str: + """The flash ld to pin for this board.""" # A per-board override preserves a layout the board shipped with # (see d1_wroom_02 in boards.py) - return board_ld_script(board_data) + return board_ld_script(BOARDS[board]) @coroutine_with_priority(CoroPriority.PLATFORM) @@ -435,10 +413,9 @@ async def to_code(config: ConfigType) -> None: ) if config[CONF_BOARD] in BOARDS: - ld_script = _choose_ld_script(config[CONF_BOARD], ver) - - if ld_script is not None: - cg.add_platformio_option("board_build.ldscript", ld_script) + cg.add_platformio_option( + "board_build.ldscript", _choose_ld_script(config[CONF_BOARD]) + ) CORE.add_job(add_pin_initial_states_array) CORE.add_job(finalize_waveform_config) diff --git a/esphome/components/nextion/nextion_upload_arduino.cpp b/esphome/components/nextion/nextion_upload_arduino.cpp index f02f32d5ca..944fa1db47 100644 --- a/esphome/components/nextion/nextion_upload_arduino.cpp +++ b/esphome/components/nextion/nextion_upload_arduino.cpp @@ -209,14 +209,8 @@ bool Nextion::upload_tft(uint32_t baud_rate, bool exit_reparse) { http_client.setTimeout(this->tft_upload_http_timeout_); bool begin_status = false; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 7, 0) http_client.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); -#elif USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) - http_client.setFollowRedirects(true); -#endif -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) http_client.setRedirectLimit(3); -#endif begin_status = http_client.begin(*this->get_wifi_client_(), this->tft_url_.c_str()); if (!begin_status) { this->connection_state_.is_updating_ = false; diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index cfdbc1a968..63df9fbfa5 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -40,11 +40,6 @@ #include #include -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(2, 4, 0) -extern "C" { -#include -}; -#endif #endif #ifdef USE_RP2 diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index b4a91fb3cd..031da1b355 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -21,7 +21,6 @@ extern "C" { #include "lwip/apps/sntp.h" #include "lwip/netif.h" // struct netif #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) #include "LwipDhcpServer.h" #if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) #include @@ -30,7 +29,6 @@ extern "C" { #define wifi_softap_set_dhcps_lease_time(time) dhcpSoftAP.set_dhcps_lease_time(time) #define wifi_softap_set_dhcps_offer_option(offer, mode) dhcpSoftAP.set_dhcps_offer_option(offer, mode) #endif -#endif } #include "esphome/core/application.h" @@ -293,7 +291,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { conf.bssid_set = 0; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) if (ap.password_.empty()) { conf.threshold.authmode = AUTH_OPEN; } else { @@ -310,7 +307,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { } } conf.threshold.rssi = -127; -#endif ETS_UART_INTR_DISABLE(); bool ret = wifi_station_set_config_current(&conf); @@ -602,7 +598,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) case EVENT_OPMODE_CHANGED: { auto it = event->event_info.opmode_changed; ESP_LOGV(TAG, "Changed Mode old=%s new=%s", LOG_STR_ARG(get_op_mode_str(it.old_opmode)), @@ -620,7 +615,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#endif default: break; } @@ -705,7 +699,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.bssid = nullptr; config.channel = 0; config.show_hidden = 1; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE; // Use shorter dwell times for roaming scans - we only need to detect strong // nearby APs, not do a thorough survey. This also reduces off-channel time @@ -724,7 +717,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.scan_time.active.min = roaming ? SCAN_ACTIVE_MIN_ROAMING_MS : SCAN_ACTIVE_MIN_DEFAULT_MS; config.scan_time.active.max = roaming ? SCAN_ACTIVE_MAX_ROAMING_MS : SCAN_ACTIVE_MAX_DEFAULT_MS; } -#endif bool ret = wifi_station_scan(&config, &WiFiComponent::s_wifi_scan_done_callback); if (!ret) { ESP_LOGV(TAG, "wifi_station_scan failed"); @@ -830,7 +822,7 @@ bool WiFiComponent::wifi_ap_ip_config_(const optional &manual_ip) { return false; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) +#if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) dhcpSoftAP.begin(&info); #endif diff --git a/esphome/core/log.h b/esphome/core/log.h index 272e516808..14d24412ef 100644 --- a/esphome/core/log.h +++ b/esphome/core/log.h @@ -18,7 +18,6 @@ #ifdef USE_STORE_LOG_STR_IN_FLASH #include "WString.h" -#include "esphome/core/defines.h" // for USE_ARDUINO_VERSION_CODE #endif // Include ESP-IDF/Arduino based logging methods here so they don't undefine ours later @@ -177,20 +176,7 @@ struct LogString; #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 0) #define LOG_STR_ARG(s) ((PGM_P) (s)) -#else -// Pre-Arduino 2.5, we can't pass a PSTR() to printf(). Emulate support by copying the message to a -// local buffer first. String length is limited to 63 characters. -// https://github.com/esp8266/Arduino/commit/6280e98b0360f85fdac2b8f10707fffb4f6e6e31 -#define LOG_STR_ARG(s) \ - ({ \ - char __buf[64]; \ - __buf[63] = '\0'; \ - strncpy_P(__buf, (PGM_P) (s), 63); \ - __buf; \ - }) -#endif #define LOG_STR(s) (reinterpret_cast(PSTR(s))) #define LOG_STR_LITERAL(s) LOG_STR_ARG(LOG_STR(s)) diff --git a/tests/unit_tests/components/esp8266/test_boards.py b/tests/unit_tests/components/esp8266/test_boards.py index df0e536d42..78213a762a 100644 --- a/tests/unit_tests/components/esp8266/test_boards.py +++ b/tests/unit_tests/components/esp8266/test_boards.py @@ -1,11 +1,7 @@ """Tests for the per-board linker-script rule.""" -import pytest - from esphome.components.esp8266 import _choose_ld_script from esphome.components.esp8266.boards import BOARDS, board_ld_script -import esphome.config_validation as cv -from esphome.core import EsphomeError def test_d1_wroom_02_keeps_its_shipped_layout() -> None: @@ -21,13 +17,6 @@ def test_default_boards_use_the_flash_size_layout() -> None: def test_choose_ld_script_paths() -> None: - """Old cores get the size default, overriding boards hard-error there - (a substituted layout would wipe flash-backed state), modern cores - honor the override.""" - assert _choose_ld_script("nodemcuv2", cv.Version(2, 3, 0)) is None - assert _choose_ld_script("nodemcuv2", cv.Version(2, 4, 2)) == "eagle.flash.4m.ld" - assert _choose_ld_script("d1_wroom_02", cv.Version(2, 7, 4)) == ( - "eagle.flash.2m64.ld" - ) - with pytest.raises(EsphomeError, match="cannot honor"): - _choose_ld_script("d1_wroom_02", cv.Version(2, 4, 2)) + """Default boards get the size layout, overriding boards keep theirs.""" + assert _choose_ld_script("nodemcuv2") == "eagle.flash.4m.ld" + assert _choose_ld_script("d1_wroom_02") == "eagle.flash.2m64.ld" diff --git a/tests/unit_tests/components/esp8266/test_framework_version.py b/tests/unit_tests/components/esp8266/test_framework_version.py new file mode 100644 index 0000000000..0107aff8dd --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_framework_version.py @@ -0,0 +1,23 @@ +"""Tests for the Arduino framework version floor.""" + +import pytest + +from esphome.components.esp8266 import _arduino_check_versions +import esphome.config_validation as cv +from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION + + +def test_versions_before_3_are_rejected() -> None: + with pytest.raises(cv.Invalid, match="no longer supported") as excinfo: + _arduino_check_versions({CONF_VERSION: "2.7.4"}) + assert excinfo.value.path == [CONF_VERSION] + + +def test_supported_versions_pass() -> None: + value = _arduino_check_versions({CONF_VERSION: "3.0.2"}) + assert value[CONF_VERSION] == "3.0.2" + assert "espressif8266@3.2.0" in value[CONF_PLATFORM_VERSION] + + value = _arduino_check_versions({CONF_VERSION: "recommended"}) + assert value[CONF_VERSION] == "3.1.2" + assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION] diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index bd0a620e10..9f415344ae 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -21,17 +21,12 @@ def _build_path(tmp_path: Path) -> None: def test_framework_package_version() -> None: assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" - # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path) - assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0" # A future major bump needs its own encoding, not a doomed registry lookup with pytest.raises(EsphomeError, match="not supported yet"): framework.framework_package_version(cv.Version(4, 0, 0)) - # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release - # keeps this encoding - with pytest.raises(EsphomeError, match="older package encoding"): - framework.framework_package_version(cv.Version(2, 6, 2)) - assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0" - assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0" + # Cores before 3.x cannot build ESPHome (C++20) and are rejected + with pytest.raises(EsphomeError, match="requires core 3"): + framework.framework_package_version(cv.Version(2, 7, 4)) def test_format_framework_arduino_version_pins_all_series() -> None: @@ -39,10 +34,10 @@ def test_format_framework_arduino_version_pins_all_series() -> None: era, including the 4.x rejection it now shares with the installer.""" from esphome.components.esp8266 import _format_framework_arduino_version as fmt - assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0" - assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0" - assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0" assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" + # Pre-3 cores are rejected with the version line anchored + with pytest.raises(cv.Invalid, match="requires core 3"): + fmt(cv.Version(2, 7, 4)) # Anchored to the framework version line, not a bare EsphomeError with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: fmt(cv.Version(4, 0, 0)) From cb0c2bdaca67440249b415f4bb831c3906b83bbe Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:38:55 +1200 Subject: [PATCH 033/178] [esp32_ble] Reference count BLE advertising (#18943) --- esphome/components/esp32_ble/ble.cpp | 35 +++++++++++++++---- esphome/components/esp32_ble/ble.h | 13 +++++++ .../esp32_ble_beacon/esp32_ble_beacon.cpp | 2 ++ .../components/esp32_ble_server/__init__.py | 12 +++++++ .../esp32_ble_server/ble_server.cpp | 21 +++++++++-- .../components/esp32_ble_server/ble_server.h | 11 ++++++ .../esp32_improv/esp32_improv_component.cpp | 20 ++++++++++- .../esp32_improv/esp32_improv_component.h | 3 ++ .../esp32_ble_server/config/improv_only.yaml | 13 +++++++ .../config/manufacturer_data_only.yaml | 9 +++++ .../esp32_ble_server/config/own_service.yaml | 14 ++++++++ .../esp32_ble_server/test_esp32_ble_server.py | 28 +++++++++++++++ 12 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/esp32_ble_server/config/improv_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/own_service.yaml diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 6e6fb0e30d..fc95760cf8 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -100,21 +100,38 @@ void ESP32BLE::disable() { #ifdef USE_ESP32_BLE_ADVERTISING void ESP32BLE::advertising_start() { this->advertising_init_(); - if (!this->is_active()) + this->advertising_ref_count_++; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_stop() { + if (this->advertising_ref_count_ == 0) return; - this->advertising_->start(); + this->advertising_ref_count_--; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_refresh() { + if (this->advertising_ == nullptr || !this->is_active()) + return; + // Advertise while any component still needs it, otherwise stop + if (this->advertising_ref_count_ == 0) { + this->advertising_->stop(); + } else { + this->advertising_->start(); + } } void ESP32BLE::advertising_set_service_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_service_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_manufacturer_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_manufacturer_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_service_data_and_name(std::span data, bool include_name) { @@ -136,7 +153,7 @@ void ESP32BLE::advertising_set_service_data_and_name(std::span da this->advertising_->set_service_data(data); } - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_register_raw_advertisement_callback(std::function &&callback) { @@ -147,13 +164,13 @@ void ESP32BLE::advertising_register_raw_advertisement_callback(std::functionadvertising_init_(); this->advertising_->add_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_remove_service_uuid(ESPBTUUID uuid) { this->advertising_init_(); this->advertising_->remove_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } #endif @@ -575,6 +592,10 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { } this->state_ = BLE_COMPONENT_STATE_ACTIVE; +#ifdef USE_ESP32_BLE_ADVERTISING + // Requests made before the stack was up (or before it was re-enabled) take effect now + this->advertising_refresh(); +#endif } } diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 2a355a6c8b..7d2d0438a4 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -114,7 +114,17 @@ class ESP32BLE final : public Component { void set_name(const char *name) { this->name_ = name; } #ifdef USE_ESP32_BLE_ADVERTISING + /** Request advertising on behalf of a component. + * + * Requests are reference counted: advertising runs until every component that called + * advertising_start() has released it again with advertising_stop(). Each component must + * pair its calls, so nothing advertises until something actually asks for it. + */ void advertising_start(); + /// Release a request made with advertising_start(); advertising stops at the last release. + void advertising_stop(); + /// Apply the current payload and request count: advertise while requested, otherwise stop. + void advertising_refresh(); void advertising_set_service_data(const std::vector &data); void advertising_set_manufacturer_data(const std::vector &data); void advertising_set_appearance(uint16_t appearance) { this->appearance_ = appearance; } @@ -226,6 +236,9 @@ class ESP32BLE final : public Component { // 1-byte aligned members (grouped together to minimize padding) BLEComponentState state_{BLE_COMPONENT_STATE_OFF}; // 1 byte (uint8_t enum) bool enable_on_boot_{}; // 1 byte +#ifdef USE_ESP32_BLE_ADVERTISING + uint8_t advertising_ref_count_{0}; // 1 byte, number of components requesting advertising +#endif #ifdef ESPHOME_ESP32_BLE_EXTENDED_AUTH_PARAMS optional auth_req_mode_; diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp index 9f1723430b..ab728f9f6f 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp @@ -67,6 +67,8 @@ void ESP32BLEBeacon::setup() { this->on_advertise_(); } }); + // A beacon always needs the device to advertise, and never releases the request + global_ble->advertising_start(); } void ESP32BLEBeacon::on_advertise_() { diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 855a3be29b..d8095cd702 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -596,6 +596,18 @@ async def to_code(config): cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) cg.add(var.set_max_clients(config[CONF_MAX_CLIENTS])) + # Only advertise for the server itself when the configuration gives clients something to + # find. A server that is auto-loaded purely to host a runtime service (esp32_improv) stays + # silent until that service asks for advertising. + cg.add( + var.set_advertising_required( + CONF_MANUFACTURER_DATA in config + or any( + not uuid_is(service_config[CONF_UUID], DEVICE_INFORMATION_SERVICE_UUID) + for service_config in config[CONF_SERVICES] + ) + ) + ) if CONF_MANUFACTURER_DATA in config: cg.add(var.set_manufacturer_data(config[CONF_MANUFACTURER_DATA])) for service_config in config[CONF_SERVICES]: diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp index 2dea1666bb..45679b9b98 100644 --- a/esphome/components/esp32_ble_server/ble_server.cpp +++ b/esphome/components/esp32_ble_server/ble_server.cpp @@ -81,6 +81,7 @@ void BLEServer::loop() { if (this->device_information_service_->is_running()) { this->state_ = RUNNING; this->restart_advertising_(); + this->request_advertising_(); ESP_LOGD(TAG, "BLE server setup successfully"); } else if (this->device_information_service_->is_created()) { this->device_information_service_->start(); @@ -98,6 +99,20 @@ void BLEServer::restart_advertising_() { } } +void BLEServer::request_advertising_() { + if (!this->advertising_required_ || this->advertising_requested_) + return; + this->advertising_requested_ = true; + this->parent_->advertising_start(); +} + +void BLEServer::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + this->parent_->advertising_stop(); +} + BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles) { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE char uuid_buf[esp32_ble::UUID_STR_LEN]; @@ -170,7 +185,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga this->add_client_(param->connect.conn_id); // Resume advertising so additional clients can discover and connect if (this->client_count_ < this->max_clients_) { - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); } this->dispatch_callbacks_(CallbackType::ON_CONNECT, param->connect.conn_id); break; @@ -178,7 +193,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga case ESP_GATTS_DISCONNECT_EVT: { ESP_LOGD(TAG, "BLE Client disconnected"); this->remove_client_(param->disconnect.conn_id); - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); this->dispatch_callbacks_(CallbackType::ON_DISCONNECT, param->disconnect.conn_id); break; } @@ -226,6 +241,8 @@ void BLEServer::remove_client_(uint16_t conn_id) { } void BLEServer::ble_before_disabled_event_handler() { + // Advertising is re-requested once the server is running again after BLE is re-enabled + this->release_advertising_(); // Delete all clients this->client_count_ = 0; // Delete all services diff --git a/esphome/components/esp32_ble_server/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h index fdd92812cd..7869c73cc5 100644 --- a/esphome/components/esp32_ble_server/ble_server.h +++ b/esphome/components/esp32_ble_server/ble_server.h @@ -38,6 +38,13 @@ class BLEServer final : public Component, public Parented { this->restart_advertising_(); } + /** Whether this server needs the device to advertise so clients can find and connect to it. + * + * False for a server that only hosts services created at runtime (e.g. esp32_improv), which + * request advertising themselves for as long as they need it. + */ + void set_advertising_required(bool required) { this->advertising_required_ = required; } + void set_max_clients(uint8_t max_clients) { this->max_clients_ = max_clients; } uint8_t get_max_clients() const { return this->max_clients_; } @@ -82,6 +89,8 @@ class BLEServer final : public Component, public Parented { }; void restart_advertising_(); + void request_advertising_(); + void release_advertising_(); int8_t find_client_index_(uint16_t conn_id) const; void add_client_(uint16_t conn_id); @@ -93,6 +102,8 @@ class BLEServer final : public Component, public Parented { std::vector manufacturer_data_{}; esp_gatt_if_t gatts_if_{0}; bool registered_{false}; + bool advertising_required_{true}; + bool advertising_requested_{false}; uint16_t clients_[USE_ESP32_BLE_MAX_CONNECTIONS]{}; uint8_t client_count_{0}; diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 4756fba637..9ec6eb7bab 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -112,6 +112,7 @@ void ESP32ImprovComponent::loop() { this->state_callback_.call(this->state_, this->error_state_); #endif } + this->release_advertising_(); this->incoming_data_.clear(); return; } @@ -143,8 +144,9 @@ void ESP32ImprovComponent::loop() { ESP_LOGV(TAG, "Starting with device name advertising"); this->advertising_device_name_ = true; this->last_name_adv_time_ = App.get_loop_component_start_time(); + // Set the payload before requesting, so advertising starts exactly once esp32_ble::global_ble->advertising_set_service_data_and_name(std::span{}, true); - esp32_ble::global_ble->advertising_start(); + this->request_advertising_(); // Set initial state based on whether we have an authorizer this->set_state_(this->get_initial_state_(), false); @@ -326,6 +328,8 @@ void ESP32ImprovComponent::stop() { this->set_timeout("end-service", STOP_ADVERTISING_DELAY, [this] { if (this->state_ == improv::STATE_STOPPED || this->service_ == nullptr) return; + // Release first so removing the service UUID does not restart advertising on the way out + this->release_advertising_(); this->service_->stop(); this->set_state_(improv::STATE_STOPPED); }); @@ -520,6 +524,20 @@ void ESP32ImprovComponent::update_advertising_type_() { } } +void ESP32ImprovComponent::request_advertising_() { + if (this->advertising_requested_) + return; + this->advertising_requested_ = true; + esp32_ble::global_ble->advertising_start(); +} + +void ESP32ImprovComponent::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + esp32_ble::global_ble->advertising_stop(); +} + improv::State ESP32ImprovComponent::get_initial_state_() const { #ifdef USE_BINARY_SENSOR // If we have an authorizer, start in awaiting authorization state diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index 414948c977..a40d60552a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -104,8 +104,11 @@ class ESP32ImprovComponent final : public Component, public improv_base::ImprovB bool status_indicator_state_{false}; uint32_t last_name_adv_time_{0}; bool advertising_device_name_{false}; + bool advertising_requested_{false}; void set_status_indicator_state_(bool state); void update_advertising_type_(); + void request_advertising_(); + void release_advertising_(); void set_state_(improv::State state, bool update_advertising = true); void set_error_(improv::Error error); diff --git a/tests/component_tests/esp32_ble_server/config/improv_only.yaml b/tests/component_tests/esp32_ble_server/config/improv_only.yaml new file mode 100644 index 0000000000..8a5c3ba638 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/improv_only.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32 + +wifi: + ssid: MySSID + password: password1 + +# esp32_ble_server is only auto-loaded here, so it has no services of its own. +esp32_improv: + authorizer: none diff --git a/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml new file mode 100644 index 0000000000..b7bdae4af7 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + manufacturer_data: [0x72, 0x04, 0x00, 0x23] diff --git a/tests/component_tests/esp32_ble_server/config/own_service.yaml b/tests/component_tests/esp32_ble_server/config/own_service.yaml new file mode 100644 index 0000000000..c7ef0287b0 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/own_service.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + services: + - uuid: 2a24b789-7aab-4535-af3e-ee76a35cc12d + characteristics: + - uuid: cad48e28-7fbe-41cf-bae9-d77a6c233423 + read: true + value: [1, 2, 3, 4] diff --git a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py index 88307d0dcf..4b7ab79a81 100644 --- a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py +++ b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py @@ -1,5 +1,10 @@ """Tests for esp32_ble_server configuration helpers.""" +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + import pytest from esphome.components.esp32_ble_server import ( @@ -45,3 +50,26 @@ def test_uuid_is_matches_descriptor_short_strings(uuid16) -> None: assert uuid_is(uuid16, uuid16) assert uuid_is(f"{uuid16:04X}", uuid16) assert uuid_is(f"{uuid16:08X}", uuid16) + + +@pytest.mark.parametrize( + ("config_file", "required"), + [ + # Auto-loaded by esp32_improv only: nothing to find until Improv asks for it + ("improv_only.yaml", False), + # The configuration defines a service clients are meant to connect to + ("own_service.yaml", True), + # Manufacturer data is only useful if it is actually broadcast + ("manufacturer_data_only.yaml", True), + ], +) +def test_advertising_required( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + required: bool, +) -> None: + """The server only requests advertising when the configuration needs it.""" + main_cpp = generate_main(component_config_path(config_file)) + + assert f"set_advertising_required({str(required).lower()})" in main_cpp From e36445fa5feb4db65586186ec823a225339b8885 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sat, 5 Sep 2026 06:00:25 -0500 Subject: [PATCH 034/178] [usb_uart] Keep the comm interface number valid when its claim fails (#18968) --- esphome/components/usb_uart/usb_uart.cpp | 12 +++++++----- esphome/components/usb_uart/usb_uart.h | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/esphome/components/usb_uart/usb_uart.cpp b/esphome/components/usb_uart/usb_uart.cpp index cf66e4c369..60b7fe4e9c 100644 --- a/esphome/components/usb_uart/usb_uart.cpp +++ b/esphome/components/usb_uart/usb_uart.cpp @@ -434,11 +434,12 @@ void USBUartTypeCdcAcm::on_connected() { auto err_comm = usb_host_interface_claim(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number, 0); if (err_comm != ESP_OK) { + // Continue anyway: the interface number stays valid for CDC request addressing ESP_LOGW(TAG, "Could not claim comm interface %d: %s", channel->cdc_dev_.interrupt_interface_number, esp_err_to_name(err_comm)); - channel->cdc_dev_.interrupt_interface_number = 0xFF; // Mark as unavailable, but continue anyway } else { ESP_LOGD(TAG, "Claimed comm interface %d", channel->cdc_dev_.interrupt_interface_number); + channel->cdc_dev_.interrupt_interface_claimed = true; } } auto err = @@ -465,14 +466,15 @@ void USBUartTypeCdcAcm::on_disconnected() { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); } - if (channel->cdc_dev_.notify_ep != nullptr) { + // Only tear down the notify pipe when we claimed its interface ourselves; + // no transfer is ever submitted on it, so there is nothing else to cancel. + if (channel->cdc_dev_.notify_ep != nullptr && channel->cdc_dev_.interrupt_interface_claimed) { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); } - if (channel->cdc_dev_.interrupt_interface_number != 0xFF && - channel->cdc_dev_.interrupt_interface_number != channel->cdc_dev_.bulk_interface_number) { + if (channel->cdc_dev_.interrupt_interface_claimed) { usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number); - channel->cdc_dev_.interrupt_interface_number = 0xFF; + channel->cdc_dev_.interrupt_interface_claimed = false; } usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.bulk_interface_number); // Reset the input and output started flags to their initial state to avoid the possibility of spurious restarts diff --git a/esphome/components/usb_uart/usb_uart.h b/esphome/components/usb_uart/usb_uart.h index 00b34fb942..9d87bf964c 100644 --- a/esphome/components/usb_uart/usb_uart.h +++ b/esphome/components/usb_uart/usb_uart.h @@ -34,7 +34,10 @@ struct CdcEps { const usb_ep_desc_t *in_ep; const usb_ep_desc_t *out_ep; uint8_t bulk_interface_number; + // Also the wIndex target for CDC class requests (SET_LINE_CODING etc.), so it + // must remain valid even when the interface itself is not claimed. uint8_t interrupt_interface_number; + bool interrupt_interface_claimed{false}; }; enum CH34xChipType : uint8_t { From 745eb3010910a400c14527d0dd8e1fd5fa7ac984 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:07:15 +0200 Subject: [PATCH 035/178] Bump bundled esphome-device-builder to 1.14.1 (#18981) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7952616496..2d4ddbef5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 RUN \ platformio settings set enable_telemetry No \ From 7089dae3b63f57db6435202c14668001d0f0b595 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:25:58 +0000 Subject: [PATCH 036/178] Bump bundled esphome-device-builder to 1.14.2 (#18988) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d4ddbef5d..b5170864a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 RUN \ platformio settings set enable_telemetry No \ From 011497d6eeed511d55ec556db334f154f9dfc514 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:28:02 +0200 Subject: [PATCH 037/178] Bump bundled esphome-device-builder to 1.14.3 (#18996) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5170864a3..e875851bfb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 RUN \ platformio settings set enable_telemetry No \ From 18220e0b3940727d2f0657247ba0b749403b9db4 Mon Sep 17 00:00:00 2001 From: Ricardo Sanz Date: Sun, 6 Sep 2026 23:03:07 +0200 Subject: [PATCH 038/178] [climate][template] New template climate component (#14455) --- esphome/components/climate/__init__.py | 13 + .../components/template/climate/__init__.py | 465 ++++++++++++++++++ .../components/template/climate/automation.h | 57 +++ .../template/climate/template_climate.cpp | 164 ++++++ .../template/climate/template_climate.h | 92 ++++ esphome/config_validation.py | 1 + .../template/test_template_climate.py | 145 ++++++ tests/components/climate/common.yaml | 3 +- tests/components/template/common-base.yaml | 113 +++++ .../fixtures/template_climate_basic.yaml | 72 +++ .../template_climate_custom_modes.yaml | 47 ++ .../template_climate_nonoptimistic.yaml | 56 +++ .../template_climate_on_control_ordering.yaml | 26 + .../template_climate_publish_all_fields.yaml | 63 +++ .../template_climate_sensor_push.yaml | 49 ++ .../template_climate_set_actions.yaml | 89 ++++ ...emplate_climate_two_point_temperature.yaml | 52 ++ .../test_template_climate_basic.py | 146 ++++++ .../test_template_climate_custom_modes.py | 98 ++++ .../test_template_climate_nonoptimistic.py | 107 ++++ ...st_template_climate_on_control_ordering.py | 83 ++++ ...est_template_climate_publish_all_fields.py | 96 ++++ .../test_template_climate_sensor_push.py | 88 ++++ .../test_template_climate_set_actions.py | 114 +++++ ..._template_climate_two_point_temperature.py | 118 +++++ 25 files changed, 2355 insertions(+), 2 deletions(-) create mode 100644 esphome/components/template/climate/__init__.py create mode 100644 esphome/components/template/climate/automation.h create mode 100644 esphome/components/template/climate/template_climate.cpp create mode 100644 esphome/components/template/climate/template_climate.h create mode 100644 tests/component_tests/template/test_template_climate.py create mode 100644 tests/integration/fixtures/template_climate_basic.yaml create mode 100644 tests/integration/fixtures/template_climate_custom_modes.yaml create mode 100644 tests/integration/fixtures/template_climate_nonoptimistic.yaml create mode 100644 tests/integration/fixtures/template_climate_on_control_ordering.yaml create mode 100644 tests/integration/fixtures/template_climate_publish_all_fields.yaml create mode 100644 tests/integration/fixtures/template_climate_sensor_push.yaml create mode 100644 tests/integration/fixtures/template_climate_set_actions.yaml create mode 100644 tests/integration/fixtures/template_climate_two_point_temperature.yaml create mode 100644 tests/integration/test_template_climate_basic.py create mode 100644 tests/integration/test_template_climate_custom_modes.py create mode 100644 tests/integration/test_template_climate_nonoptimistic.py create mode 100644 tests/integration/test_template_climate_on_control_ordering.py create mode 100644 tests/integration/test_template_climate_publish_all_fields.py create mode 100644 tests/integration/test_template_climate_sensor_push.py create mode 100644 tests/integration/test_template_climate_set_actions.py create mode 100644 tests/integration/test_template_climate_two_point_temperature.py diff --git a/esphome/components/climate/__init__.py b/esphome/components/climate/__init__.py index 80dd913fba..3fbca1a6d0 100644 --- a/esphome/components/climate/__init__.py +++ b/esphome/components/climate/__init__.py @@ -125,6 +125,19 @@ CLIMATE_SWING_MODES = { validate_climate_swing_mode = cv.enum(CLIMATE_SWING_MODES, upper=True) +ClimateAction = climate_ns.enum("ClimateAction") +CLIMATE_ACTIONS = { + "OFF": ClimateAction.CLIMATE_ACTION_OFF, + "COOLING": ClimateAction.CLIMATE_ACTION_COOLING, + "HEATING": ClimateAction.CLIMATE_ACTION_HEATING, + "IDLE": ClimateAction.CLIMATE_ACTION_IDLE, + "DRYING": ClimateAction.CLIMATE_ACTION_DRYING, + "FAN": ClimateAction.CLIMATE_ACTION_FAN, + "DEFROSTING": ClimateAction.CLIMATE_ACTION_DEFROSTING, +} + +validate_climate_action = cv.enum(CLIMATE_ACTIONS, upper=True) + CONF_MIN_HUMIDITY = "min_humidity" CONF_MAX_HUMIDITY = "max_humidity" CONF_TARGET_HUMIDITY = "target_humidity" diff --git a/esphome/components/template/climate/__init__.py b/esphome/components/template/climate/__init__.py new file mode 100644 index 0000000000..c39ea8f80e --- /dev/null +++ b/esphome/components/template/climate/__init__.py @@ -0,0 +1,465 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import climate, sensor +from esphome.components.climate import climate_ns +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTION, + CONF_CURRENT_TEMPERATURE, + CONF_CUSTOM_FAN_MODE, + CONF_CUSTOM_FAN_MODES, + CONF_CUSTOM_PRESET, + CONF_CUSTOM_PRESETS, + CONF_FAN_MODE, + CONF_HUMIDITY_SENSOR, + CONF_ID, + CONF_INITIAL_STATE, + CONF_MODE, + CONF_OPTIMISTIC, + CONF_PRESET, + CONF_RESTORE_MODE, + CONF_SENSOR, + CONF_SUPPORTED_FAN_MODES, + CONF_SUPPORTED_MODES, + CONF_SUPPORTED_PRESETS, + CONF_SUPPORTED_SWING_MODES, + CONF_SWING_MODE, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.types import ConfigType + +from .. import template_ns + +CONF_CURRENT_HUMIDITY = "current_humidity" +CONF_TARGET_HUMIDITY = "target_humidity" +CONF_SUPPORTS_ACTION = "supports_action" +CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE = "supports_two_point_target_temperature" +CONF_SUPPORTS_TARGET_HUMIDITY = "supports_target_humidity" +CONF_SUPPORTS_CURRENT_TEMPERATURE = "supports_current_temperature" +CONF_SUPPORTS_CURRENT_HUMIDITY = "supports_current_humidity" +CONF_SET_MODE_ACTION = "set_mode_action" +CONF_SET_TARGET_TEMPERATURE_ACTION = "set_target_temperature_action" +CONF_SET_TARGET_TEMPERATURE_LOW_ACTION = "set_target_temperature_low_action" +CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION = "set_target_temperature_high_action" +CONF_SET_TARGET_HUMIDITY_ACTION = "set_target_humidity_action" +CONF_SET_FAN_MODE_ACTION = "set_fan_mode_action" +CONF_SET_CUSTOM_FAN_MODE_ACTION = "set_custom_fan_mode_action" +CONF_SET_SWING_MODE_ACTION = "set_swing_mode_action" +CONF_SET_PRESET_ACTION = "set_preset_action" +CONF_SET_CUSTOM_PRESET_ACTION = "set_custom_preset_action" + +TemplateClimate = template_ns.class_("TemplateClimate", climate.Climate, cg.Component) +TemplateClimatePublishAction = template_ns.class_( + "TemplateClimatePublishAction", + automation.Action, + cg.Parented.template(TemplateClimate), +) + +TemplateClimateRestoreMode = template_ns.enum( + "TemplateClimateRestoreMode", is_class=True +) +CLIMATE_RESTORE_MODES = { + "NO_RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + "RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +} + +# Per-field actions that forward a requested value on. The third item is the type of `x`. +SET_ACTIONS = ( + (CONF_SET_MODE_ACTION, "get_set_mode_trigger", climate.ClimateMode), + ( + CONF_SET_TARGET_TEMPERATURE_ACTION, + "get_set_target_temperature_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + "get_set_target_temperature_low_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + "get_set_target_temperature_high_trigger", + cg.float_, + ), + (CONF_SET_TARGET_HUMIDITY_ACTION, "get_set_target_humidity_trigger", cg.float_), + (CONF_SET_FAN_MODE_ACTION, "get_set_fan_mode_trigger", climate.ClimateFanMode), + ( + CONF_SET_CUSTOM_FAN_MODE_ACTION, + "get_set_custom_fan_mode_trigger", + cg.StringRef, + ), + ( + CONF_SET_SWING_MODE_ACTION, + "get_set_swing_mode_trigger", + climate.ClimateSwingMode, + ), + (CONF_SET_PRESET_ACTION, "get_set_preset_trigger", climate.ClimatePreset), + (CONF_SET_CUSTOM_PRESET_ACTION, "get_set_custom_preset_trigger", cg.StringRef), +) + +# supports_* keys have no default so that an omitted key can mean "derive it from the sensor or +# set action that makes the trait useful", which is not expressible once a default fills it in. +DERIVED_SUPPORTS = ( + (CONF_SUPPORTS_CURRENT_TEMPERATURE, (CONF_SENSOR,)), + (CONF_SUPPORTS_CURRENT_HUMIDITY, (CONF_HUMIDITY_SENSOR,)), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + ), + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, (CONF_SET_TARGET_HUMIDITY_ACTION,)), +) + + +# Custom fan modes/presets are opaque user-defined strings with no build-time correctness check +# elsewhere (Climate::set_supported_custom_fan_modes()/set_supported_custom_presets() don't block +# empty entries), so reject empty ones here -- they could never be selected at runtime anyway. +validate_custom_climate_string = cv.All(cv.string_strict, cv.Length(min=1)) + + +def _validate_two_point(config: ConfigType) -> ConfigType: + has_low = CONF_TARGET_TEMPERATURE_LOW in config + has_high = CONF_TARGET_TEMPERATURE_HIGH in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE_LOW}' and '{CONF_TARGET_TEMPERATURE_HIGH}' must be used together" + ) + if (has_low or has_high) and CONF_TARGET_TEMPERATURE in config: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' cannot be used together with " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}'" + ) + return config + + +def _validate_set_actions(config: ConfigType) -> ConfigType: + has_low = CONF_SET_TARGET_TEMPERATURE_LOW_ACTION in config + has_high = CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}' and " + f"'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}' must be used together" + ) + if (has_low or has_high) and CONF_SET_TARGET_TEMPERATURE_ACTION in config: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_ACTION}' cannot be used together with " + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}'/'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}'" + ) + return config + + +def _resolve_supports(config: ConfigType) -> ConfigType: + # An explicit true stays valid without either, since climate.template.publish can report the + # value; an explicit false that contradicts the configuration is an error, not a silent override. + for key, sources in DERIVED_SUPPORTS: + configured = [source for source in sources if source in config] + if key not in config: + config[key] = bool(configured) + elif not config[key] and configured: + raise cv.Invalid( + f"'{key}' cannot be false while '{configured[0]}' is configured", + path=[key], + ) + return config + + +def _validate_initial_state(config: ConfigType) -> ConfigType: + # Climate keeps target_temperature and target_temperature_low in a union, so writing the wrong + # one of the pair corrupts the setpoint with no runtime complaint. + if (initial_state := config.get(CONF_INITIAL_STATE)) is None: + return config + + two_point = config[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] + if two_point and CONF_TARGET_TEMPERATURE in initial_state: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' is not available while " + f"'{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' is enabled; use " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}' instead", + path=[CONF_INITIAL_STATE, CONF_TARGET_TEMPERATURE], + ) + if not two_point: + for key in (CONF_TARGET_TEMPERATURE_LOW, CONF_TARGET_TEMPERATURE_HIGH): + if key in initial_state: + raise cv.Invalid( + f"'{key}' requires '{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' to be enabled", + path=[CONF_INITIAL_STATE, key], + ) + if ( + CONF_TARGET_HUMIDITY in initial_state + and not config[CONF_SUPPORTS_TARGET_HUMIDITY] + ): + raise cv.Invalid( + f"'{CONF_TARGET_HUMIDITY}' requires '{CONF_SUPPORTS_TARGET_HUMIDITY}' to be enabled", + path=[CONF_INITIAL_STATE, CONF_TARGET_HUMIDITY], + ) + return config + + +# Same settable fields as climate.template.publish, minus current_temperature/current_humidity/ +# action: those are reported values (from a sensor or the device), not meaningful static defaults. +INITIAL_STATE_SCHEMA = cv.All( + cv.Schema( + { + cv.Optional(CONF_MODE): climate.validate_climate_mode, + cv.Optional(CONF_TARGET_TEMPERATURE): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.temperature, + cv.Optional(CONF_TARGET_HUMIDITY): cv.percentage_int, + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): climate.validate_climate_fan_mode, + cv.Exclusive( + CONF_CUSTOM_FAN_MODE, "fan_mode" + ): validate_custom_climate_string, + cv.Optional(CONF_SWING_MODE): climate.validate_climate_swing_mode, + cv.Exclusive(CONF_PRESET, "preset"): climate.validate_climate_preset, + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): validate_custom_climate_string, + } + ), + _validate_two_point, +) + +CONFIG_SCHEMA = cv.All( + climate.climate_schema(TemplateClimate) + .extend( + { + cv.Optional(CONF_SENSOR): cv.use_id(sensor.Sensor), + cv.Optional(CONF_HUMIDITY_SENSOR): cv.use_id(sensor.Sensor), + # action only ever arrives through climate.template.publish, so unlike the other + # supports_* keys there is no set action to derive it from. + cv.Optional(CONF_SUPPORTS_ACTION, default=False): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_HUMIDITY): cv.boolean, + cv.Optional(CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_TARGET_HUMIDITY): cv.boolean, + cv.Required(CONF_SUPPORTED_MODES): cv.All( + cv.ensure_list(climate.validate_climate_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_FAN_MODES): cv.All( + cv.ensure_list(climate.validate_climate_fan_mode), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_FAN_MODES): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_SWING_MODES): cv.All( + cv.ensure_list(climate.validate_climate_swing_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_PRESETS): cv.All( + cv.ensure_list(climate.validate_climate_preset), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_PRESETS): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_OPTIMISTIC, default=True): cv.boolean, + cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum( + CLIMATE_RESTORE_MODES, upper=True + ), + cv.Optional(CONF_INITIAL_STATE): INITIAL_STATE_SCHEMA, + cv.Optional(CONF_SET_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_HUMIDITY_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_FAN_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_CUSTOM_FAN_MODE_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_SWING_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_PRESET_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_CUSTOM_PRESET_ACTION): automation.validate_automation( + single=True + ), + } + ) + .extend(cv.COMPONENT_SCHEMA), + _validate_set_actions, + _resolve_supports, + _validate_initial_state, +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await climate.register_climate(var, config) + + if (sens := config.get(CONF_SENSOR)) is not None: + cg.add(var.set_sensor(await cg.get_variable(sens))) + + if (sens := config.get(CONF_HUMIDITY_SENSOR)) is not None: + cg.add(var.set_humidity_sensor(await cg.get_variable(sens))) + + for key, flag in ( + (CONF_SUPPORTS_ACTION, climate_ns.CLIMATE_SUPPORTS_ACTION), + ( + CONF_SUPPORTS_CURRENT_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_CURRENT_TEMPERATURE, + ), + (CONF_SUPPORTS_CURRENT_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_CURRENT_HUMIDITY), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_TARGET_HUMIDITY), + ): + if config[key]: + cg.add(var.add_feature_flags(flag)) + + for mode in config[CONF_SUPPORTED_MODES]: + cg.add(var.add_supported_mode(mode)) + + for mode in config.get(CONF_SUPPORTED_FAN_MODES, []): + cg.add(var.add_supported_fan_mode(mode)) + + if CONF_CUSTOM_FAN_MODES in config: + cg.add( + var.set_supported_custom_fan_modes( + cg.ArrayInitializer(*config[CONF_CUSTOM_FAN_MODES]) + ) + ) + + for mode in config.get(CONF_SUPPORTED_SWING_MODES, []): + cg.add(var.add_supported_swing_mode(mode)) + + for preset in config.get(CONF_SUPPORTED_PRESETS, []): + cg.add(var.add_supported_preset(preset)) + + if CONF_CUSTOM_PRESETS in config: + cg.add( + var.set_supported_custom_presets( + cg.ArrayInitializer(*config[CONF_CUSTOM_PRESETS]) + ) + ) + + for key, trigger_getter, arg_type in SET_ACTIONS: + if (conf := config.get(key)) is not None: + await automation.build_automation( + getattr(var, trigger_getter)(), [(arg_type, "x")], conf + ) + + cg.add(var.set_optimistic(config[CONF_OPTIMISTIC])) + cg.add(var.set_restore_mode(config[CONF_RESTORE_MODE])) + + if (initial_state := config.get(CONF_INITIAL_STATE)) is not None: + if (v := initial_state.get(CONF_MODE)) is not None: + cg.add(var.set_mode(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add(var.set_target_temperature_high(v)) + if (v := initial_state.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(v)) + if (v := initial_state.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(v)) + if (v := initial_state.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(v)) + if (v := initial_state.get(CONF_SWING_MODE)) is not None: + cg.add(var.set_swing_mode(v)) + if (v := initial_state.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(v)) + if (v := initial_state.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(v)) + + +CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.use_id(TemplateClimate), + cv.Optional(CONF_CURRENT_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_CURRENT_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_TARGET_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_MODE): cv.templatable(climate.validate_climate_mode), + cv.Optional(CONF_ACTION): cv.templatable(climate.validate_climate_action), + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): cv.templatable( + climate.validate_climate_fan_mode + ), + cv.Exclusive(CONF_CUSTOM_FAN_MODE, "fan_mode"): cv.templatable( + validate_custom_climate_string + ), + cv.Optional(CONF_SWING_MODE): cv.templatable( + climate.validate_climate_swing_mode + ), + cv.Exclusive(CONF_PRESET, "preset"): cv.templatable( + climate.validate_climate_preset + ), + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): cv.templatable( + validate_custom_climate_string + ), + } + ), + _validate_two_point, +) + + +@automation.register_action( + "climate.template.publish", + TemplateClimatePublishAction, + CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA, + synchronous=True, +) +async def climate_template_publish_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + + if (v := config.get(CONF_CURRENT_TEMPERATURE)) is not None: + cg.add(var.set_current_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_CURRENT_HUMIDITY)) is not None: + cg.add(var.set_current_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add( + var.set_target_temperature_high(await cg.templatable(v, args, cg.float_)) + ) + if (v := config.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_MODE)) is not None: + cg.add(var.set_mode(await cg.templatable(v, args, climate.ClimateMode))) + if (v := config.get(CONF_ACTION)) is not None: + cg.add(var.set_action(await cg.templatable(v, args, climate.ClimateAction))) + if (v := config.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(await cg.templatable(v, args, climate.ClimateFanMode))) + if (v := config.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(await cg.templatable(v, args, cg.std_string))) + if (v := config.get(CONF_SWING_MODE)) is not None: + cg.add( + var.set_swing_mode(await cg.templatable(v, args, climate.ClimateSwingMode)) + ) + if (v := config.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(await cg.templatable(v, args, climate.ClimatePreset))) + if (v := config.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(await cg.templatable(v, args, cg.std_string))) + + return var diff --git a/esphome/components/template/climate/automation.h b/esphome/components/template/climate/automation.h new file mode 100644 index 0000000000..49a79ace2f --- /dev/null +++ b/esphome/components/template/climate/automation.h @@ -0,0 +1,57 @@ +#pragma once + +#include "template_climate.h" +#include "esphome/core/automation.h" + +namespace esphome::template_ { + +template +class TemplateClimatePublishAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(float, current_temperature) + TEMPLATABLE_VALUE(float, current_humidity) + TEMPLATABLE_VALUE(float, target_temperature) + TEMPLATABLE_VALUE(float, target_temperature_low) + TEMPLATABLE_VALUE(float, target_temperature_high) + TEMPLATABLE_VALUE(float, target_humidity) + TEMPLATABLE_VALUE(climate::ClimateMode, mode) + TEMPLATABLE_VALUE(climate::ClimateAction, action) + TEMPLATABLE_VALUE(climate::ClimateFanMode, fan_mode) + TEMPLATABLE_VALUE(std::string, custom_fan_mode) + TEMPLATABLE_VALUE(climate::ClimateSwingMode, swing_mode) + TEMPLATABLE_VALUE(climate::ClimatePreset, preset) + TEMPLATABLE_VALUE(std::string, custom_preset) + + void play(const Ts &...x) override { + if (this->current_temperature_.has_value()) + this->parent_->current_temperature = this->current_temperature_.value(x...); + if (this->current_humidity_.has_value()) + this->parent_->current_humidity = this->current_humidity_.value(x...); + if (this->target_temperature_.has_value()) + this->parent_->set_target_temperature(this->target_temperature_.value(x...)); + if (this->target_temperature_low_.has_value()) + this->parent_->set_target_temperature_low(this->target_temperature_low_.value(x...)); + if (this->target_temperature_high_.has_value()) + this->parent_->set_target_temperature_high(this->target_temperature_high_.value(x...)); + if (this->target_humidity_.has_value()) + this->parent_->set_target_humidity(this->target_humidity_.value(x...)); + if (this->mode_.has_value()) + this->parent_->set_mode(this->mode_.value(x...)); + if (this->action_.has_value()) + this->parent_->action = this->action_.value(x...); + if (this->fan_mode_.has_value()) + this->parent_->set_fan_mode(this->fan_mode_.value(x...)); + if (this->custom_fan_mode_.has_value()) + this->parent_->set_custom_fan_mode(StringRef(this->custom_fan_mode_.value(x...))); + if (this->swing_mode_.has_value()) + this->parent_->set_swing_mode(this->swing_mode_.value(x...)); + if (this->preset_.has_value()) + this->parent_->set_preset(this->preset_.value(x...)); + if (this->custom_preset_.has_value()) + this->parent_->set_custom_preset(StringRef(this->custom_preset_.value(x...))); + + this->parent_->publish_state(); + } +}; + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.cpp b/esphome/components/template/climate/template_climate.cpp new file mode 100644 index 0000000000..a7a4d2ccab --- /dev/null +++ b/esphome/components/template/climate/template_climate.cpp @@ -0,0 +1,164 @@ +#include "template_climate.h" +#include "esphome/core/log.h" + +namespace esphome::template_ { + +static const char *const TAG = "template.climate"; + +void TemplateClimate::setup() { + if (this->restore_mode_ == TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE) { + auto restore = this->restore_state_(); + if (restore.has_value()) { + restore->apply(this); + } + } + + // Sensors publish every reading, not just changes, so only re-publish when the value moved. + // NAN means the sensor went unavailable and is passed through rather than dropped; the second + // check stops an unavailable sensor re-publishing forever, since NAN never equals NAN. +#ifdef USE_SENSOR + if (this->sensor_ != nullptr) { + this->current_temperature = this->sensor_->state; + this->sensor_->add_on_state_callback([this](float state) { + if (state != this->current_temperature && !(std::isnan(state) && std::isnan(this->current_temperature))) { + this->current_temperature = state; + this->publish_state(); + } + }); + } + + if (this->humidity_sensor_ != nullptr) { + this->current_humidity = this->humidity_sensor_->state; + this->humidity_sensor_->add_on_state_callback([this](float state) { + if (state != this->current_humidity && !(std::isnan(state) && std::isnan(this->current_humidity))) { + this->current_humidity = state; + this->publish_state(); + } + }); + } +#endif +} + +void TemplateClimate::dump_config() { + LOG_CLIMATE("", "Template Climate", this); + ESP_LOGCONFIG(TAG, " Optimistic: %s", YESNO(this->optimistic_)); +} + +void TemplateClimate::control(const climate::ClimateCall &call) { + // Each field present fires its set_*_action; on_control sees the whole call. optimistic: true + // also applies the values right away, false waits for a climate.template.publish report. + if (auto mode = call.get_mode()) { + if (this->optimistic_) + this->mode = *mode; + this->set_mode_trigger_.trigger(*mode); + } + + if (auto target_temp = call.get_target_temperature()) { + if (this->optimistic_) + this->target_temperature = *target_temp; + this->set_target_temperature_trigger_.trigger(*target_temp); + } + + if (auto target_temp_low = call.get_target_temperature_low()) { + if (this->optimistic_) + this->target_temperature_low = *target_temp_low; + this->set_target_temperature_low_trigger_.trigger(*target_temp_low); + } + + if (auto target_temp_high = call.get_target_temperature_high()) { + if (this->optimistic_) + this->target_temperature_high = *target_temp_high; + this->set_target_temperature_high_trigger_.trigger(*target_temp_high); + } + + if (auto target_humidity = call.get_target_humidity()) { + if (this->optimistic_) + this->target_humidity = *target_humidity; + this->set_target_humidity_trigger_.trigger(*target_humidity); + } + + if (auto fan_mode = call.get_fan_mode()) { + if (this->optimistic_) + this->set_fan_mode_(*fan_mode); + this->set_fan_mode_trigger_.trigger(*fan_mode); + } + + if (call.has_custom_fan_mode()) { + if (this->optimistic_) + this->set_custom_fan_mode_(call.get_custom_fan_mode()); + this->set_custom_fan_mode_trigger_.trigger(call.get_custom_fan_mode()); + } + + if (auto swing_mode = call.get_swing_mode()) { + if (this->optimistic_) + this->swing_mode = *swing_mode; + this->set_swing_mode_trigger_.trigger(*swing_mode); + } + + if (auto preset = call.get_preset()) { + if (this->optimistic_) + this->set_preset_(*preset); + this->set_preset_trigger_.trigger(*preset); + } + + if (call.has_custom_preset()) { + if (this->optimistic_) + this->set_custom_preset_(call.get_custom_preset()); + this->set_custom_preset_trigger_.trigger(call.get_custom_preset()); + } + + if (this->optimistic_) + this->publish_state(); +} + +// A climate.template.publish report (and initial_state:) never goes through ClimateCall::validate_(), +// so check here instead -- otherwise a typo is published as state the receiving end will reject. +void TemplateClimate::set_mode(climate::ClimateMode mode) { + if (!this->traits_.supports_mode(mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported mode %u", this->get_name().c_str(), static_cast(mode)); + return; + } + this->mode = mode; +} + +void TemplateClimate::set_swing_mode(climate::ClimateSwingMode swing_mode) { + if (!this->traits_.supports_swing_mode(swing_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported swing mode %u", this->get_name().c_str(), static_cast(swing_mode)); + return; + } + this->swing_mode = swing_mode; +} + +void TemplateClimate::set_fan_mode(climate::ClimateFanMode fan_mode) { + if (!this->traits_.supports_fan_mode(fan_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported fan mode %u", this->get_name().c_str(), static_cast(fan_mode)); + return; + } + this->set_fan_mode_(fan_mode); +} + +void TemplateClimate::set_preset(climate::ClimatePreset preset) { + if (!this->traits_.supports_preset(preset)) { + ESP_LOGW(TAG, "'%s' - Unsupported preset %u", this->get_name().c_str(), static_cast(preset)); + return; + } + this->set_preset_(preset); +} + +void TemplateClimate::set_custom_fan_mode(StringRef mode) { + if (this->find_custom_fan_mode_(mode.c_str(), mode.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom fan mode '%s'", this->get_name().c_str(), mode.c_str()); + return; + } + this->set_custom_fan_mode_(mode); +} + +void TemplateClimate::set_custom_preset(StringRef preset) { + if (this->find_custom_preset_(preset.c_str(), preset.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom preset '%s'", this->get_name().c_str(), preset.c_str()); + return; + } + this->set_custom_preset_(preset); +} + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.h b/esphome/components/template/climate/template_climate.h new file mode 100644 index 0000000000..5448488c34 --- /dev/null +++ b/esphome/components/template/climate/template_climate.h @@ -0,0 +1,92 @@ +#pragma once + +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/components/climate/climate.h" +#ifdef USE_SENSOR +#include "esphome/components/sensor/sensor.h" +#endif + +namespace esphome::template_ { + +enum class TemplateClimateRestoreMode { + TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +}; + +class TemplateClimate final : public climate::Climate, public Component { + public: + void setup() override; + void dump_config() override; + + climate::ClimateTraits traits() override { return this->traits_; } + + void add_feature_flags(uint32_t flags) { this->traits_.add_feature_flags(flags); } + +#ifdef USE_SENSOR + // The matching feature flag is added from codegen, so the configuration alone decides it. + void set_sensor(sensor::Sensor *sensor) { this->sensor_ = sensor; } + void set_humidity_sensor(sensor::Sensor *sensor) { this->humidity_sensor_ = sensor; } +#endif + + void add_supported_mode(climate::ClimateMode mode) { this->traits_.add_supported_mode(mode); } + void add_supported_fan_mode(climate::ClimateFanMode mode) { this->traits_.add_supported_fan_mode(mode); } + void add_supported_swing_mode(climate::ClimateSwingMode mode) { this->traits_.add_supported_swing_mode(mode); } + void add_supported_preset(climate::ClimatePreset preset) { this->traits_.add_supported_preset(preset); } + + void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } + void set_restore_mode(TemplateClimateRestoreMode restore_mode) { this->restore_mode_ = restore_mode; } + + // Fired from control() for each field the call carries, so a device-backed config can forward + // it on. Which of these are configured also decides the two-point/target-humidity traits. + Trigger *get_set_mode_trigger() { return &this->set_mode_trigger_; } + Trigger *get_set_target_temperature_trigger() { return &this->set_target_temperature_trigger_; } + Trigger *get_set_target_temperature_low_trigger() { return &this->set_target_temperature_low_trigger_; } + Trigger *get_set_target_temperature_high_trigger() { return &this->set_target_temperature_high_trigger_; } + Trigger *get_set_target_humidity_trigger() { return &this->set_target_humidity_trigger_; } + Trigger *get_set_fan_mode_trigger() { return &this->set_fan_mode_trigger_; } + Trigger *get_set_custom_fan_mode_trigger() { return &this->set_custom_fan_mode_trigger_; } + Trigger *get_set_swing_mode_trigger() { return &this->set_swing_mode_trigger_; } + Trigger *get_set_preset_trigger() { return &this->set_preset_trigger_; } + Trigger *get_set_custom_preset_trigger() { return &this->set_custom_preset_trigger_; } + + // Used by TemplateClimatePublishAction, which is not a Climate subclass and so cannot reach the + // protected setters, and by codegen to apply `initial_state:` before setup() runs. + void set_target_temperature(float value) { this->target_temperature = value; } + void set_target_temperature_low(float value) { this->target_temperature_low = value; } + void set_target_temperature_high(float value) { this->target_temperature_high = value; } + void set_target_humidity(float value) { this->target_humidity = value; } + void set_mode(climate::ClimateMode mode); + void set_swing_mode(climate::ClimateSwingMode mode); + void set_fan_mode(climate::ClimateFanMode mode); + void set_custom_fan_mode(const char *mode) { this->set_custom_fan_mode(StringRef(mode)); } + void set_custom_fan_mode(StringRef mode); + void set_preset(climate::ClimatePreset preset); + void set_custom_preset(const char *preset) { this->set_custom_preset(StringRef(preset)); } + void set_custom_preset(StringRef preset); + + protected: + void control(const climate::ClimateCall &call) override; + + climate::ClimateTraits traits_; + bool optimistic_{false}; + TemplateClimateRestoreMode restore_mode_{TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE}; + +#ifdef USE_SENSOR + sensor::Sensor *sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; +#endif + + Trigger set_mode_trigger_; + Trigger set_target_temperature_trigger_; + Trigger set_target_temperature_low_trigger_; + Trigger set_target_temperature_high_trigger_; + Trigger set_target_humidity_trigger_; + Trigger set_fan_mode_trigger_; + Trigger set_custom_fan_mode_trigger_; + Trigger set_swing_mode_trigger_; + Trigger set_preset_trigger_; + Trigger set_custom_preset_trigger_; +}; + +} // namespace esphome::template_ diff --git a/esphome/config_validation.py b/esphome/config_validation.py index aff39201e8..685a9d04b3 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -133,6 +133,7 @@ Upper = vol.Upper Length = vol.Length Exclusive = vol.Exclusive Inclusive = vol.Inclusive +Unique = vol.Unique ALLOW_EXTRA = vol.ALLOW_EXTRA UNDEFINED = vol.UNDEFINED RequiredFieldInvalid = vol.RequiredFieldInvalid diff --git a/tests/component_tests/template/test_template_climate.py b/tests/component_tests/template/test_template_climate.py new file mode 100644 index 0000000000..304991ea64 --- /dev/null +++ b/tests/component_tests/template/test_template_climate.py @@ -0,0 +1,145 @@ +"""Tests for template climate config validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.template.climate import ( + CONF_SET_TARGET_HUMIDITY_ACTION, + CONF_SET_TARGET_TEMPERATURE_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SUPPORTS_CURRENT_HUMIDITY, + CONF_SUPPORTS_CURRENT_TEMPERATURE, + CONF_SUPPORTS_TARGET_HUMIDITY, + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + CONF_TARGET_HUMIDITY, + _resolve_supports, + _validate_initial_state, + _validate_set_actions, +) +from esphome.const import ( + CONF_HUMIDITY_SENSOR, + CONF_INITIAL_STATE, + CONF_SENSOR, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.types import ConfigType + + +def test_supports_current_temperature_derived_from_sensor() -> None: + config: ConfigType = {CONF_SENSOR: "some_sensor"} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_without_sensor() -> None: + assert _resolve_supports({})[CONF_SUPPORTS_CURRENT_TEMPERATURE] is False + + +def test_supports_current_temperature_explicit_true_without_sensor_allowed() -> None: + # The value can still be reported with climate.template.publish. + config: ConfigType = {CONF_SUPPORTS_CURRENT_TEMPERATURE: True} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_supports_current_humidity_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_HUMIDITY_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_HUMIDITY: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_two_point_derived_from_set_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + assert _resolve_supports(config)[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] is True + + +def test_two_point_false_with_set_action_rejected() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_target_humidity_derived_from_set_action() -> None: + config: ConfigType = {CONF_SET_TARGET_HUMIDITY_ACTION: [{}]} + assert _resolve_supports(config)[CONF_SUPPORTS_TARGET_HUMIDITY] is True + + +def test_set_target_temperature_low_requires_high() -> None: + config: ConfigType = {CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}]} + with pytest.raises(cv.Invalid, match="must be used together"): + _validate_set_actions(config) + + +def test_set_target_temperature_conflicts_with_two_point_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + with pytest.raises(cv.Invalid, match="cannot be used together"): + _validate_set_actions(config) + + +def test_initial_state_target_temperature_rejected_with_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_TEMPERATURE: 21.0}, + } + with pytest.raises(cv.Invalid, match="is not available"): + _validate_initial_state(config) + + +def test_initial_state_two_point_values_rejected_without_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + }, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_target_humidity_rejected_without_support() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_HUMIDITY: 50}, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_matching_two_point_accepted() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: True, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + CONF_TARGET_HUMIDITY: 50, + }, + } + assert _validate_initial_state(config) is config diff --git a/tests/components/climate/common.yaml b/tests/components/climate/common.yaml index c28fde8eeb..49386a16d5 100644 --- a/tests/components/climate/common.yaml +++ b/tests/components/climate/common.yaml @@ -30,8 +30,7 @@ climate: - switch.turn_on: climate_heater_switch - switch.turn_off: climate_cooler_switch # Thermostat-based climate so climate.control: action variants get build - # coverage (bang_bang doesn't support fan modes, presets, etc.). Climate - # has no template platform, so thermostat is the right vehicle. + # coverage (bang_bang doesn't support fan modes, presets, etc.). - platform: thermostat id: climate_test_thermostat name: Test Thermostat diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index 92a1fc8eda..02aedaf167 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -25,6 +25,27 @@ esphome: away: !lambda "return true;" is_on: !lambda "return false;" + - climate.template.publish: + id: template_climate + current_temperature: 21.0 + mode: HEAT + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE + target_temperature: 22.0 + + # Templated + - climate.template.publish: + id: template_climate + current_temperature: !lambda "return 21.5f;" + mode: !lambda "return climate::CLIMATE_MODE_COOL;" + target_temperature: !lambda "return 23.0f;" + + - climate.template.publish: + id: template_climate_custom_modes + custom_fan_mode: "turbo" + custom_preset: "eco_plus" + # Test C++ API: set_template() with stateless lambda (no captures) # NOTE: set_template() is not intended to be a public API, but we test it to ensure it doesn't break. - lambda: |- @@ -513,6 +534,98 @@ alarm_control_panel: codes: - "1234" +climate: + - platform: template + id: template_climate + name: "Template Climate" + optimistic: true + sensor: template_template_sens + supports_action: true + supports_current_humidity: true + restore_mode: NO_RESTORE + initial_state: + mode: HEAT + target_temperature: 21.0 + fan_mode: LOW + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_action: + - logger.log: + format: "set_target_temperature_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.1f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + on_control: + - logger.log: "on_control fired" + on_state: + - logger.log: "on_state fired" + + - platform: template + id: template_climate_custom_modes + name: "Template Climate Custom Modes" + optimistic: true + sensor: template_template_sens + supported_modes: + - "OFF" + - HEAT + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + initial_state: + custom_fan_mode: eco + custom_preset: max + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + water_heater: - platform: template id: template_water_heater diff --git a/tests/integration/fixtures/template_climate_basic.yaml b/tests/integration/fixtures/template_climate_basic.yaml new file mode 100644 index 0000000000..51558b4875 --- /dev/null +++ b/tests/integration/fixtures/template_climate_basic.yaml @@ -0,0 +1,72 @@ +esphome: + name: tmpl-clim-basic + on_boot: + - climate.template.publish: + id: test_climate + action: IDLE +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Basic Climate + optimistic: true + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 55.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: "OFF" + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE diff --git a/tests/integration/fixtures/template_climate_custom_modes.yaml b/tests/integration/fixtures/template_climate_custom_modes.yaml new file mode 100644 index 0000000000..9dbfe60cb9 --- /dev/null +++ b/tests/integration/fixtures/template_climate_custom_modes.yaml @@ -0,0 +1,47 @@ +esphome: + name: tmpl-clim-custom +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Custom Mode Climate + optimistic: true + sensor: test_climate_current_temperature + supported_modes: + - "OFF" + - HEAT + - COOL + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + on_control: + - lambda: |- + if (x.has_custom_fan_mode()) + ESP_LOGD("test", "on_control custom_fan_mode=%s", x.get_custom_fan_mode().c_str()); + if (x.has_custom_preset()) + ESP_LOGD("test", "on_control custom_preset=%s", x.get_custom_preset().c_str()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + custom_fan_mode: "eco" + custom_preset: "max" diff --git a/tests/integration/fixtures/template_climate_nonoptimistic.yaml b/tests/integration/fixtures/template_climate_nonoptimistic.yaml new file mode 100644 index 0000000000..2b0c7ee132 --- /dev/null +++ b/tests/integration/fixtures/template_climate_nonoptimistic.yaml @@ -0,0 +1,56 @@ +esphome: + name: tmpl-clim-nonopt +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Template Climate Nonoptimistic + optimistic: false + supported_modes: + - "OFF" + - HEAT + - COOL + - FAN_ONLY + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + - AWAY + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +button: + - platform: template + id: simulate_device_confirmation + name: Simulate Device Confirmation + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + target_temperature: 22.5 + fan_mode: HIGH + swing_mode: VERTICAL + preset: AWAY diff --git a/tests/integration/fixtures/template_climate_on_control_ordering.yaml b/tests/integration/fixtures/template_climate_on_control_ordering.yaml new file mode 100644 index 0000000000..8366a6d21e --- /dev/null +++ b/tests/integration/fixtures/template_climate_on_control_ordering.yaml @@ -0,0 +1,26 @@ +esphome: + name: tmpl-clim-oc-order +host: +api: +logger: + +# on_control fires with the full ClimateCall (arg `x`) from the base Climate component's +# ClimateCall::perform(), before validate_()/control() run -- so when the lambda action below +# runs, the entity's own .mode is still the OLD value, even though x.get_mode() already reports +# the NEW requested value. on_state fires afterward, once control() has applied it. +climate: + - platform: template + id: test_climate + name: Test On Control Ordering + optimistic: true + supported_modes: + - "OFF" + - HEAT + on_control: + - lambda: |- + ESP_LOGD("test", "on_control requested_mode=%d current_mode_before_apply=%d", + x.get_mode().has_value() ? (int) *x.get_mode() : -1, + (int) id(test_climate).mode); + on_state: + - lambda: |- + ESP_LOGD("test", "on_state mode=%d", (int) x.mode); diff --git a/tests/integration/fixtures/template_climate_publish_all_fields.yaml b/tests/integration/fixtures/template_climate_publish_all_fields.yaml new file mode 100644 index 0000000000..e57fcc4508 --- /dev/null +++ b/tests/integration/fixtures/template_climate_publish_all_fields.yaml @@ -0,0 +1,63 @@ +esphome: + name: tmpl-clim-publish-all +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Publish All Fields + optimistic: true + # current_temperature/current_humidity/action are only sent over the API at all if their + # trait is advertised: current_temperature/current_humidity because a sensor/humidity_sensor + # is referenced below, action because supports_action is set. The sensors' fixed readings + # match what climate.template.publish pushes, so the sensor callback (guarded to only publish + # on an actual change) doesn't produce an extra, unexpected state update of its own. + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + supported_fan_modes: + - AUTO + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + on_control: + # Should never fire in this test: climate.template.publish is a pure bypass and must not + # re-trigger on_control as if the entity were freshly commanded. + - logger.log: "on_control fired" + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 20.0f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 60.0f;" + update_interval: 10ms + +button: + - platform: template + id: publish_all + name: Publish All + on_press: + - climate.template.publish: + id: test_climate + current_temperature: 20.0 + current_humidity: 60.0 + target_temperature: 23.0 + mode: HEAT + action: HEATING + fan_mode: HIGH + swing_mode: VERTICAL + preset: ECO diff --git a/tests/integration/fixtures/template_climate_sensor_push.yaml b/tests/integration/fixtures/template_climate_sensor_push.yaml new file mode 100644 index 0000000000..1fc004335d --- /dev/null +++ b/tests/integration/fixtures/template_climate_sensor_push.yaml @@ -0,0 +1,49 @@ +esphome: + name: tmpl-clim-sensor-push +host: +api: +logger: + +# No lambda/update_interval: these sensors only ever report a value when a button below +# publishes one (standing in for e.g. a BLE scan callback in a real config). +sensor: + - platform: template + id: room_temperature + name: Room Temperature + - platform: template + id: room_humidity + name: Room Humidity + +climate: + - platform: template + id: test_climate + name: Test Sensor Push Climate + optimistic: true + sensor: room_temperature + humidity_sensor: room_humidity + supported_modes: + - "OFF" + - HEAT + +button: + - platform: template + id: publish_temperature + name: Publish Temperature + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_temperature_same + name: Publish Temperature Same Value + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_humidity + name: Publish Humidity + on_press: + - sensor.template.publish: + id: room_humidity + state: 65.0 diff --git a/tests/integration/fixtures/template_climate_set_actions.yaml b/tests/integration/fixtures/template_climate_set_actions.yaml new file mode 100644 index 0000000000..b247367f64 --- /dev/null +++ b/tests/integration/fixtures/template_climate_set_actions.yaml @@ -0,0 +1,89 @@ +esphome: + name: tmpl-clim-set-act +host: +api: +logger: + +# Every settable field forwards its requested value to a set_*_action. supports_two_point and +# supports_target_humidity are not declared here: they are derived from the low/high and humidity +# set actions being present. +climate: + - platform: template + id: test_climate + name: Test Set Actions + optimistic: false + restore_mode: NO_RESTORE + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + custom_fan_modes: + - turbo + custom_presets: + - eco_plus + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_low_action: + - logger.log: + format: "set_target_temperature_low_action %.1f" + args: ["x"] + set_target_temperature_high_action: + - logger.log: + format: "set_target_temperature_high_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.0f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + +button: + - platform: template + id: report_device_state + name: Report Device State + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + + - platform: template + id: report_unsupported_mode + name: Report Unsupported Mode + on_press: + - climate.template.publish: + id: test_climate + mode: DRY diff --git a/tests/integration/fixtures/template_climate_two_point_temperature.yaml b/tests/integration/fixtures/template_climate_two_point_temperature.yaml new file mode 100644 index 0000000000..ec10785ee8 --- /dev/null +++ b/tests/integration/fixtures/template_climate_two_point_temperature.yaml @@ -0,0 +1,52 @@ +esphome: + name: tmpl-clim-two-point +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Two-Point Heatpump + optimistic: true + sensor: test_climate_current_temperature + supports_two_point_target_temperature: true + supports_target_humidity: true + supported_modes: + - "OFF" + - HEAT_COOL + - HEAT + - COOL + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature_low().has_value()) + ESP_LOGD("test", "on_control target_temperature_low=%.1f", *x.get_target_temperature_low()); + if (x.get_target_temperature_high().has_value()) + ESP_LOGD("test", "on_control target_temperature_high=%.1f", *x.get_target_temperature_high()); + if (x.get_target_humidity().has_value()) + ESP_LOGD("test", "on_control target_humidity=%.1f", *x.get_target_humidity()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 21.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT_COOL + target_temperature_low: 18.0 + target_temperature_high: 24.0 + target_humidity: 50.0 diff --git a/tests/integration/test_template_climate_basic.py b/tests/integration/test_template_climate_basic.py new file mode 100644 index 0000000000..431fd4e3e8 --- /dev/null +++ b/tests/integration/test_template_climate_basic.py @@ -0,0 +1,146 @@ +"""Integration test for template climate: sensor-pushed measured values, on_control + publish +for the settable ones. + +current_temperature/current_humidity are pushed by a referenced sensor/humidity_sensor (no +polling); action is set once at boot via climate.template.publish, since it has no sensor +equivalent. mode/target_temperature/fan_mode/swing_mode/preset are plain internal state: +on_control fires exactly once per command (never before the first one), and +climate.template.publish simulates the device reporting its own state independent of any prior +command -- that report is authoritative, overriding whatever was optimistically applied earlier. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-basic" + + +@pytest.mark.asyncio +async def test_template_climate_basic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Sensor-pushed measured values, on_control + publish for settable ones.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + # Advertised capabilities come straight from the supported_*/custom_* config lists. + assert ClimateMode.OFF in test_climate.supported_modes + assert ClimateMode.HEAT in test_climate.supported_modes + assert ClimateMode.COOL in test_climate.supported_modes + + assert ClimateFanMode.AUTO in test_climate.supported_fan_modes + assert ClimateFanMode.LOW in test_climate.supported_fan_modes + assert ClimateFanMode.HIGH in test_climate.supported_fan_modes + + assert ClimateSwingMode.OFF in test_climate.supported_swing_modes + assert ClimateSwingMode.VERTICAL in test_climate.supported_swing_modes + + assert ClimatePreset.NONE in test_climate.supported_presets + assert ClimatePreset.ECO in test_climate.supported_presets + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.current_temperature == pytest.approx(22.5, abs=0.1) + assert initial.current_humidity == pytest.approx(55.0, abs=0.1) + assert initial.action == ClimateAction.IDLE + assert initial.mode == ClimateMode.OFF + # Nothing was commanded yet: on_control must not have fired. + assert not log_lines + + # Commands apply optimistically and on_control fires with the same values. + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + client.climate_command(test_climate.key, target_temperature=22.5) + state = await wait_for_climate_state() + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.HIGH) + state = await wait_for_climate_state() + assert state.fan_mode == ClimateFanMode.HIGH + + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + state = await wait_for_climate_state() + assert state.swing_mode == ClimateSwingMode.VERTICAL + + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + state = await wait_for_climate_state() + assert state.preset == ClimatePreset.ECO + + await asyncio.sleep(0.2) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + # Exactly one on_control log line per command, none extra (e.g. from a stray republish). + assert len(log_lines) == 5 + + # measured values are untouched by any of the above (no set action exists for them). + assert state.current_temperature == pytest.approx(22.5, abs=0.1) + assert state.current_humidity == pytest.approx(55.0, abs=0.1) + assert state.action == ClimateAction.IDLE + + # The device's report is authoritative and overrides everything commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.OFF + assert state.fan_mode == ClimateFanMode.AUTO + assert state.swing_mode == ClimateSwingMode.OFF + assert state.preset == ClimatePreset.NONE diff --git a/tests/integration/test_template_climate_custom_modes.py b/tests/integration/test_template_climate_custom_modes.py new file mode 100644 index 0000000000..4817fe1ddf --- /dev/null +++ b/tests/integration/test_template_climate_custom_modes.py @@ -0,0 +1,98 @@ +"""Integration test for template climate: custom fan modes and presets. + +Same on_control (forward) + climate.template.publish (device report, authoritative) pattern as +the enum-based mode/preset fields, but for the custom string variants. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-custom" + + +@pytest.mark.asyncio +async def test_template_climate_custom_modes( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Custom fan mode/preset: traits, on_control forwarding, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + assert set(test_climate.supported_custom_fan_modes) == { + "turbo", + "silent", + "eco", + } + assert set(test_climate.supported_custom_presets) == { + "eco_plus", + "power_save", + "max", + } + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.custom_fan_mode == "" + assert initial.custom_preset == "" + + client.climate_command(test_climate.key, custom_fan_mode="turbo") + state = await wait_for_climate_state() + assert state.custom_fan_mode == "turbo" + + client.climate_command(test_climate.key, custom_preset="power_save") + state = await wait_for_climate_state() + assert state.custom_preset == "power_save" + + await asyncio.sleep(0.2) + assert any("on_control custom_fan_mode=turbo" in line for line in log_lines) + assert any("on_control custom_preset=power_save" in line for line in log_lines) + + # The device's report is authoritative and overrides what was commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.custom_fan_mode == "eco" + assert state.custom_preset == "max" diff --git a/tests/integration/test_template_climate_nonoptimistic.py b/tests/integration/test_template_climate_nonoptimistic.py new file mode 100644 index 0000000000..e922ec31b9 --- /dev/null +++ b/tests/integration/test_template_climate_nonoptimistic.py @@ -0,0 +1,107 @@ +"""Integration test for template climate: optimistic: false. + +A command still fires on_control (so a real device-backed config can forward it out), but must +NOT change the entity's own state -- only an explicit climate.template.publish call (standing in +for the device confirming the command actually took effect) does that. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-nonopt" + + +@pytest.mark.asyncio +async def test_template_climate_nonoptimistic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Nonoptimistic: a command doesn't change state until explicitly published.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + state_updates: list[aioesphomeapi.ClimateState] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + confirm_button = require_entity( + entities, "simulate_device_confirmation", ButtonInfo + ) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.mode == ClimateMode.OFF + + # Send every settable field in one command. on_control must fire with all of them, but + # nothing may be applied to the entity's own state -- no ClimateState update at all. + client.climate_command( + test_climate.key, + mode=ClimateMode.HEAT, + target_temperature=22.5, + fan_mode=ClimateFanMode.HIGH, + swing_mode=ClimateSwingMode.VERTICAL, + preset=ClimatePreset.AWAY, + ) + await asyncio.sleep(0.3) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + assert not state_updates, ( + "optimistic: false must not publish a state until climate.template.publish reports it" + ) + + # The device confirms the command actually took effect. + client.button_command(confirm_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.AWAY diff --git a/tests/integration/test_template_climate_on_control_ordering.py b/tests/integration/test_template_climate_on_control_ordering.py new file mode 100644 index 0000000000..8d212b3ccb --- /dev/null +++ b/tests/integration/test_template_climate_on_control_ordering.py @@ -0,0 +1,83 @@ +"""Integration test: on_control fires before control()/on_state, with the full ClimateCall. + +on_control's lambda argument exposes get_mode()/etc. on the *requested* ClimateCall, while the +entity's own .mode field still reflects the state *before* control() applies the change -- +proving the firing order is on_control, then control(), then on_state. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-oc-order" + + +@pytest.mark.asyncio +async def test_template_climate_on_control_ordering( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """on_control sees the requested value while the entity's own state is still the old one.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line or "on_state " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + await asyncio.sleep(0.2) + + # on_control saw the new requested mode (3 == CLIMATE_MODE_HEAT) while the entity's own + # state was still the old one (0 == CLIMATE_MODE_OFF) -- proving it fired before control(). + assert any( + "on_control requested_mode=3 current_mode_before_apply=0" in line + for line in log_lines + ) + # on_state fired afterward, reporting the now-applied mode. + assert any("on_state mode=3" in line for line in log_lines) + + control_index = next( + i for i, line in enumerate(log_lines) if "on_control " in line + ) + state_index = next(i for i, line in enumerate(log_lines) if "on_state " in line) + assert control_index < state_index, "on_control must fire before on_state" diff --git a/tests/integration/test_template_climate_publish_all_fields.py b/tests/integration/test_template_climate_publish_all_fields.py new file mode 100644 index 0000000000..9c4262b311 --- /dev/null +++ b/tests/integration/test_template_climate_publish_all_fields.py @@ -0,0 +1,96 @@ +"""Integration test for template climate: climate.template.publish covering every field at once. + +A single climate.template.publish call resolves into exactly one ClimateState update, and never +triggers on_control (which would misrepresent a device state report as a fresh command). This also +exercises that a sensor/humidity_sensor whose reading matches what's about to be published doesn't +sneak in an extra state update of its own (the sensor callback only re-publishes on an actual +change). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-publish-all" + + +@pytest.mark.asyncio +async def test_template_climate_publish_all_fields( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """One climate.template.publish call setting every field resolves to one state update.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + on_control_count = 0 + + def on_log_line(line: str) -> None: + nonlocal on_control_count + if "on_control fired" in line: + on_control_count += 1 + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + publish_button = require_entity(entities, "publish_all", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.button_command(publish_button.key) + try: + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + except TimeoutError: + pytest.fail("Timeout waiting for the published climate state") + + assert state.current_temperature == pytest.approx(20.0, abs=0.1) + assert state.current_humidity == pytest.approx(60.0, abs=0.1) + assert state.target_temperature == pytest.approx(23.0, abs=0.1) + assert state.mode == ClimateMode.HEAT + assert state.action == ClimateAction.HEATING + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.ECO + + # Give any stray extra update (there shouldn't be one) a moment to arrive. + await asyncio.sleep(0.2) + assert len(state_updates) == 1, ( + f"Expected exactly one ClimateState update, got {len(state_updates)}" + ) + assert on_control_count == 0, ( + "climate.template.publish must not trigger on_control" + ) diff --git a/tests/integration/test_template_climate_sensor_push.py b/tests/integration/test_template_climate_sensor_push.py new file mode 100644 index 0000000000..1db4da81ed --- /dev/null +++ b/tests/integration/test_template_climate_sensor_push.py @@ -0,0 +1,88 @@ +"""Integration test for template climate: current_temperature/current_humidity live sensor push. + +A *later* change to a backing sensor's value -- not just its initial reading at boot -- propagates +into a new climate state via add_on_state_callback. Re-publishing the same sensor value again must +not cause a redundant climate state update. +""" + +from __future__ import annotations + +import asyncio +import math + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-sensor-push" + + +@pytest.mark.asyncio +async def test_template_climate_sensor_push( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A later change to the backing sensor pushes a new climate state; an unchanged republish does not.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + publish_temp = require_entity(entities, "publish_temperature", ButtonInfo) + publish_temp_same = require_entity( + entities, "publish_temperature_same", ButtonInfo + ) + publish_humidity = require_entity(entities, "publish_humidity", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Neither backing sensor has published anything yet. + assert math.isnan(initial.current_temperature) + assert math.isnan(initial.current_humidity) + + # A later sensor reading -- not the initial one -- pushes a new climate state. + client.button_command(publish_temp.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_temperature == pytest.approx(24.0, abs=0.1) + + client.button_command(publish_humidity.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_humidity == pytest.approx(65.0, abs=0.1) + + # Re-publishing the same temperature must not cause a redundant climate state update. + updates_before = len(state_updates) + client.button_command(publish_temp_same.key) + await asyncio.sleep(0.3) + assert len(state_updates) == updates_before, ( + "Re-publishing an unchanged sensor reading must not republish the climate state" + ) diff --git a/tests/integration/test_template_climate_set_actions.py b/tests/integration/test_template_climate_set_actions.py new file mode 100644 index 0000000000..0b1eb80874 --- /dev/null +++ b/tests/integration/test_template_climate_set_actions.py @@ -0,0 +1,114 @@ +"""Integration test: each settable field forwards its value to the matching set_*_action. + +With optimistic: false the entity state stays put until climate.template.publish reports the +device's actual state back, so the actions are the only thing that reacts to a command. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-set-act" + + +@pytest.mark.asyncio +async def test_template_climate_set_actions( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Every set_*_action fires with the requested value; state waits for a publish.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "_action " in line or "Unsupported" in line: + log_lines.append(line) + + def logged(fragment: str) -> bool: + return any(fragment in line for line in log_lines) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + report_button = require_entity(entities, "report_device_state", ButtonInfo) + unsupported_button = require_entity( + entities, "report_unsupported_mode", ButtonInfo + ) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Both traits are derived from the low/high and humidity set actions, not declared. + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + client.climate_command( + test_climate.key, target_temperature_low=18.0, target_temperature_high=24.0 + ) + client.climate_command(test_climate.key, target_humidity=55) + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.LOW) + client.climate_command(test_climate.key, custom_fan_mode="turbo") + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + client.climate_command(test_climate.key, custom_preset="eco_plus") + + for _ in range(50): + await asyncio.sleep(0.1) + if logged("set_custom_preset_action eco_plus"): + break + + assert logged("set_mode_action 3") # CLIMATE_MODE_HEAT + assert logged("set_target_temperature_low_action 18.0") + assert logged("set_target_temperature_high_action 24.0") + assert logged("set_target_humidity_action 55") + assert logged("set_fan_mode_action 3") # CLIMATE_FAN_LOW + assert logged("set_custom_fan_mode_action turbo") + assert logged("set_swing_mode_action 2") # CLIMATE_SWING_VERTICAL + assert logged("set_preset_action 5") # CLIMATE_PRESET_ECO + assert logged("set_custom_preset_action eco_plus") + + # optimistic: false, so none of the commands above touched the entity's own state -- + # a device report is what actually moves it. + client.button_command(report_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + + # A publish naming a mode outside supported_modes warns instead of publishing it. + client.button_command(unsupported_button.key) + for _ in range(50): + await asyncio.sleep(0.1) + if logged("Unsupported mode"): + break + assert logged("Unsupported mode") diff --git a/tests/integration/test_template_climate_two_point_temperature.py b/tests/integration/test_template_climate_two_point_temperature.py new file mode 100644 index 0000000000..9270b59ffc --- /dev/null +++ b/tests/integration/test_template_climate_two_point_temperature.py @@ -0,0 +1,118 @@ +"""Integration tests for template climate: two-point target temperature + humidity. + +Covers the supports_two_point_target_temperature/supports_target_humidity boolean flags plus +on_control (forwarding commands out) and climate.template.publish (the device reporting its own +authoritative state, independent of any prior command -- e.g. a device that owns its own setpoint, +changed via a physical remote). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-two-point" + + +@pytest.mark.asyncio +async def test_template_climate_two_point_temperature( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Two-point target temperature + humidity: booleans, on_control, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + test_climate = climate_infos[0] + assert test_climate.name == "Test Two-Point Heatpump" + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Nothing has been published yet: settable fields have no sensor to seed them from, so + # the entity starts at ESPHome's plain defaults. current_temperature is pushed by the + # referenced sensor, which has already settled by the time we get here. + assert initial.mode == ClimateMode.OFF + assert initial.current_temperature == pytest.approx(21.0, abs=0.1) + + # The device reports its actual state for the first time. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT_COOL + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) + + # Commands apply optimistically (settable fields are plain internal state), and on_control + # fires with the same values so a real config could forward them to the device. + client.climate_command( + test_climate.key, target_temperature_low=19.0, target_temperature_high=25.0 + ) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(19.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(25.0, abs=0.1) + await asyncio.sleep(0.2) + assert any( + "on_control target_temperature_low=19.0" in line for line in log_lines + ) + assert any( + "on_control target_temperature_high=25.0" in line for line in log_lines + ) + + client.climate_command(test_climate.key, target_humidity=45.0) + state = await wait_for_climate_state() + assert state.target_humidity == pytest.approx(45.0, abs=0.1) + await asyncio.sleep(0.2) + assert any("on_control target_humidity=45.0" in line for line in log_lines) + + # The device's next report is authoritative and overrides whatever was optimistically + # applied above -- this is the whole point of climate.template.publish: a device that owns + # its own state (e.g. changed by a physical remote) always wins. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) From 95ab3fb4f29aed85f762f2699484fea9e756b4f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 6 Sep 2026 23:59:40 +0200 Subject: [PATCH 039/178] [ota] Offer encryption with the api key so enabling it works over OTA (#18979) --- THREAT_MODEL.md | 55 ++- esphome/__main__.py | 14 +- esphome/components/api/__init__.py | 4 +- esphome/components/api/api_connection.cpp | 6 +- .../components/api/api_frame_helper_noise.cpp | 2 +- esphome/components/api/api_server.cpp | 37 +- esphome/components/api/api_server.h | 12 +- esphome/components/esphome/ota/__init__.py | 130 +++--- .../components/esphome/ota/ota_esphome.cpp | 83 ++-- esphome/components/esphome/ota/ota_esphome.h | 13 +- .../esphome/ota/ota_esphome_noise.cpp | 91 +++-- esphome/components/noise/__init__.py | 36 +- esphome/components/noise/noise.cpp | 9 + esphome/components/noise/noise.h | 16 +- esphome/components/noise/noise_handshake.cpp | 5 +- esphome/components/noise/noise_handshake.h | 6 +- esphome/core/defines.h | 3 + esphome/espota2.py | 122 +++++- esphome/wizard.py | 18 +- .../noise/test_encryption_key.py | 14 +- tests/component_tests/ota/test_esphome_ota.py | 242 +++++++++--- .../ota/test_esphome_ota_api_key_offer.yaml | 11 + ...st_esphome_ota_api_key_offer_password.yaml | 12 + .../test_esphome_ota_encryption_required.yaml | 12 + .../ota/test_esphome_ota_own_key.yaml | 11 + .../ota/test_esphome_ota_plain.yaml | 9 + .../ota/test_esphome_ota_runtime_api_key.yaml | 10 + .../components/noise/test_noise_handshake.cpp | 18 +- .../noise/test_noise_primitives.cpp | 13 +- tests/components/ota/api_key_offer.yaml | 12 + tests/components/ota/api_runtime_key.yaml | 10 + .../ota/test-api_key_offer.esp32-idf.yaml | 2 + .../ota/test-api_key_offer.esp8266-ard.yaml | 2 + .../ota/test-api_runtime_key.esp32-idf.yaml | 2 + .../ota/test-api_runtime_key.esp8266-ard.yaml | 2 + tests/integration/conftest.py | 7 + tests/integration/const.py | 7 + .../host_ota_api_key_offer_with_password.yaml | 12 + .../host_ota_provisioned_api_key.yaml | 10 + .../test_api_zero_psk_provisioning.py | 51 ++- tests/integration/test_host_ota.py | 373 ++++++++++++------ tests/unit_tests/test_espota2_noise.py | 136 ++++++- tests/unit_tests/test_main.py | 114 +++++- tests/unit_tests/test_wizard.py | 31 +- 44 files changed, 1342 insertions(+), 443 deletions(-) create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_encryption_required.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_own_key.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_plain.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml create mode 100644 tests/components/ota/api_key_offer.yaml create mode 100644 tests/components/ota/api_runtime_key.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp8266-ard.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml create mode 100644 tests/integration/fixtures/host_ota_provisioned_api_key.yaml diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index b4f557e55b..11656ff0b7 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -125,30 +125,47 @@ design is optimal or that it will not change. ## OTA update encryption The `esphome` OTA platform optionally encrypts updates with the same Noise -`NNpsk0` pattern the native API uses; one key protects the device. With an -`encryption:` block configured the guarantees are: the firmware image is -confidential in transit, the uploader is authenticated by the pre-shared key, -and the plaintext negotiation preceding the handshake is bound into the -handshake prologue, so stripping or tampering with it fails the first MAC. -Both ends fail closed with no override: a device built with a key refuses +`NNpsk0` pattern the native API uses; one key protects the device. A device +whose `api:` block has an encryption key, static in the YAML or provisioned at +runtime, compiles in the transport and offers it on every OTA connection once +it holds a key, so an uploader presenting that key gets the guarantees below +even without an `ota: encryption:` block; only that block makes the device +require encryption. The guarantees are: the firmware image is confidential in +transit, the uploader is authenticated by the pre-shared key, and the plaintext +negotiation preceding the handshake is bound into the handshake prologue, so +stripping or tampering with it fails the first MAC. With `ota: encryption:` +configured both ends fail closed with no override: the device refuses plaintext uploads, and the CLI refuses to send plaintext when a key is -configured. +configured. Without that block the CLI tries a static api key when the device +offers and, until 2027.3.0, falls back to plaintext with a warning when the +offer is missing or the handshake fails; a runtime provisioned key never +reaches the CLI, so those uploads stay plaintext. -Defeating any of that without the key is in scope: a keyed device accepting a -plaintext or downgraded upload, getting past the MAC, or recovering image -contents from captured traffic. +Defeating any of that without the key is in scope: a device that requires +encryption accepting a plaintext or downgraded upload, getting past the MAC, +or recovering image contents from captured traffic. The following are **not** vulnerabilities, by design: -- Plaintext OTA on a device with no `encryption:` block. That is the - documented default, authenticated (if at all) by the OTA password. -- The enablement window: turning encryption on takes one last upload of the - encryption-enabled firmware over the existing plaintext channel, with the - pre-existing plaintext exposure. -- The web OTA `/update` endpoint alongside encryption. The `web_server` - component keeps it always reachable, and `captive_portal:` auto-loads it - for the fallback AP window; validation warns about both combinations, and - the operator keeps the recovery path. +- Plaintext OTA on a device with no `ota: encryption:` block, including one + that offers encryption because it has an api key. That is the documented + default, authenticated (if at all) by the OTA password. An uploader that + takes the offer skips the password; the key authenticates it. With a + runtime provisioned key and no `provisioning:` window, whoever provisions + the key gains that upload path too; validation warns about the pair. +- The CLI plaintext fallback until 2027.3.0: without `ota: encryption:` an + active attacker who strips the offer or breaks the handshake can make a + keyed CLI upload plaintext, with the pre-existing plaintext exposure. A + device that requires encryption still refuses that upload. +- The enablement window: firmware built with a static api key already offers + encryption, so turning on `ota: encryption:` is itself an encrypted upload. + Older firmware needs one last plaintext upload of an offering build, with + the pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. With the `web_server` + or `prometheus` component the shared listener is always up, so the endpoint + stays reachable and validation warns about that combination; + `captive_portal:` alone brings the listener up only for the fallback AP + window, which is the intended recovery path, so that is not warned about. - CLI retry behavior on transport or MAC failures; every attempt renegotiates a fresh handshake with fresh ephemerals, so retrying does not weaken authentication. diff --git a/esphome/__main__.py b/esphome/__main__.py index b3d58ad13b..30e97f55eb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1335,12 +1335,14 @@ def _upload_via_native_api( break from esphome import espota2 + from esphome.components.noise import static_encryption_key remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) # Fail closed: an encryption block whose key did not resolve must never # fall back to a plaintext upload noise_psk = None + plaintext_fallback = False if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: noise_psk = encryption_conf.get(CONF_KEY) if not noise_psk: @@ -1351,6 +1353,10 @@ def _upload_via_native_api( # Ensure the key is a string, as required by the underlying OTA implementation. # It arrives here as a SensitiveStr which aioesphomeapi rejects. noise_psk = str(noise_psk) + elif api_key := static_encryption_key(config.get(CONF_API) or {}): + # Remove before 2027.3.0: the api key is tried, falling back to plaintext + noise_psk = str(api_key) + plaintext_fallback = True def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1382,7 +1388,13 @@ def _upload_via_native_api( _validate_bootloader_binary(binary) return espota2.run_ota( - network_devices, remote_port, password, binary, ota_type, noise_psk + network_devices, + remote_port, + password, + binary, + ota_type, + noise_psk, + plaintext_fallback=plaintext_fallback, ) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3568318dad..6202e127bf 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -14,6 +14,7 @@ from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, encryption_schema, + new_psk_progmem, validate_encryption_key, ) from esphome.config_helpers import filter_source_files_from_defines, get_logger_level @@ -589,8 +590,7 @@ async def to_code(config: ConfigType) -> None: if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None: if key := encryption_config.get(CONF_KEY): - decoded = decode_encryption_key(key) - cg.add(var.set_noise_psk(list(decoded))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) cg.add_define("USE_API_NOISE_PSK_FROM_YAML") else: # No key provided, but encryption desired diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9c609aa047..da4b7d7702 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2161,7 +2161,10 @@ void APIConnection::on_homeassistant_action_response(const HomeassistantActionRe bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg) { NoiseEncryptionSetKeyResponse resp; resp.success = false; - +#ifdef USE_API_NOISE_PSK_FROM_YAML + // A yaml key cannot be changed at runtime, so no decode or save path is built + ESP_LOGW(TAG, "Key set in YAML"); +#else #ifdef USE_PROVISIONING // Refuse to set a key once the provisioning window has closed (defense in depth; // such connections are already rejected at hello). @@ -2196,6 +2199,7 @@ bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptio } #endif } +#endif // USE_API_NOISE_PSK_FROM_YAML return this->send_message(resp); } diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 138dbdddba..29b2858aee 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -548,7 +548,7 @@ APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) { * @return 0 on success, -1 on error (check errno) */ APIError APINoiseFrameHelper::init_handshake_() { - int err = this->handshake_.init(this->ctx_.get_psk(), prologue_.data(), prologue_.size()); + int err = this->handshake_.init(this->ctx_, prologue_.data(), prologue_.size()); APIError aerr = handle_noise_error_(err, LOG_STR("noise_handshake_init"), APIError::HANDSHAKESTATE_SETUP_FAILED); if (aerr != APIError::OK) return aerr; diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 43d35363d3..78ebe5c38e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -41,13 +41,13 @@ void APIServer::setup() { ControllerRegistry::register_controller(this); #ifdef USE_API_NOISE + // Always reserve the slot: flash preferences are positional on esp8266, so + // a yaml key build must keep the layout of a runtime key build uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); - #ifndef USE_API_NOISE_PSK_FROM_YAML - // Only load saved PSK if not set from YAML - if (this->load_and_apply_noise_psk_()) { + // A cleared record loads fine but holds no key + if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { ESP_LOGD(TAG, "Loaded saved Noise PSK"); } #endif @@ -550,6 +550,7 @@ const std::vector &APIServer::get_sta #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active) { if (!this->noise_pref_.save(&new_psk)) { @@ -583,22 +584,19 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString } bool APIServer::load_and_apply_noise_psk_() { - SavedNoisePsk saved{}; - if (!this->noise_pref_.load(&saved)) + // Load into a temp so a failed read cannot disturb the key in use + SavedNoisePsk loaded{}; + if (!this->noise_pref_.load(&loaded)) return false; - this->set_noise_psk(saved.psk); + this->saved_psk_ = loaded; + // An unprovisioned device stores the reserved all-zeros key, which is no key + const bool has_key = !noise::NoiseContext::is_all_zeros(this->saved_psk_.psk); + this->noise_ctx_.set_psk(has_key ? this->saved_psk_.psk.data() : nullptr); return true; } bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else - auto &old_psk = this->noise_ctx_.get_psk(); - if (std::equal(old_psk.begin(), old_psk.end(), psk.begin())) { + if (this->saved_psk_.psk == psk) { ESP_LOGW(TAG, "New PSK matches old"); return true; } @@ -614,15 +612,8 @@ bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { } #endif return result; -#endif } bool APIServer::clear_noise_psk(bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else SavedNoisePsk empty_psk{}; bool result = this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"), make_active); @@ -634,8 +625,8 @@ bool APIServer::clear_noise_psk(bool make_active) { } #endif return result; -#endif } +#endif // USE_API_NOISE_PSK_FROM_YAML #endif #ifdef USE_HOMEASSISTANT_TIME diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 072a583901..618ea4eb11 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -76,9 +76,14 @@ class APIServer final : public Component, APIBuffer &get_shared_buffer_ref() { return shared_write_buffer_; } #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML + // Runtime key changes exist for the provisioning path only (not lambdas); + // with a yaml key they compile out bool save_noise_psk(noise::psk_t psk, bool make_active = true); bool clear_noise_psk(bool make_active = true); - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } noise::NoiseContext &get_noise_ctx() { return this->noise_ctx_; } #endif // USE_API_NOISE @@ -275,10 +280,12 @@ class APIServer final : public Component, #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active); // Load saved PSK from preferences and apply it. Returns true on success. bool load_and_apply_noise_psk_(); +#endif // USE_API_NOISE_PSK_FROM_YAML #endif // USE_API_NOISE #ifdef USE_API_HOMEASSISTANT_STATES // Helper methods to reduce code duplication @@ -358,6 +365,9 @@ class APIServer final : public Component, #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; +#ifndef USE_API_NOISE_PSK_FROM_YAML + SavedNoisePsk saved_psk_{}; // backs noise_ctx_ for a runtime provisioned key +#endif ESPPreferenceObject noise_pref_; #endif // USE_API_NOISE }; diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 1fec9e5c9b..f5eb878260 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -2,12 +2,12 @@ import logging import esphome.codegen as cg from esphome.components.noise import ( - decode_encryption_key, encryption_schema, - is_reserved_key, + new_psk_progmem, + static_encryption_key, ) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code -from esphome.config_helpers import merge_config +from esphome.config_helpers import filter_source_files_from_defines, merge_config import esphome.config_validation as cv from esphome.const import ( CONF_API, @@ -31,7 +31,6 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" -CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -41,11 +40,10 @@ DEPENDENCIES = ["network"] def AUTO_LOAD(config: ConfigType) -> list[str]: - """Auto-load noise only when encryption is configured.""" + """Auto-load noise only when encryption is configured; the api key offer + inherits it from the api component.""" base = ["sha256", "socket"] - # A falsy config is a tooling probe for the maximal set (None from - # dependency resolution, {} from the components-graph platform probe); - # a validated config always carries defaults, never empty + # A falsy config is a tooling probe for the maximal set if not config or CONF_ENCRYPTION in config: return base + ["noise"] return base @@ -132,12 +130,56 @@ def ota_esphome_final_validate(config: ConfigType) -> None: _validate_no_password_with_encryption(ota_conf) if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: _resolve_encryption_key(encryption_conf, api_conf) - if any( - conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf - ) and any( - CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + elif CONF_PASSWORD in ota_conf and static_encryption_key(api_conf) is not None: + _LOGGER.warning( + "'%s' %s wastes significant flash and RAM (about 3.5 KB and 60 " + "bytes plus the password on the heap): the device already offers " + "encryption with the '%s' %s %s, which authenticates any uploader " + "that takes it, and a password only matters for uploaders without " + "encryption support; remove '%s' and add '%s' under '%s' so " + "uploads use the key and encryption is required", + CONF_OTA, + CONF_PASSWORD, + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_PASSWORD, + CONF_ENCRYPTION, + CONF_OTA, + ) + elif ( + CONF_PASSWORD in ota_conf + and CONF_ENCRYPTION in api_conf + and not api_conf[CONF_ENCRYPTION].get(CONF_KEY) + ): + # The CLI still needs the password; whoever provisions the key skips it + _LOGGER.warning( + "The '%s' %s %s provisioned at runtime also authenticates OTA " + "uploads once provisioned; '%s' %s then only guards plaintext " + "uploads. Whoever provisions the key can upload firmware " + "without the password, so add a 'provisioning:' block to limit " + "when that is possible", + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_OTA, + CONF_PASSWORD, + ) + # web_server and prometheus keep the shared listener up; the captive + # portal's copy only exists on the fallback AP and is the recovery path + if ( + (CONF_WEB_SERVER in full_conf or "prometheus" in full_conf) + and any(conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf) + and any( + CONF_ENCRYPTION in conf + for conf in merged_ota_esphome_configs_by_port.values() + ) ): - _warn_web_server_ota(full_conf) + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -152,33 +194,11 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) -def _warn_web_server_ota(full_conf: ConfigType) -> None: - """The web_server ota platform accepts the same image over plaintext HTTP - with basic auth, bypassing the encryption; warn rather than fail so the - operator keeps the recovery path.""" - if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: - # The captive_portal auto-load: the endpoint only exists while the - # fallback AP is active - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform (auto-loaded " - "by captive_portal); the plaintext /update endpoint stays " - "reachable while the fallback AP is active", - CONF_WEB_SERVER, - ) - else: - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform; its " - "plaintext /update endpoint accepts the same image", - CONF_WEB_SERVER, - ) - - def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: """Resolve the one encryption key per device into the ota block. An explicit ota key must match the api key, a bare block inherits it, - a runtime provisioned api key cannot be inherited, and the all-zeros - provisioning sentinel is rejected (the device treats it as no key). + a runtime provisioned api key cannot be inherited. """ api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) if ota_key := encryption_conf.get(CONF_KEY): @@ -201,11 +221,6 @@ def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) - ) else: encryption_conf[CONF_KEY] = api_key - if is_reserved_key(encryption_conf[CONF_KEY]): - raise cv.Invalid( - f"The all-zeros {CONF_KEY} is reserved and provides no protection; " - f"generate a real key with: openssl rand -base64 32" - ) # Also called on merged same-port configs in final validate, where schemas @@ -267,15 +282,9 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate -def FILTER_SOURCE_FILES() -> list[str]: - """Filter out the noise transport when no ota entry configures encryption.""" - for ota_conf in CORE.config.get(CONF_OTA, []): - if ( - ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME - and ota_conf.get(CONF_ENCRYPTION) is not None - ): - return [] - return ["ota_esphome_noise.cpp"] +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} +) @coroutine_with_priority(CoroPriority.OTA_UPDATES) @@ -296,11 +305,24 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") - if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: - # A missing key was resolved from the api component in final validate. - key = encryption_conf[CONF_KEY] + # One key per device: an api encryption block supplies it (static or + # runtime) and offers; the ota block only adds the requirement + api_conf = CORE.config.get(CONF_API) or {} + encryption_conf = config.get(CONF_ENCRYPTION) + own_key = None + if encryption_conf is not None and static_encryption_key(api_conf) is None: + own_key = encryption_conf[CONF_KEY] + if own_key is not None: cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + elif CONF_ENCRYPTION in api_conf: + cg.add_define("USE_OTA_ENCRYPTION") + cg.add_define("USE_OTA_ENCRYPTION_FROM_API") + if static_encryption_key(api_conf) is None: + # The key arrives at runtime, so the offer has to look for it + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 396a47bc52..1005ed214b 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,4 +1,7 @@ #include "ota_esphome.h" +#ifdef USE_OTA_ENCRYPTION_FROM_API +#include "esphome/components/api/api_server.h" +#endif #ifdef USE_OTA #ifdef USE_OTA_PASSWORD #include "esphome/components/sha256/sha256.h" @@ -26,6 +29,16 @@ namespace esphome { static const char *const TAG = "esphome.ota"; + +#ifdef USE_OTA_ENCRYPTION +const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { +#ifdef USE_OTA_ENCRYPTION_FROM_API + return api::global_api_server->get_noise_ctx(); +#else + return this->noise_ctx_; +#endif +} +#endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -97,18 +110,30 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, "Over-The-Air updates:\n" " Address: %s:%u\n" - " Version: %d", - network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION); + " Version: %d" +#ifdef USE_OTA_ENCRYPTION + "\n Encryption: %s" +#endif + , + network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION +#ifdef USE_OTA_ENCRYPTION_REQUIRED + , + LOG_STR_LITERAL("required") +#elif defined(USE_OTA_ENCRYPTION_PROVISIONED) + // A runtime provisioned key may not exist yet + , + this->noise_context_().has_psk() ? LOG_STR_LITERAL("offered, plaintext accepted") + : LOG_STR_LITERAL("offered once the api key is provisioned") +#elif defined(USE_OTA_ENCRYPTION) + , + LOG_STR_LITERAL("offered, plaintext accepted") +#endif + ); #ifdef USE_OTA_PASSWORD if (!this->password_.empty()) { ESP_LOGCONFIG(TAG, " Password configured"); } #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { - ESP_LOGCONFIG(TAG, " Encryption configured"); - } -#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -154,10 +179,22 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +// Noise needs the extended protocol: the prologue binds the 2-byte feature ack +static constexpr uint8_t CLIENT_NOISE_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +inline bool ESPHomeOTAComponent::extended_proto_() const { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // FEATURE_READ already refused every client without the extended protocol + return true; +#else + return (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; +#endif +} + void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. /// @@ -241,12 +278,9 @@ void ESPHomeOTAComponent::handle_handshake_() { this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); -#ifdef USE_OTA_ENCRYPTION - // Fail closed: with a PSK configured the client must negotiate encryption - // (which requires the extended protocol); refuse plaintext uploads. - static constexpr uint8_t NOISE_REQUIRED_FEATURES = - CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; - if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // `ota: encryption:` requires the client to negotiate encryption + if ((this->ota_features_ & CLIENT_NOISE_FEATURES) != CLIENT_NOISE_FEATURES) { ESP_LOGW(TAG, "Client does not support encryption"); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); return; @@ -261,18 +295,21 @@ void ESPHomeOTAComponent::handle_handshake_() { // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte // legacy response. - this->extended_proto_ = (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; - if (this->extended_proto_) { + if (this->extended_proto_()) { static_assert(HANDSHAKE_BUF_SIZE >= 2, "handshake_buf_ must hold the 2-byte extended-protocol feature ack"); this->handshake_buf_[0] = ota::OTA_RESPONSE_FEATURE_FLAGS; this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // A runtime provisioned key may not exist yet + if (this->noise_context_().has_psk()) { this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; } +#elif defined(USE_OTA_ENCRYPTION) + // A yaml key always exists: validation rejects the all-zeros key + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; #endif } else { this->handshake_buf_[0] = @@ -284,15 +321,15 @@ void ESPHomeOTAComponent::handle_handshake_() { case OTAState::FEATURE_ACK: { static constexpr size_t STANDARD_PROTO_ACK_SIZE = 1; static constexpr size_t EXTENDED_PROTO_ACK_SIZE = 2; - const size_t ack_size = this->extended_proto_ ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; + const size_t ack_size = this->extended_proto_() ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } #ifdef USE_OTA_ENCRYPTION - // With a PSK configured the rest of the session runs inside the noise - // transport; the client sends the first handshake frame next, so there - // is nothing to do until data arrives. - if (this->noise_ctx_.has_psk()) { + // Latch the offer actually sent: a key activating between the two + // states must not start a session the client never expects + if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -412,7 +449,7 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge auth OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); - if (this->extended_proto_) { + if (this->extended_proto_()) { // Read ota type, 1 byte if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index fd164b8138..c6f710b3fc 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,8 +44,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#ifdef USE_OTA_ENCRYPTION - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif /// Manually set the port OTA should listen on @@ -85,9 +86,12 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; + // The api server's live context when the api has encryption, else our own + const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); + size_t noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len); bool noise_try_write_frame_(); void noise_send_reject_(const LogString *reason); ssize_t noise_decrypt_(uint8_t *buf, size_t len); @@ -144,7 +148,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION +#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#endif std::unique_ptr noise_; #endif // USE_OTA_ENCRYPTION @@ -166,6 +172,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; + // Derived from the feature byte; storing it would pad the trailing bytes + bool extended_proto_() const; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; @@ -179,7 +187,6 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint8_t auth_buf_pos_{0}; uint8_t auth_type_{0}; // Store auth type to know which hasher to use #endif // USE_OTA_PASSWORD - bool extended_proto_{false}; }; } // namespace esphome diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7f8331cf96..7401413d6d 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -3,6 +3,7 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise.h" #include "esphome/components/ota/ota_backend.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -40,24 +41,17 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // A provisioned key cleared between the offer and here is not guarded: the + // session runs on the zero key load_psk fills in and fails the client's MAC. + // Default-init: the frame buffer is written before it is read // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); - if (this->noise_ == nullptr) { - ESP_LOGW(TAG, "Session allocation failed"); - this->cleanup_connection_(); - return false; - } - + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + PROLOGUE_FEATURE_ACK_LEN]; -#ifdef USE_ESP8266 - memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#else - std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#endif + progmem_memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; // Magic bytes, already validated in MAGIC_READ std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); @@ -71,9 +65,13 @@ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; *p++ = server_feature_flags; - int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + // The caller only starts a session when the context holds a key + int err = this->noise_ == nullptr ? NOISE_ERROR_NO_MEMORY + : this->noise_->handshake.init(this->noise_context_(), prologue, sizeof(prologue)); if (err != 0) { - ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + // Raw noise codes throughout: the name table would cost flash in builds + // where only the OTA uses noise + ESP_LOGW(TAG, "Session init: %d", err); this->cleanup_connection_(); return false; } @@ -105,14 +103,16 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { s.frame_pos = 0; s.frame_len = 0; if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { - ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + ESP_LOGW(TAG, "Client rejected the handshake: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); this->cleanup_connection_(); return false; } int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); if (err != 0) { - ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); - this->noise_send_reject_(noise::reject_reason_for(err)); + // A MAC failure here almost always means the uploader has a different key + const LogString *reason = noise::reject_reason_for(err); + ESP_LOGW(TAG, "Handshake read: %s (%d)", LOG_STR_ARG(reason), err); + this->noise_send_reject_(reason); this->cleanup_connection_(); return false; } @@ -123,7 +123,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { int err = s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); if (err != 0) { - ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake write: %d", err); this->cleanup_connection_(); return false; } @@ -138,7 +138,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { int err = s.handshake.split(s.send_cipher, s.recv_cipher); if (err != 0) { - ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake split: %d", err); this->cleanup_connection_(); return false; } @@ -154,33 +154,41 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { } } +/// Payload length from a frame header, or 0 (logged) when the indicator or +/// the length is out of range. Callers pass min_len >= 1 so 0 is never valid. +size_t ESPHomeOTAComponent::noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len) { + const size_t payload_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || payload_len < min_len || payload_len > max_len) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], payload_len); + return 0; + } + return payload_len; +} + /// Non-blocking read of one handshake frame into the session buffer. bool ESPHomeOTAComponent::noise_try_read_frame_() { NoiseSession &s = *this->noise_; - while (s.frame_pos < noise::FRAME_HEADER_SIZE) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { - return false; + while (true) { + // The header first, then the body once the header says how long it is + const uint16_t want = s.frame_len == 0 ? noise::FRAME_HEADER_SIZE : s.frame_len; + if (s.frame_pos < want) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, want - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise"))) { + return false; + } + s.frame_pos += read; + continue; } - s.frame_pos += read; - } - if (s.frame_len == 0) { - const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); - if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { - ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + if (s.frame_len != 0) { + return true; + } + const size_t payload_len = this->noise_frame_payload_len_(s.frame_buf, 1, 1 + noise::MAX_HANDSHAKE_SIZE); + if (payload_len == 0) { this->cleanup_connection_(); return false; } s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; } - while (s.frame_pos < s.frame_len) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { - return false; - } - s.frame_pos += read; - } - return true; } /// Non-blocking write of the pending session-buffer frame. @@ -214,7 +222,7 @@ ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { noise_buffer_set_inout(mbuf, buf, len, len); int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Decrypt: %d", err); return -1; } return mbuf.size; @@ -229,9 +237,8 @@ ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min if (!this->readall_(header, sizeof(header))) { return -1; } - const size_t ciphertext_len = encode_uint16(header[1], header[2]); - if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { - ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + const size_t ciphertext_len = this->noise_frame_payload_len_(header, min_ciphertext, max_ciphertext); + if (ciphertext_len == 0) { return -1; } if (!this->readall_(buf, ciphertext_len)) { @@ -267,7 +274,7 @@ bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Encrypt: %d", err); return false; } noise::write_frame_header(frame, mbuf.size); diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 0f9328a482..a1d9444fc0 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -4,7 +4,9 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_KEY +from esphome.const import CONF_ENCRYPTION, CONF_KEY +from esphome.core import ID +from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] @@ -23,6 +25,14 @@ def validate_encryption_key(value: Any) -> str: if len(decoded) != 32: raise cv.Invalid("Encryption key must be base64 and 32 bytes long") + if not any(decoded): + # The device treats the all-zeros key as no key at all (it is the + # provisioning sentinel), so it must never reach a build + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"omit the {CONF_KEY} to provision it at runtime, or generate a real " + "key with: openssl rand -base64 32" + ) # Return original data for roundtrip conversion return value @@ -45,15 +55,6 @@ def decode_encryption_key(value: str) -> bytes: return decoded -def is_reserved_key(value: str) -> bool: - """Whether the key is the reserved all-zeros provisioning sentinel. - - The device treats it as no key configured, so consumers that require a - real key must reject it. - """ - return not any(decode_encryption_key(value)) - - ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), @@ -61,6 +62,21 @@ ENCRYPTION_SCHEMA = cv.Schema( ) +def static_encryption_key(conf: ConfigType) -> str | None: + """The build time key of a component config; None without one or when + the key is provisioned at runtime.""" + return (conf.get(CONF_ENCRYPTION) or {}).get(CONF_KEY) or None + + +def new_psk_progmem(parent_id: ID, key: str) -> MockObj: + """Emit the decoded key as a PROGMEM array; the component keeps a pointer + so the key never occupies RAM.""" + return cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decode_encryption_key(key)), + ) + + def encryption_schema(config: ConfigType | None) -> ConfigType: # A bare `encryption:` block is valid; a missing key means the consumer # falls back to its keyless behavior (api provisioning, ota inheriting diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 95fab322db..4806706167 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,5 +1,6 @@ #include "noise.h" #ifdef USE_NOISE +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -15,6 +16,14 @@ namespace esphome::noise { static const char *const TAG = "noise"; +void NoiseContext::load_psk(psk_t &out) const { + if (this->psk_ == nullptr) { + out.fill(0); + return; + } + progmem_memcpy(out.data(), this->psk_, out.size()); +} + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index f9da8d35b8..1033d5423c 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -23,16 +23,16 @@ class NoiseContext { } return acc == 0; } - void set_psk(psk_t psk) { - this->psk_ = psk; - this->has_psk_ = !is_all_zeros(psk); - } - const psk_t &get_psk() const { return this->psk_; } - bool has_psk() const { return this->has_psk_; } + /// psk points at 32 bytes that outlive the context (PROGMEM or caller owned + /// RAM); nullptr means no key. Runtime callers map the all-zeros key to + /// nullptr themselves; validation keeps it out of yaml. + void set_psk(const uint8_t *psk) { this->psk_ = psk; } + /// Copy the key out (flash-aware on ESP8266); all zeros when none is set. + void load_psk(psk_t &out) const; + bool has_psk() const { return this->psk_ != nullptr; } protected: - psk_t psk_{}; - bool has_psk_{false}; + const uint8_t *psk_{nullptr}; }; /// Convert a noise error code to a readable error diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 6d426de012..cc7fa603c4 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -20,7 +20,7 @@ NoiseResponderHandshake::~NoiseResponderHandshake() { } } -int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len) { +int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len) { if (this->handshake_ != nullptr) { noise_handshakestate_free(this->handshake_); this->handshake_ = nullptr; @@ -44,6 +44,9 @@ int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, siz HANDSHAKE_STEP_LOG("noise_handshakestate_new_by_id", err); return err; } + // noise-c keeps its own copy, so the key only passes through the stack here + psk_t psk; + ctx.load_psk(psk); err = noise_handshakestate_set_pre_shared_key(this->handshake_, psk.data(), psk.size()); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_set_pre_shared_key", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index 30596f35c2..bf1aa8cb7f 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -36,9 +36,9 @@ class NoiseResponderHandshake { NoiseResponderHandshake(const NoiseResponderHandshake &) = delete; NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; - /// Create and start the handshake with the given PSK and prologue. A - /// repeated call frees the previous handshake state and starts over. - [[nodiscard]] int init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len); + /// Create and start the handshake with the context's PSK and the prologue. + /// A repeated call frees the previous handshake state and starts over. + [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. [[nodiscard]] Action action() const; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 526adf74f0..9dd1e0ced6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -244,6 +244,9 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION +#define USE_OTA_ENCRYPTION_FROM_API +#define USE_OTA_ENCRYPTION_PROVISIONED +#define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ac4cbeeb7c..ce403c398d 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -202,6 +202,49 @@ class OTANetworkError(OTAError): """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" +# Remove before 2027.3.0 +class OTAEncryptionFallback(OTAError): + """The encrypted attempt failed and the caller may retry in plaintext.""" + + +# Remove before 2027.3.0 +PLAINTEXT_FALLBACK_NOTICE = ( + "A device with an api encryption key offers encryption after this " + "install; add 'encryption:' under 'ota: platform: esphome' to require it. " + "This plaintext fallback is removed in 2027.3.0." +) + + +# Remove before 2027.3.0 +class _EncryptionAttempt: + """The key an upload tries and whether it may fall back to plaintext; + a rejected handshake falls back at once, a transport fault only on repeat.""" + + def __init__(self, noise_psk: str | None, plaintext_fallback: bool) -> None: + self.noise_psk = noise_psk + self.plaintext_fallback = plaintext_fallback + self.handshake_faults = 0 + + def handshake_fault_falls_back(self) -> bool: + self.handshake_faults += 1 + return self.plaintext_fallback and self.handshake_faults >= 2 + + def downgrade(self, reason: str) -> None: + _LOGGER.warning( + "%s. Retrying in plaintext; a device that requires encryption " + "refuses it. %s", + reason, + PLAINTEXT_FALLBACK_NOTICE, + ) + self.noise_psk = None + self.plaintext_fallback = False + + +# Remove before 2027.3.0: only the fallback decision needs this distinction +class OTAHandshakeNetworkError(OTANetworkError): + """A transport failure inside the noise handshake; retrying encrypted may succeed.""" + + def _committed_error(err: OTANetworkError) -> OTAError: """Wrap a network failure that happened once the device had the full image. @@ -464,6 +507,7 @@ def perform_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> None: # Validate up front; an out-of-range value would only surface as a # ValueError deep inside send_check, bypassing OTAError handling @@ -528,19 +572,28 @@ def perform_ota( else: features = 0 - if noise_psk: - # Fail closed: never fall back to a plaintext upload when an - # encryption key is configured, an active attacker could otherwise - # strip the feature flag and capture the image (it contains the wifi - # credentials and the api encryption key). - if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if noise_psk and not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if plaintext_fallback: + # Remove before 2027.3.0: older firmware that cannot encrypt still + # gets its update on this connection + _LOGGER.warning( + "The device did not offer OTA encryption; continuing in plaintext. %s", + PLAINTEXT_FALLBACK_NOTICE, + ) + noise_psk = None + else: + # Fail closed: an attacker could otherwise strip the offer and + # capture the image (wifi credentials, api key) raise OTAError( "An OTA encryption key is configured but the device did not " "offer encryption; refusing to send the image in plaintext. " - "If the running firmware predates OTA encryption, first update " - "it without the 'ota: encryption:' block (over a trusted " - "network or via USB), then restore the block and upload again." + "The running firmware predates ESPHome 2026.9.0 or has no " + "'api: encryption: key'. With an api key, install once " + "without the 'ota: encryption:' block (that build offers " + "encryption), then restore it; otherwise flash by serial or " + "the web_server OTA platform." ) + if noise_psk: # The prologue binds every negotiation byte both sides saw, so any # tampering with the plaintext preamble breaks the handshake. prologue = ( @@ -549,8 +602,18 @@ def perform_ota( + bytes([RESPONSE_OK, version, features_to_send]) + bytes([RESPONSE_FEATURE_FLAGS, features]) ) + # Built outside the try: a local failure must never downgrade the upload sock = NoiseSocketWrapper(sock, noise_psk, prologue) - sock.do_handshake() + try: + sock.do_handshake() + except OTANetworkError as err: + # A transport fault: retry encrypted before considering plaintext + raise OTAHandshakeNetworkError(str(err)) from err + except OTAError as err: + # Remove before 2027.3.0 + if plaintext_fallback: + raise OTAEncryptionFallback(str(err)) from err + raise _LOGGER.info("Encrypted connection established") if ota_type != OTA_TYPE_UPDATE_APP: @@ -757,6 +820,7 @@ def run_ota_impl_( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -795,7 +859,9 @@ def run_ota_impl_( total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" reached_device = False - for attempt in range(total_attempts): + attempt = 0 + encryption = _EncryptionAttempt(noise_psk, plaintext_fallback) + while attempt < total_attempts: af, socktype, _, _, sa = res[attempt % len(res)] if reached_device or attempt >= len(res): _LOGGER.info( @@ -815,17 +881,40 @@ def run_ota_impl_( sock.close() _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) last_error = f"connecting to {sa[0]} failed: {err}" + attempt += 1 continue _LOGGER.info("Connected to %s", sa[0]) reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) + perform_ota( + sock, + password, + file_handle, + filename, + ota_type, + encryption.noise_psk, + encryption.plaintext_fallback, + ) + except OTAEncryptionFallback as err: + # Same address and attempt budget: not a network retry + last_error = str(err) + encryption.downgrade(last_error) + continue + except OTAHandshakeNetworkError as err: + last_error = str(err) + if encryption.handshake_fault_falls_back(): + encryption.downgrade(last_error) + continue + _LOGGER.warning("%s", last_error) + attempt += 1 + continue except OTANetworkError as err: # Transient network failure; retry last_error = str(err) _LOGGER.warning("%s", last_error) + attempt += 1 continue except OTAError as err: # Device-reported error (wrong password, wrong flash size, ...); @@ -847,10 +936,17 @@ def run_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: try: return run_ota_impl_( - remote_host, remote_port, password, filename, ota_type, noise_psk + remote_host, + remote_port, + password, + filename, + ota_type, + noise_psk, + plaintext_fallback, ) except OTAError as err: _LOGGER.error(err) diff --git a/esphome/wizard.py b/esphome/wizard.py index f7706928e9..897d5f60a1 100644 --- a/esphome/wizard.py +++ b/esphome/wizard.py @@ -148,11 +148,13 @@ def wizard_file(**kwargs: Unpack[WizardFileKwargs]) -> str: if "api_encryption_key" in kwargs: config += f' encryption:\n key: "{kwargs["api_encryption_key"]}"\n' - # Configure OTA + # The api key also secures OTA; a password only serves older uploaders config += "\nota:\n" config += " - platform: esphome\n" if "ota_password" in kwargs: config += f' password: "{kwargs["ota_password"]}"' + elif "api_encryption_key" in kwargs: + config += " encryption:" # Configuring wifi config += "\n\nwifi:\n" @@ -529,20 +531,9 @@ def wizard(path: Path) -> int: safe_print() safe_print("You'll need this key when adding the device to Home Assistant.") sleep(1) - - safe_print() - safe_print( - f"Do you want to set a {color(AnsiFore.GREEN, 'password')} for OTA updates? " - "This can be insecure if you do not trust the WiFi network." - ) - safe_print() - sleep(0.25) - safe_print("Press ENTER for no password") - ota_password = safe_input(color(AnsiFore.BOLD_WHITE, "(password): ")) else: ssid, psk = "", "" api_encryption_key = None - ota_password = "" kwargs = { "path": path, @@ -553,10 +544,9 @@ def wizard(path: Path) -> int: "psk": psk, "type": "basic", } + # The api key also secures OTA updates, so the wizard sets no OTA password if api_encryption_key: kwargs["api_encryption_key"] = api_encryption_key - if ota_password: - kwargs["ota_password"] = ota_password if not wizard_write(**kwargs): return 1 diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 10f1eb3d4c..2b79bd5464 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,11 +5,7 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import ( - decode_encryption_key, - is_reserved_key, - validate_encryption_key, -) +from esphome.components.noise import decode_encryption_key, validate_encryption_key KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -41,6 +37,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: decode_encryption_key("AAECAw==") -def test_is_reserved_key() -> None: - assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") - assert not is_reserved_key(KEY) +def test_validate_encryption_key_rejects_all_zeros() -> None: + """The all-zeros key is the provisioning sentinel the device treats as no + key, so it never reaches a build.""" + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + validate_encryption_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 873f162555..d3092294dc 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable import logging from typing import Any @@ -14,6 +15,7 @@ from esphome.components.esphome.ota import ( _validate_no_password_with_encryption, ota_esphome_final_validate, ) +from esphome.components.noise import static_encryption_key from esphome.const import ( CONF_API, CONF_ENCRYPTION, @@ -115,7 +117,6 @@ def test_non_esphome_ota_unaffected() -> None: API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" -ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" def test_encryption_key_inherited_from_api() -> None: @@ -197,36 +198,6 @@ def test_encryption_without_any_key_rejected() -> None: fv.full_config.reset(token) -def test_encryption_explicit_all_zeros_key_rejected() -> None: - """The all-zeros key is the provisioning sentinel; the device would treat - it as no PSK and accept plaintext, so it must fail validation.""" - full_conf = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) - ], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - -def test_encryption_inherited_all_zeros_key_rejected() -> None: - """An all-zeros api key must not silently disable ota encryption either.""" - full_conf = { - CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, - CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: """Same-port configs with different encryption keys raise.""" full_conf = { @@ -295,13 +266,14 @@ def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None fv.full_config.reset(token) +@pytest.mark.parametrize("component", ["web_server", "prometheus"]) def test_encryption_with_web_server_ota_warns( - caplog: pytest.LogCaptureFixture, + caplog: pytest.LogCaptureFixture, component: str ) -> None: - """With the web_server component the plaintext /update endpoint is always - on; the combination validates with a warning.""" + """web_server and prometheus keep the shared listener up, so the + plaintext /update endpoint is always on and the combination warns.""" full_conf = { - "web_server": {}, + component: {}, CONF_OTA: [ _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, @@ -316,12 +288,12 @@ def test_encryption_with_web_server_ota_warns( fv.full_config.reset(token) -def test_encryption_with_captive_portal_web_server_ota_warns( +def test_encryption_with_captive_portal_does_not_warn( caplog: pytest.LogCaptureFixture, ) -> None: """captive_portal auto-loads the web_server ota platform without the - web_server component; encryption stays usable and only warns, so the - fallback AP recovery path is not lost.""" + web_server component; its endpoint only exists while the fallback AP is + active and is the intended recovery path, so there is no warning.""" full_conf = { "captive_portal": {}, CONF_OTA: [ @@ -333,7 +305,10 @@ def test_encryption_with_captive_portal_web_server_ota_warns( try: with caplog.at_level(logging.WARNING): ota_esphome_final_validate({}) - assert any("captive_portal" in record.message for record in caplog.records) + assert not any( + "OTA encryption does not cover" in record.message + for record in caplog.records + ) esphome_conf = next( conf for conf in fv.full_config.get()[CONF_OTA] @@ -344,6 +319,100 @@ def test_encryption_with_captive_portal_web_server_ota_warns( fv.full_config.reset(token) +def test_password_with_api_key_warns(caplog: pytest.LogCaptureFixture) -> None: + """A static api key makes the device offer encryption and the CLI take + it, so the password is dead weight; the config validates with a warning.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("wastes significant flash" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_password_with_runtime_api_key_warns_differently( + caplog: pytest.LogCaptureFixture, +) -> None: + """The CLI still needs the password, but the provisioned key also + authenticates uploads; the warning says so without the flash advice.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + messages = [r.message for r in caplog.records] + assert any("provisioned at runtime also authenticates" in m for m in messages) + assert not any("wastes significant flash" in m for m in messages) + finally: + fv.full_config.reset(token) + + +def test_password_without_api_key_no_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """Without an api key there is no offer, so nothing to warn about.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("authenticates" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_web_server_component_without_ota_platform_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """The web_server component alone has no /update endpoint.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any( + "OTA encryption does not cover" in r.message for r in caplog.records + ) + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_platform_alone_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the web_server component starts the shared listener, so the ota + platform on its own never exposes /update.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("plaintext /update" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + def test_web_server_ota_without_encryption_unaffected() -> None: """web_server ota stays valid alongside an unencrypted esphome entry.""" full_conf = { @@ -370,20 +439,87 @@ def test_auto_load_pulls_noise_only_for_encryption() -> None: assert "noise" in AUTO_LOAD({}) -def test_filter_source_files_excludes_noise_without_encryption() -> None: - """The noise transport source compiles only for encrypted builds.""" - old_config = CORE.config - try: - CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} - assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] - CORE.config = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) - ] - } - assert FILTER_SOURCE_FILES() == [] - finally: - CORE.config = old_config +def test_static_encryption_key() -> None: + """Only a build-time key counts; a runtime provisioned one does not.""" + assert static_encryption_key({}) is None + assert static_encryption_key({CONF_ENCRYPTION: {}}) is None + assert static_encryption_key({CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) == API_KEY + + +@pytest.mark.parametrize( + ("yaml_name", "defines_present", "defines_absent"), + [ + # An api key alone compiles the transport in without requiring it; + # the device uses the api server's key, not a copy + ( + "api_key_offer", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A password still guards plaintext uploads on an offering device + ( + "api_key_offer_password", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # The ota encryption block is what makes the device refuse plaintext + ( + "encryption_required", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + }, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # Without api encryption the ota key is the device's own + ( + "own_key", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, + {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A key provisioned at runtime lives in the api server; the device + # offers with it once provisioned and never requires it + ( + "runtime_api_key", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + {"USE_OTA_ENCRYPTION_REQUIRED"}, + ), + # No api encryption at all keeps the noise glue out of the build + ( + "plain", + set(), + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + ), + ], +) +def test_encryption_offer_codegen( + generate_main: Callable[[str], str], + yaml_name: str, + defines_present: set[str], + defines_absent: set[str], +) -> None: + main_cpp = generate_main( + f"tests/component_tests/ota/test_esphome_ota_{yaml_name}.yaml" + ) + defines = {define.name for define in CORE.defines} + assert defines_present <= defines + assert not (defines_absent & defines) + encrypted = "USE_OTA_ENCRYPTION" in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) + # The noise transport source compiles only when the define is set + assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) def test_password_with_encryption_rejected() -> None: diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml new file mode 100644 index 0000000000..ca26eb9f46 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-offer + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml new file mode 100644 index 0000000000..1e23975690 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-offer-password + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml new file mode 100644 index 0000000000..36690038d8 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-encryption-required + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + encryption: diff --git a/tests/component_tests/ota/test_esphome_ota_own_key.yaml b/tests/component_tests/ota/test_esphome_ota_own_key.yaml new file mode 100644 index 0000000000..b6d1e4200d --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_own_key.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-own-key + +host: + +api: + +ota: + - platform: esphome + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/component_tests/ota/test_esphome_ota_plain.yaml b/tests/component_tests/ota/test_esphome_ota_plain.yaml new file mode 100644 index 0000000000..c5ca7afcf0 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_plain.yaml @@ -0,0 +1,9 @@ +esphome: + name: ota-plain + +host: + +api: + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml new file mode 100644 index 0000000000..8825335141 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: ota-runtime-key + +host: + +api: + encryption: + +ota: + - platform: esphome diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index d879a26c43..f2081f2965 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -68,6 +68,14 @@ class Initiator { static const uint8_t PROLOGUE[] = {'t', 'e', 's', 't', 'p', 'r', 'o', 'l', 'o', 'g', 'u', 'e'}; +// The context only points at the key and init() copies it before returning, +// so a temporary context over a temporary key is safe within one call +static NoiseContext ctx_for(const psk_t &psk) { + NoiseContext ctx; + ctx.set_psk(psk.data()); + return ctx; +} + static psk_t make_psk(uint8_t seed) { psk_t psk; for (size_t i = 0; i < psk.size(); i++) { @@ -102,7 +110,7 @@ TEST(NoiseResponderHandshakeTest, MessageMethodsErrorBeforeInit) { TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); @@ -155,8 +163,8 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // proves the restart took effect; the old state surviving would fail the // MAC here. NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); - ASSERT_EQ(responder.init(make_psk(9), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(9)), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(make_psk(9), PROLOGUE, sizeof(PROLOGUE)); @@ -168,7 +176,7 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { TEST(NoiseResponderHandshakeTest, WrongPskFailsWithMacFailure) { NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); Initiator initiator(make_psk(200), PROLOGUE, sizeof(PROLOGUE)); uint8_t msg[MAX_HANDSHAKE_SIZE]; @@ -185,7 +193,7 @@ TEST(NoiseResponderHandshakeTest, MismatchedPrologueFailsWithMacFailure) { // tampered preamble must fail even with the right key. const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); static const uint8_t TAMPERED[] = {'x'}; Initiator initiator(psk, TAMPERED, sizeof(TAMPERED)); diff --git a/tests/components/noise/test_noise_primitives.cpp b/tests/components/noise/test_noise_primitives.cpp index 018be9f717..8687c4b963 100644 --- a/tests/components/noise/test_noise_primitives.cpp +++ b/tests/components/noise/test_noise_primitives.cpp @@ -17,12 +17,17 @@ TEST(NoiseContextTest, AllZerosPskIsReserved) { EXPECT_FALSE(NoiseContext::is_all_zeros(psk)); NoiseContext ctx; + psk_t loaded; EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(zeros); - EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, zeros); + ctx.set_psk(psk.data()); EXPECT_TRUE(ctx.has_psk()); - EXPECT_EQ(ctx.get_psk(), psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, psk); + // Callers map the reserved key to nullptr; the context just stores what it is given + ctx.set_psk(nullptr); + EXPECT_FALSE(ctx.has_psk()); } TEST(WireFormatTest, FrameHeaderIsIndicatorPlusBigEndianLength) { diff --git a/tests/components/ota/api_key_offer.yaml b/tests/components/ota/api_key_offer.yaml new file mode 100644 index 0000000000..8d1814bf7e --- /dev/null +++ b/tests/components/ota/api_key_offer.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3290 + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/components/ota/api_runtime_key.yaml b/tests/components/ota/api_runtime_key.yaml new file mode 100644 index 0000000000..8976c92f96 --- /dev/null +++ b/tests/components/ota/api_runtime_key.yaml @@ -0,0 +1,10 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + +ota: + - platform: esphome + port: 3291 diff --git a/tests/components/ota/test-api_key_offer.esp32-idf.yaml b/tests/components/ota/test-api_key_offer.esp32-idf.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_key_offer.esp8266-ard.yaml b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp32-idf.yaml b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6777e6cabc..15c5860879 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -162,6 +162,13 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) +@pytest.fixture +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Host preferences persist per device name; give the test its own so a + provisioned key never leaks into another run.""" + monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) + + @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" diff --git a/tests/integration/const.py b/tests/integration/const.py index 6876bbd443..e35d4673af 100644 --- a/tests/integration/const.py +++ b/tests/integration/const.py @@ -9,6 +9,13 @@ API_CONNECTION_TIMEOUT = 30.0 # seconds PORT_WAIT_TIMEOUT = 30.0 # seconds PORT_POLL_INTERVAL = 0.1 # seconds +# The well-known all-zeros provisioning PSK, a key to provision over it, and +# the time the device takes to activate a newly saved key (100 ms timer plus +# margin) +ZERO_PSK = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +PROVISIONING_PSK = b"bm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm4=" +KEY_ACTIVATION_DELAY = 0.5 # seconds + # Process shutdown timeouts SIGINT_TIMEOUT = 5.0 # seconds SIGTERM_TIMEOUT = 2.0 # seconds diff --git a/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml new file mode 100644 index 0000000000..1dedcc9ee1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + password: "hunter2" +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml new file mode 100644 index 0000000000..aa0a9a66c9 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index bcea2a2471..f315335d1b 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -10,34 +10,40 @@ from __future__ import annotations import asyncio import base64 +import socket from aioesphomeapi import InvalidEncryptionKeyAPIError, RequiresEncryptionAPIError import pytest -from .types import APIClientConnectedFactory, RunCompiledFunction +from .conftest import run_binary_and_wait_for_port +from .const import KEY_ACTIVATION_DELAY, LOCALHOST, PROVISIONING_PSK, ZERO_PSK +from .types import ( + APIClientConnectedFactory, + CompileFunction, + ConfigWriter, + RunCompiledFunction, +) -# The well-known provisioning PSK: base64 of 32 zero bytes -ZERO_PSK = base64.b64encode(bytes(32)).decode() -# A real key to provision -NEW_KEY = base64.b64encode(b"n" * 32) -# Time for the device to activate a newly saved key (100ms timer plus margin) -KEY_ACTIVATION_DELAY = 0.5 - - -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so every run starts unprovisioned.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) +pytestmark = pytest.mark.usefixtures("isolated_preferences") +NEW_KEY = PROVISIONING_PSK @pytest.mark.asyncio async def test_api_zero_psk_provisioning( yaml_config: str, - run_compiled: RunCompiledFunction, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], api_client_connected: APIClientConnectedFactory, ) -> None: - """Exercise the reject paths, then provision a key over the zero-PSK channel.""" - async with run_compiled(yaml_config): + """Exercise the reject paths, provision a key over the zero-PSK channel, + and check the key comes back from preferences on the next boot.""" + port, port_socket = reserved_tcp_port + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + port_socket.close() + + async with run_binary_and_wait_for_port(binary_path, LOCALHOST, port): # --- Pre-provisioning reject paths (device state is unchanged) --- # A wrong (non-zero) PSK fails against the zero provisioning PSK @@ -97,6 +103,19 @@ async def test_api_zero_psk_provisioning( async with api_client_connected(timeout=5) as client: await client.device_info() + # The key is loaded from preferences on the next boot + lines: list[str] = [] + async with run_binary_and_wait_for_port( + binary_path, LOCALHOST, port, line_callback=lines.append + ): + async with api_client_connected(noise_psk=NEW_KEY.decode()) as client: + device_info = await client.device_info() + assert device_info.api_encryption_provisionable is False + with pytest.raises(InvalidEncryptionKeyAPIError): + async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client: + await client.device_info() + assert any("Loaded saved Noise PSK" in line for line in lines) + @pytest.mark.asyncio async def test_api_zero_psk_provisioning_plaintext( diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 4e74814534..f8c122c6e1 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -8,9 +8,12 @@ instance covers the FD_CLOEXEC path. from __future__ import annotations import asyncio +import base64 from collections.abc import Generator from contextlib import contextmanager +from dataclasses import dataclass import functools +from pathlib import Path import socket import pytest @@ -18,10 +21,18 @@ import pytest from esphome import espota2 from .conftest import run_binary, wait_and_connect_api_client -from .const import LOCALHOST, PORT_POLL_INTERVAL, PORT_WAIT_TIMEOUT -from .types import CompileFunction, ConfigWriter +from .const import ( + KEY_ACTIVATION_DELAY, + LOCALHOST, + PORT_POLL_INTERVAL, + PORT_WAIT_TIMEOUT, + PROVISIONING_PSK, + ZERO_PSK, +) +from .types import APIClientConnectedFactory, CompileFunction, ConfigWriter DEVICE_NAME = "host-ota-test" +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @contextmanager @@ -35,6 +46,14 @@ def _reserve_port() -> Generator[tuple[int, socket.socket]]: s.close() +async def _wait_for_line(lines: list[str], needle: str, timeout: float = 5.0) -> None: + """The config dump prints after every setup, a little after the api port + opens, so wait for it rather than assert on the lines seen so far.""" + async with asyncio.timeout(timeout): + while not any(needle in line for line in lines): + await asyncio.sleep(PORT_POLL_INTERVAL) + + async def _wait_for_port(host: str, port: int, timeout: float) -> None: """Poll until a TCP port accepts connections, or raise TimeoutError.""" loop = asyncio.get_running_loop() @@ -51,6 +70,102 @@ async def _wait_for_port(host: str, port: int, timeout: float) -> None: raise TimeoutError(f"Port {port} on {host} did not open within {timeout}s") +async def _build( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> tuple[int, int, Path]: + """Reserve an OTA port, compile the fixture with it, and release both + ports right before the binary is started.""" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + config_path = await write_yaml_config( + yaml_config.replace("__OTA_PORT__", str(ota_port)) + ) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + return api_port, ota_port, binary_path + + +async def _run_ota( + ota_port: int, + password: str | None, + binary_path: Path, + noise_psk: str | None, + plaintext_fallback: bool = False, +) -> int: + """espota2 is blocking; run it in the executor and return its exit code.""" + rc, _ = await asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + password, + binary_path, + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, + ), + ) + return rc + + +@dataclass +class _Device: + """A running host binary and the checks every successful OTA repeats: + a safe reboot, the api port back up, and the pid preserved by execv.""" + + api_port: int + ota_port: int + binary_path: Path + proc: asyncio.subprocess.Process | None = None + reboots: int = 0 + + def __post_init__(self) -> None: + self._rebooted = asyncio.Event() + + def on_log(self, line: str) -> None: + if "Rebooting safely" in line: + self.reboots += 1 + self._rebooted.set() + + async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: + async with asyncio.timeout(timeout): + while self.reboots < count: + self._rebooted.clear() + await self._rebooted.wait() + + async def ota( + self, + password: str | None, + noise_psk: str | None, + msg: str, + plaintext_fallback: bool = False, + ) -> None: + """Upload, then expect the re-exec with the pid preserved.""" + pid_before = self.proc.pid + expected_reboots = self.reboots + 1 + rc = await _run_ota( + self.ota_port, password, self.binary_path, noise_psk, plaintext_fallback + ) + assert rc == 0, msg + await self.wait_reboot(expected_reboots) + await _wait_for_port(LOCALHOST, self.api_port, PORT_WAIT_TIMEOUT) + assert self.proc.returncode is None, "process exited instead of execing" + assert self.proc.pid == pid_before + + async def refused_ota( + self, password: str | None, noise_psk: str | None, msg: str + ) -> None: + """Upload must fail and the device must keep running.""" + rc = await _run_ota(self.ota_port, password, self.binary_path, noise_psk) + assert rc == 1, msg + await asyncio.sleep(0.5) + assert self.proc.returncode is None, "process died on rejected OTA" + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -59,57 +174,34 @@ async def test_host_ota_self_update( reserved_tcp_port: tuple[int, socket.socket], ) -> None: """Self-OTA: upload the running binary back to itself, expect re-exec.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + staged = asyncio.Event() - loop = asyncio.get_running_loop() - ota_staged = loop.create_future() - rebooted = loop.create_future() + def on_log(line: str) -> None: + if "OTA staged at" in line: + staged.set() + dev.on_log(line) - def on_log(line: str) -> None: - if not ota_staged.done() and "OTA staged at" in line: - ota_staged.set_result(True) - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_before = await client.device_info() + assert info_before.name == DEVICE_NAME - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - async with wait_and_connect_api_client(port=api_port) as client: - info_before = await client.device_info() - assert info_before.name == DEVICE_NAME + await dev.ota(None, None, "espota2 reported failure") + assert staged.is_set() - # espota2 is blocking; run in executor. - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "espota2 reported failure" + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_after = await client.device_info() + assert info_after.name == info_before.name - await asyncio.wait_for(ota_staged, timeout=10.0) - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - - # execv preserves pid; mismatch means external respawn. - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info_after = await client.device_info() - assert info_after.name == DEVICE_NAME - assert info_after.name == info_before.name - - # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "second OTA failed -- listener leaked across execv" - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.pid == pid_before + # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). + await dev.ota(None, None, "second OTA failed -- listener leaked across execv") @pytest.mark.asyncio @@ -121,51 +213,110 @@ async def test_host_ota_encrypted( ) -> None: """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" pytest.importorskip("aioesphomeapi.noise") - noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await dev.refused_ota( + None, None, "plaintext upload to an encrypted device must fail" + ) + await dev.ota(None, API_KEY, "encrypted OTA reported failure") - loop = asyncio.get_running_loop() - rebooted = loop.create_future() - def on_log(line: str) -> None: - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) +@pytest.mark.asyncio +async def test_host_ota_api_key_offer_with_password( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + caplog: pytest.LogCaptureFixture, +) -> None: + """With only an api key the device offers encryption without requiring + it: the password still guards plaintext uploads, the key alone + authenticates an encrypted one, and until 2027.3.0 a failed encrypted + attempt falls back to plaintext.""" + pytest.importorskip("aioesphomeapi.noise") + wrong_key = base64.b64encode(b"w" * 32).decode() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "Encryption: offered") - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid + await dev.refused_ota( + None, None, "plaintext upload without the password must fail" + ) + await dev.ota( + "hunter2", None, "plaintext upload with the password must succeed" + ) + await dev.ota(None, API_KEY, "encrypted upload with the api key must succeed") - # A plaintext upload must be refused with the device unharmed - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + # Remove before 2027.3.0: a wrong key falls back to plaintext, which + # the password still guards + with caplog.at_level("WARNING", logger="esphome.espota2"): + await dev.ota( + "hunter2", + wrong_key, + "the plaintext retry with the password must succeed", + plaintext_fallback=True, ) - assert rc == 1, "plaintext upload to an encrypted device must fail" - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected plaintext OTA" + assert any("Retrying in plaintext" in r.message for r in caplog.records) + await dev.ota( + None, + API_KEY, + "the right api key encrypts without touching the fallback", + plaintext_fallback=True, + ) - # The encrypted upload goes through and the device re-execs - rc, _ = await loop.run_in_executor( - None, - functools.partial( - espota2.run_ota, - LOCALHOST, - ota_port, - None, - binary_path, - noise_psk=noise_psk, - ), - ) - assert rc == 0, "encrypted OTA reported failure" - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_preferences") +async def test_host_ota_provisioned_api_key( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """A key provisioned over the api feeds the OTA offer: plaintext works + while unprovisioned, the provisioned key encrypts, the key loaded from + preferences on the next boot keeps encrypting, and plaintext stays + accepted because only the ota block requires encryption.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "once the api key is provisioned") + + await dev.ota( + None, None, "plaintext upload to an unprovisioned device must succeed" + ) + + async with api_client_connected( + port=dev.api_port, noise_psk=ZERO_PSK + ) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + key = PROVISIONING_PSK.decode() + await dev.ota( + None, key, "encrypted upload with the provisioned key must succeed" + ) + await dev.ota(None, key, "the key loaded at boot must feed the OTA offer") + await dev.ota(None, None, "plaintext must stay accepted on an offering device") @pytest.mark.asyncio @@ -177,33 +328,25 @@ async def test_host_ota_rejects_garbage( integration_test_dir, ) -> None: """Bogus payload is rejected and the device keeps running.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # 192 bytes that are neither ELF nor Mach-O. + bogus_path = integration_test_dir / "bogus.bin" + bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) - # 192 bytes that are neither ELF nor Mach-O. - bogus_path = integration_test_dir / "bogus.bin" - bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) + async with run_binary(dev.binary_path) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + rc = await _run_ota(dev.ota_port, None, bogus_path, None) + assert rc == 1 + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected OTA" + assert proc.pid == pid_before - api_socket.close() - ota_socket.close() - - async with run_binary(binary_path) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - - loop = asyncio.get_running_loop() - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, bogus_path - ) - assert rc == 1 - - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected OTA" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info = await client.device_info() - assert info.name == DEVICE_NAME + async with wait_and_connect_api_client(port=dev.api_port) as client: + info = await client.device_info() + assert info.name == DEVICE_NAME diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py index 5b43d05530..439220f09c 100644 --- a/tests/unit_tests/test_espota2_noise.py +++ b/tests/unit_tests/test_espota2_noise.py @@ -10,12 +10,15 @@ when the installed aioesphomeapi predates the noise module. from __future__ import annotations import base64 +from collections.abc import Callable import hashlib import io +import logging from pathlib import Path import socket import sys import threading +from typing import Any from unittest.mock import Mock, patch import pytest @@ -65,8 +68,12 @@ class FakeEncryptedDevice(threading.Thread): offer_noise: bool = True, require_noise: bool = True, prologue_features_override: int | None = None, + connections: int = 1, + drop_handshakes: int = 0, ) -> None: super().__init__(daemon=True) + self.connections = connections + self.drop_handshakes = drop_handshakes # hang up mid-handshake this many times self.psk = psk self.version = version self.offer_noise = offer_noise @@ -81,10 +88,11 @@ class FakeEncryptedDevice(threading.Thread): def run(self) -> None: try: - sock, _ = self.listener.accept() - sock.settimeout(10) - with sock: - self._serve(sock) + for _ in range(self.connections): + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) except Exception as err: # noqa: BLE001 - surfaced via join_and_check self.error = err finally: @@ -109,8 +117,23 @@ class FakeEncryptedDevice(threading.Thread): return server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) - if not (self.offer_noise and noise_negotiated): - return # the client fails closed; nothing further arrives + if not (noise_negotiated and self.offer_noise): + # A device that does not require encryption continues in + # plaintext whatever the client asked for, like older firmware + try: + self._transfer( + lambda byte: sock.sendall(bytes([byte])), + lambda length: _recv_exact(sock, length), + lambda remaining: _recv_exact( + sock, min(remaining, espota2.UPLOAD_BLOCK_SIZE) + ), + ) + except ConnectionError: + # A keyed client without fallback fails closed and hangs up + if noise_negotiated and not self.offer_noise: + return + raise + return from cryptography.exceptions import InvalidTag from noise.connection import NoiseConnection @@ -134,6 +157,9 @@ class FakeEncryptedDevice(threading.Thread): msg1 = _recv_frame(sock) assert msg1[0] == 0x00 + if self.drop_handshakes > 0: + self.drop_handshakes -= 1 + return # a transport fault: the socket closes with no reply try: proto.read_message(msg1[1:]) except InvalidTag: @@ -149,6 +175,20 @@ class FakeEncryptedDevice(threading.Thread): assert len(plaintext) == length, "control units must be one per frame" return plaintext + def recv_data(_remaining: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + return plaintext + + self._transfer(send_byte, recv_unit, recv_data) + + def _transfer( + self, + send_byte: Callable[[int], None], + recv_unit: Callable[[int], bytes], + recv_data: Callable[[int], bytes], + ) -> None: + """The post-handshake exchange, identical over both transports.""" send_byte(espota2.RESPONSE_AUTH_OK) recv_unit(1) # ota type size = int.from_bytes(recv_unit(4), "big") @@ -159,9 +199,7 @@ class FakeEncryptedDevice(threading.Thread): received = b"" acked = 0 while len(received) < size: - plaintext = proto.decrypt(_recv_frame(sock)) - assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT - received += plaintext + received += recv_data(size - len(received)) if self.version >= espota2.OTA_VERSION_2_0: while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( len(received) == size and acked < size @@ -176,7 +214,10 @@ class FakeEncryptedDevice(threading.Thread): def _upload( - device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None + device: FakeEncryptedDevice, + firmware: bytes, + noise_psk: str | None, + plaintext_fallback: bool = False, ) -> None: device.start() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -184,12 +225,35 @@ def _upload( sock.connect(("127.0.0.1", device.port)) try: espota2.perform_ota( - sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + sock, + None, + io.BytesIO(firmware), + Path("firmware.bin"), + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, ) finally: sock.close() +def _run_ota( + device: FakeEncryptedDevice, firmware: bytes, tmp_path: Path, noise_psk: str +) -> int: + """Drive the retry loop, which is where the plaintext fallback reconnects.""" + path = tmp_path / "firmware.bin" + path.write_bytes(firmware) + device.start() + rc, _ = espota2.run_ota( + "127.0.0.1", + device.port, + None, + path, + noise_psk=noise_psk, + plaintext_fallback=True, + ) + return rc + + def test_encrypted_upload_success() -> None: """A full encrypted v2 upload spanning several 8192-byte blocks.""" pytest.importorskip("aioesphomeapi.noise") @@ -240,6 +304,56 @@ def test_client_fails_closed_when_device_lacks_encryption() -> None: device.join_and_check() +# Remove before 2027.3.0 +def test_fallback_when_device_does_not_offer(caplog: pytest.LogCaptureFixture) -> None: + """The api key is tried opportunistically; an older device that cannot + encrypt still gets its update, with a warning.""" + firmware = b"firmware" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + _upload(device, firmware, PSK, plaintext_fallback=True) + device.join_and_check() + assert device.received == firmware + assert any("fallback is removed in 2027.3.0" in r.message for r in caplog.records) + + +# Remove before 2027.3.0 +@pytest.mark.parametrize( + ("device_kwargs", "expected_rc", "fell_back"), + [ + # A wrong key against an offering device reconnects in plaintext + ({"psk": OTHER_PSK, "require_noise": False, "connections": 2}, 0, True), + # The plaintext retry is refused by a device that requires encryption + ({"psk": OTHER_PSK, "require_noise": True, "connections": 2}, 1, True), + # A dropped connection inside the handshake is retried encrypted + ({"require_noise": False, "connections": 2, "drop_handshakes": 1}, 0, False), + # A second transport fault inside the handshake falls back + ({"require_noise": False, "connections": 3, "drop_handshakes": 2}, 0, True), + ], + ids=["wrong_key", "wrong_key_required", "one_fault", "two_faults"], +) +def test_fallback_through_the_retry_loop( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, + device_kwargs: dict[str, Any], + expected_rc: int, + fell_back: bool, +) -> None: + pytest.importorskip("aioesphomeapi.noise") + firmware = b"firmware" + device = FakeEncryptedDevice(**device_kwargs) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + rc = _run_ota(device, firmware, tmp_path, PSK) + device.join_and_check() + assert rc == expected_rc + assert (device.received == firmware) is (expected_rc == 0) + assert ( + any("Retrying in plaintext" in r.message for r in caplog.records) is fell_back + ) + if expected_rc == 1: + assert any("requires an encrypted OTA" in r.message for r in caplog.records) + + def test_plaintext_client_gets_encryption_required_error() -> None: """A client without a key gets the device's 0x94 error message.""" device = FakeEncryptedDevice() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 5372a7203d..8fb9b7376e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -2108,7 +2108,13 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + "secret", + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2140,10 +2146,77 @@ def test_upload_program_ota_encryption_key( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=False, ) +def test_upload_program_ota_api_key_opportunistic( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Without an ota encryption block the api key is tried with a plaintext + fallback (removed in 2027.3.0).""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: key}}, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=True, + ) + + +@pytest.mark.parametrize( + "api_conf", + [{}, {CONF_ENCRYPTION: {}}], + ids=["no_encryption", "runtime_key"], +) +def test_upload_program_ota_no_usable_api_key_stays_plaintext( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + api_conf: dict[str, Any], +) -> None: + """A missing or runtime provisioned api key gives the uploader nothing + to try.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_API: api_conf, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert mock_run_ota.call_args.args[5] is None + assert mock_run_ota.call_args.kwargs == {"plaintext_fallback": False} + + def test_upload_program_ota_encryption_without_key_fails_closed( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2194,7 +2267,13 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + Path("custom.bin"), + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2250,6 +2329,7 @@ def test_upload_program_ota_partition_table_with_file_arg( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2312,6 +2392,7 @@ def test_upload_program_ota_partition_table_mqttip( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2500,6 +2581,7 @@ def test_upload_program_ota_bootloader_with_file_arg( bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, None, + plaintext_fallback=False, ) @@ -2988,7 +3070,13 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -3038,7 +3126,13 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.50"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5211,6 +5305,7 @@ def test_upload_program_ota_static_ip_with_mqttip( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5261,6 +5356,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5438,7 +5534,13 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) diff --git a/tests/unit_tests/test_wizard.py b/tests/unit_tests/test_wizard.py index 244e4eb5a1..f57ae71ae6 100644 --- a/tests/unit_tests/test_wizard.py +++ b/tests/unit_tests/test_wizard.py @@ -37,7 +37,6 @@ def wizard_answers() -> list[str]: "nodemcuv2", # board "SSID", # ssid "psk", # wifi password - "", # ota password (empty for no password) ] @@ -101,6 +100,25 @@ def test_config_file_should_include_ota(default_config: dict[str, Any]): assert "ota:" in config +def test_config_file_should_use_encryption_when_api_key_set( + default_config: dict[str, Any], +): + """ + With an API encryption key and no OTA password the OTA block reuses the key + """ + # Given + default_config["api_encryption_key"] = ( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + ) + + # When + config = wz.wizard_file(**default_config) + + # Then + assert "ota:\n - platform: esphome\n encryption:" in config + assert "password" not in config.split("ota:")[1].split("wifi:")[0] + + def test_config_file_should_include_ota_when_password_set( default_config: dict[str, Any], ): @@ -630,15 +648,15 @@ def test_wizard_write_protects_existing_config( assert config_file.read_text() == original_content -def test_wizard_accepts_ota_password( +def test_wizard_uses_the_api_key_for_ota( tmp_path: Path, monkeypatch: MonkeyPatch, wizard_answers: list[str] ): """ - The wizard should pass ota_password to wizard_write when the user provides one + The wizard generates an api key and does not ask for an OTA password; + the key secures OTA updates """ # Given - wizard_answers[5] = "my_ota_password" # Set OTA password config_file = tmp_path / "test.yaml" input_mock = MagicMock(side_effect=wizard_answers) monkeypatch.setattr("builtins.input", input_mock) @@ -653,8 +671,9 @@ def test_wizard_accepts_ota_password( # Then assert retval == 0 call_kwargs = wizard_write_mock.call_args.kwargs - assert "ota_password" in call_kwargs - assert call_kwargs["ota_password"] == "my_ota_password" + assert "api_encryption_key" in call_kwargs + assert "ota_password" not in call_kwargs + assert input_mock.call_count == len(wizard_answers) def test_wizard_accepts_rpipico_board(tmp_path: Path, monkeypatch: MonkeyPatch): From 96b1a03ea493a7281158907c6dd98184a48c05f2 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:05:16 +0000 Subject: [PATCH 040/178] Bump bundled esphome-device-builder to 1.14.4 (#19006) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e875851bfb..da76ab7b6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 RUN \ platformio settings set enable_telemetry No \ From c1aa41f276e4bc2b05f4b45031229623d93ab84a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:12:52 +0200 Subject: [PATCH 041/178] [noise] Bump noise-c to 0.1.24 and libsodium to 1.10021.6 (#18989) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index a1d9444fc0..4de706120e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.21") + cg.add_library("esphome/noise-c", "0.1.24") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.4") + cg.add_library("esphome/libsodium", "1.10021.6") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index fcf7caa7c7..779a05e7de 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.21 ; used by noise (api, ota) + esphome/noise-c@0.1.24 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index a263d7937f..4f7f5a4a4c 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.21") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.21") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.21"] + assert libs == ["esphome/noise-c @ 0.1.24"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.21", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.24", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.21"] + assert cls.calls == ["esphome/noise-c @ 0.1.24"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.21"] is None + assert compats["esphome/noise-c @ 0.1.24"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.21"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 379ef52ebd..fb79885736 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1576,7 +1576,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1596,7 +1596,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 5d2ddc658c3db2431fb71dfc78dc2df885f1cf78 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:26:44 +0200 Subject: [PATCH 042/178] [mdns] Guard LEAmDNS main loop calls against lwIP re-entrancy on ESP8266 (#18990) --- esphome/components/mdns/__init__.py | 2 + esphome/components/mdns/mdns_esp8266.cpp | 51 +++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index f039bb69f0..c8020104b3 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -192,6 +192,8 @@ async def to_code(config: ConfigType) -> None: if CORE.using_arduino: if CORE.is_esp8266: cg.add_library("ESP8266mDNS", None) + # No MDNS global in the build; mdns_esp8266.cpp owns a guarded MDNSResponder + cg.add_build_flag("-DNO_GLOBAL_MDNS") elif CORE.is_rp2: cg.add_library("LEAmDNS", None) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index 1f0b3c9519..0e600d3bac 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -13,8 +13,47 @@ namespace esphome::mdns { +// Main-loop calls into LEAmDNS that send (update() and close(); begin(), addService() and +// the scheduled restart never reach a send) can yield inside UdpContext::sendTimeout(); a +// packet arriving then re-enters LEAmDNS from lwIP on the same UdpContext and both sides +// free the same tx pbufs (#18760). Received packets stay queued during such a call and are +// processed from the main loop afterwards. +class GuardedMDNSResponder : public ::esp8266::MDNSImplementation::MDNSResponder { + public: + void update_guarded() { this->run_guarded_(&GuardedMDNSResponder::update); } + void close_guarded() { this->run_guarded_(&GuardedMDNSResponder::close); } + + private: + void run_guarded_(bool (GuardedMDNSResponder::*fn)()) { + UdpContext *ctx = this->m_pUDPContext; + if (ctx == nullptr) { + (this->*fn)(); + return; + } + // Set every time: a restart replaces the context together with its stock handler. Only + // begin() and the scheduled netif callback restart, never update() or close(), so the + // context cannot change underneath this call. + ctx->onRx([this]() { + if (!this->in_loop_call_) { + this->_callProcess(); + } + }); + this->in_loop_call_ = true; + (this->*fn)(); + // close() releases the context; a yield in here queues further packets for this loop too + while (this->m_pUDPContext != nullptr && this->m_pUDPContext->next()) { + this->_parseMessage(); + } + this->in_loop_call_ = false; + } + + volatile bool in_loop_call_{false}; +}; + +static GuardedMDNSResponder mdns_responder; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + static void register_esp8266(MDNSComponent *, StaticVector &services) { - MDNS.begin(App.get_name().c_str()); + mdns_responder.begin(App.get_name().c_str()); for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is @@ -30,10 +69,10 @@ static void register_esp8266(MDNSComponent *, StaticVectoris_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) return; #endif - MDNS.update(); + mdns_responder.update_guarded(); }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } @@ -81,7 +120,7 @@ void MDNSComponent::on_ip_state(const network::IPAddresses &ips, const network:: #endif void MDNSComponent::on_shutdown() { - MDNS.close(); + mdns_responder.close_guarded(); delay(10); } From e0e85db822309dd8fe17043552405c8c8b8374af Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:28:31 +0200 Subject: [PATCH 043/178] [core] Show the other downloader's progress while a prefetch job waits on its lock (#18983) --- esphome/framework_helpers.py | 61 +++++++++++++- esphome/platformio/prefetch.py | 87 ++++++++++---------- esphome/platformio/registry.py | 67 ++++++++++----- tests/unit_tests/conftest.py | 39 ++++++++- tests/unit_tests/test_framework_helpers.py | 17 ++++ tests/unit_tests/test_platformio_prefetch.py | 87 ++++++++++++++++++-- tests/unit_tests/test_platformio_registry.py | 73 ++++++++++++++-- 7 files changed, 348 insertions(+), 83 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 82bc0d3727..fc2a18a6ec 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -23,6 +23,7 @@ from esphome.net_retry import ( ) if TYPE_CHECKING: + from filelock import FileLock import requests PathType = str | os.PathLike @@ -909,6 +910,61 @@ def _part_path(dest: Path) -> Path: return dest.with_name(dest.name + ".part") +def downloaded_bytes(dest: Path, size: int | None = None) -> int: + """Bytes of ``dest`` on disk (its ``.part`` while streaming), capped at ``size``.""" + done = 0 + for candidate in (_part_path(dest), dest): + try: + done = candidate.stat().st_size + break + except FileNotFoundError: + continue + return done if size is None else min(done, size) + + +# Short lock-acquire slices so a waiting worker still observes Ctrl-C +_DOWNLOAD_LOCK_POLL = 1 + +# Waiting on another process's download; past this the caller leaves the +# file to its holder (the later sequential install waits on the same lock) +DOWNLOAD_LOCK_TIMEOUT = 60 + + +class DownloadLockUnavailable(OSError): + """The lock file cannot be used at all (a lock-less filesystem).""" + + +def wait_for_download_lock( + lock: "FileLock", + tracker: Callable[[int], None], + on_disk: Callable[[], int], + name: str, +) -> None: + """Acquire ``lock``, reporting ``on_disk()`` to ``tracker`` each poll so the + bar follows the holder's download. Raises filelock's ``Timeout`` once + ``DOWNLOAD_LOCK_TIMEOUT`` seconds pass.""" + from filelock import Timeout + + deadline = time.monotonic() + DOWNLOAD_LOCK_TIMEOUT + waiting = False + while True: + try: + lock.acquire(timeout=_DOWNLOAD_LOCK_POLL) + return + except Timeout: + pass + except OSError as err: + # Distinct from an OSError out of on_disk(), which must not + # read as "locks unsupported" + raise DownloadLockUnavailable(*err.args) from err + if not waiting: + waiting = True + _LOGGER.info("Waiting for another process downloading %s", name) + tracker(on_disk()) # raises when the batch is cancelled + if time.monotonic() >= deadline: + raise Timeout(lock.lock_file) + + def discard_partial_download(dest: Path) -> None: """Remove ``dest`` and the resume sidecars of an abandoned download.""" part = _part_path(dest) @@ -1319,10 +1375,7 @@ def download_from_mirrors( ) # Tick with the bytes already on disk so a combined bar holds # steady during the backoff instead of rewinding to zero - done = 0 - if progress is not None: - part = _part_path(path_target) - done = part.stat().st_size if part.is_file() else 0 + done = downloaded_bytes(path_target) if progress is not None else 0 _cancellable_sleep(delay, progress, done) # 3. Report every attempted URL if all mirrors failed. failures spans diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 5097239065..17a06cb9c1 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -33,11 +33,14 @@ import time from typing import Any, NamedTuple from esphome.framework_helpers import ( + DownloadLockUnavailable, content_length, discard_partial_download, + downloaded_bytes, failure_reason, resume_fetch_job, run_batch_downloads, + wait_for_download_lock, warn_prefetch_failures, ) from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree @@ -61,16 +64,10 @@ _RESOLVE_WORKERS = 8 # A hung child must not block the build; downloads resume on the next run _PREFETCH_TIMEOUT = 20 * 60 -# Waiting on another process's URL download; past this, leave it to pio -_DOWNLOAD_LOCK_TIMEOUT = 60 - # Child exit for a handled, already-warned failure; 1 would collide with # the interpreter's own import-failure exit _EXIT_HANDLED = 3 -# Short lock-acquire slices so a waiting worker still observes Ctrl-C -_URI_LOCK_POLL = 1 - # Resolution errored (vs a clean skip); suppresses the warm sentinel _RESOLVE_FAILED = object() @@ -462,51 +459,54 @@ def _uri_jobs( def _serialized_fetch_job( - dl_path: Path, lock_path: str, body: Any, unlocked_ok: bool = True + dl_path: Path, + lock_path: str, + body: Any, + size: int, + stream_dest: Path | None = None, + unlocked_ok: bool = True, ) -> Any: - """Wrap ``body`` so the shared destination is single-writer. - - Interleaved writers truncate each other's ``.part`` bytes (see - registry.py). The bounded poll observes Ctrl-C via the tracker; a - blown deadline is a clean skip (the holder's copy is what the build - needs). On a lock-less filesystem a sha256-verified body runs - unlocked with one warning; a checksum-less one - (``unlocked_ok=False``) is a counted failure instead. + """Wrap ``body`` so the shared destination is single-writer (interleaved + writers truncate each other's ``.part``, see registry.py). A blown deadline + is a clean skip. On a lock-less filesystem a sha256-verified body runs + unlocked with one warning; a checksum-less one (``unlocked_ok=False``) fails. """ + def on_disk() -> int: + # A URL job's holder streams beside the staging path until it + # promotes; after that only dl_path is left + done = downloaded_bytes(dl_path, size) + if not done and stream_dest is not None: + done = downloaded_bytes(stream_dest, size) + return done + def run(tracker: Any) -> None: from filelock import FileLock, Timeout # fallback_to_soft would leave a stale marker on lock-less # filesystems that blocks every later build (see git.py) lock = FileLock(lock_path, fallback_to_soft=False) - deadline = time.monotonic() + _DOWNLOAD_LOCK_TIMEOUT - while True: - try: - lock.acquire(timeout=_URI_LOCK_POLL) - break - except Timeout: - tracker(0) # raises when the batch is cancelled - if time.monotonic() >= deadline: - # Another process is fetching this same file; its copy - # is what the build needs (a large framework archive - # can hold the lock far longer than this deadline) - _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) - return - except OSError as err: - if not unlocked_ok: - # A body with no checksum to catch interleaved corruption - raise - lock = None - _LOGGER.warning( - "Could not lock %s (%s); downloading unlocked", - dl_path.name, - err, - ) - break + try: + wait_for_download_lock(lock, tracker, on_disk, dl_path.name) + except Timeout: + # The holder's copy is what the build needs (a large + # framework archive can outlast this deadline) + _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) + return + except DownloadLockUnavailable as err: + if not unlocked_ok: + # A body with no checksum to catch interleaved corruption + raise + lock = None + _LOGGER.warning( + "Could not lock %s (%s); downloading unlocked", + dl_path.name, + err, + ) try: if dl_path.is_file(): - return # another process finished it while we waited + tracker(size) # another process finished it while we waited + return body(tracker) finally: if lock is not None: @@ -540,6 +540,7 @@ def _registry_fetch_job( dl_path, f"{dl_path}.esphome.lock", resume_fetch_job(url, dl_path, sha256=checksum, size=size), + size, ) def run(tracker: Any) -> None: @@ -571,9 +572,9 @@ def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: tmp.replace(dl_path) def run(tracker: Any) -> None: - _serialized_fetch_job(dl_path, f"{tmp}.lock", promote, unlocked_ok=False)( - tracker - ) + _serialized_fetch_job( + dl_path, f"{tmp}.lock", promote, size, tmp, unlocked_ok=False + )(tracker) if dl_path.is_file(): # Won or lost, the race is over; staging files left behind # are dead weight PlatformIO's cache never prunes diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index 9538a28ff4..75df82da0e 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -17,8 +17,10 @@ from esphome.framework_helpers import ( archive_extract_all, download_from_mirrors, download_with_resume, + downloaded_bytes, rmdir, run_batch_downloads, + wait_for_download_lock, ) from esphome.net_retry import fetch_with_retry, http_request @@ -164,11 +166,17 @@ class _PendingArchive(NamedTuple): name: str version: str dest: Path + archive: Path url: str sha256: str size: int +def _archive_path(downloads_dir: Path, name: str, version: str) -> Path: + """The one archive path the prefetch and the sequential install share.""" + return downloads_dir / f"{name}-{version}" + + def _already_installed(dest: Path) -> bool: """Whether ``dest`` holds a completed install (extraction marker).""" return (dest / ".esphome_extracted").is_file() @@ -187,18 +195,18 @@ def prefetch_packages( lock as ``install_package``: the archive's ``.part`` file is shared, and two concurrent writers would truncate each other's bytes. """ - from filelock import FileLock + from filelock import FileLock, Timeout pending: list[_PendingArchive] = [] - seen: set[str] = set() + seen: set[Path] = set() for name, version, dest, mirrors in packages: if mirrors or (dest / ".esphome_extracted").is_file(): continue - archive_name = f"{name}-{version}" - if archive_name in seen: + archive = _archive_path(downloads_dir, name, version) + if archive in seen: # A duplicate entry would race itself between two workers continue - seen.add(archive_name) + seen.add(archive) try: url, sha256, size = registry_download(name, version) except EsphomeError as err: @@ -207,10 +215,9 @@ def prefetch_packages( continue if not size: continue - archive = downloads_dir / archive_name if archive.is_file() and archive.stat().st_size == size: continue - pending.append(_PendingArchive(name, version, dest, url, sha256, size)) + pending.append(_PendingArchive(name, version, dest, archive, url, sha256, size)) if len(pending) < 2: return downloads_dir.mkdir(parents=True, exist_ok=True) @@ -222,20 +229,36 @@ def prefetch_packages( def _fetch(entry: _PendingArchive, tracker: Callable[[int], None]) -> None: entry.dest.parent.mkdir(parents=True, exist_ok=True) - with FileLock(f"{entry.dest}.lock", fallback_to_soft=False): - # Marker re-check: a concurrent build may have installed (and - # deleted the archive of) this package while we waited; - # re-downloading would orphan a fresh copy in downloads_dir - # no branch: the thread tracer misses the skip edge; both - # arms of _already_installed are pinned directly - if not _already_installed(entry.dest): # pragma: no branch - download_with_resume( - entry.url, - downloads_dir / f"{entry.name}-{entry.version}", - sha256=entry.sha256, - size=entry.size, - progress=tracker, - ) + + def on_disk() -> int: + if done := downloaded_bytes(entry.archive, entry.size): + return done + # The holder deletes the archive once it has installed it + return entry.size if _already_installed(entry.dest) else 0 + + lock = FileLock(f"{entry.dest}.lock", fallback_to_soft=False) + try: + wait_for_download_lock(lock, tracker, on_disk, entry.name) + except Timeout: + # install_package waits on this same lock and verifies the + # holder's copy + _LOGGER.debug("Leaving %s to its current downloader", entry.name) + return + try: + if _already_installed(entry.dest): + # A concurrent build installed it while we waited; a + # re-download would orphan a fresh copy in downloads_dir + tracker(entry.size) + return + download_with_resume( + entry.url, + entry.archive, + sha256=entry.sha256, + size=entry.size, + progress=tracker, + ) + finally: + lock.release() failures = run_batch_downloads( "Downloading packages", @@ -288,7 +311,7 @@ def install_package( rmdir(dest, msg=f"Clean up incomplete {name} install") # Persistent location so an interrupted download resumes across runs. downloads_dir.mkdir(parents=True, exist_ok=True) - archive = downloads_dir / f"{name}-{version}" + archive = _archive_path(downloads_dir, name, version) _LOGGER.info("Downloading %s %s ...", name, version) if mirrors: _LOGGER.warning( diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 9de8f715ef..ad9c0bb11f 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -9,7 +9,7 @@ not be part of a unit test suite. """ -from collections.abc import Generator +from collections.abc import Callable, Generator import os from pathlib import Path import sys @@ -137,3 +137,40 @@ def mock_get_component() -> Generator[Mock, None, None]: """Mock get_component for config module.""" with patch("esphome.config.get_component") as mock: yield mock + + +@pytest.fixture +def held_lock() -> Callable[..., Callable[..., None]]: + """Factory for a ``FileLock.acquire`` fake held by another downloader. + + Each poll writes the next chunk to ``part`` (or runs it, for a callable) + and raises ``Timeout``; when the chunks run out the part is removed, + ``land()`` runs, and the acquire succeeds (also for any later job, so + ``land`` must be idempotent). + """ + from filelock import Timeout + + def make( + part: Path, + chunks: list[bytes | Callable[[], None]], + land: Callable[[], None], + ) -> Callable[..., None]: + polls = iter(chunks) + + def acquire(*args, **kwargs) -> None: + try: + chunk = next(polls) + except StopIteration: + part.unlink(missing_ok=True) + land() + return + if callable(chunk): + chunk() + else: + part.parent.mkdir(parents=True, exist_ok=True) + part.write_bytes(chunk) + raise Timeout("held") + + return acquire + + return make diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index fcc5572f51..22b34c9df5 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2353,3 +2353,20 @@ def test_discard_partial_download_logs_undeletable( ): framework_helpers.discard_partial_download(dest) assert "Could not remove" in caplog.text + + +def test_downloaded_bytes_reports_what_is_on_disk(tmp_path: Path) -> None: + """Part file first, then the landed file, both capped at size; else 0.""" + dest = tmp_path / "archive" + assert framework_helpers.downloaded_bytes(dest, 4) == 0 + part = tmp_path / "archive.part" + part.write_bytes(b"ab") + assert framework_helpers.downloaded_bytes(dest, 4) == 2 + part.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 + part.unlink() + dest.write_bytes(b"abc") + assert framework_helpers.downloaded_bytes(dest, 4) == 3 + assert framework_helpers.downloaded_bytes(dest) == 3 + dest.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index fb79885736..77490fd861 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -454,23 +454,96 @@ def test_uri_fetch_job_waits_out_a_briefly_held_lock(tmp_path: Path) -> None: assert dl_path.read_bytes() == b"data" -def test_lock_deadline_leaves_download_to_the_holder(tmp_path: Path) -> None: - """A lock held past the deadline means another process is fetching the - same file; skipping cleanly beats a misleading failure warning. The - tracker is still polled so a parked worker observes cancellation.""" +@pytest.mark.parametrize("staged", [b"", b"ab"]) +def test_lock_deadline_leaves_download_to_the_holder( + tmp_path: Path, staged: bytes +) -> None: + """A lock held past the deadline is another process's download; skip + cleanly, polling the tracker with what the holder has staged so far.""" dl_path = tmp_path / "archive" + (tmp_path / "archive.prefetch.part").write_bytes(staged) ticks: list[int] = [] with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) mock_download.assert_not_called() - assert ticks == [0] + assert ticks == [len(staged)] assert not dl_path.exists() +@pytest.mark.parametrize( + ("job", "part_name", "chunks", "expected"), + [ + ( + lambda dl_path: pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + ), + "archive.part", + [b"a", b"abc"], + [1, 3, 4], + ), + ( + lambda dl_path: pf._uri_fetch_job( + MagicMock(), "https://x/a.zip", dl_path, 4 + ), + "archive.prefetch.part", + [b"ab"], + [2, 4], + ), + ], + ids=["registry", "uri"], +) +def test_lock_wait_reports_the_holders_progress( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + held_lock, + job, + part_name: str, + chunks: list[bytes], + expected: list[int], +) -> None: + """A waiting job reports the holder's part file (the staging one for a + URL job), then the full size once the holder lands the archive.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / part_name, chunks, lambda: dl_path.write_bytes(b"abcd") + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + caplog.at_level(logging.INFO), + ): + job(dl_path)(ticks.append) + mock_download.assert_not_called() + assert ticks == expected + assert caplog.text.count("Waiting for another process downloading archive") == 1 + + +def test_uri_lock_wait_prefers_the_landed_archive(tmp_path: Path, held_lock) -> None: + """Between the holder's promotion rename and its release the staging + part is gone; the landed cache file is credited instead of 0.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / "archive.prefetch.part", + [b"ab", lambda: dl_path.write_bytes(b"abcd")], + lambda: None, + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) + mock_download.assert_not_called() + assert ticks == [2, 4, 4] + + def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: """A registry job that lost the download race to another process must not stamp a nonexistent archive into pio's usage.db.""" @@ -479,7 +552,7 @@ def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( lambda done: None diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 6ba8691c4e..9d5f6c4ce5 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -8,6 +8,7 @@ import os from pathlib import Path from unittest.mock import MagicMock, patch +from filelock import Timeout import pytest from esphome.core import EsphomeError @@ -540,16 +541,13 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: dest = tmp_path / "a" dest.mkdir() - from contextlib import contextmanager - - @contextmanager - def marker_appears_under_lock(path, **kwargs): + def marker_appears_under_lock(*args, **kwargs): # Simulates the concurrent build finishing while we waited (dest / ".esphome_extracted").touch() - yield with ( - patch("filelock.FileLock", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.acquire", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.release"), patch.object(registry, "download_with_resume") as mock_download, patch.object( registry, "registry_download", side_effect=_resolve_for({"a": 10}) @@ -559,6 +557,69 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: mock_download.assert_not_called() +def test_prefetch_packages_waits_with_the_holders_progress( + tmp_path: Path, held_lock +) -> None: + """A worker parked on another build's lock reports that build's part + file, then the full size once the marker appears.""" + dest = tmp_path / "a" + dest.mkdir() + ticks: list[int] = [] + part = tmp_path / "dl" / "a-1.0.part" + + def installed_and_pruned() -> None: + # install_package touches the marker, then unlinks the archive + (dest / ".esphome_extracted").touch() + part.unlink() + + acquire = held_lock( + part, + [lambda: None, b"abc", installed_and_pruned], + (dest / ".esphome_extracted").touch, + ) + + def fake_batch(header, jobs): + for _name, _size, fetch in jobs: + fetch(ticks.append) + return [] + + with ( + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + patch.object(registry, "run_batch_downloads", side_effect=fake_batch), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", dest, []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + assert ticks == [0, 3, 10, 10] + mock_download.assert_called_once() + + +def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( + tmp_path: Path, +) -> None: + """Past the deadline the worker skips; install_package waits on the same + lock later and verifies whatever the holder produced.""" + with ( + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", tmp_path / "a", []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + def test_already_installed_probe(tmp_path: Path) -> None: """Both arms of the marker probe the prefetch worker keys on.""" dest = tmp_path / "pkg" From 8434dc5474e433a61a250800b489674ec5116d84 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:29:20 +1200 Subject: [PATCH 044/178] [esp32_hosted] Add ESP-NOW-over-hosted shim for the ESP32-P4 (#17712) --- esphome/components/esp32_hosted/__init__.py | 36 ++ .../esp32_hosted/esp_now_hosted.cpp | 467 ++++++++++++++++++ .../esp32_hosted/esp_now_hosted_rpc.h | 128 +++++ esphome/components/espnow/__init__.py | 20 + esphome/core/defines.h | 1 + script/ci-custom.py | 17 +- .../test-espnow.esp32-p4-idf.yaml | 5 + tests/unit_tests/components/test_espnow.py | 48 ++ 8 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 esphome/components/esp32_hosted/esp_now_hosted.cpp create mode 100644 esphome/components/esp32_hosted/esp_now_hosted_rpc.h create mode 100644 tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml create mode 100644 tests/unit_tests/components/test_espnow.py diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index ab9455250c..21626e432b 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -37,6 +37,25 @@ CONF_HANDSHAKE_PIN = "handshake_pin" CONF_SDIO_FREQUENCY = "sdio_frequency" CONF_SPI_MODE = "spi_mode" +# ESP-NOW-over-hosted shim (esp_now_hosted.cpp). esp-hosted proxies esp_wifi.h +# but not esp_now.h (espressif/esp-hosted-mcu#19), and esp_wifi_remote injects +# the esp_now.h header on the ESP32-P4 host with no implementation, leaving the +# esp_now_* symbols undefined at link. On a P4 host, esp_now_hosted.cpp DEFINES +# those symbols and forwards each call to the co-processor over esp-hosted's +# CustomRpc "peer data transfer" channel, so ESPHome's `espnow` component links +# and runs unchanged (proven on a Tab5, 2026-07-20). The .cpp is guarded to +# CONFIG_IDF_TARGET_ESP32P4 so it compiles to nothing on hosts with a native +# ESP-NOW stack. CustomRpc needs these two host-side Kconfig options. Host +# registers 3 handlers (RESP, RECV, SEND); the coprocessor registers 1 (REQ); +# we ask for 8 to leave room for other CustomRpc extensions alongside. +# +# The coprocessor must run the matching custom firmware (a parallel effort in +# esphome/esp-hosted-firmware). esp_now_hosted_rpc.h here is the canonical copy +# of the wire contract and MUST stay byte-identical to the copy that coprocessor +# firmware uses — the packed structs are the on-wire layout, so any divergence +# silently corrupts every ESP-NOW frame. +_MAX_CUSTOM_MSG_HANDLERS = 8 + # Shared fields for both transport modes BASE_SCHEMA = cv.Schema( { @@ -262,6 +281,23 @@ async def to_code(config: ConfigType) -> None: else: _configure_spi(config) + # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see + # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not + # gated on the `espnow` component being present: the shim is tiny and the + # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options + # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so + # coupling the two keeps the build consistent. When `espnow` is absent the + # symbols are simply unused and never register a callback at runtime. + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + add_define("USE_ESP_NOW_HOSTED") + # esp-hosted's CustomRpc ("peer data transfer") path — off by default. + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS + ) + # Place the transport mempool in PSRAM. Required on memory-tight host # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM # mempool allocation fails at boot with `sdio_mempool_create` assert. diff --git a/esphome/components/esp32_hosted/esp_now_hosted.cpp b/esphome/components/esp32_hosted/esp_now_hosted.cpp new file mode 100644 index 0000000000..ad29b208fe --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted.cpp @@ -0,0 +1,467 @@ +/* + * esp_now_hosted — host-side shim implementing over esp-hosted + * CustomRpc, so ESPHome's `espnow` component can run on a radio-less host + * (e.g. the ESP32-P4) whose radio lives on an esp-hosted co-processor. + * + * A radio-less host has no native ESP-NOW. esp_wifi_remote INJECTS the full + * esp_now.h header (types + declarations) but ships NO implementation, so every + * esp_now_* symbol is an undefined reference at link time. This translation + * unit provides those definitions; each forwards to the co-processor over + * CustomRpc (see esphome/esp-hosted-firmware for the matching coprocessor + * handlers). No esp-hosted or esp_wifi_remote source is patched, and there is no + * duplicate-symbol clash because nothing else defines these symbols here. + * + * See esp_now_hosted_rpc.h for the wire protocol. + */ + +#include "sdkconfig.h" + +// Only build the shim on the radio-less host. On chips with a native ESP-NOW +// stack (S3, C6, …) the real symbols exist and this file must stay empty to +// avoid duplicate definitions. +#if defined(CONFIG_IDF_TARGET_ESP32P4) + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" + +#include "esp_idf_version.h" +#include "esp_log.h" +#include "esp_timer.h" + +#include // injected declarations we are now DEFINING +#include // wifi_pkt_rx_ctrl_t, wifi_tx_info_t + +// esp_hosted_misc.h (host) ships WITHOUT an extern "C" guard, so including it +// from C++ would give its declarations C++ linkage and the real C symbols in +// libesp_hosted would go unresolved at link. Wrap it. (Verified vs +// esp_hosted 2.12.9.) +extern "C" { +#include "esp_hosted_misc.h" // esp_hosted_{send_custom_data,register_custom_callback} +} + +#include "esp_now_hosted_rpc.h" + +namespace { + +const char *const TAG = "esp_now_hosted"; + +// One outstanding request at a time. ESPHome drives esp_now_* from the main +// loop; the matching response and the async RECV/SEND events all arrive on the +// single esp-hosted RPC RX thread. Serializing requests keeps the shared +// response slot race-free; a sequence number stops a late/stale response from +// being mistaken for ours. +SemaphoreHandle_t g_req_mutex = nullptr; +SemaphoreHandle_t g_resp_sem = nullptr; // given when the matching RESP lands +bool g_setup_done = false; // set only after setup fully succeeds +uint8_t g_seq = 0; +volatile uint8_t g_expect_seq = 0; +volatile int32_t g_resp_status = 0; +uint8_t g_resp_ret[16]; +volatile uint16_t g_resp_ret_len = 0; + +// Written from the main loop (register/unregister/deinit), read from the +// esp-hosted RX thread (on_recv/on_send). volatile for the same reason the +// g_resp_* globals are: force the RX thread to observe an updated pointer +// (e.g. a nulling by esp_now_deinit) rather than a cached one. +volatile esp_now_recv_cb_t g_recv_cb = nullptr; +volatile esp_now_send_cb_t g_send_cb = nullptr; + +// Local mirror of the co-processor's peer table. ESPHome's espnow component +// calls esp_now_is_peer_exist() on the main loop for every received frame +// (twice) and every send; forwarding each as a blocking RPC round-trip stalls +// the loop. The shim is the only path that mutates the co-processor peer table +// (add/del/deinit all go through here), so this mirror is authoritative and +// esp_now_is_peer_exist() can answer from it with no round-trip. +// +// esp_now_* are public C symbols: any component or user lambda may call them, +// and although ESPHome's espnow touches peers only from the main loop today +// (its RX/TX callbacks merely enqueue), the shim cannot rely on that. A short +// spinlock keeps the mirror consistent from any task/core, matching native +// esp_now_*'s own internal thread-safety. The critical sections are a bounded +// (<=20-entry) scan, so they stay tiny. ESP_NOW_MAX_TOTAL_PEER_NUM is 20. +constexpr size_t ESP_NOW_HOSTED_MAX_PEERS = 20; +uint8_t g_peer_cache[ESP_NOW_HOSTED_MAX_PEERS][6]; +size_t g_peer_count = 0; +portMUX_TYPE g_peer_lock = portMUX_INITIALIZER_UNLOCKED; + +// Caller must hold g_peer_lock. +int peer_cache_find_locked(const uint8_t *mac) { + for (size_t i = 0; i < g_peer_count; i++) { + if (memcmp(g_peer_cache[i], mac, 6) == 0) + return static_cast(i); + } + return -1; +} + +bool peer_cache_contains(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const bool found = peer_cache_find_locked(mac) >= 0; + portEXIT_CRITICAL(&g_peer_lock); + return found; +} + +void peer_cache_add(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + if (peer_cache_find_locked(mac) < 0 && g_peer_count < ESP_NOW_HOSTED_MAX_PEERS) + memcpy(g_peer_cache[g_peer_count++], mac, 6); + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_remove(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const int idx = peer_cache_find_locked(mac); + if (idx >= 0) { + g_peer_count--; + if (static_cast(idx) != g_peer_count) // move the last entry into the gap + memcpy(g_peer_cache[idx], g_peer_cache[g_peer_count], 6); + } + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_clear() { + portENTER_CRITICAL(&g_peer_lock); + g_peer_count = 0; + portEXIT_CRITICAL(&g_peer_lock); +} + +// ── CustomRpc event handlers (run on the esp-hosted RPC RX thread) ────────── +// Keep them short and non-blocking. In particular they MUST NOT call back into +// any esp_now_* shim function: that would try to take g_req_mutex / wait on the +// RX thread that delivers the response, and deadlock. + +void on_resp(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + if (len < sizeof(esp_now_hosted_resp_t)) { + ESP_LOGW(TAG, "RESP too short: %u bytes", static_cast(len)); + return; + } + const auto *r = reinterpret_cast(data); + if (r->seq != g_expect_seq) { // late response from a timed-out request (expected) + ESP_LOGV(TAG, "dropping stale RESP seq %u (want %u)", r->seq, g_expect_seq); + return; + } + g_resp_status = r->status; + uint16_t rl = r->ret_len; + if (rl > sizeof(g_resp_ret)) { + // Larger than any real opcode return — a likely wire-format drift signal. + ESP_LOGW(TAG, "RESP ret_len %u exceeds buffer, clamping (wire drift?)", rl); + rl = sizeof(g_resp_ret); + } + if (len >= sizeof(esp_now_hosted_resp_t) + rl) { + memcpy(g_resp_ret, r->ret, rl); + } else { + // Truncated frame: fail closed. Never hand the caller stale bytes left in + // g_resp_ret by a previous response, and don't let request() report a + // zeroed payload as success — override the status to an error. + ESP_LOGW(TAG, "RESP truncated: claims %u ret bytes, frame too short", rl); + rl = 0; + g_resp_status = ESP_ERR_INVALID_RESPONSE; + } + g_resp_ret_len = rl; + xSemaphoreGive(g_resp_sem); +} + +void on_recv(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once: esp_now_unregister_recv_cb()/deinit() (via + // the espnow component's disable()) can null it on the main loop between the + // guard and the call, which would otherwise turn the call into a null-deref. + const esp_now_recv_cb_t cb = g_recv_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_recv_evt_t)) { + ESP_LOGW(TAG, "RECV too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); + if (len < sizeof(esp_now_hosted_recv_evt_t) + e->data_len) { + ESP_LOGW(TAG, "RECV data_len %u exceeds frame", e->data_len); + return; + } + + // ESPHome dereferences info->rx_ctrl->{rssi,timestamp}; give it a real one. + wifi_pkt_rx_ctrl_t rx_ctrl; + memset(&rx_ctrl, 0, sizeof(rx_ctrl)); + rx_ctrl.rssi = e->rssi; + rx_ctrl.channel = e->channel; + rx_ctrl.timestamp = static_cast(esp_timer_get_time()); + + esp_now_recv_info_t info; + info.src_addr = const_cast(e->src_addr); + info.des_addr = const_cast(e->des_addr); + info.rx_ctrl = &rx_ctrl; + cb(&info, e->data, static_cast(e->data_len)); +} + +void on_send(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once (see on_recv): disable()/deinit() can null it + // on the main loop concurrently with this RX-thread callback. + const esp_now_send_cb_t cb = g_send_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_send_evt_t)) { + ESP_LOGW(TAG, "SEND evt too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0) + // IDF >= 5.5: esp_now_send_cb_t takes esp_now_send_info_t (== wifi_tx_info_t), + // whose des_addr is a POINTER (not an inline array). Point it at the event's + // MAC (valid for this callback) — do NOT memcpy into it (that writes NULL and + // faults). ESPHome reads only info->des_addr. + esp_now_send_info_t si; + memset(&si, 0, sizeof(si)); + si.des_addr = const_cast(e->des_addr); + cb(&si, static_cast(e->status)); +#else + cb(e->des_addr, static_cast(e->status)); +#endif +} + +esp_err_t ensure_setup() { + // Gate on g_setup_done, not on g_req_mutex: a failure part-way through (a + // semaphore that did not allocate, a callback that did not register) must not + // leave a later call thinking setup completed. Semaphore creation is guarded + // so a retry after a partial failure does not leak the earlier handles. + if (g_setup_done) + return ESP_OK; + if (g_req_mutex == nullptr) + g_req_mutex = xSemaphoreCreateMutex(); + if (g_resp_sem == nullptr) + g_resp_sem = xSemaphoreCreateBinary(); + if (g_req_mutex == nullptr || g_resp_sem == nullptr) + return ESP_ERR_NO_MEM; + esp_err_t err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RESP, on_resp, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RECV, on_recv, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_SEND, on_send, nullptr)) != ESP_OK) + return err; + g_setup_done = true; + return ESP_OK; +} + +// Send one request envelope. With wait=true (default) block until the matching +// response (or timeout); with wait=false return as soon as the frame is handed +// to the transport (fire-and-forget, used by esp_now_send). +// +// `tail` is an optional second chunk written straight after `payload`. Callers +// with a fixed header plus a bulk body (esp_now_send) pass the two separately +// so they never need a build buffer of their own: both chunks are laid into the +// request buffer here, under g_req_mutex, which keeps concurrent callers from +// racing and saves a full copy of the body on every transmit. +esp_err_t request(uint8_t opcode, const void *payload, uint16_t plen, void *ret, uint16_t ret_cap, uint16_t *ret_len, + bool wait = true, const void *tail = nullptr, uint16_t tail_len = 0) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + if (plen > ESP_NOW_HOSTED_MAX_PAYLOAD || tail_len > ESP_NOW_HOSTED_MAX_PAYLOAD - plen) + return ESP_ERR_INVALID_SIZE; + const uint16_t total_len = static_cast(plen + tail_len); + + if (xSemaphoreTake(g_req_mutex, portMAX_DELAY) != pdTRUE) + return ESP_FAIL; + + static uint8_t buf[sizeof(esp_now_hosted_req_t) + ESP_NOW_HOSTED_MAX_PAYLOAD]; // guarded by g_req_mutex + auto *req = reinterpret_cast(buf); + req->opcode = opcode; + req->seq = ++g_seq; + req->payload_len = total_len; + if (plen != 0) + memcpy(req->payload, payload, plen); + if (tail_len != 0) + memcpy(req->payload + plen, tail, tail_len); + g_expect_seq = req->seq; + + xSemaphoreTake(g_resp_sem, 0); // drain any stale signal before sending + err = esp_hosted_send_custom_data(ESP_NOW_HOSTED_MSG_REQ, buf, sizeof(esp_now_hosted_req_t) + total_len); + if (err != ESP_OK) { + xSemaphoreGive(g_req_mutex); + return err; + } + if (!wait) { + // Fire-and-forget (esp_now_send): the co-processor enqueues the frame and + // reports the real TX result later via the async SEND event, exactly like + // native esp_now_send. Returning here keeps the main loop off the ~100 ms+ + // RPC round-trip. The matching RESP is ignored (seq won't match the next + // waited request, so on_resp drops it). + xSemaphoreGive(g_req_mutex); + return ESP_OK; + } + if (xSemaphoreTake(g_resp_sem, pdMS_TO_TICKS(ESP_NOW_HOSTED_TIMEOUT_MS)) != pdTRUE) { + ESP_LOGW(TAG, "opcode %u timed out", opcode); + xSemaphoreGive(g_req_mutex); + return ESP_ERR_TIMEOUT; + } + + const int32_t status = g_resp_status; + if (ret != nullptr && ret_cap != 0) { + uint16_t n = g_resp_ret_len < ret_cap ? g_resp_ret_len : ret_cap; + memcpy(ret, const_cast(g_resp_ret), n); + if (ret_len != nullptr) + *ret_len = n; + } + xSemaphoreGive(g_req_mutex); + return static_cast(status); +} + +} // namespace + +// ── The surface, defined for the radio-less host ──────────────── +extern "C" { + +esp_err_t esp_now_init(void) { return request(ESP_NOW_HOSTED_OP_INIT, nullptr, 0, nullptr, 0, nullptr); } + +esp_err_t esp_now_deinit(void) { + g_recv_cb = nullptr; + g_send_cb = nullptr; + peer_cache_clear(); // the co-processor drops all peers on deinit + return request(ESP_NOW_HOSTED_OP_DEINIT, nullptr, 0, nullptr, 0, nullptr); +} + +esp_err_t esp_now_get_version(uint32_t *version) { + uint32_t v = 0; + uint16_t rl = 0; + esp_err_t err = request(ESP_NOW_HOSTED_OP_GET_VERSION, nullptr, 0, &v, sizeof(v), &rl); + if (version != nullptr) + *version = v; + return err; +} + +esp_err_t esp_now_register_recv_cb(esp_now_recv_cb_t cb) { + // Only arm the callback once the CustomRpc handlers are actually registered, + // so a failed setup leaves g_recv_cb null rather than falsely "registered". + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_recv_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_recv_cb(void) { + g_recv_cb = nullptr; + return ESP_OK; +} +esp_err_t esp_now_register_send_cb(esp_now_send_cb_t cb) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_send_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_send_cb(void) { + g_send_cb = nullptr; + return ESP_OK; +} + +static esp_err_t add_or_mod_peer(uint8_t opcode, const esp_now_peer_info_t *peer, bool wait) { + if (peer == nullptr) + return ESP_ERR_ESPNOW_ARG; + esp_now_hosted_peer_t p; + memset(&p, 0, sizeof(p)); + memcpy(p.peer_addr, peer->peer_addr, 6); + memcpy(p.lmk, peer->lmk, 16); + p.channel = peer->channel; + p.ifidx = static_cast(peer->ifidx); + p.encrypt = peer->encrypt ? 1 : 0; + return request(opcode, &p, sizeof(p), nullptr, 0, nullptr, wait); +} +esp_err_t esp_now_add_peer(const esp_now_peer_info_t *peer) { + // Fire-and-forget (wait=false): adding a peer is a blocking RPC round-trip, + // and ESPHome's espnow calls it on the main loop when a device joins the mesh + // — under co-processor load that stalls the UI (peer-churn stutter). Issue it + // without waiting and mirror it locally. Safe against a following + // esp_now_send to the same peer: both ride the same in-order CustomRpc + // channel (mutex-serialized on the host) and the co-processor processes REQs + // FIFO, so ADD_PEER is applied before the SEND. Trade-off: a co-processor-side + // failure (e.g. peer table full) is no longer reported synchronously — the + // same limitation as esp_now_send — but ESPHome only adds peers it validated. + esp_err_t err = add_or_mod_peer(ESP_NOW_HOSTED_OP_ADD_PEER, peer, /*wait=*/false); + if (err == ESP_OK) + peer_cache_add(peer->peer_addr); // keep the local mirror in sync + return err; +} +esp_err_t esp_now_mod_peer(const esp_now_peer_info_t *peer) { + // mod_peer changes a peer's parameters, not its existence, so the cache is + // unaffected. Kept synchronous — it is not on any hot path (espnow never + // calls it), so the extra round-trip does not matter and the status is useful. + return add_or_mod_peer(ESP_NOW_HOSTED_OP_MOD_PEER, peer, /*wait=*/true); +} + +esp_err_t esp_now_del_peer(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return ESP_ERR_ESPNOW_ARG; + // Fire-and-forget for the same reason as add_peer (peer churn on the main + // loop). Removal is order-independent, so this is strictly safe. + esp_err_t err = request(ESP_NOW_HOSTED_OP_DEL_PEER, peer_addr, 6, nullptr, 0, nullptr, /*wait=*/false); + if (err == ESP_OK) + peer_cache_remove(peer_addr); // keep the local mirror in sync + return err; +} + +bool esp_now_is_peer_exist(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return false; + // Answered from the local mirror — no RPC round-trip. ESPHome's espnow calls + // this on the main loop for every received frame and every send, so a + // blocking round-trip here would stall rendering under mesh traffic. + return peer_cache_contains(peer_addr); +} + +esp_err_t esp_now_send(const uint8_t *peer_addr, const uint8_t *data, size_t len) { + if (len > ESP_NOW_HOSTED_MAX_FRAME) + return ESP_ERR_ESPNOW_ARG; + if (data == nullptr && len != 0) // native esp_now_send treats this as an arg error + return ESP_ERR_ESPNOW_ARG; + // Only the small fixed header is built here; the caller's frame goes over as + // the request tail, so request() lays both into its own buffer under + // g_req_mutex. esp_now_send is a public C symbol and may be called from any + // task, and a shared build buffer here would let two callers corrupt each + // other's frame. Passing the body through also drops a full-frame copy per + // transmit, on the path this shim exists to keep quick. + uint8_t hdr[sizeof(esp_now_hosted_send_req_t)]; + auto *s = reinterpret_cast(hdr); + s->has_addr = peer_addr != nullptr ? 1 : 0; + if (peer_addr != nullptr) + memcpy(s->peer_addr, peer_addr, 6); + else + memset(s->peer_addr, 0, 6); + s->data_len = static_cast(len); + // Fire-and-forget (wait=false): native esp_now_send returns once the frame is + // queued, with the real TX result delivered later through the send callback. + // The co-processor mirrors that — it acks enqueue immediately and reports the + // outcome via the async SEND event (on_send -> on_send_report). Waiting for + // the RPC RESP here would block the main loop for the full round-trip on + // every transmit. + return request(ESP_NOW_HOSTED_OP_SEND, hdr, sizeof(hdr), nullptr, 0, nullptr, /*wait=*/false, data, + static_cast(len)); +} + +esp_err_t esp_now_set_pmk(const uint8_t *pmk) { + if (pmk == nullptr) + return ESP_ERR_ESPNOW_ARG; + return request(ESP_NOW_HOSTED_OP_SET_PMK, pmk, 16, nullptr, 0, nullptr); +} + +// Remainder of the surface. Not used by ESPHome's espnow component +// today; provided so the whole header links and future callers get a defined +// (if unimplemented) symbol rather than a link error. Wire them through +// CustomRpc if a use case appears. +esp_err_t esp_now_get_peer(const uint8_t * /*peer_addr*/, esp_now_peer_info_t * /*peer*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_now_fetch_peer(bool /*from_head*/, esp_now_peer_info_t * /*peer*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_get_peer_num(esp_now_peer_num_t * /*num*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_set_wake_window(uint16_t /*window*/) { + return ESP_ERR_NOT_SUPPORTED; // power-save wake window is not forwarded; don't claim success +} +esp_err_t esp_now_set_peer_rate_config(const uint8_t * /*peer_addr*/, esp_now_rate_config_t * /*cfg*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_wifi_config_espnow_rate(wifi_interface_t /*ifx*/, wifi_phy_rate_t /*rate*/) { + return ESP_ERR_NOT_SUPPORTED; +} + +} // extern "C" + +#endif // CONFIG_IDF_TARGET_ESP32P4 diff --git a/esphome/components/esp32_hosted/esp_now_hosted_rpc.h b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h new file mode 100644 index 0000000000..bf68c759ee --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h @@ -0,0 +1,128 @@ +/* + * esp_now_hosted — ESP-NOW-over-CustomRpc wire protocol. + * + * Shared, byte-for-byte-identical contract between: + * - the host shim (esphome/components/esp32_hosted/esp_now_hosted.cpp) + * - the coprocessor firmware (esphome/esp-hosted-firmware) + * + * It rides esp-hosted's CustomRpc channel (RPC ID 388, "peer data transfer", + * available since esp-hosted v2.8.1), teaching the radio-less host <-> radio + * co-processor link to carry esp_now.h, which esp-hosted itself does not proxy + * (Espressif issue espressif/esp-hosted-mcu#19). + * + * KEEP THE TWO COPIES IN SYNC. The canonical copy lives here; the coprocessor + * firmware uses a verbatim copy. Both sides are little-endian, so these packed + * structs are wire-compatible with no byte-swapping. + */ + +#ifndef ESP_NOW_HOSTED_RPC_H +#define ESP_NOW_HOSTED_RPC_H + +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* ── CustomRpc message IDs (any uint32_t except 0xFFFFFFFF) ────────────────── + * One REQ handler slot on the device; three event handler slots on the host. + * The bytes spell "now" + index, a private range unlikely to clash with other + * CustomRpc users (e.g. the stock peer_data_transfer example's 1..6). */ +#define ESP_NOW_HOSTED_MSG_REQ 0x6E6F7701u /* host -> device : request envelope */ +#define ESP_NOW_HOSTED_MSG_RESP 0x6E6F7702u /* device -> host : reply to a REQ */ +#define ESP_NOW_HOSTED_MSG_RECV 0x6E6F7703u /* device -> host : async RX frame */ +#define ESP_NOW_HOSTED_MSG_SEND 0x6E6F7704u /* device -> host : async TX status */ + +/* ── Request opcodes ────────────────────────────────────────────────────── */ +enum { + ESP_NOW_HOSTED_OP_INIT = 1, /* esp_now_init + register device recv/send cbs */ + ESP_NOW_HOSTED_OP_DEINIT = 2, /* unregister cbs + esp_now_deinit */ + ESP_NOW_HOSTED_OP_ADD_PEER = 3, /* payload: esp_now_hosted_peer_t */ + ESP_NOW_HOSTED_OP_DEL_PEER = 4, /* payload: 6-byte peer MAC */ + ESP_NOW_HOSTED_OP_IS_PEER_EXIST = 5, /* payload: 6-byte MAC; ret: 1 byte bool */ + ESP_NOW_HOSTED_OP_SEND = 6, /* payload: esp_now_hosted_send_req_t */ + ESP_NOW_HOSTED_OP_GET_VERSION = 7, /* ret: uint32 version */ + ESP_NOW_HOSTED_OP_SET_PMK = 8, /* payload: 16-byte PMK */ + ESP_NOW_HOSTED_OP_MOD_PEER = 9, /* payload: esp_now_hosted_peer_t */ +}; + +/* Largest ESP-NOW payload we forward. ESP-NOW v2 (IDF >= 5.4) is 1470 B; well + * under esp-hosted's 8166 B CustomRpc cap, so the shim never truncates. */ +#define ESP_NOW_HOSTED_MAX_FRAME 1470u +/* Envelope slack for the largest opcode payload (a SEND req wrapping a frame). */ +#define ESP_NOW_HOSTED_MAX_PAYLOAD (ESP_NOW_HOSTED_MAX_FRAME + 16u) +/* Host request/response round-trip timeout over the transport. Generous: + * normal RTT is sub-millisecond, but Wi-Fi/BLE contention on the co-processor + * can stall the RX thread. */ +#define ESP_NOW_HOSTED_TIMEOUT_MS 2000 + +/* ── Envelopes ──────────────────────────────────────────────────────────── */ + +/* These payloads are shared verbatim with the C co-processor firmware, so they + * use C's `typedef struct {...} name;` idiom rather than C++ `using` aliases, + * which would not compile there. Silence clang-tidy's modernize-use-using for + * the shared struct block. */ +// NOLINTBEGIN(modernize-use-using) +typedef struct { + uint8_t opcode; /* one of ESP_NOW_HOSTED_OP_* */ + uint8_t seq; /* wraps 0..255; echoed in the response for matching */ + uint16_t payload_len; /* bytes of opcode-specific payload that follow */ + uint8_t payload[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_req_t; + +typedef struct { + uint8_t opcode; /* echoes the request opcode */ + uint8_t seq; /* echoes the request seq */ + int32_t status; /* esp_err_t from the native call on the co-processor */ + uint16_t ret_len; /* bytes of return payload that follow */ + uint8_t ret[]; /* flexible (e.g. version u32, is_peer_exist bool) */ +} __attribute__((packed)) esp_now_hosted_resp_t; + +/* ── Opcode payloads ────────────────────────────────────────────────────── */ + +/* esp_now_peer_info_t minus the host-only `priv` pointer, which is meaningless + * across the transport and never set by ESPHome's espnow component. */ +typedef struct { + uint8_t peer_addr[6]; + uint8_t lmk[16]; + uint8_t channel; /* 0 = current channel */ + uint8_t ifidx; /* wifi_interface_t (0=STA, 1=AP) */ + uint8_t encrypt; /* bool */ +} __attribute__((packed)) esp_now_hosted_peer_t; + +typedef struct { + uint8_t has_addr; /* 0 => peer_addr is NULL (broadcast to all peers) */ + uint8_t peer_addr[6]; + uint16_t data_len; + uint8_t data[]; /* flexible, up to ESP_NOW_HOSTED_MAX_FRAME */ +} __attribute__((packed)) esp_now_hosted_send_req_t; + +/* ── Async events (device -> host) ──────────────────────────────────────── */ + +/* Reconstructed on the host into an esp_now_recv_info_t + a minimal + * wifi_pkt_rx_ctrl_t. ESPHome's espnow reads info->src_addr, info->des_addr, + * info->rx_ctrl->rssi and info->rx_ctrl->timestamp. */ +typedef struct { + uint8_t src_addr[6]; + uint8_t des_addr[6]; + int8_t rssi; + uint8_t channel; + uint16_t data_len; + uint8_t data[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_recv_evt_t; + +typedef struct { + uint8_t des_addr[6]; + uint8_t status; /* esp_now_send_status_t (0 = success) */ +} __attribute__((packed)) esp_now_hosted_send_evt_t; +// NOLINTEND(modernize-use-using) + +#ifdef __cplusplus +} +#endif + +#endif /* ESP_NOW_HOSTED_RPC_H */ diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 5541a6ee97..14d099ec06 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -3,6 +3,7 @@ from typing import Any from esphome import automation, core import esphome.codegen as cg from esphome.components import wifi +from esphome.components.esp32 import VARIANT_ESP32P4, get_esp32_variant from esphome.components.udp import CONF_ON_RECEIVE import esphome.config_validation as cv from esphome.const import ( @@ -17,6 +18,7 @@ from esphome.const import ( ) from esphome.core import CORE, HexInt from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] @@ -132,6 +134,24 @@ CONFIG_SCHEMA = cv.All( ) +def _validate_variant(config: ConfigType) -> ConfigType: + # ESP-NOW rides the Wi-Fi PHY. Radio-less esp32 variants have no native + # ESP-NOW; only the ESP32-P4 has a path, via the esp32_hosted shim that + # supplies the esp_now_* symbols. Fail here with a clear message instead of + # letting the build reach an "undefined reference to esp_now_*" link error. + variant = get_esp32_variant() + if wifi.variant_has_wifi(variant): + return config + if variant != VARIANT_ESP32P4: + raise cv.Invalid(f"ESP-NOW is not supported on {variant} (no Wi-Fi radio)") + if "esp32_hosted" not in fv.full_config.get(): + raise cv.Invalid(f"ESP-NOW on {variant} requires the esp32_hosted component") + return config + + +FINAL_VALIDATE_SCHEMA = _validate_variant + + async def _trigger_to_code(config: ConfigType) -> MockObj: if address := config.get(CONF_ADDRESS): address = address.parts diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9dd1e0ced6..eaece6d5ff 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_ESP32_HOSTED #define USE_ESP32_HOSTED_HTTP_UPDATE #define USE_ESP32_IMPROV_STATE_CALLBACK +#define USE_ESP_NOW_HOSTED #define USE_EVENT #define USE_FAN #define USE_GPIO_BINARY_SENSOR_INTERRUPT diff --git a/script/ci-custom.py b/script/ci-custom.py index f481fda860..e2b7cd8d37 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -294,6 +294,9 @@ def highlight(s): "esphome/components/socket/headers.h", "esphome/core/defines.h", "esphome/components/http_request/httplib.h", + # Shared C wire header (byte-identical with the co-processor firmware); + # these are protocol constants and constexpr is C++-only. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_no_defines(fname, match): @@ -816,6 +819,10 @@ def lint_relative_py_import(fname: Path, line, col, content): "esphome/components/host/helpers.cpp", "esphome/components/zephyr/helpers.cpp", "esphome/components/http_request/httplib.h", + # Global extern "C" esp_now_* linker symbols + shared C wire header; + # neither can live in a C++ namespace. + "esphome/components/esp32_hosted/esp_now_hosted.cpp", + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: @@ -841,7 +848,15 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) +@lint_content_check( + include=["*.h"], + exclude=[ + "esphome/core/entity_types.h", + # Shared C wire header; uses a classic #ifndef guard for portability + # across the co-processor firmware repo it stays byte-identical with. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + ], +) def lint_pragma_once(fname, content): if "#pragma once" not in content: return ( diff --git a/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml new file mode 100644 index 0000000000..fab0a64ab8 --- /dev/null +++ b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml @@ -0,0 +1,5 @@ +# Exercises the ESP-NOW-over-hosted shim: on the ESP32-P4 host, esp32_hosted +# supplies the esp_now_* symbols that the espnow component links against. +packages: + esp32_hosted: !include common.yaml + espnow: !include ../espnow/common.yaml diff --git a/tests/unit_tests/components/test_espnow.py b/tests/unit_tests/components/test_espnow.py new file mode 100644 index 0000000000..21305c2b33 --- /dev/null +++ b/tests/unit_tests/components/test_espnow.py @@ -0,0 +1,48 @@ +"""Tests for the espnow component's final validation.""" + +import pytest + +from esphome.components.esp32.const import ( + VARIANT_ESP32C3, + VARIANT_ESP32H2, + VARIANT_ESP32P4, +) +from esphome.components.espnow import _validate_variant +import esphome.config_validation as cv +import esphome.final_validate as fv +from esphome.types import ConfigType + + +def _run( + monkeypatch, variant: str, full_config: dict, config: ConfigType +) -> ConfigType: + monkeypatch.setattr("esphome.components.espnow.get_esp32_variant", lambda: variant) + token = fv.full_config.set(full_config) + try: + return _validate_variant(config) + finally: + fv.full_config.reset(token) + + +def test_variant_with_native_wifi_passes(monkeypatch) -> None: + """A variant with a native Wi-Fi PHY needs no shim; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32C3, {}, config) is config + + +def test_radioless_non_p4_variant_rejected(monkeypatch) -> None: + """Radio-less variants without any ESP-NOW path are rejected outright.""" + with pytest.raises(cv.Invalid, match="not supported"): + _run(monkeypatch, VARIANT_ESP32H2, {}, {}) + + +def test_p4_without_esp32_hosted_rejected(monkeypatch) -> None: + """The P4 needs the esp32_hosted shim to supply the esp_now_* symbols.""" + with pytest.raises(cv.Invalid, match="esp32_hosted"): + _run(monkeypatch, VARIANT_ESP32P4, {}, {}) + + +def test_p4_with_esp32_hosted_passes(monkeypatch) -> None: + """The P4 with esp32_hosted present validates; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32P4, {"esp32_hosted": {}}, config) is config From 9ba4477ada0f207b7426fe83b37fde69c9ed9947 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:46:22 +1200 Subject: [PATCH 045/178] Bump version to 2026.9.0b2 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 7b2d21027a..060de51d3a 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0b1 +PROJECT_NUMBER = 2026.9.0b2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 378da14197..287804ace3 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0b1" +__version__ = "2026.9.0b2" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 5e37872da24f4626dc7ea8bdd61f4c5534f6c0ee Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:32:18 +1200 Subject: [PATCH 046/178] [ci] Sync pre-commit revs and prek version from requirements files (#19026) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 13 +- .../workflows/sync-dependency-versions.yml | 94 ++++++++ .pre-commit-config.yaml | 5 +- AGENTS.md | 2 +- requirements_dev.txt | 4 +- requirements_test.txt | 9 +- script/sync_dependency_versions.py | 164 +++++++++++++ tests/script/test_sync_dependency_versions.py | 219 ++++++++++++++++++ 8 files changed, 498 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/sync-dependency-versions.yml create mode 100755 script/sync_dependency_versions.py create mode 100644 tests/script/test_sync_dependency_versions.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7c93b3b86..173d2c227a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,11 +244,20 @@ jobs: steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Read prek version from requirements_test.txt + id: prek + # requirements_test.txt is the only place the version is pinned, so a + # Dependabot bump there is picked up here without a second edit. + run: | + if ! version=$(sed -nE 's/^prek==([^[:space:]#]+).*/\1/p' requirements_test.txt) || [ -z "$version" ]; then + echo "::error::No prek== pin found in requirements_test.txt." + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" - name: Run prek uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 with: - # Keep in sync with requirements_test.txt. - prek-version: "0.4.11" + prek-version: ${{ steps.prek.outputs.version }} # This job only runs on pull requests, so nothing ever populates # the cache on dev. Every run would miss and then write a per-pull # request copy, which is what the old seed-cache job existed to diff --git a/.github/workflows/sync-dependency-versions.yml b/.github/workflows/sync-dependency-versions.yml new file mode 100644 index 0000000000..5599691ed1 --- /dev/null +++ b/.github/workflows/sync-dependency-versions.yml @@ -0,0 +1,94 @@ +# Keeps pre-commit hook revs in sync with the requirements files. +# +# Dependabot only bumps the pins in requirements*.txt. Some of those tools +# are pinned again as hook revs in .pre-commit-config.yaml. This workflow +# runs script/sync_dependency_versions.py against the pull request branch +# and pushes a commit with the revs updated. + +name: Sync dependency versions + +on: + # pull_request_target rather than pull_request so the App secret is + # available on Dependabot pull requests (pull_request runs opened by + # Dependabot only see Dependabot secrets). The job below only touches + # branches in this repository and only ever executes the script from the + # base branch checkout, so fork code never runs with the token. + pull_request_target: + types: [opened, synchronize, reopened] + paths: + - requirements_dev.txt + - requirements_test.txt + - .pre-commit-config.yaml + - script/sync_dependency_versions.py + +# The push to the pull request branch uses the App token minted below, so +# the workflow's GITHUB_TOKEN does not need any scopes. +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + sync: + name: Sync pinned versions + runs-on: ubuntu-latest + # Same-repository branches only: a push to a fork is not possible with + # this token, and it keeps untrusted heads out of a privileged job. + if: >- + github.repository == 'esphome/esphome' + && github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} + # A push made with the workflow's own GITHUB_TOKEN would not start + # CI on the new commit; a push with the App token does. + permission-contents: write # git push of the sync commit to the pull request branch + + - name: Check out base branch + # Provides the script that runs below. Deliberately the base branch + # so the pull request cannot change what executes here. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + + - name: Check out pull request branch + # No allow-unsafe-pr-checkout here on purpose: checkout v7 only + # refuses heads that live in a different repository, and the job + # condition above already limits runs to same-repository branches. + # Leaving it off keeps that refusal as a backstop for fork heads. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.ref }} + path: pull-request + token: ${{ steps.generate-token.outputs.token }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install yamlrocks + # The script edits YAML through yamlrocks. Take the pin from the + # base branch requirements so this workflow has no copy of its own. + run: pip install "$(grep -E '^yamlrocks==' requirements_test.txt | cut -d'#' -f1)" + + - name: Sync pinned versions + run: python script/sync_dependency_versions.py --root pull-request + + - name: Push changes + working-directory: pull-request + run: | + if git diff --quiet; then + echo "All pinned versions already match the requirements files." + exit 0 + fi + git config user.name "esphome[bot]" + git config user.email "115708604+esphome[bot]@users.noreply.github.com" + git commit -am "Sync pinned tool versions with requirements files" + git push diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ea799aa4d..1af0e19273 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ --- # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks - ci: autoupdate_commit_msg: 'pre-commit: autoupdate' autoupdate_schedule: off # Disabled until ruff versions are synced between deps and pre-commit @@ -11,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.3 + rev: v0.16.5 hooks: # Run the linter. - id: ruff @@ -42,7 +41,7 @@ repos: - id: pyupgrade args: [--py312-plus] - repo: https://github.com/adrienverge/yamllint.git - rev: v1.37.1 + rev: v1.38.0 hooks: - id: yamllint exclude: ^(\.clang-format|\.clang-tidy)$ diff --git a/AGENTS.md b/AGENTS.md index 15b92c4deb..98bdd58ec5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -840,7 +840,7 @@ file does, and it is the authority when they disagree. The most useful starting cv.rename_key( CONF_OLD_KEY, CONF_NEW_KEY, removed_in="2026.6.0", component="my_component" ), - cv.Schema({ ... }), + cv.Schema({...}), ) ``` For other deprecations, warn manually during validation: diff --git a/requirements_dev.txt b/requirements_dev.txt index f2cf855d6b..ee94a2401a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ # Useful stuff when working in a development environment -clang-format==13.0.1 # also change in .pre-commit-config.yaml and Dockerfile when updating +clang-format==13.0.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py clang-tidy==22.1.8 -yamllint==1.38.0 # also change in .pre-commit-config.yaml when updating +yamllint==1.38.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py diff --git a/requirements_test.txt b/requirements_test.txt index 897445a4cb..ef70a5ac0c 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,8 +1,9 @@ pylint==4.0.8 -flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.16.5 # also change in .pre-commit-config.yaml when updating -pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.5.1 # also change in .github/workflows/ci.yml when updating +flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.5 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +prek==0.5.1 # .github/workflows/ci.yml reads this pin +yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests pytest==9.1.1 diff --git a/script/sync_dependency_versions.py b/script/sync_dependency_versions.py new file mode 100755 index 0000000000..a97a58b3b0 --- /dev/null +++ b/script/sync_dependency_versions.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Keep pre-commit hook revs in sync with the requirements files. + +Dependabot only bumps the ``package==version`` pins in ``requirements*.txt``. +Some of those tools are pinned a second time as hook ``rev`` values in +``.pre-commit-config.yaml``. This script treats the requirements files as +the source of truth and rewrites the revs to match, editing the config +through yamlrocks so comments and layout survive. + +Run without arguments to apply the changes in place, or with ``--check`` to +only report drift (exit status 1 when anything is out of sync). +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path +import re +import sys +from typing import Any + +import yamlrocks + +REPO_ROOT = Path(__file__).resolve().parent.parent +PRECOMMIT_CONFIG = ".pre-commit-config.yaml" + + +class SyncError(Exception): + """A pin could not be located in a requirements file or the config.""" + + +@dataclass(frozen=True) +class SyncTarget: + """A requirements pin and the pre-commit repo whose rev mirrors it.""" + + package: str + requirements_file: str + repo: str + + +SYNC_TARGETS: tuple[SyncTarget, ...] = ( + SyncTarget( + "ruff", "requirements_test.txt", "https://github.com/astral-sh/ruff-pre-commit" + ), + SyncTarget("flake8", "requirements_test.txt", "https://github.com/PyCQA/flake8"), + SyncTarget( + "pyupgrade", "requirements_test.txt", "https://github.com/asottile/pyupgrade" + ), + SyncTarget( + "clang-format", + "requirements_dev.txt", + "https://github.com/pre-commit/mirrors-clang-format", + ), + SyncTarget( + "yamllint", + "requirements_dev.txt", + "https://github.com/adrienverge/yamllint.git", + ), +) + + +def read_requirement_version(requirements: str, package: str) -> str | None: + """Return the ``==`` pin for ``package`` or None when it is not pinned.""" + pattern = re.compile( + rf"^{re.escape(package)}==(?P[^\s#]+)", + re.MULTILINE | re.IGNORECASE, + ) + match = pattern.search(requirements) + return match.group("version") if match else None + + +def find_repo_entry(doc: Any, repo: str) -> Any: + """Return the single ``- repo:`` block for ``repo`` in a pre-commit doc.""" + try: + entries = [entry for entry in doc["repos"] if entry["repo"] == repo] + except KeyError as err: + raise SyncError(f"malformed pre-commit config, missing key {err}") from None + if len(entries) != 1: + raise SyncError( + f"expected exactly one block for repo {repo}, found {len(entries)}" + ) + return entries[0] + + +def current_rev(entry: Any, repo: str) -> tuple[str, str]: + """Split the block's rev into its tag prefix (``v`` or empty) and version.""" + if "rev" not in entry: + raise SyncError(f"repo {repo} has no rev") + rev = entry["rev"] + if not isinstance(rev, str): + # A rev such as ``1.0`` parses as a number and cannot be compared or + # rewritten safely; quote it in the config instead. + raise SyncError(f"rev of repo {repo} is not a string: {rev!r}") + prefix = "v" if rev.startswith("v") else "" + return prefix, rev.removeprefix("v") + + +def sync(root: Path, *, write: bool) -> list[str]: + """Bring every hook rev in line with its requirements pin. + + Returns one description per rev that was (or, when ``write`` is False, + would be) changed. Raises SyncError when a pin cannot be found, which + means SYNC_TARGETS has gone stale and needs updating by hand. + """ + config_path = root / PRECOMMIT_CONFIG + doc = yamlrocks.loads(config_path.read_bytes(), option=yamlrocks.OPT_ROUND_TRIP) + requirements: dict[str, str] = {} + changes: list[str] = [] + for target in SYNC_TARGETS: + if target.requirements_file not in requirements: + requirements[target.requirements_file] = ( + root / target.requirements_file + ).read_text() + version = read_requirement_version( + requirements[target.requirements_file], target.package + ) + if version is None: + raise SyncError( + f"{target.requirements_file}: no '{target.package}==' pin found" + ) + + entry = find_repo_entry(doc, target.repo) + prefix, current = current_rev(entry, target.repo) + if current == version: + continue + changes.append(f"{target.package}: {current} -> {version}") + entry["rev"] = f"{prefix}{version}" + + if changes and write: + config_path.write_bytes(doc.to_yaml()) + return changes + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--check", + action="store_true", + help="report drift without modifying any file; exit 1 if out of sync", + ) + parser.add_argument( + "--root", + type=Path, + default=REPO_ROOT, + help="repository checkout to operate on (default: this checkout)", + ) + args = parser.parse_args(argv) + + try: + changes = sync(args.root, write=not args.check) + except SyncError as err: + print(f"error: {err}", file=sys.stderr) + return 1 + + for change in changes: + print(change) + if args.check and changes: + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/tests/script/test_sync_dependency_versions.py b/tests/script/test_sync_dependency_versions.py new file mode 100644 index 0000000000..787c8112d9 --- /dev/null +++ b/tests/script/test_sync_dependency_versions.py @@ -0,0 +1,219 @@ +"""Unit tests for script/sync_dependency_versions.py.""" + +from pathlib import Path +import subprocess +import sys + +import pytest +import yamlrocks + +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) + +import sync_dependency_versions as sync_mod # noqa: E402 + +PRECOMMIT = """\ +# See https://pre-commit.com for more information +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.1.0 + hooks: + - id: ruff + - repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + - repo: https://github.com/asottile/pyupgrade + rev: v3.0.0 + hooks: + - id: pyupgrade + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v13.0.1 + hooks: + - id: clang-format + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.0.0 + hooks: + - id: yamllint + - repo: local + hooks: + - id: pylint +""" + +REQ_TEST = """\ +pylint==4.0.8 +flake8==7.1.0 +ruff==0.2.0 # comment +pyupgrade==3.0.0 +""" + +REQ_DEV = """\ +clang-format==13.0.1 +yamllint==1.0.0 +""" + +RUFF_REPO = "https://github.com/astral-sh/ruff-pre-commit" +DUPLICATE_RUFF_BLOCK = f" - repo: {RUFF_REPO}\n rev: v0.3.0\n hooks: []\n" + +EXPECTED_DRIFT = ["ruff: 0.1.0 -> 0.2.0", "flake8: 7.0.0 -> 7.1.0"] +EXPECTED_PRECOMMIT = PRECOMMIT.replace("rev: v0.1.0", "rev: v0.2.0").replace( + "rev: 7.0.0", "rev: 7.1.0" +) + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + """A fake checkout where ruff (v-prefixed) and flake8 (bare) have drifted.""" + (tmp_path / ".pre-commit-config.yaml").write_text(PRECOMMIT) + (tmp_path / "requirements_test.txt").write_text(REQ_TEST) + (tmp_path / "requirements_dev.txt").write_text(REQ_DEV) + return tmp_path + + +def _load(text: str) -> object: + return yamlrocks.loads(text.encode(), option=yamlrocks.OPT_ROUND_TRIP) + + +@pytest.mark.parametrize( + ("requirements", "expected"), + [ + ("prek==0.5.1 # comment\n", "0.5.1"), + ("Prek==0.5.1\n", "0.5.1"), + ("other==1.0\nprek==0.5.1\n", "0.5.1"), + ("prek>=0.5.1\n", None), + ("prek-extra==0.5.1\n", None), + ("", None), + ], +) +def test_read_requirement_version(requirements: str, expected: str | None) -> None: + assert sync_mod.read_requirement_version(requirements, "prek") == expected + + +def test_find_repo_entry() -> None: + entry = sync_mod.find_repo_entry(_load(PRECOMMIT), RUFF_REPO) + assert entry["rev"] == "v0.1.0" + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ("hooks: []\n", "missing key 'repos'"), + ("repos:\n - rev: 1.0.0\n", "missing key 'repo'"), + (PRECOMMIT + DUPLICATE_RUFF_BLOCK, "found 2"), + ("repos:\n - repo: other\n rev: 1.0.0\n", "found 0"), + ], +) +def test_find_repo_entry_errors(text: str, message: str) -> None: + with pytest.raises(sync_mod.SyncError, match=message): + sync_mod.find_repo_entry(_load(text), RUFF_REPO) + + +@pytest.mark.parametrize( + ("rev", "expected"), + [("v0.1.0", ("v", "0.1.0")), ("7.0.0", ("", "7.0.0")), ("'1.0'", ("", "1.0"))], +) +def test_current_rev(rev: str, expected: tuple[str, str]) -> None: + doc = _load(f"repos:\n - repo: {RUFF_REPO}\n rev: {rev}\n") + assert sync_mod.current_rev(doc["repos"][0], RUFF_REPO) == expected + + +@pytest.mark.parametrize( + ("block", "message"), + [(" hooks: []\n", "has no rev"), (" rev: 1.0\n", "not a string: 1.0")], +) +def test_current_rev_errors(block: str, message: str) -> None: + doc = _load(f"repos:\n - repo: {RUFF_REPO}\n{block}") + with pytest.raises(sync_mod.SyncError, match=message): + sync_mod.current_rev(doc["repos"][0], RUFF_REPO) + + +def test_sync_reports_without_writing(root: Path) -> None: + assert sync_mod.sync(root, write=False) == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == PRECOMMIT + + +def test_sync_writes_keeps_layout_and_is_idempotent(root: Path) -> None: + assert sync_mod.sync(root, write=True) == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == EXPECTED_PRECOMMIT + assert sync_mod.sync(root, write=True) == [] + + +def test_sync_does_not_touch_a_config_that_matches(root: Path) -> None: + (root / ".pre-commit-config.yaml").write_text(EXPECTED_PRECOMMIT) + before = (root / ".pre-commit-config.yaml").stat().st_mtime_ns + assert sync_mod.sync(root, write=True) == [] + assert (root / ".pre-commit-config.yaml").stat().st_mtime_ns == before + + +def test_sync_missing_requirement_pin(root: Path) -> None: + (root / "requirements_dev.txt").write_text("") + with pytest.raises(sync_mod.SyncError, match="no 'clang-format==' pin"): + sync_mod.sync(root, write=True) + + +def test_sync_propagates_config_errors(root: Path) -> None: + (root / ".pre-commit-config.yaml").write_text(PRECOMMIT + DUPLICATE_RUFF_BLOCK) + with pytest.raises(sync_mod.SyncError, match="found 2"): + sync_mod.sync(root, write=True) + + +def test_main_check_reports_drift( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert sync_mod.main(["--check", "--root", str(root)]) == 1 + assert capsys.readouterr().out.splitlines() == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == PRECOMMIT + + +def test_main_writes_then_check_is_clean( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert sync_mod.main(["--root", str(root)]) == 0 + assert capsys.readouterr().out.splitlines() == EXPECTED_DRIFT + assert sync_mod.main(["--check", "--root", str(root)]) == 0 + assert capsys.readouterr().out == "" + + +def test_main_reports_sync_error( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (root / "requirements_dev.txt").write_text("") + assert sync_mod.main(["--root", str(root)]) == 1 + assert ( + "error: requirements_dev.txt: no 'clang-format==' pin" + in capsys.readouterr().err + ) + + +def test_main_defaults_to_repo_root(monkeypatch: pytest.MonkeyPatch) -> None: + seen: dict[str, object] = {} + + def fake_sync(root: Path, *, write: bool) -> list[str]: + seen["root"] = root + seen["write"] = write + return [] + + monkeypatch.setattr(sync_mod, "sync", fake_sync) + assert sync_mod.main([]) == 0 + assert seen == {"root": sync_mod.REPO_ROOT, "write": True} + + +def test_repository_is_in_sync() -> None: + """The real checkout must match; a failure here means a rev has drifted. + + Also proves every SYNC_TARGETS entry still resolves in the real files. + """ + assert sync_mod.sync(sync_mod.REPO_ROOT, write=False) == [] + + +def test_cli_entry_point(root: Path) -> None: + """Run the script the way the workflow does, as a subprocess.""" + script = Path(sync_mod.__file__) + result = subprocess.run( + [sys.executable, str(script), "--check", "--root", str(root)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 1 + assert result.stdout.splitlines() == EXPECTED_DRIFT From 390742cf9ba113ea89bc88a05582c4c409a1bd63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 23:47:46 +0000 Subject: [PATCH 047/178] Bump ruff from 0.16.5 to 0.16.6 (#19022) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- esphome/api_client.py | 4 +--- esphome/components/debug/sensor.py | 6 +----- esphome/components/debug/text_sensor.py | 6 +----- esphome/components/esp32/const.py | 11 ++--------- esphome/components/nextion/display.py | 7 +------ esphome/happy_eyeballs.py | 5 +---- requirements_test.txt | 2 +- 8 files changed, 9 insertions(+), 34 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1af0e19273..95e6f0f73e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.5 + rev: v0.16.6 hooks: # Run the linter. - id: ruff diff --git a/esphome/api_client.py b/esphome/api_client.py index fb41075de8..2b93b4790f 100644 --- a/esphome/api_client.py +++ b/esphome/api_client.py @@ -23,9 +23,7 @@ from esphome.util import safe_print if TYPE_CHECKING: from collections.abc import Callable - from aioesphomeapi.api_pb2 import ( - SubscribeLogsResponse, # pylint: disable=no-name-in-module - ) + from aioesphomeapi.api_pb2 import SubscribeLogsResponse # pylint: disable=no-name-in-module _LOGGER = logging.getLogger(__name__) diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index e53cb0d1e4..80d1daa81f 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -23,11 +23,7 @@ from esphome.const import ( ) from esphome.types import ConfigType -from . import ( # noqa: F401 pylint: disable=unused-import - CONF_DEBUG_ID, - FILTER_SOURCE_FILES, - DebugComponent, -) +from . import CONF_DEBUG_ID, FILTER_SOURCE_FILES, DebugComponent # noqa: F401 pylint: disable=unused-import DEPENDENCIES = ["debug"] diff --git a/esphome/components/debug/text_sensor.py b/esphome/components/debug/text_sensor.py index 9d4fcc1b42..2e02af67cb 100644 --- a/esphome/components/debug/text_sensor.py +++ b/esphome/components/debug/text_sensor.py @@ -9,11 +9,7 @@ from esphome.const import ( ) from esphome.types import ConfigType -from . import ( # noqa: F401 pylint: disable=unused-import - CONF_DEBUG_ID, - FILTER_SOURCE_FILES, - DebugComponent, -) +from . import CONF_DEBUG_ID, FILTER_SOURCE_FILES, DebugComponent # noqa: F401 pylint: disable=unused-import DEPENDENCIES = ["debug"] diff --git a/esphome/components/esp32/const.py b/esphome/components/esp32/const.py index e7d8a66e7a..a0c9809c50 100644 --- a/esphome/components/esp32/const.py +++ b/esphome/components/esp32/const.py @@ -3,18 +3,11 @@ import esphome.codegen as cg # Re-exported for the many esp32-side users; defined in esphome.const # and esphome.espidf so the upload/logs fast path can use them without # importing this package. -from esphome.const import ( # noqa: F401 # pylint: disable=unused-import - KEY_ESP32, - KEY_FLASH_SIZE, - KEY_IDF_VERSION, - KEY_VARIANT, -) +from esphome.const import KEY_ESP32, KEY_FLASH_SIZE, KEY_IDF_VERSION, KEY_VARIANT # noqa: F401 # pylint: disable=unused-import # Back compat for external components only; in-tree callers import it # from esphome.espidf directly. -from esphome.espidf import ( # noqa: F401 # pylint: disable=unused-import - variant_to_idf_target, -) +from esphome.espidf import variant_to_idf_target # noqa: F401 # pylint: disable=unused-import KEY_BOARD = "board" KEY_SDKCONFIG_OPTIONS = "sdkconfig_options" diff --git a/esphome/components/nextion/display.py b/esphome/components/nextion/display.py index 3f5ba94b40..a5894bdaf7 100644 --- a/esphome/components/nextion/display.py +++ b/esphome/components/nextion/display.py @@ -14,12 +14,7 @@ from esphome.const import ( ) from esphome.core import CORE, TimePeriod -from . import ( # noqa: F401 pylint: disable=unused-import - FILTER_SOURCE_FILES, - Nextion, - nextion_ns, - nextion_ref, -) +from . import FILTER_SOURCE_FILES, Nextion, nextion_ns, nextion_ref # noqa: F401 pylint: disable=unused-import from .base_component import ( CONF_AUTO_WAKE_ON_TOUCH, CONF_COMMAND_SPACING, diff --git a/esphome/happy_eyeballs.py b/esphome/happy_eyeballs.py index 35092e7daa..8b0d020862 100644 --- a/esphome/happy_eyeballs.py +++ b/esphome/happy_eyeballs.py @@ -69,10 +69,7 @@ def _make_create_connection() -> Callable[..., socket.socket]: from aiohappyeyeballs import start_connection from urllib3.exceptions import LocationParseError - from urllib3.util.connection import ( # noqa: PLC2701 - _set_socket_options, - allowed_gai_family, - ) + from urllib3.util.connection import _set_socket_options, allowed_gai_family # noqa: PLC2701 from urllib3.util.timeout import _DEFAULT_TIMEOUT # noqa: PLC2701 from esphome import async_thread diff --git a/requirements_test.txt b/requirements_test.txt index ef70a5ac0c..9fd82b7509 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.8 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -ruff==0.16.5 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.6 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py prek==0.5.1 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py From 89a56298c231a138080a8604deea8ebb5a630369 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:00:28 +0000 Subject: [PATCH 048/178] Bump prek from 0.5.1 to 0.5.2 (#19021) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 9fd82b7509..cd0427f33e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.8 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.6 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -prek==0.5.1 # .github/workflows/ci.yml reads this pin +prek==0.5.2 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests From 50ca38119873fc717db2ec00ef9b614d5539921b Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:10:55 +0200 Subject: [PATCH 049/178] [i2s_audio] Keep a start request that arrives while the speaker task stops (#19027) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../components/i2s_audio/speaker/i2s_audio_speaker.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 5e271e671e..1c2eb12904 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -91,7 +91,14 @@ void I2SAudioSpeakerBase::loop() { this->speaker_task_handle_ = nullptr; this->stop_i2s_driver_(); - xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + // ALL_BITS includes COMMAND_START. Take the bits from the clear itself, not from the snapshot at + // the top of loop(): the audio source's task can raise a start at any point above, including + // during stop_i2s_driver_(), and nothing would ever re-issue it. + const EventBits_t bits_before_clear = xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + if (bits_before_clear & SpeakerEventGroupBits::COMMAND_START) { + ESP_LOGD(TAG, "Start requested while stopping; keeping the request"); + xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); + } this->status_clear_error(); this->on_task_stopped(); From 56c3361b9adafd3f1f433987d04f27f549a4d965 Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:13:03 +0200 Subject: [PATCH 050/178] [audio] Do not treat MP3_STREAM_INFO_CHANGED as a fatal decoder error (#19028) Co-authored-by: Claude --- esphome/components/audio/audio_decoder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index fe9ad9c9ad..051395606c 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -313,9 +313,10 @@ FileDecoderState AudioDecoder::decode_mp3_() { this->output_transfer_buffer_->increase_buffer_length( this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); } - } else if (result == micro_mp3::MP3_STREAM_INFO_READY) { - // First successful header parse: capture stream info and resize the output buffer to fit one full frame. - // microMP3 always outputs 16-bit PCM. + } else if (result == micro_mp3::MP3_STREAM_INFO_READY || result == micro_mp3::MP3_STREAM_INFO_CHANGED) { + // Header parsed: capture stream info and resize the output buffer to fit one full frame. + // microMP3 always outputs 16-bit PCM. MP3_STREAM_INFO_CHANGED is handled identically: despite its + // negative value it is documented as recoverable, so it must not reach the catch-all below. this->audio_stream_info_ = audio::AudioStreamInfo(16, this->mp3_decoder_->get_channels(), this->mp3_decoder_->get_sample_rate()); this->free_buffer_required_ = From 62eafc477d9ab731b1ce5b8d493d5b69c7e5b468 Mon Sep 17 00:00:00 2001 From: Ryan Ronnander <61520+ryan-ronnander@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:09:02 -0400 Subject: [PATCH 051/178] [mqtt] Restore brightness flag in light discovery (#18950) --- esphome/components/mqtt/mqtt_light.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/mqtt/mqtt_light.cpp b/esphome/components/mqtt/mqtt_light.cpp index aa47bdf996..a8b52a3839 100644 --- a/esphome/components/mqtt/mqtt_light.cpp +++ b/esphome/components/mqtt/mqtt_light.cpp @@ -67,6 +67,9 @@ void MQTTJSONLightComponent::send_discovery(JsonObject root, mqtt::SendDiscovery if (traits.supports_color_mode(ColorMode::RGB_COLD_WARM_WHITE)) color_modes.add(ESPHOME_F("rgbww")); + if (traits.supports_color_capability(ColorCapability::BRIGHTNESS)) + root[ESPHOME_F("brightness")] = true; + if (traits.supports_color_mode(ColorMode::COLOR_TEMPERATURE) || traits.supports_color_mode(ColorMode::COLD_WARM_WHITE)) { root[MQTT_MIN_MIREDS] = traits.get_min_mireds(); From 639ce609bf70332146f25d04cdf5ac6a15bb4ee2 Mon Sep 17 00:00:00 2001 From: AndreKR Date: Tue, 8 Sep 2026 03:13:51 +0200 Subject: [PATCH 052/178] [logger] Fix garbled stack traces (#17939) --- esphome/components/logger/logger_esp32.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index 05fc959ceb..c3d777299d 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -5,6 +5,7 @@ #include #include +#include #ifdef USE_LOGGER_UART_SELECTION_USB_SERIAL_JTAG #include @@ -76,7 +77,11 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { uart_config.parity = UART_PARITY_DISABLE; uart_config.stop_bits = UART_STOP_BITS_1; uart_config.flow_ctrl = UART_HW_FLOWCTRL_DISABLE; +#if SOC_UART_SUPPORT_XTAL_CLK + uart_config.source_clk = UART_SCLK_XTAL; +#else uart_config.source_clk = UART_SCLK_DEFAULT; +#endif uart_param_config(uart_num, &uart_config); // The logger only writes to UART, never reads, so use the minimum RX buffer. // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). From e6aa575f2e960f406cc8edaccb9719dc11f2a92b Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Mon, 7 Sep 2026 18:27:49 -0700 Subject: [PATCH 053/178] [dallas_temp] filter 85 temp from sensor reset (#17877) Co-authored-by: Samuel Sieb --- esphome/components/dallas_temp/dallas_temp.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esphome/components/dallas_temp/dallas_temp.cpp b/esphome/components/dallas_temp/dallas_temp.cpp index ab4a8c458f..c418362ced 100644 --- a/esphome/components/dallas_temp/dallas_temp.cpp +++ b/esphome/components/dallas_temp/dallas_temp.cpp @@ -6,6 +6,7 @@ namespace esphome::dallas_temp { static const char *const TAG = "dallas.temp.sensor"; static const uint8_t DALLAS_MODEL_DS18S20 = 0x10; +static const uint8_t DALLAS_MODEL_DS18B20 = 0x28; static const uint8_t DALLAS_COMMAND_START_CONVERSION = 0x44; static const uint8_t DALLAS_COMMAND_READ_SCRATCH_PAD = 0xBE; static const uint8_t DALLAS_COMMAND_WRITE_SCRATCH_PAD = 0x4E; @@ -154,7 +155,14 @@ float DallasTemperatureSensor::get_temp_c_() { default: break; } - + // undocumented test for powerup measurement of 85 + // https://github.com/cpetrich/counterfeit_DS18B20#solution-to-the-85-c-problem + if ((this->address_ & 0xff) == DALLAS_MODEL_DS18B20) { + if ((temp == 85 * 16) && (this->scratch_pad_[6] == 0xc)) { + ESP_LOGD(TAG, "dropping reading caused by sensor reset"); + return NAN; + } + } return temp / 16.0f; } From d34ffaf3928ef4a5fdaccc94f30b3ab59f6b75c4 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 7 Sep 2026 18:57:50 -0700 Subject: [PATCH 054/178] [ble_client] Report Established from nodes that never read services (#17920) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/ble_client/automation.h | 34 +++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/esphome/components/ble_client/automation.h b/esphome/components/ble_client/automation.h index 94eeb83b3e..93aae23b6a 100644 --- a/esphome/components/ble_client/automation.h +++ b/esphome/components/ble_client/automation.h @@ -22,6 +22,23 @@ class Automation { static const char *const TAG; }; +// Base for nodes that never read the parent's services. +// The parent releases its services only once every node reports Established, so a node that never +// reports it keeps that memory allocated for the life of the connection. +class BLEClientServicelessNode : public BLEClientNode { + public: + // Final so that Established is always reported on SEARCH_CMPL, before the derived node sees the event. + void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) final { + if (event == ESP_GATTC_SEARCH_CMPL_EVT) + this->node_state = espbt::ClientState::ESTABLISHED; + this->on_gattc_event(event, gattc_if, param); + } + + protected: + // Derived nodes handle GATT events here rather than by overriding the handler above. + virtual void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) {} +}; + // implement on_connect automation. class BLEClientConnectTrigger final : public Trigger<>, public BLEClientNode { public: @@ -61,7 +78,7 @@ class BLEClientDisconnectTrigger final : public Trigger<>, public BLEClientNode } }; -class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientNode { +class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientServicelessNode { public: explicit BLEClientPasskeyRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -71,7 +88,7 @@ class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientN } }; -class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientNode { +class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientPasskeyNotificationTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -82,7 +99,7 @@ class BLEClientPasskeyNotificationTrigger final : public Trigger, publ } }; -class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientNode { +class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientNumericComparisonRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -315,19 +332,17 @@ template class BLEClientRemoveBondAction final : public Action class BLEClientConnectAction final : public Action, public BLEClientNode { +template class BLEClientConnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientConnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { case ESP_GATTC_SEARCH_CMPL_EVT: - this->node_state = espbt::ClientState::ESTABLISHED; this->parent()->run_later([this]() { this->play_next_tuple_(this->var_); }); break; // if the connection is closed, terminate the automation chain. @@ -364,14 +379,13 @@ template class BLEClientConnectAction final : public Action var_{}; }; -template class BLEClientDisconnectAction final : public Action, public BLEClientNode { +template class BLEClientDisconnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientDisconnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { From 574762f07861b7008225bb7de89164ed697836da Mon Sep 17 00:00:00 2001 From: Davide D M Date: Tue, 8 Sep 2026 03:59:05 +0200 Subject: [PATCH 055/178] [debug] Check reboot source pref on ESP_RST_WDT and guard against empty source (#17537) --- esphome/components/debug/debug_esp32.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esphome/components/debug/debug_esp32.cpp b/esphome/components/debug/debug_esp32.cpp index 969cd840cf..8e1a67224e 100644 --- a/esphome/components/debug/debug_esp32.cpp +++ b/esphome/components/debug/debug_esp32.cpp @@ -66,11 +66,15 @@ const char *DebugComponent::get_reset_reason_(std::spanmake_preference(REBOOT_MAX_LEN, fnv1_hash_extend(fnv1_hash(REBOOT_KEY), App.get_name().c_str())); char reboot_source[REBOOT_MAX_LEN]{}; - if (pref.load(&reboot_source)) { + if (pref.load(&reboot_source) && reboot_source[0] != '\0') { reboot_source[REBOOT_MAX_LEN - 1] = '\0'; snprintf(buf, size, "Reboot request from %s", reboot_source); } else { From 94e5c3839d8372e95a320cd1796ca500de75be91 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:17:22 +1200 Subject: [PATCH 056/178] [udp] Use cv.invalid for relocated packet_transport options (#19032) --- esphome/components/udp/__init__.py | 16 +++------ tests/unit_tests/components/udp/__init__.py | 0 tests/unit_tests/components/udp/test_init.py | 37 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 tests/unit_tests/components/udp/__init__.py create mode 100644 tests/unit_tests/components/udp/test_init.py diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index a782d875b9..d96a731e9c 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -1,5 +1,4 @@ -from collections.abc import Callable -from typing import Any, NoReturn +from typing import Any from esphome import automation from esphome.automation import Trigger @@ -48,17 +47,10 @@ UDP_SCHEMA = cv.Schema( ) -def is_relocated(option: str) -> Callable[[Any], NoReturn]: - def validator(value: Any) -> NoReturn: - raise cv.Invalid( - f"The '{option}' option should now be configured in the 'packet_transport' component" - ) - - return validator - - RELOCATED = { - cv.Optional(x): is_relocated(x) + cv.Optional(x): cv.invalid( + f"The '{x}' option should now be configured in the 'packet_transport' component" + ) for x in ( CONF_PROVIDERS, CONF_ENCRYPTION, diff --git a/tests/unit_tests/components/udp/__init__.py b/tests/unit_tests/components/udp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/udp/test_init.py b/tests/unit_tests/components/udp/test_init.py new file mode 100644 index 0000000000..5afc92e9c6 --- /dev/null +++ b/tests/unit_tests/components/udp/test_init.py @@ -0,0 +1,37 @@ +"""Tests for the udp component configuration schema.""" + +from __future__ import annotations + +import pytest + +from esphome.components import udp +from esphome.components.packet_transport import ( + CONF_BINARY_SENSORS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_PROVIDERS, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, +) +import esphome.config_validation as cv + + +@pytest.mark.parametrize( + "option", + [ + CONF_PROVIDERS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, + CONF_BINARY_SENSORS, + ], +) +def test_relocated_option_rejected(option: str) -> None: + """Options that moved to packet_transport raise a pointing error.""" + with pytest.raises(cv.Invalid) as exc_info: + udp.CONFIG_SCHEMA({option: True}) + assert ( + f"The '{option}' option should now be configured in the 'packet_transport' component" + in str(exc_info.value) + ) From 5722ccba372857c175e75de217b2e455ca57f88d Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:36:04 -0400 Subject: [PATCH 057/178] [tuya] Build without a network component (#18948) --- esphome/components/tuya/tuya.cpp | 17 +++++++++-- .../tuya/test-no-network.bk72xx-ard.yaml | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/components/tuya/test-no-network.bk72xx-ard.yaml diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index 82fb96d787..f9b4fe2453 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -1,10 +1,13 @@ #include "tuya.h" -#include "esphome/components/network/util.h" #include "esphome/core/gpio.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/util.h" +#ifdef USE_NETWORK +#include "esphome/components/network/util.h" +#endif + #ifdef USE_WIFI #include "esphome/components/wifi/wifi_component.h" #endif @@ -22,6 +25,14 @@ static const int MAX_RETRIES = 5; // Max bytes to log for datapoint values (larger values are truncated) static constexpr size_t MAX_DATAPOINT_LOG_BYTES = 16; +static bool network_is_connected() { +#ifdef USE_NETWORK + return network::is_connected(); +#else + return false; +#endif +} + void Tuya::setup() { this->set_interval("heartbeat", 15000, [this] { this->send_empty_command_(TuyaCommandType::HEARTBEAT); }); if (this->status_pin_ != nullptr) { @@ -554,14 +565,14 @@ void Tuya::send_empty_command_(TuyaCommandType command) { } void Tuya::set_status_pin_() { - bool is_network_ready = network::is_connected() && remote_is_connected(); + bool is_network_ready = network_is_connected() && remote_is_connected(); this->status_pin_->digital_write(is_network_ready); } uint8_t Tuya::get_wifi_status_code_() { uint8_t status = 0x02; - if (network::is_connected()) { + if (network_is_connected()) { status = 0x03; // Protocol version 3 also supports specifying when connected to "the cloud" diff --git a/tests/components/tuya/test-no-network.bk72xx-ard.yaml b/tests/components/tuya/test-no-network.bk72xx-ard.yaml new file mode 100644 index 0000000000..64207e94e3 --- /dev/null +++ b/tests/components/tuya/test-no-network.bk72xx-ard.yaml @@ -0,0 +1,29 @@ +# Tuya without any network component (no wifi/ethernet/api), as used on +# serial-only or BLE-only Tuya MCU boards. Regression test for +# https://github.com/esphome/esphome/issues/18942 +substitutions: + status_pin: P6 + +packages: + uart: !include ../../test_build_components/common/uart/bk72xx-ard.yaml + +tuya: + status_pin: ${status_pin} + +binary_sensor: + - platform: tuya + id: tuya_presence + sensor_datapoint: 101 + +sensor: + - platform: tuya + id: tuya_light_intensity + sensor_datapoint: 103 + +number: + - platform: tuya + id: tuya_far_detection + number_datapoint: 109 + min_value: 0 + max_value: 600 + step: 1 From 53075e41391a706a52d69885f70057cc9616c675 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 06:42:30 +0200 Subject: [PATCH 058/178] [core] Skip PlatformIO's private-package authorization probe (#18823) --- esphome/platformio/library.py | 6 ++- esphome/platformio/prefetch.py | 2 + esphome/platformio/runner.py | 14 ++++++- tests/unit_tests/test_platformio_library.py | 19 ++++++++++ tests/unit_tests/test_platformio_prefetch.py | 14 +++++++ tests/unit_tests/test_platformio_runner.py | 40 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 3ff60f8aaa..fb6779b807 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -616,11 +616,15 @@ def _make_registry_client() -> Any: elsewhere, not by the PlatformIO registry. """ from platformio.package.manager._registry import PackageManagerRegistryMixin + from platformio.registry.client import RegistryClient class _Registry(PackageManagerRegistryMixin): def __init__(self) -> None: - self._registry_client = None self.pkg_type = "library" + self._registry_client = RegistryClient() + # The probe sleeps ~500 ms per lookup (see runner.patch_registry_private_packages); + # instance-level so the ESPHome process never patches PlatformIO's class + self._registry_client.allowed_private_packages = lambda: False @staticmethod def is_system_compatible(value: Any, custom_system: Any = None) -> bool: diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 17a06cb9c1..e648192b73 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -951,8 +951,10 @@ def main(argv: list[str]) -> int: """Subprocess entry point: ``prefetch ``.""" from esphome.core import CORE from esphome.log import setup_log + from esphome.platformio.runner import patch_registry_private_packages signal.signal(signal.SIGTERM, _sigterm) + patch_registry_private_packages() raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") try: level = int(raw_level) if raw_level is not None else logging.INFO diff --git a/esphome/platformio/runner.py b/esphome/platformio/runner.py index 9bb2205a90..b9fbdec38d 100644 --- a/esphome/platformio/runner.py +++ b/esphome/platformio/runner.py @@ -2,7 +2,8 @@ Invoked via ``python -m esphome.platformio.runner`` instead of ``python -m platformio`` so that the patches (incremental rebuild -preservation, download retries) apply inside the subprocess. Running +preservation, download retries, skipping the private-package probe) apply +inside the subprocess. Running PlatformIO in a subprocess keeps its ``sys.path`` mutations and other global state from leaking into the ESPHome process. """ @@ -105,6 +106,16 @@ def patch_file_downloader() -> None: FileDownloader.__init__ = patched_init +def patch_registry_private_packages() -> None: + """Skip PlatformIO's private-package probe; it sleeps ~500 ms per lookup. + + ESPHome never uses private packages, so the answer is always False. + """ + from platformio.registry.client import RegistryClient + + RegistryClient.allowed_private_packages = staticmethod(lambda: False) # type: ignore[method-assign] + + _IGNORE_LIB_WARNINGS = "(?:Hash|Update)" # Regex patterns matched against each line of PlatformIO output. Lines that # match are dropped by RedirectText before they reach the parent process. @@ -152,6 +163,7 @@ FILTER_PLATFORMIO_LINES = [ def main() -> int: patch_structhash() patch_file_downloader() + patch_registry_private_packages() # Wrap stdout/stderr with RedirectText before PlatformIO runs: # diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 3bae39b3c1..512c883c37 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -7,6 +7,7 @@ exercised in their own test modules).""" import json import logging from pathlib import Path +from unittest.mock import Mock import pytest @@ -228,6 +229,24 @@ def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): _resolve_registry_version("owner", "pkg", set()) +def test_make_registry_client_skips_private_package_probe(monkeypatch): + """Our client answers the probe locally without patching PlatformIO's class.""" + from platformio.account.client import AccountClient + from platformio.registry.client import RegistryClient + + pio_probe = RegistryClient.__dict__["allowed_private_packages"] + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + client = lib._make_registry_client().get_registry_client_instance() + + assert client.allowed_private_packages() is False + assert RegistryClient.__dict__["allowed_private_packages"] is pio_probe + + def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: """Stub the registry lookup so tests never touch the network.""" monkeypatch.setattr( diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 77490fd861..14c52dda8d 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1225,6 +1225,20 @@ def test_main_runs_prefetch(tmp_path: Path) -> None: mock_prefetch.assert_called_once_with(tmp_path, "testenv") +def test_main_skips_private_package_probe_before_prefetch(tmp_path: Path) -> None: + """The registry probe patch is applied before any package manager runs.""" + order: list[str] = [] + with ( + patch.object(pf, "_prefetch", side_effect=lambda *_: order.append("prefetch")), + patch( + "esphome.platformio.runner.patch_registry_private_packages", + side_effect=lambda: order.append("patch"), + ), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + assert order == ["patch", "prefetch"] + + def test_main_bad_argv_is_a_distinct_exit( caplog: pytest.LogCaptureFixture, ) -> None: diff --git a/tests/unit_tests/test_platformio_runner.py b/tests/unit_tests/test_platformio_runner.py index f375aa457a..007455f45a 100644 --- a/tests/unit_tests/test_platformio_runner.py +++ b/tests/unit_tests/test_platformio_runner.py @@ -6,7 +6,9 @@ from collections.abc import Callable import io import sys from types import ModuleType +from unittest.mock import Mock +from platformio.registry.client import RegistryClient import pytest from esphome.platformio import runner @@ -30,6 +32,7 @@ def _prepare_main( monkeypatch.setattr(sys, "stderr", stream) monkeypatch.setattr(runner, "patch_structhash", lambda: None) monkeypatch.setattr(runner, "patch_file_downloader", lambda: None) + monkeypatch.setattr(runner, "patch_registry_private_packages", lambda: None) platformio = ModuleType("platformio") platformio_main = ModuleType("platformio.__main__") @@ -91,3 +94,40 @@ def test_main_still_filters_a_drained_partial_line( assert runner.main() == 0 assert buf.getvalue() == b"" + + +def test_main_applies_registry_private_packages_patch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The probe is patched before PlatformIO runs.""" + order: list[str] = [] + _prepare_main(monkeypatch, lambda: order.append("pio") or 0) + monkeypatch.setattr( + runner, "patch_registry_private_packages", lambda: order.append("patch") + ) + + assert runner.main() == 0 + assert order == ["patch", "pio"] + + +# Snapshot PlatformIO's own probe at import, before any test can patch it +_PIO_PROBE = RegistryClient.__dict__["allowed_private_packages"] + + +def test_patch_registry_private_packages_skips_account_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Answers False without touching the account client.""" + from platformio.account.client import AccountClient + + monkeypatch.setattr(RegistryClient, "allowed_private_packages", _PIO_PROBE) + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + runner.patch_registry_private_packages() + + assert RegistryClient.allowed_private_packages() is False + assert RegistryClient().allowed_private_packages() is False From a23f7bb5693a3ef0ec23e0bcc49f6e30561ae986 Mon Sep 17 00:00:00 2001 From: Gytis Date: Tue, 8 Sep 2026 08:31:30 +0200 Subject: [PATCH 059/178] [lvgl] Add missing label dependency to qrcode, keyboard and tabview (#18387) --- esphome/components/lvgl/widgets/keyboard.py | 3 +- esphome/components/lvgl/widgets/qrcode.py | 3 +- esphome/components/lvgl/widgets/tabview.py | 3 +- .../lvgl/config/keyboard_no_label.yaml | 32 +++++++++++++++++ .../lvgl/config/qrcode_no_label.yaml | 34 ++++++++++++++++++ .../lvgl/config/tabview_no_label.yaml | 35 +++++++++++++++++++ .../lvgl/test_widget_label_dependency.py | 32 +++++++++++++++++ 7 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/config/keyboard_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/qrcode_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/tabview_no_label.yaml create mode 100644 tests/component_tests/lvgl/test_widget_label_dependency.py diff --git a/esphome/components/lvgl/widgets/keyboard.py b/esphome/components/lvgl/widgets/keyboard.py index bcd2d2ae59..65516513a6 100644 --- a/esphome/components/lvgl/widgets/keyboard.py +++ b/esphome/components/lvgl/widgets/keyboard.py @@ -15,6 +15,7 @@ from ..defines import ( from ..types import LvCompound, LvType from . import Widget, WidgetType, get_widgets from .buttonmatrix import CONF_BUTTONMATRIX +from .label import CONF_LABEL from .textarea import CONF_TEXTAREA, lv_textarea_t CONF_KEYBOARD = "keyboard" @@ -49,7 +50,7 @@ class KeyboardType(WidgetType): ) def get_uses(self): - return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX + return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX, CONF_LABEL async def to_code(self, w: Widget, config: dict): add_lv_use("KEY_LISTENER") diff --git a/esphome/components/lvgl/widgets/qrcode.py b/esphome/components/lvgl/widgets/qrcode.py index df76ab6bb0..59af9168aa 100644 --- a/esphome/components/lvgl/widgets/qrcode.py +++ b/esphome/components/lvgl/widgets/qrcode.py @@ -10,6 +10,7 @@ from ..types import lv_obj_t from . import Widget, WidgetType from .canvas import CONF_CANVAS from .img import CONF_IMAGE +from .label import CONF_LABEL CONF_QRCODE = "qrcode" CONF_DARK_COLOR = "dark_color" @@ -41,7 +42,7 @@ class QrCodeType(WidgetType): ) def get_uses(self): - return CONF_CANVAS, CONF_IMAGE + return CONF_CANVAS, CONF_IMAGE, CONF_LABEL async def to_code(self, w: Widget, config): await w.set_property( diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index ee252ecf0b..77c88c48ff 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -28,6 +28,7 @@ from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t, lv_obj_t_ptr from . import Widget, WidgetType, add_widgets, get_widgets, set_obj_properties from .button import button_spec from .buttonmatrix import CONF_BUTTONMATRIX, buttonmatrix_spec +from .label import CONF_LABEL from .obj import obj_spec CONF_TABVIEW = "tabview" @@ -74,7 +75,7 @@ class TabviewType(WidgetType): ) def get_uses(self): - return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON + return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON, CONF_LABEL async def to_code(self, w: Widget, config: dict): await w.set_property( diff --git a/tests/component_tests/lvgl/config/keyboard_no_label.yaml b/tests/component_tests/lvgl/config/keyboard_no_label.yaml new file mode 100644 index 0000000000..7a45a537d3 --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_no_label.yaml @@ -0,0 +1,32 @@ +esphome: + name: test-keyboard-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - keyboard: + id: keyboard_widget diff --git a/tests/component_tests/lvgl/config/qrcode_no_label.yaml b/tests/component_tests/lvgl/config/qrcode_no_label.yaml new file mode 100644 index 0000000000..8bb1aafdd6 --- /dev/null +++ b/tests/component_tests/lvgl/config/qrcode_no_label.yaml @@ -0,0 +1,34 @@ +esphome: + name: test-qrcode-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - qrcode: + id: qr_widget + size: 100 + text: "esphome.io" diff --git a/tests/component_tests/lvgl/config/tabview_no_label.yaml b/tests/component_tests/lvgl/config/tabview_no_label.yaml new file mode 100644 index 0000000000..a3c16ab347 --- /dev/null +++ b/tests/component_tests/lvgl/config/tabview_no_label.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-tabview-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - tabview: + id: tabview_widget + tabs: + - name: "Tab 1" + id: tab_1 diff --git a/tests/component_tests/lvgl/test_widget_label_dependency.py b/tests/component_tests/lvgl/test_widget_label_dependency.py new file mode 100644 index 0000000000..9d3e24c8c5 --- /dev/null +++ b/tests/component_tests/lvgl/test_widget_label_dependency.py @@ -0,0 +1,32 @@ +"""Widgets whose LVGL C implementation creates or references labels +internally (tab titles, key legends, the QR canvas fallback) must declare +the label dependency in ``get_uses()``. Otherwise a config that contains +no ``label`` widget of its own compiles LVGL without ``LV_USE_LABEL`` and +fails at C compile time with undefined ``lv_label_*`` symbols. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.lvgl import defines as df + + +@pytest.mark.parametrize( + "yaml_file", + [ + "qrcode_no_label.yaml", + "keyboard_no_label.yaml", + "tabview_no_label.yaml", + ], +) +def test_label_less_config_enables_lv_use_label( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + yaml_file: str, +) -> None: + generate_main(component_config_path(yaml_file)) + assert "LV_USE_LABEL" in df.get_defines() From 10a9baff746613b52df73ff07895e177cf1a0f81 Mon Sep 17 00:00:00 2001 From: Bryan Li Date: Mon, 7 Sep 2026 23:36:42 -0700 Subject: [PATCH 060/178] [rf_bridge] Fix bucket sniffing with Portisch firmware (#17683) Co-authored-by: Bryan Li Co-authored-by: Claude Fable 5 --- esphome/components/rf_bridge/rf_bridge.cpp | 109 +++++++++++++++++---- esphome/components/rf_bridge/rf_bridge.h | 13 +++ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/esphome/components/rf_bridge/rf_bridge.cpp b/esphome/components/rf_bridge/rf_bridge.cpp index 549cce72df..a4a4da5d8c 100644 --- a/esphome/components/rf_bridge/rf_bridge.cpp +++ b/esphome/components/rf_bridge/rf_bridge.cpp @@ -18,6 +18,16 @@ void RFBridgeComponent::ack_() { } bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { + if (this->bucket_frame_candidate_ && byte == RF_CODE_START) { + // A queued next frame proves the trailing 0x55 really was the bucket + // frame's terminator: Portisch builds pulse entries from alternating + // signal edges, so the two level bits inside one pulse byte are always + // opposite — 0xAA (two high-level nibbles) cannot occur in pulse data. + // Finalize before this byte starts the new frame, so back-to-back + // deliveries are split even when loop() never observed a quiet gap + // between them. + this->finish_bucket_frame_(); + } size_t at = this->rx_buffer_.size(); this->rx_buffer_.push_back(byte); const uint8_t *raw = &this->rx_buffer_[0]; @@ -84,26 +94,21 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { break; } case RF_CODE_RFIN_BUCKET: { - if (byte != RF_CODE_STOP) { - return true; + if (at == 2) { + // The count byte: Portisch sends at most 7 buckets + sync, so 0 or + // >8 cannot be a genuine capture — reject before it can occupy the + // buffer for a full frame timeout. + return byte != 0 && byte <= B1_MAX_BUCKET_COUNT; } - - uint8_t buckets = raw[2] << 1; - std::string str; - char next_byte[3]; // 2 hex chars + null - - for (uint32_t i = 0; i <= at; i++) { - buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); - str += next_byte; - if ((i > 3) && buckets) { - buckets--; - } - if ((i < 3) || (buckets % 2) || (i == at - 1)) { - str += " "; - } - } - ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); - break; + // 0x55 is legal DATA inside a B1 frame: bucket durations are sent + // with only their HIGH byte masked to 7 bits, so a duration such as + // 0x0155 puts a raw 0x55 low byte inside the table — the first 0x55 + // must therefore not end the capture. The header declares the table + // length (raw[2] pairs), so a 0x55 there is always data; one at or + // past the first pulse index is a terminator CANDIDATE, confirmed + // once the UART goes quiet (finish_bucket_frame_ in loop()). + this->bucket_frame_candidate_ = byte == RF_CODE_STOP && at >= 3 + static_cast(raw[2]) * 2; + return true; } default: ESP_LOGW(TAG, "Unknown action: 0x%02X", action); @@ -119,6 +124,47 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { return false; } +void RFBridgeComponent::finish_bucket_frame_() { + if (this->rx_buffer_.size() < 4) { + // The candidate flag requires a header + non-empty bucket table, so + // this cannot happen while flag and buffer stay consistent; guard the + // raw[2] / size-1 reads against any future divergence anyway. + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + return; + } + const uint8_t *raw = this->rx_buffer_.data(); + const size_t at = this->rx_buffer_.size() - 1; + + uint8_t buckets = raw[2] << 1; + std::string str; + char next_byte[3]; // 2 hex chars + null + + for (uint32_t i = 0; i <= at; i++) { + buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); + str += next_byte; + if ((i > 3) && buckets) { + buckets--; + } + if ((i < 3) || (buckets % 2) || (i == at - 1)) { + str += " "; + } + } + ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); + + // Deliberately NOT ACKed: Portisch's B1 command handler leaves its + // last_sniffing_command at the previous mode (RF_CODE_RFIN), and its + // host-ACK handler re-arms sniffing from that stale value — so ACKing a + // bucket delivery silently reverts the radio to standard sniffing and + // ends bucket capture. Its delivery path is fire-and-forget and never + // waits for a host ACK. Stock Itead firmware never sends B1 frames, so + // suppressing this ACK cannot change stock-firmware behavior. + // https://github.com/esphome/esphome/issues/17682 + + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; +} + void RFBridgeComponent::write_byte_str_(const std::string &codes) { uint8_t code; int size = codes.length(); @@ -130,12 +176,31 @@ void RFBridgeComponent::write_byte_str_(const std::string &codes) { void RFBridgeComponent::loop() { const uint32_t now = App.get_loop_component_start_time(); - if (now - this->last_bridge_byte_ > 50) { + size_t avail = this->available(); + if (avail == 0 && this->bucket_frame_candidate_ && now - this->last_bridge_byte_ > BUCKET_CANDIDATE_QUIET_MS) { + // The trailing 0x55 was followed by UART quiet, so it really was the + // frame terminator and not an interior data byte. + this->finish_bucket_frame_(); + this->last_bridge_byte_ = now; + } + const bool receiving_bucket = this->rx_buffer_.size() >= 2 && this->rx_buffer_[1] == RF_CODE_RFIN_BUCKET; + if (receiving_bucket) { + // Never declare an in-progress bucket frame dead while its continuation + // bytes are already queued: a stalled loop() otherwise discards a live + // frame that the UART buffer proves is still arriving. + if (avail == 0 && now - this->last_bridge_byte_ > BUCKET_FRAME_TIMEOUT_MS) { + ESP_LOGD(TAG, "Discarding incomplete RFBridge Bucket frame (%u bytes)", + static_cast(this->rx_buffer_.size())); + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + this->last_bridge_byte_ = now; + } + } else if (now - this->last_bridge_byte_ > 50) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; this->last_bridge_byte_ = now; } - size_t avail = this->available(); while (avail > 0) { uint8_t buf[64]; size_t to_read = std::min(avail, sizeof(buf)); @@ -146,12 +211,14 @@ void RFBridgeComponent::loop() { for (size_t i = 0; i < to_read; i++) { if (this->rx_buffer_.size() > MAX_RX_BUFFER_SIZE) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } if (this->parse_bridge_byte_(buf[i])) { ESP_LOGVV(TAG, "Parsed: 0x%02X", buf[i]); this->last_bridge_byte_ = now; } else { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } } } diff --git a/esphome/components/rf_bridge/rf_bridge.h b/esphome/components/rf_bridge/rf_bridge.h index 5ad75650ab..cbb1880ec5 100644 --- a/esphome/components/rf_bridge/rf_bridge.h +++ b/esphome/components/rf_bridge/rf_bridge.h @@ -30,6 +30,17 @@ static const uint8_t RF_CODE_BEEP = 0xC0; static const uint8_t RF_CODE_STOP = 0x55; static const uint8_t RF_DEBOUNCE = 200; static const size_t MAX_RX_BUFFER_SIZE = 512; +// ~10 byte times at 19200 baud: long enough to prove the UART went quiet +// after a possible bucket-frame terminator, short enough to finish well +// before the next radio capture can be delivered. +static const uint32_t BUCKET_CANDIDATE_QUIET_MS = 5; +// Portisch drains a B1 frame's header, bucket table, and pulse data as +// separate UART writes, so an in-progress bucket frame tolerates a longer +// inter-region gap than the generic 50 ms inter-byte timeout. +static const uint32_t BUCKET_FRAME_TIMEOUT_MS = 250; +// Portisch's uart_put_RF_buckets sends at most 7 buckets plus the sync +// bucket, so a B1 count byte above 8 (or 0) is malformed for any protocol. +static const uint8_t B1_MAX_BUCKET_COUNT = 8; struct RFBridgeData { uint16_t sync; @@ -67,10 +78,12 @@ class RFBridgeComponent final : public uart::UARTDevice, public Component { void ack_(); void decode_(); bool parse_bridge_byte_(uint8_t byte); + void finish_bucket_frame_(); void write_byte_str_(const std::string &codes); std::vector rx_buffer_; uint32_t last_bridge_byte_{0}; + bool bucket_frame_candidate_{false}; CallbackManager data_callback_; CallbackManager advanced_data_callback_; From 28588310e74f93140e020fcaf6f95584412f671a Mon Sep 17 00:00:00 2001 From: raykholo Date: Tue, 8 Sep 2026 02:57:33 -0400 Subject: [PATCH 061/178] [anova] Re-assert temperature unit on every poll cycle (#17141) --- esphome/components/anova/anova.cpp | 107 ++++++++++++++--------------- esphome/components/anova/anova.h | 13 +++- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/esphome/components/anova/anova.cpp b/esphome/components/anova/anova.cpp index 6e382872e2..b0769bb622 100644 --- a/esphome/components/anova/anova.cpp +++ b/esphome/components/anova/anova.cpp @@ -13,7 +13,7 @@ void Anova::dump_config() { LOG_CLIMATE("", "Anova BLE Cooker", this); } void Anova::setup() { this->codec_ = make_unique(); - this->current_request_ = 0; + this->poll_step_ = PollStep::IDLE; } void Anova::loop() { @@ -22,6 +22,15 @@ void Anova::loop() { this->disable_loop(); } +void Anova::write_request_(AnovaPacket *pkt) { + auto status = + esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, + pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); + if (status) { + ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } +} + void Anova::control(const ClimateCall &call) { auto mode_val = call.get_mode(); if (mode_val.has_value()) { @@ -38,22 +47,11 @@ void Anova::control(const ClimateCall &call) { ESP_LOGW(TAG, "Unsupported mode: %d", mode); return; } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(pkt); } auto target_temp = call.get_target_temperature(); if (target_temp.has_value()) { - auto *pkt = this->codec_->get_set_target_temp_request(*target_temp); - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(this->codec_->get_set_target_temp_request(*target_temp)); } } @@ -62,6 +60,7 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ case ESP_GATTC_DISCONNECT_EVT: { this->current_temperature = NAN; this->target_temperature = NAN; + this->poll_step_ = PollStep::IDLE; this->publish_state(); break; } @@ -83,8 +82,8 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } case ESP_GATTC_REG_FOR_NOTIFY_EVT: { this->node_state = espbt::ClientState::ESTABLISHED; - this->current_request_ = 0; - this->update(); + this->poll_step_ = PollStep::IDLE; + this->update(); // begin the first poll cycle immediately break; } case ESP_GATTC_NOTIFY_EVT: { @@ -101,33 +100,30 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ this->mode = this->codec_->running_ ? climate::CLIMATE_MODE_HEAT : climate::CLIMATE_MODE_OFF; } if (this->codec_->has_unit()) { - this->fahrenheit_ = (this->codec_->unit_ == 'f'); - ESP_LOGD(TAG, "Anova units is %s", this->fahrenheit_ ? "fahrenheit" : "celsius"); - this->current_request_++; + ESP_LOGD(TAG, "Anova units is %s", (this->codec_->unit_ == 'f') ? "fahrenheit" : "celsius"); } this->publish_state(); - if (this->current_request_ > 1) { - AnovaPacket *pkt = nullptr; - switch (this->current_request_++) { - case 2: - pkt = this->codec_->get_read_target_temp_request(); - break; - case 3: - pkt = this->codec_->get_read_current_temp_request(); - break; - default: - this->current_request_ = 1; - break; - } - if (pkt != nullptr) { - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - } + // Advance the poll cycle to its next request based on the reply we got. + switch (this->poll_step_) { + case PollStep::SET_UNIT: + this->poll_step_ = PollStep::STATUS; + this->write_request_(this->codec_->get_read_device_status_request()); + break; + case PollStep::STATUS: + this->poll_step_ = PollStep::TARGET; + this->write_request_(this->codec_->get_read_target_temp_request()); + break; + case PollStep::TARGET: + this->poll_step_ = PollStep::CURRENT; + this->write_request_(this->codec_->get_read_current_temp_request()); + break; + case PollStep::CURRENT: + this->poll_step_ = PollStep::IDLE; // full cycle complete + break; + default: + // A reply to an ad-hoc control() write, outside a managed cycle. + break; } break; } @@ -136,27 +132,26 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } } -void Anova::set_unit_of_measurement(const char *unit) { this->fahrenheit_ = !strncmp(unit, "f", 1); } +void Anova::set_unit_of_measurement(const char *unit) { this->want_fahrenheit_ = !strncmp(unit, "f", 1); } void Anova::update() { if (this->node_state != espbt::ClientState::ESTABLISHED) return; - - if (this->current_request_ < 2) { - AnovaPacket *pkt; - if (this->current_request_ == 0) { - pkt = this->codec_->get_set_unit_request(this->fahrenheit_ ? 'f' : 'c'); - } else { - pkt = this->codec_->get_read_device_status_request(); - } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - this->current_request_++; + if (this->poll_step_ != PollStep::IDLE) { + // The previous cycle never finished within a full polling interval -- a + // reply was missed or a write failed. Restart the cycle rather than stall; + // the polling interval itself acts as the timeout. A late reply from the + // abandoned cycle is harmless: state decoding happens on every notify + // regardless of step, and each notify sends at most one follow-up request. + ESP_LOGW(TAG, "[%s] Poll cycle incomplete (step %u); restarting cycle", this->parent_->address_str(), + static_cast(this->poll_step_)); } + // Re-assert the configured unit at the start of every poll cycle, then fall + // through the status/temperature reads via the notification handler. Always + // command the configured unit (want_fahrenheit_) -- never the last value the + // device reported, or a drift to 'c' would lock itself in. + this->poll_step_ = PollStep::SET_UNIT; + this->write_request_(this->codec_->get_set_unit_request(this->want_fahrenheit_ ? 'f' : 'c')); } } // namespace esphome::anova diff --git a/esphome/components/anova/anova.h b/esphome/components/anova/anova.h index 49b1100c37..a0fa03df01 100644 --- a/esphome/components/anova/anova.h +++ b/esphome/components/anova/anova.h @@ -37,11 +37,20 @@ class Anova final : public climate::Climate, public esphome::ble_client::BLEClie void set_unit_of_measurement(const char *unit); protected: + // A poll cycle re-asserts the configured unit, then reads device state. + // Re-asserting every cycle prevents the cooker from silently reverting to + // its default (Celsius); previously the unit was only set once on + // connection, so a drift persisted (and corrupted the F/C interpretation of + // subsequent readings) until the BLE link was re-established. + enum class PollStep : uint8_t { SET_UNIT, STATUS, TARGET, CURRENT, IDLE }; + + void write_request_(AnovaPacket *pkt); + std::unique_ptr codec_; void control(const climate::ClimateCall &call) override; uint16_t char_handle_; - uint8_t current_request_; - bool fahrenheit_; + bool want_fahrenheit_{true}; // configured target unit; never overwritten by device replies + PollStep poll_step_{PollStep::IDLE}; }; } // namespace esphome::anova From 1700a40b7cc0cb4a033ba7f2d7c27c1ba65ea0f7 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:59:55 +1200 Subject: [PATCH 062/178] [core] Restore the shared git hooks after post-checkout runs script/setup (#19036) --- script/git-hooks/post-checkout | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/script/git-hooks/post-checkout b/script/git-hooks/post-checkout index 8f4085ae6e..853c2b0352 100755 --- a/script/git-hooks/post-checkout +++ b/script/git-hooks/post-checkout @@ -14,7 +14,31 @@ top=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0 [ -x "$top/venv/bin/python" ] && exit 0 [ -x "$top/script/setup" ] || exit 0 +# Every worktree shares the hooks directory of the checkout it was created +# from, and the script/setup run below is the one from whichever branch was just +# checked out. Older branches install their own pre-commit hook without checking +# for a worktree: that moves the shared hook aside as pre-commit.legacy and +# replaces it with one tied to this worktree's virtual environment, so commits +# break in every checkout. To rule that out, the hooks directory is copied +# before script/setup runs and put back exactly as it was afterwards, including +# removing any file script/setup added. +hooks=$(git rev-parse --path-format=absolute --git-path hooks 2>/dev/null) || exit 0 +snap=$(mktemp -d "$hooks/.post-checkout.XXXXXX") || exit 0 +cp -p "$hooks"/* "$snap"/ 2>/dev/null + # Clear VIRTUAL_ENV so a checkout made from a shell with an environment already # activated still gets its own, rather than having the active one repointed at # this working tree. -exec env -u VIRTUAL_ENV "$top/script/setup" +env -u VIRTUAL_ENV "$top/script/setup" +status=$? + +for f in "$hooks"/*; do + [ -e "$snap/${f##*/}" ] || rm -f "$f" +done +# Files are moved rather than copied so a hook that is still running, such as +# this one, is swapped out atomically instead of being rewritten in place. +for f in "$snap"/*; do + cmp -s "$f" "$hooks/${f##*/}" 2>/dev/null || mv -f "$f" "$hooks/${f##*/}" +done +rm -rf "$snap" +exit $status From 227ca90aad10d3084e3e8a45a13c46aea65a6c3f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:00:08 +1200 Subject: [PATCH 063/178] [core] Restore the shared git hooks after post-checkout runs script/setup (#19036) From f191d5e0c384ca785e562e652cc03cdaf21a99d4 Mon Sep 17 00:00:00 2001 From: John <34163498+CircuitSetup@users.noreply.github.com> Date: Tue, 8 Sep 2026 04:02:49 -0400 Subject: [PATCH 064/178] [atm90e32] Verify offset calibration writes (#18701) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/atm90e32/atm90e32.cpp | 360 ++++++++++-------- esphome/components/atm90e32/atm90e32.h | 71 ++-- tests/components/atm90e32/__init__.py | 5 + .../offset_register_verification_test.cpp | 62 +++ 4 files changed, 322 insertions(+), 176 deletions(-) create mode 100644 tests/components/atm90e32/__init__.py create mode 100644 tests/components/atm90e32/offset_register_verification_test.cpp diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp index d948b3741d..23701e7834 100644 --- a/esphome/components/atm90e32/atm90e32.cpp +++ b/esphome/components/atm90e32/atm90e32.cpp @@ -9,6 +9,10 @@ namespace esphome::atm90e32 { static const char *const TAG = "atm90e32"; +static const LogString *offset_calibration_name(bool power_offsets) { + return power_offsets ? LOG_STR("Power offset") : LOG_STR("Offset"); +} + static uint32_t pref_hash(const char *prefix, const char *name_space) { auto hash = fnv1_hash(prefix); return fnv1_hash_extend(hash, name_space); @@ -203,13 +207,12 @@ void ATM90E32Component::setup() { // Initialize flash storage for power offset calibrations uint32_t po_hash = pref_hash("_power_offset_calibration_", cs); - this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); + this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); bool migrated_power_offset = false; if (has_distinct_legacy_namespace) { uint32_t legacy_po_hash = pref_hash("_power_offset_calibration_", legacy_cs); - auto legacy_power_offset_pref = - global_preferences->make_preference(legacy_po_hash, true); - PowerOffsetCalibration power_offset_data[3]{}; + auto legacy_power_offset_pref = global_preferences->make_preference(legacy_po_hash, true); + OffsetCalibration power_offset_data[3]{}; int migration_status = migrate_legacy_pref_if_needed(this->power_offset_pref_, legacy_power_offset_pref, &power_offset_data); migrated_power_offset = migration_status > 0; @@ -224,20 +227,20 @@ void ATM90E32Component::setup() { global_preferences->sync(); } - this->restore_offset_calibrations_(); - this->restore_power_offset_calibrations_(); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } else { ESP_LOGI(TAG, "[CALIBRATION][%s] Power & Voltage/Current offset calibration is disabled. Using config file values.", cs); for (uint8_t phase = 0; phase < 3; ++phase) { this->write16_(this->voltage_offset_registers[phase], - static_cast(this->offset_phase_[phase].voltage_offset_)); + static_cast(this->offset_phase_[phase].first_offset)); this->write16_(this->current_offset_registers[phase], - static_cast(this->offset_phase_[phase].current_offset_)); + static_cast(this->offset_phase_[phase].second_offset)); this->write16_(this->power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].active_power_offset)); + static_cast(this->power_offset_phase_[phase].first_offset)); this->write16_(this->reactive_power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].reactive_power_offset)); + static_cast(this->power_offset_phase_[phase].second_offset)); } } @@ -317,8 +320,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].voltage_offset_, - this->config_offset_phase_[phase].current_offset_, this->offset_phase_[phase].current_offset_); + this->config_offset_phase_[phase].first_offset, this->offset_phase_[phase].first_offset, + this->config_offset_phase_[phase].second_offset, this->offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -335,10 +338,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].active_power_offset, - this->config_power_offset_phase_[phase].reactive_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->config_power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].first_offset, + this->config_power_offset_phase_[phase].second_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -372,7 +373,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\\n", cs); } @@ -385,8 +386,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); } @@ -756,36 +756,68 @@ void ATM90E32Component::save_gain_calibration_to_memory_() { } } -void ATM90E32Component::save_offset_calibration_to_memory_() { +void ATM90E32Component::finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); - bool success = this->offset_pref_.save(&this->offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_offset_calibration_ = true; - for (bool &phase : this->offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save offset calibration to memory!", cs); - } -} + const LogString *name = offset_calibration_name(power_offsets); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; -void ATM90E32Component::save_power_offset_calibration_to_memory_() { - const char *cs = this->get_calibration_id_(); - bool success = this->power_offset_pref_.save(&this->power_offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_power_offset_calibration_ = true; - for (bool &phase : this->power_offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Power offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save power offset calibration to memory!", cs); + const bool writes_verified = this->verify_offset_writes_(type); + bool saved = false; + bool synced = false; + if (writes_verified) { + saved = preference->save(offsets); + synced = global_preferences->sync(); } + + if (writes_verified && saved && synced) { + this->using_saved_calibrations_ = true; + *has_stored = true; + *restored = true; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration saved to memory. %s calibration completed and verified.", cs, + LOG_STR_ARG(name), LOG_STR_ARG(name)); + return; + } + + if (writes_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save %s calibration to memory!", cs, LOG_STR_ARG(name)); + } + + for (uint8_t phase = 0; phase < 3; phase++) { + this->write_offsets_to_registers_(phase, previous[phase].first_offset, previous[phase].second_offset, type); + } + const bool rollback_verified = this->verify_offset_writes_(type); + + bool rollback_persisted = false; + if (writes_verified) { + OffsetCalibration rollback[3]{}; + prepare_offset_rollback(previous, previous_restored, rollback); + const bool rollback_saved = preference->save(&rollback); + const bool rollback_synced = global_preferences->sync(); + rollback_persisted = rollback_saved && rollback_synced; + if (!rollback_saved || !rollback_synced) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to persist restored %s calibration values!", cs, LOG_STR_ARG(name)); + } + } + + *restored = previous_restored; + if (rollback_persisted) + *has_stored = previous_restored; + this->using_saved_calibrations_ = previous_using_saved; + if (!rollback_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; rollback readback verification failed.", cs, + LOG_STR_ARG(name)); + return; + } + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; previous values restored.", cs, LOG_STR_ARG(name)); } void ATM90E32Component::run_offset_calibrations() { @@ -803,11 +835,16 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->offset_phase_[0], this->offset_phase_[1], this->offset_phase_[2]}; + const bool previous_restored = this->restored_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = calibrate_offset(phase, true); int16_t current_offset = calibrate_offset(phase, false); - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); @@ -815,7 +852,8 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] ==================================================================\n", cs); - this->save_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); } void ATM90E32Component::run_power_offset_calibrations() { @@ -834,18 +872,25 @@ void ATM90E32Component::run_power_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->power_offset_phase_[0], this->power_offset_phase_[1], + this->power_offset_phase_[2]}; + const bool previous_restored = this->restored_power_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; ++phase) { int16_t active_offset = calibrate_power_offset(phase, false); int16_t reactive_offset = calibrate_power_offset(phase, true); - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - this->save_power_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } void ATM90E32Component::write_gains_to_registers_() { @@ -859,35 +904,26 @@ void ATM90E32Component::write_gains_to_registers_() { this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } -void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset) { - // Save to runtime - this->offset_phase_[phase].voltage_offset_ = voltage_offset; - this->phase_[phase].voltage_offset_ = voltage_offset; +void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + OffsetCalibration &offsets = power_offsets ? this->power_offset_phase_[phase] : this->offset_phase_[phase]; + offsets.first_offset = first_offset; + offsets.second_offset = second_offset; + if (power_offsets) { + this->phase_[phase].active_power_offset_ = first_offset; + this->phase_[phase].reactive_power_offset_ = second_offset; + } else { + this->phase_[phase].voltage_offset_ = first_offset; + this->phase_[phase].current_offset_ = second_offset; + } - // Save to flash-storable struct - this->offset_phase_[phase].current_offset_ = current_offset; - this->phase_[phase].current_offset_ = current_offset; - - // Write to registers + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(voltage_offset_registers[phase], static_cast(voltage_offset)); - this->write16_(current_offset_registers[phase], static_cast(current_offset)); - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); -} - -void ATM90E32Component::write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset) { - // Save to runtime - this->phase_[phase].active_power_offset_ = p_offset; - this->phase_[phase].reactive_power_offset_ = q_offset; - - // Save to flash-storable struct - this->power_offset_phase_[phase].active_power_offset = p_offset; - this->power_offset_phase_[phase].reactive_power_offset = q_offset; - - // Write to registers - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(this->power_offset_registers[phase], static_cast(p_offset)); - this->write16_(this->reactive_power_offset_registers[phase], static_cast(q_offset)); + this->write16_(first_registers[phase], static_cast(first_offset)); + this->write16_(second_registers[phase], static_cast(second_offset)); this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } @@ -947,89 +983,78 @@ void ATM90E32Component::restore_gain_calibrations_() { ESP_LOGW(TAG, "[CALIBRATION][%s] No stored gain calibrations found. Using config file values.", cs); } -void ATM90E32Component::restore_offset_calibrations_() { +void ATM90E32Component::restore_offset_calibrations_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); + const LogString *name = power_offsets ? LOG_STR("power offset") : LOG_STR("offset"); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + OffsetCalibration(*config_offsets)[3] = + power_offsets ? &this->config_power_offset_phase_ : &this->config_offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; + const bool *has_first = power_offsets ? this->has_config_active_power_offset_ : this->has_config_voltage_offset_; + const bool *has_second = power_offsets ? this->has_config_reactive_power_offset_ : this->has_config_current_offset_; + for (uint8_t i = 0; i < 3; ++i) - this->config_offset_phase_[i] = this->offset_phase_[i]; - - bool have_data = this->offset_pref_.load(&this->offset_phase_); + (*config_offsets)[i] = (*offsets)[i]; + const bool have_data = preference->load(offsets); bool all_zero = true; if (have_data) { - for (auto &phase : this->offset_phase_) { - if (phase.voltage_offset_ != 0 || phase.current_offset_ != 0) { + for (const auto &phase : *offsets) { + if (phase.first_offset != 0 || phase.second_offset != 0) { all_zero = false; break; } } } - if (have_data && !all_zero) { - this->restored_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; phase++) { - auto &offset = this->offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_voltage_offset_[phase] && - offset.voltage_offset_ != this->config_offset_phase_[phase].voltage_offset_) - mismatch = true; - if (this->has_config_current_offset_[phase] && - offset.current_offset_ != this->config_offset_phase_[phase].current_offset_) - mismatch = true; - if (mismatch) - this->offset_calibration_mismatch_[phase] = true; + *has_stored = have_data && !all_zero; + *restored = false; + for (uint8_t phase = 0; phase < 3; phase++) { + mismatches[phase] = false; + if (*has_stored) { + mismatches[phase] = + (has_first[phase] && (*offsets)[phase].first_offset != (*config_offsets)[phase].first_offset) || + (has_second[phase] && (*offsets)[phase].second_offset != (*config_offsets)[phase].second_offset); } - } else { + } + + if (!*has_stored) { for (uint8_t phase = 0; phase < 3; phase++) - this->offset_phase_[phase] = this->config_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored offset calibrations found. Using default values.", cs); + (*offsets)[phase] = (*config_offsets)[phase]; + ESP_LOGW(TAG, "[CALIBRATION][%s] No stored %s calibrations found. Using default values.", cs, LOG_STR_ARG(name)); } for (uint8_t phase = 0; phase < 3; phase++) { - write_offsets_to_registers_(phase, this->offset_phase_[phase].voltage_offset_, - this->offset_phase_[phase].current_offset_); + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); } -} - -void ATM90E32Component::restore_power_offset_calibrations_() { - const char *cs = this->get_calibration_id_(); - for (uint8_t i = 0; i < 3; ++i) - this->config_power_offset_phase_[i] = this->power_offset_phase_[i]; - - bool have_data = this->power_offset_pref_.load(&this->power_offset_phase_); - - bool all_zero = true; - if (have_data) { - for (auto &phase : this->power_offset_phase_) { - if (phase.active_power_offset != 0 || phase.reactive_power_offset != 0) { - all_zero = false; - break; - } - } + const bool initial_values_verified = this->verify_offset_writes_(type); + if (initial_values_verified) { + const auto state = resolve_offset_restore_state(*has_stored, true, false); + *restored = state.restored; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration values verified.", cs, LOG_STR_ARG(name)); + return; } - if (have_data && !all_zero) { - this->restored_power_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; ++phase) { - auto &offset = this->power_offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_active_power_offset_[phase] && - offset.active_power_offset != this->config_power_offset_phase_[phase].active_power_offset) - mismatch = true; - if (this->has_config_reactive_power_offset_[phase] && - offset.reactive_power_offset != this->config_power_offset_phase_[phase].reactive_power_offset) - mismatch = true; - if (mismatch) - this->power_offset_calibration_mismatch_[phase] = true; - } + this->using_saved_calibrations_ = false; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + for (uint8_t phase = 0; phase < 3; phase++) { + (*offsets)[phase] = (*config_offsets)[phase]; + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); + } + const auto state = resolve_offset_restore_state(*has_stored, false, this->verify_offset_writes_(type)); + *restored = state.restored; + if (state.values_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore failed verification; config values verified.", cs, + LOG_STR_ARG(name)); } else { - for (uint8_t phase = 0; phase < 3; ++phase) - this->power_offset_phase_[phase] = this->config_power_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored power offsets found. Using default values.", cs); - } - - for (uint8_t phase = 0; phase < 3; ++phase) { - write_power_offsets_to_registers_(phase, this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore and config fallback both failed verification.", cs, + LOG_STR_ARG(name)); } } @@ -1084,14 +1109,14 @@ void ATM90E32Component::clear_gain_calibrations() { void ATM90E32Component::clear_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_offset_calibration_) { + if (!this->has_stored_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored offset calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\n", cs); return; @@ -1104,10 +1129,11 @@ void ATM90E32Component::clear_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = - this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].voltage_offset_ : 0; + this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].first_offset : 0; int16_t current_offset = - this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].current_offset_ : 0; - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); } @@ -1117,6 +1143,7 @@ void ATM90E32Component::clear_offset_calibrations() { this->offset_pref_.save(&zero_offsets); // Clear stored values in flash global_preferences->sync(); + this->has_stored_offset_calibration_ = false; this->restored_offset_calibration_ = false; for (bool &phase : this->offset_calibration_mismatch_) phase = false; @@ -1126,15 +1153,14 @@ void ATM90E32Component::clear_offset_calibrations() { void ATM90E32Component::clear_power_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_power_offset_calibration_) { + if (!this->has_stored_power_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored power offsets to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); return; @@ -1147,20 +1173,21 @@ void ATM90E32Component::clear_power_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t active_offset = - this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].active_power_offset : 0; - int16_t reactive_offset = this->has_config_reactive_power_offset_[phase] - ? this->config_power_offset_phase_[phase].reactive_power_offset - : 0; - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].first_offset : 0; + int16_t reactive_offset = + this->has_config_reactive_power_offset_[phase] ? this->config_power_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - PowerOffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; + OffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; this->power_offset_pref_.save(&zero_power_offsets); global_preferences->sync(); + this->has_stored_power_offset_calibration_ = false; this->restored_power_offset_calibration_ = false; for (bool &phase : this->power_offset_calibration_mismatch_) phase = false; @@ -1215,6 +1242,31 @@ bool ATM90E32Component::verify_gain_writes_() { return success; // Return true if all writes were successful, false otherwise } +bool ATM90E32Component::verify_offset_writes_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + const char *cs = this->get_calibration_id_(); + const LogString *name = offset_calibration_name(power_offsets); + const LogString *first_name = power_offsets ? LOG_STR("active") : LOG_STR("voltage"); + const LogString *second_name = power_offsets ? LOG_STR("reactive") : LOG_STR("current"); + const OffsetCalibration *offsets = power_offsets ? this->power_offset_phase_ : this->offset_phase_; + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; + bool success = true; + for (uint8_t phase = 0; phase < 3; phase++) { + const uint16_t first = this->read16_(first_registers[phase]); + const uint16_t second = this->read16_(second_registers[phase]); + if (!offset_register_value_matches(first, offsets[phase].first_offset) || + !offset_register_value_matches(second, offsets[phase].second_offset)) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s readback failed for Phase %s: %s %d/%d, %s %d/%d.", cs, LOG_STR_ARG(name), + phase_labels[phase], LOG_STR_ARG(first_name), static_cast(first), offsets[phase].first_offset, + LOG_STR_ARG(second_name), static_cast(second), offsets[phase].second_offset); + success = false; + } + } + return success; +} + #ifdef USE_TEXT_SENSOR void ATM90E32Component::check_phase_status() { uint16_t state0 = this->read16_(ATM90E32_REGISTER_EMMSTATE0); diff --git a/esphome/components/atm90e32/atm90e32.h b/esphome/components/atm90e32/atm90e32.h index c636e5065a..fe7d903962 100644 --- a/esphome/components/atm90e32/atm90e32.h +++ b/esphome/components/atm90e32/atm90e32.h @@ -13,6 +13,40 @@ namespace esphome::atm90e32 { +inline bool offset_register_value_matches(uint16_t actual, int16_t expected) { + return actual == static_cast(expected); +} + +struct OffsetCalibration { + int16_t first_offset{0}; + int16_t second_offset{0}; +}; + +static_assert(sizeof(OffsetCalibration[3]) == 12, "Offset calibration preference layout must remain compatible"); + +enum class OffsetCalibrationType : uint8_t { + OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT, + OFFSET_CALIBRATION_TYPE_POWER, +}; + +struct OffsetRestoreState { + bool restored; + bool values_verified; +}; + +inline OffsetRestoreState resolve_offset_restore_state(bool has_stored_values, bool initial_values_verified, + bool fallback_values_verified) { + if (initial_values_verified) + return {has_stored_values, true}; + return {false, fallback_values_verified}; +} + +inline void prepare_offset_rollback(const OffsetCalibration (&previous)[3], bool had_stored_values, + OffsetCalibration (&rollback)[3]) { + for (uint8_t phase = 0; phase < 3; phase++) + rollback[phase] = had_stored_values ? previous[phase] : OffsetCalibration{}; +} + class ATM90E32Component final : public PollingComponent, public spi::SPIDevice { @@ -71,19 +105,19 @@ class ATM90E32Component final : public PollingComponent, this->has_config_current_gain_[phase] = true; } void set_voltage_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].voltage_offset_ = offset; + this->offset_phase_[phase].first_offset = offset; this->has_config_voltage_offset_[phase] = true; } void set_current_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].current_offset_ = offset; + this->offset_phase_[phase].second_offset = offset; this->has_config_current_offset_[phase] = true; } void set_active_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].active_power_offset = offset; + this->power_offset_phase_[phase].first_offset = offset; this->has_config_active_power_offset_[phase] = true; } void set_reactive_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].reactive_power_offset = offset; + this->power_offset_phase_[phase].second_offset = offset; this->has_config_reactive_power_offset_[phase] = true; } void set_freq_sensor(sensor::Sensor *freq_sensor) { freq_sensor_ = freq_sensor; } @@ -171,16 +205,16 @@ class ATM90E32Component final : public PollingComponent, float get_chip_temperature_(); bool get_publish_interval_flag_() { return publish_interval_flag_; }; void set_publish_interval_flag_(bool flag) { publish_interval_flag_ = flag; }; - void restore_offset_calibrations_(); - void restore_power_offset_calibrations_(); + void restore_offset_calibrations_(OffsetCalibrationType type); void restore_gain_calibrations_(); - void save_offset_calibration_to_memory_(); void save_gain_calibration_to_memory_(); - void save_power_offset_calibration_to_memory_(); - void write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset); - void write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset); + void finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type); + void write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type); void write_gains_to_registers_(); bool verify_gain_writes_(); + bool verify_offset_writes_(OffsetCalibrationType type); bool validate_spi_read_(uint16_t expected, const char *context = nullptr); void log_calibration_status_(); const char *get_calibration_id_(); @@ -219,19 +253,10 @@ class ATM90E32Component final : public PollingComponent, uint32_t cumulative_reverse_active_energy_{0}; } phase_[3]; - struct OffsetCalibration { - int16_t voltage_offset_{0}; - int16_t current_offset_{0}; - } offset_phase_[3]; - + OffsetCalibration offset_phase_[3]; OffsetCalibration config_offset_phase_[3]; - - struct PowerOffsetCalibration { - int16_t active_power_offset{0}; - int16_t reactive_power_offset{0}; - } power_offset_phase_[3]; - - PowerOffsetCalibration config_power_offset_phase_[3]; + OffsetCalibration power_offset_phase_[3]; + OffsetCalibration config_power_offset_phase_[3]; struct GainCalibration { uint16_t voltage_gain{1}; @@ -265,6 +290,8 @@ class ATM90E32Component final : public PollingComponent, bool enable_offset_calibration_{false}; bool enable_gain_calibration_{false}; const char *instance_id_{nullptr}; + bool has_stored_offset_calibration_{false}; + bool has_stored_power_offset_calibration_{false}; bool restored_offset_calibration_{false}; bool restored_power_offset_calibration_{false}; bool restored_gain_calibration_{false}; diff --git a/tests/components/atm90e32/__init__.py b/tests/components/atm90e32/__init__.py new file mode 100644 index 0000000000..37d6797e2d --- /dev/null +++ b/tests/components/atm90e32/__init__.py @@ -0,0 +1,5 @@ +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + manifest.dependencies = manifest.dependencies + ["sensor", "spi"] diff --git a/tests/components/atm90e32/offset_register_verification_test.cpp b/tests/components/atm90e32/offset_register_verification_test.cpp new file mode 100644 index 0000000000..3bb3eb76ea --- /dev/null +++ b/tests/components/atm90e32/offset_register_verification_test.cpp @@ -0,0 +1,62 @@ +#include + +#include "esphome/components/atm90e32/atm90e32.h" + +namespace esphome::atm90e32::testing { + +TEST(ATM90E32OffsetRegisterVerification, AcceptsExactSignedReadback) { + EXPECT_TRUE(offset_register_value_matches(0x007B, 123)); + EXPECT_TRUE(offset_register_value_matches(0xFF85, -123)); +} + +TEST(ATM90E32OffsetRegisterVerification, RejectsMismatchedReadback) { + EXPECT_FALSE(offset_register_value_matches(0x007C, 123)); + EXPECT_FALSE(offset_register_value_matches(0xFF84, -123)); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedStoredValuesAsRestored) { + const auto state = resolve_offset_restore_state(true, true, false); + + EXPECT_TRUE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedConfigFallbackAsNotRestored) { + const auto state = resolve_offset_restore_state(true, false, true); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsFailedConfigFallbackAsUnverified) { + const auto state = resolve_offset_restore_state(true, false, false); + + EXPECT_FALSE(state.restored); + EXPECT_FALSE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsConfigWithoutStoredValuesAsNotRestored) { + const auto state = resolve_offset_restore_state(false, true, false); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetPersistence, RollsBackStoredValuesOrZeroSentinel) { + const OffsetCalibration previous[3]{{1, -1}, {2, -2}, {3, -3}}; + OffsetCalibration rollback[3]{}; + + prepare_offset_rollback(previous, true, rollback); + for (uint8_t phase = 0; phase < 3; phase++) { + EXPECT_EQ(rollback[phase].first_offset, previous[phase].first_offset); + EXPECT_EQ(rollback[phase].second_offset, previous[phase].second_offset); + } + + prepare_offset_rollback(previous, false, rollback); + for (const auto &phase : rollback) { + EXPECT_EQ(phase.first_offset, 0); + EXPECT_EQ(phase.second_offset, 0); + } +} + +} // namespace esphome::atm90e32::testing From f91486305ff20743d086651517d652360ab58ff7 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:21:48 +0200 Subject: [PATCH 065/178] Bump bundled esphome-device-builder to 1.14.5 (#19040) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index da76ab7b6a..ac84ee4689 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 RUN \ platformio settings set enable_telemetry No \ From 1ce0bed3f672d3a4699dad0cbfd8617c3b8950e5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 18:10:45 +0200 Subject: [PATCH 066/178] [core] Share compiled binaries across modbus integration tests (#18945) --- script/helpers.py | 17 + tests/integration/README.md | 7 + tests/integration/conftest.py | 416 ++++++++++++++---- .../fixtures/sensor_filters_batch_window.yaml | 58 --- .../uart_mock_modbus_client_read_write.yaml | 111 ----- .../fixtures/uart_mock_modbus_custom_pdu.yaml | 88 ---- ...t_mock_modbus_deprecated_write_buffer.yaml | 106 ----- .../uart_mock_modbus_lambda_invert.yaml | 95 ---- .../uart_mock_modbus_lambda_write.yaml | 97 ---- .../fixtures/uart_mock_modbus_loopback.yaml | 233 ++++++++++ ...roller.yaml => uart_mock_modbus_mesh.yaml} | 121 ++++- .../uart_mock_modbus_register_offset.yaml | 138 ------ .../fixtures/uart_mock_modbus_server.yaml | 124 ------ ...ock_modbus_server_controller_multiple.yaml | 116 ----- ... => uart_mock_modbus_server_injected.yaml} | 55 ++- tests/integration/host_prefs.py | 14 +- .../test_api_zero_psk_provisioning.py | 1 - .../test_host_preferences_suspend_resume.py | 11 +- tests/integration/test_light_initial_state.py | 8 - tests/integration/test_uart_mock_modbus.py | 23 +- tests/script/test_helpers.py | 28 ++ 21 files changed, 805 insertions(+), 1062 deletions(-) delete mode 100644 tests/integration/fixtures/sensor_filters_batch_window.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml create mode 100644 tests/integration/fixtures/uart_mock_modbus_loopback.yaml rename tests/integration/fixtures/{uart_mock_modbus_server_controller.yaml => uart_mock_modbus_mesh.yaml} (58%) delete mode 100644 tests/integration/fixtures/uart_mock_modbus_register_offset.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_server.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml rename tests/integration/fixtures/{uart_mock_modbus_server_read_write.yaml => uart_mock_modbus_server_injected.yaml} (52%) diff --git a/script/helpers.py b/script/helpers.py index bf22e15808..a8a237118f 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -1104,6 +1104,10 @@ def get_components_per_integration_fixture() -> dict[str, set[str]]: _TEST_FUNC_RE = re.compile(r"async def (test_\w+)") +# Any usage form (decorator, pytestmark assignment or list element); only +# test_*.py files are scanned, so the marker docs elsewhere cannot false-hit +_SHARED_YAML_USE_RE = re.compile(r"\bmark\.shared_yaml") +_SHARED_YAML_ARG_RE = re.compile(r"\(\s*[\"'](\w+)[\"']\s*\)") @cache @@ -1123,6 +1127,19 @@ def get_fixture_to_test_files() -> dict[str, frozenset[str]]: for func in _TEST_FUNC_RE.findall(content): base_name = func.replace("test_", "").partition("[")[0] result.setdefault(base_name, set()).add(rel_path) + # Shared fixtures are named by marker, not by a test function; each + # decorator must carry a string literal or its fixture would silently + # map to no tests + for use in _SHARED_YAML_USE_RE.finditer(content): + arg = _SHARED_YAML_ARG_RE.match(content, use.end()) + if arg is None: + line = content.count("\n", 0, use.start()) + 1 + raise ValueError( + f"{rel_path}:{line}: shared_yaml marker must take a " + "single-line string literal so CI test selection can map " + "its fixture" + ) + result.setdefault(arg.group(1), set()).add(rel_path) return {k: frozenset(v) for k, v in result.items()} diff --git a/tests/integration/README.md b/tests/integration/README.md index 790d9a3a11..bee20409e8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -21,6 +21,13 @@ The `yaml_config` fixture automatically loads YAML configurations based on the t - The fixture file must exist or the test will fail with a clear error message - The fixture automatically injects a dynamic port number into the API configuration +Tests marked `@pytest.mark.shared_yaml("name")` load `fixtures/name.yaml` instead +of the test-named file and compile it in a shared, hash-keyed build directory, so +the whole group pays one full compile and each test only a relink. The marker +argument must be a single-line string literal (CI test selection maps fixtures to +test files by scanning for it), and marked tests must hand the `yaml_config` +content to `run_compiled` unmodified. + ### Key Fixtures - `run_compiled` - Combines write, compile, and run operations into a single context manager diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 15c5860879..78e0b1a36c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,17 +4,22 @@ from __future__ import annotations import asyncio from collections.abc import AsyncGenerator, Callable, Generator -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress import fcntl +from functools import cache +import hashlib import logging import os from pathlib import Path import platform +import re +import shutil import signal import socket import subprocess import sys import tempfile +import time from typing import TextIO from aioesphomeapi import APIClient, APIConnectionError, LogParser, ReconnectLogic @@ -23,7 +28,13 @@ import pytest_asyncio import esphome.config from esphome.core import CORE -from esphome.helpers import get_usable_cpu_count +from esphome.helpers import ( + get_usable_cpu_count, + read_file, + rmtree, + write_file, + write_file_if_changed, +) from esphome.platformio.toolchain import get_idedata from .const import ( @@ -56,6 +67,21 @@ import pty # not available on Windows pytest.register_assert_rewrite("tests.integration.entity_utils") +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "shared_yaml(name): load fixtures/.yaml and compile it in a shared, " + "hash-keyed incremental build directory", + ) + + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +# CI caches parts of this path; keep in sync with ci.yml integration-tests. +INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests" + + def _get_platformio_env(cache_dir: Path) -> dict[str, str]: """Get environment variables for PlatformIO with shared cache.""" env = os.environ.copy() @@ -78,7 +104,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: ) # Compile with THIS tree's esphome sources, not wherever the venv's editable # install points (which may be a different git worktree or checkout). - repo_root = str(Path(__file__).resolve().parent.parent.parent) + repo_root = str(REPO_ROOT) existing = env.get("PYTHONPATH") env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{existing}" if existing else repo_root return env @@ -88,8 +114,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: def shared_platformio_cache() -> Generator[Path]: """Initialize a shared PlatformIO cache for all integration tests.""" # Use a dedicated directory for integration tests to avoid conflicts. - # CI caches parts of this path; keep in sync with ci.yml integration-tests. - test_cache_dir = Path.home() / ".esphome-integration-tests" + test_cache_dir = INTEGRATION_TESTS_ROOT cache_dir = test_cache_dir / "platformio" # Use a lock file in the home directory to ensure only one process initializes the cache @@ -112,7 +137,9 @@ def shared_platformio_cache() -> Generator[Path]: init_dir = Path(tmpdir) fixture_path = Path(__file__).parent / "fixtures" / "cache_init.yaml" config_path = init_dir / "cache_init.yaml" - config_path.write_text(fixture_path.read_text()) + config_path.write_text( + fixture_path.read_text(encoding="utf-8"), encoding="utf-8" + ) # Run compilation to populate the cache # We must succeed here to avoid race conditions where multiple @@ -162,13 +189,6 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) -@pytest.fixture -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Host preferences persist per device name; give the test its own so a - provisioned key never leaks into another run.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) - - @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" @@ -188,21 +208,29 @@ def unused_tcp_port(reserved_tcp_port: tuple[int, socket.socket]) -> int: return reserved_tcp_port[0] +@pytest.fixture(autouse=True) +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """Give every test its own host prefs dir; prefs are keyed only by device + name, which tests sharing a fixture also share.""" + prefdir = tmp_path / "prefs" + monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir)) + return prefdir + + @pytest_asyncio.fixture async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> str: """Load YAML configuration based on test name.""" - # Get the test function name - test_name: str = request.node.name - # Extract the base test name (remove test_ prefix and any parametrization) - base_name = test_name.replace("test_", "").partition("[")[0] + shared_name = _shared_yaml_name(request) + # Base test name: test_ prefix and any parametrization stripped + base_name = shared_name or request.node.name.replace("test_", "").partition("[")[0] # Load the fixture file - fixture_path = Path(__file__).parent / "fixtures" / f"{base_name}.yaml" + fixture_path = FIXTURES_DIR / f"{base_name}.yaml" if not fixture_path.exists(): raise FileNotFoundError(f"Fixture file not found: {fixture_path}") loop = asyncio.get_running_loop() - content = await loop.run_in_executor(None, fixture_path.read_text) + content = await loop.run_in_executor(None, read_file, fixture_path) # Replace the port in the config if it contains api section if "api:" in content: @@ -226,11 +254,13 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s # Replace external component path placeholder if present if "EXTERNAL_COMPONENT_PATH" in content: - external_components_path = str( - Path(__file__).parent / "fixtures" / "external_components" - ) + external_components_path = str(FIXTURES_DIR / "external_components") content = content.replace("EXTERNAL_COMPONENT_PATH", external_components_path) + if shared_name is not None: + # _compile verifies the marked test compiles this content unmodified + request.node._shared_yaml_content = content + return content @@ -240,24 +270,218 @@ async def write_yaml_config( ) -> AsyncGenerator[ConfigWriter]: """Write YAML configuration to a file.""" # Get the test name for default filename - test_name = request.node.name - base_name = test_name.replace("test_", "").split("[")[0] + base_name = request.node.name.replace("test_", "").partition("[")[0] async def _write_config(content: str, filename: str | None = None) -> Path: if filename is None: filename = f"{base_name}.yaml" config_path = integration_test_dir / filename loop = asyncio.get_running_loop() - await loop.run_in_executor(None, config_path.write_text, content) + await loop.run_in_executor(None, write_file, config_path, content) return config_path yield _write_config +# Deliberately not CI-cached (ci.yml caches only platformio/ subpaths); stale +# dirs for a fixture are pruned when its content hash changes. +SHARED_BUILDS_ROOT = INTEGRATION_TESTS_ROOT / "builds" + +# In the dir name (not just the hash) so pruning stays inside this checkout +_REPO_KEY = hashlib.sha256(str(REPO_ROOT).encode()).hexdigest()[:8] + +# Give a contended shared build lock time for a full cold compile ahead of us +_SHARED_LOCK_TIMEOUT_S = 900 +_SHARED_LOCK_POLL_S = 0.1 +_SHARED_LOCK_REPORT_S = 30 + +# Reclaims dirs orphaned by fixture renames or deleted checkouts +_STALE_BUILD_MAX_AGE_S = 30 * 24 * 3600 + +# ELF path per shared build dir; constant once compiled, so resolve it only once +_shared_elf_paths: dict[Path, Path] = {} + +# Dirs this process already swept; pruning is session-scoped work +_pruned_dirs: set[Path] = set() + + +def _shared_yaml_name(request: pytest.FixtureRequest) -> str | None: + """Name passed to the shared_yaml marker, or None when unmarked.""" + marker = request.node.get_closest_marker("shared_yaml") + if marker is None: + return None + # Exactly one \w+ positional arg: the name doubles as a build dir + # component, and CI test selection (script/helpers.py) parses the same shape + if ( + len(marker.args) != 1 + or marker.kwargs + or not re.fullmatch(r"\w+", str(marker.args[0])) + ): + raise ValueError( + "shared_yaml marker requires exactly one \\w+ fixture name literal" + ) + return marker.args[0] + + +def _shared_build_prefix(name: str) -> str: + return f"{name}-{_REPO_KEY}-" + + +@cache +def _shared_build_dir(name: str) -> Path: + """Dir keyed by checkout and fixture source, before per-test injections.""" + key = hashlib.sha256((FIXTURES_DIR / f"{name}.yaml").read_bytes()).hexdigest()[:16] + return SHARED_BUILDS_ROOT / (_shared_build_prefix(name) + key) + + +def _read_stamp(stamp: Path, shared_dir: Path) -> Path | None: + """ELF path recorded by the last completed compile, or None.""" + try: + text = stamp.read_text(encoding="utf-8").strip() + except FileNotFoundError: + return None + except OSError as err: + print(f"Cannot read {stamp}: {err}") + return None + if not text: + print(f"Ignoring empty stamp {stamp}") + return None + built = Path(text) + # Never trust a stamp pointing outside its own build dir as an unlink target + if shared_dir.resolve() in built.resolve().parents: + return built + print(f"Ignoring stamp {stamp} pointing outside {shared_dir}") + return None + + +def _unused_since(stale: Path, cutoff: float) -> bool: + """Whether a build dir looks untouched since cutoff; unknown counts as used.""" + # Newest of the .built stamp (rewritten by every completed compile) and the + # dir itself (freshened by a worker claiming the dir before locking) + newest: float | None = None + for probe in (stale / ".built", stale): + try: + mtime = probe.stat().st_mtime + except FileNotFoundError: + continue + except NotADirectoryError: + return True # a stray file where a dir should be; reclaimable + except OSError as err: + print(f"Cannot age-probe {stale}: {err}") + return False # unknown never authorizes deletion + newest = mtime if newest is None else max(newest, mtime) + return newest is not None and newest < cutoff + + +def _prune_stale_builds(name: str, keep: Path) -> None: + """Remove outdated build dirs (blocking, run in executor): this checkout's + other dirs for the fixture, plus anything untouched for 30 days. Tolerates + other workers pruning the same dirs concurrently.""" + cutoff = time.time() - _STALE_BUILD_MAX_AGE_S + prefix = _shared_build_prefix(name) + for stale in SHARED_BUILDS_ROOT.iterdir(): + if stale == keep: + continue + same_fixture = stale.name.startswith(prefix) + if not same_fixture and not _unused_since(stale, cutoff): + continue + # Creating .lock bumps the dir mtime, so remember whether the re-probe + # under the lock can trust it + lock_preexisting = (stale / ".lock").exists() + try: + lock_file = (stale / ".lock").open("w") + except FileNotFoundError: + continue # pruned by another worker meanwhile + except NotADirectoryError: + print(f"Removing stray file {stale}") + stale.unlink(missing_ok=True) + continue + except OSError as err: + print(f"Cannot prune {stale}: {err}") + continue + with lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + continue # still in use by another run + # Re-probe under the lock: a worker freshens its dir before + # locking, so a just-claimed dir no longer looks unused. A dir + # whose .lock we just created cannot be held by anyone, and our + # own open bumped its mtime, so its pre-open probe stands + if ( + lock_preexisting + and not same_fixture + and not _unused_since(stale, cutoff) + ): + continue + # rmtree tolerates races; a leftover partial tree only costs a + # rebuild, since the ELF is deleted before every compile + try: + rmtree(stale) + except OSError as err: + print(f"Failed to prune {stale}: {err}") + + +async def _run_esphome_compile( + config_path: Path, cwd: Path, env: dict[str, str] +) -> None: + """Run `esphome compile`, retrying up to 3 times on a segfault.""" + max_retries = 3 + for attempt in range(max_retries): + # Compile using subprocess, inheriting stdout/stderr to show progress + proc = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "esphome", + "compile", + str(config_path), + cwd=cwd, + stdout=None, # Inherit stdout + stderr=None, # Inherit stderr + stdin=asyncio.subprocess.DEVNULL, + # Start in a new process group to isolate signal handling + start_new_session=True, + env=env, + close_fds=False, + ) + await proc.wait() + + if proc.returncode == 0: + break + if proc.returncode == -11 and attempt < max_retries - 1: + # Segfault (-11 = SIGSEGV), retry + print( + f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..." + ) + await asyncio.sleep(1) # Brief pause before retry + continue + raise RuntimeError( + f"Failed to compile {config_path}, return code: {proc.returncode}. " + f"Run with 'pytest -s' to see compilation output." + ) + + +def _resolve_compiled_binary(config_path: Path) -> Path: + """Load the config to learn the compiled ELF path (blocking, run in executor).""" + CORE.reset() # Reset CORE state between test runs + CORE.config_path = config_path + config = esphome.config.read_config( + {"command": "compile", "config": str(config_path)} + ) + if config is None: + raise RuntimeError(f"Failed to read config from {config_path}") + idedata = get_idedata(config) + binary_path = Path(idedata.firmware_elf_path) + if not binary_path.exists(): + raise RuntimeError(f"Compiled binary not found at {binary_path}") + return binary_path + + @pytest_asyncio.fixture async def compile_esphome( integration_test_dir: Path, shared_platformio_cache: Path, + request: pytest.FixtureRequest, ) -> AsyncGenerator[CompileFunction]: """Compile an ESPHome configuration and return the binary path.""" @@ -265,66 +489,96 @@ async def compile_esphome( # Use the shared PlatformIO cache for faster compilation # This avoids re-downloading dependencies for each test env = _get_platformio_env(shared_platformio_cache) - - # Retry compilation up to 3 times if we get a segfault - max_retries = 3 - for attempt in range(max_retries): - # Compile using subprocess, inheriting stdout/stderr to show progress - proc = await asyncio.create_subprocess_exec( - sys.executable, - "-m", - "esphome", - "compile", - str(config_path), - cwd=integration_test_dir, - stdout=None, # Inherit stdout - stderr=None, # Inherit stderr - stdin=asyncio.subprocess.DEVNULL, - # Start in a new process group to isolate signal handling - start_new_session=True, - env=env, - close_fds=False, - ) - await proc.wait() - - if proc.returncode == 0: - # Success! - break - if proc.returncode == -11 and attempt < max_retries - 1: - # Segfault (-11 = SIGSEGV), retry - print( - f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..." - ) - await asyncio.sleep(1) # Brief pause before retry - continue - # Other error or final retry - raise RuntimeError( - f"Failed to compile {config_path}, return code: {proc.returncode}. " - f"Run with 'pytest -s' to see compilation output." - ) - - # Load the config to get idedata (blocking call, must use executor) loop = asyncio.get_running_loop() - def _read_config_and_get_binary(): - CORE.reset() # Reset CORE state between test runs - CORE.config_path = config_path - config = esphome.config.read_config( - {"command": "compile", "config": str(config_path)} + name = _shared_yaml_name(request) + if name is None: + await _run_esphome_compile(config_path, integration_test_dir, env) + return await loop.run_in_executor( + None, _resolve_compiled_binary, config_path ) - if config is None: - raise RuntimeError(f"Failed to read config from {config_path}") - # Get the compiled binary path - idedata = get_idedata(config) - return Path(idedata.firmware_elf_path) - - binary_path = await loop.run_in_executor(None, _read_config_and_get_binary) - - if not binary_path.exists(): - raise RuntimeError(f"Compiled binary not found at {binary_path}") - - return binary_path + # Shared fixture: build in a hash-keyed dir so tests sharing a config + # pay one full compile and later only a main.cpp (port) rebuild + relink + shared_dir = _shared_build_dir(name) + shared_dir.mkdir(parents=True, exist_ok=True) + # Freshen the dir before locking so a concurrent age sweep, which + # re-probes under the lock, never reaps a dir a worker just claimed; + # if a peer reaped it already, the guarded lock open recreates it + with suppress(FileNotFoundError): + os.utime(shared_dir) + if shared_dir not in _pruned_dirs: + _pruned_dirs.add(shared_dir) + await loop.run_in_executor(None, _prune_stale_builds, name, shared_dir) + shared_config = shared_dir / f"{name}.yaml" + private_binary = integration_test_dir / f"{name}.elf" + content = await loop.run_in_executor(None, read_file, config_path) + if content != getattr(request.node, "_shared_yaml_content", None): + # The dir is keyed by the fixture source; a mutated config would be + # cached under a hash that does not describe it + raise RuntimeError( + "shared_yaml tests must compile the yaml_config content unmodified" + ) + # flock serializes concurrent xdist workers; closing the fd releases it. + # Hand-rolled rather than filelock.FileLock: non-blocking retries keep + # the wait cancellable, while a blocking acquire in an executor thread + # would survive test cancellation holding the fd + try: + lock_file = (shared_dir / ".lock").open("w") + except FileNotFoundError: + # A peer run pruning divergent hashes reaped the dir between our + # mkdir and this open; recreate it and pay a full rebuild + shared_dir.mkdir(parents=True, exist_ok=True) + lock_file = (shared_dir / ".lock").open("w") + with lock_file: + start = time.monotonic() + last_report = start + while True: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + now = time.monotonic() + if now - start > _SHARED_LOCK_TIMEOUT_S: + raise RuntimeError( + f"Timed out waiting for the {shared_dir} lock" + ) from None + if now - last_report >= _SHARED_LOCK_REPORT_S: + last_report = now + print( + f"Waited {now - start:.0f}s for another worker's " + f"build of {shared_dir.name}" + ) + await asyncio.sleep(_SHARED_LOCK_POLL_S) + # .built carries the ELF path of the last completed compile, so + # later workers skip the config re-read in _resolve_compiled_binary + stamp = shared_dir / ".built" + if (built := _shared_elf_paths.get(shared_dir)) is None: + built = await loop.run_in_executor(None, _read_stamp, stamp, shared_dir) + # Delete the ELF before compiling: whatever exists afterwards is + # this compile's output, so no staleness check is ever needed. + # With no usable stamp, sweep any leftover at the known layout + if built is not None: + built.unlink(missing_ok=True) + else: + # Layout-agnostic: ESPHOME_BUILD_PATH can move the build tree + for leftover in shared_dir.rglob("program"): + if leftover.is_file(): + leftover.unlink() + await loop.run_in_executor( + None, write_file_if_changed, shared_config, content + ) + await _run_esphome_compile(shared_config, shared_dir, env) + if built is None or not built.exists(): + built = await loop.run_in_executor( + None, _resolve_compiled_binary, shared_config + ) + _shared_elf_paths[shared_dir] = built + await loop.run_in_executor(None, write_file, stamp, str(built)) + # Copy out before unlocking: another worker may relink firmware.elf + # while this test is still running its private copy + await loop.run_in_executor(None, shutil.copy2, built, private_binary) + return private_binary yield _compile diff --git a/tests/integration/fixtures/sensor_filters_batch_window.yaml b/tests/integration/fixtures/sensor_filters_batch_window.yaml deleted file mode 100644 index 58a254c215..0000000000 --- a/tests/integration/fixtures/sensor_filters_batch_window.yaml +++ /dev/null @@ -1,58 +0,0 @@ -esphome: - name: test-batch-window-filters - -host: -api: - batch_delay: 0ms # Disable batching to receive all state updates -logger: - level: DEBUG - -# Template sensor that we'll use to publish values -sensor: - - platform: template - name: "Source Sensor" - id: source_sensor - accuracy_decimals: 2 - - # Batch window filters (window_size == send_every) - use streaming filters - - platform: copy - source_id: source_sensor - name: "Min Sensor" - id: min_sensor - filters: - - min: - window_size: 5 - send_every: 5 - send_first_at: 1 - - - platform: copy - source_id: source_sensor - name: "Max Sensor" - id: max_sensor - filters: - - max: - window_size: 5 - send_every: 5 - send_first_at: 1 - - - platform: copy - source_id: source_sensor - name: "Moving Avg Sensor" - id: moving_avg_sensor - filters: - - sliding_window_moving_average: - window_size: 5 - send_every: 5 - send_first_at: 1 - -# Button to trigger publishing test values -button: - - platform: template - name: "Publish Values Button" - id: publish_button - on_press: - - lambda: |- - // Publish 10 values: 1.0, 2.0, ..., 10.0 - for (int i = 1; i <= 10; i++) { - id(source_sensor).publish_state(float(i)); - } diff --git a/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml b/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml deleted file mode 100644 index 1f89889c95..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml +++ /dev/null @@ -1,111 +0,0 @@ -esphome: - name: uart-mock-modbus-cli-rw - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -# Two virtual buses looped back to each other: the client's transmissions reach the server and the -# server's replies reach the client. auto_start so forwarding is active before the button fires. -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_client - data: !lambda return data; - - id: virtual_uart_client - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: stored_1 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_client - id: virtual_modbus_client - role: client - turnaround_time: 10ms - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - registers: - # Writable + readable register: the read publishes what it returns, so the test can confirm the - # write half of the 0x17 ran before the read half (Modbus 6.17). - - address: 0x01 - value_type: U_WORD - read_lambda: |- - id(srv_read_1).publish_state(id(stored_1)); - return id(stored_1); - write_lambda: |- - id(stored_1) = x; - id(srv_write_1).publish_state(x); - return true; - # Read-only register, returned together with 0x01 by the 2-register read half. - - address: 0x02 - value_type: U_WORD - read_lambda: return 0x00AA; - -sensor: - # Server-side observations. - - platform: template - name: "srv_write_1" - id: srv_write_1 - - platform: template - name: "srv_read_1" - id: srv_read_1 - # Client-side read-back: the values the client's on_response received. - - platform: template - name: "client_read_0" - id: client_read_0 - - platform: template - name: "client_read_1" - id: client_read_1 - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - on_press: - # FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction. - - modbus_client.read_write_multiple_registers: - address: 0x01 - read_address: 0x0001 - read_count: 2 - write_address: 0x0001 - values: [0x1234] - on_response: - then: - - lambda: |- - // values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002. - if (values.size() >= 2) { - id(client_read_0).publish_state(values[0]); - id(client_read_1).publish_state(values[1]); - } diff --git a/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml b/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml deleted file mode 100644 index 188abf90f1..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml +++ /dev/null @@ -1,88 +0,0 @@ -esphome: - name: uart-mock-modbus-custom-pdu - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 259; - -sensor: - # Plain read to confirm the controller <-> server link is up. - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "plain_read" - address: 0x01 - register_type: holding - value_type: U_WORD - # Custom PDU: read holding register 0x0001, count 1. The PDU is - # {function code, address hi, address lo, count hi, count lo}; the device - # address and CRC are added by the hub. The lambda parses the response payload - # (the register value, big-endian). - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "custom_read" - custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01] - lambda: |- - if (data.size() < 2) return {}; - return (float) ((data[0] << 8) | data[1]); - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml b/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml deleted file mode 100644 index f378e3de43..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml +++ /dev/null @@ -1,106 +0,0 @@ -esphome: - name: uart-mock-modbus-dep-buffer - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg10 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x10 - value_type: U_WORD - read_lambda: return id(reg10); - write_lambda: |- - id(reg10) = x; - return true; - -# A number whose write_lambda uses the DEPRECATED buffer parameter (fills `payload` with a legacy raw -# frame as words: device address + function code + data) instead of the new item->write_* API. The write -# must still land with its legacy semantics, and the one-time deprecation warning must fire only once per -# entity no matter how many writes happen. -number: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "buf_number" - id: buf_number - address: 0x10 - register_type: holding - value_type: U_WORD - min_value: 0 - max_value: 1000 - step: 1 - write_lambda: |- - // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0010, value. - payload.push_back(0x0106); - payload.push_back(0x0010); - payload.push_back((uint16_t) x); - return {}; - -# Reports the server-side register so the test can observe that the deprecated buffer write landed. -sensor: - - platform: template - name: "written_value" - id: written_value - update_interval: 0.5s - lambda: "return id(reg10);" - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # The test drives the writes via number_command; the mock is autostart. diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml deleted file mode 100644 index 41afce70d6..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml +++ /dev/null @@ -1,95 +0,0 @@ -esphome: - name: uart-mock-modbus-lambda-invert - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg40 - type: uint16_t - initial_value: "5" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x40 - value_type: U_WORD - read_lambda: return id(reg40); - write_lambda: id(reg40) = x; return true; - -# An active-low holding switch: the write_lambda inverts the wire value, but the entity must still -# report the REQUESTED state. assumed_state keeps the register unpolled, so the published state comes -# only from write_state() - turning ON writes 0x0000 yet the switch shows ON. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "invert_switch" - register_type: holding - address: 0x40 - assumed_state: true - write_lambda: |- - return !x; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_40" - address: 0x40 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml deleted file mode 100644 index 86e17ea0d7..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml +++ /dev/null @@ -1,97 +0,0 @@ -esphome: - name: uart-mock-modbus-lambda-write - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg30 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x30 - value_type: U_WORD - read_lambda: return id(reg30); - write_lambda: id(reg30) = x; return true; - -# A COIL-type switch (assumed_state, write-only) whose write_lambda ignores its own coil type and instead -# drives a HOLDING-REGISTER write on the mock server through the entity itself: `item` IS the command, so -# item->write_single_register() sends a register write from a coil entity (cross-type). Returning nothing -# (an empty optional) tells the write path the lambda already dispatched the frame - no default coil write. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "cross_switch" - register_type: coil - address: 0x00 - assumed_state: true - write_lambda: |- - item->write_single_register(0x30, x ? 1234 : 0); - return {}; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_30" - address: 0x30 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_loopback.yaml b/tests/integration/fixtures/uart_mock_modbus_loopback.yaml new file mode 100644 index 0000000000..7212bfb2b2 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_loopback.yaml @@ -0,0 +1,233 @@ +esphome: + name: uart-mock-modbus-loopback + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +# Shared loopback fixture (see the shared_yaml markers in the test file); +# register spaces are disjoint so each test only observes its own entities. +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg10 + type: uint16_t + initial_value: "100" + - id: reg11 + type: uint16_t + initial_value: "200" + - id: reg12 + type: uint16_t + initial_value: "300" + - id: reg13 + type: uint16_t + initial_value: "0xABCD" + - id: reg30 + type: uint16_t + initial_value: "0" + - id: reg40 + type: uint16_t + initial_value: "5" + - id: reg50 + type: uint16_t + initial_value: "0" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 259; + - address: 0x10 + value_type: U_WORD + read_lambda: return id(reg10); + write_lambda: id(reg10) = x; return true; + - address: 0x11 + value_type: U_WORD + read_lambda: return id(reg11); + write_lambda: id(reg11) = x; return true; + - address: 0x12 + value_type: U_WORD + read_lambda: return id(reg12); + write_lambda: id(reg12) = x; return true; + - address: 0x13 + value_type: U_WORD + read_lambda: return id(reg13); + - address: 0x30 + value_type: U_WORD + read_lambda: return id(reg30); + write_lambda: id(reg30) = x; return true; + - address: 0x40 + value_type: U_WORD + read_lambda: return id(reg40); + write_lambda: id(reg40) = x; return true; + - address: 0x50 + value_type: U_WORD + read_lambda: return id(reg50); + write_lambda: id(reg50) = x; return true; + +# Byte-based offset: 2 bytes -> register 0x11 (the old code folded it in as a +# register count, hitting 0x12). assumed_state keeps the switch write-only. +switch: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "offset_switch" + register_type: holding + address: 0x10 + offset: 2 + assumed_state: true + # Reading switch, byte offset 6 -> register 0x13; the pre-fix resolution (0x16) + # would draw ILLEGAL_DATA_ADDRESS and never publish. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "read_offset_switch" + register_type: holding + address: 0x10 + offset: 6 + bitmask: 0x1 + # Coil switch whose write_lambda dispatches a holding-register write via `item`; + # returning an empty optional suppresses the default coil write. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "cross_switch" + register_type: coil + address: 0x00 + assumed_state: true + write_lambda: |- + item->write_single_register(0x30, x ? 1234 : 0); + return {}; + # Active-low: the write_lambda inverts the wire value but the entity must still + # report the requested state (assumed_state keeps the register unpolled). + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "invert_switch" + register_type: holding + address: 0x40 + assumed_state: true + write_lambda: |- + return !x; + +# Uses the deprecated buffer parameter (legacy raw frame as words); the write +# must land and the deprecation warning must fire only once per entity. +number: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "buf_number" + id: buf_number + address: 0x50 + register_type: holding + value_type: U_WORD + min_value: 0 + max_value: 1000 + step: 1 + write_lambda: |- + // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0050, value. + payload.push_back(0x0106); + payload.push_back(0x0050); + payload.push_back((uint16_t) x); + return {}; + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "plain_read" + address: 0x01 + register_type: holding + value_type: U_WORD + # Custom PDU: read holding register 0x0001; device address and CRC are added + # by the hub. The lambda parses the big-endian register value. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "custom_read" + custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01] + lambda: |- + if (data.size() < 2) return {}; + return (float) ((data[0] << 8) | data[1]); + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_10" + address: 0x10 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_11" + address: 0x11 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_12" + address: 0x12 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_30" + address: 0x30 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_40" + address: 0x40 + register_type: holding + value_type: U_WORD + # Reports the server-side register so the test can observe that the deprecated buffer write landed. + - platform: template + name: "written_value" + id: written_value + update_interval: 0.5s + lambda: "return id(reg50);" + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # Nothing to start (mock is autostart); tests drive entities directly diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml b/tests/integration/fixtures/uart_mock_modbus_mesh.yaml similarity index 58% rename from tests/integration/fixtures/uart_mock_modbus_server_controller.yaml rename to tests/integration/fixtures/uart_mock_modbus_mesh.yaml index 4a5d280a2f..69edd614d7 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_mesh.yaml @@ -1,5 +1,5 @@ esphome: - name: uart-mock-modbus-server-contro + name: uart-mock-modbus-mesh host: api: @@ -17,13 +17,14 @@ uart: baud_rate: 115200 port: /dev/null +# Shared 3-bus mesh (see the shared_yaml markers): addr 1 = typed read-only +# registers, addr 5 = the read/write 0x17 target, addr 2/3 on the second +# server hub. auto_start everywhere: the controller polls at boot, so the +# forwarding must already be live or early requests generate warnings. +# Every test presses Start Scenario, so all merged actions fire in every test. uart_mock: - id: virtual_uart_server baud_rate: 9600 - # auto_start must be true for loopback fixtures: the modbus controller - # polls on its update_interval immediately at boot, so the uart_mock - # forwarding must already be active or early requests are lost and - # generate modbus warnings. auto_start: true debug: on_tx: @@ -31,35 +32,68 @@ uart_mock: - uart_mock.inject_rx: id: virtual_uart_controller data: !lambda return data; - - id: virtual_uart_controller + - uart_mock.inject_rx: + id: virtual_uart_server_2 + data: !lambda return data; + - id: virtual_uart_server_2 baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above + auto_start: true debug: on_tx: - then: - uart_mock.inject_rx: id: virtual_uart_server data: !lambda return data; + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + - uart_mock.inject_rx: + id: virtual_uart_server_2 + data: !lambda return data; + +globals: + - id: stored_1 + type: uint16_t + initial_value: "0" modbus: - uart_id: virtual_uart_server id: virtual_modbus_server role: server + - uart_id: virtual_uart_server_2 + id: virtual_modbus_server_2 + role: server - uart_id: virtual_uart_controller - id: virtual_modbus_controller + id: virtual_modbus_client role: client turnaround_time: 10ms modbus_controller: - address: 1 - modbus_id: virtual_modbus_controller + modbus_id: virtual_modbus_client id: modbus_controller_1 update_interval: 1s + - address: 2 + modbus_id: virtual_modbus_client + id: modbus_controller_2 + update_interval: 1s + - address: 3 + modbus_id: virtual_modbus_client + id: modbus_controller_3 + update_interval: 1s modbus_server: - address: 1 modbus_id: virtual_modbus_server - id: modbus_server_1 registers: - address: 0x01 value_type: U_WORD @@ -103,6 +137,34 @@ modbus_server: - address: 0x28 value_type: FP32_R read_lambda: return 3.14; + - address: 5 + modbus_id: virtual_modbus_server + registers: + # Writable + readable register: srv_write_1 plus the client's read-back + # confirm the write half of the 0x17 ran before the read half (Modbus 6.17). + - address: 0x01 + value_type: U_WORD + read_lambda: return id(stored_1); + write_lambda: |- + id(stored_1) = x; + id(srv_write_1).publish_state(x); + return true; + # Read-only register, returned together with 0x01 by the 2-register read half. + - address: 0x02 + value_type: U_WORD + read_lambda: return 0x00AA; + - address: 2 + modbus_id: virtual_modbus_server_2 + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 919; + - address: 3 + modbus_id: virtual_modbus_server_2 + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 929; sensor: - platform: modbus_controller @@ -195,9 +257,46 @@ sensor: address: 0x28 register_type: holding value_type: FP32_R + - platform: modbus_controller + modbus_controller_id: modbus_controller_2 + name: "multi_reg_a" + address: 0x01 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_3 + name: "multi_reg_b" + address: 0x01 + register_type: holding + value_type: U_WORD + # client_read_write observations, server- and client-side. + - platform: template + name: "srv_write_1" + id: srv_write_1 + - platform: template + name: "client_read_0" + id: client_read_0 + - platform: template + name: "client_read_1" + id: client_read_1 button: - platform: template name: "Start Scenario" id: start_scenario_btn - # This test does not have anything to start (mock is autostart) + on_press: + # FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction. + - modbus_client.read_write_multiple_registers: + address: 5 + read_address: 0x0001 + read_count: 2 + write_address: 0x0001 + values: [0x1234] + on_response: + then: + - lambda: |- + // values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002. + if (values.size() >= 2) { + id(client_read_0).publish_state(values[0]); + id(client_read_1).publish_state(values[1]); + } diff --git a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml b/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml deleted file mode 100644 index 21c451aa99..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml +++ /dev/null @@ -1,138 +0,0 @@ -esphome: - name: uart-mock-modbus-reg-offset - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg10 - type: uint16_t - initial_value: "100" - - id: reg11 - type: uint16_t - initial_value: "200" - - id: reg12 - type: uint16_t - initial_value: "300" - - id: reg13 - type: uint16_t - initial_value: "0xABCD" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x10 - value_type: U_WORD - read_lambda: return id(reg10); - write_lambda: id(reg10) = x; return true; - - address: 0x11 - value_type: U_WORD - read_lambda: return id(reg11); - write_lambda: id(reg11) = x; return true; - - address: 0x12 - value_type: U_WORD - read_lambda: return id(reg12); - write_lambda: id(reg12) = x; return true; - - address: 0x13 - value_type: U_WORD - read_lambda: return id(reg13); - write_lambda: id(reg13) = x; return true; - -# A holding-register switch at 0x10 with a 2-BYTE offset. offset is byte-based, so the write must target -# register 0x10 + 2/2 = 0x11. The old (pre-fix) behavior folded offset into the address as a register -# count, hitting 0x12 instead. assumed_state keeps the switch write-only so it does not read any register. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "offset_switch" - register_type: holding - address: 0x10 - offset: 2 - assumed_state: true - # A holding-register switch that READS its state. Byte offset 6 -> register 0x10 + 6/2 = 0x13. Post-fix - # the switch itself resolves to 0x13 (the even byte offset folds into the address as whole registers) and - # joins the 0x10..0x13 range, so no separate 0x13 sensor is needed. Pre-fix the whole byte offset folds - # into the address (0x16), where the server answers ILLEGAL_DATA_ADDRESS and the switch never publishes. - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "read_offset_switch" - register_type: holding - address: 0x10 - offset: 6 - bitmask: 0x1 - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_10" - address: 0x10 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_11" - address: 0x11 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_12" - address: 0x12 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_server.yaml b/tests/integration/fixtures/uart_mock_modbus_server.yaml deleted file mode 100644 index cc5a59e242..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_server.yaml +++ /dev/null @@ -1,124 +0,0 @@ -esphome: - name: uart-mock-modbus-server-test - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_dev - baud_rate: 9600 - rx_full_threshold: 120 - rx_timeout: 2 - auto_start: false - debug: - injections: - - delay: 100ms - inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read) - - delay: 100ms - # Read holding register 7 on device 2 - # Reply from device 2 - # Read holding register 5 on device 1 (read_after_peer_response) - inject_rx: - [ - 0x02, - 0x03, - 0x00, - 0x07, - 0x00, - 0x01, - 0x35, - 0xF8, - 0x02, - 0x03, - 0x02, - 0x00, - 0xF0, - 0xFC, - 0x00, - 0x01, - 0x03, - 0x00, - 0x05, - 0x00, - 0x01, - 0x94, - 0x0B, - ] - - delay: 100ms - inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response - - delay: 100ms - # Read holding register 7 on device 2, with no response - # Read holding register A on device 1 (read_after_peer_timeout) - inject_rx: - [ - 0x02, - 0x03, - 0x00, - 0x07, - 0x00, - 0x01, - 0x35, - 0xF8, - 0x01, - 0x03, - 0x00, - 0x0A, - 0x00, - 0x01, - 0xA4, - 0x08, - ] - -modbus: - uart_id: virtual_uart_dev - role: server - -modbus_server: - - address: 1 - registers: - - address: 0x03 - value_type: U_WORD - read_lambda: |- - id(basic_read).publish_state(1); - return 1; - - address: 0x05 - value_type: U_WORD - read_lambda: |- - id(read_after_peer_response).publish_state(1); - return 1; - - address: 0x0A - value_type: U_WORD - read_lambda: |- - id(read_after_peer_timeout).publish_state(1); - return 1; - -sensor: - - platform: template - name: "basic_read" - id: basic_read - - platform: template - name: "read_after_peer_response" - id: read_after_peer_response - - platform: template - name: "read_after_peer_timeout" - id: read_after_peer_timeout - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - on_press: - - lambda: "id(virtual_uart_dev).start_scenario();" diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml b/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml deleted file mode 100644 index 18423be6d5..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml +++ /dev/null @@ -1,116 +0,0 @@ -esphome: - name: uart-mock-modbus-server-mult - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - # auto_start must be true for loopback fixtures: the modbus controller - # polls on its update_interval immediately at boot, so the uart_mock - # forwarding must already be active or early requests are lost and - # generate modbus warnings. - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_server_2 - data: !lambda return data; - - id: virtual_uart_server_2 - baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_server_2 - data: !lambda return data; - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_server_2 - id: virtual_modbus_server_2 - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_client - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_client - update_interval: 1s - id: modbus_controller_1 - - address: 2 - modbus_id: virtual_modbus_client - update_interval: 1s - id: modbus_controller_2 - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 919; - - address: 2 - modbus_id: virtual_modbus_server_2 - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 929; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_u_word" - address: 0x01 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_2 - name: "reg_u_word_2" - address: 0x01 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml b/tests/integration/fixtures/uart_mock_modbus_server_injected.yaml similarity index 52% rename from tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml rename to tests/integration/fixtures/uart_mock_modbus_server_injected.yaml index e998861c2d..2cd1c610f1 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_server_injected.yaml @@ -1,5 +1,5 @@ esphome: - name: uart-mock-modbus-srv-rw + name: uart-mock-modbus-srv-injected host: api: @@ -17,6 +17,8 @@ uart: baud_rate: 115200 port: /dev/null +# Shared server-role fixture (see the shared_yaml markers in the test file); +# the injections concatenate and each test waits only on its own sensors. uart_mock: - id: virtual_uart_dev baud_rate: 9600 @@ -25,18 +27,31 @@ uart_mock: auto_start: false debug: injections: - # FC 0x17 Read/Write Multiple Registers on device 1: - # write reg 0x0001 = 0x1234 (qty 1), then read regs 0x0001..0x0002 (qty 2). - # Per Modbus 6.17 the write is performed before the read, so reg 0x0001 must - # read back the just-written 0x1234 in the same request. + - delay: 100ms + inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read) + - delay: 100ms + # Read holding register 7 on device 2, its reply, then read holding + # register 5 on device 1 (read_after_peer_response) + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8, + 0x02, 0x03, 0x02, 0x00, 0xF0, 0xFC, + 0x00, 0x01, 0x03, 0x00, 0x05, 0x00, 0x01, 0x94, 0x0B] + - delay: 100ms + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response + - delay: 100ms + # Read holding register 7 on device 2 with no response, then read + # holding register A on device 1 (read_after_peer_timeout) + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8, + 0x01, 0x03, 0x00, 0x0A, 0x00, 0x01, 0xA4, 0x08] + # FC 0x17 on device 1: write reg 0x0001 = 0x1234 then read 0x0001..0x0002; + # per Modbus 6.17 the write runs first, so 0x0001 must read back 0x1234. - delay: 100ms inject_rx: [0x01, 0x17, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x02, 0x12, 0x34, 0x49, 0xD8] - # FC 0x17: write reg 0x0003 = 0x5678 (qty 1), then read reg 0x0003 (qty 1) - + # FC 0x17: write reg 0x0006 = 0x5678 (qty 1), then read reg 0x0006 (qty 1) - # a write and read targeting a different register block. - delay: 100ms inject_rx: - [0x01, 0x17, 0x00, 0x03, 0x00, 0x01, 0x00, 0x03, 0x00, 0x01, 0x02, 0x56, 0x78, 0x9B, 0x10] + [0x01, 0x17, 0x00, 0x06, 0x00, 0x01, 0x00, 0x06, 0x00, 0x01, 0x02, 0x56, 0x78, 0x8B, 0x55] globals: - id: stored_1 @@ -70,8 +85,18 @@ modbus_server: read_lambda: |- id(rw_read_2).publish_state(0x00AA); return 0x00AA; - # Second writable + readable register, targeted by the second request. - address: 0x03 + value_type: U_WORD + read_lambda: |- + id(basic_read).publish_state(1); + return 1; + - address: 0x05 + value_type: U_WORD + read_lambda: |- + id(read_after_peer_response).publish_state(1); + return 1; + # Second writable + readable register, targeted by the second FC 0x17 request. + - address: 0x06 value_type: U_WORD read_lambda: |- id(rw_read_3).publish_state(id(stored_3)); @@ -80,8 +105,22 @@ modbus_server: id(stored_3) = x; id(rw_write_3).publish_state(x); return true; + - address: 0x0A + value_type: U_WORD + read_lambda: |- + id(read_after_peer_timeout).publish_state(1); + return 1; sensor: + - platform: template + name: "basic_read" + id: basic_read + - platform: template + name: "read_after_peer_response" + id: read_after_peer_response + - platform: template + name: "read_after_peer_timeout" + id: read_after_peer_timeout - platform: template name: "rw_write_1" id: rw_write_1 diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index c7f21d8a01..5f526dce5f 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -1,7 +1,7 @@ """Helpers for manipulating the host platform's preferences file. ESPHome's host platform stores preferences in -``~/.esphome/prefs/.prefs`` using a simple binary layout that +``$ESPHOME_PREFDIR/.prefs`` using a simple binary layout that mirrors ``HostPreferences::sync()``: ``[uint32_t key][uint8_t len][uint8_t data[len]]`` per entry. @@ -11,13 +11,21 @@ boot (e.g. forcing safe mode) or to clear stale state between runs. from __future__ import annotations +import os from pathlib import Path import struct def host_prefs_path(device_name: str) -> Path: - """Return the on-disk prefs file path for a host-platform device.""" - return Path.home() / ".esphome" / "prefs" / f"{device_name}.prefs" + """Return the on-disk prefs file path for a host-platform device. + + Requires ESPHOME_PREFDIR, which the autouse isolated_preferences fixture + sets; refusing the ~/.esphome/prefs fallback keeps tests off real user + data if the fixture is ever bypassed.""" + prefdir = os.environ.get("ESPHOME_PREFDIR") + if not prefdir: + raise RuntimeError("ESPHOME_PREFDIR is not set; refusing the real prefs dir") + return Path(prefdir) / f"{device_name}.prefs" def clear_host_prefs(device_name: str) -> None: diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index f315335d1b..d103167a00 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -24,7 +24,6 @@ from .types import ( RunCompiledFunction, ) -pytestmark = pytest.mark.usefixtures("isolated_preferences") NEW_KEY = PROVISIONING_PSK diff --git a/tests/integration/test_host_preferences_suspend_resume.py b/tests/integration/test_host_preferences_suspend_resume.py index ab08d5c440..5f08d5519e 100644 --- a/tests/integration/test_host_preferences_suspend_resume.py +++ b/tests/integration/test_host_preferences_suspend_resume.py @@ -41,15 +41,6 @@ async def _poll_until_exists(path: Path) -> None: await asyncio.sleep(0.05) -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> Path: - """Keep host preferences per-test so this test never touches the real - ~/.esphome/prefs and never races other tests over ESPHOME_PREFDIR.""" - prefdir = tmp_path / "prefs" - monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir)) - return prefdir / f"{DEVICE_NAME}.prefs" - - @pytest.mark.asyncio async def test_host_preferences_suspend_resume( yaml_config: str, @@ -58,7 +49,7 @@ async def test_host_preferences_suspend_resume( isolated_preferences: Path, ) -> None: """Test that a running syncer flushes, a suspended one doesn't, and resume restores flushing.""" - pref_file = isolated_preferences + pref_file = isolated_preferences / f"{DEVICE_NAME}.prefs" loop = asyncio.get_running_loop() saved_in_memory = loop.create_future() diff --git a/tests/integration/test_light_initial_state.py b/tests/integration/test_light_initial_state.py index 657e273fe7..12ebf7c4a1 100644 --- a/tests/integration/test_light_initial_state.py +++ b/tests/integration/test_light_initial_state.py @@ -11,14 +11,6 @@ from .state_utils import InitialStateHelper, require_entity from .types import APIClientConnectedFactory, RunCompiledFunction -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so RESTORE_AND_ON never loads a stale value left - behind by a previous run (host preferences otherwise persist to ~/.esphome/prefs, - keyed only by device name).""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) - - @pytest.mark.asyncio async def test_light_initial_state( yaml_config: str, diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index 864275f5ed..232e1fb654 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -173,6 +173,7 @@ async def test_uart_mock_modbus_no_threshold( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_server_injected") @pytest.mark.asyncio async def test_uart_mock_modbus_server( yaml_config: str, @@ -203,6 +204,7 @@ async def test_uart_mock_modbus_server( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_server_injected") @pytest.mark.asyncio async def test_uart_mock_modbus_server_read_write( yaml_config: str, @@ -231,8 +233,8 @@ async def test_uart_mock_modbus_server_read_write( "rw_write_1": 4660, # 0x1234 written to reg 0x0001 "rw_read_1": 4660, # reg 0x0001 reads back the just-written value "rw_read_2": 170, # 0x00AA read from reg 0x0002 in the same request - "rw_write_3": 22136, # 0x5678 written to reg 0x0003 - "rw_read_3": 22136, # reg 0x0003 reads back the just-written value + "rw_write_3": 22136, # 0x5678 written to reg 0x0006 + "rw_read_3": 22136, # reg 0x0006 reads back the just-written value } ) @@ -241,7 +243,8 @@ async def test_uart_mock_modbus_server_read_write( api_client_connected() as client, ): await tracker.setup_and_start_scenario(client) - await tracker.await_all(futures) + # The FC 0x17 injections fire last, behind four earlier 100ms delays + await tracker.await_all(futures, timeout=4.0) _assert_no_modbus_errors(error_log_lines, warning_log_lines) @@ -296,6 +299,7 @@ async def test_uart_mock_modbus_server_read_write_invalid( ) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_server_controller( yaml_config: str, @@ -485,6 +489,7 @@ async def test_uart_mock_modbus_server_controller_bits( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_server_controller_multiple( yaml_config: str, @@ -495,7 +500,7 @@ async def test_uart_mock_modbus_server_controller_multiple( line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() - expected_values = {"reg_u_word": 919, "reg_u_word_2": 929} + expected_values = {"multi_reg_a": 919, "multi_reg_b": 929} tracker = SensorTracker(list(expected_values.keys())) futures = tracker.expect_all(expected_values) @@ -706,6 +711,7 @@ async def test_uart_mock_modbus_shared_address( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_custom_pdu( yaml_config: str, @@ -932,6 +938,7 @@ async def test_uart_mock_modbus_broadcast_write( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_client_read_write( yaml_config: str, @@ -947,9 +954,7 @@ async def test_uart_mock_modbus_client_read_write( """ line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() - tracker = SensorTracker( - ["srv_write_1", "srv_read_1", "client_read_0", "client_read_1"] - ) + tracker = SensorTracker(["srv_write_1", "client_read_0", "client_read_1"]) futures = tracker.expect_all( { "srv_write_1": 4660, # server wrote 0x1234 to reg 0x0001 @@ -967,6 +972,7 @@ async def test_uart_mock_modbus_client_read_write( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_register_offset( yaml_config: str, @@ -1022,6 +1028,7 @@ async def test_uart_mock_modbus_register_offset( ) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_lambda_write( yaml_config: str, @@ -1058,6 +1065,7 @@ async def test_uart_mock_modbus_lambda_write( await tracker.await_change(wrote_30, "reg_30", timeout=4.0) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_lambda_invert( yaml_config: str, @@ -1113,6 +1121,7 @@ async def test_uart_mock_modbus_lambda_invert( ) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_deprecated_write_buffer( yaml_config: str, diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 7d4059da2f..8f82a121c6 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -2122,6 +2122,34 @@ def test_get_cpp_changed_components_independent_of_cwd( ) == ["time"] +def test_fixture_map_includes_shared_yaml_markers() -> None: + """Fixtures named only by shared_yaml markers must map to their test file.""" + helpers.get_fixture_to_test_files.cache_clear() + mapping = helpers.get_fixture_to_test_files() + for fixture in ( + "uart_mock_modbus_loopback", + "uart_mock_modbus_mesh", + "uart_mock_modbus_server_injected", + ): + assert mapping[fixture] == frozenset( + {"tests/integration/test_uart_mock_modbus.py"} + ) + + +def test_no_orphan_integration_fixtures() -> None: + """Every fixture must reach CI test selection; an orphan selects nothing.""" + helpers.get_fixture_to_test_files.cache_clear() + mapping = helpers.get_fixture_to_test_files() + fixtures_dir = (Path(__file__).parent.parent / "integration" / "fixtures").resolve() + fixtures = list(fixtures_dir.glob("*.yaml")) + assert fixtures, f"no fixtures found under {fixtures_dir}" + # cache_init is covered via INTEGRATION_TESTS_TRIGGER_FILES instead + orphans = [ + f.stem for f in fixtures if f.stem != "cache_init" and f.stem not in mapping + ] + assert not orphans, f"fixtures invisible to CI test selection: {orphans}" + + def test_lpt_partition_balances_skewed_weights() -> None: """Heavy items spread across groups instead of clustering.""" items = [f"i{n}" for n in range(6)] From 3926612281789284df820f21f159f8cf1bb24969 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 15:46:17 -0400 Subject: [PATCH 067/178] [core] Fix use-after-free when deleting a running StaticTask (#19048) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../micro_wake_word/micro_wake_word.cpp | 4 +-- .../mixer/speaker/mixer_speaker.cpp | 4 +-- .../resampler/speaker/resampler_speaker.cpp | 4 +-- .../speaker/media_player/audio_pipeline.cpp | 11 +++++-- esphome/core/static_task.cpp | 30 ++++++++++++++----- esphome/core/static_task.h | 17 +++++++---- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 3dadb78077..cebfe8e791 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -446,9 +446,9 @@ void MicroWakeWord::loop() { xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STOPPING); } - if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & EventGroupBits::TASK_STOPPED) && this->inference_task_.deallocate()) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 6128dc3767..0b79010773 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -382,8 +382,8 @@ void MixerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, MIXER_TASK_STATE_STOPPING); } - if (event_group_bits & MIXER_TASK_STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS); this->all_stopped_since_ms_ = 0; diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index f1ebd180cc..edda00ae06 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -153,8 +153,8 @@ void ResamplerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::STATE_STOPPING); } - if (event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::ALL_BITS); } diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index 010f0c50b3..c286a9d7d6 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -202,8 +202,15 @@ AudioPipelineState AudioPipeline::process_state() { if (!this->is_playing_) { // The tasks have been stopped for two ``process_state`` calls in a row, so delete the tasks if (this->read_task_.is_created() || this->decode_task_.is_created()) { - this->read_task_.deallocate(); - this->decode_task_.deallocate(); + // Both are attempted every time; a task that is still running on the other core is freed by a + // subsequent call, and freeing an already freed task succeeds without doing anything + bool read_task_freed = this->read_task_.deallocate(); + bool decode_task_freed = this->decode_task_.deallocate(); + if (!read_task_freed || !decode_task_freed) { + // A task is still running on the other core, so keep the pipeline in its current state and try + // again on the next call + return AudioPipelineState::PLAYING; + } if (this->hard_stop_) { // Stop command was sent, so immediately end the playback this->speaker_->stop(); diff --git a/esphome/core/static_task.cpp b/esphome/core/static_task.cpp index 4cfead44c2..4301108315 100644 --- a/esphome/core/static_task.cpp +++ b/esphome/core/static_task.cpp @@ -40,16 +40,31 @@ bool StaticTask::create(TaskFunction_t fn, const char *name, uint32_t stack_size return true; } -void StaticTask::destroy() { - if (this->handle_ != nullptr) { - TaskHandle_t handle = this->handle_; - this->handle_ = nullptr; - vTaskDelete(handle); +bool StaticTask::destroy() { + if (this->handle_ == nullptr) { + return true; } + + // Suspending takes the task off the ready and event lists, so nothing can schedule it again. It only asks + // the other core to yield though, so the task may still be running on it for a moment. + vTaskSuspend(this->handle_); + if (eTaskGetState(this->handle_) != eSuspended) { + // The task is still running on the other core and using its stack. Deleting it now would only put it on + // the termination list and return, so the caller has to try again once it has been swapped out. + return false; + } + + // The task cannot run again, so the delete completes right away instead of being left to the idle task. + TaskHandle_t handle = this->handle_; + this->handle_ = nullptr; + vTaskDelete(handle); + return true; } -void StaticTask::deallocate() { - this->destroy(); +bool StaticTask::deallocate() { + if (!this->destroy()) { + return false; + } if (this->stack_buffer_ != nullptr) { RAMAllocator allocator(this->use_psram_ ? RAMAllocator::ALLOC_EXTERNAL : RAMAllocator::ALLOC_INTERNAL); @@ -57,6 +72,7 @@ void StaticTask::deallocate() { this->stack_buffer_ = nullptr; this->stack_size_ = 0; } + return true; } } // namespace esphome diff --git a/esphome/core/static_task.h b/esphome/core/static_task.h index 5fd5b38f9e..e2996abeda 100644 --- a/esphome/core/static_task.h +++ b/esphome/core/static_task.h @@ -11,6 +11,7 @@ namespace esphome { /** Helper for FreeRTOS static task management. * Bundles TaskHandle_t, StaticTask_t, and the stack buffer into one object with create/destroy methods. + * Call destroy() and deallocate() from another task: a task cannot free the stack it is still running on. */ class StaticTask { public: @@ -23,7 +24,7 @@ class StaticTask { /// @brief Allocate stack and create task. /// @param fn Task function /// @param name Task name (for debug) - /// @param stack_size Stack size in StackType_t words + /// @param stack_size Stack size in bytes (StackType_t is a byte on ESP-IDF) /// @param param Parameter passed to task function /// @param priority FreeRTOS task priority /// @param use_psram If true, allocate stack in PSRAM; otherwise internal RAM @@ -31,11 +32,17 @@ class StaticTask { bool create(TaskFunction_t fn, const char *name, uint32_t stack_size, void *param, UBaseType_t priority, bool use_psram); - /// @brief Delete the task but keep the stack buffer allocated for reuse by a subsequent create() call. - void destroy(); + /// @brief Delete the task, keeping the stack buffer allocated for reuse by a subsequent create() call. + /// The task must have finished its work and parked itself, either suspended or blocked indefinitely: it is + /// suspended here so that it cannot be scheduled again, and it is given no chance to clean up. + /// @return true if the task was deleted; false if it is still running on another core, in which case the + /// caller should try again later. + bool destroy(); - /// @brief Delete the task (if running) and free the stack buffer. - void deallocate(); + /// @brief Delete the task (if created) and free the stack buffer. + /// @return true if the stack buffer was freed; false if the task is still running on another core, in + /// which case the caller should try again later. + bool deallocate(); protected: TaskHandle_t handle_{nullptr}; From 4ab9298ab3eedddbd45507785b4d2453ae867bba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:11:36 +0200 Subject: [PATCH 068/178] Bump esptool from 5.3.1 to 5.4.0 (#19023) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd3f7446f3..dfddbed00b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ tzlocal==5.4.4 # from time tzdata>=2026.3 # from time pyserial==3.5 platformio==6.1.19 -esptool==5.3.1 +esptool==5.4.0 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi From 5bb112f407e8edac9576a7eea1eafb9d94cb1f47 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:04:34 -0400 Subject: [PATCH 069/178] [audio][i2s_audio][micro_wake_word][microphone][mixer][resampler][speaker] Replace use_count() checks with lock and null test (#19046) --- esphome/components/audio/audio_reader.cpp | 3 +++ esphome/components/audio/audio_transfer_buffer.cpp | 12 ++++++------ .../i2s_audio/speaker/i2s_audio_speaker.cpp | 4 ++-- .../components/micro_wake_word/micro_wake_word.cpp | 2 +- esphome/components/microphone/microphone_source.h | 2 +- esphome/components/mixer/speaker/mixer_speaker.cpp | 12 ++++++------ .../resampler/speaker/resampler_speaker.cpp | 6 +++--- .../speaker/media_player/audio_pipeline.cpp | 12 +++++++----- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/esphome/components/audio/audio_reader.cpp b/esphome/components/audio/audio_reader.cpp index 4678ed548c..e69f33ac2d 100644 --- a/esphome/components/audio/audio_reader.cpp +++ b/esphome/components/audio/audio_reader.cpp @@ -58,6 +58,9 @@ esp_err_t AudioReader::add_sink(const std::weak_ptr &ou if (current_audio_file_ != nullptr) { // A transfer buffer isn't ncessary for a local file this->file_ring_buffer_ = output_ring_buffer.lock(); + if (this->file_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; + } return ESP_OK; } diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index a611549e58..01fd4bb68a 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -51,14 +51,14 @@ void AudioTransferBuffer::increase_buffer_length(size_t bytes) { this->buffer_le void AudioTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } } void AudioSinkTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } #ifdef USE_SPEAKER @@ -69,7 +69,7 @@ void AudioSinkTransferBuffer::clear_buffered_data() { } bool AudioTransferBuffer::has_buffered_data() const { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); @@ -144,7 +144,7 @@ size_t AudioSourceTransferBuffer::transfer_data_from_source(TickType_t ticks_to_ size_t bytes_to_read = AudioTransferBuffer::free(); size_t bytes_read = 0; if (bytes_to_read > 0) { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_read = this->ring_buffer_->read((void *) this->get_buffer_end(), bytes_to_read, ticks_to_wait); } @@ -161,7 +161,7 @@ size_t AudioSinkTransferBuffer::transfer_data_to_sink(TickType_t ticks_to_wait, bytes_written = this->speaker_->play(this->data_start_, this->available(), ticks_to_wait); } else #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_written = this->ring_buffer_->write_without_replacement((void *) this->data_start_, this->available(), ticks_to_wait); } else if (this->sink_callback_ != nullptr) { @@ -186,7 +186,7 @@ bool AudioSinkTransferBuffer::has_buffered_data() const { return (this->speaker_->has_buffered_data() || (this->available() > 0)); } #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1c2eb12904..b78a151ee4 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -218,8 +218,8 @@ size_t I2SAudioSpeakerBase::play(const uint8_t *data, size_t length, TickType_t } bool I2SAudioSpeakerBase::has_buffered_data() const { - if (this->audio_ring_buffer_.use_count() > 0) { - std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + if (temp_ring_buffer != nullptr) { return temp_ring_buffer->available() > 0; } return false; diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index cebfe8e791..cf239be696 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -129,7 +129,7 @@ void MicroWakeWord::setup() { return; } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (this->ring_buffer_.use_count() > 1) { + if (temp_ring_buffer != nullptr) { // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task // to drain it - reset() is a consumer operation and must run on the inference task's thread. // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. diff --git a/esphome/components/microphone/microphone_source.h b/esphome/components/microphone/microphone_source.h index 7be3b8cdb5..d7a3352432 100644 --- a/esphome/components/microphone/microphone_source.h +++ b/esphome/components/microphone/microphone_source.h @@ -48,7 +48,7 @@ class MicrophoneSource final { template void add_data_callback(F &&data_callback) { this->mic_->add_data_callback([this, data_callback](const std::vector &data) { if (this->enabled_ || this->passive_) { - if (this->processed_samples_.use_count() == 0) { + if (this->processed_samples_ == nullptr) { // Create vector if its unused this->processed_samples_ = std::make_shared>(); } diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 0b79010773..ef21da65c5 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -218,7 +218,7 @@ size_t SourceSpeaker::play(const uint8_t *data, size_t length, TickType_t ticks_ } size_t bytes_written = 0; std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer.use_count() > 0) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); if (bytes_written > 0) { @@ -250,14 +250,14 @@ esp_err_t SourceSpeaker::start_() { // avoids unnecessary single-frame splices. const size_t ring_buffer_size = (this->audio_stream_info_.ms_to_bytes(this->buffer_duration_ms_) / bytes_per_frame) * bytes_per_frame; - if (this->audio_source_.use_count() == 0) { + if (this->audio_source_ == nullptr) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); this->ring_buffer_ = temp_ring_buffer; } - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { return ESP_ERR_NO_MEM; } @@ -278,7 +278,7 @@ void SourceSpeaker::stop() { this->send_command_(SOURCE_SPEAKER_COMMAND_STOP); } void SourceSpeaker::finish() { this->send_command_(SOURCE_SPEAKER_COMMAND_FINISH); } bool SourceSpeaker::has_buffered_data() const { - return ((this->audio_source_.use_count() > 0) && this->audio_source_->has_buffered_data()); + return ((this->audio_source_ != nullptr) && this->audio_source_->has_buffered_data()); } void SourceSpeaker::set_mute_state(bool mute_state) { @@ -496,7 +496,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (speaker->is_running() && !speaker->get_pause_state()) { // Speaker is running and not paused, so it possibly can provide audio data std::shared_ptr audio_source = speaker->get_audio_source().lock(); - if (audio_source.use_count() == 0) { + if (audio_source == nullptr) { // No audio source allocated, so skip processing this speaker continue; } diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index edda00ae06..16d2d5dc9e 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -235,7 +235,7 @@ size_t ResamplerSpeaker::play(const uint8_t *data, size_t length, TickType_t tic bytes_written = this->output_speaker_->play(data, length, ticks_to_wait); } else { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); } else { @@ -299,7 +299,7 @@ bool ResamplerSpeaker::has_buffered_data() const { bool has_ring_buffer_data = false; if (this->requires_resampling_()) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { has_ring_buffer_data = (temp_ring_buffer->available() > 0); } } @@ -342,7 +342,7 @@ void ResamplerSpeaker::resample_task(void *params) { std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( this_resampler->audio_stream_info_.ms_to_bytes(this_resampler->buffer_duration_ms_)); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { this_resampler->ring_buffer_ = temp_ring_buffer; diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index c286a9d7d6..509984cfa2 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -322,17 +322,17 @@ void AudioPipeline::read_task(void *params) { if (err == ESP_OK) { size_t file_ring_buffer_size = this_pipeline->buffer_size_; - std::shared_ptr temp_ring_buffer; + std::shared_ptr temp_ring_buffer = this_pipeline->raw_file_ring_buffer_.lock(); - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(file_ring_buffer_size); this_pipeline->raw_file_ring_buffer_ = temp_ring_buffer; } - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { - reader->add_sink(this_pipeline->raw_file_ring_buffer_); + err = reader->add_sink(temp_ring_buffer); } } @@ -403,7 +403,9 @@ void AudioPipeline::decode_task(void *params) { make_unique(this_pipeline->transfer_buffer_size_, this_pipeline->transfer_buffer_size_); esp_err_t err = decoder->start(this_pipeline->current_audio_file_type_); - decoder->add_source(this_pipeline->raw_file_ring_buffer_); + if (err == ESP_OK) { + err = decoder->add_source(this_pipeline->raw_file_ring_buffer_); + } if (err != ESP_OK) { // Send specific error message From 006f31af9308fd85212cec8b5a9816273608dbba Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:05:02 -0400 Subject: [PATCH 070/178] [i2s_audio] Fix spurious driver failure (#19045) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index b78a151ee4..1382a87046 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -118,21 +118,24 @@ void I2SAudioSpeakerBase::loop() { break; } + // Still starting up or winding down from a previous run + if ((this->tx_handle_ != nullptr) || (this->speaker_task_handle_ != nullptr)) { + break; + } + if (this->start_i2s_driver(this->audio_stream_info_) != ESP_OK) { ESP_LOGE(TAG, "Driver failed to start; retrying in 1 second"); this->status_momentary_error("driver-failure", 1000); break; } - if (this->speaker_task_handle_ == nullptr) { - xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, - &this->speaker_task_handle_); + xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, + &this->speaker_task_handle_); - if (this->speaker_task_handle_ == nullptr) { - ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); - this->status_momentary_error("task-failure", 1000); - this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt - } + if (this->speaker_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); + this->status_momentary_error("task-failure", 1000); + this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt } break; case speaker::STATE_RUNNING: // Intentional fallthrough From 8f511a365a471d1614e7578a03ceb3c0dbc4470f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:07:06 +0200 Subject: [PATCH 071/178] [noise] Bump noise-c to 0.1.26 and libsodium to 1.10021.8 (#19030) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 4de706120e..d17ebf235e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.24") + cg.add_library("esphome/noise-c", "0.1.26") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.6") + cg.add_library("esphome/libsodium", "1.10021.8") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 779a05e7de..738773d1b5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.24 ; used by noise (api, ota) + esphome/noise-c@0.1.26 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 4f7f5a4a4c..00f22ca138 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.24"] + assert libs == ["esphome/noise-c @ 0.1.26"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.24", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 0.1.26", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.24"] + assert cls.calls == ["esphome/noise-c @ 0.1.26"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.24"] is None + assert compats["esphome/noise-c @ 0.1.26"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 14c52dda8d..b03bff19a2 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 6c5ab89d5f818ac501855479ea776984c5d3f16a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:08:16 +0200 Subject: [PATCH 072/178] [esphome][core] Give a lost OTA chunk ack time to be retransmitted (#19041) --- esphome/components/esphome/ota/ota_esphome.cpp | 5 ++++- esphome/espota2.py | 9 ++++++--- tests/unit_tests/test_espota2.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 1005ed214b..f853ed6a2d 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -41,7 +41,10 @@ const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake -static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer +// Milliseconds for data transfer. Covers the lwIP retransmit run seen in +// practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits +// longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries +static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/espota2.py b/esphome/espota2.py index ce403c398d..c683ffa323 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -96,6 +96,10 @@ UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # across the addresses on top of that. EXTRA_UPLOAD_ATTEMPTS = 2 UPLOAD_RETRY_DELAY = 5.0 +# Data phase timeout; must stay longer than the device's OTA_SOCKET_TIMEOUT_DATA +# (105 s) so a stalled session is gone before a retry, and long enough for lwIP +# to get a lost chunk ack through after the retransmit run seen in practice +DATA_PHASE_TIMEOUT = 160.0 _LOGGER = logging.getLogger(__name__) @@ -694,8 +698,7 @@ def perform_ota( _LOGGER.info("Handshake complete") - # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures - sock.settimeout(90.0) + sock.settimeout(DATA_PHASE_TIMEOUT) if extended_proto: send_check(sock, ota_type, "ota type") @@ -854,7 +857,7 @@ def run_ota_impl_( # clean up a half-open connection (its handshake watchdog runs at 20s); # moving on to the next address family stays immediate. Known limitation: # a silent mid-transfer drop with no reset can wedge the device until its - # 90s data timeout, which outlasts this budget; the retries target the + # 105s data timeout, which outlasts this budget; the retries target the # common failures where the device resets or closes the link promptly. total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 8867e2c215..2d65e8e079 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -416,6 +416,9 @@ def test_perform_ota_no_auth( "Update took 14.00 seconds (prepare 2.00, upload 5.00, commit 7.00)" in caplog.text ) + # The data phase timeout must outlast the device's 105 s data timeout + mock_socket.settimeout.assert_any_call(espota2.DATA_PHASE_TIMEOUT) + assert espota2.DATA_PHASE_TIMEOUT > 105.0 @pytest.mark.usefixtures("mock_time") From b947094f45f7bc8b193db6a75b732c9bdbcce41b Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:31:18 -0400 Subject: [PATCH 073/178] [sendspin] Add codec preference list to the media source (#19047) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/sendspin/__init__.py | 26 ++++-- .../sendspin/media_source/__init__.py | 31 +++++++ .../sendspin/test_media_source.py | 90 +++++++++++++++++++ .../sendspin/common-media_source.yaml | 1 + 4 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 tests/component_tests/sendspin/test_media_source.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 570fd3fadd..8ef11a7f90 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -30,6 +30,7 @@ CONF_SENDSPIN_ID = "sendspin_id" CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" +CONF_CODECS = "codecs" # Matches ARTWORK_MAX_SLOTS in sendspin-cpp. MAX_ARTWORK_SLOTS = 4 @@ -44,6 +45,20 @@ CODEC_FORMAT_OPUS = SendspinCodecFormat.enum("OPUS") CODEC_FORMAT_PCM = SendspinCodecFormat.enum("PCM") CODEC_FORMAT_UNSUPPORTED = SendspinCodecFormat.enum("UNSUPPORTED") +CODEC_FLAC = "flac" +CODEC_OPUS = "opus" +CODEC_PCM = "pcm" + +CODECS = { + CODEC_FLAC: CODEC_FORMAT_FLAC, + CODEC_OPUS: CODEC_FORMAT_OPUS, + CODEC_PCM: CODEC_FORMAT_PCM, +} + +# Opus only supports 48 kHz audio, so it is left out of the default list at other rates. +DEFAULT_CODECS = [CODEC_FLAC, CODEC_OPUS, CODEC_PCM] +OPUS_SAMPLE_RATE = 48000 + SendspinImageFormat = sendspin_library_ns.enum("SendspinImageFormat", is_class=True) IMAGE_FORMAT_JPEG = SendspinImageFormat.enum("JPEG") IMAGE_FORMAT_PNG = SendspinImageFormat.enum("PNG") @@ -286,16 +301,13 @@ async def to_code(config: ConfigType) -> None: if data.player_support: cg.add_define("USE_SENDSPIN_PLAYER", True) - # Configures the player role. We always assume support for 16 bits per sample mono and stereo FLAC, Opus, and PCM at the configured sample rate - # (with Opus only supported at 48 kHz since that's the only sample rate it supports). Users can configure the specific formats via the Sendspin server + # Configures the player role. Each configured codec is advertised for 16 bits per sample + # mono and stereo at the configured sample rate. The order is a preference order, both for + # the codecs themselves and for stereo over mono. player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - # OPUS only supports 48 kHz audio - codecs = [CODEC_FORMAT_FLAC] - if sample_rate == 48000: - codecs.append(CODEC_FORMAT_OPUS) - codecs.append(CODEC_FORMAT_PCM) + codecs = player_cfg[CONF_CODECS] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index 6af244d41f..6a9f1f18ba 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -13,11 +13,16 @@ from esphome.cpp_generator import MockObj, TemplateArgsType from esphome.types import ConfigType from .. import ( + CODEC_OPUS, + CODECS, + CONF_CODECS, CONF_DECODE_MEMORY, CONF_FIXED_DELAY, CONF_INITIAL_STATIC_DELAY, CONF_SENDSPIN_ID, + DEFAULT_CODECS, MEMORY_LOCATIONS, + OPUS_SAMPLE_RATE, SendspinHub, register_player_config, request_controller_support, @@ -49,10 +54,32 @@ DisableStaticDelayAdjustmentAction = sendspin_ns.class_( ) +def _resolve_codecs(config: ConfigType) -> ConfigType: + """Validate the codec preference list, filling in the default when it is not set.""" + sample_rate = config[CONF_SAMPLE_RATE] + if (codecs := config.get(CONF_CODECS)) is None: + config[CONF_CODECS] = [ + codec + for codec in DEFAULT_CODECS + if codec != CODEC_OPUS or sample_rate == OPUS_SAMPLE_RATE + ] + return config + + if len(set(codecs)) != len(codecs): + raise cv.Invalid("Each codec may only be listed once", path=[CONF_CODECS]) + if CODEC_OPUS in codecs and sample_rate != OPUS_SAMPLE_RATE: + raise cv.Invalid( + f"Codec '{CODEC_OPUS}' requires a {CONF_SAMPLE_RATE} of {OPUS_SAMPLE_RATE}", + path=[CONF_CODECS], + ) + return config + + def _register(config: ConfigType) -> ConfigType: request_controller_support() register_player_config( { + CONF_CODECS: config[CONF_CODECS], CONF_SAMPLE_RATE: config[CONF_SAMPLE_RATE], CONF_BUFFER_SIZE: config[CONF_BUFFER_SIZE], CONF_INITIAL_STATIC_DELAY: config[CONF_INITIAL_STATIC_DELAY], @@ -85,9 +112,13 @@ CONFIG_SCHEMA = cv.All( min=16000, max=96000 ), cv.Optional(CONF_DECODE_MEMORY): cv.one_of(*MEMORY_LOCATIONS, lower=True), + cv.Optional(CONF_CODECS): cv.All( + cv.ensure_list(cv.enum(CODECS, lower=True)), cv.Length(min=1) + ), } ), cv.only_on_esp32, + _resolve_codecs, _register, ) diff --git a/tests/component_tests/sendspin/test_media_source.py b/tests/component_tests/sendspin/test_media_source.py new file mode 100644 index 0000000000..6c2f79198d --- /dev/null +++ b/tests/component_tests/sendspin/test_media_source.py @@ -0,0 +1,90 @@ +"""Validation tests for the sendspin media_source platform. + +These cover the codec preference list, whose rejection branches a compile test +cannot reach: a `test*.yaml` can only assert that a configuration is accepted. +""" + +from typing import Any + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import CONF_CODECS, _get_data +from esphome.components.sendspin.media_source import CONFIG_SCHEMA +from esphome.const import PlatformFramework +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _media_source_config(**overrides: Any) -> ConfigType: + """Build a minimal valid media source config, allowing field overrides.""" + config: ConfigType = { + "id": "sendspin_media_source", + "sendspin_id": "sendspin_hub", + } + config.update(overrides) + return config + + +def test_default_codecs_at_48_khz(set_core_config: SetCoreConfigCallable) -> None: + """Every codec is advertised when the sample rate suits all of them.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config()) + + assert config[CONF_CODECS] == ["flac", "opus", "pcm"] + + +def test_default_codecs_drop_opus_at_other_rates( + set_core_config: SetCoreConfigCallable, +) -> None: + """Opus only supports 48 kHz, so it leaves the default list at other rates.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config(sample_rate=44100)) + + assert config[CONF_CODECS] == ["flac", "pcm"] + + +def test_configured_order_is_preserved(set_core_config: SetCoreConfigCallable) -> None: + """The list is a preference order, so it reaches the player role as written.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA(_media_source_config(codecs=["pcm", "flac"])) + + assert _get_data().player_config[CONF_CODECS] == ["pcm", "flac"] + + +def test_empty_codec_list_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A player with no codecs at all could never be given a stream.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="length of value must be at least 1"): + CONFIG_SCHEMA(_media_source_config(codecs=[])) + + +def test_duplicate_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A repeated codec has no meaning in a preference order.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="may only be listed once"): + CONFIG_SCHEMA(_media_source_config(codecs=["flac", "flac"])) + + +def test_unknown_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """Only codecs the player role can decode are accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="Unknown value"): + CONFIG_SCHEMA(_media_source_config(codecs=["mp3"])) + + +def test_opus_at_wrong_sample_rate_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + """Asking for Opus at a rate it cannot handle fails rather than silently + dropping the stated preference.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="requires a sample_rate of 48000"): + CONFIG_SCHEMA(_media_source_config(codecs=["opus"], sample_rate=44100)) diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 1977b79c04..0c136fbd43 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,3 +9,4 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal + codecs: [pcm, opus, flac] From 823d79c948eb4474423200d5a251210c31482b68 Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:10:55 +0200 Subject: [PATCH 074/178] [i2s_audio] Keep a start request that arrives while the speaker task stops (#19027) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../components/i2s_audio/speaker/i2s_audio_speaker.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 5e271e671e..1c2eb12904 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -91,7 +91,14 @@ void I2SAudioSpeakerBase::loop() { this->speaker_task_handle_ = nullptr; this->stop_i2s_driver_(); - xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + // ALL_BITS includes COMMAND_START. Take the bits from the clear itself, not from the snapshot at + // the top of loop(): the audio source's task can raise a start at any point above, including + // during stop_i2s_driver_(), and nothing would ever re-issue it. + const EventBits_t bits_before_clear = xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + if (bits_before_clear & SpeakerEventGroupBits::COMMAND_START) { + ESP_LOGD(TAG, "Start requested while stopping; keeping the request"); + xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); + } this->status_clear_error(); this->on_task_stopped(); From 628ebe23ec389d770e822f18de22753c167dff6f Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:13:03 +0200 Subject: [PATCH 075/178] [audio] Do not treat MP3_STREAM_INFO_CHANGED as a fatal decoder error (#19028) Co-authored-by: Claude --- esphome/components/audio/audio_decoder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index fe9ad9c9ad..051395606c 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -313,9 +313,10 @@ FileDecoderState AudioDecoder::decode_mp3_() { this->output_transfer_buffer_->increase_buffer_length( this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); } - } else if (result == micro_mp3::MP3_STREAM_INFO_READY) { - // First successful header parse: capture stream info and resize the output buffer to fit one full frame. - // microMP3 always outputs 16-bit PCM. + } else if (result == micro_mp3::MP3_STREAM_INFO_READY || result == micro_mp3::MP3_STREAM_INFO_CHANGED) { + // Header parsed: capture stream info and resize the output buffer to fit one full frame. + // microMP3 always outputs 16-bit PCM. MP3_STREAM_INFO_CHANGED is handled identically: despite its + // negative value it is documented as recoverable, so it must not reach the catch-all below. this->audio_stream_info_ = audio::AudioStreamInfo(16, this->mp3_decoder_->get_channels(), this->mp3_decoder_->get_sample_rate()); this->free_buffer_required_ = From e7f45a0d315442dcf789997d6e28de72f082e28a Mon Sep 17 00:00:00 2001 From: Ryan Ronnander <61520+ryan-ronnander@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:09:02 -0400 Subject: [PATCH 076/178] [mqtt] Restore brightness flag in light discovery (#18950) --- esphome/components/mqtt/mqtt_light.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/mqtt/mqtt_light.cpp b/esphome/components/mqtt/mqtt_light.cpp index aa47bdf996..a8b52a3839 100644 --- a/esphome/components/mqtt/mqtt_light.cpp +++ b/esphome/components/mqtt/mqtt_light.cpp @@ -67,6 +67,9 @@ void MQTTJSONLightComponent::send_discovery(JsonObject root, mqtt::SendDiscovery if (traits.supports_color_mode(ColorMode::RGB_COLD_WARM_WHITE)) color_modes.add(ESPHOME_F("rgbww")); + if (traits.supports_color_capability(ColorCapability::BRIGHTNESS)) + root[ESPHOME_F("brightness")] = true; + if (traits.supports_color_mode(ColorMode::COLOR_TEMPERATURE) || traits.supports_color_mode(ColorMode::COLD_WARM_WHITE)) { root[MQTT_MIN_MIREDS] = traits.get_min_mireds(); From d5cff6e9dfcdfce156483eecf205e64169a56dee Mon Sep 17 00:00:00 2001 From: AndreKR Date: Tue, 8 Sep 2026 03:13:51 +0200 Subject: [PATCH 077/178] [logger] Fix garbled stack traces (#17939) --- esphome/components/logger/logger_esp32.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index 05fc959ceb..c3d777299d 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -5,6 +5,7 @@ #include #include +#include #ifdef USE_LOGGER_UART_SELECTION_USB_SERIAL_JTAG #include @@ -76,7 +77,11 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { uart_config.parity = UART_PARITY_DISABLE; uart_config.stop_bits = UART_STOP_BITS_1; uart_config.flow_ctrl = UART_HW_FLOWCTRL_DISABLE; +#if SOC_UART_SUPPORT_XTAL_CLK + uart_config.source_clk = UART_SCLK_XTAL; +#else uart_config.source_clk = UART_SCLK_DEFAULT; +#endif uart_param_config(uart_num, &uart_config); // The logger only writes to UART, never reads, so use the minimum RX buffer. // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). From 934086365217965f95c21bda0593b3f2ec960615 Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Mon, 7 Sep 2026 18:27:49 -0700 Subject: [PATCH 078/178] [dallas_temp] filter 85 temp from sensor reset (#17877) Co-authored-by: Samuel Sieb --- esphome/components/dallas_temp/dallas_temp.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esphome/components/dallas_temp/dallas_temp.cpp b/esphome/components/dallas_temp/dallas_temp.cpp index ab4a8c458f..c418362ced 100644 --- a/esphome/components/dallas_temp/dallas_temp.cpp +++ b/esphome/components/dallas_temp/dallas_temp.cpp @@ -6,6 +6,7 @@ namespace esphome::dallas_temp { static const char *const TAG = "dallas.temp.sensor"; static const uint8_t DALLAS_MODEL_DS18S20 = 0x10; +static const uint8_t DALLAS_MODEL_DS18B20 = 0x28; static const uint8_t DALLAS_COMMAND_START_CONVERSION = 0x44; static const uint8_t DALLAS_COMMAND_READ_SCRATCH_PAD = 0xBE; static const uint8_t DALLAS_COMMAND_WRITE_SCRATCH_PAD = 0x4E; @@ -154,7 +155,14 @@ float DallasTemperatureSensor::get_temp_c_() { default: break; } - + // undocumented test for powerup measurement of 85 + // https://github.com/cpetrich/counterfeit_DS18B20#solution-to-the-85-c-problem + if ((this->address_ & 0xff) == DALLAS_MODEL_DS18B20) { + if ((temp == 85 * 16) && (this->scratch_pad_[6] == 0xc)) { + ESP_LOGD(TAG, "dropping reading caused by sensor reset"); + return NAN; + } + } return temp / 16.0f; } From 199acdf5222a923d5c7951af2e0ea632f45ba220 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 7 Sep 2026 18:57:50 -0700 Subject: [PATCH 079/178] [ble_client] Report Established from nodes that never read services (#17920) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/ble_client/automation.h | 34 +++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/esphome/components/ble_client/automation.h b/esphome/components/ble_client/automation.h index 94eeb83b3e..93aae23b6a 100644 --- a/esphome/components/ble_client/automation.h +++ b/esphome/components/ble_client/automation.h @@ -22,6 +22,23 @@ class Automation { static const char *const TAG; }; +// Base for nodes that never read the parent's services. +// The parent releases its services only once every node reports Established, so a node that never +// reports it keeps that memory allocated for the life of the connection. +class BLEClientServicelessNode : public BLEClientNode { + public: + // Final so that Established is always reported on SEARCH_CMPL, before the derived node sees the event. + void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) final { + if (event == ESP_GATTC_SEARCH_CMPL_EVT) + this->node_state = espbt::ClientState::ESTABLISHED; + this->on_gattc_event(event, gattc_if, param); + } + + protected: + // Derived nodes handle GATT events here rather than by overriding the handler above. + virtual void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) {} +}; + // implement on_connect automation. class BLEClientConnectTrigger final : public Trigger<>, public BLEClientNode { public: @@ -61,7 +78,7 @@ class BLEClientDisconnectTrigger final : public Trigger<>, public BLEClientNode } }; -class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientNode { +class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientServicelessNode { public: explicit BLEClientPasskeyRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -71,7 +88,7 @@ class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientN } }; -class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientNode { +class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientPasskeyNotificationTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -82,7 +99,7 @@ class BLEClientPasskeyNotificationTrigger final : public Trigger, publ } }; -class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientNode { +class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientNumericComparisonRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -315,19 +332,17 @@ template class BLEClientRemoveBondAction final : public Action class BLEClientConnectAction final : public Action, public BLEClientNode { +template class BLEClientConnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientConnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { case ESP_GATTC_SEARCH_CMPL_EVT: - this->node_state = espbt::ClientState::ESTABLISHED; this->parent()->run_later([this]() { this->play_next_tuple_(this->var_); }); break; // if the connection is closed, terminate the automation chain. @@ -364,14 +379,13 @@ template class BLEClientConnectAction final : public Action var_{}; }; -template class BLEClientDisconnectAction final : public Action, public BLEClientNode { +template class BLEClientDisconnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientDisconnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { From 442e4a1ec2c70bfc8507aa1cf5f471402806d9b0 Mon Sep 17 00:00:00 2001 From: Davide D M Date: Tue, 8 Sep 2026 03:59:05 +0200 Subject: [PATCH 080/178] [debug] Check reboot source pref on ESP_RST_WDT and guard against empty source (#17537) --- esphome/components/debug/debug_esp32.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esphome/components/debug/debug_esp32.cpp b/esphome/components/debug/debug_esp32.cpp index 969cd840cf..8e1a67224e 100644 --- a/esphome/components/debug/debug_esp32.cpp +++ b/esphome/components/debug/debug_esp32.cpp @@ -66,11 +66,15 @@ const char *DebugComponent::get_reset_reason_(std::spanmake_preference(REBOOT_MAX_LEN, fnv1_hash_extend(fnv1_hash(REBOOT_KEY), App.get_name().c_str())); char reboot_source[REBOOT_MAX_LEN]{}; - if (pref.load(&reboot_source)) { + if (pref.load(&reboot_source) && reboot_source[0] != '\0') { reboot_source[REBOOT_MAX_LEN - 1] = '\0'; snprintf(buf, size, "Reboot request from %s", reboot_source); } else { From c9729244af79e5b36a2b712c2fa5b91efa6a504f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:17:22 +1200 Subject: [PATCH 081/178] [udp] Use cv.invalid for relocated packet_transport options (#19032) --- esphome/components/udp/__init__.py | 16 +++------ tests/unit_tests/components/udp/__init__.py | 0 tests/unit_tests/components/udp/test_init.py | 37 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 tests/unit_tests/components/udp/__init__.py create mode 100644 tests/unit_tests/components/udp/test_init.py diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index a782d875b9..d96a731e9c 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -1,5 +1,4 @@ -from collections.abc import Callable -from typing import Any, NoReturn +from typing import Any from esphome import automation from esphome.automation import Trigger @@ -48,17 +47,10 @@ UDP_SCHEMA = cv.Schema( ) -def is_relocated(option: str) -> Callable[[Any], NoReturn]: - def validator(value: Any) -> NoReturn: - raise cv.Invalid( - f"The '{option}' option should now be configured in the 'packet_transport' component" - ) - - return validator - - RELOCATED = { - cv.Optional(x): is_relocated(x) + cv.Optional(x): cv.invalid( + f"The '{x}' option should now be configured in the 'packet_transport' component" + ) for x in ( CONF_PROVIDERS, CONF_ENCRYPTION, diff --git a/tests/unit_tests/components/udp/__init__.py b/tests/unit_tests/components/udp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/udp/test_init.py b/tests/unit_tests/components/udp/test_init.py new file mode 100644 index 0000000000..5afc92e9c6 --- /dev/null +++ b/tests/unit_tests/components/udp/test_init.py @@ -0,0 +1,37 @@ +"""Tests for the udp component configuration schema.""" + +from __future__ import annotations + +import pytest + +from esphome.components import udp +from esphome.components.packet_transport import ( + CONF_BINARY_SENSORS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_PROVIDERS, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, +) +import esphome.config_validation as cv + + +@pytest.mark.parametrize( + "option", + [ + CONF_PROVIDERS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, + CONF_BINARY_SENSORS, + ], +) +def test_relocated_option_rejected(option: str) -> None: + """Options that moved to packet_transport raise a pointing error.""" + with pytest.raises(cv.Invalid) as exc_info: + udp.CONFIG_SCHEMA({option: True}) + assert ( + f"The '{option}' option should now be configured in the 'packet_transport' component" + in str(exc_info.value) + ) From ca864c22b4c0810e9e4779bfa6d95b3597cc8abe Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:36:04 -0400 Subject: [PATCH 082/178] [tuya] Build without a network component (#18948) --- esphome/components/tuya/tuya.cpp | 17 +++++++++-- .../tuya/test-no-network.bk72xx-ard.yaml | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/components/tuya/test-no-network.bk72xx-ard.yaml diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index 82fb96d787..f9b4fe2453 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -1,10 +1,13 @@ #include "tuya.h" -#include "esphome/components/network/util.h" #include "esphome/core/gpio.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/util.h" +#ifdef USE_NETWORK +#include "esphome/components/network/util.h" +#endif + #ifdef USE_WIFI #include "esphome/components/wifi/wifi_component.h" #endif @@ -22,6 +25,14 @@ static const int MAX_RETRIES = 5; // Max bytes to log for datapoint values (larger values are truncated) static constexpr size_t MAX_DATAPOINT_LOG_BYTES = 16; +static bool network_is_connected() { +#ifdef USE_NETWORK + return network::is_connected(); +#else + return false; +#endif +} + void Tuya::setup() { this->set_interval("heartbeat", 15000, [this] { this->send_empty_command_(TuyaCommandType::HEARTBEAT); }); if (this->status_pin_ != nullptr) { @@ -554,14 +565,14 @@ void Tuya::send_empty_command_(TuyaCommandType command) { } void Tuya::set_status_pin_() { - bool is_network_ready = network::is_connected() && remote_is_connected(); + bool is_network_ready = network_is_connected() && remote_is_connected(); this->status_pin_->digital_write(is_network_ready); } uint8_t Tuya::get_wifi_status_code_() { uint8_t status = 0x02; - if (network::is_connected()) { + if (network_is_connected()) { status = 0x03; // Protocol version 3 also supports specifying when connected to "the cloud" diff --git a/tests/components/tuya/test-no-network.bk72xx-ard.yaml b/tests/components/tuya/test-no-network.bk72xx-ard.yaml new file mode 100644 index 0000000000..64207e94e3 --- /dev/null +++ b/tests/components/tuya/test-no-network.bk72xx-ard.yaml @@ -0,0 +1,29 @@ +# Tuya without any network component (no wifi/ethernet/api), as used on +# serial-only or BLE-only Tuya MCU boards. Regression test for +# https://github.com/esphome/esphome/issues/18942 +substitutions: + status_pin: P6 + +packages: + uart: !include ../../test_build_components/common/uart/bk72xx-ard.yaml + +tuya: + status_pin: ${status_pin} + +binary_sensor: + - platform: tuya + id: tuya_presence + sensor_datapoint: 101 + +sensor: + - platform: tuya + id: tuya_light_intensity + sensor_datapoint: 103 + +number: + - platform: tuya + id: tuya_far_detection + number_datapoint: 109 + min_value: 0 + max_value: 600 + step: 1 From 866ddb6e5729f11e2a3a107fa7145dc596b9a17a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 06:42:30 +0200 Subject: [PATCH 083/178] [core] Skip PlatformIO's private-package authorization probe (#18823) --- esphome/platformio/library.py | 6 ++- esphome/platformio/prefetch.py | 2 + esphome/platformio/runner.py | 14 ++++++- tests/unit_tests/test_platformio_library.py | 19 ++++++++++ tests/unit_tests/test_platformio_prefetch.py | 14 +++++++ tests/unit_tests/test_platformio_runner.py | 40 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 3ff60f8aaa..fb6779b807 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -616,11 +616,15 @@ def _make_registry_client() -> Any: elsewhere, not by the PlatformIO registry. """ from platformio.package.manager._registry import PackageManagerRegistryMixin + from platformio.registry.client import RegistryClient class _Registry(PackageManagerRegistryMixin): def __init__(self) -> None: - self._registry_client = None self.pkg_type = "library" + self._registry_client = RegistryClient() + # The probe sleeps ~500 ms per lookup (see runner.patch_registry_private_packages); + # instance-level so the ESPHome process never patches PlatformIO's class + self._registry_client.allowed_private_packages = lambda: False @staticmethod def is_system_compatible(value: Any, custom_system: Any = None) -> bool: diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 17a06cb9c1..e648192b73 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -951,8 +951,10 @@ def main(argv: list[str]) -> int: """Subprocess entry point: ``prefetch ``.""" from esphome.core import CORE from esphome.log import setup_log + from esphome.platformio.runner import patch_registry_private_packages signal.signal(signal.SIGTERM, _sigterm) + patch_registry_private_packages() raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") try: level = int(raw_level) if raw_level is not None else logging.INFO diff --git a/esphome/platformio/runner.py b/esphome/platformio/runner.py index 9bb2205a90..b9fbdec38d 100644 --- a/esphome/platformio/runner.py +++ b/esphome/platformio/runner.py @@ -2,7 +2,8 @@ Invoked via ``python -m esphome.platformio.runner`` instead of ``python -m platformio`` so that the patches (incremental rebuild -preservation, download retries) apply inside the subprocess. Running +preservation, download retries, skipping the private-package probe) apply +inside the subprocess. Running PlatformIO in a subprocess keeps its ``sys.path`` mutations and other global state from leaking into the ESPHome process. """ @@ -105,6 +106,16 @@ def patch_file_downloader() -> None: FileDownloader.__init__ = patched_init +def patch_registry_private_packages() -> None: + """Skip PlatformIO's private-package probe; it sleeps ~500 ms per lookup. + + ESPHome never uses private packages, so the answer is always False. + """ + from platformio.registry.client import RegistryClient + + RegistryClient.allowed_private_packages = staticmethod(lambda: False) # type: ignore[method-assign] + + _IGNORE_LIB_WARNINGS = "(?:Hash|Update)" # Regex patterns matched against each line of PlatformIO output. Lines that # match are dropped by RedirectText before they reach the parent process. @@ -152,6 +163,7 @@ FILTER_PLATFORMIO_LINES = [ def main() -> int: patch_structhash() patch_file_downloader() + patch_registry_private_packages() # Wrap stdout/stderr with RedirectText before PlatformIO runs: # diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 3bae39b3c1..512c883c37 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -7,6 +7,7 @@ exercised in their own test modules).""" import json import logging from pathlib import Path +from unittest.mock import Mock import pytest @@ -228,6 +229,24 @@ def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): _resolve_registry_version("owner", "pkg", set()) +def test_make_registry_client_skips_private_package_probe(monkeypatch): + """Our client answers the probe locally without patching PlatformIO's class.""" + from platformio.account.client import AccountClient + from platformio.registry.client import RegistryClient + + pio_probe = RegistryClient.__dict__["allowed_private_packages"] + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + client = lib._make_registry_client().get_registry_client_instance() + + assert client.allowed_private_packages() is False + assert RegistryClient.__dict__["allowed_private_packages"] is pio_probe + + def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: """Stub the registry lookup so tests never touch the network.""" monkeypatch.setattr( diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 77490fd861..14c52dda8d 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1225,6 +1225,20 @@ def test_main_runs_prefetch(tmp_path: Path) -> None: mock_prefetch.assert_called_once_with(tmp_path, "testenv") +def test_main_skips_private_package_probe_before_prefetch(tmp_path: Path) -> None: + """The registry probe patch is applied before any package manager runs.""" + order: list[str] = [] + with ( + patch.object(pf, "_prefetch", side_effect=lambda *_: order.append("prefetch")), + patch( + "esphome.platformio.runner.patch_registry_private_packages", + side_effect=lambda: order.append("patch"), + ), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + assert order == ["patch", "prefetch"] + + def test_main_bad_argv_is_a_distinct_exit( caplog: pytest.LogCaptureFixture, ) -> None: diff --git a/tests/unit_tests/test_platformio_runner.py b/tests/unit_tests/test_platformio_runner.py index f375aa457a..007455f45a 100644 --- a/tests/unit_tests/test_platformio_runner.py +++ b/tests/unit_tests/test_platformio_runner.py @@ -6,7 +6,9 @@ from collections.abc import Callable import io import sys from types import ModuleType +from unittest.mock import Mock +from platformio.registry.client import RegistryClient import pytest from esphome.platformio import runner @@ -30,6 +32,7 @@ def _prepare_main( monkeypatch.setattr(sys, "stderr", stream) monkeypatch.setattr(runner, "patch_structhash", lambda: None) monkeypatch.setattr(runner, "patch_file_downloader", lambda: None) + monkeypatch.setattr(runner, "patch_registry_private_packages", lambda: None) platformio = ModuleType("platformio") platformio_main = ModuleType("platformio.__main__") @@ -91,3 +94,40 @@ def test_main_still_filters_a_drained_partial_line( assert runner.main() == 0 assert buf.getvalue() == b"" + + +def test_main_applies_registry_private_packages_patch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The probe is patched before PlatformIO runs.""" + order: list[str] = [] + _prepare_main(monkeypatch, lambda: order.append("pio") or 0) + monkeypatch.setattr( + runner, "patch_registry_private_packages", lambda: order.append("patch") + ) + + assert runner.main() == 0 + assert order == ["patch", "pio"] + + +# Snapshot PlatformIO's own probe at import, before any test can patch it +_PIO_PROBE = RegistryClient.__dict__["allowed_private_packages"] + + +def test_patch_registry_private_packages_skips_account_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Answers False without touching the account client.""" + from platformio.account.client import AccountClient + + monkeypatch.setattr(RegistryClient, "allowed_private_packages", _PIO_PROBE) + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + runner.patch_registry_private_packages() + + assert RegistryClient.allowed_private_packages() is False + assert RegistryClient().allowed_private_packages() is False From f8a4cfa945ef765e469daa03edbe263346a03154 Mon Sep 17 00:00:00 2001 From: Gytis Date: Tue, 8 Sep 2026 08:31:30 +0200 Subject: [PATCH 084/178] [lvgl] Add missing label dependency to qrcode, keyboard and tabview (#18387) --- esphome/components/lvgl/widgets/keyboard.py | 3 +- esphome/components/lvgl/widgets/qrcode.py | 3 +- esphome/components/lvgl/widgets/tabview.py | 3 +- .../lvgl/config/keyboard_no_label.yaml | 32 +++++++++++++++++ .../lvgl/config/qrcode_no_label.yaml | 34 ++++++++++++++++++ .../lvgl/config/tabview_no_label.yaml | 35 +++++++++++++++++++ .../lvgl/test_widget_label_dependency.py | 32 +++++++++++++++++ 7 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/config/keyboard_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/qrcode_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/tabview_no_label.yaml create mode 100644 tests/component_tests/lvgl/test_widget_label_dependency.py diff --git a/esphome/components/lvgl/widgets/keyboard.py b/esphome/components/lvgl/widgets/keyboard.py index bcd2d2ae59..65516513a6 100644 --- a/esphome/components/lvgl/widgets/keyboard.py +++ b/esphome/components/lvgl/widgets/keyboard.py @@ -15,6 +15,7 @@ from ..defines import ( from ..types import LvCompound, LvType from . import Widget, WidgetType, get_widgets from .buttonmatrix import CONF_BUTTONMATRIX +from .label import CONF_LABEL from .textarea import CONF_TEXTAREA, lv_textarea_t CONF_KEYBOARD = "keyboard" @@ -49,7 +50,7 @@ class KeyboardType(WidgetType): ) def get_uses(self): - return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX + return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX, CONF_LABEL async def to_code(self, w: Widget, config: dict): add_lv_use("KEY_LISTENER") diff --git a/esphome/components/lvgl/widgets/qrcode.py b/esphome/components/lvgl/widgets/qrcode.py index df76ab6bb0..59af9168aa 100644 --- a/esphome/components/lvgl/widgets/qrcode.py +++ b/esphome/components/lvgl/widgets/qrcode.py @@ -10,6 +10,7 @@ from ..types import lv_obj_t from . import Widget, WidgetType from .canvas import CONF_CANVAS from .img import CONF_IMAGE +from .label import CONF_LABEL CONF_QRCODE = "qrcode" CONF_DARK_COLOR = "dark_color" @@ -41,7 +42,7 @@ class QrCodeType(WidgetType): ) def get_uses(self): - return CONF_CANVAS, CONF_IMAGE + return CONF_CANVAS, CONF_IMAGE, CONF_LABEL async def to_code(self, w: Widget, config): await w.set_property( diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index ee252ecf0b..77c88c48ff 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -28,6 +28,7 @@ from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t, lv_obj_t_ptr from . import Widget, WidgetType, add_widgets, get_widgets, set_obj_properties from .button import button_spec from .buttonmatrix import CONF_BUTTONMATRIX, buttonmatrix_spec +from .label import CONF_LABEL from .obj import obj_spec CONF_TABVIEW = "tabview" @@ -74,7 +75,7 @@ class TabviewType(WidgetType): ) def get_uses(self): - return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON + return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON, CONF_LABEL async def to_code(self, w: Widget, config: dict): await w.set_property( diff --git a/tests/component_tests/lvgl/config/keyboard_no_label.yaml b/tests/component_tests/lvgl/config/keyboard_no_label.yaml new file mode 100644 index 0000000000..7a45a537d3 --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_no_label.yaml @@ -0,0 +1,32 @@ +esphome: + name: test-keyboard-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - keyboard: + id: keyboard_widget diff --git a/tests/component_tests/lvgl/config/qrcode_no_label.yaml b/tests/component_tests/lvgl/config/qrcode_no_label.yaml new file mode 100644 index 0000000000..8bb1aafdd6 --- /dev/null +++ b/tests/component_tests/lvgl/config/qrcode_no_label.yaml @@ -0,0 +1,34 @@ +esphome: + name: test-qrcode-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - qrcode: + id: qr_widget + size: 100 + text: "esphome.io" diff --git a/tests/component_tests/lvgl/config/tabview_no_label.yaml b/tests/component_tests/lvgl/config/tabview_no_label.yaml new file mode 100644 index 0000000000..a3c16ab347 --- /dev/null +++ b/tests/component_tests/lvgl/config/tabview_no_label.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-tabview-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - tabview: + id: tabview_widget + tabs: + - name: "Tab 1" + id: tab_1 diff --git a/tests/component_tests/lvgl/test_widget_label_dependency.py b/tests/component_tests/lvgl/test_widget_label_dependency.py new file mode 100644 index 0000000000..9d3e24c8c5 --- /dev/null +++ b/tests/component_tests/lvgl/test_widget_label_dependency.py @@ -0,0 +1,32 @@ +"""Widgets whose LVGL C implementation creates or references labels +internally (tab titles, key legends, the QR canvas fallback) must declare +the label dependency in ``get_uses()``. Otherwise a config that contains +no ``label`` widget of its own compiles LVGL without ``LV_USE_LABEL`` and +fails at C compile time with undefined ``lv_label_*`` symbols. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.lvgl import defines as df + + +@pytest.mark.parametrize( + "yaml_file", + [ + "qrcode_no_label.yaml", + "keyboard_no_label.yaml", + "tabview_no_label.yaml", + ], +) +def test_label_less_config_enables_lv_use_label( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + yaml_file: str, +) -> None: + generate_main(component_config_path(yaml_file)) + assert "LV_USE_LABEL" in df.get_defines() From c3ce07755f32292af3da6466aa1fc4a2cfeca07d Mon Sep 17 00:00:00 2001 From: Bryan Li Date: Mon, 7 Sep 2026 23:36:42 -0700 Subject: [PATCH 085/178] [rf_bridge] Fix bucket sniffing with Portisch firmware (#17683) Co-authored-by: Bryan Li Co-authored-by: Claude Fable 5 --- esphome/components/rf_bridge/rf_bridge.cpp | 109 +++++++++++++++++---- esphome/components/rf_bridge/rf_bridge.h | 13 +++ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/esphome/components/rf_bridge/rf_bridge.cpp b/esphome/components/rf_bridge/rf_bridge.cpp index 549cce72df..a4a4da5d8c 100644 --- a/esphome/components/rf_bridge/rf_bridge.cpp +++ b/esphome/components/rf_bridge/rf_bridge.cpp @@ -18,6 +18,16 @@ void RFBridgeComponent::ack_() { } bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { + if (this->bucket_frame_candidate_ && byte == RF_CODE_START) { + // A queued next frame proves the trailing 0x55 really was the bucket + // frame's terminator: Portisch builds pulse entries from alternating + // signal edges, so the two level bits inside one pulse byte are always + // opposite — 0xAA (two high-level nibbles) cannot occur in pulse data. + // Finalize before this byte starts the new frame, so back-to-back + // deliveries are split even when loop() never observed a quiet gap + // between them. + this->finish_bucket_frame_(); + } size_t at = this->rx_buffer_.size(); this->rx_buffer_.push_back(byte); const uint8_t *raw = &this->rx_buffer_[0]; @@ -84,26 +94,21 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { break; } case RF_CODE_RFIN_BUCKET: { - if (byte != RF_CODE_STOP) { - return true; + if (at == 2) { + // The count byte: Portisch sends at most 7 buckets + sync, so 0 or + // >8 cannot be a genuine capture — reject before it can occupy the + // buffer for a full frame timeout. + return byte != 0 && byte <= B1_MAX_BUCKET_COUNT; } - - uint8_t buckets = raw[2] << 1; - std::string str; - char next_byte[3]; // 2 hex chars + null - - for (uint32_t i = 0; i <= at; i++) { - buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); - str += next_byte; - if ((i > 3) && buckets) { - buckets--; - } - if ((i < 3) || (buckets % 2) || (i == at - 1)) { - str += " "; - } - } - ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); - break; + // 0x55 is legal DATA inside a B1 frame: bucket durations are sent + // with only their HIGH byte masked to 7 bits, so a duration such as + // 0x0155 puts a raw 0x55 low byte inside the table — the first 0x55 + // must therefore not end the capture. The header declares the table + // length (raw[2] pairs), so a 0x55 there is always data; one at or + // past the first pulse index is a terminator CANDIDATE, confirmed + // once the UART goes quiet (finish_bucket_frame_ in loop()). + this->bucket_frame_candidate_ = byte == RF_CODE_STOP && at >= 3 + static_cast(raw[2]) * 2; + return true; } default: ESP_LOGW(TAG, "Unknown action: 0x%02X", action); @@ -119,6 +124,47 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { return false; } +void RFBridgeComponent::finish_bucket_frame_() { + if (this->rx_buffer_.size() < 4) { + // The candidate flag requires a header + non-empty bucket table, so + // this cannot happen while flag and buffer stay consistent; guard the + // raw[2] / size-1 reads against any future divergence anyway. + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + return; + } + const uint8_t *raw = this->rx_buffer_.data(); + const size_t at = this->rx_buffer_.size() - 1; + + uint8_t buckets = raw[2] << 1; + std::string str; + char next_byte[3]; // 2 hex chars + null + + for (uint32_t i = 0; i <= at; i++) { + buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); + str += next_byte; + if ((i > 3) && buckets) { + buckets--; + } + if ((i < 3) || (buckets % 2) || (i == at - 1)) { + str += " "; + } + } + ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); + + // Deliberately NOT ACKed: Portisch's B1 command handler leaves its + // last_sniffing_command at the previous mode (RF_CODE_RFIN), and its + // host-ACK handler re-arms sniffing from that stale value — so ACKing a + // bucket delivery silently reverts the radio to standard sniffing and + // ends bucket capture. Its delivery path is fire-and-forget and never + // waits for a host ACK. Stock Itead firmware never sends B1 frames, so + // suppressing this ACK cannot change stock-firmware behavior. + // https://github.com/esphome/esphome/issues/17682 + + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; +} + void RFBridgeComponent::write_byte_str_(const std::string &codes) { uint8_t code; int size = codes.length(); @@ -130,12 +176,31 @@ void RFBridgeComponent::write_byte_str_(const std::string &codes) { void RFBridgeComponent::loop() { const uint32_t now = App.get_loop_component_start_time(); - if (now - this->last_bridge_byte_ > 50) { + size_t avail = this->available(); + if (avail == 0 && this->bucket_frame_candidate_ && now - this->last_bridge_byte_ > BUCKET_CANDIDATE_QUIET_MS) { + // The trailing 0x55 was followed by UART quiet, so it really was the + // frame terminator and not an interior data byte. + this->finish_bucket_frame_(); + this->last_bridge_byte_ = now; + } + const bool receiving_bucket = this->rx_buffer_.size() >= 2 && this->rx_buffer_[1] == RF_CODE_RFIN_BUCKET; + if (receiving_bucket) { + // Never declare an in-progress bucket frame dead while its continuation + // bytes are already queued: a stalled loop() otherwise discards a live + // frame that the UART buffer proves is still arriving. + if (avail == 0 && now - this->last_bridge_byte_ > BUCKET_FRAME_TIMEOUT_MS) { + ESP_LOGD(TAG, "Discarding incomplete RFBridge Bucket frame (%u bytes)", + static_cast(this->rx_buffer_.size())); + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + this->last_bridge_byte_ = now; + } + } else if (now - this->last_bridge_byte_ > 50) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; this->last_bridge_byte_ = now; } - size_t avail = this->available(); while (avail > 0) { uint8_t buf[64]; size_t to_read = std::min(avail, sizeof(buf)); @@ -146,12 +211,14 @@ void RFBridgeComponent::loop() { for (size_t i = 0; i < to_read; i++) { if (this->rx_buffer_.size() > MAX_RX_BUFFER_SIZE) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } if (this->parse_bridge_byte_(buf[i])) { ESP_LOGVV(TAG, "Parsed: 0x%02X", buf[i]); this->last_bridge_byte_ = now; } else { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } } } diff --git a/esphome/components/rf_bridge/rf_bridge.h b/esphome/components/rf_bridge/rf_bridge.h index 5ad75650ab..cbb1880ec5 100644 --- a/esphome/components/rf_bridge/rf_bridge.h +++ b/esphome/components/rf_bridge/rf_bridge.h @@ -30,6 +30,17 @@ static const uint8_t RF_CODE_BEEP = 0xC0; static const uint8_t RF_CODE_STOP = 0x55; static const uint8_t RF_DEBOUNCE = 200; static const size_t MAX_RX_BUFFER_SIZE = 512; +// ~10 byte times at 19200 baud: long enough to prove the UART went quiet +// after a possible bucket-frame terminator, short enough to finish well +// before the next radio capture can be delivered. +static const uint32_t BUCKET_CANDIDATE_QUIET_MS = 5; +// Portisch drains a B1 frame's header, bucket table, and pulse data as +// separate UART writes, so an in-progress bucket frame tolerates a longer +// inter-region gap than the generic 50 ms inter-byte timeout. +static const uint32_t BUCKET_FRAME_TIMEOUT_MS = 250; +// Portisch's uart_put_RF_buckets sends at most 7 buckets plus the sync +// bucket, so a B1 count byte above 8 (or 0) is malformed for any protocol. +static const uint8_t B1_MAX_BUCKET_COUNT = 8; struct RFBridgeData { uint16_t sync; @@ -67,10 +78,12 @@ class RFBridgeComponent final : public uart::UARTDevice, public Component { void ack_(); void decode_(); bool parse_bridge_byte_(uint8_t byte); + void finish_bucket_frame_(); void write_byte_str_(const std::string &codes); std::vector rx_buffer_; uint32_t last_bridge_byte_{0}; + bool bucket_frame_candidate_{false}; CallbackManager data_callback_; CallbackManager advanced_data_callback_; From 7660dd7fa7059a6154e65797c26e45d27aa8bf78 Mon Sep 17 00:00:00 2001 From: raykholo Date: Tue, 8 Sep 2026 02:57:33 -0400 Subject: [PATCH 086/178] [anova] Re-assert temperature unit on every poll cycle (#17141) --- esphome/components/anova/anova.cpp | 107 ++++++++++++++--------------- esphome/components/anova/anova.h | 13 +++- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/esphome/components/anova/anova.cpp b/esphome/components/anova/anova.cpp index 6e382872e2..b0769bb622 100644 --- a/esphome/components/anova/anova.cpp +++ b/esphome/components/anova/anova.cpp @@ -13,7 +13,7 @@ void Anova::dump_config() { LOG_CLIMATE("", "Anova BLE Cooker", this); } void Anova::setup() { this->codec_ = make_unique(); - this->current_request_ = 0; + this->poll_step_ = PollStep::IDLE; } void Anova::loop() { @@ -22,6 +22,15 @@ void Anova::loop() { this->disable_loop(); } +void Anova::write_request_(AnovaPacket *pkt) { + auto status = + esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, + pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); + if (status) { + ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } +} + void Anova::control(const ClimateCall &call) { auto mode_val = call.get_mode(); if (mode_val.has_value()) { @@ -38,22 +47,11 @@ void Anova::control(const ClimateCall &call) { ESP_LOGW(TAG, "Unsupported mode: %d", mode); return; } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(pkt); } auto target_temp = call.get_target_temperature(); if (target_temp.has_value()) { - auto *pkt = this->codec_->get_set_target_temp_request(*target_temp); - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(this->codec_->get_set_target_temp_request(*target_temp)); } } @@ -62,6 +60,7 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ case ESP_GATTC_DISCONNECT_EVT: { this->current_temperature = NAN; this->target_temperature = NAN; + this->poll_step_ = PollStep::IDLE; this->publish_state(); break; } @@ -83,8 +82,8 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } case ESP_GATTC_REG_FOR_NOTIFY_EVT: { this->node_state = espbt::ClientState::ESTABLISHED; - this->current_request_ = 0; - this->update(); + this->poll_step_ = PollStep::IDLE; + this->update(); // begin the first poll cycle immediately break; } case ESP_GATTC_NOTIFY_EVT: { @@ -101,33 +100,30 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ this->mode = this->codec_->running_ ? climate::CLIMATE_MODE_HEAT : climate::CLIMATE_MODE_OFF; } if (this->codec_->has_unit()) { - this->fahrenheit_ = (this->codec_->unit_ == 'f'); - ESP_LOGD(TAG, "Anova units is %s", this->fahrenheit_ ? "fahrenheit" : "celsius"); - this->current_request_++; + ESP_LOGD(TAG, "Anova units is %s", (this->codec_->unit_ == 'f') ? "fahrenheit" : "celsius"); } this->publish_state(); - if (this->current_request_ > 1) { - AnovaPacket *pkt = nullptr; - switch (this->current_request_++) { - case 2: - pkt = this->codec_->get_read_target_temp_request(); - break; - case 3: - pkt = this->codec_->get_read_current_temp_request(); - break; - default: - this->current_request_ = 1; - break; - } - if (pkt != nullptr) { - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - } + // Advance the poll cycle to its next request based on the reply we got. + switch (this->poll_step_) { + case PollStep::SET_UNIT: + this->poll_step_ = PollStep::STATUS; + this->write_request_(this->codec_->get_read_device_status_request()); + break; + case PollStep::STATUS: + this->poll_step_ = PollStep::TARGET; + this->write_request_(this->codec_->get_read_target_temp_request()); + break; + case PollStep::TARGET: + this->poll_step_ = PollStep::CURRENT; + this->write_request_(this->codec_->get_read_current_temp_request()); + break; + case PollStep::CURRENT: + this->poll_step_ = PollStep::IDLE; // full cycle complete + break; + default: + // A reply to an ad-hoc control() write, outside a managed cycle. + break; } break; } @@ -136,27 +132,26 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } } -void Anova::set_unit_of_measurement(const char *unit) { this->fahrenheit_ = !strncmp(unit, "f", 1); } +void Anova::set_unit_of_measurement(const char *unit) { this->want_fahrenheit_ = !strncmp(unit, "f", 1); } void Anova::update() { if (this->node_state != espbt::ClientState::ESTABLISHED) return; - - if (this->current_request_ < 2) { - AnovaPacket *pkt; - if (this->current_request_ == 0) { - pkt = this->codec_->get_set_unit_request(this->fahrenheit_ ? 'f' : 'c'); - } else { - pkt = this->codec_->get_read_device_status_request(); - } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - this->current_request_++; + if (this->poll_step_ != PollStep::IDLE) { + // The previous cycle never finished within a full polling interval -- a + // reply was missed or a write failed. Restart the cycle rather than stall; + // the polling interval itself acts as the timeout. A late reply from the + // abandoned cycle is harmless: state decoding happens on every notify + // regardless of step, and each notify sends at most one follow-up request. + ESP_LOGW(TAG, "[%s] Poll cycle incomplete (step %u); restarting cycle", this->parent_->address_str(), + static_cast(this->poll_step_)); } + // Re-assert the configured unit at the start of every poll cycle, then fall + // through the status/temperature reads via the notification handler. Always + // command the configured unit (want_fahrenheit_) -- never the last value the + // device reported, or a drift to 'c' would lock itself in. + this->poll_step_ = PollStep::SET_UNIT; + this->write_request_(this->codec_->get_set_unit_request(this->want_fahrenheit_ ? 'f' : 'c')); } } // namespace esphome::anova diff --git a/esphome/components/anova/anova.h b/esphome/components/anova/anova.h index 49b1100c37..a0fa03df01 100644 --- a/esphome/components/anova/anova.h +++ b/esphome/components/anova/anova.h @@ -37,11 +37,20 @@ class Anova final : public climate::Climate, public esphome::ble_client::BLEClie void set_unit_of_measurement(const char *unit); protected: + // A poll cycle re-asserts the configured unit, then reads device state. + // Re-asserting every cycle prevents the cooker from silently reverting to + // its default (Celsius); previously the unit was only set once on + // connection, so a drift persisted (and corrupted the F/C interpretation of + // subsequent readings) until the BLE link was re-established. + enum class PollStep : uint8_t { SET_UNIT, STATUS, TARGET, CURRENT, IDLE }; + + void write_request_(AnovaPacket *pkt); + std::unique_ptr codec_; void control(const climate::ClimateCall &call) override; uint16_t char_handle_; - uint8_t current_request_; - bool fahrenheit_; + bool want_fahrenheit_{true}; // configured target unit; never overwritten by device replies + PollStep poll_step_{PollStep::IDLE}; }; } // namespace esphome::anova From f8b2e53609051bf6ac9a626305a7e6304c67393f Mon Sep 17 00:00:00 2001 From: John <34163498+CircuitSetup@users.noreply.github.com> Date: Tue, 8 Sep 2026 04:02:49 -0400 Subject: [PATCH 087/178] [atm90e32] Verify offset calibration writes (#18701) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/atm90e32/atm90e32.cpp | 360 ++++++++++-------- esphome/components/atm90e32/atm90e32.h | 71 ++-- tests/components/atm90e32/__init__.py | 5 + .../offset_register_verification_test.cpp | 62 +++ 4 files changed, 322 insertions(+), 176 deletions(-) create mode 100644 tests/components/atm90e32/__init__.py create mode 100644 tests/components/atm90e32/offset_register_verification_test.cpp diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp index d948b3741d..23701e7834 100644 --- a/esphome/components/atm90e32/atm90e32.cpp +++ b/esphome/components/atm90e32/atm90e32.cpp @@ -9,6 +9,10 @@ namespace esphome::atm90e32 { static const char *const TAG = "atm90e32"; +static const LogString *offset_calibration_name(bool power_offsets) { + return power_offsets ? LOG_STR("Power offset") : LOG_STR("Offset"); +} + static uint32_t pref_hash(const char *prefix, const char *name_space) { auto hash = fnv1_hash(prefix); return fnv1_hash_extend(hash, name_space); @@ -203,13 +207,12 @@ void ATM90E32Component::setup() { // Initialize flash storage for power offset calibrations uint32_t po_hash = pref_hash("_power_offset_calibration_", cs); - this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); + this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); bool migrated_power_offset = false; if (has_distinct_legacy_namespace) { uint32_t legacy_po_hash = pref_hash("_power_offset_calibration_", legacy_cs); - auto legacy_power_offset_pref = - global_preferences->make_preference(legacy_po_hash, true); - PowerOffsetCalibration power_offset_data[3]{}; + auto legacy_power_offset_pref = global_preferences->make_preference(legacy_po_hash, true); + OffsetCalibration power_offset_data[3]{}; int migration_status = migrate_legacy_pref_if_needed(this->power_offset_pref_, legacy_power_offset_pref, &power_offset_data); migrated_power_offset = migration_status > 0; @@ -224,20 +227,20 @@ void ATM90E32Component::setup() { global_preferences->sync(); } - this->restore_offset_calibrations_(); - this->restore_power_offset_calibrations_(); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } else { ESP_LOGI(TAG, "[CALIBRATION][%s] Power & Voltage/Current offset calibration is disabled. Using config file values.", cs); for (uint8_t phase = 0; phase < 3; ++phase) { this->write16_(this->voltage_offset_registers[phase], - static_cast(this->offset_phase_[phase].voltage_offset_)); + static_cast(this->offset_phase_[phase].first_offset)); this->write16_(this->current_offset_registers[phase], - static_cast(this->offset_phase_[phase].current_offset_)); + static_cast(this->offset_phase_[phase].second_offset)); this->write16_(this->power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].active_power_offset)); + static_cast(this->power_offset_phase_[phase].first_offset)); this->write16_(this->reactive_power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].reactive_power_offset)); + static_cast(this->power_offset_phase_[phase].second_offset)); } } @@ -317,8 +320,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].voltage_offset_, - this->config_offset_phase_[phase].current_offset_, this->offset_phase_[phase].current_offset_); + this->config_offset_phase_[phase].first_offset, this->offset_phase_[phase].first_offset, + this->config_offset_phase_[phase].second_offset, this->offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -335,10 +338,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].active_power_offset, - this->config_power_offset_phase_[phase].reactive_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->config_power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].first_offset, + this->config_power_offset_phase_[phase].second_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -372,7 +373,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\\n", cs); } @@ -385,8 +386,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); } @@ -756,36 +756,68 @@ void ATM90E32Component::save_gain_calibration_to_memory_() { } } -void ATM90E32Component::save_offset_calibration_to_memory_() { +void ATM90E32Component::finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); - bool success = this->offset_pref_.save(&this->offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_offset_calibration_ = true; - for (bool &phase : this->offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save offset calibration to memory!", cs); - } -} + const LogString *name = offset_calibration_name(power_offsets); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; -void ATM90E32Component::save_power_offset_calibration_to_memory_() { - const char *cs = this->get_calibration_id_(); - bool success = this->power_offset_pref_.save(&this->power_offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_power_offset_calibration_ = true; - for (bool &phase : this->power_offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Power offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save power offset calibration to memory!", cs); + const bool writes_verified = this->verify_offset_writes_(type); + bool saved = false; + bool synced = false; + if (writes_verified) { + saved = preference->save(offsets); + synced = global_preferences->sync(); } + + if (writes_verified && saved && synced) { + this->using_saved_calibrations_ = true; + *has_stored = true; + *restored = true; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration saved to memory. %s calibration completed and verified.", cs, + LOG_STR_ARG(name), LOG_STR_ARG(name)); + return; + } + + if (writes_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save %s calibration to memory!", cs, LOG_STR_ARG(name)); + } + + for (uint8_t phase = 0; phase < 3; phase++) { + this->write_offsets_to_registers_(phase, previous[phase].first_offset, previous[phase].second_offset, type); + } + const bool rollback_verified = this->verify_offset_writes_(type); + + bool rollback_persisted = false; + if (writes_verified) { + OffsetCalibration rollback[3]{}; + prepare_offset_rollback(previous, previous_restored, rollback); + const bool rollback_saved = preference->save(&rollback); + const bool rollback_synced = global_preferences->sync(); + rollback_persisted = rollback_saved && rollback_synced; + if (!rollback_saved || !rollback_synced) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to persist restored %s calibration values!", cs, LOG_STR_ARG(name)); + } + } + + *restored = previous_restored; + if (rollback_persisted) + *has_stored = previous_restored; + this->using_saved_calibrations_ = previous_using_saved; + if (!rollback_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; rollback readback verification failed.", cs, + LOG_STR_ARG(name)); + return; + } + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; previous values restored.", cs, LOG_STR_ARG(name)); } void ATM90E32Component::run_offset_calibrations() { @@ -803,11 +835,16 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->offset_phase_[0], this->offset_phase_[1], this->offset_phase_[2]}; + const bool previous_restored = this->restored_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = calibrate_offset(phase, true); int16_t current_offset = calibrate_offset(phase, false); - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); @@ -815,7 +852,8 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] ==================================================================\n", cs); - this->save_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); } void ATM90E32Component::run_power_offset_calibrations() { @@ -834,18 +872,25 @@ void ATM90E32Component::run_power_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->power_offset_phase_[0], this->power_offset_phase_[1], + this->power_offset_phase_[2]}; + const bool previous_restored = this->restored_power_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; ++phase) { int16_t active_offset = calibrate_power_offset(phase, false); int16_t reactive_offset = calibrate_power_offset(phase, true); - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - this->save_power_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } void ATM90E32Component::write_gains_to_registers_() { @@ -859,35 +904,26 @@ void ATM90E32Component::write_gains_to_registers_() { this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } -void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset) { - // Save to runtime - this->offset_phase_[phase].voltage_offset_ = voltage_offset; - this->phase_[phase].voltage_offset_ = voltage_offset; +void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + OffsetCalibration &offsets = power_offsets ? this->power_offset_phase_[phase] : this->offset_phase_[phase]; + offsets.first_offset = first_offset; + offsets.second_offset = second_offset; + if (power_offsets) { + this->phase_[phase].active_power_offset_ = first_offset; + this->phase_[phase].reactive_power_offset_ = second_offset; + } else { + this->phase_[phase].voltage_offset_ = first_offset; + this->phase_[phase].current_offset_ = second_offset; + } - // Save to flash-storable struct - this->offset_phase_[phase].current_offset_ = current_offset; - this->phase_[phase].current_offset_ = current_offset; - - // Write to registers + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(voltage_offset_registers[phase], static_cast(voltage_offset)); - this->write16_(current_offset_registers[phase], static_cast(current_offset)); - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); -} - -void ATM90E32Component::write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset) { - // Save to runtime - this->phase_[phase].active_power_offset_ = p_offset; - this->phase_[phase].reactive_power_offset_ = q_offset; - - // Save to flash-storable struct - this->power_offset_phase_[phase].active_power_offset = p_offset; - this->power_offset_phase_[phase].reactive_power_offset = q_offset; - - // Write to registers - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(this->power_offset_registers[phase], static_cast(p_offset)); - this->write16_(this->reactive_power_offset_registers[phase], static_cast(q_offset)); + this->write16_(first_registers[phase], static_cast(first_offset)); + this->write16_(second_registers[phase], static_cast(second_offset)); this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } @@ -947,89 +983,78 @@ void ATM90E32Component::restore_gain_calibrations_() { ESP_LOGW(TAG, "[CALIBRATION][%s] No stored gain calibrations found. Using config file values.", cs); } -void ATM90E32Component::restore_offset_calibrations_() { +void ATM90E32Component::restore_offset_calibrations_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); + const LogString *name = power_offsets ? LOG_STR("power offset") : LOG_STR("offset"); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + OffsetCalibration(*config_offsets)[3] = + power_offsets ? &this->config_power_offset_phase_ : &this->config_offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; + const bool *has_first = power_offsets ? this->has_config_active_power_offset_ : this->has_config_voltage_offset_; + const bool *has_second = power_offsets ? this->has_config_reactive_power_offset_ : this->has_config_current_offset_; + for (uint8_t i = 0; i < 3; ++i) - this->config_offset_phase_[i] = this->offset_phase_[i]; - - bool have_data = this->offset_pref_.load(&this->offset_phase_); + (*config_offsets)[i] = (*offsets)[i]; + const bool have_data = preference->load(offsets); bool all_zero = true; if (have_data) { - for (auto &phase : this->offset_phase_) { - if (phase.voltage_offset_ != 0 || phase.current_offset_ != 0) { + for (const auto &phase : *offsets) { + if (phase.first_offset != 0 || phase.second_offset != 0) { all_zero = false; break; } } } - if (have_data && !all_zero) { - this->restored_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; phase++) { - auto &offset = this->offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_voltage_offset_[phase] && - offset.voltage_offset_ != this->config_offset_phase_[phase].voltage_offset_) - mismatch = true; - if (this->has_config_current_offset_[phase] && - offset.current_offset_ != this->config_offset_phase_[phase].current_offset_) - mismatch = true; - if (mismatch) - this->offset_calibration_mismatch_[phase] = true; + *has_stored = have_data && !all_zero; + *restored = false; + for (uint8_t phase = 0; phase < 3; phase++) { + mismatches[phase] = false; + if (*has_stored) { + mismatches[phase] = + (has_first[phase] && (*offsets)[phase].first_offset != (*config_offsets)[phase].first_offset) || + (has_second[phase] && (*offsets)[phase].second_offset != (*config_offsets)[phase].second_offset); } - } else { + } + + if (!*has_stored) { for (uint8_t phase = 0; phase < 3; phase++) - this->offset_phase_[phase] = this->config_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored offset calibrations found. Using default values.", cs); + (*offsets)[phase] = (*config_offsets)[phase]; + ESP_LOGW(TAG, "[CALIBRATION][%s] No stored %s calibrations found. Using default values.", cs, LOG_STR_ARG(name)); } for (uint8_t phase = 0; phase < 3; phase++) { - write_offsets_to_registers_(phase, this->offset_phase_[phase].voltage_offset_, - this->offset_phase_[phase].current_offset_); + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); } -} - -void ATM90E32Component::restore_power_offset_calibrations_() { - const char *cs = this->get_calibration_id_(); - for (uint8_t i = 0; i < 3; ++i) - this->config_power_offset_phase_[i] = this->power_offset_phase_[i]; - - bool have_data = this->power_offset_pref_.load(&this->power_offset_phase_); - - bool all_zero = true; - if (have_data) { - for (auto &phase : this->power_offset_phase_) { - if (phase.active_power_offset != 0 || phase.reactive_power_offset != 0) { - all_zero = false; - break; - } - } + const bool initial_values_verified = this->verify_offset_writes_(type); + if (initial_values_verified) { + const auto state = resolve_offset_restore_state(*has_stored, true, false); + *restored = state.restored; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration values verified.", cs, LOG_STR_ARG(name)); + return; } - if (have_data && !all_zero) { - this->restored_power_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; ++phase) { - auto &offset = this->power_offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_active_power_offset_[phase] && - offset.active_power_offset != this->config_power_offset_phase_[phase].active_power_offset) - mismatch = true; - if (this->has_config_reactive_power_offset_[phase] && - offset.reactive_power_offset != this->config_power_offset_phase_[phase].reactive_power_offset) - mismatch = true; - if (mismatch) - this->power_offset_calibration_mismatch_[phase] = true; - } + this->using_saved_calibrations_ = false; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + for (uint8_t phase = 0; phase < 3; phase++) { + (*offsets)[phase] = (*config_offsets)[phase]; + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); + } + const auto state = resolve_offset_restore_state(*has_stored, false, this->verify_offset_writes_(type)); + *restored = state.restored; + if (state.values_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore failed verification; config values verified.", cs, + LOG_STR_ARG(name)); } else { - for (uint8_t phase = 0; phase < 3; ++phase) - this->power_offset_phase_[phase] = this->config_power_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored power offsets found. Using default values.", cs); - } - - for (uint8_t phase = 0; phase < 3; ++phase) { - write_power_offsets_to_registers_(phase, this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore and config fallback both failed verification.", cs, + LOG_STR_ARG(name)); } } @@ -1084,14 +1109,14 @@ void ATM90E32Component::clear_gain_calibrations() { void ATM90E32Component::clear_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_offset_calibration_) { + if (!this->has_stored_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored offset calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\n", cs); return; @@ -1104,10 +1129,11 @@ void ATM90E32Component::clear_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = - this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].voltage_offset_ : 0; + this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].first_offset : 0; int16_t current_offset = - this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].current_offset_ : 0; - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); } @@ -1117,6 +1143,7 @@ void ATM90E32Component::clear_offset_calibrations() { this->offset_pref_.save(&zero_offsets); // Clear stored values in flash global_preferences->sync(); + this->has_stored_offset_calibration_ = false; this->restored_offset_calibration_ = false; for (bool &phase : this->offset_calibration_mismatch_) phase = false; @@ -1126,15 +1153,14 @@ void ATM90E32Component::clear_offset_calibrations() { void ATM90E32Component::clear_power_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_power_offset_calibration_) { + if (!this->has_stored_power_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored power offsets to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); return; @@ -1147,20 +1173,21 @@ void ATM90E32Component::clear_power_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t active_offset = - this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].active_power_offset : 0; - int16_t reactive_offset = this->has_config_reactive_power_offset_[phase] - ? this->config_power_offset_phase_[phase].reactive_power_offset - : 0; - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].first_offset : 0; + int16_t reactive_offset = + this->has_config_reactive_power_offset_[phase] ? this->config_power_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - PowerOffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; + OffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; this->power_offset_pref_.save(&zero_power_offsets); global_preferences->sync(); + this->has_stored_power_offset_calibration_ = false; this->restored_power_offset_calibration_ = false; for (bool &phase : this->power_offset_calibration_mismatch_) phase = false; @@ -1215,6 +1242,31 @@ bool ATM90E32Component::verify_gain_writes_() { return success; // Return true if all writes were successful, false otherwise } +bool ATM90E32Component::verify_offset_writes_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + const char *cs = this->get_calibration_id_(); + const LogString *name = offset_calibration_name(power_offsets); + const LogString *first_name = power_offsets ? LOG_STR("active") : LOG_STR("voltage"); + const LogString *second_name = power_offsets ? LOG_STR("reactive") : LOG_STR("current"); + const OffsetCalibration *offsets = power_offsets ? this->power_offset_phase_ : this->offset_phase_; + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; + bool success = true; + for (uint8_t phase = 0; phase < 3; phase++) { + const uint16_t first = this->read16_(first_registers[phase]); + const uint16_t second = this->read16_(second_registers[phase]); + if (!offset_register_value_matches(first, offsets[phase].first_offset) || + !offset_register_value_matches(second, offsets[phase].second_offset)) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s readback failed for Phase %s: %s %d/%d, %s %d/%d.", cs, LOG_STR_ARG(name), + phase_labels[phase], LOG_STR_ARG(first_name), static_cast(first), offsets[phase].first_offset, + LOG_STR_ARG(second_name), static_cast(second), offsets[phase].second_offset); + success = false; + } + } + return success; +} + #ifdef USE_TEXT_SENSOR void ATM90E32Component::check_phase_status() { uint16_t state0 = this->read16_(ATM90E32_REGISTER_EMMSTATE0); diff --git a/esphome/components/atm90e32/atm90e32.h b/esphome/components/atm90e32/atm90e32.h index c636e5065a..fe7d903962 100644 --- a/esphome/components/atm90e32/atm90e32.h +++ b/esphome/components/atm90e32/atm90e32.h @@ -13,6 +13,40 @@ namespace esphome::atm90e32 { +inline bool offset_register_value_matches(uint16_t actual, int16_t expected) { + return actual == static_cast(expected); +} + +struct OffsetCalibration { + int16_t first_offset{0}; + int16_t second_offset{0}; +}; + +static_assert(sizeof(OffsetCalibration[3]) == 12, "Offset calibration preference layout must remain compatible"); + +enum class OffsetCalibrationType : uint8_t { + OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT, + OFFSET_CALIBRATION_TYPE_POWER, +}; + +struct OffsetRestoreState { + bool restored; + bool values_verified; +}; + +inline OffsetRestoreState resolve_offset_restore_state(bool has_stored_values, bool initial_values_verified, + bool fallback_values_verified) { + if (initial_values_verified) + return {has_stored_values, true}; + return {false, fallback_values_verified}; +} + +inline void prepare_offset_rollback(const OffsetCalibration (&previous)[3], bool had_stored_values, + OffsetCalibration (&rollback)[3]) { + for (uint8_t phase = 0; phase < 3; phase++) + rollback[phase] = had_stored_values ? previous[phase] : OffsetCalibration{}; +} + class ATM90E32Component final : public PollingComponent, public spi::SPIDevice { @@ -71,19 +105,19 @@ class ATM90E32Component final : public PollingComponent, this->has_config_current_gain_[phase] = true; } void set_voltage_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].voltage_offset_ = offset; + this->offset_phase_[phase].first_offset = offset; this->has_config_voltage_offset_[phase] = true; } void set_current_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].current_offset_ = offset; + this->offset_phase_[phase].second_offset = offset; this->has_config_current_offset_[phase] = true; } void set_active_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].active_power_offset = offset; + this->power_offset_phase_[phase].first_offset = offset; this->has_config_active_power_offset_[phase] = true; } void set_reactive_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].reactive_power_offset = offset; + this->power_offset_phase_[phase].second_offset = offset; this->has_config_reactive_power_offset_[phase] = true; } void set_freq_sensor(sensor::Sensor *freq_sensor) { freq_sensor_ = freq_sensor; } @@ -171,16 +205,16 @@ class ATM90E32Component final : public PollingComponent, float get_chip_temperature_(); bool get_publish_interval_flag_() { return publish_interval_flag_; }; void set_publish_interval_flag_(bool flag) { publish_interval_flag_ = flag; }; - void restore_offset_calibrations_(); - void restore_power_offset_calibrations_(); + void restore_offset_calibrations_(OffsetCalibrationType type); void restore_gain_calibrations_(); - void save_offset_calibration_to_memory_(); void save_gain_calibration_to_memory_(); - void save_power_offset_calibration_to_memory_(); - void write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset); - void write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset); + void finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type); + void write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type); void write_gains_to_registers_(); bool verify_gain_writes_(); + bool verify_offset_writes_(OffsetCalibrationType type); bool validate_spi_read_(uint16_t expected, const char *context = nullptr); void log_calibration_status_(); const char *get_calibration_id_(); @@ -219,19 +253,10 @@ class ATM90E32Component final : public PollingComponent, uint32_t cumulative_reverse_active_energy_{0}; } phase_[3]; - struct OffsetCalibration { - int16_t voltage_offset_{0}; - int16_t current_offset_{0}; - } offset_phase_[3]; - + OffsetCalibration offset_phase_[3]; OffsetCalibration config_offset_phase_[3]; - - struct PowerOffsetCalibration { - int16_t active_power_offset{0}; - int16_t reactive_power_offset{0}; - } power_offset_phase_[3]; - - PowerOffsetCalibration config_power_offset_phase_[3]; + OffsetCalibration power_offset_phase_[3]; + OffsetCalibration config_power_offset_phase_[3]; struct GainCalibration { uint16_t voltage_gain{1}; @@ -265,6 +290,8 @@ class ATM90E32Component final : public PollingComponent, bool enable_offset_calibration_{false}; bool enable_gain_calibration_{false}; const char *instance_id_{nullptr}; + bool has_stored_offset_calibration_{false}; + bool has_stored_power_offset_calibration_{false}; bool restored_offset_calibration_{false}; bool restored_power_offset_calibration_{false}; bool restored_gain_calibration_{false}; diff --git a/tests/components/atm90e32/__init__.py b/tests/components/atm90e32/__init__.py new file mode 100644 index 0000000000..37d6797e2d --- /dev/null +++ b/tests/components/atm90e32/__init__.py @@ -0,0 +1,5 @@ +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + manifest.dependencies = manifest.dependencies + ["sensor", "spi"] diff --git a/tests/components/atm90e32/offset_register_verification_test.cpp b/tests/components/atm90e32/offset_register_verification_test.cpp new file mode 100644 index 0000000000..3bb3eb76ea --- /dev/null +++ b/tests/components/atm90e32/offset_register_verification_test.cpp @@ -0,0 +1,62 @@ +#include + +#include "esphome/components/atm90e32/atm90e32.h" + +namespace esphome::atm90e32::testing { + +TEST(ATM90E32OffsetRegisterVerification, AcceptsExactSignedReadback) { + EXPECT_TRUE(offset_register_value_matches(0x007B, 123)); + EXPECT_TRUE(offset_register_value_matches(0xFF85, -123)); +} + +TEST(ATM90E32OffsetRegisterVerification, RejectsMismatchedReadback) { + EXPECT_FALSE(offset_register_value_matches(0x007C, 123)); + EXPECT_FALSE(offset_register_value_matches(0xFF84, -123)); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedStoredValuesAsRestored) { + const auto state = resolve_offset_restore_state(true, true, false); + + EXPECT_TRUE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedConfigFallbackAsNotRestored) { + const auto state = resolve_offset_restore_state(true, false, true); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsFailedConfigFallbackAsUnverified) { + const auto state = resolve_offset_restore_state(true, false, false); + + EXPECT_FALSE(state.restored); + EXPECT_FALSE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsConfigWithoutStoredValuesAsNotRestored) { + const auto state = resolve_offset_restore_state(false, true, false); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetPersistence, RollsBackStoredValuesOrZeroSentinel) { + const OffsetCalibration previous[3]{{1, -1}, {2, -2}, {3, -3}}; + OffsetCalibration rollback[3]{}; + + prepare_offset_rollback(previous, true, rollback); + for (uint8_t phase = 0; phase < 3; phase++) { + EXPECT_EQ(rollback[phase].first_offset, previous[phase].first_offset); + EXPECT_EQ(rollback[phase].second_offset, previous[phase].second_offset); + } + + prepare_offset_rollback(previous, false, rollback); + for (const auto &phase : rollback) { + EXPECT_EQ(phase.first_offset, 0); + EXPECT_EQ(phase.second_offset, 0); + } +} + +} // namespace esphome::atm90e32::testing From f89b9e704c7dfabce1e8ce670dd2c6e7aa9ea086 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:21:48 +0200 Subject: [PATCH 088/178] Bump bundled esphome-device-builder to 1.14.5 (#19040) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index da76ab7b6a..ac84ee4689 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 RUN \ platformio settings set enable_telemetry No \ From 9b6facb20d5461dfaf47fd3993a7b4601f082c34 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 15:46:17 -0400 Subject: [PATCH 089/178] [core] Fix use-after-free when deleting a running StaticTask (#19048) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../micro_wake_word/micro_wake_word.cpp | 4 +-- .../mixer/speaker/mixer_speaker.cpp | 4 +-- .../resampler/speaker/resampler_speaker.cpp | 4 +-- .../speaker/media_player/audio_pipeline.cpp | 11 +++++-- esphome/core/static_task.cpp | 30 ++++++++++++++----- esphome/core/static_task.h | 17 +++++++---- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 3dadb78077..cebfe8e791 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -446,9 +446,9 @@ void MicroWakeWord::loop() { xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STOPPING); } - if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & EventGroupBits::TASK_STOPPED) && this->inference_task_.deallocate()) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 6128dc3767..0b79010773 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -382,8 +382,8 @@ void MixerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, MIXER_TASK_STATE_STOPPING); } - if (event_group_bits & MIXER_TASK_STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS); this->all_stopped_since_ms_ = 0; diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index f1ebd180cc..edda00ae06 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -153,8 +153,8 @@ void ResamplerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::STATE_STOPPING); } - if (event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::ALL_BITS); } diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index 010f0c50b3..c286a9d7d6 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -202,8 +202,15 @@ AudioPipelineState AudioPipeline::process_state() { if (!this->is_playing_) { // The tasks have been stopped for two ``process_state`` calls in a row, so delete the tasks if (this->read_task_.is_created() || this->decode_task_.is_created()) { - this->read_task_.deallocate(); - this->decode_task_.deallocate(); + // Both are attempted every time; a task that is still running on the other core is freed by a + // subsequent call, and freeing an already freed task succeeds without doing anything + bool read_task_freed = this->read_task_.deallocate(); + bool decode_task_freed = this->decode_task_.deallocate(); + if (!read_task_freed || !decode_task_freed) { + // A task is still running on the other core, so keep the pipeline in its current state and try + // again on the next call + return AudioPipelineState::PLAYING; + } if (this->hard_stop_) { // Stop command was sent, so immediately end the playback this->speaker_->stop(); diff --git a/esphome/core/static_task.cpp b/esphome/core/static_task.cpp index 4cfead44c2..4301108315 100644 --- a/esphome/core/static_task.cpp +++ b/esphome/core/static_task.cpp @@ -40,16 +40,31 @@ bool StaticTask::create(TaskFunction_t fn, const char *name, uint32_t stack_size return true; } -void StaticTask::destroy() { - if (this->handle_ != nullptr) { - TaskHandle_t handle = this->handle_; - this->handle_ = nullptr; - vTaskDelete(handle); +bool StaticTask::destroy() { + if (this->handle_ == nullptr) { + return true; } + + // Suspending takes the task off the ready and event lists, so nothing can schedule it again. It only asks + // the other core to yield though, so the task may still be running on it for a moment. + vTaskSuspend(this->handle_); + if (eTaskGetState(this->handle_) != eSuspended) { + // The task is still running on the other core and using its stack. Deleting it now would only put it on + // the termination list and return, so the caller has to try again once it has been swapped out. + return false; + } + + // The task cannot run again, so the delete completes right away instead of being left to the idle task. + TaskHandle_t handle = this->handle_; + this->handle_ = nullptr; + vTaskDelete(handle); + return true; } -void StaticTask::deallocate() { - this->destroy(); +bool StaticTask::deallocate() { + if (!this->destroy()) { + return false; + } if (this->stack_buffer_ != nullptr) { RAMAllocator allocator(this->use_psram_ ? RAMAllocator::ALLOC_EXTERNAL : RAMAllocator::ALLOC_INTERNAL); @@ -57,6 +72,7 @@ void StaticTask::deallocate() { this->stack_buffer_ = nullptr; this->stack_size_ = 0; } + return true; } } // namespace esphome diff --git a/esphome/core/static_task.h b/esphome/core/static_task.h index 5fd5b38f9e..e2996abeda 100644 --- a/esphome/core/static_task.h +++ b/esphome/core/static_task.h @@ -11,6 +11,7 @@ namespace esphome { /** Helper for FreeRTOS static task management. * Bundles TaskHandle_t, StaticTask_t, and the stack buffer into one object with create/destroy methods. + * Call destroy() and deallocate() from another task: a task cannot free the stack it is still running on. */ class StaticTask { public: @@ -23,7 +24,7 @@ class StaticTask { /// @brief Allocate stack and create task. /// @param fn Task function /// @param name Task name (for debug) - /// @param stack_size Stack size in StackType_t words + /// @param stack_size Stack size in bytes (StackType_t is a byte on ESP-IDF) /// @param param Parameter passed to task function /// @param priority FreeRTOS task priority /// @param use_psram If true, allocate stack in PSRAM; otherwise internal RAM @@ -31,11 +32,17 @@ class StaticTask { bool create(TaskFunction_t fn, const char *name, uint32_t stack_size, void *param, UBaseType_t priority, bool use_psram); - /// @brief Delete the task but keep the stack buffer allocated for reuse by a subsequent create() call. - void destroy(); + /// @brief Delete the task, keeping the stack buffer allocated for reuse by a subsequent create() call. + /// The task must have finished its work and parked itself, either suspended or blocked indefinitely: it is + /// suspended here so that it cannot be scheduled again, and it is given no chance to clean up. + /// @return true if the task was deleted; false if it is still running on another core, in which case the + /// caller should try again later. + bool destroy(); - /// @brief Delete the task (if running) and free the stack buffer. - void deallocate(); + /// @brief Delete the task (if created) and free the stack buffer. + /// @return true if the stack buffer was freed; false if the task is still running on another core, in + /// which case the caller should try again later. + bool deallocate(); protected: TaskHandle_t handle_{nullptr}; From 0a1e2acbcba4521391742824c617c8cf206beb63 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:04:34 -0400 Subject: [PATCH 090/178] [audio][i2s_audio][micro_wake_word][microphone][mixer][resampler][speaker] Replace use_count() checks with lock and null test (#19046) --- esphome/components/audio/audio_reader.cpp | 3 +++ esphome/components/audio/audio_transfer_buffer.cpp | 12 ++++++------ .../i2s_audio/speaker/i2s_audio_speaker.cpp | 4 ++-- .../components/micro_wake_word/micro_wake_word.cpp | 2 +- esphome/components/microphone/microphone_source.h | 2 +- esphome/components/mixer/speaker/mixer_speaker.cpp | 12 ++++++------ .../resampler/speaker/resampler_speaker.cpp | 6 +++--- .../speaker/media_player/audio_pipeline.cpp | 12 +++++++----- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/esphome/components/audio/audio_reader.cpp b/esphome/components/audio/audio_reader.cpp index 4678ed548c..e69f33ac2d 100644 --- a/esphome/components/audio/audio_reader.cpp +++ b/esphome/components/audio/audio_reader.cpp @@ -58,6 +58,9 @@ esp_err_t AudioReader::add_sink(const std::weak_ptr &ou if (current_audio_file_ != nullptr) { // A transfer buffer isn't ncessary for a local file this->file_ring_buffer_ = output_ring_buffer.lock(); + if (this->file_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; + } return ESP_OK; } diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index a611549e58..01fd4bb68a 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -51,14 +51,14 @@ void AudioTransferBuffer::increase_buffer_length(size_t bytes) { this->buffer_le void AudioTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } } void AudioSinkTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } #ifdef USE_SPEAKER @@ -69,7 +69,7 @@ void AudioSinkTransferBuffer::clear_buffered_data() { } bool AudioTransferBuffer::has_buffered_data() const { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); @@ -144,7 +144,7 @@ size_t AudioSourceTransferBuffer::transfer_data_from_source(TickType_t ticks_to_ size_t bytes_to_read = AudioTransferBuffer::free(); size_t bytes_read = 0; if (bytes_to_read > 0) { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_read = this->ring_buffer_->read((void *) this->get_buffer_end(), bytes_to_read, ticks_to_wait); } @@ -161,7 +161,7 @@ size_t AudioSinkTransferBuffer::transfer_data_to_sink(TickType_t ticks_to_wait, bytes_written = this->speaker_->play(this->data_start_, this->available(), ticks_to_wait); } else #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_written = this->ring_buffer_->write_without_replacement((void *) this->data_start_, this->available(), ticks_to_wait); } else if (this->sink_callback_ != nullptr) { @@ -186,7 +186,7 @@ bool AudioSinkTransferBuffer::has_buffered_data() const { return (this->speaker_->has_buffered_data() || (this->available() > 0)); } #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1c2eb12904..b78a151ee4 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -218,8 +218,8 @@ size_t I2SAudioSpeakerBase::play(const uint8_t *data, size_t length, TickType_t } bool I2SAudioSpeakerBase::has_buffered_data() const { - if (this->audio_ring_buffer_.use_count() > 0) { - std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + if (temp_ring_buffer != nullptr) { return temp_ring_buffer->available() > 0; } return false; diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index cebfe8e791..cf239be696 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -129,7 +129,7 @@ void MicroWakeWord::setup() { return; } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (this->ring_buffer_.use_count() > 1) { + if (temp_ring_buffer != nullptr) { // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task // to drain it - reset() is a consumer operation and must run on the inference task's thread. // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. diff --git a/esphome/components/microphone/microphone_source.h b/esphome/components/microphone/microphone_source.h index 7be3b8cdb5..d7a3352432 100644 --- a/esphome/components/microphone/microphone_source.h +++ b/esphome/components/microphone/microphone_source.h @@ -48,7 +48,7 @@ class MicrophoneSource final { template void add_data_callback(F &&data_callback) { this->mic_->add_data_callback([this, data_callback](const std::vector &data) { if (this->enabled_ || this->passive_) { - if (this->processed_samples_.use_count() == 0) { + if (this->processed_samples_ == nullptr) { // Create vector if its unused this->processed_samples_ = std::make_shared>(); } diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 0b79010773..ef21da65c5 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -218,7 +218,7 @@ size_t SourceSpeaker::play(const uint8_t *data, size_t length, TickType_t ticks_ } size_t bytes_written = 0; std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer.use_count() > 0) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); if (bytes_written > 0) { @@ -250,14 +250,14 @@ esp_err_t SourceSpeaker::start_() { // avoids unnecessary single-frame splices. const size_t ring_buffer_size = (this->audio_stream_info_.ms_to_bytes(this->buffer_duration_ms_) / bytes_per_frame) * bytes_per_frame; - if (this->audio_source_.use_count() == 0) { + if (this->audio_source_ == nullptr) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); this->ring_buffer_ = temp_ring_buffer; } - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { return ESP_ERR_NO_MEM; } @@ -278,7 +278,7 @@ void SourceSpeaker::stop() { this->send_command_(SOURCE_SPEAKER_COMMAND_STOP); } void SourceSpeaker::finish() { this->send_command_(SOURCE_SPEAKER_COMMAND_FINISH); } bool SourceSpeaker::has_buffered_data() const { - return ((this->audio_source_.use_count() > 0) && this->audio_source_->has_buffered_data()); + return ((this->audio_source_ != nullptr) && this->audio_source_->has_buffered_data()); } void SourceSpeaker::set_mute_state(bool mute_state) { @@ -496,7 +496,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (speaker->is_running() && !speaker->get_pause_state()) { // Speaker is running and not paused, so it possibly can provide audio data std::shared_ptr audio_source = speaker->get_audio_source().lock(); - if (audio_source.use_count() == 0) { + if (audio_source == nullptr) { // No audio source allocated, so skip processing this speaker continue; } diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index edda00ae06..16d2d5dc9e 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -235,7 +235,7 @@ size_t ResamplerSpeaker::play(const uint8_t *data, size_t length, TickType_t tic bytes_written = this->output_speaker_->play(data, length, ticks_to_wait); } else { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); } else { @@ -299,7 +299,7 @@ bool ResamplerSpeaker::has_buffered_data() const { bool has_ring_buffer_data = false; if (this->requires_resampling_()) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { has_ring_buffer_data = (temp_ring_buffer->available() > 0); } } @@ -342,7 +342,7 @@ void ResamplerSpeaker::resample_task(void *params) { std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( this_resampler->audio_stream_info_.ms_to_bytes(this_resampler->buffer_duration_ms_)); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { this_resampler->ring_buffer_ = temp_ring_buffer; diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index c286a9d7d6..509984cfa2 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -322,17 +322,17 @@ void AudioPipeline::read_task(void *params) { if (err == ESP_OK) { size_t file_ring_buffer_size = this_pipeline->buffer_size_; - std::shared_ptr temp_ring_buffer; + std::shared_ptr temp_ring_buffer = this_pipeline->raw_file_ring_buffer_.lock(); - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(file_ring_buffer_size); this_pipeline->raw_file_ring_buffer_ = temp_ring_buffer; } - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { - reader->add_sink(this_pipeline->raw_file_ring_buffer_); + err = reader->add_sink(temp_ring_buffer); } } @@ -403,7 +403,9 @@ void AudioPipeline::decode_task(void *params) { make_unique(this_pipeline->transfer_buffer_size_, this_pipeline->transfer_buffer_size_); esp_err_t err = decoder->start(this_pipeline->current_audio_file_type_); - decoder->add_source(this_pipeline->raw_file_ring_buffer_); + if (err == ESP_OK) { + err = decoder->add_source(this_pipeline->raw_file_ring_buffer_); + } if (err != ESP_OK) { // Send specific error message From ac79173f4ae83ff10c6a571140091be6ec7878b1 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:05:02 -0400 Subject: [PATCH 091/178] [i2s_audio] Fix spurious driver failure (#19045) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index b78a151ee4..1382a87046 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -118,21 +118,24 @@ void I2SAudioSpeakerBase::loop() { break; } + // Still starting up or winding down from a previous run + if ((this->tx_handle_ != nullptr) || (this->speaker_task_handle_ != nullptr)) { + break; + } + if (this->start_i2s_driver(this->audio_stream_info_) != ESP_OK) { ESP_LOGE(TAG, "Driver failed to start; retrying in 1 second"); this->status_momentary_error("driver-failure", 1000); break; } - if (this->speaker_task_handle_ == nullptr) { - xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, - &this->speaker_task_handle_); + xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, + &this->speaker_task_handle_); - if (this->speaker_task_handle_ == nullptr) { - ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); - this->status_momentary_error("task-failure", 1000); - this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt - } + if (this->speaker_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); + this->status_momentary_error("task-failure", 1000); + this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt } break; case speaker::STATE_RUNNING: // Intentional fallthrough From 9c16aba6f78af2657cd9e5876d0e771c72692fcd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:07:06 +0200 Subject: [PATCH 092/178] [noise] Bump noise-c to 0.1.26 and libsodium to 1.10021.8 (#19030) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 4de706120e..d17ebf235e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.24") + cg.add_library("esphome/noise-c", "0.1.26") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.6") + cg.add_library("esphome/libsodium", "1.10021.8") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 779a05e7de..738773d1b5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.24 ; used by noise (api, ota) + esphome/noise-c@0.1.26 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 4f7f5a4a4c..00f22ca138 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.24"] + assert libs == ["esphome/noise-c @ 0.1.26"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.24", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 0.1.26", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.24"] + assert cls.calls == ["esphome/noise-c @ 0.1.26"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.24"] is None + assert compats["esphome/noise-c @ 0.1.26"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 14c52dda8d..b03bff19a2 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 42fffd16fef3d08f58b5ace59b0545e551c07e64 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:08:16 +0200 Subject: [PATCH 093/178] [esphome][core] Give a lost OTA chunk ack time to be retransmitted (#19041) --- esphome/components/esphome/ota/ota_esphome.cpp | 5 ++++- esphome/espota2.py | 9 ++++++--- tests/unit_tests/test_espota2.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 1005ed214b..f853ed6a2d 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -41,7 +41,10 @@ const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake -static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer +// Milliseconds for data transfer. Covers the lwIP retransmit run seen in +// practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits +// longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries +static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/espota2.py b/esphome/espota2.py index ce403c398d..c683ffa323 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -96,6 +96,10 @@ UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # across the addresses on top of that. EXTRA_UPLOAD_ATTEMPTS = 2 UPLOAD_RETRY_DELAY = 5.0 +# Data phase timeout; must stay longer than the device's OTA_SOCKET_TIMEOUT_DATA +# (105 s) so a stalled session is gone before a retry, and long enough for lwIP +# to get a lost chunk ack through after the retransmit run seen in practice +DATA_PHASE_TIMEOUT = 160.0 _LOGGER = logging.getLogger(__name__) @@ -694,8 +698,7 @@ def perform_ota( _LOGGER.info("Handshake complete") - # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures - sock.settimeout(90.0) + sock.settimeout(DATA_PHASE_TIMEOUT) if extended_proto: send_check(sock, ota_type, "ota type") @@ -854,7 +857,7 @@ def run_ota_impl_( # clean up a half-open connection (its handshake watchdog runs at 20s); # moving on to the next address family stays immediate. Known limitation: # a silent mid-transfer drop with no reset can wedge the device until its - # 90s data timeout, which outlasts this budget; the retries target the + # 105s data timeout, which outlasts this budget; the retries target the # common failures where the device resets or closes the link promptly. total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 8867e2c215..2d65e8e079 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -416,6 +416,9 @@ def test_perform_ota_no_auth( "Update took 14.00 seconds (prepare 2.00, upload 5.00, commit 7.00)" in caplog.text ) + # The data phase timeout must outlast the device's 105 s data timeout + mock_socket.settimeout.assert_any_call(espota2.DATA_PHASE_TIMEOUT) + assert espota2.DATA_PHASE_TIMEOUT > 105.0 @pytest.mark.usefixtures("mock_time") From d2bc056f0ab3ea93e62c7ca468f7a3da17a3f421 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:31:18 -0400 Subject: [PATCH 094/178] [sendspin] Add codec preference list to the media source (#19047) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/sendspin/__init__.py | 26 ++++-- .../sendspin/media_source/__init__.py | 31 +++++++ .../sendspin/test_media_source.py | 90 +++++++++++++++++++ .../sendspin/common-media_source.yaml | 1 + 4 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 tests/component_tests/sendspin/test_media_source.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 570fd3fadd..8ef11a7f90 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -30,6 +30,7 @@ CONF_SENDSPIN_ID = "sendspin_id" CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" +CONF_CODECS = "codecs" # Matches ARTWORK_MAX_SLOTS in sendspin-cpp. MAX_ARTWORK_SLOTS = 4 @@ -44,6 +45,20 @@ CODEC_FORMAT_OPUS = SendspinCodecFormat.enum("OPUS") CODEC_FORMAT_PCM = SendspinCodecFormat.enum("PCM") CODEC_FORMAT_UNSUPPORTED = SendspinCodecFormat.enum("UNSUPPORTED") +CODEC_FLAC = "flac" +CODEC_OPUS = "opus" +CODEC_PCM = "pcm" + +CODECS = { + CODEC_FLAC: CODEC_FORMAT_FLAC, + CODEC_OPUS: CODEC_FORMAT_OPUS, + CODEC_PCM: CODEC_FORMAT_PCM, +} + +# Opus only supports 48 kHz audio, so it is left out of the default list at other rates. +DEFAULT_CODECS = [CODEC_FLAC, CODEC_OPUS, CODEC_PCM] +OPUS_SAMPLE_RATE = 48000 + SendspinImageFormat = sendspin_library_ns.enum("SendspinImageFormat", is_class=True) IMAGE_FORMAT_JPEG = SendspinImageFormat.enum("JPEG") IMAGE_FORMAT_PNG = SendspinImageFormat.enum("PNG") @@ -286,16 +301,13 @@ async def to_code(config: ConfigType) -> None: if data.player_support: cg.add_define("USE_SENDSPIN_PLAYER", True) - # Configures the player role. We always assume support for 16 bits per sample mono and stereo FLAC, Opus, and PCM at the configured sample rate - # (with Opus only supported at 48 kHz since that's the only sample rate it supports). Users can configure the specific formats via the Sendspin server + # Configures the player role. Each configured codec is advertised for 16 bits per sample + # mono and stereo at the configured sample rate. The order is a preference order, both for + # the codecs themselves and for stereo over mono. player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - # OPUS only supports 48 kHz audio - codecs = [CODEC_FORMAT_FLAC] - if sample_rate == 48000: - codecs.append(CODEC_FORMAT_OPUS) - codecs.append(CODEC_FORMAT_PCM) + codecs = player_cfg[CONF_CODECS] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index 6af244d41f..6a9f1f18ba 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -13,11 +13,16 @@ from esphome.cpp_generator import MockObj, TemplateArgsType from esphome.types import ConfigType from .. import ( + CODEC_OPUS, + CODECS, + CONF_CODECS, CONF_DECODE_MEMORY, CONF_FIXED_DELAY, CONF_INITIAL_STATIC_DELAY, CONF_SENDSPIN_ID, + DEFAULT_CODECS, MEMORY_LOCATIONS, + OPUS_SAMPLE_RATE, SendspinHub, register_player_config, request_controller_support, @@ -49,10 +54,32 @@ DisableStaticDelayAdjustmentAction = sendspin_ns.class_( ) +def _resolve_codecs(config: ConfigType) -> ConfigType: + """Validate the codec preference list, filling in the default when it is not set.""" + sample_rate = config[CONF_SAMPLE_RATE] + if (codecs := config.get(CONF_CODECS)) is None: + config[CONF_CODECS] = [ + codec + for codec in DEFAULT_CODECS + if codec != CODEC_OPUS or sample_rate == OPUS_SAMPLE_RATE + ] + return config + + if len(set(codecs)) != len(codecs): + raise cv.Invalid("Each codec may only be listed once", path=[CONF_CODECS]) + if CODEC_OPUS in codecs and sample_rate != OPUS_SAMPLE_RATE: + raise cv.Invalid( + f"Codec '{CODEC_OPUS}' requires a {CONF_SAMPLE_RATE} of {OPUS_SAMPLE_RATE}", + path=[CONF_CODECS], + ) + return config + + def _register(config: ConfigType) -> ConfigType: request_controller_support() register_player_config( { + CONF_CODECS: config[CONF_CODECS], CONF_SAMPLE_RATE: config[CONF_SAMPLE_RATE], CONF_BUFFER_SIZE: config[CONF_BUFFER_SIZE], CONF_INITIAL_STATIC_DELAY: config[CONF_INITIAL_STATIC_DELAY], @@ -85,9 +112,13 @@ CONFIG_SCHEMA = cv.All( min=16000, max=96000 ), cv.Optional(CONF_DECODE_MEMORY): cv.one_of(*MEMORY_LOCATIONS, lower=True), + cv.Optional(CONF_CODECS): cv.All( + cv.ensure_list(cv.enum(CODECS, lower=True)), cv.Length(min=1) + ), } ), cv.only_on_esp32, + _resolve_codecs, _register, ) diff --git a/tests/component_tests/sendspin/test_media_source.py b/tests/component_tests/sendspin/test_media_source.py new file mode 100644 index 0000000000..6c2f79198d --- /dev/null +++ b/tests/component_tests/sendspin/test_media_source.py @@ -0,0 +1,90 @@ +"""Validation tests for the sendspin media_source platform. + +These cover the codec preference list, whose rejection branches a compile test +cannot reach: a `test*.yaml` can only assert that a configuration is accepted. +""" + +from typing import Any + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import CONF_CODECS, _get_data +from esphome.components.sendspin.media_source import CONFIG_SCHEMA +from esphome.const import PlatformFramework +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _media_source_config(**overrides: Any) -> ConfigType: + """Build a minimal valid media source config, allowing field overrides.""" + config: ConfigType = { + "id": "sendspin_media_source", + "sendspin_id": "sendspin_hub", + } + config.update(overrides) + return config + + +def test_default_codecs_at_48_khz(set_core_config: SetCoreConfigCallable) -> None: + """Every codec is advertised when the sample rate suits all of them.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config()) + + assert config[CONF_CODECS] == ["flac", "opus", "pcm"] + + +def test_default_codecs_drop_opus_at_other_rates( + set_core_config: SetCoreConfigCallable, +) -> None: + """Opus only supports 48 kHz, so it leaves the default list at other rates.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config(sample_rate=44100)) + + assert config[CONF_CODECS] == ["flac", "pcm"] + + +def test_configured_order_is_preserved(set_core_config: SetCoreConfigCallable) -> None: + """The list is a preference order, so it reaches the player role as written.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA(_media_source_config(codecs=["pcm", "flac"])) + + assert _get_data().player_config[CONF_CODECS] == ["pcm", "flac"] + + +def test_empty_codec_list_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A player with no codecs at all could never be given a stream.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="length of value must be at least 1"): + CONFIG_SCHEMA(_media_source_config(codecs=[])) + + +def test_duplicate_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A repeated codec has no meaning in a preference order.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="may only be listed once"): + CONFIG_SCHEMA(_media_source_config(codecs=["flac", "flac"])) + + +def test_unknown_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """Only codecs the player role can decode are accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="Unknown value"): + CONFIG_SCHEMA(_media_source_config(codecs=["mp3"])) + + +def test_opus_at_wrong_sample_rate_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + """Asking for Opus at a rate it cannot handle fails rather than silently + dropping the stated preference.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="requires a sample_rate of 48000"): + CONFIG_SCHEMA(_media_source_config(codecs=["opus"], sample_rate=44100)) diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 1977b79c04..0c136fbd43 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,3 +9,4 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal + codecs: [pcm, opus, flac] From 008677298ada0a95adeef14c5ac88d1895d5fb2f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:54:44 +1200 Subject: [PATCH 095/178] Bump version to 2026.9.0b3 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 060de51d3a..97ce92240c 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0b2 +PROJECT_NUMBER = 2026.9.0b3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 287804ace3..b013098f33 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0b2" +__version__ = "2026.9.0b3" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From fd598057efdfa689a10b53329753a936a15016a1 Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Wed, 9 Sep 2026 14:22:23 +0200 Subject: [PATCH 096/178] [sendspin] Fix codec enum codegen when codecs is not set (#19055) --- esphome/components/sendspin/__init__.py | 2 +- tests/components/sendspin/common-media_source.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 8ef11a7f90..c1970ab132 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -307,7 +307,7 @@ async def to_code(config: ConfigType) -> None: player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - codecs = player_cfg[CONF_CODECS] + codecs = [CODECS[codec] for codec in player_cfg[CONF_CODECS]] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 0c136fbd43..1977b79c04 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,4 +9,3 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal - codecs: [pcm, opus, flac] From 58ca3456845ca130ac106796225e8ea4cb9c5107 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:42:09 -0400 Subject: [PATCH 097/178] [ci] Refresh integration test durations (#19049) --- .../integration_test_durations.json | 293 +++++++++--------- 1 file changed, 152 insertions(+), 141 deletions(-) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json index 5a5aac3b22..b4a7f4e1ae 100644 --- a/tests/integration/integration_test_durations.json +++ b/tests/integration/integration_test_durations.json @@ -1,143 +1,154 @@ { - "tests/integration/test_action_concurrent_reentry.py": 57.91, - "tests/integration/test_addressable_light_transition.py": 21.25, - "tests/integration/test_alarm_control_panel_state_transitions.py": 70.71, - "tests/integration/test_api_action_metadata.py": 66.6, - "tests/integration/test_api_action_responses.py": 36.1, - "tests/integration/test_api_action_timeout.py": 68.86, - "tests/integration/test_api_conditional_memory.py": 15.48, - "tests/integration/test_api_custom_services.py": 18.77, - "tests/integration/test_api_get_time_response_timezone.py": 21.08, - "tests/integration/test_api_homeassistant.py": 65.59, - "tests/integration/test_api_homeassistant_action_no_subscriber.py": 18.44, - "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 15.05, - "tests/integration/test_api_list_entities_backpressure.py": 13.88, - "tests/integration/test_api_message_size_batching.py": 29.98, - "tests/integration/test_api_reboot_timeout.py": 16.05, - "tests/integration/test_api_string_lambda.py": 15.31, - "tests/integration/test_api_vv_logging.py": 19.28, - "tests/integration/test_api_zero_psk_provisioning.py": 31.5, - "tests/integration/test_areas_and_devices.py": 24.95, - "tests/integration/test_automation_wait_actions.py": 20.92, - "tests/integration/test_automations.py": 35.19, - "tests/integration/test_batch_delay_zero_rapid_transitions.py": 17.99, - "tests/integration/test_binary_sensor_autorepeat_filter.py": 20.39, - "tests/integration/test_binary_sensor_invalidate_state.py": 18.41, - "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 24.69, - "tests/integration/test_build_info.py": 18.7, - "tests/integration/test_camera_mock.py": 16.23, - "tests/integration/test_climate_control_action.py": 21.14, - "tests/integration/test_climate_custom_modes.py": 20.74, - "tests/integration/test_continuation_actions.py": 16.81, - "tests/integration/test_cover_control_action.py": 20.34, - "tests/integration/test_crc8_helper.py": 9.36, - "tests/integration/test_device_id_in_state.py": 44.67, - "tests/integration/test_duplicate_entities.py": 23.58, - "tests/integration/test_entity_icon.py": 34.35, - "tests/integration/test_fan_turn_on_action.py": 24.23, - "tests/integration/test_fnv1_hash_object_id.py": 16.21, - "tests/integration/test_fnv1a_hash.py": 13.38, - "tests/integration/test_gpio_expander_cache.py": 13.06, - "tests/integration/test_host_logger_thread_safety.py": 23.66, - "tests/integration/test_host_mode_basic.py": 8.01, - "tests/integration/test_host_mode_batch_delay.py": 21.0, - "tests/integration/test_host_mode_climate_basic_state.py": 22.14, - "tests/integration/test_host_mode_climate_control.py": 19.39, - "tests/integration/test_host_mode_empty_string_options.py": 21.76, - "tests/integration/test_host_mode_entity_fields.py": 29.61, - "tests/integration/test_host_mode_fan_preset.py": 20.01, - "tests/integration/test_host_mode_many_entities.py": 39.08, - "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.92, - "tests/integration/test_host_mode_noise_encryption.py": 42.42, - "tests/integration/test_host_mode_reconnect.py": 3.41, - "tests/integration/test_host_mode_sensor.py": 22.96, - "tests/integration/test_host_ota.py": 29.5, - "tests/integration/test_host_preferences.py": 16.06, - "tests/integration/test_host_preferences_suspend_resume.py": 18.71, - "tests/integration/test_improv_serial_uart.py": 20.22, - "tests/integration/test_large_message_batching.py": 26.56, - "tests/integration/test_legacy_area.py": 22.72, - "tests/integration/test_legacy_climate_compat.py": 14.13, - "tests/integration/test_legacy_fan_compat.py": 14.33, - "tests/integration/test_light_automations.py": 18.81, - "tests/integration/test_light_binary_effect_off_phase.py": 8.38, - "tests/integration/test_light_calls.py": 21.88, - "tests/integration/test_light_constant_brightness.py": 59.45, - "tests/integration/test_light_control_action.py": 31.91, - "tests/integration/test_light_dim_relative_action.py": 14.43, - "tests/integration/test_light_effect_zero_brightness.py": 25.05, - "tests/integration/test_light_initial_state.py": 18.97, - "tests/integration/test_light_toggle_action.py": 17.44, - "tests/integration/test_lock_automations.py": 18.9, - "tests/integration/test_logger_buffered_recursion_guard.py": 18.2, - "tests/integration/test_loop_disable_enable.py": 63.35, - "tests/integration/test_loop_interval_decoupling.py": 17.7, - "tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.56, - "tests/integration/test_micros_to_millis.py": 15.89, - "tests/integration/test_multi_click_trigger.py": 17.23, - "tests/integration/test_multi_device_preferences.py": 19.4, - "tests/integration/test_noise_encryption_key_protection.py": 72.59, - "tests/integration/test_object_id_api_verification.py": 19.22, - "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 16.77, - "tests/integration/test_object_id_no_friendly_name.py": 45.8, - "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 86.73, - "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 40.4, - "tests/integration/test_online_image_bmp.py": 37.24, - "tests/integration/test_oversized_payloads.py": 55.75, - "tests/integration/test_preference_key_stability.py": 25.49, - "tests/integration/test_runtime_stats.py": 29.81, - "tests/integration/test_safe_mode_loop_runs.py": 6.26, - "tests/integration/test_scheduler_blocking_warning.py": 37.98, - "tests/integration/test_scheduler_bulk_cleanup.py": 18.67, - "tests/integration/test_scheduler_defer_cancel.py": 18.46, - "tests/integration/test_scheduler_defer_cancel_regular.py": 16.34, - "tests/integration/test_scheduler_defer_fifo_simple.py": 18.26, - "tests/integration/test_scheduler_defer_stress.py": 17.74, - "tests/integration/test_scheduler_heap_stress.py": 3.89, - "tests/integration/test_scheduler_internal_id_no_collision.py": 20.01, - "tests/integration/test_scheduler_interval_reschedule.py": 16.29, - "tests/integration/test_scheduler_interval_zero_coerced.py": 16.09, - "tests/integration/test_scheduler_null_name.py": 14.69, - "tests/integration/test_scheduler_numeric_id_test.py": 17.08, - "tests/integration/test_scheduler_pool.py": 19.88, - "tests/integration/test_scheduler_rapid_cancellation.py": 4.42, - "tests/integration/test_scheduler_recursive_timeout.py": 4.3, - "tests/integration/test_scheduler_removed_item_race.py": 15.49, - "tests/integration/test_scheduler_self_keyed.py": 25.77, - "tests/integration/test_scheduler_simultaneous_callbacks.py": 14.84, - "tests/integration/test_scheduler_string_test.py": 15.42, - "tests/integration/test_script_array_params.py": 12.73, - "tests/integration/test_script_delay_params.py": 12.69, - "tests/integration/test_script_queued.py": 20.38, - "tests/integration/test_script_queued_idle_loop.py": 25.06, - "tests/integration/test_script_wait_on_boot.py": 15.67, - "tests/integration/test_select_stringref_trigger.py": 19.48, - "tests/integration/test_sensor_filters_delta.py": 27.62, - "tests/integration/test_sensor_filters_ring_buffer.py": 20.27, - "tests/integration/test_sensor_filters_sliding_window.py": 56.28, - "tests/integration/test_sensor_filters_value_list.py": 20.6, - "tests/integration/test_sensor_timeout_filter.py": 22.21, - "tests/integration/test_socket_wake_gate_tcp.py": 16.37, - "tests/integration/test_status_flags.py": 29.68, - "tests/integration/test_strftime_to.py": 17.42, - "tests/integration/test_syslog.py": 18.39, - "tests/integration/test_template_alarm_control_panel_many_sensors.py": 25.61, - "tests/integration/test_template_text_save.py": 19.16, - "tests/integration/test_text_command.py": 16.43, - "tests/integration/test_text_sensor_raw_state.py": 17.19, - "tests/integration/test_uart_mock_ld2410.py": 37.0, - "tests/integration/test_uart_mock_ld2412.py": 40.82, - "tests/integration/test_uart_mock_ld2420.py": 32.7, - "tests/integration/test_uart_mock_ld2450.py": 32.84, - "tests/integration/test_uart_mock_modbus.py": 548.87, - "tests/integration/test_udp.py": 16.67, - "tests/integration/test_use_address_runtime.py": 27.26, - "tests/integration/test_valve_control_action.py": 24.58, - "tests/integration/test_varint_five_byte_device_id.py": 22.5, - "tests/integration/test_wait_until_mid_loop_timing.py": 22.05, - "tests/integration/test_wait_until_on_boot.py": 10.37, - "tests/integration/test_wait_until_ordering.py": 18.23, - "tests/integration/test_wait_until_reentrant_restart.py": 19.35, - "tests/integration/test_wake_loop_forces_phase_b.py": 17.83, - "tests/integration/test_water_heater_template.py": 25.7 + "tests/integration/test_action_concurrent_reentry.py": 30.48, + "tests/integration/test_addressable_light_transition.py": 42.1, + "tests/integration/test_alarm_control_panel_state_transitions.py": 35.76, + "tests/integration/test_api_action_metadata.py": 22.35, + "tests/integration/test_api_action_responses.py": 30.31, + "tests/integration/test_api_action_timeout.py": 34.73, + "tests/integration/test_api_conditional_memory.py": 18.35, + "tests/integration/test_api_custom_services.py": 15.99, + "tests/integration/test_api_get_time_response_timezone.py": 24.21, + "tests/integration/test_api_homeassistant.py": 33.77, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 20.8, + "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 23.55, + "tests/integration/test_api_list_entities_backpressure.py": 23.04, + "tests/integration/test_api_message_size_batching.py": 27.31, + "tests/integration/test_api_reboot_timeout.py": 29.32, + "tests/integration/test_api_string_lambda.py": 14.9, + "tests/integration/test_api_vv_logging.py": 26.25, + "tests/integration/test_api_zero_psk_provisioning.py": 38.19, + "tests/integration/test_areas_and_devices.py": 27.52, + "tests/integration/test_automation_wait_actions.py": 24.25, + "tests/integration/test_automations.py": 36.02, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 18.46, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 17.47, + "tests/integration/test_binary_sensor_invalidate_state.py": 14.79, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 21.52, + "tests/integration/test_build_info.py": 21.42, + "tests/integration/test_camera_mock.py": 17.02, + "tests/integration/test_climate_control_action.py": 26.56, + "tests/integration/test_climate_custom_modes.py": 18.82, + "tests/integration/test_continuation_actions.py": 20.39, + "tests/integration/test_cover_control_action.py": 19.91, + "tests/integration/test_crc8_helper.py": 16.73, + "tests/integration/test_device_id_in_state.py": 58.41, + "tests/integration/test_duplicate_entities.py": 30.76, + "tests/integration/test_entity_icon.py": 25.34, + "tests/integration/test_fan_turn_on_action.py": 23.64, + "tests/integration/test_fnv1_hash_object_id.py": 25.44, + "tests/integration/test_fnv1a_hash.py": 20.85, + "tests/integration/test_gpio_expander_cache.py": 14.42, + "tests/integration/test_host_logger_thread_safety.py": 21.31, + "tests/integration/test_host_mode_basic.py": 2.65, + "tests/integration/test_host_mode_batch_delay.py": 22.21, + "tests/integration/test_host_mode_climate_basic_state.py": 27.12, + "tests/integration/test_host_mode_climate_control.py": 21.57, + "tests/integration/test_host_mode_empty_string_options.py": 27.17, + "tests/integration/test_host_mode_entity_fields.py": 30.1, + "tests/integration/test_host_mode_fan_preset.py": 17.55, + "tests/integration/test_host_mode_many_entities.py": 38.98, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.82, + "tests/integration/test_host_mode_noise_encryption.py": 39.84, + "tests/integration/test_host_mode_reconnect.py": 13.1, + "tests/integration/test_host_mode_sensor.py": 22.17, + "tests/integration/test_host_ota.py": 92.05, + "tests/integration/test_host_preferences.py": 20.29, + "tests/integration/test_host_preferences_suspend_resume.py": 15.02, + "tests/integration/test_improv_serial_uart.py": 30.15, + "tests/integration/test_large_message_batching.py": 25.84, + "tests/integration/test_legacy_area.py": 21.24, + "tests/integration/test_legacy_climate_compat.py": 17.34, + "tests/integration/test_legacy_fan_compat.py": 22.6, + "tests/integration/test_light_automations.py": 29.13, + "tests/integration/test_light_binary_effect_off_phase.py": 33.99, + "tests/integration/test_light_calls.py": 26.81, + "tests/integration/test_light_constant_brightness.py": 25.0, + "tests/integration/test_light_control_action.py": 25.57, + "tests/integration/test_light_dim_relative_action.py": 21.4, + "tests/integration/test_light_effect_zero_brightness.py": 19.65, + "tests/integration/test_light_initial_state.py": 17.58, + "tests/integration/test_light_toggle_action.py": 28.28, + "tests/integration/test_lock_automations.py": 23.3, + "tests/integration/test_logger_buffered_recursion_guard.py": 22.96, + "tests/integration/test_loop_disable_enable.py": 16.18, + "tests/integration/test_loop_interval_decoupling.py": 25.19, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 20.59, + "tests/integration/test_lvgl_headless_render.py": 87.78, + "tests/integration/test_micros_to_millis.py": 18.73, + "tests/integration/test_multi_click_trigger.py": 24.2, + "tests/integration/test_multi_device_preferences.py": 20.52, + "tests/integration/test_noise_encryption_key_protection.py": 19.1, + "tests/integration/test_object_id_api_verification.py": 26.24, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 14.88, + "tests/integration/test_object_id_no_friendly_name.py": 61.27, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 82.32, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 46.03, + "tests/integration/test_online_image_bmp.py": 34.21, + "tests/integration/test_oversized_payloads.py": 62.75, + "tests/integration/test_preference_key_stability.py": 26.8, + "tests/integration/test_runtime_stats.py": 28.26, + "tests/integration/test_safe_mode_loop_runs.py": 18.14, + "tests/integration/test_scheduler_blocking_warning.py": 28.7, + "tests/integration/test_scheduler_bulk_cleanup.py": 20.73, + "tests/integration/test_scheduler_defer_cancel.py": 22.99, + "tests/integration/test_scheduler_defer_cancel_regular.py": 21.97, + "tests/integration/test_scheduler_defer_fifo_simple.py": 24.15, + "tests/integration/test_scheduler_defer_stress.py": 23.11, + "tests/integration/test_scheduler_heap_stress.py": 20.2, + "tests/integration/test_scheduler_internal_id_no_collision.py": 23.75, + "tests/integration/test_scheduler_interval_reschedule.py": 15.32, + "tests/integration/test_scheduler_interval_zero_coerced.py": 20.1, + "tests/integration/test_scheduler_null_name.py": 17.43, + "tests/integration/test_scheduler_numeric_id_test.py": 25.51, + "tests/integration/test_scheduler_pool.py": 24.22, + "tests/integration/test_scheduler_rapid_cancellation.py": 24.01, + "tests/integration/test_scheduler_recursive_timeout.py": 22.94, + "tests/integration/test_scheduler_removed_item_race.py": 23.07, + "tests/integration/test_scheduler_self_keyed.py": 18.43, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 21.99, + "tests/integration/test_scheduler_string_test.py": 17.27, + "tests/integration/test_script_array_params.py": 4.59, + "tests/integration/test_script_delay_params.py": 22.46, + "tests/integration/test_script_queued.py": 25.24, + "tests/integration/test_script_queued_idle_loop.py": 5.04, + "tests/integration/test_script_wait_on_boot.py": 21.77, + "tests/integration/test_sdl_headless_screenshot.py": 19.23, + "tests/integration/test_select_stringref_trigger.py": 19.31, + "tests/integration/test_sensor_filters_delta.py": 25.92, + "tests/integration/test_sensor_filters_ring_buffer.py": 22.39, + "tests/integration/test_sensor_filters_sliding_window.py": 57.93, + "tests/integration/test_sensor_filters_value_list.py": 20.32, + "tests/integration/test_sensor_timeout_filter.py": 25.35, + "tests/integration/test_snapshot_display.py": 19.7, + "tests/integration/test_socket_wake_gate_tcp.py": 14.5, + "tests/integration/test_status_flags.py": 33.83, + "tests/integration/test_strftime_to.py": 17.64, + "tests/integration/test_syslog.py": 24.49, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 24.81, + "tests/integration/test_template_climate_basic.py": 15.28, + "tests/integration/test_template_climate_custom_modes.py": 25.07, + "tests/integration/test_template_climate_nonoptimistic.py": 24.25, + "tests/integration/test_template_climate_on_control_ordering.py": 24.09, + "tests/integration/test_template_climate_publish_all_fields.py": 17.78, + "tests/integration/test_template_climate_sensor_push.py": 17.42, + "tests/integration/test_template_climate_set_actions.py": 23.63, + "tests/integration/test_template_climate_two_point_temperature.py": 25.19, + "tests/integration/test_template_text_save.py": 17.88, + "tests/integration/test_text_command.py": 22.71, + "tests/integration/test_text_sensor_raw_state.py": 25.17, + "tests/integration/test_uart_mock_ld2410.py": 58.15, + "tests/integration/test_uart_mock_ld2412.py": 61.14, + "tests/integration/test_uart_mock_ld2420.py": 33.87, + "tests/integration/test_uart_mock_ld2450.py": 26.06, + "tests/integration/test_uart_mock_modbus.py": 391.79, + "tests/integration/test_udp.py": 7.38, + "tests/integration/test_use_address_runtime.py": 24.09, + "tests/integration/test_valve_control_action.py": 23.22, + "tests/integration/test_varint_five_byte_device_id.py": 17.93, + "tests/integration/test_wait_until_mid_loop_timing.py": 22.26, + "tests/integration/test_wait_until_on_boot.py": 17.46, + "tests/integration/test_wait_until_ordering.py": 11.89, + "tests/integration/test_wait_until_reentrant_restart.py": 22.88, + "tests/integration/test_wake_loop_forces_phase_b.py": 16.6, + "tests/integration/test_water_heater_template.py": 19.66 } From c66fa812086f20aead9e56bf42b15f541b2e5bc8 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:09:49 +1200 Subject: [PATCH 098/178] [core] Consolidate setup scripts into a cross-platform setup.py (#18856) --- script/git-hooks/post-checkout | 46 ++- script/setup | 74 +---- script/setup.bat | 29 +- script/setup.py | 222 +++++++++++++ tests/script/test_setup.py | 562 +++++++++++++++++++++++++++++++++ 5 files changed, 827 insertions(+), 106 deletions(-) create mode 100755 script/setup.py create mode 100644 tests/script/test_setup.py diff --git a/script/git-hooks/post-checkout b/script/git-hooks/post-checkout index 853c2b0352..73c1cb0f13 100755 --- a/script/git-hooks/post-checkout +++ b/script/git-hooks/post-checkout @@ -1,27 +1,49 @@ #!/bin/sh # Prepare the dev environment for a new checkout or worktree. # -# Installed into the git hooks directory by script/setup. Deliberately tiny and -# self-contained: it stays valid on branches where script/setup does not exist, -# and simply does nothing there. +# Installed into the git hooks directory by script/setup.py. Deliberately tiny +# and self-contained: it stays valid on branches where the setup script does not +# exist, and simply does nothing there. # $3 is 1 for a branch checkout, 0 for a file checkout. [ "$3" = "1" ] || exit 0 top=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0 -# This also runs on ordinary branch switches, where there is nothing to do. +# This also runs on ordinary branch switches, where there is nothing to do. Both +# layouts are checked because git for Windows runs hooks under its own bundled +# shell, where the environment lives in venv/Scripts rather than venv/bin. [ -x "$top/venv/bin/python" ] && exit 0 -[ -x "$top/script/setup" ] || exit 0 +[ -f "$top/venv/Scripts/python.exe" ] && exit 0 + +# Branches from before the setup script moved to Python carry only the shell +# entry point, so whichever one the checked out branch has is used. +py= +if [ -f "$top/script/setup.py" ]; then + # The interpreter goes by different names across platforms, and on Windows + # "python3" is often a stub that opens the app store instead of running + # anything, so each candidate is tried before it is used. Doing nothing is the + # right outcome when none of them work. + for candidate in "python3" "python" "py -3"; do + # Unquoted on purpose: the launcher candidate is a command plus a flag. + if $candidate -c "" >/dev/null 2>&1; then + py=$candidate + break + fi + done + [ -n "$py" ] || exit 0 +elif ! [ -x "$top/script/setup" ]; then + exit 0 +fi # Every worktree shares the hooks directory of the checkout it was created -# from, and the script/setup run below is the one from whichever branch was just +# from, and the setup script run below is the one from whichever branch was just # checked out. Older branches install their own pre-commit hook without checking # for a worktree: that moves the shared hook aside as pre-commit.legacy and # replaces it with one tied to this worktree's virtual environment, so commits # break in every checkout. To rule that out, the hooks directory is copied -# before script/setup runs and put back exactly as it was afterwards, including -# removing any file script/setup added. +# before the setup script runs and put back exactly as it was afterwards, +# including removing any file the setup script added. hooks=$(git rev-parse --path-format=absolute --git-path hooks 2>/dev/null) || exit 0 snap=$(mktemp -d "$hooks/.post-checkout.XXXXXX") || exit 0 cp -p "$hooks"/* "$snap"/ 2>/dev/null @@ -29,7 +51,13 @@ cp -p "$hooks"/* "$snap"/ 2>/dev/null # Clear VIRTUAL_ENV so a checkout made from a shell with an environment already # activated still gets its own, rather than having the active one repointed at # this working tree. -env -u VIRTUAL_ENV "$top/script/setup" +unset VIRTUAL_ENV +if [ -n "$py" ]; then + # Unquoted on purpose, as above. + $py "$top/script/setup.py" +else + "$top/script/setup" +fi status=$? for f in "$hooks"/*; do diff --git a/script/setup b/script/setup index b96af6e8f3..91bcb88154 100755 --- a/script/setup +++ b/script/setup @@ -1,71 +1,7 @@ #!/usr/bin/env bash -# Set up ESPHome dev environment +# Set up ESPHome dev environment. +# +# The work is done by setup.py, which script/setup.bat also runs, so the Unix +# and Windows entry points share one implementation. -set -e - -cd "$(dirname "$0")/.." -if [ -n "$VIRTUAL_ENV" ]; then - # A virtual environment is already active (e.g. the devcontainer's pre-provisioned - # esphome-venv). Install into it rather than creating a ./venv in the workspace. - venv_state=active -elif [ -x venv/bin/python ]; then - # Reuse the environment from an earlier run, so this script can be run again - # at any time to pick up dependency changes. - venv_state=reused - source venv/bin/activate -else - venv_state=created - # --clear replaces a partial environment left behind by an interrupted run. - if [ -x "$(command -v uv)" ]; then - uv venv --clear --seed venv - else - python3 -m venv --clear venv - fi - source venv/bin/activate -fi - -if ! [ -x "$(command -v uv)" ]; then - python3 -m pip install uv -fi - -uv pip install setuptools wheel -uv pip install -e ".[dev,test]" --config-settings editable_mode=compat - -# A worktree shares one git hooks directory with the main checkout it was -# created from, so hooks are installed from the main checkout only. Installing -# from a worktree would point the shared hook at that worktree's virtual -# environment, breaking it for everyone once the worktree is removed. -git_dir="$(git rev-parse --absolute-git-dir 2>/dev/null || true)" -common_dir="$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" -if [ -n "$common_dir" ] && [ "$git_dir" = "$common_dir" ]; then - # --overwrite replaces any hook already in place. Without it, prek finds a - # previously installed pre-commit hook, moves it aside to - # .git/hooks/pre-commit.legacy and keeps calling it, so every commit would - # run both tools. - prek install --overwrite - - # Prepares the virtual environment for new checkouts and worktrees. Installed - # once here, it covers every worktree created from this checkout. - if [ -d "$common_dir/hooks" ]; then - cp script/git-hooks/post-checkout "$common_dir/hooks/post-checkout" - chmod +x "$common_dir/hooks/post-checkout" - fi -fi - -mkdir -p .temp - -echo -echo -case "$venv_state" in - created) - echo "Virtual environment created at ./venv. Run 'source venv/bin/activate' to use it." - ;; - reused) - echo "Dependencies updated in the existing ./venv. Run 'source venv/bin/activate' to use it." - ;; - active) - echo "Dependencies installed into the active virtual environment:" - echo " $VIRTUAL_ENV" - echo "It is already active in this shell, so no 'source venv/bin/activate' is needed." - ;; -esac +exec python3 "$(dirname "$0")/setup.py" "$@" diff --git a/script/setup.bat b/script/setup.bat index 809d05ae93..405121b139 100644 --- a/script/setup.bat +++ b/script/setup.bat @@ -1,28 +1 @@ -@echo off - -if defined VIRTUAL_ENV goto :install - -echo Starting the Virtual Environment -python -m venv venv -call venv/Scripts/activate -echo Running the Virtual Environment - -:install - -echo Installing required packages... - -python.exe -m pip install --upgrade pip - -pip3 install -r requirements.txt -r requirements_test.txt -r requirements_dev.txt -pip3 install setuptools wheel -pip3 install -e ".[dev,test]" --config-settings editable_mode=compat - -rem --overwrite replaces any hook already in place. Without it, prek finds a -rem previously installed pre-commit hook, moves it aside to -rem .git/hooks/pre-commit.legacy and keeps calling it, so every commit would -rem run both tools. -prek install --overwrite - -echo . -echo . -echo Virtual environment created. Run 'venv/Scripts/activate' to use it. +@python "%~dp0setup.py" %* diff --git a/script/setup.py b/script/setup.py new file mode 100755 index 0000000000..62129b8c05 --- /dev/null +++ b/script/setup.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Set up the ESPHome development environment. + +Shared implementation behind script/setup and script/setup.bat, so the Unix and +Windows entry points cannot drift apart. Uses only the standard library: it runs +before any dependency has been installed. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import sysconfig + +MIN_PYTHON = (3, 12) + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_VENV = ROOT / "venv" +POST_CHECKOUT_HOOK = ROOT / "script" / "git-hooks" / "post-checkout" + +# State of the environment the dependencies end up in, used for the closing +# message. +VENV_ACTIVE = "active" +VENV_REUSED = "reused" +VENV_CREATED = "created" + + +def bin_dir(venv: Path) -> Path: + """Return the directory holding a virtual environment's executables. + + The "venv" scheme resolves to bin on Unix and Scripts on Windows, so the + layout does not have to be hardcoded here. + """ + base = str(venv) + return Path( + sysconfig.get_path("scripts", "venv", vars={"base": base, "platbase": base}) + ) + + +def venv_python(venv: Path) -> Path: + """Return the path to a virtual environment's interpreter.""" + name = "python.exe" if os.name == "nt" else "python" + return bin_dir(venv) / name + + +def run(command: list[str], env: dict[str, str] | None = None) -> None: + """Run a command, aborting the whole script if it fails.""" + print(f"+ {' '.join(command)}", flush=True) + result = subprocess.run(command, cwd=ROOT, env=env, check=False) + if result.returncode != 0: + # Some tools fail without printing anything, so name the step that broke. + print( + f"Failed with exit code {result.returncode}: {command[0]}", file=sys.stderr + ) + raise SystemExit(result.returncode) + + +def git_output(*args: str) -> str: + """Return the trimmed output of a git command, or "" if it cannot be run.""" + try: + result = subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=False + ) + except OSError: + # Git is not required to install the dependencies, only to install hooks. + return "" + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def create_venv(venv: Path) -> None: + """Create a virtual environment, replacing anything already at the path.""" + # --clear replaces a partial environment left behind by an interrupted run. + if (uv := shutil.which("uv")) is not None: + run([uv, "venv", "--clear", "--seed", str(venv)]) + else: + run([sys.executable, "-m", "venv", "--clear", str(venv)]) + + +def venv_environment(venv: Path) -> dict[str, str]: + """Return the environment child processes need to target a virtual env. + + Equivalent to sourcing the environment's activate script: tools such as uv + and prek pick the environment up from VIRTUAL_ENV and PATH. + """ + env = dict(os.environ) + env["VIRTUAL_ENV"] = str(venv) + env.pop("PYTHONHOME", None) + path = str(bin_dir(venv)) + # An empty entry would be appended if PATH is unset, and on Unix that means + # the working directory is searched for executables. + if existing := env.get("PATH"): + path = os.pathsep.join([path, existing]) + env["PATH"] = path + return env + + +def find_uv(venv: Path, env: dict[str, str]) -> str: + """Return the path to uv, installing it into the environment if needed.""" + if (uv := shutil.which("uv", path=env["PATH"])) is not None: + return uv + run([str(venv_python(venv)), "-m", "pip", "install", "uv"], env=env) + if (uv := shutil.which("uv", path=env["PATH"])) is not None: + return uv + raise SystemExit("uv could not be installed, aborting.") + + +def install_dependencies(venv: Path, env: dict[str, str]) -> None: + """Install ESPHome and its development dependencies into the environment.""" + uv = find_uv(venv, env) + run([uv, "pip", "install", "setuptools", "wheel"], env=env) + # The dev and test extras pull in requirements_dev.txt and + # requirements_test.txt, and the package itself pulls in requirements.txt, + # so this single install covers every requirements file. + run( + [ + uv, + "pip", + "install", + "-e", + ".[dev,test]", + "--config-settings", + "editable_mode=compat", + ], + env=env, + ) + + +def install_git_hooks(env: dict[str, str]) -> None: + """Install the git hooks, but only when run from the main checkout. + + A worktree shares one git hooks directory with the main checkout it was + created from. Installing from a worktree would point the shared hook at that + worktree's virtual environment, breaking it for everyone once the worktree is + removed. + """ + git_dir = git_output("rev-parse", "--absolute-git-dir") + common_dir = git_output("rev-parse", "--path-format=absolute", "--git-common-dir") + if not git_dir or not common_dir or Path(git_dir) != Path(common_dir): + return + + prek = shutil.which("prek", path=env["PATH"]) + if prek is None: + raise SystemExit("prek was not installed, aborting.") + # --overwrite replaces any hook already in place. Without it, prek finds a + # previously installed pre-commit hook, moves it aside to + # .git/hooks/pre-commit.legacy and keeps calling it, so every commit would + # run both tools. + run([prek, "install", "--overwrite"], env=env) + + # Prepares the virtual environment for new checkouts and worktrees. Installed + # once here, it covers every worktree created from this checkout. + hooks_dir = Path(common_dir) / "hooks" + if hooks_dir.is_dir(): + installed = hooks_dir / "post-checkout" + shutil.copyfile(POST_CHECKOUT_HOOK, installed) + installed.chmod(0o755) + + +def activate_hint() -> str: + """Return the command that activates the environment this script creates.""" + activate = bin_dir(DEFAULT_VENV).relative_to(ROOT) / "activate" + if os.name == "nt": + return str(activate) + return f"source {activate.as_posix()}" + + +def report(state: str, venv: Path) -> None: + """Print the closing message for the environment that was set up.""" + location = f"./{DEFAULT_VENV.name}" + print() + print() + if state == VENV_ACTIVE: + print("Dependencies installed into the active virtual environment:") + print(f" {venv}") + print( + f"It is already active in this shell, so no '{activate_hint()}' is needed." + ) + elif state == VENV_REUSED: + print( + f"Dependencies updated in the existing {location}. " + f"Run '{activate_hint()}' to use it." + ) + else: + print( + f"Virtual environment created at {location}. " + f"Run '{activate_hint()}' to use it." + ) + + +def main() -> None: + """Set up the development environment.""" + if sys.version_info < MIN_PYTHON: + raise SystemExit( + f"ESPHome needs Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]} or newer, " + f"but this is Python {sys.version.split()[0]}." + ) + + # A virtual environment that is already active (for example the + # devcontainer's pre-provisioned esphome-venv) is installed into rather than + # creating a ./venv in the workspace. + if active := os.environ.get("VIRTUAL_ENV"): + state, venv = VENV_ACTIVE, Path(active) + elif venv_python(DEFAULT_VENV).is_file(): + # Reuse the environment from an earlier run, so this script can be run + # again at any time to pick up dependency changes. + state, venv = VENV_REUSED, DEFAULT_VENV + else: + state, venv = VENV_CREATED, DEFAULT_VENV + create_venv(venv) + + env = venv_environment(venv) + install_dependencies(venv, env) + install_git_hooks(env) + (ROOT / ".temp").mkdir(exist_ok=True) + report(state, venv) + + +if __name__ == "__main__": + main() diff --git a/tests/script/test_setup.py b/tests/script/test_setup.py new file mode 100644 index 0000000000..3e816c4b05 --- /dev/null +++ b/tests/script/test_setup.py @@ -0,0 +1,562 @@ +"""Tests for script/setup.py.""" + +import importlib.util +import os +from pathlib import Path, PurePosixPath, PureWindowsPath +import runpy +import sys +from types import ModuleType +from unittest.mock import Mock, call, patch + +import pytest + +_SCRIPT = Path(__file__).parents[2] / "script" / "setup.py" + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location("script_setup", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def script_setup() -> ModuleType: + """Fresh import of script/setup.py, isolated from other tests.""" + return _load_module() + + +# --- bin_dir / venv_python / activate_hint ----------------------------------- + + +def test_bin_dir_matches_host_layout(script_setup: ModuleType, tmp_path: Path) -> None: + """The venv scheme resolves to Scripts on Windows and bin everywhere else.""" + expected = "Scripts" if os.name == "nt" else "bin" + assert script_setup.bin_dir(tmp_path) == tmp_path / expected + + +# Both flavours are exercised on every host. Pure paths are used because a real +# Path refuses to change flavour: PosixPath cannot be built on Windows, and +# WindowsPath cannot be built on Unix. + + +def test_venv_python_posix(script_setup: ModuleType, tmp_path: Path) -> None: + with ( + patch.object( + script_setup, "bin_dir", return_value=PurePosixPath("/x/venv/bin") + ), + patch.object(script_setup.os, "name", "posix"), + ): + result = script_setup.venv_python(tmp_path) + assert result == PurePosixPath("/x/venv/bin/python") + + +def test_venv_python_nt(script_setup: ModuleType, tmp_path: Path) -> None: + with ( + patch.object( + script_setup, "bin_dir", return_value=PureWindowsPath(r"C:\x\venv\Scripts") + ), + patch.object(script_setup.os, "name", "nt"), + ): + result = script_setup.venv_python(tmp_path) + assert result == PureWindowsPath(r"C:\x\venv\Scripts\python.exe") + + +def test_activate_hint_posix(script_setup: ModuleType) -> None: + with ( + patch.object(script_setup, "ROOT", PurePosixPath("/x")), + patch.object( + script_setup, "bin_dir", return_value=PurePosixPath("/x/venv/bin") + ), + patch.object(script_setup.os, "name", "posix"), + ): + hint = script_setup.activate_hint() + assert hint == "source venv/bin/activate" + + +def test_activate_hint_nt(script_setup: ModuleType) -> None: + with ( + patch.object(script_setup, "ROOT", PureWindowsPath(r"C:\x")), + patch.object( + script_setup, "bin_dir", return_value=PureWindowsPath(r"C:\x\venv\Scripts") + ), + patch.object(script_setup.os, "name", "nt"), + ): + hint = script_setup.activate_hint() + # The nt branch returns str(activate) as-is, skipping the "source " prefix. + assert hint == r"venv\Scripts\activate" + + +# --- run ----------------------------------------------------------------- + + +def test_run_success(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run: + script_setup.run(["echo", "hi"]) + mock_run.assert_called_once_with( + ["echo", "hi"], cwd=script_setup.ROOT, env=None, check=False + ) + + +def test_run_failure_raises_system_exit_with_code( + script_setup: ModuleType, capsys: pytest.CaptureFixture[str] +) -> None: + with ( + patch.object(script_setup.subprocess, "run", return_value=Mock(returncode=7)), + pytest.raises(SystemExit) as excinfo, + ): + script_setup.run(["false"]) + assert excinfo.value.code == 7 + assert "Failed with exit code 7: false" in capsys.readouterr().err + + +# --- git_output ------------------------------------------------------------ + + +def test_git_output_success_strips_stdout(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, + "run", + return_value=Mock(returncode=0, stdout=" /repo/.git \n"), + ) as mock_run: + result = script_setup.git_output("rev-parse", "--absolute-git-dir") + assert result == "/repo/.git" + mock_run.assert_called_once_with( + ["git", "rev-parse", "--absolute-git-dir"], + cwd=script_setup.ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def test_git_output_nonzero_returncode_is_empty(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, + "run", + return_value=Mock(returncode=1, stdout="whatever"), + ): + assert script_setup.git_output("status") == "" + + +def test_git_output_oserror_is_empty(script_setup: ModuleType) -> None: + with patch.object(script_setup.subprocess, "run", side_effect=OSError("no git")): + assert script_setup.git_output("status") == "" + + +# --- create_venv ----------------------------------------------------------- + + +def test_create_venv_uses_uv_when_present( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.create_venv(venv) + mock_run.assert_called_once_with( + ["/usr/bin/uv", "venv", "--clear", "--seed", str(venv)], + cwd=script_setup.ROOT, + env=None, + check=False, + ) + + +def test_create_venv_falls_back_to_venv_module( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + with ( + patch.object(script_setup.shutil, "which", return_value=None), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.create_venv(venv) + mock_run.assert_called_once_with( + [sys.executable, "-m", "venv", "--clear", str(venv)], + cwd=script_setup.ROOT, + env=None, + check=False, + ) + + +# --- venv_environment -------------------------------------------------------- + + +def test_venv_environment_sets_virtual_env_and_prepends_path( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + venv = tmp_path / "venv" + monkeypatch.setenv("PYTHONHOME", "/somewhere") + monkeypatch.setenv("PATH", "/usr/bin:/bin") + env = script_setup.venv_environment(venv) + assert env["VIRTUAL_ENV"] == str(venv) + assert "PYTHONHOME" not in env + expected_prefix = str(script_setup.bin_dir(venv)) + os.pathsep + assert env["PATH"] == expected_prefix + "/usr/bin:/bin" + + +def test_venv_environment_path_fallback_when_unset( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + venv = tmp_path / "venv" + monkeypatch.delenv("PATH", raising=False) + env = script_setup.venv_environment(venv) + # No trailing separator: an empty PATH entry means "search the cwd". + assert env["PATH"] == str(script_setup.bin_dir(venv)) + + +# --- find_uv ----------------------------------------------------------------- + + +def test_find_uv_found_immediately(script_setup: ModuleType, tmp_path: Path) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + result = script_setup.find_uv(venv, env) + assert result == "/usr/bin/uv" + mock_run.assert_not_called() + + +def test_find_uv_installed_then_found(script_setup: ModuleType, tmp_path: Path) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", side_effect=[None, "/usr/bin/uv"]), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + result = script_setup.find_uv(venv, env) + assert result == "/usr/bin/uv" + mock_run.assert_called_once_with( + [str(script_setup.venv_python(venv)), "-m", "pip", "install", "uv"], + cwd=script_setup.ROOT, + env=env, + check=False, + ) + + +def test_find_uv_still_missing_raises_system_exit( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", side_effect=[None, None]), + patch.object(script_setup.subprocess, "run", return_value=Mock(returncode=0)), + pytest.raises(SystemExit, match="uv could not be installed"), + ): + script_setup.find_uv(venv, env) + + +# --- install_dependencies ----------------------------------------------------- + + +def test_install_dependencies_installs_setuptools_then_project( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_dependencies(venv, env) + assert mock_run.call_args_list == [ + call( + ["/usr/bin/uv", "pip", "install", "setuptools", "wheel"], + cwd=script_setup.ROOT, + env=env, + check=False, + ), + call( + [ + "/usr/bin/uv", + "pip", + "install", + "-e", + ".[dev,test]", + "--config-settings", + "editable_mode=compat", + ], + cwd=script_setup.ROOT, + env=env, + check=False, + ), + ] + + +# --- install_git_hooks --------------------------------------------------------- + + +def _fake_git_output(git_dir: str, common_dir: str): + def _run(*args: str) -> str: + if "--absolute-git-dir" in args: + return git_dir + return common_dir + + return _run + + +def test_install_git_hooks_returns_early_when_git_dir_empty( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, "git_output", side_effect=_fake_git_output("", "/repo/.git") + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_returns_early_when_common_dir_empty( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, "git_output", side_effect=_fake_git_output("/repo/.git", "") + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_returns_early_for_worktree( + script_setup: ModuleType, +) -> None: + """A worktree's git-dir differs from the shared common-dir.""" + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output("/repo/.git/worktrees/wt", "/repo/.git"), + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_missing_prek_raises_system_exit( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output("/repo/.git", "/repo/.git"), + ), + patch.object(script_setup.shutil, "which", return_value=None), + patch.object(script_setup.subprocess, "run") as mock_run, + pytest.raises(SystemExit, match="prek was not installed"), + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_happy_path_installs_hook( + script_setup: ModuleType, tmp_path: Path +) -> None: + env = {"PATH": "/usr/bin"} + common_dir = tmp_path / "repo" / ".git" + hooks_dir = common_dir / "hooks" + hooks_dir.mkdir(parents=True) + source_hook = tmp_path / "post-checkout" + source_hook.write_text("#!/bin/sh\necho post-checkout\n") + + with ( + patch.object(script_setup, "POST_CHECKOUT_HOOK", source_hook), + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output(str(common_dir), str(common_dir)), + ), + patch.object(script_setup.shutil, "which", return_value="/usr/bin/prek"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_git_hooks(env) + + mock_run.assert_called_once_with( + ["/usr/bin/prek", "install", "--overwrite"], + cwd=script_setup.ROOT, + env=env, + check=False, + ) + installed = hooks_dir / "post-checkout" + assert installed.read_text() == source_hook.read_text() + if os.name != "nt": + # Windows has no POSIX permission bits for chmod to set. + assert (installed.stat().st_mode & 0o777) == 0o755 + + +def test_install_git_hooks_skips_copy_when_hooks_dir_missing( + script_setup: ModuleType, tmp_path: Path +) -> None: + """The prek install still runs when the hooks directory does not exist.""" + env = {"PATH": "/usr/bin"} + common_dir = tmp_path / "repo" / ".git" + common_dir.mkdir(parents=True) # no "hooks" subdirectory created + + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output(str(common_dir), str(common_dir)), + ), + patch.object(script_setup.shutil, "which", return_value="/usr/bin/prek"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_git_hooks(env) + + mock_run.assert_called_once() + assert not (common_dir / "hooks").exists() + + +# --- report ------------------------------------------------------------------ + + +def test_report_active_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + venv = Path("/opt/esphome-venv") + script_setup.report(script_setup.VENV_ACTIVE, venv) + out = capsys.readouterr().out + assert "Dependencies installed into the active virtual environment:" in out + assert str(venv) in out + assert "is already active in this shell" in out + + +def test_report_reused_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + script_setup.report(script_setup.VENV_REUSED, script_setup.DEFAULT_VENV) + out = capsys.readouterr().out + assert "Dependencies updated in the existing ./venv" in out + + +def test_report_created_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + script_setup.report(script_setup.VENV_CREATED, script_setup.DEFAULT_VENV) + out = capsys.readouterr().out + assert "Virtual environment created at ./venv" in out + + +# --- main -------------------------------------------------------------------- + + +def test_main_raises_system_exit_when_python_too_old( + script_setup: ModuleType, +) -> None: + with ( + patch.object(script_setup.sys, "version_info", (3, 11, 5)), + pytest.raises(SystemExit, match="ESPHome needs Python 3.12"), + ): + script_setup.main() + + +def test_main_uses_active_virtual_env( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + active_venv = tmp_path / "active-venv" + monkeypatch.setenv("VIRTUAL_ENV", str(active_venv)) + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_not_called() + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_ACTIVE, active_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_main_reuses_existing_venv( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + default_venv = tmp_path / "venv" + python_path = script_setup.venv_python(default_venv) + python_path.parent.mkdir(parents=True) + python_path.touch() + + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "DEFAULT_VENV", default_venv), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_not_called() + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_REUSED, default_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_main_creates_new_venv( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + default_venv = tmp_path / "venv" # does not exist yet + + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "DEFAULT_VENV", default_venv), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_called_once_with(default_venv) + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_CREATED, default_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_run_as_script_calls_main(tmp_path: Path) -> None: + """The __main__ guard runs the whole flow, with every side effect stubbed.""" + completed = Mock(returncode=0, stdout="") + with ( + patch("subprocess.run", return_value=completed) as mock_run, + patch("shutil.which", return_value="/usr/bin/uv"), + patch("pathlib.Path.mkdir") as mock_mkdir, + patch.dict(os.environ, {"VIRTUAL_ENV": str(tmp_path / "env")}), + ): + runpy.run_path(str(_SCRIPT), run_name="__main__") + + # The dependency install ran, and git reported no hooks directory to touch. + assert mock_run.called + mock_mkdir.assert_called_once_with(exist_ok=True) From 4868b498cf80cf6fb6c59544a84797f74736bbfe Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:09:58 +1200 Subject: [PATCH 099/178] [ci] Ask stale PR authors to merge dev instead of rebasing (#19064) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index aa31094f81..38d2418ac6 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -33,7 +33,7 @@ jobs: and will be closed if no further activity occurs within 7 days. If you are the author of this PR, please leave a comment if you want - to keep it open. Also, please rebase your PR onto the latest dev + to keep it open. Also, please merge the latest dev branch into your branch to ensure that it's up to date with the latest changes. Thank you for your contribution! From 5ed59af9204af5e4a4638d10379375b598dd8ace Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:11:13 +1200 Subject: [PATCH 100/178] [template] Surface value metadata on template entity forms (#17545) --- .../template/binary_sensor/__init__.py | 14 +++- .../components/template/button/__init__.py | 6 +- esphome/components/template/cover/__init__.py | 7 +- esphome/components/template/event/__init__.py | 6 +- .../components/template/number/__init__.py | 9 ++- .../components/template/sensor/__init__.py | 22 +++++- .../components/template/switch/__init__.py | 7 +- .../template/text_sensor/__init__.py | 8 +- esphome/components/template/valve/__init__.py | 7 +- esphome/config_validation.py | 32 ++++++++ .../template/test_template_visibility.py | 76 +++++++++++++++++++ tests/unit_tests/test_config_validation.py | 29 +++++++ 12 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 tests/component_tests/template/test_template_visibility.py diff --git a/esphome/components/template/binary_sensor/__init__.py b/esphome/components/template/binary_sensor/__init__.py index 8f57df91c5..07028f7dff 100644 --- a/esphome/components/template/binary_sensor/__init__.py +++ b/esphome/components/template/binary_sensor/__init__.py @@ -2,7 +2,13 @@ from esphome import automation import esphome.codegen as cg from esphome.components import binary_sensor import esphome.config_validation as cv -from esphome.const import CONF_CONDITION, CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import ( + CONF_CONDITION, + CONF_DEVICE_CLASS, + CONF_ID, + CONF_LAMBDA, + CONF_STATE, +) from esphome.cpp_generator import LambdaExpression from .. import template_ns @@ -12,7 +18,11 @@ TemplateBinarySensor = template_ns.class_( ) CONFIG_SCHEMA = ( - binary_sensor.binary_sensor_schema(TemplateBinarySensor) + cv.with_visibility( + binary_sensor.binary_sensor_schema(TemplateBinarySensor), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Exclusive(CONF_LAMBDA, CONF_CONDITION): cv.returning_lambda, diff --git a/esphome/components/template/button/__init__.py b/esphome/components/template/button/__init__.py index e0101dfc8f..9c6fa13c19 100644 --- a/esphome/components/template/button/__init__.py +++ b/esphome/components/template/button/__init__.py @@ -1,10 +1,14 @@ from esphome.components import button +import esphome.config_validation as cv +from esphome.const import CONF_DEVICE_CLASS from .. import template_ns TemplateButton = template_ns.class_("TemplateButton", button.Button) -CONFIG_SCHEMA = button.button_schema(TemplateButton) +CONFIG_SCHEMA = cv.with_visibility( + button.button_schema(TemplateButton), cv.Visibility.UI, CONF_DEVICE_CLASS +) async def to_code(config): diff --git a/esphome/components/template/cover/__init__.py b/esphome/components/template/cover/__init__.py index 7cb50df84c..0e6f96e9f5 100644 --- a/esphome/components/template/cover/__init__.py +++ b/esphome/components/template/cover/__init__.py @@ -6,6 +6,7 @@ from esphome.const import ( CONF_ASSUMED_STATE, CONF_CLOSE_ACTION, CONF_CURRENT_OPERATION, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPEN_ACTION, @@ -38,7 +39,11 @@ CONF_HAS_POSITION = "has_position" CONF_TOGGLE_ACTION = "toggle_action" CONFIG_SCHEMA = ( - cover.cover_schema(TemplateCover) + cv.with_visibility( + cover.cover_schema(TemplateCover), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/components/template/event/__init__.py b/esphome/components/template/event/__init__.py index cf9c7f4c3d..bdcbd456d5 100644 --- a/esphome/components/template/event/__init__.py +++ b/esphome/components/template/event/__init__.py @@ -1,7 +1,7 @@ import esphome.codegen as cg from esphome.components import event import esphome.config_validation as cv -from esphome.const import CONF_EVENT_TYPES +from esphome.const import CONF_DEVICE_CLASS, CONF_EVENT_TYPES from .. import template_ns @@ -9,7 +9,9 @@ CODEOWNERS = ["@nohat"] TemplateEvent = template_ns.class_("TemplateEvent", event.Event, cg.Component) -CONFIG_SCHEMA = event.event_schema(TemplateEvent).extend( +CONFIG_SCHEMA = cv.with_visibility( + event.event_schema(TemplateEvent), cv.Visibility.UI, CONF_DEVICE_CLASS +).extend( { cv.Required(CONF_EVENT_TYPES): cv.ensure_list(cv.string_strict), } diff --git a/esphome/components/template/number/__init__.py b/esphome/components/template/number/__init__.py index 2f4c9cbffe..3b6485fec3 100644 --- a/esphome/components/template/number/__init__.py +++ b/esphome/components/template/number/__init__.py @@ -3,6 +3,7 @@ import esphome.codegen as cg from esphome.components import number import esphome.config_validation as cv from esphome.const import ( + CONF_DEVICE_CLASS, CONF_ID, CONF_INITIAL_VALUE, CONF_LAMBDA, @@ -12,6 +13,7 @@ from esphome.const import ( CONF_RESTORE_VALUE, CONF_SET_ACTION, CONF_STEP, + CONF_UNIT_OF_MEASUREMENT, ) from .. import template_ns @@ -46,7 +48,12 @@ def validate(config): CONFIG_SCHEMA = cv.All( - number.number_schema(TemplateNumber) + cv.with_visibility( + number.number_schema(TemplateNumber), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + CONF_UNIT_OF_MEASUREMENT, + ) .extend( { cv.Required(CONF_MAX_VALUE): cv.float_, diff --git a/esphome/components/template/sensor/__init__.py b/esphome/components/template/sensor/__init__.py index 0c875bba0f..55537a5636 100644 --- a/esphome/components/template/sensor/__init__.py +++ b/esphome/components/template/sensor/__init__.py @@ -2,7 +2,16 @@ from esphome import automation import esphome.codegen as cg from esphome.components import sensor import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import ( + CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, + CONF_FORCE_UPDATE, + CONF_ID, + CONF_LAMBDA, + CONF_STATE, + CONF_STATE_CLASS, + CONF_UNIT_OF_MEASUREMENT, +) from .. import template_ns @@ -11,9 +20,14 @@ TemplateSensor = template_ns.class_( ) CONFIG_SCHEMA = ( - sensor.sensor_schema( - TemplateSensor, - accuracy_decimals=1, + cv.with_visibility( + sensor.sensor_schema(TemplateSensor, accuracy_decimals=1), + cv.Visibility.UI, + CONF_UNIT_OF_MEASUREMENT, + CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, + CONF_STATE_CLASS, + CONF_FORCE_UPDATE, ) .extend( { diff --git a/esphome/components/template/switch/__init__.py b/esphome/components/template/switch/__init__.py index ca986365ed..37303abb0d 100644 --- a/esphome/components/template/switch/__init__.py +++ b/esphome/components/template/switch/__init__.py @@ -4,6 +4,7 @@ from esphome.components import switch import esphome.config_validation as cv from esphome.const import ( CONF_ASSUMED_STATE, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPTIMISTIC, @@ -31,7 +32,11 @@ def validate(config): CONFIG_SCHEMA = cv.All( - switch.switch_schema(TemplateSwitch) + cv.with_visibility( + switch.switch_schema(TemplateSwitch), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/components/template/text_sensor/__init__.py b/esphome/components/template/text_sensor/__init__.py index ddbdd6dadb..77f5c2ff7c 100644 --- a/esphome/components/template/text_sensor/__init__.py +++ b/esphome/components/template/text_sensor/__init__.py @@ -3,7 +3,7 @@ import esphome.codegen as cg from esphome.components import text_sensor from esphome.components.text_sensor import TextSensorPublishAction import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_STATE from .. import template_ns @@ -12,7 +12,11 @@ TemplateTextSensor = template_ns.class_( ) CONFIG_SCHEMA = ( - text_sensor.text_sensor_schema() + cv.with_visibility( + text_sensor.text_sensor_schema(), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.GenerateID(): cv.declare_id(TemplateTextSensor), diff --git a/esphome/components/template/valve/__init__.py b/esphome/components/template/valve/__init__.py index a2d0c19880..11b35dad23 100644 --- a/esphome/components/template/valve/__init__.py +++ b/esphome/components/template/valve/__init__.py @@ -6,6 +6,7 @@ from esphome.const import ( CONF_ASSUMED_STATE, CONF_CLOSE_ACTION, CONF_CURRENT_OPERATION, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPEN_ACTION, @@ -36,7 +37,11 @@ CONF_HAS_POSITION = "has_position" CONF_TOGGLE_ACTION = "toggle_action" CONFIG_SCHEMA = ( - valve.valve_schema(TemplateValve) + cv.with_visibility( + valve.valve_schema(TemplateValve), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 685a9d04b3..a38fb2ed82 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -4,6 +4,7 @@ from __future__ import annotations from collections.abc import Callable from contextlib import contextmanager, suppress +import copy from datetime import datetime from ipaddress import ( AddressValueError, @@ -419,6 +420,37 @@ class Required(vol.Required): self.visibility: Visibility | None = visibility +def with_visibility(schema: Schema, visibility: Visibility, *keys: str) -> Schema: + """Return a copy of ``schema`` with the given ``keys`` re-marked at ``visibility``. + + Lets a platform override the editor :class:`Visibility` of fields it + inherits from a shared schema builder — without that builder needing a + visibility parameter of its own. The canonical use is a ``template`` + platform promoting the value metadata its user is expected to define + (``device_class``, ``unit_of_measurement``, …) onto the main form: + + CONFIG_SCHEMA = cv.with_visibility( + sensor.sensor_schema(TemplateSensor), + cv.Visibility.UI, + CONF_DEVICE_CLASS, CONF_UNIT_OF_MEASUREMENT, + ) + + The original marker's key, default and validator are preserved; only the + visibility changes, and the input ``schema`` is left untouched. Raises if + a requested key is not present so typos fail at schema-build time. + """ + wanted = {str(k) for k in keys} + overrides = {} + for marker, validator in schema.schema.items(): + if str(marker) in wanted: + marker = copy.copy(marker) + marker.visibility = visibility + overrides[marker] = validator + if missing := wanted - {str(m) for m in overrides}: + raise ValueError(f"with_visibility: keys not in schema: {sorted(missing)}") + return schema.extend(overrides) + + class FinalExternalInvalid(Invalid): """Represents an invalid value in the final validation phase where the path should not be prepended.""" diff --git a/tests/component_tests/template/test_template_visibility.py b/tests/component_tests/template/test_template_visibility.py new file mode 100644 index 0000000000..a50a27e1f7 --- /dev/null +++ b/tests/component_tests/template/test_template_visibility.py @@ -0,0 +1,76 @@ +"""The template platforms surface value-describing metadata on the main form. + +Hardware platforms get sensible defaults for unit/device_class/etc., so those +fields fall through to the editor's advanced disclosure. A ``template`` entity +has no such defaults -- the user is expected to define them -- so the template +platforms pass ``visibility=cv.Visibility.UI`` to promote them onto the form. +""" + +from __future__ import annotations + +import importlib + +import pytest + +import esphome.config_validation as cv + + +def _markers(schema: cv.Schema) -> dict[str, object]: + s = schema + if hasattr(s, "validators"): + # cv.All -> the schema is the first validator. + s = s.validators[0] + return {str(k): k for k in s.schema} + + +@pytest.mark.parametrize( + ("platform", "fields"), + [ + ( + "sensor", + [ + "unit_of_measurement", + "accuracy_decimals", + "device_class", + "state_class", + "force_update", + ], + ), + ("binary_sensor", ["device_class"]), + ("switch", ["device_class"]), + ("cover", ["device_class"]), + ("button", ["device_class"]), + ("valve", ["device_class"]), + ("event", ["device_class"]), + ("text_sensor", ["device_class"]), + ("number", ["device_class", "unit_of_measurement"]), + ], +) +def test_template_metadata_is_ui(platform: str, fields: list[str]) -> None: + mod = importlib.import_module(f"esphome.components.template.{platform}") + markers = _markers(mod.CONFIG_SCHEMA) + for field in fields: + assert markers[field].visibility is cv.Visibility.UI, f"{platform}.{field}" + + +def test_template_sensor_promotion_preserves_defaults() -> None: + """Promoting to UI must not drop the fields' defaults.""" + from esphome.components.template.sensor import CONFIG_SCHEMA + + markers = _markers(CONFIG_SCHEMA) + assert markers["accuracy_decimals"].default() == 1 + assert markers["force_update"].default() is False + + +def test_hardware_platform_metadata_not_promoted() -> None: + """Without ``visibility=`` the builders leave metadata unset. + + Unset markers fall through to the consumer's ``Optional`` default of + advanced, so hardware platforms are unaffected by the template promotion. + """ + from esphome.components import binary_sensor, sensor + + hw_sensor = _markers(sensor.sensor_schema(device_class="temperature")) + assert hw_sensor["device_class"].visibility is None + hw_bs = _markers(binary_sensor.binary_sensor_schema(device_class="motion")) + assert hw_bs["device_class"].visibility is None diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 457b9d017b..4092b4c0d5 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -1394,6 +1394,35 @@ def test_entity_metadata_visibility_hints() -> None: assert web["web_server"].visibility is advanced +def test_with_visibility_remarks_keys() -> None: + """``with_visibility`` re-marks the named keys, preserving each field's + default and validator, without touching the other keys or the input schema. + """ + base = cv.Schema( + { + cv.Optional("a", default=7): cv.int_, + cv.Optional("b", visibility=cv.Visibility.ADVANCED): cv.string, + } + ) + promoted = cv.with_visibility(base, cv.Visibility.UI, "a") + + pm = {str(k): k for k in promoted.schema} + assert pm["a"].visibility is cv.Visibility.UI # re-marked + assert pm["a"].default() == 7 # default preserved + assert pm["b"].visibility is cv.Visibility.ADVANCED # sibling untouched + assert promoted({}) == {"a": 7} # validator/default still applied + + # The input schema is left untouched (no shared-marker mutation). + assert {str(k): k for k in base.schema}["a"].visibility is None + + +def test_with_visibility_unknown_key_raises() -> None: + """A key not present in the schema is a typo — fail at build time.""" + base = cv.Schema({cv.Optional("a"): cv.int_}) + with pytest.raises(ValueError, match="not in schema"): + cv.with_visibility(base, cv.Visibility.UI, "nope") + + def _wrap_str(value: str) -> ESPHomeDataBase: """Wrap a raw string as an ESPHomeDataBase, mimicking a YAML-loaded value.""" return make_data_base(value) From ddbd89dd2a5bb35b24217dae93a4a45ce5da476f Mon Sep 17 00:00:00 2001 From: Robin Thoni Date: Thu, 10 Sep 2026 06:06:44 +0200 Subject: [PATCH 101/178] [network] Improve `network::is_connected()` to better handle multiple interfaces (#18999) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/network/util.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/esphome/components/network/util.h b/esphome/components/network/util.h index 65a578c22f..57c5a66833 100644 --- a/esphome/components/network/util.h +++ b/esphome/components/network/util.h @@ -26,30 +26,34 @@ namespace esphome::network { /// Return whether the node is connected to the network (through wifi, eth, ...) ESPHOME_ALWAYS_INLINE inline bool is_connected() { + // With a single interface enabled the checks below collapse to `if (x) return true; return false;`, which + // clang-tidy wants folded into one return. Keep the per-interface form so every enabled interface is checked. + // NOLINTBEGIN(readability-simplify-boolean-expr) #ifdef USE_ETHERNET if (ethernet::global_eth_component != nullptr && ethernet::global_eth_component->is_connected()) return true; #endif #ifdef USE_MODEM - if (modem::global_modem_component != nullptr) - return modem::global_modem_component->is_connected(); + if (modem::global_modem_component != nullptr && modem::global_modem_component->is_connected()) + return true; #endif #ifdef USE_WIFI - if (wifi::global_wifi_component != nullptr) - return wifi::global_wifi_component->is_connected(); + if (wifi::global_wifi_component != nullptr && wifi::global_wifi_component->is_connected()) + return true; #endif #ifdef USE_OPENTHREAD - if (openthread::global_openthread_component != nullptr) - return openthread::global_openthread_component->is_connected(); + if (openthread::global_openthread_component != nullptr && openthread::global_openthread_component->is_connected()) + return true; #endif #ifdef USE_HOST return true; // Assume it's connected #endif return false; + // NOLINTEND(readability-simplify-boolean-expr) } /// Return whether the network is disabled: every configured interface with a From 05f7d5e4f1b1d5ff14f0d4c30ce984ed319ca112 Mon Sep 17 00:00:00 2001 From: Anton Sergunov Date: Thu, 10 Sep 2026 10:14:43 +0600 Subject: [PATCH 102/178] [mlx90614] pec validation (#6689) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/mlx90614/mlx90614.cpp | 185 +++++++++++++++++------ esphome/components/mlx90614/mlx90614.h | 7 +- 2 files changed, 145 insertions(+), 47 deletions(-) diff --git a/esphome/components/mlx90614/mlx90614.cpp b/esphome/components/mlx90614/mlx90614.cpp index 2d3b6631bc..508b3743d1 100644 --- a/esphome/components/mlx90614/mlx90614.cpp +++ b/esphome/components/mlx90614/mlx90614.cpp @@ -26,44 +26,129 @@ static const uint8_t MLX90614_ID4 = 0x3F; static const char *const TAG = "mlx90614"; +// The EEPROM cell has a limited number of write cycles, so stop retrying after a few failures +static constexpr uint8_t EMISSIVITY_WRITE_ATTEMPTS = 3; + +// SMBus packet error code: CRC-8 with polynomial 0x07, MSB first +static uint8_t crc8_pec(const uint8_t *data, uint8_t len) { return crc8(data, len, 0x00, 0x07, true); } + void MLX90614Component::setup() { - if (!this->write_emissivity_()) { - ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); - this->mark_failed(); + if (std::isnan(this->emissivity_)) { return; } + this->emissivity_write_attempts_ = EMISSIVITY_WRITE_ATTEMPTS; + this->try_write_emissivity_(); + if (this->emissivity_write_attempts_ != 0) { + this->status_set_warning(LOG_STR("Failed to write emissivity, will retry")); + } +} + +void MLX90614Component::try_write_emissivity_() { + if (this->emissivity_write_attempts_ == 0) { + return; + } + if (this->write_emissivity_()) { + this->emissivity_write_attempts_ = 0; + return; + } + if (--this->emissivity_write_attempts_ == 0) { + ESP_LOGE(TAG, "Giving up on writing emissivity after %u attempts", EMISSIVITY_WRITE_ATTEMPTS); + this->emissivity_write_failed_ = true; + } } bool MLX90614Component::write_emissivity_() { - if (std::isnan(this->emissivity_)) + // Skip the write when the EEPROM already holds the desired value to save write cycles + uint16_t current_emissivity; + if (this->read_register_(MLX90614_EMISSIVITY, current_emissivity) != i2c::ERROR_OK) { + return false; + } + + const auto desired_emissivity = static_cast(this->emissivity_ * 0xFFFF); + if (current_emissivity == desired_emissivity) { return true; - uint16_t value = (uint16_t) (this->emissivity_ * 65535); - if (!this->write_bytes_(MLX90614_EMISSIVITY, 0)) { - return false; } - delay(10); - if (!this->write_bytes_(MLX90614_EMISSIVITY, value)) { - return false; - } - delay(10); - return true; + + return this->write_register_(MLX90614_EMISSIVITY, desired_emissivity); } -bool MLX90614Component::write_bytes_(uint8_t reg, uint16_t data) { +bool MLX90614Component::write_register_(uint8_t reg, uint16_t data) { + // The PEC covers the whole write transaction: SLA+W, command, data low, data high uint8_t buf[5]; buf[0] = this->address_ << 1; buf[1] = reg; - buf[2] = data & 0xFF; - buf[3] = data >> 8; - buf[4] = crc8(buf, 4, 0x00, 0x07, true); - return this->write_bytes(reg, buf + 2, 3); + + // See datasheet 8.3.3.1 EEPROM write sequence + // 1. Write 0x0000 into the cell of interest (erases the cell) + buf[2] = buf[3] = 0; + buf[4] = crc8_pec(buf, 4); + auto ec = this->write_register(reg, buf + 2, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't erase register 0x%02X, error %d", reg, ec); + return false; + } + + // 2. Wait at least 5ms + delay(10); + + // 3. Write the new value + if (data != 0) { + buf[2] = data & 0xFF; + buf[3] = data >> 8; + buf[4] = crc8_pec(buf, 4); + ec = this->write_register(reg, buf + 2, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't write register 0x%02X, error %d", reg, ec); + return false; + } + // 4. Wait at least 5ms + delay(10); + } + + // 5. Read back to confirm the value was stored + uint16_t read_back; + ec = this->read_register_(reg, read_back); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't check register 0x%02X value, error %d", reg, ec); + return false; + } + + if (read_back != data) { + ESP_LOGW(TAG, "Read back mismatch on register 0x%02X. Expected 0x%04X, got 0x%04X", reg, data, read_back); + return false; + } + + return true; +} + +i2c::ErrorCode MLX90614Component::read_register_(uint8_t reg, uint16_t &data) { + // The PEC covers the whole read transaction: SLA+W, command, SLA+R, data low, data high + uint8_t buf[6]; + buf[0] = this->address_ << 1; + buf[1] = reg; + buf[2] = (this->address_ << 1) | 0x01; + + const auto ec = this->read_register(reg, buf + 3, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "i2c read error %d", ec); + return ec; + } + + const auto expected_pec = crc8_pec(buf, 5); + if (buf[5] != expected_pec) { + ESP_LOGW(TAG, "i2c CRC error. Expected 0x%02X, got 0x%02X", expected_pec, buf[5]); + return i2c::ERROR_CRC; + } + + data = encode_uint16(buf[4], buf[3]); + return i2c::ERROR_OK; } void MLX90614Component::dump_config() { ESP_LOGCONFIG(TAG, "MLX90614:"); LOG_I2C_DEVICE(this); - if (this->is_failed()) { - ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); + if (this->emissivity_write_attempts_ != 0) { + ESP_LOGW(TAG, " Emissivity not written yet, will retry"); } LOG_UPDATE_INTERVAL(this); LOG_SENSOR(" ", "Ambient", this->ambient_sensor_); @@ -71,33 +156,41 @@ void MLX90614Component::dump_config() { } void MLX90614Component::update() { - uint8_t emissivity[3]; - if (this->read_register(MLX90614_EMISSIVITY, emissivity, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; + // Temperature reads run regardless of the emissivity state so a failure still shows up as NAN + this->try_write_emissivity_(); + + // Publishes NAN on a bus or CRC failure so a stuck reading is visible instead of silently stale + auto publish_sensor = [this](sensor::Sensor *sensor, uint8_t reg) { + if (sensor == nullptr) { + return i2c::ERROR_OK; + } + + uint16_t raw; + const auto ec = this->read_register_(reg, raw); + if (ec != i2c::ERROR_OK) { + sensor->publish_state(NAN); + return ec; + } + + // Bit 15 set means the device flagged the reading as invalid + const float temperature = (raw & 0x8000) ? NAN : raw * 0.02f - 273.15f; + ESP_LOGD(TAG, "'%s': Got temperature=%.1f°C", sensor->get_name().c_str(), temperature); + sensor->publish_state(temperature); + return ec; + }; + + const auto object_ec = publish_sensor(this->object_sensor_, MLX90614_TEMPERATURE_OBJECT_1); + const auto ambient_ec = publish_sensor(this->ambient_sensor_, MLX90614_TEMPERATURE_AMBIENT); + + if (object_ec != i2c::ERROR_OK || ambient_ec != i2c::ERROR_OK) { + this->status_set_warning(LOG_STR("Failed to read some sensors")); + } else if (this->emissivity_write_failed_) { + this->status_set_warning(LOG_STR("Failed to write emissivity")); + } else if (this->emissivity_write_attempts_ != 0) { + this->status_set_warning(LOG_STR("Failed to write emissivity, will retry")); + } else { + this->status_clear_warning(); } - uint8_t raw_object[3]; - if (this->read_register(MLX90614_TEMPERATURE_OBJECT_1, raw_object, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; - } - - uint8_t raw_ambient[3]; - if (this->read_register(MLX90614_TEMPERATURE_AMBIENT, raw_ambient, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; - } - - float ambient = raw_ambient[1] & 0x80 ? NAN : encode_uint16(raw_ambient[1], raw_ambient[0]) * 0.02f - 273.15f; - float object = raw_object[1] & 0x80 ? NAN : encode_uint16(raw_object[1], raw_object[0]) * 0.02f - 273.15f; - - ESP_LOGD(TAG, "Got Temperature=%.1f°C Ambient=%.1f°C", object, ambient); - - if (this->ambient_sensor_ != nullptr && !std::isnan(ambient)) - this->ambient_sensor_->publish_state(ambient); - if (this->object_sensor_ != nullptr && !std::isnan(object)) - this->object_sensor_->publish_state(object); - this->status_clear_warning(); } } // namespace esphome::mlx90614 diff --git a/esphome/components/mlx90614/mlx90614.h b/esphome/components/mlx90614/mlx90614.h index 882ee45186..758792aced 100644 --- a/esphome/components/mlx90614/mlx90614.h +++ b/esphome/components/mlx90614/mlx90614.h @@ -18,13 +18,18 @@ class MLX90614Component final : public PollingComponent, public i2c::I2CDevice { void set_emissivity(float emissivity) { emissivity_ = emissivity; } protected: + void try_write_emissivity_(); bool write_emissivity_(); - bool write_bytes_(uint8_t reg, uint16_t data); + bool write_register_(uint8_t reg, uint16_t data); + i2c::ErrorCode read_register_(uint8_t reg, uint16_t &data); sensor::Sensor *ambient_sensor_{nullptr}; sensor::Sensor *object_sensor_{nullptr}; float emissivity_{NAN}; + // Remaining attempts to program the emissivity EEPROM cell, bounded to limit cell wear + uint8_t emissivity_write_attempts_{0}; + bool emissivity_write_failed_{false}; }; } // namespace esphome::mlx90614 From 54706e869c13abc0f688a91c0f326c779799b858 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:18:15 +0200 Subject: [PATCH 103/178] [deep_sleep] disable loop (#18962) --- esphome/components/deep_sleep/deep_sleep_bk72xx.cpp | 2 +- esphome/components/deep_sleep/deep_sleep_component.cpp | 3 ++- esphome/components/deep_sleep/deep_sleep_component.h | 5 +++++ esphome/components/deep_sleep/deep_sleep_esp32.cpp | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp index 2c97dc3211..a955095875 100644 --- a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp +++ b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp @@ -44,7 +44,7 @@ bool DeepSleepComponent::prepare_to_sleep_() { this->status_set_warning(); ESP_LOGV(TAG, "Waiting for pin to switch state to enter deep sleep..."); } - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return false; } } diff --git a/esphome/components/deep_sleep/deep_sleep_component.cpp b/esphome/components/deep_sleep/deep_sleep_component.cpp index 9a3e537e05..d33102bf4f 100644 --- a/esphome/components/deep_sleep/deep_sleep_component.cpp +++ b/esphome/components/deep_sleep/deep_sleep_component.cpp @@ -17,6 +17,7 @@ void DeepSleepComponent::setup() { void DeepSleepComponent::schedule_sleep_() { this->next_enter_deep_sleep_ = false; + this->disable_loop(); const optional run_duration = get_run_duration_(); if (run_duration.has_value()) { ESP_LOGI(TAG, "Scheduling in %" PRIu32 " ms", *run_duration); @@ -45,7 +46,7 @@ void DeepSleepComponent::loop() { void DeepSleepComponent::begin_sleep(bool manual) { if (this->prevent_ && !manual) { - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return; } diff --git a/esphome/components/deep_sleep/deep_sleep_component.h b/esphome/components/deep_sleep/deep_sleep_component.h index 208f88d707..0bbca4c5c4 100644 --- a/esphome/components/deep_sleep/deep_sleep_component.h +++ b/esphome/components/deep_sleep/deep_sleep_component.h @@ -190,6 +190,11 @@ class DeepSleepComponent final : public Component { void schedule_sleep_(); bool should_teardown_(); + void defer_sleep_() { + this->next_enter_deep_sleep_ = true; + this->enable_loop(); + } + #ifdef USE_BK72XX bool pin_prevents_sleep_(WakeUpPinItem &pin_item) const; bool get_real_pin_state_(InternalGPIOPin &pin) const { return (pin.digital_read() ^ pin.is_inverted()); } diff --git a/esphome/components/deep_sleep/deep_sleep_esp32.cpp b/esphome/components/deep_sleep/deep_sleep_esp32.cpp index 3fa1a1f1ed..20297028b2 100644 --- a/esphome/components/deep_sleep/deep_sleep_esp32.cpp +++ b/esphome/components/deep_sleep/deep_sleep_esp32.cpp @@ -100,7 +100,7 @@ bool DeepSleepComponent::prepare_to_sleep_() { this->status_set_warning(); ESP_LOGW(TAG, "Waiting for wakeup pin state change"); } - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return false; } return true; From a88ec7d90b6b19813f2e06bb012c7192b04b3c73 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:07:09 +0200 Subject: [PATCH 104/178] [logger] Flush uart before sleep in idf 6 (#18975) --- esphome/components/logger/logger_esp32.cpp | 13 +++++++++++-- sdkconfig.defaults | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index c3d777299d..8579708559 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -3,6 +3,7 @@ #include "esphome/components/esp32/crash_handler.h" #include +#include #include #include @@ -16,8 +17,10 @@ #include #endif #endif - -#include "esp_idf_version.h" +#if defined(CONFIG_PM_ENABLE) && defined(CONFIG_FREERTOS_USE_TICKLESS_IDLE) && \ + (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0)) +#include "esp_sleep.h" +#endif #include "freertos/FreeRTOS.h" #include @@ -87,6 +90,12 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). const int min_rx_buffer_size = UART_HW_FIFO_LEN(uart_num) + 1; uart_driver_install(uart_num, min_rx_buffer_size, tx_buffer_size, 0, nullptr, 0); +#if defined(CONFIG_PM_ENABLE) && defined(CONFIG_FREERTOS_USE_TICKLESS_IDLE) && \ + (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0)) + // Always flush before going to light sleep. Could be disabled for devices + // without TOP_PD or if source_clk = UART_SCLK_RTC + esp_sleep_set_console_uart_handling_mode(ESP_SLEEP_ALWAYS_FLUSH_UART); +#endif } void Logger::pre_setup() { diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 2bd702f48e..f4fe331df4 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -17,6 +17,8 @@ CONFIG_ESP_TASK_WDT_INIT=y CONFIG_ESP_TASK_WDT_PANIC=y CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=n CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1=n +CONFIG_FREERTOS_USE_TICKLESS_IDLE=y +CONFIG_PM_ENABLE=y # esp32_ble CONFIG_BT_ENABLED=y From 66f829c760358a291a9a97d90f9b981d8ac6a6ec Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:08:07 +0200 Subject: [PATCH 105/178] [zigbee] wake loop on defer/set_timeout (#19050) --- esphome/components/zigbee/time/zigbee_time_zephyr.cpp | 2 ++ esphome/components/zigbee/zigbee_esp32.cpp | 5 ++++- esphome/components/zigbee/zigbee_zephyr.cpp | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/esphome/components/zigbee/time/zigbee_time_zephyr.cpp b/esphome/components/zigbee/time/zigbee_time_zephyr.cpp index 92d238629a..3f14d0a62d 100644 --- a/esphome/components/zigbee/time/zigbee_time_zephyr.cpp +++ b/esphome/components/zigbee/time/zigbee_time_zephyr.cpp @@ -1,6 +1,7 @@ #include "zigbee_time_zephyr.h" #if defined(USE_ZIGBEE) && defined(USE_NRF52) && defined(USE_TIME) #include "esphome/core/log.h" +#include "esphome/core/application.h" namespace esphome::zigbee { @@ -47,6 +48,7 @@ void ZigbeeTime::set_epoch_time(uint32_t epoch) { this->synchronize_epoch_(epoch); this->has_time_ = true; }); + App.wake_loop_threadsafe(); } void ZigbeeTime::zcl_device_cb_(zb_bufid_t bufid) { diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index cd094306f4..4f9c70da75 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -49,7 +49,8 @@ void ZigbeeComponent::factory_reset() { void ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(ezb_bdb_comm_mode_mask_t mode) { if (!esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { - global_zigbee->set_timeout("zb_init", 10, [mode]() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(mode); }); + global_zigbee->set_timeout("zb_init", 100, [mode]() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(mode); }); + App.wake_loop_threadsafe(); return; } if (ezb_bdb_start_top_level_commissioning(mode) != EZB_ERR_NONE) { @@ -88,6 +89,7 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { global_zigbee->set_timeout("zb_init", 1000, []() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(EZB_BDB_MODE_INITIALIZATION); }); + App.wake_loop_threadsafe(); } } break; case EZB_BDB_SIGNAL_STEERING: { @@ -113,6 +115,7 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(EZB_BDB_MODE_NETWORK_STEERING); }); } + App.wake_loop_threadsafe(); } } break; case EZB_ZDO_SIGNAL_LEAVE: { diff --git a/esphome/components/zigbee/zigbee_zephyr.cpp b/esphome/components/zigbee/zigbee_zephyr.cpp index b8bb0a2036..286c83b8f5 100644 --- a/esphome/components/zigbee/zigbee_zephyr.cpp +++ b/esphome/components/zigbee/zigbee_zephyr.cpp @@ -1,10 +1,10 @@ #include "zigbee_zephyr.h" #if defined(USE_ZIGBEE) && defined(USE_NRF52) #include "esphome/core/log.h" +#include "esphome/core/application.h" #include #include #include "esphome/core/hal.h" -#include "esphome/core/wake.h" extern "C" { #include @@ -120,7 +120,7 @@ void ZigbeeComponent::zcl_device_cb(zb_bufid_t bufid) { /* Set default response value. */ p_device_cb_param->status = RET_OK; - esphome::wake_loop_threadsafe(); + App.wake_loop_threadsafe(); // endpoints are enumerated from 1 if (global_zigbee->callbacks_.size() >= endpoint) { @@ -138,6 +138,7 @@ void ZigbeeComponent::on_join_(bool factory_new) { ESP_LOGD(TAG, "Joined the network"); this->join_cb_.call(factory_new); }); + App.wake_loop_threadsafe(); } void ZigbeeComponent::on_start_() { @@ -145,6 +146,7 @@ void ZigbeeComponent::on_start_() { ESP_LOGD(TAG, "Started zigbee stack"); this->start_cb_.call(); }); + App.wake_loop_threadsafe(); } #ifdef USE_ZIGBEE_WIPE_ON_BOOT From 99241483026d54d47f8f06bdd2415e02c0cd3ebb Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:59:23 +0000 Subject: [PATCH 106/178] Bump aioesphomeapi from 46.3.0 to 46.4.0 (#19071) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dfddbed00b..72c42dad32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.4.0 click==8.3.3 -aioesphomeapi==46.3.0 +aioesphomeapi==46.4.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.151.3 puremagic==2.2.0 From 280fac11e6a8b571f6859dc4f9203e470cbbf1d4 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 10 Sep 2026 10:03:33 -0500 Subject: [PATCH 107/178] [serial_proxy] Add tap interface and port mode (#18955) Co-authored-by: puddly <32534428+puddly@users.noreply.github.com> --- esphome/components/api/api.proto | 37 +++- esphome/components/api/api_connection.cpp | 16 +- esphome/components/api/api_connection.h | 1 + esphome/components/api/api_pb2.cpp | 13 ++ esphome/components/api/api_pb2.h | 21 ++ esphome/components/api/api_pb2_dump.cpp | 18 ++ esphome/components/api/api_pb2_service.cpp | 11 ++ esphome/components/api/api_pb2_service.h | 3 + esphome/components/serial_proxy/__init__.py | 1 + .../components/serial_proxy/serial_proxy.cpp | 182 +++++++++++++++--- .../components/serial_proxy/serial_proxy.h | 103 +++++++++- esphome/core/defines.h | 1 + .../components/serial_proxy/serial_proxy.h | 3 + .../serial_proxy/test-tap.esp32-idf.yaml | 14 ++ 14 files changed, 394 insertions(+), 30 deletions(-) create mode 100644 tests/components/serial_proxy/test-tap.esp32-idf.yaml diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 3a0e0abea9..21972decad 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -77,6 +77,7 @@ service APIConnection { rpc serial_proxy_set_modem_pins(SerialProxySetModemPinsRequest) returns (void) {} rpc serial_proxy_get_modem_pins(SerialProxyGetModemPinsRequest) returns (void) {} rpc serial_proxy_request(SerialProxyRequest) returns (void) {} + rpc serial_proxy_set_mode(SerialProxySetModeRequest) returns (void) {} } @@ -2726,7 +2727,8 @@ enum SerialProxyParity { SERIAL_PROXY_PARITY_ODD = 2; } -// Configure UART parameters for a serial proxy instance +// Configure UART parameters for a serial proxy instance. Only the subscribed client may +// configure the port; others are refused with PORT_IN_USE (since API 1.17). message SerialProxyConfigureRequest { option (id) = 138; option (source) = SOURCE_CLIENT; @@ -2752,7 +2754,8 @@ message SerialProxyDataReceived { bytes data = 2; // Raw data received from the serial device } -// Write data to a serial device +// Write data to a serial device. Only the subscribed client may write; writes from +// others are ignored (since API 1.17). message SerialProxyWriteRequest { option (id) = 140; option (source) = SOURCE_CLIENT; @@ -2763,7 +2766,8 @@ message SerialProxyWriteRequest { bytes data = 2; // Raw data to write to the serial device } -// Set modem control pin states (RTS and DTR) +// Set modem control pin states (RTS and DTR). Only the subscribed client may set them; +// others are refused with PORT_IN_USE (since API 1.17). message SerialProxySetModemPinsRequest { option (id) = 141; option (source) = SOURCE_CLIENT; @@ -2802,6 +2806,7 @@ enum SerialProxyRequestType { // error the device answers with INVALID_ARGUMENT. SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3; // Acknowledges a SerialProxyConfigureRequest SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4; // Acknowledges a SerialProxySetModemPinsRequest + SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5; // Acknowledges a SerialProxySetModeRequest (since API 1.17) } enum SerialProxyStatus { @@ -2814,7 +2819,8 @@ enum SerialProxyStatus { SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6; // Invalid instance index or parameter value } -// Generic request message for simple serial proxy operations +// Generic request message for simple serial proxy operations. FLUSH requires an active +// subscription; it is refused with PORT_IN_USE otherwise (since API 1.17). message SerialProxyRequest { option (id) = 144; option (source) = SOURCE_CLIENT; @@ -2838,6 +2844,29 @@ message SerialProxyRequestResponse { string error_message = 4; // Additional detail on failure (optional) } +// How a port treats the bytes passing through it. RAW is a plain byte pipe; PROTOCOL +// activates the port's protocol-aware tap (if one is configured), letting it observe +// traffic and inject protocol bytes such as acknowledgements. Which protocol the tap +// speaks is a property of the device configuration, discoverable from the tap +// component's own API surface. A client that is about to flash firmware selects RAW +// first, which definitively disables that injection. +enum SerialProxyMode { + SERIAL_PROXY_MODE_RAW = 0; + SERIAL_PROXY_MODE_PROTOCOL = 1; +} + +// Only the subscribed client may change the mode; any other caller -- including one that +// never subscribed -- is refused with PORT_IN_USE. PROTOCOL is refused with NOT_SUPPORTED +// when the port has no protocol-aware tap configured. +message SerialProxySetModeRequest { + option (id) = 152; + option (source) = SOURCE_CLIENT; + option (ifdef) = "USE_SERIAL_PROXY"; + + uint32 instance = 1; + SerialProxyMode mode = 2; +} + // ==================== BLUETOOTH CONNECTION PARAMS ==================== message BluetoothSetConnectionParamsRequest { option (id) = 145; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index da4b7d7702..d910f6fc67 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1661,6 +1661,7 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { break; case enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE: case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE: // Response-only discriminators; never valid in a request ESP_LOGW(TAG, "Response-only serial proxy request type: %" PRIu32, static_cast(msg.type)); status = enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT; @@ -1673,6 +1674,19 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { send_serial_proxy_ack(this, msg.instance, msg.type, status); } +void APIConnection::on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE, + enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT); + return; + } + serial_proxy::SerialProxyResult result = proxies[msg.instance]->set_mode_from_client(this, msg.mode); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE, + serial_proxy_result_to_status(result)); +} + void APIConnection::send_serial_proxy_data(const SerialProxyDataReceived &msg) { if (!this->send_message(msg)) { ESP_LOGV(TAG, "Serial proxy data dropped, TCP buffer full"); @@ -1799,7 +1813,7 @@ bool APIConnection::send_hello_response_(const HelloRequest &msg) { HelloResponse resp; resp.api_version_major = 1; - resp.api_version_minor = 16; + resp.api_version_minor = 17; // Send only the version string - the client only logs this for debugging and doesn't use it otherwise resp.server_info = ESPHOME_VERSION_REF; resp.name = StringRef(App.get_name()); diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index a4c49dccf4..c19a33ca9b 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -244,6 +244,7 @@ class APIConnection final : public APIServerConnectionBase { void on_serial_proxy_set_modem_pins_request(const SerialProxySetModemPinsRequest &msg); void on_serial_proxy_get_modem_pins_request(const SerialProxyGetModemPinsRequest &msg); void on_serial_proxy_request(const SerialProxyRequest &msg); + void on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &msg); void send_serial_proxy_data(const SerialProxyDataReceived &msg); #endif diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 2de1f0a15c..7f162d9c15 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -4253,6 +4253,19 @@ uint32_t SerialProxyRequestResponse::calculate_size() const { size += ProtoSize::calc_length(1, this->error_message.size()); return size; } +bool SerialProxySetModeRequest::decode_varint(uint32_t field_id, proto_varint_value_t value) { + switch (field_id) { + case 1: + this->instance = value; + break; + case 2: + this->mode = static_cast(value); + break; + default: + return false; + } + return true; +} #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS bool BluetoothSetConnectionParamsRequest::decode_varint(uint32_t field_id, proto_varint_value_t value) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 5c3429a63a..799aaa27b5 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -356,6 +356,7 @@ enum SerialProxyRequestType : uint32_t { SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2, SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3, SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4, + SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5, }; enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_OK = 0, @@ -366,6 +367,10 @@ enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_PORT_IN_USE = 5, SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6, }; +enum SerialProxyMode : uint32_t { + SERIAL_PROXY_MODE_RAW = 0, + SERIAL_PROXY_MODE_PROTOCOL = 1, +}; #endif } // namespace enums @@ -3403,6 +3408,22 @@ class SerialProxyRequestResponse final : public ProtoMessage { protected: }; +class SerialProxySetModeRequest final : public ProtoDecodableMessage { + public: + static constexpr uint16_t MESSAGE_TYPE = 152; + static constexpr uint8_t ESTIMATED_SIZE = 6; +#ifdef HAS_PROTO_MESSAGE_DUMP + const LogString *message_name() const override { return LOG_STR("serial_proxy_set_mode_request"); } +#endif + uint32_t instance{0}; + enums::SerialProxyMode mode{}; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: + bool decode_varint(uint32_t field_id, proto_varint_value_t value) override; +}; #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS class BluetoothSetConnectionParamsRequest final : public ProtoDecodableMessage { diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index dced81ee30..bb244973a1 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -854,6 +854,8 @@ template<> const char *proto_enum_to_string(enums return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_CONFIGURE"); case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS"); + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE: + return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODE"); default: return ESPHOME_PSTR("UNKNOWN"); } @@ -878,6 +880,16 @@ template<> const char *proto_enum_to_string(enums::Ser return ESPHOME_PSTR("UNKNOWN"); } } +template<> const char *proto_enum_to_string(enums::SerialProxyMode value) { + switch (value) { + case enums::SERIAL_PROXY_MODE_RAW: + return ESPHOME_PSTR("SERIAL_PROXY_MODE_RAW"); + case enums::SERIAL_PROXY_MODE_PROTOCOL: + return ESPHOME_PSTR("SERIAL_PROXY_MODE_PROTOCOL"); + default: + return ESPHOME_PSTR("UNKNOWN"); + } +} #endif const char *HelloRequest::dump_to(DumpBuffer &out) const { @@ -2805,6 +2817,12 @@ const char *SerialProxyRequestResponse::dump_to(DumpBuffer &out) const { dump_field(out, ESPHOME_PSTR("error_message"), this->error_message); return out.c_str(); } +const char *SerialProxySetModeRequest::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("SerialProxySetModeRequest")); + dump_field(out, ESPHOME_PSTR("instance"), this->instance); + dump_field(out, ESPHOME_PSTR("mode"), static_cast(this->mode)); + return out.c_str(); +} #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS const char *BluetoothSetConnectionParamsRequest::dump_to(DumpBuffer &out) const { diff --git a/esphome/components/api/api_pb2_service.cpp b/esphome/components/api/api_pb2_service.cpp index 65c7b8858c..172062be63 100644 --- a/esphome/components/api/api_pb2_service.cpp +++ b/esphome/components/api/api_pb2_service.cpp @@ -712,6 +712,17 @@ void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const ui this->on_device_capabilities_request(); break; } +#ifdef USE_SERIAL_PROXY + case SerialProxySetModeRequest::MESSAGE_TYPE: { + SerialProxySetModeRequest msg; + msg.decode(msg_data, msg_size); +#ifdef HAS_PROTO_MESSAGE_DUMP + this->log_receive_message_(LOG_STR("on_serial_proxy_set_mode_request"), msg); +#endif + this->on_serial_proxy_set_mode_request(msg); + break; + } +#endif default: break; } diff --git a/esphome/components/api/api_pb2_service.h b/esphome/components/api/api_pb2_service.h index 6abdf7093e..a4dfd6a366 100644 --- a/esphome/components/api/api_pb2_service.h +++ b/esphome/components/api/api_pb2_service.h @@ -235,6 +235,9 @@ class APIServerConnectionBase { void on_serial_proxy_request(const SerialProxyRequest &value){}; #endif +#ifdef USE_SERIAL_PROXY + void on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &value){}; +#endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS void on_bluetooth_set_connection_params_request(const BluetoothSetConnectionParamsRequest &value){}; #endif diff --git a/esphome/components/serial_proxy/__init__.py b/esphome/components/serial_proxy/__init__.py index 4186fcf8b1..b6e780fabd 100644 --- a/esphome/components/serial_proxy/__init__.py +++ b/esphome/components/serial_proxy/__init__.py @@ -30,6 +30,7 @@ MULTI_CONF = True serial_proxy_ns = cg.esphome_ns.namespace("serial_proxy") SerialProxy = serial_proxy_ns.class_("SerialProxy", cg.Component, uart.UARTDevice) +SerialProxyTap = serial_proxy_ns.class_("SerialProxyTap") api_enums_ns = cg.esphome_ns.namespace("api").namespace("enums") SerialProxyPortType = api_enums_ns.enum("SerialProxyPortType") diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index c1c1510643..129745c1c9 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -29,26 +29,57 @@ void SerialProxy::setup() { #ifdef USE_API // instance_index_ is fixed at registration time; pre-set it so loop() only needs to update data this->outgoing_msg_.instance = this->instance_index_; +#endif +#ifdef USE_SERIAL_PROXY_TAP + // A tap sets itself up before this runs (its setup priority is higher), so it may + // already be waiting on the port -- a boot-time handshake with the device, say. Leaving + // the loop enabled is what lets that finish; without it the tap would stall until a + // client happened to subscribe. + if (this->tap_ != nullptr && this->tap_->tap_needs_port()) { + return; + } #endif // No subscriber at startup; disable loop until a client subscribes this->disable_loop(); } -void SerialProxy::loop() { -#ifdef USE_API - // Safety check — loop should only run when subscribed, but guard against races - if (this->api_connection_ == nullptr) [[unlikely]] { - this->disable_loop(); +#ifdef USE_SERIAL_PROXY_TAP +void SerialProxy::reset_mode_() { + // The mode belongs to a session, not to the port. Carrying a departed client's choice + // over to the next one would inject protocol bytes into a stream that never asked for + // them -- a firmware upload, or any client built before this request existed and so + // unable to turn it off. Guessing RAW is the safe direction: a client that wanted + // protocol handling and did not ask for it merely sends its own acknowledgements. + if (this->mode_ == api::enums::SERIAL_PROXY_MODE_RAW) { return; } + ESP_LOGD(TAG, "Session ended, returning serial proxy [%" PRIu32 "] to RAW mode", this->instance_index_); + this->mode_ = api::enums::SERIAL_PROXY_MODE_RAW; +} +#endif +void SerialProxy::loop() { +#ifdef USE_API // Detect subscriber disconnect - if (this->api_connection_->is_marked_for_removal() || !this->api_connection_->is_connection_setup() || - !api_is_connected()) { + if (this->api_connection_ != nullptr && (this->api_connection_->is_marked_for_removal() || + !this->api_connection_->is_connection_setup() || !api_is_connected())) { ESP_LOGW(TAG, "Subscriber disconnected"); this->api_connection_ = nullptr; + this->reset_mode_(); + } + + // With no subscriber there is normally nothing to do, but a tap may still need the port + // read -- it does its protocol work precisely while nobody else is listening. + if (this->api_connection_ == nullptr) [[unlikely]] { +#ifdef USE_SERIAL_PROXY_TAP + if (this->tap_ == nullptr || !this->tap_->tap_needs_port()) { + this->disable_loop(); + return; + } +#else this->disable_loop(); return; +#endif } // Read available data from UART and forward to subscribed client @@ -69,11 +100,54 @@ void __attribute__((noinline)) SerialProxy::read_and_send_(size_t available) { if (!this->read_array(buffer, to_read)) return; +#ifdef USE_SERIAL_PROXY_TAP + // Before forwarding, so a tap that answers the device (an acknowledgement, say) is not + // waiting on the network round trip to a subscriber that may not even exist. + if (this->tap_observing_()) { + this->tap_->on_device_rx(buffer, to_read); + } +#endif + + if (this->api_connection_ == nullptr) { + return; + } this->outgoing_msg_.set_data(buffer, to_read); this->api_connection_->send_serial_proxy_data(this->outgoing_msg_); } #endif +#ifdef USE_SERIAL_PROXY_TAP + +bool SerialProxy::tap_observing_() const { + if (this->tap_ == nullptr) { + return false; + } + // With no subscriber, a tap doing its own protocol work (the boot-time handshake with + // the device, say) is served regardless of mode -- nobody has chosen one yet. Once a + // subscriber holds the port, the mode alone decides, so RAW stays inert. + if (this->api_connection_ == nullptr && this->tap_->tap_needs_port()) { + return true; + } + // Otherwise the mode decides. RAW must be inert: a client that flips to RAW before + // flashing firmware is entitled to a byte pipe with nothing injecting protocol bytes + // into it, and "the tap turned out not to recognise the stream" is not good enough. + return this->mode_ == api::enums::SERIAL_PROXY_MODE_PROTOCOL; +} + +void SerialProxy::tap_pump() { +#ifdef USE_API + // Nothing would consume the bytes; leave them in the FIFO + if (!this->tap_observing_() && this->api_connection_ == nullptr) { + return; + } + const size_t available = this->available(); + if (available > 0) { + this->read_and_send_(available); + } +#endif +} +#endif + void SerialProxy::dump_config() { ESP_LOGCONFIG(TAG, "Serial Proxy [%" PRIu32 "]:\n" @@ -92,8 +166,9 @@ void SerialProxy::dump_config() { SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, uint8_t stop_bits, uint8_t data_size) { #ifdef USE_API - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring configure request from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring configure request from client without port subscription [%" PRIu32 "]", + this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -159,24 +234,80 @@ SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uin return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } +SerialProxyResult SerialProxy::set_mode_from_client(api::APIConnection *api_connection, + api::enums::SerialProxyMode mode) { +#ifdef USE_API + // Only the live subscriber may change the mode, so the mode cannot outlive a session + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring mode request from client without port subscription [%" PRIu32 "]", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; + } +#endif + // Values come from a remote client + if (mode != api::enums::SERIAL_PROXY_MODE_RAW && mode != api::enums::SERIAL_PROXY_MODE_PROTOCOL) { + ESP_LOGW(TAG, "Invalid mode: %" PRIu32, static_cast(mode)); + return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT; + } + // PROTOCOL on a port with no tap would be a silent no-op; refuse so the client knows +#ifdef USE_SERIAL_PROXY_TAP + const bool has_tap = this->tap_ != nullptr; +#else + const bool has_tap = false; +#endif + if (mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL && !has_tap) { + ESP_LOGW(TAG, "No tap on serial proxy [%" PRIu32 "]; PROTOCOL mode unavailable", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; + } + ESP_LOGD(TAG, "Serial proxy [%" PRIu32 "] mode set to %s", this->instance_index_, + mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL ? LOG_STR_LITERAL("PROTOCOL") : LOG_STR_LITERAL("RAW")); +#ifdef USE_SERIAL_PROXY_TAP + const bool leaving_protocol_mode = + this->mode_ != api::enums::SERIAL_PROXY_MODE_RAW && mode == api::enums::SERIAL_PROXY_MODE_RAW; + this->mode_ = mode; + + // Only for an explicit client request, not for reset_mode_() at the end of a session: + // an ordinary disconnect says nothing about the device, whereas a client deliberately + // asking for raw bytes usually precedes changing what the device is. + if (leaving_protocol_mode && this->tap_ != nullptr) { + this->tap_->on_protocol_disabled(); + } +#endif + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; +} + void SerialProxy::write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) { #ifdef USE_API - // Bytes from a client other than the live subscriber would interleave with the - // subscriber's traffic on the wire - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring write from client without port access [%" PRIu32 "]", this->instance_index_); + // Bytes from anyone but the live subscriber would interleave with the subscriber's + // traffic -- or with an active tap's -- on the wire + if (!this->is_subscriber_(api_connection)) { + if (this->api_connection_ != nullptr) { + ESP_LOGW(TAG, "Ignoring write from client that does not hold serial proxy [%" PRIu32 "]", this->instance_index_); + } else { + // A legacy client streaming writes without subscribing would flood WARN, one per + // request; writes are the only high-rate, unacknowledged operation, so keep this + // visible without drowning the log + ESP_LOGV(TAG, "Ignoring write from client without port subscription [%" PRIu32 "]", this->instance_index_); + } return; } #endif if (data == nullptr || len == 0) return; this->write_array(data, len); + +#ifdef USE_SERIAL_PROXY_TAP + // After the write, so the tap observes the same ordering the device does + if (this->tap_observing_()) { + this->tap_->on_client_tx(data, len); + } +#endif } SerialProxyResult SerialProxy::set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { #ifdef USE_API - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring modem pin request from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring modem pin request from client without port subscription [%" PRIu32 "]", + this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -210,8 +341,8 @@ uint32_t SerialProxy::get_modem_pins() const { SerialProxyResult SerialProxy::flush_port(api::APIConnection *api_connection) { #ifdef USE_API // Flushing stalls the port, so it gets the same ownership check as writes - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring flush from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring flush from client without port subscription [%" PRIu32 "]", this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -230,11 +361,6 @@ SerialProxyResult SerialProxy::flush_port(api::APIConnection *api_connection) { } #ifdef USE_API -bool SerialProxy::port_claimed_by_other_(api::APIConnection *api_connection) const { - return this->api_connection_ != nullptr && this->api_connection_ != api_connection && - this->api_connection_->is_connection_setup(); -} - SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type) { switch (type) { @@ -252,6 +378,10 @@ SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_conn return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } ESP_LOGW(TAG, "Previous subscriber disconnected; taking over subscription"); + // End the dead client's session before starting the new one, so its mode + // cannot leak into a session that never asked for it + this->api_connection_ = nullptr; + this->reset_mode_(); } this->api_connection_ = api_connection; this->enable_loop(); @@ -264,7 +394,15 @@ SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_conn return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } this->api_connection_ = nullptr; + this->reset_mode_(); +#ifdef USE_SERIAL_PROXY_TAP + // Keep the loop alive for a tap that still needs the port (mirrors loop()) + if (this->tap_ == nullptr || !this->tap_->tap_needs_port()) { + this->disable_loop(); + } +#else this->disable_loop(); +#endif ESP_LOGV(TAG, "API connection unsubscribed from serial proxy [%" PRIu32 "]", this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_OK; default: diff --git a/esphome/components/serial_proxy/serial_proxy.h b/esphome/components/serial_proxy/serial_proxy.h index a0e47ee686..e3f4264cfa 100644 --- a/esphome/components/serial_proxy/serial_proxy.h +++ b/esphome/components/serial_proxy/serial_proxy.h @@ -26,6 +26,7 @@ class APIConnection; namespace enums { enum SerialProxyPortType : uint32_t; enum SerialProxyRequestType : uint32_t; +enum SerialProxyMode : uint32_t; } // namespace enums } // namespace esphome::api @@ -52,6 +53,36 @@ enum class SerialProxyResult : uint8_t { /// Maximum bytes to read from UART in a single loop iteration inline constexpr size_t SERIAL_PROXY_MAX_READ_SIZE = 256; +#ifdef USE_SERIAL_PROXY_TAP +/// Observes a port's traffic without owning it, and may inject bytes of its own. +/// +/// This exists so protocol-aware behaviour can be layered onto a plain byte pipe without +/// the pipe knowing anything about the protocol: the tap is compiled in only when some +/// component asks for one, so a proxy carrying an RS485 meter pays nothing for it. +/// +/// A tap is an observer, never a gatekeeper -- it cannot suppress or alter the bytes +/// flowing in either direction, so a misbehaving tap cannot corrupt the stream. +class SerialProxyTap { + public: + /// Bytes read from the device, before they are forwarded to any subscriber. + virtual void on_device_rx(const uint8_t *data, size_t len) = 0; + + /// Bytes a subscriber sent towards the device, after they have been written. + virtual void on_client_tx(const uint8_t *data, size_t len) = 0; + + /// True when the port must keep reading even with no subscriber attached, so a tap can + /// do its own protocol work while nobody is listening. Honoured only while no + /// subscriber holds the port; with one attached, the port mode alone decides. + virtual bool tap_needs_port() const = 0; + + /// A client explicitly turned protocol handling off for this port. Distinct from the + /// automatic reset when a session ends: this one means a client intends to do something + /// else with the device -- reflash it, most likely -- so anything the tap believes about + /// it should be treated as suspect. + virtual void on_protocol_disabled() = 0; +}; +#endif + class SerialProxy final : public uart::UARTDevice, public Component { public: void setup() override; @@ -77,6 +108,9 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Get the port type api::enums::SerialProxyPortType get_port_type() const { return this->port_type_; } + /// Handle a mode change requested by an API client + SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode); + /// Configure UART parameters and apply them /// @param api_connection The API connection requesting the change /// @param baudrate Baud rate in bits per second @@ -121,13 +155,67 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Set the DTR GPIO pin (from YAML configuration) void set_dtr_pin(GPIOPin *pin) { this->dtr_pin_ = pin; } +#ifdef USE_SERIAL_PROXY_TAP + /// Attach a traffic observer. At most one, set once at setup time. + void set_tap(SerialProxyTap *tap) { this->tap_ = tap; } + + /// Write bytes originating from the tap rather than from a client. Bypasses the + /// subscriber ownership check, but only while the tap is being served bytes -- so a + /// port in RAW mode with a subscriber attached stays inert. Returns false when the + /// bytes were dropped for that reason. + bool write_from_tap(const uint8_t *data, size_t len) { + if (!this->tap_observing_()) { + return false; + } + this->write_array(data, len); + return true; + } + + /// Whether the tap is currently being served bytes. Can flip false with no callback + /// (a subscriber attaching in RAW mode, say), so a tap should check before starting + /// protocol work and when a reply seems overdue. + bool tap_is_observed() const { return this->tap_observing_(); } + + /// Resume reading after a tap's needs change. loop() disables itself when there is + /// neither a subscriber nor a tap that wants the port, so a tap starting fresh work + /// must ask for it back. Must be called from the main loop. + void tap_request_port() { this->enable_loop(); } + + /// Whether the underlying device is present. On a USB UART this tracks enumeration, so + /// a tap can notice the device being unplugged and plugged back in. + bool is_device_connected() const { return this->parent_->is_connected(); } + + /// Run one read-and-dispatch cycle immediately. Lets a tap make progress before the + /// main loop is running -- during setup, for instance, while a component is still + /// blocking on can_proceed(). Must not be called from on_device_rx() or + /// on_client_tx(): each nested cycle costs a 256-byte stack frame. + void tap_pump(); +#endif + protected: #ifdef USE_API - /// Read from UART and send to API client (slow path with 256-byte stack buffer) + /// Read from UART, hand the bytes to any tap, and forward them to a subscriber + /// (slow path with a 256-byte stack buffer) void read_and_send_(size_t available); - /// True when a live subscriber other than the given connection holds the port - bool port_claimed_by_other_(api::APIConnection *api_connection) const; + /// True when the given connection is the live subscriber. Every port operation + /// (write, configure, modem pins, flush, mode) requires this, so an unsubscribed + /// client can never share the wire with the subscriber or an active tap. + bool is_subscriber_(api::APIConnection *api_connection) const { return this->api_connection_ == api_connection; } +#endif + +#ifdef USE_SERIAL_PROXY_TAP + /// Return the port to RAW when a subscriber goes away, so the mode never outlives it + void reset_mode_(); +#else + /// Without a tap, PROTOCOL is refused, so the mode is fixed at RAW and there is + /// nothing to reset + void reset_mode_() {} +#endif + +#ifdef USE_SERIAL_PROXY_TAP + /// True when the tap should be shown the traffic passing through this port + bool tap_observing_() const; #endif /// Instance index for identifying this proxy in API messages @@ -147,6 +235,11 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Port type api::enums::SerialProxyPortType port_type_{}; +#ifdef USE_SERIAL_PROXY_TAP + /// How the bytes passing through are treated; zero is SERIAL_PROXY_MODE_RAW + api::enums::SerialProxyMode mode_{}; +#endif + /// Optional GPIO pins for modem control GPIOPin *rts_pin_{nullptr}; GPIOPin *dtr_pin_{nullptr}; @@ -154,6 +247,10 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Current modem pin states bool rts_state_{false}; bool dtr_state_{false}; + +#ifdef USE_SERIAL_PROXY_TAP + SerialProxyTap *tap_{nullptr}; +#endif }; } // namespace esphome::serial_proxy diff --git a/esphome/core/defines.h b/esphome/core/defines.h index eaece6d5ff..c3b16d833a 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -181,6 +181,7 @@ #define USE_SENSOR #define USE_SENSOR_FILTER #define USE_SERIAL_PROXY +#define USE_SERIAL_PROXY_TAP #define USE_SETUP_PRIORITY_OVERRIDE #define USE_STATUS_LED #define USE_STATUS_SENSOR diff --git a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h index 6fc20f3350..7da6fff017 100644 --- a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h +++ b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h @@ -40,6 +40,9 @@ class SerialProxy { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {} + SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) { + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } diff --git a/tests/components/serial_proxy/test-tap.esp32-idf.yaml b/tests/components/serial_proxy/test-tap.esp32-idf.yaml new file mode 100644 index 0000000000..5522e53c47 --- /dev/null +++ b/tests/components/serial_proxy/test-tap.esp32-idf.yaml @@ -0,0 +1,14 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +# Compile the tap code paths; no tap is attached, so this exercises the +# null-tap branches that a normal build never defines. +esphome: + platformio_options: + build_flags: + - "-DUSE_SERIAL_PROXY_TAP" + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + serial_proxy: !include common.yaml From a807a8f9451b172abf4cb05ca2f35c609224e414 Mon Sep 17 00:00:00 2001 From: matt123p Date: Thu, 10 Sep 2026 16:11:43 +0100 Subject: [PATCH 108/178] [es7210] Fix 4 channel microphone support (#19034) --- esphome/components/es7210/es7210.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esphome/components/es7210/es7210.cpp b/esphome/components/es7210/es7210.cpp index 892b67b270..5afc22aec4 100644 --- a/esphome/components/es7210/es7210.cpp +++ b/esphome/components/es7210/es7210.cpp @@ -153,13 +153,14 @@ bool ES7210::configure_mic_gain_() { ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC2_GAIN_REG44, 0x0f, regv)); // Configure mic 3 - ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x0b, 0x00)); + // MIC3 uses the ADC3/4 and MIC3/4 clock domains (bits 2 and 4), not the MIC1/2 domains. + ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x15, 0x00)); ES7210_ERROR_CHECK(this->write_byte(ES7210_MIC34_POWER_REG4C, 0x00)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC3_GAIN_REG45, 0x10, 0x10)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC3_GAIN_REG45, 0x0f, regv)); // Configure mic 4 - ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x0b, 0x00)); + ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x15, 0x00)); ES7210_ERROR_CHECK(this->write_byte(ES7210_MIC34_POWER_REG4C, 0x00)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC4_GAIN_REG46, 0x10, 0x10)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC4_GAIN_REG46, 0x0f, regv)); From 7564f5ff1ace9bbe107ec79107bd6606796f3156 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Thu, 10 Sep 2026 12:06:58 -0400 Subject: [PATCH 109/178] [sendspin] Add manufacturer, model, and firmware version options (#18792) Co-authored-by: J. Nick Koston --- esphome/components/sendspin/__init__.py | 32 +++++++ esphome/components/sendspin/sendspin_hub.cpp | 16 +++- esphome/components/sendspin/sendspin_hub.h | 19 +++++ .../sendspin/config/device_info_default.yaml | 12 +++ .../sendspin/config/device_info_explicit.yaml | 18 ++++ .../sendspin/config/device_info_project.yaml | 15 ++++ .../sendspin/test_device_info.py | 83 +++++++++++++++++++ tests/components/sendspin/common-hub.yaml | 3 + 8 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/sendspin/config/device_info_default.yaml create mode 100644 tests/component_tests/sendspin/config/device_info_explicit.yaml create mode 100644 tests/component_tests/sendspin/config/device_info_project.yaml create mode 100644 tests/component_tests/sendspin/test_device_info.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index c1970ab132..c21047c70a 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -6,12 +6,17 @@ from esphome.components import esp32, network, psram, socket, wifi import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, + CONF_ESPHOME, CONF_FORMAT, CONF_HEIGHT, CONF_ID, + CONF_MODEL, + CONF_NAME, + CONF_PROJECT, CONF_SAMPLE_RATE, CONF_SOURCE, CONF_TASK_STACK_IN_PSRAM, + CONF_VERSION, CONF_WIDTH, ) from esphome.core import CORE, ID @@ -27,6 +32,14 @@ DOMAIN = "sendspin" CONF_DISPLAY_OFFSET = "display_offset" CONF_SENDSPIN_ID = "sendspin_id" +CONF_FIRMWARE_VERSION = "firmware_version" +CONF_MANUFACTURER = "manufacturer" + +# An empty device information string would be sent to the server as an empty value rather than +# falling back, so reject it instead of silently substituting the fallback. The 127 byte cap keeps +# the length prefix of a protobuf string field to a single byte, matching `esphome: project:`. +DEVICE_INFO_STRING = cv.All(cv.string_strict, cv.Length(min=1), cv.ByteLength(max=127)) + CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" @@ -198,6 +211,9 @@ CONFIG_SCHEMA = cv.All( { cv.GenerateID(): cv.declare_id(SendspinHub), cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, + cv.Optional(CONF_MANUFACTURER): DEVICE_INFO_STRING, + cv.Optional(CONF_MODEL): DEVICE_INFO_STRING, + cv.Optional(CONF_FIRMWARE_VERSION): DEVICE_INFO_STRING, } ), cv.only_on_esp32, @@ -248,6 +264,22 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_task_stack_in_psram(True)) psram.request_external_task_stack() + # Device information for the server's client/hello message. Falls back to the project + # information, which is written as `manufacturer.model`. Anything still unset keeps the + # default the hub itself applies: the ESPHome name and version. + project = CORE.config[CONF_ESPHOME].get(CONF_PROJECT, {}) + project_manufacturer, _, project_model = project.get(CONF_NAME, "").partition(".") + for value, setter in ( + (config.get(CONF_MANUFACTURER) or project_manufacturer, var.set_manufacturer), + (config.get(CONF_MODEL) or project_model, var.set_model), + ( + config.get(CONF_FIRMWARE_VERSION) or project.get(CONF_VERSION), + var.set_firmware_version, + ), + ): + if value: + cg.add(setter(value)) + # sendspin-cpp library esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.7.2") diff --git a/esphome/components/sendspin/sendspin_hub.cpp b/esphome/components/sendspin/sendspin_hub.cpp index 028491284a..2cb2b90995 100644 --- a/esphome/components/sendspin/sendspin_hub.cpp +++ b/esphome/components/sendspin/sendspin_hub.cpp @@ -76,8 +76,12 @@ void SendspinHub::dump_config() { ESP_LOGCONFIG(TAG, "Sendspin Hub:\n" " Client ID: %s\n" + " Manufacturer: %s\n" + " Model: %s\n" + " Firmware version: %s\n" " Task stack in PSRAM: %s", - get_client_id_into_buffer(mac_buf), YESNO(this->task_stack_in_psram_)); + get_client_id_into_buffer(mac_buf), this->manufacturer_, this->get_product_name_(), + this->firmware_version_, YESNO(this->task_stack_in_psram_)); #ifdef USE_SENDSPIN_ARTWORK // Slot indices come from the order the image platform entries were declared, so the log is the @@ -127,15 +131,19 @@ const char *SendspinHub::get_client_id_into_buffer(std::spanmodel_ != nullptr ? this->model_ : App.get_name().c_str(); +} + sendspin::SendspinClientConfig SendspinHub::build_client_config_() { sendspin::SendspinClientConfig config; char mac_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; config.client_id = SendspinHub::get_client_id_into_buffer(mac_buf); config.name = App.get_friendly_name(); - config.product_name = App.get_name(); - config.manufacturer = "ESPHome"; - config.software_version = ESPHOME_VERSION; + config.product_name = this->get_product_name_(); + config.manufacturer = this->manufacturer_; + config.software_version = this->firmware_version_; config.httpd_psram_stack = this->task_stack_in_psram_; return config; diff --git a/esphome/components/sendspin/sendspin_hub.h b/esphome/components/sendspin/sendspin_hub.h index 7c50c3eb80..c66c7db3cc 100644 --- a/esphome/components/sendspin/sendspin_hub.h +++ b/esphome/components/sendspin/sendspin_hub.h @@ -8,6 +8,7 @@ #include "esphome/core/component.h" #include "esphome/core/helpers.h" #include "esphome/core/preferences.h" +#include "esphome/core/version.h" #include #include @@ -125,6 +126,15 @@ class SendspinHub final : public Component, void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } + /// @brief Sets the device information reported to the server in the `client/hello` message. + /// + /// Each takes a pointer to a string literal emitted by codegen, so it must stay valid for the + /// lifetime of the hub. Only called for values the configuration overrides; anything left alone + /// keeps the default described on the member below. + void set_manufacturer(const char *manufacturer) { this->manufacturer_ = manufacturer; } + void set_model(const char *model) { this->model_ = model; } + void set_firmware_version(const char *firmware_version) { this->firmware_version_ = firmware_version; } + // --- Sendspin role specific methods --- #ifdef USE_SENDSPIN_ARTWORK @@ -187,6 +197,9 @@ class SendspinHub final : public Component, /// @brief Builds the SendspinClientConfig from ESPHome configuration and platform info. sendspin::SendspinClientConfig build_client_config_(); + /// @brief Returns the product name reported to the server: the configured model, or the device name. + const char *get_product_name_() const; + /// @brief Writes the active network interface's MAC into @p buf and returns its data pointer. /// Uses the ethernet MAC if ethernet is configured, otherwise the base MAC (used by wifi). static const char *get_client_id_into_buffer(std::span buf); @@ -268,6 +281,12 @@ class SendspinHub final : public Component, CallbackManager group_update_callbacks_{}; bool task_stack_in_psram_{false}; + + // Device information sent in the `client/hello` message. Defaults apply when neither the + // sendspin configuration nor the project information supplies a value. + const char *manufacturer_{"ESPHome"}; + const char *model_{nullptr}; // nullptr reports the device name instead + const char *firmware_version_{ESPHOME_VERSION}; }; /// @brief Base class for all sendspin subcomponents. diff --git a/tests/component_tests/sendspin/config/device_info_default.yaml b/tests/component_tests/sendspin/config/device_info_default.yaml new file mode 100644 index 0000000000..669b2e99bc --- /dev/null +++ b/tests/component_tests/sendspin/config/device_info_default.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ap: + +sendspin: diff --git a/tests/component_tests/sendspin/config/device_info_explicit.yaml b/tests/component_tests/sendspin/config/device_info_explicit.yaml new file mode 100644 index 0000000000..c3fec3ead4 --- /dev/null +++ b/tests/component_tests/sendspin/config/device_info_explicit.yaml @@ -0,0 +1,18 @@ +esphome: + name: test + project: + name: project_manufacturer.project_model + version: 9.9.9 + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ap: + +sendspin: + manufacturer: Explicit Manufacturer + model: Explicit Model + firmware_version: 1.2.3 diff --git a/tests/component_tests/sendspin/config/device_info_project.yaml b/tests/component_tests/sendspin/config/device_info_project.yaml new file mode 100644 index 0000000000..395b2889fc --- /dev/null +++ b/tests/component_tests/sendspin/config/device_info_project.yaml @@ -0,0 +1,15 @@ +esphome: + name: test + project: + name: project_manufacturer.project_model + version: 9.9.9 + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ap: + +sendspin: diff --git a/tests/component_tests/sendspin/test_device_info.py b/tests/component_tests/sendspin/test_device_info.py new file mode 100644 index 0000000000..833dd398b4 --- /dev/null +++ b/tests/component_tests/sendspin/test_device_info.py @@ -0,0 +1,83 @@ +"""Tests for the device information the sendspin hub reports to the server.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import ( + CONF_FIRMWARE_VERSION, + CONF_MANUFACTURER, + CONFIG_SCHEMA, +) +from esphome.const import CONF_MODEL, PlatformFramework +from tests.component_tests.types import SetCoreConfigCallable + + +def test_explicit_device_info_wins_over_project( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Configured values take precedence over the project information.""" + main_cpp = generate_main(component_config_path("device_info_explicit.yaml")) + + assert 'set_manufacturer("Explicit Manufacturer")' in main_cpp + assert 'set_model("Explicit Model")' in main_cpp + assert 'set_firmware_version("1.2.3")' in main_cpp + + +def test_project_supplies_device_info( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Without configured values, the project name splits into manufacturer and model.""" + main_cpp = generate_main(component_config_path("device_info_project.yaml")) + + assert 'set_manufacturer("project_manufacturer")' in main_cpp + assert 'set_model("project_model")' in main_cpp + assert 'set_firmware_version("9.9.9")' in main_cpp + + +def test_no_device_info_leaves_hub_defaults( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """With neither source, nothing is emitted and the hub keeps its own defaults.""" + main_cpp = generate_main(component_config_path("device_info_default.yaml")) + + assert "set_manufacturer(" not in main_cpp + assert "set_model(" not in main_cpp + assert "set_firmware_version(" not in main_cpp + + +@pytest.mark.parametrize( + "conf_key", [CONF_MANUFACTURER, CONF_MODEL, CONF_FIRMWARE_VERSION] +) +def test_empty_device_info_rejected( + set_core_config: SetCoreConfigCallable, conf_key: str +) -> None: + """An empty string would be sent to the server as an empty value, so it is not accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA({conf_key: ""}) + + +@pytest.mark.parametrize( + "conf_key", [CONF_MANUFACTURER, CONF_MODEL, CONF_FIRMWARE_VERSION] +) +def test_device_info_capped_at_127_bytes( + set_core_config: SetCoreConfigCallable, conf_key: str +) -> None: + """The cap is in bytes so the protobuf length prefix stays a single byte.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA({conf_key: "a" * 127}) + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA({conf_key: "a" * 128}) + # 64 two-byte characters is 128 bytes. + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA({conf_key: "é" * 64}) diff --git a/tests/components/sendspin/common-hub.yaml b/tests/components/sendspin/common-hub.yaml index 7a6a9ffd4f..bd6747ee07 100644 --- a/tests/components/sendspin/common-hub.yaml +++ b/tests/components/sendspin/common-hub.yaml @@ -4,3 +4,6 @@ psram: sendspin: id: sendspin_hub_id task_stack_in_psram: true + manufacturer: Test Manufacturer + model: Test Model + firmware_version: 1.2.3 From 3e3822e5541f3562fae64b29837b79b1027af674 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:22:30 +0000 Subject: [PATCH 110/178] Bump bundled esphome-device-builder to 1.14.6 (#19072) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ac84ee4689..cfa47fbdad 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.6 RUN \ platformio settings set enable_telemetry No \ From f66ef23256f467a572517f6ee87956f5f527fa60 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 11 Sep 2026 03:20:30 -0500 Subject: [PATCH 111/178] [core] Support set_internal() during setup, log error after setup (#19069) --- esphome/core/entity_base.cpp | 9 ++++ esphome/core/entity_base.h | 27 ++++++++---- .../fixtures/set_internal_at_boot.yaml | 34 +++++++++++++++ .../integration/test_set_internal_at_boot.py | 41 +++++++++++++++++++ 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 tests/integration/fixtures/set_internal_at_boot.yaml create mode 100644 tests/integration/test_set_internal_at_boot.py diff --git a/esphome/core/entity_base.cpp b/esphome/core/entity_base.cpp index 21a5fc3706..dc27c1e56a 100644 --- a/esphome/core/entity_base.cpp +++ b/esphome/core/entity_base.cpp @@ -56,6 +56,15 @@ void EntityBase::configure_entity_(const char *name, uint32_t object_id_hash, ui this->flags_.entity_category = (entity_fields >> ENTITY_FIELD_ENTITY_CATEGORY_SHIFT) & 0x3; } +void EntityBase::set_internal(bool internal) { + // Remove the after-setup path in 2027.3.0 and ignore the call instead. + if (App.is_setup_complete()) { + ESP_LOGE(TAG, "'%s': set_internal() after setup is undefined behavior, stops working in 2027.3.0", + this->get_name().c_str()); + } + this->flags_.internal = internal; +} + // Weak default lookup functions — overridden by generated code in main.cpp __attribute__((weak)) const char *entity_device_class_lookup(uint8_t) { return ""; } __attribute__((weak)) const char *entity_uom_lookup(uint8_t) { return ""; } diff --git a/esphome/core/entity_base.h b/esphome/core/entity_base.h index f38e30bf52..8796e9f067 100644 --- a/esphome/core/entity_base.h +++ b/esphome/core/entity_base.h @@ -88,13 +88,26 @@ class EntityBase { // Get whether this Entity should be hidden outside ESPHome bool is_internal() const { return this->flags_.internal; } - // Deprecated: Calling set_internal() at runtime is undefined behavior. Components and clients - // are NOT notified of the change, the flag may have already been read during setup, and there - // is NO guarantee any consumer will observe the new value. Use the 'internal:' YAML key instead. - ESPDEPRECATED("set_internal() is undefined behavior at runtime — components and Home Assistant are NOT " - "notified. Use the 'internal:' YAML key instead. Will be removed in 2027.3.0.", - "2026.3.0") - void set_internal(bool internal) { this->flags_.internal = internal; } + // Set whether this Entity should be hidden outside ESPHome. Prefer the 'internal:' YAML key + // whenever possible: it is guaranteed and has none of the limitations below. Use this only when + // the decision can only be made at boot. Must be called before MQTT and the API read the flag: + // from on_boot at the default priority, or a setup() that runs above setup_priority::AFTER_WIFI. + // If the answer comes from a device handshake, hold setup with can_proceed() until it arrives. + // Calls after setup finishes are undefined behavior: the flag is still written and an error is + // logged, and from 2027.3.0 the call will be ignored. + // + // Known limitations. Not bugs, so no issue reports please; a PR that removes one with no RAM + // or performance cost would be considered. + // - No consumer is notified of a change, so the flag can only be decided once per boot. + // - The guard is coarse: a call from a priority below AFTER_WIFI (an on_boot with a low priority, + // or a setup() at LATE) still passes, but the API camera listener is already registered, MQTT + // (AFTER_CONNECTION) has cached the flag, and an API client that connected while setup was + // stalled on a slow component has already listed the entities, so they keep the old value. + // - Un-hiding an entity declared 'internal: true' in YAML skips the duplicate name check that + // codegen runs for exposed entities, so a name collision can surface at runtime. Entities with + // only an 'id:' are forced internal and use the id as their name. + // - Zigbee codegen skips YAML internal entities entirely, so un-hiding cannot add them to Zigbee. + void set_internal(bool internal); // Check if this object is declared to be disabled by default. // That means that when the device gets added to Home Assistant (or other clients) it should diff --git a/tests/integration/fixtures/set_internal_at_boot.yaml b/tests/integration/fixtures/set_internal_at_boot.yaml new file mode 100644 index 0000000000..b3007e9dbd --- /dev/null +++ b/tests/integration/fixtures/set_internal_at_boot.yaml @@ -0,0 +1,34 @@ +esphome: + name: set-internal-at-boot + on_boot: + then: + - lambda: |- + id(hidden_at_boot).set_internal(true); + id(shown_at_boot).set_internal(false); + +host: + +api: + actions: + - action: set_internal_late + then: + - lambda: id(untouched).set_internal(true); + +logger: + +sensor: + - platform: template + name: "Hidden At Boot" + id: hidden_at_boot + lambda: return 1.0; + + - platform: template + name: "Shown At Boot" + id: shown_at_boot + internal: true + lambda: return 2.0; + + - platform: template + name: "Untouched" + id: untouched + lambda: return 3.0; diff --git a/tests/integration/test_set_internal_at_boot.py b/tests/integration/test_set_internal_at_boot.py new file mode 100644 index 0000000000..68b0bd1080 --- /dev/null +++ b/tests/integration/test_set_internal_at_boot.py @@ -0,0 +1,41 @@ +"""Integration test for set_internal() called during and after setup.""" + +from __future__ import annotations + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_set_internal_at_boot( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """set_internal() in on_boot changes API exposure, later calls log an error.""" + waiter = LineWaiter() + + async with ( + run_compiled(yaml_config, line_callback=waiter.callback), + api_client_connected() as client, + ): + entities, services = await client.list_entities_services() + names = {entity.name for entity in entities} + + assert "Hidden At Boot" not in names + assert "Shown At Boot" in names + assert "Untouched" in names + + late = next(s for s in services if s.name == "set_internal_late") + await client.execute_service(late, {}) + await waiter.wait_for( + "'Untouched'", + "set_internal() after setup is undefined behavior", + timeout=5.0, + ) + + # Still written during the deprecation window, ignored from 2027.3.0 + entities, _ = await client.list_entities_services() + assert "Untouched" not in {entity.name for entity in entities} From 380938177c1cc0599f4df97f1368adb96f48f07a Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Fri, 11 Sep 2026 03:25:52 -0500 Subject: [PATCH 112/178] [uart] Add apply_settings_live() for in-place ESP-IDF reconfiguration (#19087) Co-authored-by: Claude Fable 5.1 --- .../uart/uart_component_esp_idf.cpp | 129 +++++++++++++----- .../components/uart/uart_component_esp_idf.h | 36 +++++ 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/esphome/components/uart/uart_component_esp_idf.cpp b/esphome/components/uart/uart_component_esp_idf.cpp index bbeb86bcdb..e5d5fbc983 100644 --- a/esphome/components/uart/uart_component_esp_idf.cpp +++ b/esphome/components/uart/uart_component_esp_idf.cpp @@ -160,6 +160,7 @@ void IDFUARTComponent::load_settings(bool dump_config) { this->mark_failed(); return; } + this->last_good_framing_ = this->framing_(); int8_t tx = this->tx_pin_ != nullptr ? this->tx_pin_->get_pin() : -1; int8_t rx = this->rx_pin_ != nullptr ? this->rx_pin_->get_pin() : -1; @@ -189,18 +190,9 @@ void IDFUARTComponent::load_settings(bool dump_config) { setup_pin_if_needed(this->tx_pin_); } - uint32_t invert = 0; - if (this->tx_pin_ != nullptr && this->tx_pin_->is_inverted()) { - invert |= UART_SIGNAL_TXD_INV; - } - if (this->rx_pin_ != nullptr && this->rx_pin_->is_inverted()) { - invert |= UART_SIGNAL_RXD_INV; - } - if (this->flow_control_pin_ != nullptr && this->flow_control_pin_->is_inverted()) { - invert |= UART_SIGNAL_RTS_INV; - } - - err = uart_set_line_inverse(this->uart_num_, invert); + // Must precede uart_set_pin() so an inverted TX line never shows the wrong idle + // level; apply_line_settings_() repeats it later for the reset registers. + err = uart_set_line_inverse(this->uart_num_, this->line_inversion_mask_()); if (err != ESP_OK) { ESP_LOGW(TAG, "uart_set_line_inverse failed: %s", esp_err_to_name(err)); this->mark_failed(); @@ -214,25 +206,7 @@ void IDFUARTComponent::load_settings(bool dump_config) { return; } - err = uart_set_rx_full_threshold(this->uart_num_, this->rx_full_threshold_); - if (err != ESP_OK) { - ESP_LOGW(TAG, "uart_set_rx_full_threshold failed: %s", esp_err_to_name(err)); - this->mark_failed(); - return; - } - - err = uart_set_rx_timeout(this->uart_num_, this->rx_timeout_); - if (err != ESP_OK) { - ESP_LOGW(TAG, "uart_set_rx_timeout failed: %s", esp_err_to_name(err)); - this->mark_failed(); - return; - } - - // Per ESP-IDF docs, uart_set_mode() must be called only after uart_driver_install(). - auto mode = this->flow_control_pin_ != nullptr ? UART_MODE_RS485_HALF_DUPLEX : UART_MODE_UART; - err = uart_set_mode(this->uart_num_, mode); - if (err != ESP_OK) { - ESP_LOGW(TAG, "uart_set_mode failed: %s", esp_err_to_name(err)); + if (this->apply_line_settings_() != ESP_OK) { this->mark_failed(); return; } @@ -250,6 +224,99 @@ void IDFUARTComponent::load_settings(bool dump_config) { } } +uint32_t IDFUARTComponent::line_inversion_mask_() { + uint32_t invert = 0; + if (this->tx_pin_ != nullptr && this->tx_pin_->is_inverted()) { + invert |= UART_SIGNAL_TXD_INV; + } + if (this->rx_pin_ != nullptr && this->rx_pin_->is_inverted()) { + invert |= UART_SIGNAL_RXD_INV; + } + if (this->flow_control_pin_ != nullptr && this->flow_control_pin_->is_inverted()) { + invert |= UART_SIGNAL_RTS_INV; + } + return invert; +} + +esp_err_t IDFUARTComponent::apply_line_settings_() { + // uart_param_config() resets these; call after every use of it. + esp_err_t err = uart_set_line_inverse(this->uart_num_, this->line_inversion_mask_()); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_line_inverse failed: %s", esp_err_to_name(err)); + return err; + } + + err = uart_set_rx_full_threshold(this->uart_num_, this->rx_full_threshold_); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_rx_full_threshold failed: %s", esp_err_to_name(err)); + return err; + } + + err = uart_set_rx_timeout(this->uart_num_, this->rx_timeout_); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_rx_timeout failed: %s", esp_err_to_name(err)); + return err; + } + + // Per ESP-IDF docs, uart_set_mode() must be called only after uart_driver_install(). + auto mode = this->flow_control_pin_ != nullptr ? UART_MODE_RS485_HALF_DUPLEX : UART_MODE_UART; + err = uart_set_mode(this->uart_num_, mode); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_mode failed: %s", esp_err_to_name(err)); + return err; + } + + return ESP_OK; +} + +void IDFUARTComponent::set_framing_(const Framing &framing) { + this->baud_rate_ = framing.baud_rate; + this->data_bits_ = framing.data_bits; + this->stop_bits_ = framing.stop_bits; + this->parity_ = framing.parity; + this->rx_full_threshold_ = framing.rx_full_threshold; +} + +esp_err_t IDFUARTComponent::apply_settings_live() { + if (this->is_failed()) { + return ESP_ERR_INVALID_STATE; + } + // No driver yet: nothing to reconfigure in place. + if (!uart_is_driver_installed(this->uart_num_)) { + this->load_settings(false); + return this->is_failed() ? ESP_FAIL : ESP_OK; + } + // Keeps the driver ring buffers; flushes both hardware FIFOs (in-flight bytes lost). + uart_config_t uart_config = this->get_config_(); + esp_err_t err = uart_param_config(this->uart_num_, &uart_config); + if (err != ESP_OK) { + // Failure leaves the registers reset; put back the last accepted framing so the + // getters still describe the hardware. + if (this->last_good_framing_.baud_rate == 0) { + ESP_LOGE(TAG, "uart_param_config (live) failed: %s; no previous framing to restore", esp_err_to_name(err)); + this->mark_failed(); + return err; + } + ESP_LOGW(TAG, "uart_param_config (live) failed: %s; restoring %" PRIu32 " baud", esp_err_to_name(err), + this->last_good_framing_.baud_rate); + this->set_framing_(this->last_good_framing_); + uart_config = this->get_config_(); + esp_err_t restore_err = uart_param_config(this->uart_num_, &uart_config); + if (restore_err != ESP_OK) { + ESP_LOGE(TAG, "UART left unconfigured after failed live reconfigure: %s", esp_err_to_name(restore_err)); + this->mark_failed(); + return err; + } + // Previous framing is live again; report the refusal (line-setting errors log). + this->apply_line_settings_(); + return err; + } + this->last_good_framing_ = this->framing_(); + // The new framing is live; a line-setting failure here only logs. + this->apply_line_settings_(); + return ESP_OK; +} + void IDFUARTComponent::dump_config() { ESP_LOGCONFIG(TAG, "UART Bus %u:", this->uart_num_); LOG_PIN(" TX Pin: ", this->tx_pin_); diff --git a/esphome/components/uart/uart_component_esp_idf.h b/esphome/components/uart/uart_component_esp_idf.h index a761d80f04..d9297bfa34 100644 --- a/esphome/components/uart/uart_component_esp_idf.h +++ b/esphome/components/uart/uart_component_esp_idf.h @@ -52,13 +52,49 @@ class IDFUARTComponent final : public UARTComponent, public Component { void load_settings(bool dump_config) override; using UARTComponent::load_settings; // also bring in the no-arg overload for convenience + /** + * Apply the current framing (baud rate, parity, data/stop bits) to the installed + * driver in place, without the delete/reinstall of load_settings(). Tasks blocked in + * the driver survive and the ring buffers are kept, but both hardware FIFOs are + * flushed: a frame in flight reaches the peer truncated and bytes not yet out of the + * RX FIFO are dropped. No lock is taken: quiesce writers first if that matters. + * rx_full_threshold is not rescaled (call set_rx_full_threshold_ms() first if it + * should follow the baud rate); a rollback restores the value from the last accepted + * configuration, undoing a standalone set_rx_full_threshold() made since. Without an + * installed driver this is a full load_settings(false) instead. + * + * @return ESP_OK once the new framing is live (a line-setting error after that only + * logs). On rejection (unreachable baud rate) the previous framing is restored and + * the driver's error returned; if the restore fails too the component is marked + * failed. ESP_ERR_INVALID_STATE if already failed; ESP_FAIL if the fallback + * load_settings() fails. + */ + esp_err_t apply_settings_live(); + void on_shutdown() override; protected: void check_logger_conflict() override; + uint32_t line_inversion_mask_(); + // Re-applies what uart_param_config() resets: inversion, RX threshold/timeout, mode. + esp_err_t apply_line_settings_(); uart_port_t uart_num_{UART_NUM_MAX}; uart_config_t get_config_(); + struct Framing { + uint32_t baud_rate; + uint8_t data_bits; + uint8_t stop_bits; + UARTParityOptions parity; + size_t rx_full_threshold; // sized for the baud rate, so rolled back with it + }; + Framing framing_() const { + return {this->baud_rate_, this->data_bits_, this->stop_bits_, this->parity_, this->rx_full_threshold_}; + } + void set_framing_(const Framing &framing); + // Last framing the driver accepted; baud_rate 0 means none yet. + Framing last_good_framing_{}; + bool has_peek_{false}; uint8_t peek_byte_; uint32_t flush_timeout_ms_{0}; ///< 0 means wait indefinitely (portMAX_DELAY). From 6a21ab4ea705cb4c885cb2590683949874d9b931 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:30:51 +1200 Subject: [PATCH 113/178] [esp32] Trim mbedTLS to client-only defaults and stub vasprintf on the C6 (#19088) --- esphome/components/esp32/__init__.py | 118 ++++++++++++++++++ esphome/components/esp32/vasprintf_stubs.cpp | 53 ++++++++ esphome/components/openthread/__init__.py | 10 ++ esphome/components/wifi/__init__.py | 7 ++ esphome/core/defines.h | 1 + .../esp32/config/mbedtls_tls_default.yaml | 14 +++ .../esp32/config/mbedtls_tls_openthread.yaml | 19 +++ .../esp32/config/mbedtls_tls_opt_out.yaml | 17 +++ .../config/mbedtls_tls_user_sdkconfig.yaml | 17 +++ .../esp32/config/mbedtls_tls_wifi_eap.yaml | 17 +++ .../esp32/config/vasprintf_stub_c6.yaml | 7 ++ .../config/vasprintf_stub_c6_full_printf.yaml | 9 ++ tests/component_tests/esp32/test_esp32.py | 99 +++++++++++++++ tests/components/esp32/test.esp32-idf.yaml | 2 + .../http_request/test.esp32-c6-idf.yaml | 4 + 15 files changed, 394 insertions(+) create mode 100644 esphome/components/esp32/vasprintf_stubs.cpp create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_default.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml create mode 100644 tests/component_tests/esp32/config/vasprintf_stub_c6.yaml create mode 100644 tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml create mode 100644 tests/components/http_request/test.esp32-c6-idf.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 3f5a34bc73..d027c9a1c6 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -189,6 +189,13 @@ PSRAM_XIP_VARIANTS = { VARIANT_ESP32S31, } +# Variants whose ROM exports a full-format vsnprintf but no vasprintf +# (esp32c6.rom.newlib-normal.ld). There, the newlib printf engine is only +# linked because esp_http_client calls vasprintf; see vasprintf_stubs.cpp. +# The other variants either export both (classic ESP32, nano-format only) or +# neither, so the engine is already in the image and the wrap saves nothing. +ROM_VSNPRINTF_WITHOUT_VASPRINTF_VARIANTS = {VARIANT_ESP32C6} + # NVS encryption (HMAC peripheral scheme) is only available on variants that # expose the HMAC peripheral (SOC_HMAC_SUPPORTED in soc_caps.h). The original # ESP32 and ESP32-C2 do not have it. New variants with an HMAC peripheral @@ -1732,6 +1739,8 @@ CONF_DISABLE_USB_SERIAL_JTAG_SECONDARY = "disable_usb_serial_jtag_secondary" CONF_DISABLE_DEV_NULL_VFS = "disable_dev_null_vfs" CONF_DISABLE_MBEDTLS_PEER_CERT = "disable_mbedtls_peer_cert" CONF_DISABLE_MBEDTLS_PKCS7 = "disable_mbedtls_pkcs7" +CONF_DISABLE_MBEDTLS_TLS_SERVER = "disable_mbedtls_tls_server" +CONF_DISABLE_MBEDTLS_TLS_EXTRAS = "disable_mbedtls_tls_extras" CONF_DISABLE_REGI2C_IN_IRAM = "disable_regi2c_in_iram" CONF_DISABLE_FATFS = "disable_fatfs" CONF_ADC_ONESHOT_IN_IRAM = "adc_oneshot_in_iram" @@ -1746,6 +1755,8 @@ KEY_VFS_TERMIOS_REQUIRED = "vfs_termios_required" KEY_USB_SERIAL_JTAG_SECONDARY_REQUIRED = "usb_serial_jtag_secondary_required" KEY_MBEDTLS_PEER_CERT_REQUIRED = "mbedtls_peer_cert_required" KEY_MBEDTLS_PKCS7_REQUIRED = "mbedtls_pkcs7_required" +KEY_MBEDTLS_TLS_SERVER_REQUIRED = "mbedtls_tls_server_required" +KEY_MBEDTLS_TLS_EXTRAS_REQUIRED = "mbedtls_tls_extras_required" KEY_FATFS_REQUIRED = "fatfs_required" KEY_MBEDTLS_SHA512_REQUIRED = "mbedtls_sha512_required" KEY_ADC_ONESHOT_IRAM_REQUIRED = "adc_oneshot_iram_required" @@ -1830,6 +1841,30 @@ def require_mbedtls_pkcs7() -> None: CORE.data[KEY_ESP32][KEY_MBEDTLS_PKCS7_REQUIRED] = True +def require_mbedtls_tls_server() -> None: + """Mark that the mbedTLS server-side TLS/DTLS handshake is required. + + Call this from components that accept TLS connections (OpenThread's DTLS + commissioner does). This prevents CONFIG_MBEDTLS_TLS_CLIENT_ONLY from + being selected. + """ + CORE.data[KEY_ESP32][KEY_MBEDTLS_TLS_SERVER_REQUIRED] = True + + +def require_mbedtls_tls_extras(options: Iterable[str] | None = None) -> None: + """Mark TLS features disabled by ``disable_mbedtls_tls_extras`` as required. + + ``options`` names the entries of ``MBEDTLS_TLS_EXTRA_OPTIONS`` to keep; + omit it to keep all of them. Call this from components that need AES-CCM, + deterministic ECDSA signing, static RSA/ECDH key exchange, TLS + renegotiation or session tickets, or that run a TLS client against + servers ESPHome cannot vet (wpa_supplicant's EAP client). A user-supplied + sdkconfig_options value is never overridden either. + """ + required = CORE.data[KEY_ESP32].setdefault(KEY_MBEDTLS_TLS_EXTRAS_REQUIRED, set()) + required.update(MBEDTLS_TLS_EXTRA_OPTIONS if options is None else options) + + def require_mbedtls_sha512() -> None: """Mark that mbedTLS SHA-384/SHA-512 support is required by a component. @@ -1987,6 +2022,8 @@ FRAMEWORK_SCHEMA = cv.Schema( cv.Optional(CONF_DISABLE_DEV_NULL_VFS, default=True): cv.boolean, cv.Optional(CONF_DISABLE_MBEDTLS_PEER_CERT, default=True): cv.boolean, cv.Optional(CONF_DISABLE_MBEDTLS_PKCS7, default=True): cv.boolean, + cv.Optional(CONF_DISABLE_MBEDTLS_TLS_SERVER, default=True): cv.boolean, + cv.Optional(CONF_DISABLE_MBEDTLS_TLS_EXTRAS, default=True): cv.boolean, cv.Optional(CONF_DISABLE_REGI2C_IN_IRAM, default=True): cv.boolean, cv.Optional(CONF_ADC_ONESHOT_IN_IRAM, default=False): cv.boolean, cv.Optional(CONF_DISABLE_FATFS, default=True): cv.boolean, @@ -2302,6 +2339,69 @@ async def _reconcile_certificate_bundle_sdkconfig() -> None: set_idf_sdkconfig_default("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", True) +# TLS features an HTTPS/MQTT client talking to a modern server never +# negotiates. Static RSA and static ECDH key exchange have no forward secrecy +# and are gone in TLS 1.3, renegotiation is deprecated, esp-tls never enables +# session tickets, AES-CCM ciphersuites are not offered by web servers, and +# deterministic ECDSA only matters when signing with a private key. Together +# they cost ~10 KB of flash whenever TLS is linked (http_request, mqtt). +# wpa_supplicant's EAP client is a second TLS client that talks to RADIUS +# servers ESPHome cannot vet, and a failed EAP handshake leaves the device +# off the network, so the wifi component re-enables all of these when eap is +# configured. +# The EC public key parsing extras stay enabled: they decide whether a peer +# certificate with a compressed point or explicit curve parameters parses, +# which no component can know ahead of time. +MBEDTLS_TLS_EXTRA_OPTIONS = ( + "CONFIG_MBEDTLS_KEY_EXCHANGE_RSA", + "CONFIG_MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA", + "CONFIG_MBEDTLS_KEY_EXCHANGE_ECDH_RSA", + "CONFIG_MBEDTLS_SSL_RENEGOTIATION", + "CONFIG_MBEDTLS_CLIENT_SSL_SESSION_TICKETS", + "CONFIG_MBEDTLS_SERVER_SSL_SESSION_TICKETS", + "CONFIG_MBEDTLS_CCM_C", + "CONFIG_MBEDTLS_ECDSA_DETERMINISTIC", +) + +# Members of the mbedTLS "TLS Protocol Role" Kconfig choice. Setting one +# member is only valid when the user has not already chosen another. +MBEDTLS_TLS_ROLE_OPTIONS = ( + "CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT", + "CONFIG_MBEDTLS_TLS_SERVER_ONLY", + "CONFIG_MBEDTLS_TLS_CLIENT_ONLY", + "CONFIG_MBEDTLS_TLS_DISABLED", +) + + +@coroutine_with_priority(CoroPriority.FINAL) +async def _reconcile_mbedtls_tls_sdkconfig( + disable_tls_server: bool, disable_tls_extras: bool +) -> None: + """Trim mbedTLS to what a TLS client needs unless a component asked otherwise. + + Runs at FINAL priority so every require_mbedtls_tls_server() and + require_mbedtls_tls_extras() call has happened. Only the server-side + handshake (~7 KB) is a separate option; nothing in ESPHome accepts TLS + connections, but OpenThread's DTLS commissioner does. A user-supplied + sdkconfig_options value always wins; for the TLS role choice, any member + the user set leaves the whole choice alone so the pair cannot conflict. + """ + data = CORE.data[KEY_ESP32] + sdkconfig = data[KEY_SDKCONFIG_OPTIONS] + if ( + disable_tls_server + and not data.get(KEY_MBEDTLS_TLS_SERVER_REQUIRED, False) + and not any(option in sdkconfig for option in MBEDTLS_TLS_ROLE_OPTIONS) + ): + add_idf_sdkconfig_option("CONFIG_MBEDTLS_TLS_CLIENT_ONLY", True) + add_idf_sdkconfig_option("CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT", False) + if disable_tls_extras: + required = data.get(KEY_MBEDTLS_TLS_EXTRAS_REQUIRED, set()) + for option in MBEDTLS_TLS_EXTRA_OPTIONS: + if option not in required: + set_idf_sdkconfig_default(option, False) + + @coroutine_with_priority(CoroPriority.FINAL) async def _reconcile_network_sdkconfig() -> None: """Reconcile WiFi/Ethernet/Bluetooth/coexistence sdkconfig flags. @@ -2566,6 +2666,17 @@ async def to_code(config): else: for symbol in ("vprintf", "printf", "fprintf", "vfprintf"): cg.add_build_flag(f"-Wl,--wrap={symbol}") + # esp_http_client calls vasprintf, which on the ESP32-C6 is the only + # reference to newlib's full printf engine (~20 KB: _svfprintf_r, + # _dtoa_r and their helpers); every other caller resolves to the + # ROM. See vasprintf_stubs.cpp. The --undefined flag is needed + # because libsrc.a is scanned before the IDF libraries that + # reference the symbol, so the stub would otherwise never be pulled + # from the archive. + if variant in ROM_VSNPRINTF_WITHOUT_VASPRINTF_VARIANTS: + cg.add_define("USE_ESP32_VASPRINTF_STUB") + cg.add_build_flag("-Wl,--wrap=vasprintf") + cg.add_build_flag("-Wl,--undefined=__wrap_vasprintf") else: cg.add_build_flag("-DUSE_ARDUINO") cg.add_build_flag("-DUSE_ESP32_FRAMEWORK_ARDUINO") @@ -2991,6 +3102,13 @@ async def to_code(config): # FINAL priority: runs after every require_certificate_bundle() call CORE.add_job(_reconcile_certificate_bundle_sdkconfig) + # FINAL priority: runs after every require_mbedtls_tls_*() call + CORE.add_job( + _reconcile_mbedtls_tls_sdkconfig, + advanced[CONF_DISABLE_MBEDTLS_TLS_SERVER], + advanced[CONF_DISABLE_MBEDTLS_TLS_EXTRAS], + ) + # FINAL: require_*() calls can come from to_code at or below this priority, so an # inline read would be iteration-order-dependent; reconcile once after every job ran. CORE.add_job( diff --git a/esphome/components/esp32/vasprintf_stubs.cpp b/esphome/components/esp32/vasprintf_stubs.cpp new file mode 100644 index 0000000000..308a58ebda --- /dev/null +++ b/esphome/components/esp32/vasprintf_stubs.cpp @@ -0,0 +1,53 @@ +/* + * Linker wrap stub for vasprintf() on variants whose ROM exports a + * full-format vsnprintf() but no vasprintf() (ESP32-C6, newlib only). + * + * On those chips every snprintf/vsnprintf call in the image resolves to + * the ROM, so the newlib printf engine (_svfprintf_r, _dtoa_r and their + * helpers, ~20 KB) is not linked at all until something references a + * printf-family function the ROM lacks. esp_http_client does exactly that + * through vasprintf() in its header and auth helpers, so adding + * http_request to a build costs the whole engine on top of the HTTP and + * TLS code itself. + * + * This stub reimplements vasprintf() on top of the ROM vsnprintf(), which + * keeps the engine out of the image. It is only compiled in when codegen + * defines USE_ESP32_VASPRINTF_STUB, which is gated on the variant's ROM + * linker script and on the same newlib condition as printf_stubs.cpp. + */ + +#include "esphome/core/defines.h" + +#if defined(USE_ESP_IDF) && defined(USE_ESP32_VASPRINTF_STUB) + +#include +#include +#include + +namespace esphome::esp32 {} + +// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) +extern "C" { + +int __wrap_vasprintf(char **strp, const char *fmt, va_list ap) { + va_list ap_copy; + va_copy(ap_copy, ap); + int len = vsnprintf(nullptr, 0, fmt, ap_copy); + va_end(ap_copy); + if (len < 0) { + return len; + } + // vasprintf's contract is a malloc'd buffer the caller releases with free() + char *buf = static_cast(malloc(static_cast(len) + 1)); // NOLINT(cppcoreguidelines-no-malloc) + if (buf == nullptr) { + return -1; + } + vsnprintf(buf, static_cast(len) + 1, fmt, ap); + *strp = buf; + return len; +} + +} // extern "C" +// NOLINTEND(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) + +#endif // USE_ESP_IDF && USE_ESP32_VASPRINTF_STUB diff --git a/esphome/components/openthread/__init__.py b/esphome/components/openthread/__init__.py index ab69f5d9ae..a71151f3ff 100644 --- a/esphome/components/openthread/__init__.py +++ b/esphome/components/openthread/__init__.py @@ -13,6 +13,8 @@ from esphome.components.esp32 import ( get_esp32_variant, include_builtin_idf_component, only_on_variant, + require_mbedtls_tls_extras, + require_mbedtls_tls_server, require_vfs_select, ) from esphome.components.mdns import MDNSComponent, enable_mdns_storage @@ -109,6 +111,14 @@ def set_sdkconfig_options(config: ConfigType) -> None: add_idf_sdkconfig_option("CONFIG_OPENTHREAD_ENABLED", True) + # OpenThread's DTLS commissioner is a TLS server, and its crypto platform + # uses AES-CCM and deterministic ECDSA directly. Keep the esp32 component + # from trimming them out of mbedTLS. + require_mbedtls_tls_server() + require_mbedtls_tls_extras( + ("CONFIG_MBEDTLS_CCM_C", "CONFIG_MBEDTLS_ECDSA_DETERMINISTIC") + ) + if not config.get(CONF_TLV): if pan_id := config.get(CONF_PAN_ID): add_idf_sdkconfig_option("CONFIG_OPENTHREAD_NETWORK_PANID", pan_id) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 1691dcc293..58803a8cdf 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -12,6 +12,7 @@ from esphome.components.esp32 import ( get_esp32_variant, only_on_variant, request_wifi, + require_mbedtls_tls_extras, ) from esphome.components.network import ( add_use_address, @@ -658,6 +659,12 @@ async def to_code(config): # Disable Enterprise WiFi support if no EAP is configured if CORE.is_esp32: add_idf_sdkconfig_option("CONFIG_ESP_WIFI_ENTERPRISE_SUPPORT", has_eap) + if has_eap: + # wpa_supplicant's EAP client negotiates with whatever the RADIUS + # server offers, and a failed handshake leaves the device off the + # network, so keep every mbedTLS client feature the esp32 platform + # would otherwise trim. + require_mbedtls_tls_extras() # Only define USE_WIFI_MANUAL_IP if any AP uses manual IP if has_manual_ip: diff --git a/esphome/core/defines.h b/esphome/core/defines.h index c3b16d833a..9144e65576 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -298,6 +298,7 @@ // ESP32-specific feature flags #ifdef USE_ESP32 #define USE_ESP32_CRASH_HANDLER +#define USE_ESP32_VASPRINTF_STUB #define USE_ESP32_INTERNAL_GPIO #define USE_MQTT_IDF_ENQUEUE #define USE_ESPHOME_TASK_LOG_BUFFER diff --git a/tests/component_tests/esp32/config/mbedtls_tls_default.yaml b/tests/component_tests/esp32/config/mbedtls_tls_default.yaml new file mode 100644 index 0000000000..b29e5de2bd --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_default.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml b/tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml new file mode 100644 index 0000000000..62ca893d2c --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml @@ -0,0 +1,19 @@ +esphome: + name: test + +esp32: + variant: esp32c6 + framework: + type: esp-idf + +network: + enable_ipv6: true + +openthread: + channel: 13 + network_name: OpenThread-8f28 + network_key: 0xdfd34f0f05cad978ec4e32b0413038ff + pan_id: 0x8f28 + ext_pan_id: 0xd63e8e3e495ebbc3 + pskc: 0xc23a76e98f1a6483639b1ac1271e2e27 + mesh_local_prefix: fd53:145f:ed22:ad81::/64 diff --git a/tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml b/tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml new file mode 100644 index 0000000000..e675848391 --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml @@ -0,0 +1,17 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + advanced: + disable_mbedtls_tls_server: false + disable_mbedtls_tls_extras: false + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml b/tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml new file mode 100644 index 0000000000..44ff047a48 --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml @@ -0,0 +1,17 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + sdkconfig_options: + CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT: y + CONFIG_MBEDTLS_CCM_C: y + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml b/tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml new file mode 100644 index 0000000000..6c78e06265 --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml @@ -0,0 +1,17 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + eap: + identity: "user@example.org" + username: "user" + password: "secret" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/vasprintf_stub_c6.yaml b/tests/component_tests/esp32/config/vasprintf_stub_c6.yaml new file mode 100644 index 0000000000..8fa28e7c0f --- /dev/null +++ b/tests/component_tests/esp32/config/vasprintf_stub_c6.yaml @@ -0,0 +1,7 @@ +esphome: + name: test + +esp32: + variant: esp32c6 + framework: + type: esp-idf diff --git a/tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml b/tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml new file mode 100644 index 0000000000..075c3913b5 --- /dev/null +++ b/tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32c6 + framework: + type: esp-idf + advanced: + enable_full_printf: true diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 759020c732..2dd2a50c83 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -11,9 +11,12 @@ import pytest from esphome.components.esp32 import ( KEY_FATFS_REQUIRED, + KEY_MBEDTLS_TLS_EXTRAS_REQUIRED, + KEY_MBEDTLS_TLS_SERVER_REQUIRED, KEY_VFS_DIR_REQUIRED, KEY_VFS_SELECT_REQUIRED, KEY_VFS_TERMIOS_REQUIRED, + MBEDTLS_TLS_EXTRA_OPTIONS, VARIANT_ESP32, VARIANTS, NetworkSdkconfigData, @@ -1339,3 +1342,99 @@ def test_esp32_s31_gpio_validation( with caplog.at_level("WARNING"): validate_supports(pin) assert "GPIO36 is a strapping PIN" in caplog.text + + +_TLS_SERVER_OPTIONS = ( + "CONFIG_MBEDTLS_TLS_CLIENT_ONLY", + "CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT", +) + + +@pytest.mark.parametrize( + ("config_file", "server", "extras"), + [ + pytest.param("mbedtls_tls_default.yaml", (True, False), False, id="default"), + pytest.param("mbedtls_tls_opt_out.yaml", (None, None), None, id="opt_out"), + pytest.param("mbedtls_tls_wifi_eap.yaml", (True, False), None, id="wifi_eap"), + ], +) +def test_mbedtls_tls_trim_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + server: tuple[bool | None, bool | None], + extras: bool | None, +) -> None: + """Client-only TLS and the unused-feature trims apply unless opted out or required.""" + generate_main(component_config_path(config_file)) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert tuple(sdkconfig.get(name) for name in _TLS_SERVER_OPTIONS) == server + assert {sdkconfig.get(name) for name in MBEDTLS_TLS_EXTRA_OPTIONS} == {extras} + + +_OPENTHREAD_EXTRAS = {"CONFIG_MBEDTLS_CCM_C", "CONFIG_MBEDTLS_ECDSA_DETERMINISTIC"} + + +def test_mbedtls_tls_openthread_keeps_only_what_it_uses( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The OpenThread config keeps the DTLS server, CCM and deterministic ECDSA; the rest is trimmed.""" + generate_main(component_config_path("mbedtls_tls_openthread.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert tuple(sdkconfig.get(name) for name in _TLS_SERVER_OPTIONS) == (None, None) + for name in MBEDTLS_TLS_EXTRA_OPTIONS: + assert sdkconfig.get(name) is (None if name in _OPENTHREAD_EXTRAS else False) + + +def test_mbedtls_tls_user_sdkconfig_wins( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A user-set TLS role member leaves the whole choice alone; other user values are kept.""" + generate_main(component_config_path("mbedtls_tls_user_sdkconfig.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert sdkconfig.get("CONFIG_MBEDTLS_TLS_CLIENT_ONLY") is None + role = sdkconfig["CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT"] + assert isinstance(role, RawSdkconfigValue) and role.value == "y" + ccm = sdkconfig["CONFIG_MBEDTLS_CCM_C"] + assert isinstance(ccm, RawSdkconfigValue) and ccm.value == "y" + assert { + sdkconfig.get(name) + for name in MBEDTLS_TLS_EXTRA_OPTIONS + if name != "CONFIG_MBEDTLS_CCM_C" + } == {False} + + +def test_mbedtls_tls_openthread_requires_server_and_extras( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The OpenThread hooks mark the DTLS server and CCM/deterministic ECDSA as required.""" + generate_main(component_config_path("mbedtls_tls_openthread.yaml")) + assert CORE.data[KEY_ESP32][KEY_MBEDTLS_TLS_SERVER_REQUIRED] is True + assert CORE.data[KEY_ESP32][KEY_MBEDTLS_TLS_EXTRAS_REQUIRED] == _OPENTHREAD_EXTRAS + + +_VASPRINTF_STUB_FLAGS = {"-Wl,--wrap=vasprintf", "-Wl,--undefined=__wrap_vasprintf"} + + +@pytest.mark.parametrize( + ("config_file", "expected"), + [ + pytest.param("vasprintf_stub_c6.yaml", True, id="c6"), + pytest.param("vasprintf_stub_c6_full_printf.yaml", False, id="c6_full_printf"), + pytest.param("exclusion_reincludes.yaml", False, id="esp32"), + ], +) +def test_vasprintf_stub_only_on_rom_vsnprintf_variants( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + expected: bool, +) -> None: + """The vasprintf wrap is emitted only where the ROM lacks vasprintf but has vsnprintf.""" + generate_main(component_config_path(config_file)) + assert (CORE.build_flags >= _VASPRINTF_STUB_FLAGS) is expected + defines = {define.name for define in CORE.defines} + assert ("USE_ESP32_VASPRINTF_STUB" in defines) is expected diff --git a/tests/components/esp32/test.esp32-idf.yaml b/tests/components/esp32/test.esp32-idf.yaml index 523e614e24..7f31fe59c6 100644 --- a/tests/components/esp32/test.esp32-idf.yaml +++ b/tests/components/esp32/test.esp32-idf.yaml @@ -17,6 +17,8 @@ esp32: disable_dev_null_vfs: true disable_mbedtls_peer_cert: true disable_mbedtls_pkcs7: true + disable_mbedtls_tls_server: true + disable_mbedtls_tls_extras: true disable_regi2c_in_iram: true disable_fatfs: true sram1_as_iram: true diff --git a/tests/components/http_request/test.esp32-c6-idf.yaml b/tests/components/http_request/test.esp32-c6-idf.yaml new file mode 100644 index 0000000000..ee2f5aa59b --- /dev/null +++ b/tests/components/http_request/test.esp32-c6-idf.yaml @@ -0,0 +1,4 @@ +substitutions: + verify_ssl: "true" + +<<: !include common.yaml From 37e9b2b7af3ae84358bf59c9270462d551ec7644 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 11 Sep 2026 14:12:40 -0500 Subject: [PATCH 114/178] [remote_base] Make protocol methods non-virtual and size receiver lists from codegen (#19084) --- AGENTS.md | 3 + esphome/components/coolix/climate.py | 3 +- esphome/components/infrared/infrared.cpp | 5 - esphome/components/infrared/infrared.h | 3 +- esphome/components/ir_rf_proxy/infrared.py | 10 +- .../components/ir_rf_proxy/ir_rf_proxy.cpp | 4 - esphome/components/ir_rf_proxy/ir_rf_proxy.h | 3 +- .../components/ir_rf_proxy/radio_frequency.py | 10 +- esphome/components/midea/climate.py | 3 +- esphome/components/midea_ir/climate.py | 6 +- esphome/components/remote_base/__init__.py | 109 ++++++++++++++++-- .../remote_base/abbwelcome_protocol.h | 6 +- .../components/remote_base/aeha_protocol.h | 6 +- .../components/remote_base/beo4_protocol.h | 6 +- .../remote_base/brennenstuhl_protocol.h | 6 +- .../components/remote_base/byronsx_protocol.h | 6 +- .../remote_base/canalsat_protocol.h | 6 +- .../components/remote_base/coolix_protocol.h | 6 +- .../components/remote_base/dish_protocol.h | 6 +- .../components/remote_base/dooya_protocol.h | 6 +- .../components/remote_base/drayton_protocol.h | 6 +- .../components/remote_base/dyson_protocol.h | 6 +- .../components/remote_base/gobox_protocol.h | 6 +- .../components/remote_base/haier_protocol.h | 6 +- esphome/components/remote_base/jvc_protocol.h | 6 +- .../components/remote_base/keeloq_protocol.h | 6 +- esphome/components/remote_base/lg_protocol.h | 6 +- .../remote_base/magiquest_protocol.h | 6 +- .../components/remote_base/midea_protocol.h | 6 +- .../components/remote_base/mirage_protocol.h | 6 +- esphome/components/remote_base/nec_protocol.h | 6 +- .../components/remote_base/nexa_protocol.h | 6 +- .../remote_base/panasonic_protocol.h | 6 +- .../components/remote_base/pioneer_protocol.h | 6 +- .../components/remote_base/pronto_protocol.h | 6 +- esphome/components/remote_base/rc5_protocol.h | 6 +- esphome/components/remote_base/rc6_protocol.h | 6 +- .../remote_base/rc_switch_protocol.cpp | 38 +++--- .../remote_base/rc_switch_protocol.h | 35 +++++- .../components/remote_base/remote_base.cpp | 39 +++++-- esphome/components/remote_base/remote_base.h | 74 ++++++++---- .../components/remote_base/roomba_protocol.h | 6 +- .../remote_base/samsung36_protocol.h | 6 +- .../components/remote_base/samsung_protocol.h | 6 +- .../components/remote_base/sony_protocol.h | 6 +- .../remote_base/symphony_protocol.h | 6 +- .../remote_base/toshiba_ac_protocol.h | 6 +- .../components/remote_base/toto_protocol.h | 6 +- .../components/remote_receiver/__init__.py | 4 +- esphome/components/toshiba/climate.py | 3 +- esphome/core/defines.h | 37 ++++++ esphome/cpp_helpers.py | 38 ++++-- .../remote_receiver/__init__.py | 0 .../remote_receiver/config/receiver_bare.yaml | 9 ++ .../config/receiver_with_dumpers.yaml | 24 ++++ .../config/receiver_with_proxies.yaml | 22 ++++ .../remote_receiver/test_slot_counts.py | 91 +++++++++++++++ .../remote_receiver/bare-common.yaml | 6 + .../remote_receiver/test-bare.esp32-idf.yaml | 5 + tests/unit_tests/test_cpp_helpers.py | 25 ++++ 60 files changed, 606 insertions(+), 201 deletions(-) create mode 100644 tests/component_tests/remote_receiver/__init__.py create mode 100644 tests/component_tests/remote_receiver/config/receiver_bare.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml create mode 100644 tests/component_tests/remote_receiver/test_slot_counts.py create mode 100644 tests/components/remote_receiver/bare-common.yaml create mode 100644 tests/components/remote_receiver/test-bare.esp32-idf.yaml diff --git a/AGENTS.md b/AGENTS.md index 98bdd58ec5..8db3cd3d62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -629,6 +629,9 @@ file does, and it is the authority when they disagree. The most useful starting _request_listener_slot() cg.add(hub.register_listener(var)) ``` + When several instances each own a list declared at the same size (one per hub of a + `MULTI_CONF` component), pass the owning object as the key, `_request_listener_slot(str(hub))`; + the define is then the largest count any one key requested instead of the total. ```cpp #ifdef MY_COMPONENT_LISTENER_COUNT void register_listener(MyComponentListener *listener); diff --git a/esphome/components/coolix/climate.py b/esphome/components/coolix/climate.py index 3eb8dbe2f4..fcca8b89db 100644 --- a/esphome/components/coolix/climate.py +++ b/esphome/components/coolix/climate.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import climate_ir +from esphome.components import climate_ir, remote_base from esphome.types import ConfigType AUTO_LOAD = ["climate_ir"] @@ -12,4 +12,5 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(CoolixClimate) async def to_code(config: ConfigType) -> None: + remote_base.request_protocol("coolix") # used from C++ await climate_ir.new_climate_ir(config) diff --git a/esphome/components/infrared/infrared.cpp b/esphome/components/infrared/infrared.cpp index 5a909738c6..83039a5a9b 100644 --- a/esphome/components/infrared/infrared.cpp +++ b/esphome/components/infrared/infrared.cpp @@ -59,11 +59,6 @@ void Infrared::setup() { // Set up traits based on configuration this->traits_.set_supports_transmitter(this->has_transmitter()); this->traits_.set_supports_receiver(this->has_receiver()); - - // Register as listener for received IR data - if (this->receiver_ != nullptr) { - this->receiver_->register_listener(this); - } } void Infrared::dump_config() { diff --git a/esphome/components/infrared/infrared.h b/esphome/components/infrared/infrared.h index b6863e37ce..afbde57be2 100644 --- a/esphome/components/infrared/infrared.h +++ b/esphome/components/infrared/infrared.h @@ -119,7 +119,8 @@ class Infrared : public Component, public EntityBase, public remote_base::Remote void dump_config() override; float get_setup_priority() const override { return setup_priority::AFTER_CONNECTION; } - /// Set the remote receiver component + /// Set the remote receiver component; the listener registration happens from codegen, see + /// remote_base.attach_receiver void set_receiver(remote_base::RemoteReceiverBase *receiver) { this->receiver_ = receiver; } /// Set the remote transmitter component void set_transmitter(remote_base::RemoteTransmitterBase *transmitter) { this->transmitter_ = transmitter; } diff --git a/esphome/components/ir_rf_proxy/infrared.py b/esphome/components/ir_rf_proxy/infrared.py index 3218889721..288bd91673 100644 --- a/esphome/components/ir_rf_proxy/infrared.py +++ b/esphome/components/ir_rf_proxy/infrared.py @@ -3,7 +3,12 @@ from typing import Any import esphome.codegen as cg -from esphome.components import infrared, remote_receiver, remote_transmitter +from esphome.components import ( + infrared, + remote_base, + remote_receiver, + remote_transmitter, +) from esphome.components.const import CONF_RECEIVER_FREQUENCY import esphome.config_validation as cv from esphome.const import CONF_CARRIER_DUTY_PERCENT, CONF_FREQUENCY @@ -82,8 +87,7 @@ async def to_code(config: dict[str, Any]) -> None: # Link receiver if specified if CONF_REMOTE_RECEIVER_ID in config: - receiver = await cg.get_variable(config[CONF_REMOTE_RECEIVER_ID]) - cg.add(var.set_receiver(receiver)) + await remote_base.attach_receiver(var, config, CONF_REMOTE_RECEIVER_ID) # Set receiver demodulation frequency if specified (metadata only, no hardware effect) if CONF_RECEIVER_FREQUENCY in config: diff --git a/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp b/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp index c13c6198cb..ceb4c9a67c 100644 --- a/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp +++ b/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp @@ -97,10 +97,6 @@ void RfProxy::setup() { // remote_transmitter/receiver always uses OOK (on-off keying) this->traits_.add_supported_modulation(radio_frequency::RadioFrequencyModulation::RADIO_FREQUENCY_MODULATION_OOK); - - if (this->receiver_ != nullptr) { - this->receiver_->register_listener(this); - } } void RfProxy::dump_config() { diff --git a/esphome/components/ir_rf_proxy/ir_rf_proxy.h b/esphome/components/ir_rf_proxy/ir_rf_proxy.h index 5fc683354b..1aa4394fe8 100644 --- a/esphome/components/ir_rf_proxy/ir_rf_proxy.h +++ b/esphome/components/ir_rf_proxy/ir_rf_proxy.h @@ -56,7 +56,8 @@ class RfProxy final : public radio_frequency::RadioFrequency { /// Set the remote transmitter component void set_transmitter(remote_base::RemoteTransmitterBase *transmitter) { this->transmitter_ = transmitter; } - /// Set the remote receiver component + /// Set the remote receiver component; the listener registration happens from codegen, see + /// remote_base.attach_receiver void set_receiver(remote_base::RemoteReceiverBase *receiver) { this->receiver_ = receiver; } /// Set the fixed carrier frequency in Hz (metadata: advertised via traits, does not tune hardware) diff --git a/esphome/components/ir_rf_proxy/radio_frequency.py b/esphome/components/ir_rf_proxy/radio_frequency.py index a243909837..28b8fd5953 100644 --- a/esphome/components/ir_rf_proxy/radio_frequency.py +++ b/esphome/components/ir_rf_proxy/radio_frequency.py @@ -1,7 +1,12 @@ """Radio Frequency platform implementation using remote_base (remote_transmitter/receiver).""" import esphome.codegen as cg -from esphome.components import radio_frequency, remote_receiver, remote_transmitter +from esphome.components import ( + radio_frequency, + remote_base, + remote_receiver, + remote_transmitter, +) import esphome.config_validation as cv from esphome.const import CONF_CARRIER_DUTY_PERCENT, CONF_FREQUENCY import esphome.final_validate as fv @@ -66,5 +71,4 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_transmitter(transmitter)) if CONF_REMOTE_RECEIVER_ID in config: - receiver = await cg.get_variable(config[CONF_REMOTE_RECEIVER_ID]) - cg.add(var.set_receiver(receiver)) + await remote_base.attach_receiver(var, config, CONF_REMOTE_RECEIVER_ID) diff --git a/esphome/components/midea/climate.py b/esphome/components/midea/climate.py index 0e03bca233..07ad02d3af 100644 --- a/esphome/components/midea/climate.py +++ b/esphome/components/midea/climate.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import climate, remote_transmitter, sensor, uart +from esphome.components import climate, remote_base, remote_transmitter, sensor, uart from esphome.components.climate import ClimateMode, ClimatePreset, ClimateSwingMode from esphome.components.remote_base import CONF_TRANSMITTER_ID import esphome.config_validation as cv @@ -280,6 +280,7 @@ async def to_code(config): cg.add(var.set_response_timeout(config[CONF_TIMEOUT].total_milliseconds)) cg.add(var.set_request_attempts(config[CONF_NUM_ATTEMPTS])) if CONF_TRANSMITTER_ID in config: + remote_base.request_protocol("midea") # ir_transmitter.h uses it from C++ cg.add_define("USE_REMOTE_TRANSMITTER") transmitter_ = await cg.get_variable(config[CONF_TRANSMITTER_ID]) cg.add(var.set_transmitter(transmitter_)) diff --git a/esphome/components/midea_ir/climate.py b/esphome/components/midea_ir/climate.py index 84bfeab0d4..e1b2b56ada 100644 --- a/esphome/components/midea_ir/climate.py +++ b/esphome/components/midea_ir/climate.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import climate_ir +from esphome.components import climate_ir, remote_base import esphome.config_validation as cv from esphome.const import CONF_USE_FAHRENHEIT from esphome.types import ConfigType @@ -19,5 +19,9 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(MideaIR).extend( async def to_code(config: ConfigType) -> None: + # midea_ir uses MideaProtocol from C++ and auto-loads coolix, whose coolix.cpp uses + # CoolixProtocol even when no coolix climate is configured + remote_base.request_protocol("midea") + remote_base.request_protocol("coolix") var = await climate_ir.new_climate_ir(config) cg.add(var.set_fahrenheit(config[CONF_USE_FAHRENHEIT])) diff --git a/esphome/components/remote_base/__init__.py b/esphome/components/remote_base/__init__.py index 19b8549f75..27b6eb9fc8 100644 --- a/esphome/components/remote_base/__init__.py +++ b/esphome/components/remote_base/__init__.py @@ -1,6 +1,11 @@ +from collections.abc import Callable +from pathlib import Path +from typing import Any + from esphome import automation import esphome.codegen as cg from esphome.components import binary_sensor +from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv from esphome.const import ( CONF_ADDRESS, @@ -40,11 +45,14 @@ from esphome.const import ( CONF_ZERO, ) from esphome.core import ID, coroutine +from esphome.cpp_generator import MockObj from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor +from esphome.types import ConfigType from esphome.util import Registry, SimpleRegistry AUTO_LOAD = ["binary_sensor"] + CONF_RECEIVER_ID = "receiver_id" CONF_TRANSMITTER_ID = "transmitter_id" CONF_FIRST = "first" @@ -90,9 +98,42 @@ REMOTE_TRANSMITTABLE_SCHEMA = cv.Schema( ) -async def register_listener(var, config): +# Listener and dumper lists are StaticVectors sized from these counts, so every registration +# must go through add_listener / add_dumper. Every receiver's list gets the same capacity, so +# the slots are keyed by receiver and the define is the largest count any one receiver needs. +LISTENER_COUNT_DEFINE = "REMOTE_BASE_LISTENER_COUNT" +DUMPER_COUNT_DEFINE = "REMOTE_BASE_DUMPER_COUNT" + + +_request_listener_slot = cg.slot_counter(LISTENER_COUNT_DEFINE) +_request_dumper_slot = cg.slot_counter(DUMPER_COUNT_DEFINE) + + +def add_listener(receiver: MockObj, listener: MockObj) -> None: + _request_listener_slot(str(receiver)) + cg.add(receiver.register_listener(listener)) + + +def add_dumper(receiver: MockObj, dumper: MockObj) -> None: + _request_dumper_slot(str(receiver)) + cg.add(receiver.register_dumper(dumper)) + + +async def register_listener(var: MockObj, config: ConfigType) -> None: receiver = await cg.get_variable(config[CONF_RECEIVER_ID]) - cg.add(receiver.register_listener(var)) + add_listener(receiver, var) + + +async def attach_receiver( + var: MockObj, config: ConfigType, key: str = CONF_RECEIVER_ID +) -> None: + """Link the configured receiver to an entity and register the entity as its listener. + + The C++ set_receiver() no longer registers the listener; the slot for it is counted here. + """ + receiver = await cg.get_variable(config[key]) + cg.add(var.set_receiver(receiver)) + add_listener(receiver, var) async def register_transmittable(var, config): @@ -100,8 +141,53 @@ async def register_transmittable(var, config): cg.add(var.set_transmitter(transmitter_)) -def register_binary_sensor(name, type, schema): - return BINARY_SENSOR_REGISTRY.register(name, type, schema) +# Registry names that share a protocol source file +def _protocol_stem(name: str) -> str: + if name.startswith("rc_switch"): + return "rc_switch" + if name == "canalsatld": + return "canalsat" + return name + + +def protocol_define(name: str) -> str: + return f"USE_REMOTE_PROTOCOL_{_protocol_stem(name).upper()}" + + +_PROTOCOL_STEMS = sorted( + path.name.removesuffix("_protocol.cpp") + for path in Path(__file__).parent.glob("*_protocol.cpp") +) + + +def request_protocol(name: str) -> None: + """Keep a protocol's source file in the build; components using it from C++ must call this.""" + if _protocol_stem(name) not in _PROTOCOL_STEMS: + raise ValueError( + f"Unknown remote protocol {name!r}; expected one of {', '.join(_PROTOCOL_STEMS)}" + ) + cg.add_define(protocol_define(name)) + + +# Only the protocol sources a configuration uses are compiled +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {f"{stem}_protocol.cpp": protocol_define(stem) for stem in _PROTOCOL_STEMS} +) + + +def register_binary_sensor( + name: str, type: MockObj, schema: cv.Schema | dict +) -> Callable[[Callable[[MockObj, ConfigType], Any]], Callable]: + registerer = BINARY_SENSOR_REGISTRY.register(name, type, schema) + + def decorator(func: Callable[[MockObj, ConfigType], Any]) -> Callable: + async def new_func(var: MockObj, config: ConfigType) -> None: + request_protocol(name) + await coroutine(func)(var, config) + + return registerer(new_func) + + return decorator def register_trigger(name, type, data_type): @@ -114,6 +200,7 @@ def register_trigger(name, type, data_type): def decorator(func): async def new_func(config): + request_protocol(name) var = cg.new_Pvariable(config[CONF_TRIGGER_ID]) await coroutine(func)(var, config) await automation.build_automation(var, [(data_type, "x")], config) @@ -131,6 +218,7 @@ def register_dumper(name, type, schema=None): def decorator(func): async def new_func(config, dumper_id): + request_protocol(name) var = cg.new_Pvariable(dumper_id) await coroutine(func)(var, config) return var @@ -171,6 +259,7 @@ def register_action(name, type_, schema): def decorator(func): async def new_func(config, action_id, template_arg, args): + request_protocol(name) var = cg.new_Pvariable(action_id, template_arg) await register_transmittable(var, config) if CONF_REPEAT in config: @@ -213,7 +302,13 @@ DUMPER_REGISTRY = Registry() def validate_dumpers(value): if isinstance(value, str) and value.lower() == "all": return validate_dumpers(list(DUMPER_REGISTRY.keys())) - return cv.validate_registry("dumper", DUMPER_REGISTRY)(value) + entries = cv.validate_registry("dumper", DUMPER_REGISTRY)(value) + # a dumper listed twice would register twice; the receiver holds one secondary dumper + return list( + { + next(k for k in entry if k in DUMPER_REGISTRY): entry for entry in entries + }.values() + ) def validate_triggers(base_schema): @@ -1439,7 +1534,7 @@ def validate_rc_switch_raw_code(value): def build_rc_switch_protocol(config): if isinstance(config, int): - return rc_switch_protocols[config] + return rc_switch_protocol(config) pl = config[CONF_PULSE_LENGTH] return RCSwitchBase( config[CONF_SYNC][0] * pl, @@ -1526,7 +1621,7 @@ RC_SWITCH_TRANSMITTER = cv.Schema( } ) -rc_switch_protocols = ns.RC_SWITCH_PROTOCOLS +rc_switch_protocol = ns.rc_switch_protocol RCSwitchData = ns.struct("RCSwitchData") RCSwitchBase = ns.class_("RCSwitchBase") RCSwitchTrigger = ns.class_("RCSwitchTrigger", RemoteReceiverTrigger) diff --git a/esphome/components/remote_base/abbwelcome_protocol.h b/esphome/components/remote_base/abbwelcome_protocol.h index 7ff32923be..a309c124ee 100644 --- a/esphome/components/remote_base/abbwelcome_protocol.h +++ b/esphome/components/remote_base/abbwelcome_protocol.h @@ -191,9 +191,9 @@ class ABBWelcomeData { class ABBWelcomeProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const ABBWelcomeData &src) override; - optional decode(RemoteReceiveData src) override; - void dump(const ABBWelcomeData &data) override; + void encode(RemoteTransmitData *dst, const ABBWelcomeData &src); + optional decode(RemoteReceiveData src); + void dump(const ABBWelcomeData &data); protected: void encode_byte_(RemoteTransmitData *dst, uint8_t data) const; diff --git a/esphome/components/remote_base/aeha_protocol.h b/esphome/components/remote_base/aeha_protocol.h index 3f4e98bd43..98a5501155 100644 --- a/esphome/components/remote_base/aeha_protocol.h +++ b/esphome/components/remote_base/aeha_protocol.h @@ -15,9 +15,9 @@ struct AEHAData { class AEHAProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const AEHAData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const AEHAData &data) override; + void encode(RemoteTransmitData *dst, const AEHAData &data); + optional decode(RemoteReceiveData src); + void dump(const AEHAData &data); private: std::string format_data_(const std::vector &data); diff --git a/esphome/components/remote_base/beo4_protocol.h b/esphome/components/remote_base/beo4_protocol.h index 30b99dbeb7..ed9d6aa671 100644 --- a/esphome/components/remote_base/beo4_protocol.h +++ b/esphome/components/remote_base/beo4_protocol.h @@ -16,9 +16,9 @@ struct Beo4Data { class Beo4Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const Beo4Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const Beo4Data &data) override; + void encode(RemoteTransmitData *dst, const Beo4Data &data); + optional decode(RemoteReceiveData src); + void dump(const Beo4Data &data); }; DECLARE_REMOTE_PROTOCOL(Beo4) diff --git a/esphome/components/remote_base/brennenstuhl_protocol.h b/esphome/components/remote_base/brennenstuhl_protocol.h index 1d5b621714..bfea463b7d 100644 --- a/esphome/components/remote_base/brennenstuhl_protocol.h +++ b/esphome/components/remote_base/brennenstuhl_protocol.h @@ -13,9 +13,9 @@ struct BrennenstuhlData { class BrennenstuhlProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const BrennenstuhlData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const BrennenstuhlData &data) override; + void encode(RemoteTransmitData *dst, const BrennenstuhlData &data); + optional decode(RemoteReceiveData src); + void dump(const BrennenstuhlData &data); }; DECLARE_REMOTE_PROTOCOL(Brennenstuhl) diff --git a/esphome/components/remote_base/byronsx_protocol.h b/esphome/components/remote_base/byronsx_protocol.h index 674fa99ea1..c71390c267 100644 --- a/esphome/components/remote_base/byronsx_protocol.h +++ b/esphome/components/remote_base/byronsx_protocol.h @@ -21,9 +21,9 @@ struct ByronSXData { class ByronSXProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const ByronSXData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const ByronSXData &data) override; + void encode(RemoteTransmitData *dst, const ByronSXData &data); + optional decode(RemoteReceiveData src); + void dump(const ByronSXData &data); }; DECLARE_REMOTE_PROTOCOL(ByronSX) diff --git a/esphome/components/remote_base/canalsat_protocol.h b/esphome/components/remote_base/canalsat_protocol.h index 5ba9115ea8..09bead18b3 100644 --- a/esphome/components/remote_base/canalsat_protocol.h +++ b/esphome/components/remote_base/canalsat_protocol.h @@ -19,9 +19,9 @@ struct CanalSatLDData : public CanalSatData {}; class CanalSatBaseProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const CanalSatData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const CanalSatData &data) override; + void encode(RemoteTransmitData *dst, const CanalSatData &data); + optional decode(RemoteReceiveData src); + void dump(const CanalSatData &data); protected: uint16_t frequency_; diff --git a/esphome/components/remote_base/coolix_protocol.h b/esphome/components/remote_base/coolix_protocol.h index d9441e8417..29a306ce29 100644 --- a/esphome/components/remote_base/coolix_protocol.h +++ b/esphome/components/remote_base/coolix_protocol.h @@ -21,9 +21,9 @@ struct CoolixData { class CoolixProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const CoolixData &data) override; - optional decode(RemoteReceiveData data) override; - void dump(const CoolixData &data) override; + void encode(RemoteTransmitData *dst, const CoolixData &data); + optional decode(RemoteReceiveData data); + void dump(const CoolixData &data); }; DECLARE_REMOTE_PROTOCOL(Coolix) diff --git a/esphome/components/remote_base/dish_protocol.h b/esphome/components/remote_base/dish_protocol.h index c89f4e78e1..f319b55f43 100644 --- a/esphome/components/remote_base/dish_protocol.h +++ b/esphome/components/remote_base/dish_protocol.h @@ -13,9 +13,9 @@ struct DishData { class DishProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DishData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DishData &data) override; + void encode(RemoteTransmitData *dst, const DishData &data); + optional decode(RemoteReceiveData src); + void dump(const DishData &data); }; DECLARE_REMOTE_PROTOCOL(Dish) diff --git a/esphome/components/remote_base/dooya_protocol.h b/esphome/components/remote_base/dooya_protocol.h index 148c7c17bc..954c3cf1d3 100644 --- a/esphome/components/remote_base/dooya_protocol.h +++ b/esphome/components/remote_base/dooya_protocol.h @@ -20,9 +20,9 @@ struct DooyaData { class DooyaProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DooyaData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DooyaData &data) override; + void encode(RemoteTransmitData *dst, const DooyaData &data); + optional decode(RemoteReceiveData src); + void dump(const DooyaData &data); }; DECLARE_REMOTE_PROTOCOL(Dooya) diff --git a/esphome/components/remote_base/drayton_protocol.h b/esphome/components/remote_base/drayton_protocol.h index 693a1bbe85..4e879f0f75 100644 --- a/esphome/components/remote_base/drayton_protocol.h +++ b/esphome/components/remote_base/drayton_protocol.h @@ -19,9 +19,9 @@ struct DraytonData { class DraytonProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DraytonData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DraytonData &data) override; + void encode(RemoteTransmitData *dst, const DraytonData &data); + optional decode(RemoteReceiveData src); + void dump(const DraytonData &data); }; DECLARE_REMOTE_PROTOCOL(Drayton) diff --git a/esphome/components/remote_base/dyson_protocol.h b/esphome/components/remote_base/dyson_protocol.h index 3473a489b2..663e50fb4b 100644 --- a/esphome/components/remote_base/dyson_protocol.h +++ b/esphome/components/remote_base/dyson_protocol.h @@ -21,9 +21,9 @@ struct DysonData { class DysonProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DysonData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DysonData &data) override; + void encode(RemoteTransmitData *dst, const DysonData &data); + optional decode(RemoteReceiveData src); + void dump(const DysonData &data); }; DECLARE_REMOTE_PROTOCOL(Dyson) diff --git a/esphome/components/remote_base/gobox_protocol.h b/esphome/components/remote_base/gobox_protocol.h index f6b278771e..0c8797af70 100644 --- a/esphome/components/remote_base/gobox_protocol.h +++ b/esphome/components/remote_base/gobox_protocol.h @@ -31,9 +31,9 @@ class GoboxProtocol : public RemoteProtocol { void dump_timings_(const RawTimings &timings) const; public: - void encode(RemoteTransmitData *dst, const GoboxData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const GoboxData &data) override; + void encode(RemoteTransmitData *dst, const GoboxData &data); + optional decode(RemoteReceiveData src); + void dump(const GoboxData &data); }; DECLARE_REMOTE_PROTOCOL(Gobox) diff --git a/esphome/components/remote_base/haier_protocol.h b/esphome/components/remote_base/haier_protocol.h index 9c45ba1a63..e1fd60411f 100644 --- a/esphome/components/remote_base/haier_protocol.h +++ b/esphome/components/remote_base/haier_protocol.h @@ -13,9 +13,9 @@ struct HaierData { class HaierProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const HaierData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const HaierData &data) override; + void encode(RemoteTransmitData *dst, const HaierData &data); + optional decode(RemoteReceiveData src); + void dump(const HaierData &data); protected: void encode_byte_(RemoteTransmitData *dst, uint8_t item); diff --git a/esphome/components/remote_base/jvc_protocol.h b/esphome/components/remote_base/jvc_protocol.h index f6e2548dea..5911664fc3 100644 --- a/esphome/components/remote_base/jvc_protocol.h +++ b/esphome/components/remote_base/jvc_protocol.h @@ -14,9 +14,9 @@ struct JVCData { class JVCProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const JVCData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const JVCData &data) override; + void encode(RemoteTransmitData *dst, const JVCData &data); + optional decode(RemoteReceiveData src); + void dump(const JVCData &data); }; DECLARE_REMOTE_PROTOCOL(JVC) diff --git a/esphome/components/remote_base/keeloq_protocol.h b/esphome/components/remote_base/keeloq_protocol.h index 432313b87b..335fbd164b 100644 --- a/esphome/components/remote_base/keeloq_protocol.h +++ b/esphome/components/remote_base/keeloq_protocol.h @@ -24,9 +24,9 @@ struct KeeloqData { class KeeloqProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const KeeloqData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const KeeloqData &data) override; + void encode(RemoteTransmitData *dst, const KeeloqData &data); + optional decode(RemoteReceiveData src); + void dump(const KeeloqData &data); }; DECLARE_REMOTE_PROTOCOL(Keeloq) diff --git a/esphome/components/remote_base/lg_protocol.h b/esphome/components/remote_base/lg_protocol.h index 9715974995..91dfbadb0c 100644 --- a/esphome/components/remote_base/lg_protocol.h +++ b/esphome/components/remote_base/lg_protocol.h @@ -16,9 +16,9 @@ struct LGData { class LGProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const LGData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const LGData &data) override; + void encode(RemoteTransmitData *dst, const LGData &data); + optional decode(RemoteReceiveData src); + void dump(const LGData &data); }; DECLARE_REMOTE_PROTOCOL(LG) diff --git a/esphome/components/remote_base/magiquest_protocol.h b/esphome/components/remote_base/magiquest_protocol.h index 18662ec759..f0d2410fe2 100644 --- a/esphome/components/remote_base/magiquest_protocol.h +++ b/esphome/components/remote_base/magiquest_protocol.h @@ -27,9 +27,9 @@ struct MagiQuestData { class MagiQuestProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const MagiQuestData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const MagiQuestData &data) override; + void encode(RemoteTransmitData *dst, const MagiQuestData &data); + optional decode(RemoteReceiveData src); + void dump(const MagiQuestData &data); }; DECLARE_REMOTE_PROTOCOL(MagiQuest) diff --git a/esphome/components/remote_base/midea_protocol.h b/esphome/components/remote_base/midea_protocol.h index 47bad6826f..85bbef1cb1 100644 --- a/esphome/components/remote_base/midea_protocol.h +++ b/esphome/components/remote_base/midea_protocol.h @@ -67,9 +67,9 @@ class MideaData { class MideaProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const MideaData &src) override; - optional decode(RemoteReceiveData src) override; - void dump(const MideaData &data) override; + void encode(RemoteTransmitData *dst, const MideaData &src); + optional decode(RemoteReceiveData src); + void dump(const MideaData &data); }; DECLARE_REMOTE_PROTOCOL(Midea) diff --git a/esphome/components/remote_base/mirage_protocol.h b/esphome/components/remote_base/mirage_protocol.h index c967e72f13..a37fb93f4f 100644 --- a/esphome/components/remote_base/mirage_protocol.h +++ b/esphome/components/remote_base/mirage_protocol.h @@ -13,9 +13,9 @@ struct MirageData { class MirageProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const MirageData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const MirageData &data) override; + void encode(RemoteTransmitData *dst, const MirageData &data); + optional decode(RemoteReceiveData src); + void dump(const MirageData &data); protected: void encode_byte_(RemoteTransmitData *dst, uint8_t item); diff --git a/esphome/components/remote_base/nec_protocol.h b/esphome/components/remote_base/nec_protocol.h index 7b310e8ba5..1337f7a8b3 100644 --- a/esphome/components/remote_base/nec_protocol.h +++ b/esphome/components/remote_base/nec_protocol.h @@ -14,9 +14,9 @@ struct NECData { class NECProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const NECData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const NECData &data) override; + void encode(RemoteTransmitData *dst, const NECData &data); + optional decode(RemoteReceiveData src); + void dump(const NECData &data); }; DECLARE_REMOTE_PROTOCOL(NEC) diff --git a/esphome/components/remote_base/nexa_protocol.h b/esphome/components/remote_base/nexa_protocol.h index ebcd2a2c11..ebf85387b0 100644 --- a/esphome/components/remote_base/nexa_protocol.h +++ b/esphome/components/remote_base/nexa_protocol.h @@ -24,9 +24,9 @@ class NexaProtocol : public RemoteProtocol { void zero(RemoteTransmitData *dst) const; void sync(RemoteTransmitData *dst) const; - void encode(RemoteTransmitData *dst, const NexaData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const NexaData &data) override; + void encode(RemoteTransmitData *dst, const NexaData &data); + optional decode(RemoteReceiveData src); + void dump(const NexaData &data); }; DECLARE_REMOTE_PROTOCOL(Nexa) diff --git a/esphome/components/remote_base/panasonic_protocol.h b/esphome/components/remote_base/panasonic_protocol.h index d13c0f2798..84df3c08b7 100644 --- a/esphome/components/remote_base/panasonic_protocol.h +++ b/esphome/components/remote_base/panasonic_protocol.h @@ -16,9 +16,9 @@ struct PanasonicData { class PanasonicProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const PanasonicData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const PanasonicData &data) override; + void encode(RemoteTransmitData *dst, const PanasonicData &data); + optional decode(RemoteReceiveData src); + void dump(const PanasonicData &data); }; DECLARE_REMOTE_PROTOCOL(Panasonic) diff --git a/esphome/components/remote_base/pioneer_protocol.h b/esphome/components/remote_base/pioneer_protocol.h index 514ab67501..d02bd3451f 100644 --- a/esphome/components/remote_base/pioneer_protocol.h +++ b/esphome/components/remote_base/pioneer_protocol.h @@ -13,9 +13,9 @@ struct PioneerData { class PioneerProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const PioneerData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const PioneerData &data) override; + void encode(RemoteTransmitData *dst, const PioneerData &data); + optional decode(RemoteReceiveData src); + void dump(const PioneerData &data); }; DECLARE_REMOTE_PROTOCOL(Pioneer) diff --git a/esphome/components/remote_base/pronto_protocol.h b/esphome/components/remote_base/pronto_protocol.h index f4f6b2144d..bfd04c5cd9 100644 --- a/esphome/components/remote_base/pronto_protocol.h +++ b/esphome/components/remote_base/pronto_protocol.h @@ -30,9 +30,9 @@ class ProntoProtocol : public RemoteProtocol { std::string compensate_and_dump_sequence_(const RawTimings &data, uint16_t timebase); public: - void encode(RemoteTransmitData *dst, const ProntoData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const ProntoData &data) override; + void encode(RemoteTransmitData *dst, const ProntoData &data); + optional decode(RemoteReceiveData src); + void dump(const ProntoData &data); }; DECLARE_REMOTE_PROTOCOL(Pronto) diff --git a/esphome/components/remote_base/rc5_protocol.h b/esphome/components/remote_base/rc5_protocol.h index dbb89e41c6..f6f0f33c6e 100644 --- a/esphome/components/remote_base/rc5_protocol.h +++ b/esphome/components/remote_base/rc5_protocol.h @@ -14,9 +14,9 @@ struct RC5Data { class RC5Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const RC5Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const RC5Data &data) override; + void encode(RemoteTransmitData *dst, const RC5Data &data); + optional decode(RemoteReceiveData src); + void dump(const RC5Data &data); }; DECLARE_REMOTE_PROTOCOL(RC5) diff --git a/esphome/components/remote_base/rc6_protocol.h b/esphome/components/remote_base/rc6_protocol.h index fda9d98ecb..c4a2e8529b 100644 --- a/esphome/components/remote_base/rc6_protocol.h +++ b/esphome/components/remote_base/rc6_protocol.h @@ -15,9 +15,9 @@ struct RC6Data { class RC6Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const RC6Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const RC6Data &data) override; + void encode(RemoteTransmitData *dst, const RC6Data &data); + optional decode(RemoteReceiveData src); + void dump(const RC6Data &data); }; DECLARE_REMOTE_PROTOCOL(RC6) diff --git a/esphome/components/remote_base/rc_switch_protocol.cpp b/esphome/components/remote_base/rc_switch_protocol.cpp index 612558ca1c..de16c55cb0 100644 --- a/esphome/components/remote_base/rc_switch_protocol.cpp +++ b/esphome/components/remote_base/rc_switch_protocol.cpp @@ -1,29 +1,21 @@ #include "rc_switch_protocol.h" + +#include +#include "esphome/core/hal.h" #include "esphome/core/log.h" namespace esphome::remote_base { static const char *const TAG = "remote.rc_switch"; -const RCSwitchBase RC_SWITCH_PROTOCOLS[9] = {RCSwitchBase(0, 0, 0, 0, 0, 0, false), - RCSwitchBase(350, 10850, 350, 1050, 1050, 350, false), - RCSwitchBase(650, 6500, 650, 1300, 1300, 650, false), - RCSwitchBase(3000, 7100, 400, 1100, 900, 600, false), - RCSwitchBase(380, 2280, 380, 1140, 1140, 380, false), - RCSwitchBase(3000, 7000, 500, 1000, 1000, 500, false), - RCSwitchBase(10350, 450, 450, 900, 900, 450, true), - RCSwitchBase(300, 9300, 150, 900, 900, 150, false), - RCSwitchBase(250, 2500, 250, 1250, 250, 250, false)}; - -RCSwitchBase::RCSwitchBase(uint32_t sync_high, uint32_t sync_low, uint32_t zero_high, uint32_t zero_low, - uint32_t one_high, uint32_t one_low, bool inverted) - : sync_high_(sync_high), - sync_low_(sync_low), - zero_high_(zero_high), - zero_low_(zero_low), - one_high_(one_high), - one_low_(one_low), - inverted_(inverted) {} +RCSwitchBase rc_switch_protocol(uint8_t index) { + RCSwitchBase protocol; + // entry 0 is the all-zero protocol, so an out of range index from a lambda transmits nothing + if (index >= std::size(RC_SWITCH_PROTOCOLS)) + index = 0; + progmem_memcpy(&protocol, &RC_SWITCH_PROTOCOLS[index], sizeof(protocol)); + return protocol; +} void RCSwitchBase::one(RemoteTransmitData *dst) const { if (!this->inverted_) { @@ -133,11 +125,11 @@ bool RCSwitchBase::decode(RemoteReceiveData &src, uint64_t *out_data, uint8_t *o optional RCSwitchBase::decode(RemoteReceiveData &src) const { RCSwitchData out; uint8_t out_nbits; - for (uint8_t i = 1; i <= 8; i++) { + for (size_t i = 1; i < std::size(RC_SWITCH_PROTOCOLS); i++) { src.reset(); const RCSwitchBase *protocol = &RC_SWITCH_PROTOCOLS[i]; if (protocol->decode(src, &out.code, &out_nbits) && out_nbits >= 3) { - out.protocol = i; + out.protocol = static_cast(i); return out; } } @@ -246,7 +238,7 @@ bool RCSwitchRawReceiver::matches(RemoteReceiveData src) { return decoded_nbits == this->nbits_ && (decoded_code & this->mask_) == (this->code_ & this->mask_); } bool RCSwitchDumper::dump(RemoteReceiveData src) { - for (uint8_t i = 1; i <= 8; i++) { + for (size_t i = 1; i < std::size(RC_SWITCH_PROTOCOLS); i++) { src.reset(); uint64_t out_data; uint8_t out_nbits; @@ -257,7 +249,7 @@ bool RCSwitchDumper::dump(RemoteReceiveData src) { buffer[j] = (out_data & ((uint64_t) 1 << (out_nbits - j - 1))) ? '1' : '0'; buffer[out_nbits] = '\0'; - ESP_LOGI(TAG, "Received RCSwitch Raw: protocol=%u data='%s'", i, buffer); + ESP_LOGI(TAG, "Received RCSwitch Raw: protocol=%u data='%s'", static_cast(i), buffer); // only send first decoded protocol return true; diff --git a/esphome/components/remote_base/rc_switch_protocol.h b/esphome/components/remote_base/rc_switch_protocol.h index 3224c04fb2..9ccea4d15a 100644 --- a/esphome/components/remote_base/rc_switch_protocol.h +++ b/esphome/components/remote_base/rc_switch_protocol.h @@ -16,9 +16,16 @@ class RCSwitchBase { public: using ProtocolData = RCSwitchData; - RCSwitchBase() = default; - RCSwitchBase(uint32_t sync_high, uint32_t sync_low, uint32_t zero_high, uint32_t zero_low, uint32_t one_high, - uint32_t one_low, bool inverted); + constexpr RCSwitchBase() = default; + constexpr RCSwitchBase(uint32_t sync_high, uint32_t sync_low, uint32_t zero_high, uint32_t zero_low, + uint32_t one_high, uint32_t one_low, bool inverted) + : sync_high_(sync_high), + sync_low_(sync_low), + zero_high_(zero_high), + zero_low_(zero_low), + one_high_(one_high), + one_low_(one_low), + inverted_(inverted) {} void one(RemoteTransmitData *dst) const; @@ -58,10 +65,28 @@ class RCSwitchBase { uint32_t zero_low_{}; uint32_t one_high_{}; uint32_t one_low_{}; - bool inverted_{}; + uint32_t inverted_{}; // bool widened so every field is a word: the table is read from flash }; -extern const RCSwitchBase RC_SWITCH_PROTOCOLS[9]; +// Constant-initialized and kept in flash on every platform. The decoder reads entries in place +// through a pointer, which ESP8266 only allows while every field is a whole word; copies out of +// the table go through rc_switch_protocol() +static_assert(sizeof(RCSwitchBase) == 7 * sizeof(uint32_t), "RCSwitchBase must stay word-only for flash reads"); +inline constexpr RCSwitchBase RC_SWITCH_PROTOCOLS[] PROGMEM = { + {0, 0, 0, 0, 0, 0, false}, + {350, 10850, 350, 1050, 1050, 350, false}, + {650, 6500, 650, 1300, 1300, 650, false}, + {3000, 7100, 400, 1100, 900, 600, false}, + {380, 2280, 380, 1140, 1140, 380, false}, + {3000, 7000, 500, 1000, 1000, 500, false}, + {10350, 450, 450, 900, 900, 450, true}, + {300, 9300, 150, 900, 900, 150, false}, + {250, 2500, 250, 1250, 250, 250, false}, +}; + +/// RAM copy of RC_SWITCH_PROTOCOLS[index] (0 when out of range) for the transmit actions and the dumper, made with +/// progmem_memcpy so no byte load ever touches the flash table on ESP8266 +RCSwitchBase rc_switch_protocol(uint8_t index); uint64_t decode_binary_string(const std::string &data); diff --git a/esphome/components/remote_base/remote_base.cpp b/esphome/components/remote_base/remote_base.cpp index 4d9bc55f21..5d1bba16b6 100644 --- a/esphome/components/remote_base/remote_base.cpp +++ b/esphome/components/remote_base/remote_base.cpp @@ -99,29 +99,48 @@ bool RemoteReceiverBinarySensorBase::on_receive(RemoteReceiveData src) { /* RemoteReceiverBase */ +// Slots are counted at code generation; a registration from C++ setup() has none +#ifdef REMOTE_BASE_LISTENER_COUNT +void RemoteReceiverBase::register_listener(RemoteReceiverListener *listener) { + if (this->listeners_.size() == REMOTE_BASE_LISTENER_COUNT) { + ESP_LOGE(TAG, "No %s slot: register it from to_code() with remote_base.add_%s", LOG_STR_LITERAL("listener"), + LOG_STR_LITERAL("listener")); + return; + } + this->listeners_.push_back(listener); +} +#endif + +#ifdef REMOTE_BASE_DUMPER_COUNT void RemoteReceiverBase::register_dumper(RemoteReceiverDumperBase *dumper) { if (dumper->is_secondary()) { - this->secondary_dumpers_.push_back(dumper); - } else { + if (this->secondary_dumper_ == nullptr) { + this->secondary_dumper_ = dumper; + return; + } + } else if (this->dumpers_.size() != REMOTE_BASE_DUMPER_COUNT) { this->dumpers_.push_back(dumper); + return; } + ESP_LOGE(TAG, "No %s slot: register it from to_code() with remote_base.add_%s", LOG_STR_LITERAL("dumper"), + LOG_STR_LITERAL("dumper")); } +#endif -void RemoteReceiverBase::call_listeners_() { +void RemoteReceiverBase::call_listeners_dumpers_() { +#ifdef REMOTE_BASE_LISTENER_COUNT for (auto *listener : this->listeners_) listener->on_receive(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_)); -} - -void RemoteReceiverBase::call_dumpers_() { +#endif +#ifdef REMOTE_BASE_DUMPER_COUNT bool success = false; for (auto *dumper : this->dumpers_) { if (dumper->dump(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_))) success = true; } - if (!success) { - for (auto *dumper : this->secondary_dumpers_) - dumper->dump(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_)); - } + if (!success && this->secondary_dumper_ != nullptr) + this->secondary_dumper_->dump(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_)); +#endif } void RemoteReceiverBinarySensorBase::dump_config() { LOG_BINARY_SENSOR("", "Remote Receiver Binary Sensor", this); } diff --git a/esphome/components/remote_base/remote_base.h b/esphome/components/remote_base/remote_base.h index 4e2ed4b71c..67e5799bca 100644 --- a/esphome/components/remote_base/remote_base.h +++ b/esphome/components/remote_base/remote_base.h @@ -1,12 +1,14 @@ +#pragma once + +#include #include #include -#pragma once - #include "esphome/components/binary_sensor/binary_sensor.h" #include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/hal.h" +#include "esphome/core/helpers.h" namespace esphome::remote_base { @@ -141,6 +143,22 @@ class RemoteRMTChannel { #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 +// Protocol shapes, checked where a protocol is used so a missing method fails at the use site +// instead of deep inside a template body. Receive-only protocols such as RCSwitchBase decode +// without encoding. +template +concept RemoteProtocolDecoder = requires(T proto, RemoteReceiveData src) { + { proto.decode(src) } -> std::same_as>; +}; +template +concept RemoteProtocolDumper = RemoteProtocolDecoder && requires(T proto, const typename T::ProtocolData &data) { + proto.dump(data); +}; +template +concept RemoteProtocolEncoder = requires(T proto, RemoteTransmitData *dst, const typename T::ProtocolData &data) { + proto.encode(dst, data); +}; + class RemoteTransmitterBase : public RemoteComponentBase { public: RemoteTransmitterBase(InternalGPIOPin *pin) : RemoteComponentBase(pin) {} @@ -162,8 +180,8 @@ class RemoteTransmitterBase : public RemoteComponentBase { this->temp_.reset(); return TransmitCall(this); } - template - void transmit(const Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { + template + void transmit(const typename Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { auto call = this->transmit(); Protocol().encode(call.get_data(), data); call.set_send_times(send_times); @@ -194,24 +212,37 @@ class RemoteReceiverDumperBase { class RemoteReceiverBase : public RemoteComponentBase { public: RemoteReceiverBase(InternalGPIOPin *pin) : RemoteComponentBase(pin) {} - void register_listener(RemoteReceiverListener *listener) { this->listeners_.push_back(listener); } + // Slots are counted at code generation; without one the call fails at compile time with the same message + // the runtime check logs +#ifdef REMOTE_BASE_LISTENER_COUNT + void register_listener(RemoteReceiverListener *listener); +#else + template void register_listener(T *) { + static_assert(sizeof(T) == 0, "No listener slot: register it from to_code() with remote_base.add_listener"); + } +#endif +#ifdef REMOTE_BASE_DUMPER_COUNT void register_dumper(RemoteReceiverDumperBase *dumper); +#else + template void register_dumper(T *) { + static_assert(sizeof(T) == 0, "No dumper slot: register it from to_code() with remote_base.add_dumper"); + } +#endif void set_tolerance(uint32_t tolerance, ToleranceMode tolerance_mode) { this->tolerance_ = tolerance; this->tolerance_mode_ = tolerance_mode; } protected: - void call_listeners_(); - void call_dumpers_(); - void call_listeners_dumpers_() { - this->call_listeners_(); - this->call_dumpers_(); - } + void call_listeners_dumpers_(); - std::vector listeners_; - std::vector dumpers_; - std::vector secondary_dumpers_; +#ifdef REMOTE_BASE_LISTENER_COUNT + StaticVector listeners_; +#endif +#ifdef REMOTE_BASE_DUMPER_COUNT + StaticVector dumpers_; + RemoteReceiverDumperBase *secondary_dumper_{nullptr}; // runs only when no primary dumper matched +#endif RawTimings temp_; uint32_t tolerance_{25}; ToleranceMode tolerance_mode_{TOLERANCE_MODE_PERCENTAGE}; @@ -229,15 +260,14 @@ class RemoteReceiverBinarySensorBase : public binary_sensor::BinarySensorInitial /* TEMPLATES */ +// Protocols are used only through their concrete type (see the RemoteProtocol* concepts); encode/decode/dump +// stay non-virtual so unused ones link out template class RemoteProtocol { public: using ProtocolData = T; - virtual void encode(RemoteTransmitData *dst, const ProtocolData &data) = 0; - virtual optional decode(RemoteReceiveData src) = 0; - virtual void dump(const ProtocolData &data) = 0; }; -template class RemoteReceiverBinarySensor : public RemoteReceiverBinarySensorBase { +template class RemoteReceiverBinarySensor : public RemoteReceiverBinarySensorBase { public: RemoteReceiverBinarySensor() : RemoteReceiverBinarySensorBase() {} @@ -255,7 +285,7 @@ template class RemoteReceiverBinarySensor : public RemoteReceiverBin T::ProtocolData data_; }; -template +template class RemoteReceiverTrigger final : public Trigger, public RemoteReceiverListener { protected: bool on_receive(RemoteReceiveData src) override { @@ -276,8 +306,8 @@ class RemoteTransmittable { void set_transmitter(RemoteTransmitterBase *transmitter) { this->transmitter_ = transmitter; } protected: - template - void transmit_(const Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { + template + void transmit_(const typename Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { this->transmitter_->transmit(data, send_times, send_wait); } RemoteTransmitterBase *transmitter_; @@ -298,7 +328,7 @@ template class RemoteTransmitterActionBase : public RemoteTransm virtual void encode(RemoteTransmitData *dst, Ts... x) = 0; }; -template class RemoteReceiverDumper : public RemoteReceiverDumperBase { +template class RemoteReceiverDumper : public RemoteReceiverDumperBase { public: bool dump(RemoteReceiveData src) override { auto proto = T(); diff --git a/esphome/components/remote_base/roomba_protocol.h b/esphome/components/remote_base/roomba_protocol.h index 3582dac398..8db025f812 100644 --- a/esphome/components/remote_base/roomba_protocol.h +++ b/esphome/components/remote_base/roomba_protocol.h @@ -12,9 +12,9 @@ struct RoombaData { class RoombaProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const RoombaData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const RoombaData &data) override; + void encode(RemoteTransmitData *dst, const RoombaData &data); + optional decode(RemoteReceiveData src); + void dump(const RoombaData &data); }; DECLARE_REMOTE_PROTOCOL(Roomba) diff --git a/esphome/components/remote_base/samsung36_protocol.h b/esphome/components/remote_base/samsung36_protocol.h index 4f15d906e7..df4e1af8d8 100644 --- a/esphome/components/remote_base/samsung36_protocol.h +++ b/esphome/components/remote_base/samsung36_protocol.h @@ -16,9 +16,9 @@ struct Samsung36Data { class Samsung36Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const Samsung36Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const Samsung36Data &data) override; + void encode(RemoteTransmitData *dst, const Samsung36Data &data); + optional decode(RemoteReceiveData src); + void dump(const Samsung36Data &data); }; DECLARE_REMOTE_PROTOCOL(Samsung36) diff --git a/esphome/components/remote_base/samsung_protocol.h b/esphome/components/remote_base/samsung_protocol.h index bb234d681d..dfa22ff85c 100644 --- a/esphome/components/remote_base/samsung_protocol.h +++ b/esphome/components/remote_base/samsung_protocol.h @@ -14,9 +14,9 @@ struct SamsungData { class SamsungProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const SamsungData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const SamsungData &data) override; + void encode(RemoteTransmitData *dst, const SamsungData &data); + optional decode(RemoteReceiveData src); + void dump(const SamsungData &data); }; DECLARE_REMOTE_PROTOCOL(Samsung) diff --git a/esphome/components/remote_base/sony_protocol.h b/esphome/components/remote_base/sony_protocol.h index eb873e8b7d..f83b2908b6 100644 --- a/esphome/components/remote_base/sony_protocol.h +++ b/esphome/components/remote_base/sony_protocol.h @@ -16,9 +16,9 @@ struct SonyData { class SonyProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const SonyData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const SonyData &data) override; + void encode(RemoteTransmitData *dst, const SonyData &data); + optional decode(RemoteReceiveData src); + void dump(const SonyData &data); }; DECLARE_REMOTE_PROTOCOL(Sony) diff --git a/esphome/components/remote_base/symphony_protocol.h b/esphome/components/remote_base/symphony_protocol.h index 7caf5eab86..40a5c2daec 100644 --- a/esphome/components/remote_base/symphony_protocol.h +++ b/esphome/components/remote_base/symphony_protocol.h @@ -17,9 +17,9 @@ struct SymphonyData { class SymphonyProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const SymphonyData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const SymphonyData &data) override; + void encode(RemoteTransmitData *dst, const SymphonyData &data); + optional decode(RemoteReceiveData src); + void dump(const SymphonyData &data); }; DECLARE_REMOTE_PROTOCOL(Symphony) diff --git a/esphome/components/remote_base/toshiba_ac_protocol.h b/esphome/components/remote_base/toshiba_ac_protocol.h index 8a853005ac..35d5af314c 100644 --- a/esphome/components/remote_base/toshiba_ac_protocol.h +++ b/esphome/components/remote_base/toshiba_ac_protocol.h @@ -14,9 +14,9 @@ struct ToshibaAcData { class ToshibaAcProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const ToshibaAcData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const ToshibaAcData &data) override; + void encode(RemoteTransmitData *dst, const ToshibaAcData &data); + optional decode(RemoteReceiveData src); + void dump(const ToshibaAcData &data); }; DECLARE_REMOTE_PROTOCOL(ToshibaAc) diff --git a/esphome/components/remote_base/toto_protocol.h b/esphome/components/remote_base/toto_protocol.h index 285c9f2125..8e965a5c73 100644 --- a/esphome/components/remote_base/toto_protocol.h +++ b/esphome/components/remote_base/toto_protocol.h @@ -16,9 +16,9 @@ struct TotoData { class TotoProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const TotoData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const TotoData &data) override; + void encode(RemoteTransmitData *dst, const TotoData &data); + optional decode(RemoteReceiveData src); + void dump(const TotoData &data); }; DECLARE_REMOTE_PROTOCOL(Toto) diff --git a/esphome/components/remote_receiver/__init__.py b/esphome/components/remote_receiver/__init__.py index 6e8c73d331..b2fd87165e 100644 --- a/esphome/components/remote_receiver/__init__.py +++ b/esphome/components/remote_receiver/__init__.py @@ -221,11 +221,11 @@ async def to_code(config: ConfigType) -> None: dumpers = await remote_base.build_dumpers(config[CONF_DUMP]) for dumper in dumpers: - cg.add(var.register_dumper(dumper)) + remote_base.add_dumper(var, dumper) triggers = await remote_base.build_triggers(config) for trigger in triggers: - cg.add(var.register_listener(trigger)) + remote_base.add_listener(var, trigger) await cg.register_component(var, config) cg.add( diff --git a/esphome/components/toshiba/climate.py b/esphome/components/toshiba/climate.py index 3b1e7352f9..e5f8544f2f 100644 --- a/esphome/components/toshiba/climate.py +++ b/esphome/components/toshiba/climate.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import climate_ir +from esphome.components import climate_ir, remote_base import esphome.config_validation as cv from esphome.const import CONF_MODEL from esphome.types import ConfigType @@ -26,5 +26,6 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(ToshibaClimate).exten async def to_code(config: ConfigType) -> None: + remote_base.request_protocol("toshiba_ac") # used from C++ var = await climate_ir.new_climate_ir(config) cg.add(var.set_model(config[CONF_MODEL])) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9144e65576..6b9b9eda43 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -137,6 +137,43 @@ #define MICRONOVA_LISTENER_COUNT 1 #define USE_MICRONOVA_WRITER #define MK2PVROUTER_LISTENER_COUNT 1 +#define REMOTE_BASE_DUMPER_COUNT 1 +#define REMOTE_BASE_LISTENER_COUNT 1 +#define USE_REMOTE_PROTOCOL_ABBWELCOME +#define USE_REMOTE_PROTOCOL_AEHA +#define USE_REMOTE_PROTOCOL_BEO4 +#define USE_REMOTE_PROTOCOL_BRENNENSTUHL +#define USE_REMOTE_PROTOCOL_BYRONSX +#define USE_REMOTE_PROTOCOL_CANALSAT +#define USE_REMOTE_PROTOCOL_COOLIX +#define USE_REMOTE_PROTOCOL_DISH +#define USE_REMOTE_PROTOCOL_DOOYA +#define USE_REMOTE_PROTOCOL_DRAYTON +#define USE_REMOTE_PROTOCOL_DYSON +#define USE_REMOTE_PROTOCOL_GOBOX +#define USE_REMOTE_PROTOCOL_HAIER +#define USE_REMOTE_PROTOCOL_JVC +#define USE_REMOTE_PROTOCOL_KEELOQ +#define USE_REMOTE_PROTOCOL_LG +#define USE_REMOTE_PROTOCOL_MAGIQUEST +#define USE_REMOTE_PROTOCOL_MIDEA +#define USE_REMOTE_PROTOCOL_MIRAGE +#define USE_REMOTE_PROTOCOL_NEC +#define USE_REMOTE_PROTOCOL_NEXA +#define USE_REMOTE_PROTOCOL_PANASONIC +#define USE_REMOTE_PROTOCOL_PIONEER +#define USE_REMOTE_PROTOCOL_PRONTO +#define USE_REMOTE_PROTOCOL_RAW +#define USE_REMOTE_PROTOCOL_RC5 +#define USE_REMOTE_PROTOCOL_RC6 +#define USE_REMOTE_PROTOCOL_RC_SWITCH +#define USE_REMOTE_PROTOCOL_ROOMBA +#define USE_REMOTE_PROTOCOL_SAMSUNG +#define USE_REMOTE_PROTOCOL_SAMSUNG36 +#define USE_REMOTE_PROTOCOL_SONY +#define USE_REMOTE_PROTOCOL_SYMPHONY +#define USE_REMOTE_PROTOCOL_TOSHIBA_AC +#define USE_REMOTE_PROTOCOL_TOTO #define SERIAL_PROXY_COUNT 2 #define SNTP_SERVER_COUNT 3 #define USE_MEDIA_PLAYER diff --git a/esphome/cpp_helpers.py b/esphome/cpp_helpers.py index 53b59cb124..fc44d27f47 100644 --- a/esphome/cpp_helpers.py +++ b/esphome/cpp_helpers.py @@ -1,4 +1,4 @@ -from collections.abc import Callable +from collections.abc import Callable, Hashable from dataclasses import dataclass, field import logging @@ -142,9 +142,10 @@ _SLOT_COUNTER_DOMAIN = "slot_counter" @dataclass class _SlotCounterState: - """Per-run slot counter state: requested counts and already-emitted defines.""" + """Per-run slot counter state: requested counts per define and key, and + already-emitted defines.""" - counts: dict[str, int] = field(default_factory=dict) + counts: dict[str, dict[Hashable, int]] = field(default_factory=dict) emitted: set[str] = field(default_factory=set) @@ -156,11 +157,13 @@ def _get_slot_counter_state() -> _SlotCounterState: def get_slot_count(define: str) -> int: - """Number of slots requested so far for `define`.""" - return _get_slot_counter_state().counts.get(define, 0) + """Value `define` would be emitted with so far: the largest count requested + under any one key, which is the plain request count when no key is used.""" + counts = _get_slot_counter_state().counts.get(define) + return max(counts.values()) if counts else 0 -def slot_counter(define: str) -> Callable[[], None]: +def slot_counter(define: str) -> Callable[..., None]: """Create a request_slot function for codegen-sized storage. The pattern behind a StaticVector listener array: a consumer's to_code @@ -169,6 +172,11 @@ def slot_counter(define: str) -> Callable[[], None]: emitted with the requested count. No requests, no define: the guarded storage and its registration method compile out entirely. + When several objects each declare the storage at the same size (one list + per receiver, per hub, ...) the caller passes the owning object as `key` + and the define becomes the largest count any one key requested, not the + total. Requests without a key share one count. + The counts live in a table under CORE.data, which clears between runs. A request arriving after the define was already emitted raises instead of silently undercounting: the define would keep the stale smaller value and @@ -179,10 +187,10 @@ def slot_counter(define: str) -> Callable[[], None]: async def emit_job() -> None: state = _get_slot_counter_state() state.emitted.add(define) - # Scheduled only by the first request, so the count is always >= 1 here. - add_define(define, state.counts[define]) + # Scheduled only by the first request, so there is at least one count here. + add_define(define, max(state.counts[define].values())) - def request_slot() -> None: + def request_slot(key: Hashable = None) -> None: state = _get_slot_counter_state() if define in state.emitted: raise ValueError( @@ -190,10 +198,16 @@ def slot_counter(define: str) -> Callable[[], None]: f"define was emitted; request slots from to_code, not from a " f"job running after FINAL emission" ) - counts = state.counts - counts[define] = (count := counts.get(define, 0) + 1) - if count == 1: + counts = state.counts.get(define) + if counts is None: + counts = state.counts[define] = {} CORE.add_job(emit_job) + elif (key is None) != (None in counts): + # a keyed and an unkeyed request would compare buckets instead of adding up + raise ValueError( + f"slot_counter('{define}'): every request must use a key, or none of them" + ) + counts[key] = counts.get(key, 0) + 1 return request_slot diff --git a/tests/component_tests/remote_receiver/__init__.py b/tests/component_tests/remote_receiver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/remote_receiver/config/receiver_bare.yaml b/tests/component_tests/remote_receiver/config/receiver_bare.yaml new file mode 100644 index 0000000000..b474194801 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_bare.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + board: esp32dev + +remote_receiver: + - id: rcvr + pin: GPIO4 diff --git a/tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml b/tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml new file mode 100644 index 0000000000..32c1b07f57 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml @@ -0,0 +1,24 @@ +esphome: + name: test + +esp32: + board: esp32dev + +logger: + +remote_receiver: + - id: rcvr + pin: GPIO4 + dump: + - nec + - rc_switch + on_nec: + then: + - logger.log: nec + +binary_sensor: + - platform: remote_receiver + name: Remote Input + nec: + address: 0x1234 + command: 0x5678 diff --git a/tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml b/tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml new file mode 100644 index 0000000000..c443a842f2 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml @@ -0,0 +1,22 @@ +esphome: + name: test + +esp32: + board: esp32dev + +remote_receiver: + - id: rcvr_ir + pin: GPIO4 + - id: rcvr_rf + pin: GPIO5 + +infrared: + - platform: ir_rf_proxy + name: IR Receiver + remote_receiver_id: rcvr_ir + +radio_frequency: + - platform: ir_rf_proxy + name: RF Receiver + frequency: 433.92MHz + remote_receiver_id: rcvr_rf diff --git a/tests/component_tests/remote_receiver/test_slot_counts.py b/tests/component_tests/remote_receiver/test_slot_counts.py new file mode 100644 index 0000000000..f381a64092 --- /dev/null +++ b/tests/component_tests/remote_receiver/test_slot_counts.py @@ -0,0 +1,91 @@ +"""Listener and dumper StaticVector sizes come from codegen slot counts.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.automation import ACTION_REGISTRY +from esphome.components import remote_base +import esphome.config_validation as cv + +from ..helpers import get_define_value + + +def test_dumper_and_listener_counts( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_with_dumpers.yaml")) + # nec and rc_switch dumpers + assert get_define_value("REMOTE_BASE_DUMPER_COUNT") == "2" + # on_nec trigger plus the remote_receiver binary sensor + assert get_define_value("REMOTE_BASE_LISTENER_COUNT") == "2" + + +def test_bare_receiver_emits_no_counts( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_bare.yaml")) + assert get_define_value("REMOTE_BASE_DUMPER_COUNT") is None + assert get_define_value("REMOTE_BASE_LISTENER_COUNT") is None + + +def test_proxy_receivers_count_as_listeners( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_with_proxies.yaml")) + # one proxy entity listens on each of the two receivers; every receiver's list gets the + # capacity of the busiest one, so this is the largest per receiver count, not the sum + assert get_define_value("REMOTE_BASE_LISTENER_COUNT") == "1" + assert get_define_value("REMOTE_BASE_DUMPER_COUNT") is None + + +def test_only_used_protocol_sources_are_compiled( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_with_dumpers.yaml")) + excluded = set(remote_base.FILTER_SOURCE_FILES()) + assert "nec_protocol.cpp" not in excluded + assert "rc_switch_protocol.cpp" not in excluded + assert "sony_protocol.cpp" in excluded + assert "remote_base.cpp" not in excluded + + +def test_every_registry_name_maps_to_a_protocol_source() -> None: + """A registry name must resolve to a source file or request_protocol rejects it.""" + names = ( + set(remote_base.BINARY_SENSOR_REGISTRY) + | set(remote_base.DUMPER_REGISTRY) + | {key.removeprefix("on_") for key in remote_base.TRIGGER_REGISTRY} + | { + key.removeprefix("remote_transmitter.transmit_") + for key in ACTION_REGISTRY + if key.startswith("remote_transmitter.transmit_") + } + ) + assert len(names) > 40 + for name in names: + assert remote_base._protocol_stem(name) in remote_base._PROTOCOL_STEMS, name + + +def test_request_protocol_rejects_unknown_names() -> None: + """A misspelled protocol would otherwise surface only as a link error.""" + with pytest.raises(ValueError, match="Unknown remote protocol 'toshiba'"): + remote_base.request_protocol("toshiba") + + +def test_dump_list_is_deduplicated_across_forms() -> None: + dumpers = remote_base.validate_dumpers(["raw", {"raw": None}, "nec", "nec"]) + assert [ + next(k for k in entry if k in remote_base.DUMPER_REGISTRY) for entry in dumpers + ] == ["raw", "nec"] + + +@pytest.mark.parametrize("bad", [["nec", None], [5]]) +def test_dump_list_rejects_invalid_entries_with_a_validation_error(bad: list) -> None: + with pytest.raises(cv.Invalid): + remote_base.validate_dumpers(bad) diff --git a/tests/components/remote_receiver/bare-common.yaml b/tests/components/remote_receiver/bare-common.yaml new file mode 100644 index 0000000000..c100c5c2da --- /dev/null +++ b/tests/components/remote_receiver/bare-common.yaml @@ -0,0 +1,6 @@ +# A receiver with no dumpers and no listeners compiles both lists out. +# Only built while remote_receiver is tested in isolation: the counts are global defines, +# so this variant cannot be merged with configs that register any. +remote_receiver: + - id: rcvr_bare + pin: ${pin} diff --git a/tests/components/remote_receiver/test-bare.esp32-idf.yaml b/tests/components/remote_receiver/test-bare.esp32-idf.yaml new file mode 100644 index 0000000000..152853b65f --- /dev/null +++ b/tests/components/remote_receiver/test-bare.esp32-idf.yaml @@ -0,0 +1,5 @@ +substitutions: + pin: GPIO2 + +packages: + bare: !include bare-common.yaml diff --git a/tests/unit_tests/test_cpp_helpers.py b/tests/unit_tests/test_cpp_helpers.py index 1c0e0d0a93..725c1daebb 100644 --- a/tests/unit_tests/test_cpp_helpers.py +++ b/tests/unit_tests/test_cpp_helpers.py @@ -187,6 +187,31 @@ def test_slot_counter_emits_requested_count() -> None: assert _define_value("TEST_SLOT_COUNT") == "2" +def test_slot_counter_keyed_emits_largest_count() -> None: + """Keyed requests size storage every key declares at the same capacity: + the define is the busiest key's count, not the total over all keys.""" + request = ch.slot_counter("TEST_SLOT_COUNT_KEYED") + request("rx_a") + request("rx_a") + request("rx_a") + request("rx_b") + assert ch.get_slot_count("TEST_SLOT_COUNT_KEYED") == 3 + ch.CORE.flush_tasks() + assert _define_value("TEST_SLOT_COUNT_KEYED") == "3" + + +def test_slot_counter_rejects_mixed_keyed_and_unkeyed_requests() -> None: + """A keyed and an unkeyed request for one define cannot be sized together.""" + request = ch.slot_counter("TEST_SLOT_COUNT_MIXED") + request("rx_a") + with pytest.raises(ValueError, match="TEST_SLOT_COUNT_MIXED"): + request() + unkeyed = ch.slot_counter("TEST_SLOT_COUNT_MIXED_2") + unkeyed() + with pytest.raises(ValueError, match="TEST_SLOT_COUNT_MIXED_2"): + unkeyed("rx_a") + + def test_slot_counter_without_requests_emits_nothing() -> None: """No requests, no job, no define — the guarded storage compiles out.""" ch.slot_counter("TEST_SLOT_COUNT_UNUSED") From 2578f17dc8705b0a18ce4a67f6de4bad8b2c6101 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:25:41 -0500 Subject: [PATCH 115/178] Bump bundled esphome-device-builder to 1.14.7 (#19096) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index cfa47fbdad..6f500dbe6f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.6 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.7 RUN \ platformio settings set enable_telemetry No \ From 2b71d5496d1c415e335a637c0839259d2ba7f39a Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:45:02 -0400 Subject: [PATCH 116/178] [const] Centralize definition of `CONF_MANUFACTURER` (#19098) --- esphome/components/const/__init__.py | 1 + esphome/components/esp32_ble_server/__init__.py | 2 +- esphome/components/sendspin/__init__.py | 2 +- tests/component_tests/sendspin/test_device_info.py | 7 ++----- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 49a625e3f1..256ab5c0a3 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -30,6 +30,7 @@ CONF_KEYS = "keys" CONF_LABEL = "label" CONF_LIBRETINY = "libretiny" CONF_LOOP = "loop" +CONF_MANUFACTURER = "manufacturer" CONF_NOX_INDEX = "nox_index" CONF_ON_PACKET = "on_packet" CONF_ON_RECEIVE = "on_receive" diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index d8095cd702..118ae06e42 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -3,6 +3,7 @@ import encodings from esphome import automation import esphome.codegen as cg from esphome.components import esp32_ble +from esphome.components.const import CONF_MANUFACTURER from esphome.components.esp32 import request_bluetooth from esphome.components.esp32_ble import BTLoggers, bt_uuid import esphome.config_validation as cv @@ -41,7 +42,6 @@ CONF_DESCRIPTORS = "descriptors" CONF_ENDIANNESS = "endianness" CONF_FIRMWARE_VERSION = "firmware_version" CONF_INDICATE = "indicate" -CONF_MANUFACTURER = "manufacturer" CONF_MANUFACTURER_DATA = "manufacturer_data" CONF_MAX_CLIENTS = "max_clients" CONF_ON_WRITE = "on_write" diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index c21047c70a..fda4d4f954 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -3,6 +3,7 @@ from dataclasses import dataclass, field from esphome import automation import esphome.codegen as cg from esphome.components import esp32, network, psram, socket, wifi +from esphome.components.const import CONF_MANUFACTURER import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, @@ -33,7 +34,6 @@ CONF_DISPLAY_OFFSET = "display_offset" CONF_SENDSPIN_ID = "sendspin_id" CONF_FIRMWARE_VERSION = "firmware_version" -CONF_MANUFACTURER = "manufacturer" # An empty device information string would be sent to the server as an empty value rather than # falling back, so reject it instead of silently substituting the fallback. The 127 byte cap keeps diff --git a/tests/component_tests/sendspin/test_device_info.py b/tests/component_tests/sendspin/test_device_info.py index 833dd398b4..61c10676da 100644 --- a/tests/component_tests/sendspin/test_device_info.py +++ b/tests/component_tests/sendspin/test_device_info.py @@ -8,11 +8,8 @@ from pathlib import Path import pytest from esphome import config_validation as cv -from esphome.components.sendspin import ( - CONF_FIRMWARE_VERSION, - CONF_MANUFACTURER, - CONFIG_SCHEMA, -) +from esphome.components.const import CONF_MANUFACTURER +from esphome.components.sendspin import CONF_FIRMWARE_VERSION, CONFIG_SCHEMA from esphome.const import CONF_MODEL, PlatformFramework from tests.component_tests.types import SetCoreConfigCallable From 9f9df85aa00bbac9d4c0db6905e538a80f279834 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:04:48 +0000 Subject: [PATCH 117/178] Bump aioesphomeapi from 46.4.0 to 46.4.1 (#19104) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72c42dad32..c73887a39d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.4.0 click==8.3.3 -aioesphomeapi==46.4.0 +aioesphomeapi==46.4.1 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.151.3 puremagic==2.2.0 From ff9b2a1c83edb7b542fa04f93407b86a5c04bcde Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 11 Sep 2026 18:20:40 -0500 Subject: [PATCH 118/178] [remote_receiver] Size the RMT ring buffer from receive symbols by default (#19100) --- .../components/remote_receiver/__init__.py | 14 ++++--- .../remote_receiver/remote_receiver.h | 4 +- .../remote_receiver/remote_receiver_rmt.cpp | 38 +++++++++++-------- .../config/receiver_buffer_size.yaml | 10 +++++ .../config/receiver_esp32_c2.yaml | 12 ++++++ .../config/receiver_esp8266.yaml | 9 +++++ .../remote_receiver/test_buffer_size.py | 28 ++++++++++++++ .../remote_receiver/test_slot_counts.py | 4 +- 8 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_esp8266.yaml create mode 100644 tests/component_tests/remote_receiver/test_buffer_size.py diff --git a/esphome/components/remote_receiver/__init__.py b/esphome/components/remote_receiver/__init__.py index b2fd87165e..6eaecf7ab0 100644 --- a/esphome/components/remote_receiver/__init__.py +++ b/esphome/components/remote_receiver/__init__.py @@ -114,15 +114,18 @@ CONFIG_SCHEMA = remote_base.validate_triggers( cv.Optional(CONF_TOLERANCE, default="25%"): validate_tolerance, cv.SplitDefault( CONF_BUFFER_SIZE, - esp32="10000b", - esp32_c2="1000b", - esp32_c61="1000b", + esp32=cv.UNDEFINED, + # the pulse ring needs a size; only RMT targets size themselves in setup() + **{ + f"esp32_{variant.removeprefix('ESP32').lower()}": "1000b" + for variant in esp32_rmt.VARIANTS_NO_RMT + }, esp8266="1000b", bk72xx="1000b", ln882x="1000b", rtl87xx="1000b", rp2="1000b", - ): cv.validate_bytes, + ): cv.All(cv.validate_bytes, cv.int_range(min=64)), cv.Optional(CONF_FILTER, default="50us"): cv.All( cv.positive_time_period_microseconds, cv.Range(max=TimePeriod(microseconds=4294967295)), @@ -233,7 +236,8 @@ async def to_code(config: ConfigType) -> None: config[CONF_TOLERANCE][CONF_VALUE], config[CONF_TOLERANCE][CONF_TYPE] ) ) - cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) + if CONF_BUFFER_SIZE in config: + cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) cg.add(var.set_filter_us(config[CONF_FILTER])) cg.add(var.set_idle_us(config[CONF_IDLE])) diff --git a/esphome/components/remote_receiver/remote_receiver.h b/esphome/components/remote_receiver/remote_receiver.h index f9ec054fe3..e59a8b2557 100644 --- a/esphome/components/remote_receiver/remote_receiver.h +++ b/esphome/components/remote_receiver/remote_receiver.h @@ -47,7 +47,7 @@ struct RemoteReceiverComponentStore { /// The position last read from volatile uint32_t buffer_read{0}; bool overflow{false}; - uint32_t buffer_size{1000}; + uint32_t buffer_size{0}; uint32_t receive_size{0}; uint32_t filter_symbols{0}; esp_err_t error{ESP_OK}; @@ -101,7 +101,7 @@ class RemoteReceiverComponent final : public remote_base::RemoteReceiverBase, HighFrequencyLoopRequester high_freq_; #endif - uint32_t buffer_size_{}; + uint32_t buffer_size_{}; // 0 on RMT targets: sized from receive_symbols in setup() uint32_t filter_us_{10}; uint32_t idle_us_{10000}; }; diff --git a/esphome/components/remote_receiver/remote_receiver_rmt.cpp b/esphome/components/remote_receiver/remote_receiver_rmt.cpp index 632ca9763a..4eebbbb16f 100644 --- a/esphome/components/remote_receiver/remote_receiver_rmt.cpp +++ b/esphome/components/remote_receiver/remote_receiver_rmt.cpp @@ -10,6 +10,7 @@ namespace esphome::remote_receiver { static const char *const TAG = "remote_receiver"; +static constexpr uint32_t DEFAULT_BUFFER_SLOTS = 4; static bool IRAM_ATTR HOT rmt_callback(rmt_channel_handle_t channel, const rmt_rx_done_event_data_t *event, void *arg) { RemoteReceiverComponentStore *store = (RemoteReceiverComponentStore *) arg; @@ -104,7 +105,11 @@ void RemoteReceiverComponent::setup() { this->store_.config.signal_range_max_ns = this->idle_us_ * 1000; this->store_.filter_symbols = this->filter_symbols_; this->store_.receive_size = this->receive_symbols_ * sizeof(rmt_symbol_word_t); - this->store_.buffer_size = std::max((event_size + this->store_.receive_size) * 2, this->buffer_size_); + // one slot per pending rmt_receive; two are the floor (one filling while one is decoded), and + // the default of four covers a few frames queued across a stalled loop pass + const uint32_t slot_size = event_size + this->store_.receive_size; + this->store_.buffer_size = + this->buffer_size_ != 0 ? std::max(slot_size * 2, this->buffer_size_) : slot_size * DEFAULT_BUFFER_SLOTS; this->store_.buffer = new uint8_t[this->store_.buffer_size]; error = rmt_receive(this->channel_, (uint8_t *) this->store_.buffer + event_size, this->store_.receive_size, &this->store_.config); @@ -117,20 +122,23 @@ void RemoteReceiverComponent::setup() { } void RemoteReceiverComponent::dump_config() { - ESP_LOGCONFIG(TAG, - "Remote Receiver:\n" - " Clock resolution: %" PRIu32 " hz\n" - " RMT symbols: %" PRIu32 "\n" - " Filter symbols: %" PRIu32 "\n" - " Receive symbols: %" PRIu32 "\n" - " Tolerance: %" PRIu32 "%s\n" - " Carrier frequency: %" PRIu32 " hz\n" - " Carrier duty: %u%%\n" - " Filter out pulses shorter than: %" PRIu32 " us\n" - " Signal is done after %" PRIu32 " us of no changes", - this->clock_resolution_, this->rmt_symbols_, this->filter_symbols_, this->receive_symbols_, - this->tolerance_, (this->tolerance_mode_ == remote_base::TOLERANCE_MODE_TIME) ? " us" : "%", - this->carrier_frequency_, this->carrier_duty_percent_, this->filter_us_, this->idle_us_); + ESP_LOGCONFIG( + TAG, + "Remote Receiver:\n" + " Clock resolution: %" PRIu32 " hz\n" + " RMT symbols: %" PRIu32 "\n" + " Filter symbols: %" PRIu32 "\n" + " Receive symbols: %" PRIu32 "\n" + " Buffer size: %" PRIu32 " bytes\n" + " Tolerance: %" PRIu32 "%s\n" + " Carrier frequency: %" PRIu32 " hz\n" + " Carrier duty: %u%%\n" + " Filter out pulses shorter than: %" PRIu32 " us\n" + " Signal is done after %" PRIu32 " us of no changes", + this->clock_resolution_, this->rmt_symbols_, this->filter_symbols_, this->receive_symbols_, + this->store_.buffer_size, this->tolerance_, + (this->tolerance_mode_ == remote_base::TOLERANCE_MODE_TIME) ? LOG_STR_LITERAL(" us") : LOG_STR_LITERAL("%"), + this->carrier_frequency_, this->carrier_duty_percent_, this->filter_us_, this->idle_us_); LOG_PIN(" Pin: ", this->pin_); if (this->is_failed()) { ESP_LOGE(TAG, "Configuring RMT driver failed: %s (%s)", esp_err_to_name(this->error_code_), diff --git a/tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml b/tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml new file mode 100644 index 0000000000..0b334954eb --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml @@ -0,0 +1,10 @@ +esphome: + name: test + +esp32: + board: esp32dev + +remote_receiver: + - id: rcvr + pin: GPIO4 + buffer_size: 2kb diff --git a/tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml b/tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml new file mode 100644 index 0000000000..c4497fefd8 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + board: esp32-c2-devkitm-1 + variant: esp32c2 + framework: + type: esp-idf + +remote_receiver: + - id: rcvr + pin: GPIO4 diff --git a/tests/component_tests/remote_receiver/config/receiver_esp8266.yaml b/tests/component_tests/remote_receiver/config/receiver_esp8266.yaml new file mode 100644 index 0000000000..f22d00d630 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_esp8266.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +remote_receiver: + - id: rcvr + pin: GPIO4 diff --git a/tests/component_tests/remote_receiver/test_buffer_size.py b/tests/component_tests/remote_receiver/test_buffer_size.py new file mode 100644 index 0000000000..9bfd12d9f5 --- /dev/null +++ b/tests/component_tests/remote_receiver/test_buffer_size.py @@ -0,0 +1,28 @@ +"""buffer_size reaches the receiver when set, and always on the pulse ring targets.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_explicit_buffer_size_is_passed_through( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("receiver_buffer_size.yaml")) + assert "rcvr->set_buffer_size(2000);" in main_cpp + + +def test_pulse_ring_target_keeps_a_default( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("receiver_esp8266.yaml")) + assert "rcvr->set_buffer_size(1000);" in main_cpp + + +def test_esp32_variant_without_rmt_keeps_a_default( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("receiver_esp32_c2.yaml")) + assert "rcvr->set_buffer_size(1000);" in main_cpp diff --git a/tests/component_tests/remote_receiver/test_slot_counts.py b/tests/component_tests/remote_receiver/test_slot_counts.py index f381a64092..4d69e6d923 100644 --- a/tests/component_tests/remote_receiver/test_slot_counts.py +++ b/tests/component_tests/remote_receiver/test_slot_counts.py @@ -27,7 +27,9 @@ def test_bare_receiver_emits_no_counts( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], ) -> None: - generate_main(component_config_path("receiver_bare.yaml")) + main_cpp = generate_main(component_config_path("receiver_bare.yaml")) + # the RMT ring is sized in setup() unless buffer_size is set + assert "set_buffer_size" not in main_cpp assert get_define_value("REMOTE_BASE_DUMPER_COUNT") is None assert get_define_value("REMOTE_BASE_LISTENER_COUNT") is None From eecea15f4f714af7dd7278cd2433ee3c800db92f Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:42:08 +1000 Subject: [PATCH 119/178] [core][lvgl] Migrate codegen helpers from LVGL to core code (#19105) Co-authored-by: Claude Sonnet 5 --- esphome/components/lvgl/automation.py | 3 +- esphome/components/lvgl/defines.py | 43 +---------- esphome/components/lvgl/lv_validation.py | 4 +- esphome/components/lvgl/widgets/__init__.py | 3 +- esphome/cpp_generator.py | 39 ++++++++++ tests/unit_tests/test_cpp_generator.py | 79 +++++++++++++++++++++ 6 files changed, 122 insertions(+), 49 deletions(-) diff --git a/esphome/components/lvgl/automation.py b/esphome/components/lvgl/automation.py index a62f466413..c23a36c389 100644 --- a/esphome/components/lvgl/automation.py +++ b/esphome/components/lvgl/automation.py @@ -14,7 +14,7 @@ from esphome.const import ( CONF_TIMEOUT, ) from esphome.core import Lambda -from esphome.cpp_generator import TemplateArguments, get_variable +from esphome.cpp_generator import StaticCastExpression, TemplateArguments, get_variable from esphome.cpp_types import nullptr from .defines import ( @@ -30,7 +30,6 @@ from .defines import ( CONF_SHOW_SNOW, CONF_TOP_LAYER, PARTS, - StaticCastExpression, add_warning, get_focused_widgets, get_options, diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 1eee8041f9..73fc58736b 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -10,12 +10,7 @@ from typing import Any from esphome import codegen as cg, config_validation as cv from esphome.const import CONF_ITEMS from esphome.core import CORE, ID, Lambda -from esphome.cpp_generator import ( - CallExpression, - LambdaExpression, - MockObj, - MockObjClass, -) +from esphome.cpp_generator import MockObj, StaticCastExpression, call_lambda from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor from esphome.types import Expression, SafeExpType @@ -157,17 +152,6 @@ def get_refreshed_widgets() -> set: return _get_data(KEY_REFRESHED_WIDGETS, set()) -class StaticCastExpression(Expression): - __slots__ = ("type", "exp") - - def __init__(self, type: Any, exp: SafeExpType): - self.type = str(type) - self.exp = cg.safe_exp(exp) - - def __str__(self): - return f"static_cast<{self.type}>({self.exp})" - - def add_define(macro: str, value="1"): lv_defines = get_defines() value = str(value) @@ -192,31 +176,6 @@ def addr(arg) -> MockObj: return MockObj(f"&{arg}") -def call_lambda(lamb: LambdaExpression) -> Expression: - """ - Given a lambda, either reduce to a simple expression or call it, possibly with parameters - from the surrounding context - :param lamb: - :return: - """ - expr = lamb.content.strip() - if expr.startswith("return") and expr.endswith(";"): - # Convert a lambda returning a simple expression to just that expression - expr = cg.RawExpression(expr[6:-1].strip()) - # Don't cast if the return type is a class - if isinstance(lamb.return_type, MockObjClass): - return expr - return StaticCastExpression(lamb.return_type, expr) - # If lambda has parameters, call it with their names - # Parameter names come from hardcoded component code (like "x", "it", "event") - # not from user input, so they're safe to use directly - if lamb.parameters and lamb.parameters.parameters: - return CallExpression( - lamb, *[MockObj(x.id) for x in lamb.parameters.parameters] - ) - return CallExpression(lamb) - - class LValidator: """ A validator for a particular type used in LVGL. Usable in configs as a validator, also diff --git a/esphome/components/lvgl/lv_validation.py b/esphome/components/lvgl/lv_validation.py index 42352b9602..6f86e49e51 100644 --- a/esphome/components/lvgl/lv_validation.py +++ b/esphome/components/lvgl/lv_validation.py @@ -16,7 +16,7 @@ from esphome.const import ( CONF_VALUE, ) from esphome.core import CORE, ID, Lambda -from esphome.cpp_generator import MockObj +from esphome.cpp_generator import MockObj, StaticCastExpression, call_lambda from esphome.cpp_types import ESPTime, int32, uint32 from esphome.helpers import cpp_string_escape from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor @@ -33,9 +33,7 @@ from .defines import ( LV_FONTS, LValidator, LvConstant, - StaticCastExpression, add_lv_use, - call_lambda, get_esphome_fonts_used, get_lv_fonts_used, get_lv_images_used, diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index c9099e3c3a..a524fe761f 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -16,7 +16,7 @@ from esphome.const import ( ) from esphome.core import ID, EsphomeError, TimePeriod from esphome.coroutine import FakeAwaitable -from esphome.cpp_generator import MockObj +from esphome.cpp_generator import MockObj, call_lambda from esphome.schema_extractors import EnableSchemaExtraction from esphome.types import Expression @@ -42,7 +42,6 @@ from ..defines import ( STATES, LValidator, add_lv_use, - call_lambda, get_styles_used, get_theme_widget_map, get_widget_map, diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index e6b8c0de42..173002438a 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -1187,3 +1187,42 @@ class MockObjClass(MockObj): def __repr__(self): return f"MockObjClass<{str(self.base)}, parents={self._parents}>" + + +class StaticCastExpression(Expression): + __slots__ = ("type", "exp") + + def __init__(self, type: Any, exp: SafeExpType): + self.type = str(type) + self.exp = safe_exp(exp) + + def __str__(self): + return f"static_cast<{self.type}>({self.exp})" + + +def call_lambda(lamb: LambdaExpression) -> Expression: + """ + Given a lambda, either reduce to a simple expression or call it, possibly with parameters + from the surrounding context. + This is for use only with value-returning lambdas, used in places where the value of a lambda call is needed. + :param lamb: The LambdaExpression to call or reduce + :return: An Expression representing the result of calling the lambda or reducing it to a simple expression + """ + # Developer error if this is called with a lambda that doesn't have a return type + assert lamb.return_type is not None, "Lambda must have a return type to be called" + expr = lamb.content.strip() + if re.match(r"^return\b", expr) and expr.endswith(";"): + # Convert a lambda returning a simple expression to just that expression + expr = RawExpression(expr[6:-1].strip()) + # Don't cast if the return type is a class + if isinstance(lamb.return_type, MockObjClass): + return expr + return StaticCastExpression(lamb.return_type, expr) + # If lambda has parameters, call it with their names + # Parameter names come from hardcoded component code (like "x", "it", "event") + # not from user input, so they're safe to use directly + if lamb.parameters and lamb.parameters.parameters: + return CallExpression( + lamb, *[MockObj(x.id) for x in lamb.parameters.parameters] + ) + return CallExpression(lamb) diff --git a/tests/unit_tests/test_cpp_generator.py b/tests/unit_tests/test_cpp_generator.py index 81ae586e23..052513ce97 100644 --- a/tests/unit_tests/test_cpp_generator.py +++ b/tests/unit_tests/test_cpp_generator.py @@ -85,6 +85,15 @@ class TestCallExpression: assert actual == 'my_function(1, "2", false)' +class TestStaticCastExpression: + def test_str(self): + target = cg.StaticCastExpression(ct.bool_, 42) + + actual = str(target) + + assert actual == "static_cast(42)" + + class TestStructInitializer: def test_str(self): target = cg.StructInitializer( @@ -229,6 +238,76 @@ class TestLambdaExpression: ) +class TestCallLambda: + """Tests for the call_lambda() function.""" + + def test_call_lambda__return_expression_casts_to_return_type(self): + """A lambda body that is just a return statement reduces to the + expression, cast to the lambda's return type.""" + lamb = cg.LambdaExpression(("return foo + 1;",), (), "", ct.bool_) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.StaticCastExpression) + assert str(result) == "static_cast(foo + 1)" + + def test_call_lambda__return_expression_with_class_return_type_no_cast(self): + """A class return type is not cast, since static_cast doesn't apply + to arbitrary class types.""" + mock_class = cg.MockObjClass("foo::Bar", parents=()) + lamb = cg.LambdaExpression(("return get_bar();",), (), "", mock_class) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.RawExpression) + assert str(result) == "get_bar()" + + def test_call_lambda__no_return_with_parameters_calls_with_names(self): + """A multi-statement lambda with parameters is called with the + parameter names as arguments.""" + lamb = cg.LambdaExpression( + ("do_something(x, y);",), ((int, "x"), (float, "y")), "=", ct.bool_ + ) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.CallExpression) + assert str(result) == ( + "[=](int32_t x, float y) -> bool {\n do_something(x, y);\n}(x, y)" + ) + + def test_call_lambda__no_return_type_raises(self): + """Calling a lambda with no declared return type is a developer + error: call_lambda is only for value-returning lambdas.""" + lamb = cg.LambdaExpression(("do_something();",), (), "=") + + with pytest.raises(AssertionError): + cg.call_lambda(lamb) + + def test_call_lambda__identifier_starting_with_return_is_not_a_return_statement( + self, + ): + """A body that merely starts with the substring "return" (e.g. a call + to a function named returnValue()) must not be mistaken for a return + statement -- the match requires a word boundary after "return".""" + lamb = cg.LambdaExpression(("returnValue();",), (), "=", ct.bool_) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.CallExpression) + assert str(result) == "[=]() -> bool {\n returnValue();\n}()" + + def test_call_lambda__no_return_no_parameters_calls_with_no_args(self): + """A multi-statement lambda without parameters is called with no + arguments.""" + lamb = cg.LambdaExpression(("do_something();",), (), "", ct.bool_) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.CallExpression) + assert str(result) == "[]() -> bool {\n do_something();\n}()" + + class TestLiterals: @pytest.mark.parametrize( "target, expected", From ebb9037ea1bf802334299b7c38b26dac352d028a Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Fri, 11 Sep 2026 22:08:23 -0500 Subject: [PATCH 120/178] [bridge] New component and `cdc_acm_uart` platform (#11689) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 Co-authored-by: J. Nick Koston --- CODEOWNERS | 3 + esphome/components/bridge/__init__.py | 4 + esphome/components/cdc_acm_uart/__init__.py | 1 + .../cdc_acm_uart/bridge/__init__.py | 114 +++++ .../bridge/cdc_acm_uart_bridge.cpp | 468 ++++++++++++++++++ .../cdc_acm_uart/bridge/cdc_acm_uart_bridge.h | 117 +++++ esphome/components/usb_cdc_acm/usb_cdc_acm.h | 34 ++ .../usb_cdc_acm/usb_cdc_acm_esp32.cpp | 26 +- script/analyze_component_buses.py | 1 + .../component_tests/cdc_acm_uart/__init__.py | 0 .../component_tests/cdc_acm_uart/test_init.py | 154 ++++++ tests/component_tests/conftest.py | 11 +- tests/component_tests/types.py | 3 +- tests/components/cdc_acm_uart/common.yaml | 18 + .../components/cdc_acm_uart/common_dual.yaml | 12 + .../cdc_acm_uart/test.esp32-p4-idf.yaml | 15 + .../cdc_acm_uart/test.esp32-s2-idf.yaml | 14 + .../cdc_acm_uart/test.esp32-s3-idf.yaml | 17 + 18 files changed, 983 insertions(+), 29 deletions(-) create mode 100644 esphome/components/bridge/__init__.py create mode 100644 esphome/components/cdc_acm_uart/__init__.py create mode 100644 esphome/components/cdc_acm_uart/bridge/__init__.py create mode 100644 esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp create mode 100644 esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h create mode 100644 tests/component_tests/cdc_acm_uart/__init__.py create mode 100644 tests/component_tests/cdc_acm_uart/test_init.py create mode 100644 tests/components/cdc_acm_uart/common.yaml create mode 100644 tests/components/cdc_acm_uart/common_dual.yaml create mode 100644 tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml create mode 100644 tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml create mode 100644 tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml diff --git a/CODEOWNERS b/CODEOWNERS index f91bc00ae5..246a210c7c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -100,6 +100,7 @@ esphome/components/bmp581_i2c/* @danielkent-net @kahrendt esphome/components/bmp581_spi/* @danielkent-net @kahrendt esphome/components/bp1658cj/* @Cossid esphome/components/bp5758d/* @Cossid +esphome/components/bridge/* @kbx81 esphome/components/bthome_mithermometer/* @nagyrobi esphome/components/button/* @esphome/core esphome/components/bytebuffer/* @clydebarrow @@ -111,6 +112,8 @@ esphome/components/captive_portal/* @esphome/core esphome/components/cc1101/* @gabest11 @lygris esphome/components/ccs811/* @habbie esphome/components/cd74hc4067/* @asoehlke +esphome/components/cdc_acm_uart/* @kbx81 +esphome/components/cdc_acm_uart/bridge/* @kbx81 esphome/components/ch422g/* @clydebarrow @jesterret esphome/components/ch423/* @dwmw2 esphome/components/chsc6x/* @kkosik20 diff --git a/esphome/components/bridge/__init__.py b/esphome/components/bridge/__init__.py new file mode 100644 index 0000000000..49811b0181 --- /dev/null +++ b/esphome/components/bridge/__init__.py @@ -0,0 +1,4 @@ +CODEOWNERS = ["@kbx81"] +DOMAIN = "bridge" + +IS_PLATFORM_COMPONENT = True diff --git a/esphome/components/cdc_acm_uart/__init__.py b/esphome/components/cdc_acm_uart/__init__.py new file mode 100644 index 0000000000..516af84856 --- /dev/null +++ b/esphome/components/cdc_acm_uart/__init__.py @@ -0,0 +1 @@ +CODEOWNERS = ["@kbx81"] diff --git a/esphome/components/cdc_acm_uart/bridge/__init__.py b/esphome/components/cdc_acm_uart/bridge/__init__.py new file mode 100644 index 0000000000..cee048df5d --- /dev/null +++ b/esphome/components/cdc_acm_uart/bridge/__init__.py @@ -0,0 +1,114 @@ +from esphome import pins +import esphome.codegen as cg +from esphome.components import esp32, uart, usb_cdc_acm +from esphome.components.bridge import DOMAIN as BRIDGE_DOMAIN +from esphome.components.esp32 import VARIANT_ESP32P4, VARIANT_ESP32S2, VARIANT_ESP32S3 +import esphome.config_validation as cv +from esphome.const import CONF_DEBUG, CONF_ID, CONF_UART_ID +import esphome.final_validate as fv +from esphome.types import ConfigType + +CODEOWNERS = ["@kbx81"] +DEPENDENCIES = ["tinyusb", "uart", "usb_cdc_acm"] + +CONF_DTR_PIN = "dtr_pin" +CONF_RTS_PIN = "rts_pin" +CONF_USB_CDC_ACM_ID = "usb_cdc_acm_id" + +cdc_acm_uart_ns = cg.esphome_ns.namespace("cdc_acm_uart") +CDCACMUARTBridge = cdc_acm_uart_ns.class_("CDCACMUARTBridge", cg.Component) + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(CDCACMUARTBridge), + cv.Required(CONF_UART_ID): cv.use_id(uart.IDFUARTComponent), + cv.Required(CONF_USB_CDC_ACM_ID): cv.use_id(usb_cdc_acm.USBCDCACMInstance), + cv.Optional(CONF_DTR_PIN): pins.gpio_output_pin_schema, + cv.Optional(CONF_RTS_PIN): pins.gpio_output_pin_schema, + } + ).extend(cv.COMPONENT_SCHEMA), + # Narrower than usb_cdc_acm's variant list on purpose: S31/H4 untested on + # hardware; extend once verified. + esp32.only_on_variant( + supported=[VARIANT_ESP32P4, VARIANT_ESP32S2, VARIANT_ESP32S3], + ), +) + + +def _subtree_references_uart(node: object, uart_id: str) -> bool: + """Return True if any dict in the subtree has a uart_id entry naming this bus.""" + if isinstance(node, dict): + return any( + (key == CONF_UART_ID and str(value) == uart_id) + or _subtree_references_uart(value, uart_id) + for key, value in node.items() + ) + if isinstance(node, list): + return any(_subtree_references_uart(item, uart_id) for item in node) + return False + + +def _reject_debug(uart_conf: ConfigType) -> ConfigType: + # The worker tasks use the IDF driver directly, so the uart debugger never sees + # bridge traffic and its dummy_receiver would drain RX bytes on the main loop. + if CONF_DEBUG in uart_conf: + raise cv.Invalid( + "A bridged UART cannot use 'debug'; the bridge bypasses the UART " + "component's read/write path.", + [CONF_DEBUG], + ) + return uart_conf + + +def _final_validate(config: ConfigType) -> ConfigType: + full_config = fv.full_config.get() + # Bridges of any platform must own their interfaces exclusively; shared ring + # buffers and overwritten callbacks would corrupt both streams silently. The + # seen-set is keyed on the bridge domain so future platforms share it. + # Other components bind either interface through the same uart_id key (the CDC + # instance is itself a uart::UARTComponent) and would race the worker tasks. + # Bare `id:` references (a uart.write action) cannot be distinguished; not caught. + data = full_config.data.setdefault(BRIDGE_DOMAIN, {}) + for conf_key, label in ( + (CONF_UART_ID, "UART"), + (CONF_USB_CDC_ACM_ID, "USB CDC-ACM interface"), + ): + owned_id = str(config[conf_key]) + used = data.setdefault(conf_key, set()) + if owned_id in used: + raise cv.Invalid( + f"The {label} '{owned_id}' is already bridged by another 'bridge' " + f"instance; each bridge requires its own {label}.", + [conf_key], + ) + used.add(owned_id) + for domain, domain_conf in full_config.items(): + if domain == BRIDGE_DOMAIN: + continue + if _subtree_references_uart(domain_conf, owned_id): + raise cv.Invalid( + f"The {label} '{owned_id}' is also used by '{domain}'; a bridge " + f"requires exclusive use of its {label}.", + [conf_key], + ) + + fv.id_declaration_match_schema(_reject_debug)(config[CONF_UART_ID]) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + +async def to_code(config: ConfigType) -> None: + uart_component = await cg.get_variable(config[CONF_UART_ID]) + usb_cdc = await cg.get_variable(config[CONF_USB_CDC_ACM_ID]) + var = cg.new_Pvariable(config[CONF_ID], uart_component, usb_cdc) + await cg.register_component(var, config) + + if dtr_pin_config := config.get(CONF_DTR_PIN): + dtr_pin = await cg.gpio_pin_expression(dtr_pin_config) + cg.add(var.set_dtr_pin(dtr_pin)) + if rts_pin_config := config.get(CONF_RTS_PIN): + rts_pin = await cg.gpio_pin_expression(rts_pin_config) + cg.add(var.set_rts_pin(rts_pin)) diff --git a/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp new file mode 100644 index 0000000000..042688bfe6 --- /dev/null +++ b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp @@ -0,0 +1,468 @@ +#if defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) +#include "cdc_acm_uart_bridge.h" +#include "esphome/core/application.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/ringbuf.h" +#include "driver/uart.h" +#include "soc/soc_caps.h" + +namespace esphome::cdc_acm_uart { + +static const char *const TAG = "cdc_acm_uart"; + +static constexpr size_t UART_TASK_STACK_SIZE = 4096; +static constexpr size_t RINGBUF_RETRY_CHUNK_SIZE = 64; +static constexpr uint32_t LOG_THROTTLE_MS = 1000; +static constexpr uint32_t UART_RELOAD_SETTLE_MS = 20; +// Above the default priority but below the USB/Wi-Fi system tasks. +static constexpr UBaseType_t TASK_PRIORITY = 4; + +static bool should_log_now(uint32_t *last_ms, uint32_t interval_ms) { + uint32_t now = millis(); + if ((now - *last_ms) >= interval_ms) { + *last_ms = now; + return true; + } + return false; +} + +static bool ringbuf_send_with_retry(RingbufHandle_t ringbuf, const uint8_t *data, size_t len, uint32_t *log_ms) { + if (len == 0) { + return true; + } + + if (xRingbufferSend(ringbuf, data, len, pdMS_TO_TICKS(1)) == pdTRUE) { + return true; + } + + size_t offset = 0; + while (offset < len) { + size_t chunk = std::min(RINGBUF_RETRY_CHUNK_SIZE, len - offset); + if (xRingbufferSend(ringbuf, data + offset, chunk, pdMS_TO_TICKS(1)) != pdTRUE) { + if (should_log_now(log_ms, LOG_THROTTLE_MS)) { + ESP_LOGW(TAG, "USB TX buffer full; some data is lost"); + } + return false; + } + offset += chunk; + } + return true; +} + +void CDCACMUARTBridge::setup() { + // Line state starts deasserted (no host yet); active-low DTR#/RTS# wiring is + // handled by configuring the pins inverted, so deasserted idles HIGH. + if (this->dtr_pin_ != nullptr) { + this->dtr_pin_->setup(); + this->dtr_pin_->digital_write(false); + } + + if (this->rts_pin_ != nullptr) { + this->rts_pin_->setup(); + this->rts_pin_->digital_write(false); + } + + // A failed UART never assigned its port number, so the worker tasks would run + // against an indeterminate port. + if (this->uart_parent_->is_failed()) { + ESP_LOGE(TAG, "UART parent failed; aborting"); + this->mark_failed(); + return; + } + + this->configured_baud_rate_ = this->uart_parent_->get_baud_rate(); + this->configured_parity_ = this->uart_parent_->get_parity(); + this->configured_stop_bits_ = this->uart_parent_->get_stop_bits(); + this->configured_data_bits_ = this->uart_parent_->get_data_bits(); + + // usb_cdc_acm sets up first (priority IO > HARDWARE). Any interface failing marks + // the hub failed, and a failed hub no longer runs loop(), so line coding and line + // state events would never reach this bridge even if its own interface is healthy. + if (this->usb_cdc_parent_->get_parent()->is_failed()) { + ESP_LOGE(TAG, "USB CDC ACM failed; aborting"); + this->mark_failed(); + return; + } + + // Per-instance task names (keyed on the CDC interface number) keep task dumps + // unambiguous with multiple bridges. + char tx_task_name[] = "cdc_uart_tx_0"; + char rx_task_name[] = "cdc_uart_rx_0"; + const char itf_char = format_hex_char(this->usb_cdc_parent_->get_itf()); + tx_task_name[sizeof(tx_task_name) - 2] = itf_char; + rx_task_name[sizeof(rx_task_name) - 2] = itf_char; + + xTaskCreate(uart_tx_task_fn, tx_task_name, UART_TASK_STACK_SIZE, this, TASK_PRIORITY, &this->uart_tx_task_handle_); + if (this->uart_tx_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Failed to create UART TX task"); + this->mark_failed(); + return; + } + + xTaskCreate(uart_rx_task_fn, rx_task_name, UART_TASK_STACK_SIZE, this, TASK_PRIORITY, &this->uart_rx_task_handle_); + if (this->uart_rx_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Failed to create UART RX task"); + vTaskDelete(this->uart_tx_task_handle_); + this->uart_tx_task_handle_ = nullptr; + this->mark_failed(); + return; + } + + // Only register callbacks once both tasks exist, so a failed setup never drives + // DTR/RTS from a dead bridge. + this->usb_cdc_parent_->set_line_state_callback([this](bool dtr, bool rts) { this->set_line_state(dtr, rts); }); + this->usb_cdc_parent_->set_line_coding_callback([this](uint32_t, uint8_t, uint8_t, uint8_t) { + this->host_coding_seen_ = true; + // Another component owns the UART's framing while paused; resume() re-syncs. + if (this->paused_ == 0) { + this->set_line_coding(); + } + }); + + // Release the workers only now: until here a failed setup may still delete the TX + // task, which is safe only while it is parked and owns nothing in the driver. + xTaskNotifyGive(this->uart_tx_task_handle_); + xTaskNotifyGive(this->uart_rx_task_handle_); + + // loop() only services line-coding reloads; stay off the main loop until one is + // scheduled. + this->disable_loop(); +} + +void CDCACMUARTBridge::dump_config() { + ESP_LOGCONFIG(TAG, + "CDC-ACM UART Bridge:\n" + " UART Bus: %u\n" + " USB CDC Interface: %u", + this->uart_parent_->get_hw_serial_number(), this->usb_cdc_parent_->get_itf()); + LOG_PIN(" DTR Pin: ", this->dtr_pin_); + LOG_PIN(" RTS Pin: ", this->rts_pin_); +} + +void CDCACMUARTBridge::on_shutdown() { + // The UART (BUS) shuts down after this component (HARDWARE) and deletes its driver, + // freeing the ring buffer and mutexes the worker tasks block on. Suspending the + // tasks unlinks them from those objects first. + if (this->uart_rx_task_handle_ != nullptr) { + vTaskSuspend(this->uart_rx_task_handle_); + } + if (this->uart_tx_task_handle_ != nullptr) { + vTaskSuspend(this->uart_tx_task_handle_); + } +} + +void CDCACMUARTBridge::loop() { + switch (this->state_) { + case MainState::MAIN_STATE_RELOAD_PENDING: + if ((App.get_loop_component_start_time() - this->reload_requested_at_) < UART_RELOAD_SETTLE_MS) { + return; + } + // Deliberately not gated on tx_idle_(): a host that re-codes the line mid-stream + // wants the new framing now, and its own in-flight bytes are its concern. + // apply_settings_live() rewrites the framing registers without reinstalling the + // driver, so the worker tasks blocked inside it are undisturbed. + this->uart_parent_->apply_settings_live(); + this->state_ = MainState::MAIN_STATE_RUNNING; + break; + case MainState::MAIN_STATE_PAUSING: + case MainState::MAIN_STATE_RESUMING: + // Let a host write that was in flight drain, FIFO included, before a reload + // flushes the FIFOs and truncates it. + if (!this->tx_idle_()) { + return; + } + if (this->state_ == MainState::MAIN_STATE_PAUSING) { + this->restore_configured_framing_(); + this->state_ = MainState::MAIN_STATE_PAUSED; + } else { + this->finish_resume_(); + } + break; + default: + break; + } + this->disable_loop(); +} + +void CDCACMUARTBridge::set_line_coding() { + if (!this->sync_host_framing_()) { + return; + } + // Coalesce rapid line-coding updates from the host. + this->reload_requested_at_ = App.get_loop_component_start_time(); + this->state_ = MainState::MAIN_STATE_RELOAD_PENDING; + // Main-loop context (via USBCDCACMInstance::process_events_). + this->enable_loop(); +} + +bool CDCACMUARTBridge::sync_host_framing_() { + // usb_cdc_acm has already translated the wire coding onto the CDC instance (main + // loop); mirror it here so the framing translation has a single source of truth. + bool changed = false; + + // Reject 0 (the CDC B0/hang-up encoding; older IDF revisions divide by the rate) + // and rates above the SoC ceiling. Anything in between is the driver's call, + // matching what a YAML-configured UART accepts. + const uint32_t baud = this->usb_cdc_parent_->get_baud_rate(); + if (baud == 0 || baud > SOC_UART_BITRATE_MAX) { + ESP_LOGW(TAG, "Ignoring unsupported baud rate %" PRIu32 " from host; keeping %" PRIu32, baud, + this->uart_parent_->get_baud_rate()); + } else if (this->uart_parent_->get_baud_rate() != baud) { + this->uart_parent_->set_baud_rate(baud); + changed = true; + } + + const uint8_t stop_bits = this->usb_cdc_parent_->get_stop_bits(); + if (this->uart_parent_->get_stop_bits() != stop_bits) { + this->uart_parent_->set_stop_bits(stop_bits); + changed = true; + } + + const auto parity = this->usb_cdc_parent_->get_parity(); + if (this->uart_parent_->get_parity() != parity) { + this->uart_parent_->set_parity(parity); + changed = true; + } + + // USB CDC permits data-bit counts the UART cannot represent (up to 16). + const uint8_t data_bits = this->usb_cdc_parent_->get_data_bits(); + if (data_bits < 5 || data_bits > 8) { + ESP_LOGW(TAG, "Ignoring unsupported data bits %u from host; keeping %u", data_bits, + this->uart_parent_->get_data_bits()); + } else if (this->uart_parent_->get_data_bits() != data_bits) { + this->uart_parent_->set_data_bits(data_bits); + changed = true; + } + + if (changed) { + ESP_LOGV(TAG, "Line coding: baud=%" PRIu32 ", data_bits=%u, stop_bits=%u, parity=%u", + this->uart_parent_->get_baud_rate(), this->uart_parent_->get_data_bits(), + this->uart_parent_->get_stop_bits(), static_cast(this->uart_parent_->get_parity())); + } + return changed; +} + +void CDCACMUARTBridge::pause() { + if (this->state_ == MainState::MAIN_STATE_PAUSING || this->state_ == MainState::MAIN_STATE_PAUSED) { + return; + } + this->paused_ = 1; + // A null RX task means setup() has not completed (or failed): nothing to stop, and + // the framing snapshot does not exist yet. Should setup() run later, the RX task + // starts parked. + if (this->uart_rx_task_handle_ == nullptr) { + this->state_ = MainState::MAIN_STATE_PAUSED; + return; + } + // Drops a coalesced host reload or a pending resume; loop() restores the framing + // once any host write in flight has drained. + this->state_ = MainState::MAIN_STATE_PAUSING; + this->enable_loop(); +} + +void CDCACMUARTBridge::resume() { + if (this->state_ != MainState::MAIN_STATE_PAUSING && this->state_ != MainState::MAIN_STATE_PAUSED) { + return; + } + if (this->uart_rx_task_handle_ == nullptr) { + this->paused_ = 0; + this->state_ = MainState::MAIN_STATE_RUNNING; + return; + } + // A restore still waiting on the TX side is moot: the host's framing is kept. + if (!this->tx_idle_()) { + this->state_ = MainState::MAIN_STATE_RESUMING; + this->enable_loop(); + return; + } + this->finish_resume_(); + this->disable_loop(); +} + +void CDCACMUARTBridge::finish_resume_() { + // Take the bus back at a known framing before either task runs again: the host's + // if it ever sent one, else the YAML framing (the other owner may have changed it). + if (this->host_coding_seen_) { + this->sync_host_framing_(); + this->uart_parent_->apply_settings_live(); + } else { + this->restore_configured_framing_(); + } + this->paused_ = 0; + this->state_ = MainState::MAIN_STATE_RUNNING; + this->drive_line_state_(); + xTaskNotifyGive(this->uart_rx_task_handle_); +} + +bool CDCACMUARTBridge::tx_idle_() { + const auto uart_num = static_cast(this->uart_parent_->get_hw_serial_number()); + return this->tx_busy_ == 0 && uart_wait_tx_done(uart_num, 0) == ESP_OK; +} + +void CDCACMUARTBridge::restore_configured_framing_() { + // Always applied: the cached settings can lead the hardware by a pending reload, + // so they are no proof of what is live. + this->uart_parent_->set_baud_rate(this->configured_baud_rate_); + this->uart_parent_->set_parity(this->configured_parity_); + this->uart_parent_->set_stop_bits(this->configured_stop_bits_); + this->uart_parent_->set_data_bits(this->configured_data_bits_); + this->uart_parent_->apply_settings_live(); +} + +void CDCACMUARTBridge::set_line_state(bool dtr, bool rts) { + ESP_LOGV(TAG, "Line state: DTR=%d, RTS=%d", dtr, rts); + this->host_dtr_ = dtr; + this->host_rts_ = rts; + // Frozen while paused: a host opening the port must not reset a peer that another + // component is talking to. + if (this->paused_ == 0) { + this->drive_line_state_(); + } +} + +void CDCACMUARTBridge::drive_line_state_() { + if (this->dtr_pin_ != nullptr) { + this->dtr_pin_->digital_write(this->host_dtr_); + } + if (this->rts_pin_ != nullptr) { + this->rts_pin_->digital_write(this->host_rts_); + } +} + +void CDCACMUARTBridge::uart_rx_task_fn(void *arg) { + auto *bridge = static_cast(arg); + bridge->uart_rx_task_(); +} + +void CDCACMUARTBridge::uart_tx_task_fn(void *arg) { + auto *bridge = static_cast(arg); + bridge->uart_tx_task_(); +} + +void CDCACMUARTBridge::uart_rx_task_() { + TaskHandle_t usb_tx_handle = this->usb_cdc_parent_->get_tx_task_handle(); + RingbufHandle_t usb_tx_ringbuf = this->usb_cdc_parent_->get_tx_ringbuf(); + uart_port_t uart_num = static_cast(this->uart_parent_->get_hw_serial_number()); + // Back-dated so a problem within the first LOG_THROTTLE_MS of uptime still logs. + uint32_t tx_full_log_ms = millis() - LOG_THROTTLE_MS; + uint32_t err_log_ms = millis() - LOG_THROTTLE_MS; + + uint8_t *data = this->uart_rx_buffer_.data(); + const size_t buf_size = this->uart_rx_buffer_.size(); + + // Released by setup() once both tasks exist. + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + + while (true) { + if (this->paused_ != 0) { + // Parked until resume() notifies; nothing is read, so the other owner sees + // every byte. + this->rx_parked_ = 1; + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + this->rx_parked_ = 0; + continue; + } + + // Block until at least one byte is available from UART. + int total_rx_size = uart_read_bytes(uart_num, data, 1, pdMS_TO_TICKS(UART_RX_WAIT_MS)); + if (total_rx_size < 0) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "UART read failed: %d", total_rx_size); + } + vTaskDelay(pdMS_TO_TICKS(10)); + continue; + } + if (total_rx_size == 0) { + continue; + } + // pause() landed during the read: don't forward a byte to a host that is gone. + if (this->paused_ != 0) { + continue; + } + + // Drain the currently buffered burst without waiting. + while (true) { + int rx_data_size = uart_read_bytes(uart_num, data + total_rx_size, buf_size - total_rx_size, 0); + if (rx_data_size < 0) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "UART read failed: %d", rx_data_size); + } + break; + } + if (rx_data_size == 0) { + break; + } + ESP_LOGV(TAG, "UART RX: %d bytes", rx_data_size); + total_rx_size += rx_data_size; + if (total_rx_size >= (int) buf_size) { + break; + } + } + + ringbuf_send_with_retry(usb_tx_ringbuf, data, total_rx_size, &tx_full_log_ms); + + ESP_LOGV(TAG, "UART RX: waking up USB TX task"); + xTaskNotifyGive(usb_tx_handle); + } +} + +void CDCACMUARTBridge::uart_tx_task_() { + RingbufHandle_t usb_rx_ringbuf = this->usb_cdc_parent_->get_rx_ringbuf(); + uart_port_t uart_num = static_cast(this->uart_parent_->get_hw_serial_number()); + uint8_t *data_to_uart = this->uart_tx_buffer_.data(); + const size_t buf_size = this->uart_tx_buffer_.size(); + size_t rx_size; + // Back-dated so a problem within the first LOG_THROTTLE_MS of uptime still logs. + uint32_t err_log_ms = millis() - LOG_THROTTLE_MS; + uint32_t drop_log_ms = millis() - LOG_THROTTLE_MS; + + // Released by setup() once both tasks exist. + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + + while (true) { + ESP_LOGV(TAG, "Waiting for data to send to UART"); + esp_err_t ret = usb_cdc_acm::ringbuf_read_bytes(usb_rx_ringbuf, data_to_uart, buf_size, &rx_size, portMAX_DELAY); + + if (ret != ESP_OK) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "USB RX RingBuf read failed"); + } + // Yield: this task runs above the main loop, so a persistent failure must not + // become a tight loop. + vTaskDelay(pdMS_TO_TICKS(10)); + continue; + } + + // Another component owns the UART; host bytes must not interleave with its traffic. + // tx_busy_ goes up before the check so is_paused() cannot miss a write in flight. + this->tx_busy_ = 1; + if (this->paused_ != 0) { + this->tx_busy_ = 0; + if (should_log_now(&drop_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGW(TAG, "Paused; dropping %zu bytes from host", rx_size); + } + continue; + } + + ESP_LOGV(TAG, "Sending %zu bytes to UART", rx_size); + // Signed: uart_write_bytes() returns -1 on error. + int xfer_size = uart_write_bytes(uart_num, data_to_uart, rx_size); + this->tx_busy_ = 0; + + if (xfer_size < 0) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "UART write failed: %d", xfer_size); + } + } else if (static_cast(xfer_size) != rx_size) { + ESP_LOGW(TAG, "UART write incomplete (%d/%zu bytes)", xfer_size, rx_size); + } + } +} + +} // namespace esphome::cdc_acm_uart +#endif diff --git a/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h new file mode 100644 index 0000000000..64522c86bd --- /dev/null +++ b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h @@ -0,0 +1,117 @@ +#pragma once +#if defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) +#include "esphome/components/uart/uart_component_esp_idf.h" +#include "esphome/components/usb_cdc_acm/usb_cdc_acm.h" +#include "esphome/core/component.h" + +#include +#include +#include "sdkconfig.h" + +namespace esphome::cdc_acm_uart { + +class CDCACMUARTBridge final : public Component { + public: + // Upper bound on the RX task's blocking read, so pause() takes effect without + // aborting the read. Arriving bytes still unblock it immediately. + static constexpr uint32_t UART_RX_WAIT_MS = 250; + + CDCACMUARTBridge(uart::IDFUARTComponent *uart_parent, usb_cdc_acm::USBCDCACMInstance *usb_cdc_parent) + : uart_parent_(uart_parent), usb_cdc_parent_(usb_cdc_parent) {} + + void setup() override; + void loop() override; + void dump_config() override; + void on_shutdown() override; + float get_setup_priority() const override { return setup_priority::HARDWARE; } + + void set_dtr_pin(GPIOPin *dtr_pin) { this->dtr_pin_ = dtr_pin; } + void set_rts_pin(GPIOPin *rts_pin) { this->rts_pin_ = rts_pin; } + + void set_line_coding(); + void set_line_state(bool dtr, bool rts); + + /** + * Stop forwarding in both directions and hand the UART back to its configured + * framing, so another component may use the bus. Main-loop only. The RX task parks + * within UART_RX_WAIT_MS (a byte it was already reading is discarded). A host write + * already in flight is allowed to drain first, which at low baud rates can take + * seconds; the framing is restored only after that, so poll is_paused() rather than + * waiting a fixed interval. Host bytes not yet written to the UART are discarded. + * The DTR/RTS outputs hold their state while paused and follow the host again on + * resume(). + */ + void pause(); + /** + * Re-apply the host's line coding and line state, then resume forwarding. Main-loop + * only. Deferred until any host write still draining has finished, so the reload + * never truncates it. + */ + void resume(); + /// True once both worker tasks are off the bus and the configured framing is restored. + /// With no RX task (setup() failed or has not run) there is nothing to wait for. + bool is_paused() const { + return this->state_ == MainState::MAIN_STATE_PAUSED && + (this->uart_rx_task_handle_ == nullptr || this->rx_parked_ != 0); + } + + protected: + static void uart_rx_task_fn(void *arg); + static void uart_tx_task_fn(void *arg); + void uart_rx_task_(); + void uart_tx_task_(); + void restore_configured_framing_(); + // True when the TX task has no write in flight and the UART TX FIFO has drained. + bool tx_idle_(); + void finish_resume_(); + void drive_line_state_(); + // Copy the host's line coding onto the UART settings; true if anything changed. + bool sync_host_framing_(); + + TaskHandle_t uart_rx_task_handle_{nullptr}; + TaskHandle_t uart_tx_task_handle_{nullptr}; + + GPIOPin *dtr_pin_{nullptr}; + GPIOPin *rts_pin_{nullptr}; + + uint32_t reload_requested_at_{0}; + + // Worker staging, each sized to the CDC ring buffer it feeds or drains. + std::array uart_rx_buffer_{}; + std::array uart_tx_buffer_{}; + + uart::IDFUARTComponent *uart_parent_; + usb_cdc_acm::USBCDCACMInstance *usb_cdc_parent_; + + // YAML framing, captured at setup; the host's line coding overwrites the UART's + // settings, so pause() needs the original to restore. + uint32_t configured_baud_rate_{0}; + uart::UARTParityOptions configured_parity_{uart::UART_CONFIG_PARITY_NONE}; + uint8_t configured_stop_bits_{0}; + uint8_t configured_data_bits_{0}; + + // Written on the main loop, read by both worker tasks. uint8_t rather than bool: + // GCC on Xtensa emits an out-of-line call for atomic. + std::atomic paused_{0}; + // Raised by the RX task while parked and by the TX task around each UART write, so + // the pause hand-off knows when the bus is actually free. + std::atomic rx_parked_{0}; + std::atomic tx_busy_{0}; + // Main-loop state; paused_ mirrors it for the worker tasks. + enum class MainState : uint8_t { + MAIN_STATE_RUNNING, + MAIN_STATE_RELOAD_PENDING, // host line coding debounced, forwarding continues + MAIN_STATE_PAUSING, // waiting for TX idle to restore the configured framing + MAIN_STATE_PAUSED, + MAIN_STATE_RESUMING, // resume() requested while a host write still drains + }; + MainState state_{MainState::MAIN_STATE_RUNNING}; + // Host line state, recorded even while paused so resume() can re-drive the pins. + bool host_dtr_{false}; + bool host_rts_{false}; + // True once the host has sent any line coding; resume() then re-syncs to it. + bool host_coding_seen_{false}; +}; + +} // namespace esphome::cdc_acm_uart +#endif diff --git a/esphome/components/usb_cdc_acm/usb_cdc_acm.h b/esphome/components/usb_cdc_acm/usb_cdc_acm.h index d8eb91586a..83cb5de89f 100644 --- a/esphome/components/usb_cdc_acm/usb_cdc_acm.h +++ b/esphome/components/usb_cdc_acm/usb_cdc_acm.h @@ -7,15 +7,47 @@ #include "esphome/core/lock_free_queue.h" #include "esphome/components/uart/uart_component.h" +#include #include +#include #include #include "freertos/ringbuf.h" +#include "esp_err.h" #include "tinyusb_cdc_acm.h" namespace esphome::usb_cdc_acm { static const uint8_t EVENT_QUEUE_SIZE = 12; +// Drain up to out_buf_sz bytes from a byte ring buffer, handling FreeRTOS's wrapped +// case with a second read. Shared with the cdc_acm_uart bridge platform, whose worker +// tasks drain the same ring buffers. +inline esp_err_t ringbuf_read_bytes(RingbufHandle_t ring_buf, uint8_t *out_buf, size_t out_buf_sz, size_t *rx_data_size, + TickType_t x_ticks_to_wait) { + size_t read_sz; + uint8_t *buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, x_ticks_to_wait, out_buf_sz)); + + if (buf == nullptr) { + return ESP_FAIL; + } + + memcpy(out_buf, buf, read_sz); + vRingbufferReturnItem(ring_buf, (void *) buf); + *rx_data_size = read_sz; + + // Buffer's data can be wrapped, in which case we should perform another read + if (*rx_data_size < out_buf_sz) { + buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, 0, out_buf_sz - *rx_data_size)); + if (buf != nullptr) { + memcpy(out_buf + *rx_data_size, buf, read_sz); + vRingbufferReturnItem(ring_buf, (void *) buf); + *rx_data_size += read_sz; + } + } + + return ESP_OK; +} + // Callback types for line coding and line state changes using LineCodingCallback = std::function; using LineStateCallback = std::function; @@ -103,6 +135,8 @@ class USBCDCACMInstance final : public uart::UARTComponent, public Parented usb_tx_staging_{}; // Non-zero while the TX task holds bytes it has pulled from the ring buffer but not // yet handed to TinyUSB; lets flush() account for data that is in neither the ring // buffer nor TinyUSB's FIFO. diff --git a/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp b/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp index e46369660d..7aa7b46b7b 100644 --- a/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp +++ b/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp @@ -104,30 +104,6 @@ static void tinyusb_cdc_line_coding_changed_callback(int itf, cdcacm_event_t *ev instance->queue_line_coding_event(bit_rate, stop_bits, parity, data_bits); } -static esp_err_t ringbuf_read_bytes(RingbufHandle_t ring_buf, uint8_t *out_buf, size_t out_buf_sz, size_t *rx_data_size, - TickType_t x_ticks_to_wait) { - size_t read_sz; - uint8_t *buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, x_ticks_to_wait, out_buf_sz)); - - if (buf == nullptr) { - return ESP_FAIL; - } - - memcpy(out_buf, buf, read_sz); - vRingbufferReturnItem(ring_buf, (void *) buf); - *rx_data_size = read_sz; - - // Buffer's data can be wrapped, in which case we should perform another read - buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, 0, out_buf_sz - *rx_data_size)); - if (buf != nullptr) { - memcpy(out_buf + *rx_data_size, buf, read_sz); - vRingbufferReturnItem(ring_buf, (void *) buf); - *rx_data_size += read_sz; - } - - return ESP_OK; -} - //============================================================================== // USBCDCACMInstance Implementation //============================================================================== @@ -192,7 +168,7 @@ void USBCDCACMInstance::usb_tx_task_fn(void *arg) { } void USBCDCACMInstance::usb_tx_task() { - uint8_t data[CONFIG_TINYUSB_CDC_TX_BUFSIZE] = {0}; + uint8_t *data = this->usb_tx_staging_.data(); size_t tx_data_size = 0; // Back-dated so a stall within the first LOG_THROTTLE_MS of uptime still logs // immediately (unsigned arithmetic keeps this wrap-safe). diff --git a/script/analyze_component_buses.py b/script/analyze_component_buses.py index b8ee3066bd..b805d5155a 100755 --- a/script/analyze_component_buses.py +++ b/script/analyze_component_buses.py @@ -81,6 +81,7 @@ ISOLATED_SIGNATURE_PREFIX = "isolated_" # NOTE: This should be kept in sync with both test_build_components and split_components_for_ci.py ISOLATED_COMPONENTS = { "animation": "Has display lambda in common.yaml that requires existing display platform - breaks when merged without display", + "cdc_acm_uart": "Depends on tinyusb which conflicts with usb_host", "esphome": "Defines devices/areas in esphome: section that are referenced in other sections - breaks when merged", "ethernet": "Defines ethernet: which conflicts with wifi: used by most components", "ethernet_info": "Related to ethernet component which conflicts with wifi", diff --git a/tests/component_tests/cdc_acm_uart/__init__.py b/tests/component_tests/cdc_acm_uart/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/cdc_acm_uart/test_init.py b/tests/component_tests/cdc_acm_uart/test_init.py new file mode 100644 index 0000000000..7bbf163fc3 --- /dev/null +++ b/tests/component_tests/cdc_acm_uart/test_init.py @@ -0,0 +1,154 @@ +"""Tests for the bridge cdc_acm_uart platform's final validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.cdc_acm_uart import bridge +from esphome.components.cdc_acm_uart.bridge import CONF_USB_CDC_ACM_ID +from esphome.config import Config +from esphome.const import CONF_DEBUG, CONF_ID, CONF_UART_ID, PlatformFramework +from esphome.core import ID +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + +_final_validate = bridge._final_validate + + +def _set_esp32_s3(set_core_config: SetCoreConfigCallable, **kwargs) -> None: + from esphome.components.esp32 import KEY_VARIANT, VARIANT_ESP32S3 + + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_VARIANT: VARIANT_ESP32S3}, + **kwargs, + ) + + +def _full_config(uarts: list[ConfigType] | None = None, **domains) -> Config: + """A full config declaring uart_0 and uart_1 (plus any extra entries), as the ID + pass leaves it, so the debug check can resolve a uart_id to its declaration.""" + uarts = uarts or [{CONF_ID: ID("uart_0")}, {CONF_ID: ID("uart_1")}] + full = Config() + full["uart"] = uarts + for index, uart_conf in enumerate(uarts): + full.declare_ids.append((uart_conf[CONF_ID], ["uart", index, CONF_ID])) + full.update(domains) + return full + + +def _bridge_config(uart_id: str, cdc_id: str) -> dict: + return {CONF_UART_ID: ID(uart_id), CONF_USB_CDC_ACM_ID: ID(cdc_id)} + + +def test_accepts_distinct_uart_and_cdc_interfaces( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3(set_core_config, full_config=_full_config()) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + _final_validate(_bridge_config("uart_1", "cdc_acm_2")) + + +def test_rejects_two_bridges_sharing_a_uart( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3(set_core_config, full_config=_full_config()) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + with pytest.raises(cv.Invalid, match="already bridged"): + _final_validate(_bridge_config("uart_0", "cdc_acm_2")) + + +def test_rejects_two_bridges_sharing_a_cdc_interface( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3(set_core_config, full_config=_full_config()) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + with pytest.raises(cv.Invalid, match="already bridged"): + _final_validate(_bridge_config("uart_1", "cdc_acm_1")) + + +def test_rejects_uart_shared_with_another_component( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3( + set_core_config, + full_config=_full_config( + sensor=[{"platform": "pzemac", CONF_UART_ID: ID("uart_0")}], + ), + ) + with pytest.raises(cv.Invalid, match="exclusive"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_rejects_cdc_interface_shared_with_another_component( + set_core_config: SetCoreConfigCallable, +) -> None: + # The CDC instance is itself a uart::UARTComponent, so other components can bind + # it as a plain UART via uart_id -- that must be rejected just like UART sharing. + _set_esp32_s3( + set_core_config, + full_config=_full_config( + sensor=[{"platform": "pzemac", CONF_UART_ID: ID("cdc_acm_1")}], + ), + ) + with pytest.raises(cv.Invalid, match="exclusive"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_rejects_uart_referenced_from_nested_config( + set_core_config: SetCoreConfigCallable, +) -> None: + # References can sit arbitrarily deep, e.g. inside an automation's action list. + _set_esp32_s3( + set_core_config, + full_config=_full_config( + binary_sensor=[ + { + "platform": "gpio", + "on_press": [{"then": [{CONF_UART_ID: ID("uart_0")}]}], + } + ], + ), + ) + with pytest.raises(cv.Invalid, match="exclusive"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_ignores_other_components_on_other_uarts( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3( + set_core_config, + full_config=_full_config( + sensor=[{"platform": "pzemac", CONF_UART_ID: ID("uart_1")}], + # The bridge domain itself is skipped: this bridge's own entry (and any + # bridge-vs-bridge sharing, which the seen-set already rejects) must not + # trip the exclusivity scan. + bridge=[_bridge_config("uart_0", "cdc_acm_1")], + ), + ) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_rejects_debug_on_bridged_uart( + set_core_config: SetCoreConfigCallable, +) -> None: + # The bridge talks to the IDF driver directly, so the uart debugger would see + # nothing and its dummy_receiver would steal RX bytes. + _set_esp32_s3( + set_core_config, + full_config=_full_config(uarts=[{CONF_ID: ID("uart_0"), CONF_DEBUG: {}}]), + ) + with pytest.raises(cv.Invalid, match="debug"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_allows_debug_on_other_uart( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3( + set_core_config, + full_config=_full_config( + uarts=[{CONF_ID: ID("uart_0")}, {CONF_ID: ID("uart_1"), CONF_DEBUG: {}}] + ), + ) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) diff --git a/tests/component_tests/conftest.py b/tests/component_tests/conftest.py index 4f0b786cc2..b5eceeedf6 100644 --- a/tests/component_tests/conftest.py +++ b/tests/component_tests/conftest.py @@ -60,7 +60,7 @@ def reset_core() -> Generator[None]: @pytest.fixture(autouse=True) def reset_full_config() -> Generator[None]: """Give each test a clean final-validate config and restore it after.""" - token = final_validate.full_config.set({}) + token = final_validate.full_config.set(Config()) yield final_validate.full_config.reset(token) @@ -75,7 +75,7 @@ def set_core_config() -> Generator[SetCoreConfigCallable]: *, core_data: ConfigType | None = None, platform_data: ConfigType | None = None, - full_config: dict[str, ConfigType] | None = None, + full_config: dict[str, ConfigType] | Config | None = None, ) -> None: platform, framework = platform_framework.value @@ -94,7 +94,12 @@ def set_core_config() -> Generator[SetCoreConfigCallable]: CORE.data[platform.value] = platform_data config.path_context.set([]) - final_validate.full_config.set(full_config or Config()) + # Production always installs a Config (a FinalValidateConfig), never a plain dict. + if not isinstance(full_config, Config): + full = Config() + full.update(full_config or {}) + full_config = full + final_validate.full_config.set(full_config) yield setter diff --git a/tests/component_tests/types.py b/tests/component_tests/types.py index ee9d317339..3587517bde 100644 --- a/tests/component_tests/types.py +++ b/tests/component_tests/types.py @@ -4,6 +4,7 @@ from __future__ import annotations from typing import Protocol +from esphome.config import Config from esphome.const import PlatformFramework from esphome.types import ConfigType @@ -18,5 +19,5 @@ class SetCoreConfigCallable(Protocol): *, core_data: ConfigType | None = None, platform_data: ConfigType | None = None, - full_config: dict[str, ConfigType] | None = None, + full_config: dict[str, ConfigType] | Config | None = None, ) -> None: ... diff --git a/tests/components/cdc_acm_uart/common.yaml b/tests/components/cdc_acm_uart/common.yaml new file mode 100644 index 0000000000..6c43dfc18b --- /dev/null +++ b/tests/components/cdc_acm_uart/common.yaml @@ -0,0 +1,18 @@ +tinyusb: + id: tinyusb_test + usb_lang_id: 0x0123 + usb_manufacturer_str: ESPHomeTestManufacturer + usb_product_id: 0x1234 + usb_product_str: ESPHomeTestProduct + usb_serial_str: ESPHomeTestSerialNumber + usb_vendor_id: 0x2345 + +uart: + - id: uart_0 + tx_pin: 14 + rx_pin: 13 + baud_rate: 115200 + +usb_cdc_acm: + interfaces: + - id: cdc_acm_1 diff --git a/tests/components/cdc_acm_uart/common_dual.yaml b/tests/components/cdc_acm_uart/common_dual.yaml new file mode 100644 index 0000000000..0ce817fbc2 --- /dev/null +++ b/tests/components/cdc_acm_uart/common_dual.yaml @@ -0,0 +1,12 @@ +# Second UART/CDC pair for a two-bridge setup. Kept out of common.yaml because the +# ESP32-S2 has only two UART controllers and the logger occupies one, so a second +# uart there would fail at runtime. +uart: + - id: uart_1 + tx_pin: 15 + rx_pin: 16 + baud_rate: 115200 + +usb_cdc_acm: + interfaces: + - id: cdc_acm_2 diff --git a/tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml b/tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml new file mode 100644 index 0000000000..aa9ec8079f --- /dev/null +++ b/tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml @@ -0,0 +1,15 @@ +packages: + cdc_acm_uart: !include common.yaml + cdc_acm_uart_dual: !include common_dual.yaml + +bridge: + - platform: cdc_acm_uart + uart_id: uart_0 + usb_cdc_acm_id: cdc_acm_1 + dtr_pin: 40 + rts_pin: 41 + - platform: cdc_acm_uart + uart_id: uart_1 + usb_cdc_acm_id: cdc_acm_2 + dtr_pin: 20 + rts_pin: 21 diff --git a/tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml b/tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml new file mode 100644 index 0000000000..0beeb80bfa --- /dev/null +++ b/tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml @@ -0,0 +1,14 @@ +# ESP32-S2 has no USB_SERIAL_JTAG, so the logger defaults to USB_CDC, which shares +# the USB OTG peripheral with tinyusb. Use a hardware UART for logging instead. +logger: + hardware_uart: UART0 + +packages: + cdc_acm_uart: !include common.yaml + +bridge: + - platform: cdc_acm_uart + uart_id: uart_0 + usb_cdc_acm_id: cdc_acm_1 + dtr_pin: 40 + rts_pin: 41 diff --git a/tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml b/tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml new file mode 100644 index 0000000000..cbb1fc2a3a --- /dev/null +++ b/tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml @@ -0,0 +1,17 @@ +packages: + cdc_acm_uart: !include common.yaml + cdc_acm_uart_dual: !include common_dual.yaml + +bridge: + - platform: cdc_acm_uart + uart_id: uart_0 + usb_cdc_acm_id: cdc_acm_1 + dtr_pin: 40 + rts_pin: 41 + - platform: cdc_acm_uart + uart_id: uart_1 + usb_cdc_acm_id: cdc_acm_2 + # GPIO19/20 are USB D-/D+ on the S3 (which the CDC side itself uses); use + # unrelated free pins here. + dtr_pin: 17 + rts_pin: 18 From 533002c41e58d48fbc87b65e7c13122996ff5ff2 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:12:08 +1000 Subject: [PATCH 121/178] [lvgl] Fix crash when using lvgl.list.add (#19177) --- esphome/components/lvgl/widgets/lv_list.py | 4 ++++ tests/components/lvgl/lvgl-package.yaml | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/esphome/components/lvgl/widgets/lv_list.py b/esphome/components/lvgl/widgets/lv_list.py index 83cbfb5ef9..7711e8bfe4 100644 --- a/esphome/components/lvgl/widgets/lv_list.py +++ b/esphome/components/lvgl/widgets/lv_list.py @@ -227,6 +227,7 @@ LIST_ID_SCHEMA = cv.Schema({cv.Required(CONF_ID): cv.use_id(lv_list_t)}) ) async def list_add_text_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_add_text(w: Widget): text = await lv_text.process(config[CONF_TEXT]) @@ -370,6 +371,7 @@ async def list_add_to_code(config, action_id, template_arg, args): _register_lv_uses(w_type_name, w_conf) _register_dynamic_widget_style_uses(w_conf) widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_add(w: Widget): index = None @@ -503,6 +505,7 @@ LIST_REMOVE_SCHEMA = LIST_ID_SCHEMA.extend( ) async def list_remove_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_remove(w: Widget): index = await lv_int.process(config[CONF_INDEX]) @@ -536,6 +539,7 @@ async def list_remove_to_code(config, action_id, template_arg, args): ) async def list_clear_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_clear(w: Widget): await _wait_list_triggers_completed() diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 07c492db35..bd2e77ee8c 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -30,6 +30,18 @@ binary_sensor: widget: button_button state: pressed +globals: + - id: counter + type: int + +script: + - id: add_row + then: + - lvgl.list.add: + id: test_list_id + label: + text: row + lvgl: id: lvgl_id rotation: 90 @@ -1291,7 +1303,7 @@ lvgl: then: - logger.log: format: "table selected row %u col %u" - args: [row, column] + args: [(unsigned)row, (unsigned)column] on_click: then: - lvgl.table.cell.update: @@ -1347,10 +1359,12 @@ lvgl: - logger.log: format: "list entry added at %d" args: [list_index] + - lambda: "id(counter)++;" on_remove: - logger.log: format: "list entry removed at %d" args: [list_index] + - lambda: "id(counter)--;" on_click: - lvgl.list.add_text: id: test_list_id From 5bbfe12e4ff5bc24603f1d9e23c6cade81f2096e Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:38:27 +1000 Subject: [PATCH 122/178] [core] Isolate contextvars per task in the coroutine runner (#19238) --- esphome/coroutine.py | 18 ++++++++++-- tests/unit_tests/test_coroutine.py | 45 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/esphome/coroutine.py b/esphome/coroutine.py index 3ce94cc979..8a82536227 100644 --- a/esphome/coroutine.py +++ b/esphome/coroutine.py @@ -45,6 +45,7 @@ the last `yield` expression defines what is returned. from __future__ import annotations from collections.abc import Awaitable, Callable, Generator, Iterator +import contextvars import enum import functools import heapq @@ -277,14 +278,22 @@ class _Task: id_number: int, iterator: Iterator[None], original_function: Any, + context: contextvars.Context, ): self.priority = priority self.id_number = id_number self.iterator = iterator self.original_function = original_function + self.context = context def with_priority(self, priority: float) -> _Task: - return _Task(priority, self.id_number, self.iterator, self.original_function) + return _Task( + priority, + self.id_number, + self.iterator, + self.original_function, + self.context, + ) @property def _cmp_tuple(self) -> tuple[float, int]: @@ -321,7 +330,10 @@ class FakeEventLoop: coro = coroutine(func) gen = coro(*args, **kwargs) prio = getattr(coro, "priority", 0.0) - task = _Task(prio, self._task_counter, gen, func) + # Each task gets its own copy of the current context, isolating any + # contextvars it sets from other tasks the scheduler interleaves it with + # (mirrors what asyncio.Task does internally). + task = _Task(prio, self._task_counter, gen, func, contextvars.copy_context()) self._task_counter += 1 heapq.heappush(self._pending_tasks, task) @@ -352,7 +364,7 @@ class FakeEventLoop: ) try: - next(task.iterator) + task.context.run(next, task.iterator) # Decrease priority over time, so that if this task is blocked # due to a dependency others will clear the dependency # This could be improved with a less naive approach diff --git a/tests/unit_tests/test_coroutine.py b/tests/unit_tests/test_coroutine.py index e12c273294..0a8fb59cb8 100644 --- a/tests/unit_tests/test_coroutine.py +++ b/tests/unit_tests/test_coroutine.py @@ -1,5 +1,7 @@ """Tests for the coroutine module.""" +import contextvars + import pytest from esphome.coroutine import CoroPriority, FakeEventLoop, coroutine_with_priority @@ -217,3 +219,46 @@ def test_custom_priority_between_enum_values() -> None: # Check execution order assert execution_order == ["core", "custom", "diagnostics"] + + +def test_context_isolated_between_interleaved_tasks() -> None: + """Test that a contextvar set in one task does not leak into another task that the scheduler interleaves with it.""" + my_var: contextvars.ContextVar[str] = contextvars.ContextVar("my_var") + seen: dict[str, str] = {} + + def task_a(): + my_var.set("a") + yield # suspend so task_b can run before task_a resumes + seen["a"] = my_var.get() + + def task_b(): + my_var.set("b") + yield + seen["b"] = my_var.get() + + loop = FakeEventLoop() + loop.add_job(task_a) + loop.add_job(task_b) + loop.flush_tasks() + + assert seen == {"a": "a", "b": "b"} + + +def test_context_inherits_ambient_value_at_schedule_time() -> None: + """Test that a job sees whatever contextvar value was set before it was scheduled.""" + my_var: contextvars.ContextVar[str] = contextvars.ContextVar("my_var") + token = my_var.set("ambient") + seen: dict[str, str] = {} + + def task(): + seen["value"] = my_var.get() + yield + + try: + loop = FakeEventLoop() + loop.add_job(task) + loop.flush_tasks() + finally: + my_var.reset(token) + + assert seen == {"value": "ambient"} From eea66fc32b3b971d55573c69939d1885035a2d93 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 08:06:35 -0500 Subject: [PATCH 123/178] Bump bundled esphome-device-builder to 1.14.8 (#19250) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f500dbe6f..bdbbe798ce 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.7 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.8 RUN \ platformio settings set enable_telemetry No \ From 6b08aa60e660a238ae710415827fdd7bcd1d8438 Mon Sep 17 00:00:00 2001 From: David Coulson <23066302+davidcoulson@users.noreply.github.com> Date: Sun, 13 Sep 2026 12:25:27 -0400 Subject: [PATCH 124/178] [bluetooth_proxy] Add an advertisement filter hook (#19220) Co-authored-by: Claude Opus 5 Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../components/bluetooth_proxy/__init__.py | 11 +++++++ .../bluetooth_proxy/bluetooth_proxy.cpp | 12 +++++++ .../bluetooth_proxy/bluetooth_proxy.h | 32 +++++++++++++++++++ esphome/core/defines.h | 2 ++ .../test_advertisement_filter.py | 13 ++++++++ ...est-advertisement-filter.esp32-s3-idf.yaml | 12 +++++++ 6 files changed, 82 insertions(+) create mode 100644 tests/component_tests/bluetooth_proxy/test_advertisement_filter.py create mode 100644 tests/components/bluetooth_proxy/test-advertisement-filter.esp32-s3-idf.yaml diff --git a/esphome/components/bluetooth_proxy/__init__.py b/esphome/components/bluetooth_proxy/__init__.py index 1b761849a5..c87ad7f595 100644 --- a/esphome/components/bluetooth_proxy/__init__.py +++ b/esphome/components/bluetooth_proxy/__init__.py @@ -395,6 +395,17 @@ async def _to_code_ble_hub(config: ConfigType) -> None: await _connections_to_code(var, config) +def enable_advertisement_filter() -> None: + """Compile the advertisement filter hook into bluetooth_proxy. + + Called by external filtering components from to_code(). The define behind + this is an implementation detail; do not emit it directly. + + Public API for external components. Do not remove. + """ + cg.add_define("USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER") + + async def to_code(config: ConfigType) -> None: if CORE.is_esp32: await _to_code_esp32(config) diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index 878d3cd44e..cb37057cd4 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -94,6 +94,15 @@ void BluetoothProxy::on_raw_advertisement_(const ble_device_base::RawAdvertiseme if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) return; +#ifdef USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER + // Ask the filter before the packet is queued, so a dropped advertisement never + // reaches the batch or the network. + if (this->advertisement_filter_.is_set() && !this->advertisement_filter_.should_forward(raw)) { + ESP_LOGVV(TAG, "Filtered packet from %012" PRIX64, raw.address); + return; + } +#endif + auto &adv = this->response_.advertisements[this->response_.advertisements_len]; adv.address = raw.address; adv.rssi = raw.rssi; @@ -184,6 +193,9 @@ void BluetoothProxy::dump_config() { " Adapter MAC: %s", scan_mode, mac_out); #endif +#ifdef USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER + ESP_LOGCONFIG(TAG, " Advertisement filter: %s", YESNO(this->advertisement_filter_.is_set())); +#endif } #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index e233c38b56..567109dc60 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -97,6 +97,29 @@ static_assert(pending_reply_round_trips(0xABCD112233445566ULL, 0x000011223344556 static_assert(PendingReply{}.empty()); #endif +#ifdef USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER +/// Predicate slot letting an external component drop advertisements before they +/// are queued for the API. Same shape as +/// ble_device_base::RawAdvertisementCallback. Runs on the advertisement hot +/// path, so it must be cheap and must not block. +/// +/// Usage: +/// proxy->set_advertisement_filter({this, [](void *self, const ble_device_base::RawAdvertisement &adv) { +/// return static_cast(self)->should_forward(adv); +/// }}); +/// +/// Returning false drops the advertisement. Not called at all while the API is +/// disconnected, which matters to a stateful filter. Compiled in only when an +/// external component calls bluetooth_proxy.enable_advertisement_filter(). +struct AdvertisementFilter { + void *instance{nullptr}; + bool (*fn)(void *instance, const ble_device_base::RawAdvertisement &adv){nullptr}; + /// A default-constructed slot is "no filter"; the proxy guards on this. + bool is_set() const { return this->fn != nullptr; } + bool should_forward(const ble_device_base::RawAdvertisement &adv) const { return this->fn(this->instance, adv); } +}; +#endif // USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER + class BluetoothProxy final : public Component { #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS // Allow the connection to update connections_free_response_ @@ -162,6 +185,11 @@ class BluetoothProxy final : public Component { void set_active(bool active) { this->active_ = active; } bool has_active() { return this->active_; } +#ifdef USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER + /// One subscriber; a later call replaces an earlier one. + void set_advertisement_filter(AdvertisementFilter filter) { this->advertisement_filter_ = filter; } +#endif + uint32_t get_legacy_version() const { if (!this->active_) { return LEGACY_PASSIVE_ONLY_VERSION; @@ -330,6 +358,10 @@ class BluetoothProxy final : public Component { // start on an even word, closing two alignment holes. uint32_t last_advertisement_flush_time_{0}; +#ifdef USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER + AdvertisementFilter advertisement_filter_{}; +#endif + // BLE advertisement batching api::BluetoothLERawAdvertisementsResponse response_; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 6b9b9eda43..fe06cc3418 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -329,6 +329,8 @@ #else #define BLUETOOTH_PROXY_MAX_CONNECTIONS 0 #endif +// Defined here so static analysis parses the slot and its call site. +#define USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER #define BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE 16 #endif diff --git a/tests/component_tests/bluetooth_proxy/test_advertisement_filter.py b/tests/component_tests/bluetooth_proxy/test_advertisement_filter.py new file mode 100644 index 0000000000..84d8b0677f --- /dev/null +++ b/tests/component_tests/bluetooth_proxy/test_advertisement_filter.py @@ -0,0 +1,13 @@ +"""The codegen hook external filtering components use to turn on the filter slot.""" + +from esphome.components import bluetooth_proxy +from esphome.core import CORE + + +def test_enable_advertisement_filter_emits_define() -> None: + """External components call this rather than emitting the define.""" + bluetooth_proxy.enable_advertisement_filter() + + assert "USE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER" in { + define.name for define in CORE.defines + } diff --git a/tests/components/bluetooth_proxy/test-advertisement-filter.esp32-s3-idf.yaml b/tests/components/bluetooth_proxy/test-advertisement-filter.esp32-s3-idf.yaml new file mode 100644 index 0000000000..f46f4814c2 --- /dev/null +++ b/tests/components/bluetooth_proxy/test-advertisement-filter.esp32-s3-idf.yaml @@ -0,0 +1,12 @@ +# Compile the gated filter path; no external component is in-tree to call +# enable_advertisement_filter(), so the define is forced here. +<<: !include common.yaml + +esphome: + build_flags: + - "-DUSE_BLUETOOTH_PROXY_ADVERTISEMENT_FILTER" + +esp32_ble_tracker: + +bluetooth_proxy: + active: true From fc8611a2122c77f94861d7b320a30b86654cbe74 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:14:26 -0500 Subject: [PATCH 125/178] [noise] Bump noise-c to 0.1.30 and libsodium to 1.10021.11 (#19062) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index d17ebf235e..6067fde164 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.26") + cg.add_library("esphome/noise-c", "0.1.30") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.8") + cg.add_library("esphome/libsodium", "1.10021.11") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 738773d1b5..0e334ac5b4 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.26 ; noise (api, ota) + esphome/noise-c@0.1.30 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.26 ; noise (api, ota) + esphome/noise-c@0.1.30 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.26 ; used by noise (api, ota) + esphome/noise-c@0.1.30 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 00f22ca138..0dce00785b 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 1.0") == "noise-c" + assert mod.spec_key("esphome/noise-c@1.0") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.26\n" + " esphome/noise-c @ 1.0\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.26\n" + " esphome/noise-c @ 1.0\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.26"] + assert libs == ["esphome/noise-c @ 1.0"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 1.0", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.26", - "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 1.0", + "esphome/noise-c @ 1.0", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.26"] + assert cls.calls == ["esphome/noise-c @ 1.0"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.26": [ + "esphome/noise-c @ 1.0": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 1.0", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.26"] is None + assert compats["esphome/noise-c @ 1.0"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.26": [ + "esphome/noise-c @ 1.0": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) + mod.parallel_install(cls, ["esphome/noise-c @ 1.0"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 1.0"}) cls.deps = { - "esphome/noise-c @ 0.1.26": [ + "esphome/noise-c @ 1.0": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) + mod.parallel_install(cls, ["esphome/noise-c @ 1.0"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index b03bff19a2..774493ecf4 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@1.0", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@1.0", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 4f0faac1486d03e33f0e18bef2301ea77a3c31db Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:16:37 -0500 Subject: [PATCH 126/178] [core] Add FixedVector::try_init so callers can handle an exhausted heap (#19253) --- esphome/core/helpers.h | 52 ++++++++++++++++++++------ script/cpp_unit_test.py | 3 +- tests/components/core/test_helpers.cpp | 19 ++++++++++ 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index a0afb03124..987c54a5b0 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #endif #ifdef USE_ESP32 +#include #include #endif @@ -539,7 +541,15 @@ template inline void init_array_from(std::array &des } } -/// Fixed-capacity vector - allocates once at runtime, never reallocates +// Abort with a reason that reaches the panic output on ESP32. Elsewhere the literal is dropped +// before it can land in rodata, which is RAM on ESP8266 +#ifdef USE_ESP32 +#define ESPHOME_ABORT_WITH_REASON(reason) esp_system_abort(reason) +#else +#define ESPHOME_ABORT_WITH_REASON(reason) abort() +#endif + +/// Fixed-capacity vector - sized once through init() or try_init(); push_back never reallocates /// This avoids std::vector template overhead (_M_realloc_insert, _M_default_append) /// when size is known at initialization but not at compile time template class FixedVector { @@ -562,8 +572,7 @@ template class FixedVector { void cleanup_() { if (data_ != nullptr) { destroy_elements_(); - // Free raw memory - ::operator delete(data_); + free(data_); // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) } } @@ -632,16 +641,27 @@ template class FixedVector { // Allocate capacity - can be called multiple times to reinit // IMPORTANT: After calling init(), you MUST use push_back() to add elements. // Direct assignment via operator[] does NOT update the size counter. + // Aborts on exhaustion; use try_init() to handle failure. void init(size_t n) { + if (!try_init(n)) + ESPHOME_ABORT_WITH_REASON("FixedVector: out of memory"); + } + + // Same as init(), but returns false when memory is exhausted; the previous storage is freed either way + bool try_init(size_t n) { cleanup_(); reset_(); - if (n > 0) { - // Allocate raw memory without calling constructors - // sizeof(T) is correct here for any type T (value types, pointers, etc.) - // NOLINTNEXTLINE(bugprone-sizeof-expression) - data_ = static_cast(::operator new(n * sizeof(T))); - capacity_ = n; - } + if (n == 0) + return true; + if (n > SIZE_MAX / sizeof(T)) + return false; // the byte count would wrap into a small block + // sizeof(T) is correct here for any type T (value types, pointers, etc.) + // NOLINTNEXTLINE(bugprone-sizeof-expression,cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory) + data_ = static_cast(malloc(n * sizeof(T))); + if (data_ == nullptr) + return false; + capacity_ = n; + return true; } // Clear the vector (destroy all elements, reset size to 0, keep capacity) @@ -738,14 +758,22 @@ template class FixedVector { template class SmallBufferWithHeapFallback { public: explicit SmallBufferWithHeapFallback(size_t size) { + static_assert(std::is_trivially_default_constructible_v && std::is_trivially_destructible_v, + "the heap fallback leaves elements unconstructed"); if (size <= STACK_SIZE) { this->buffer_ = this->stack_buffer_; } else { - this->heap_buffer_ = new T[size]; + if (size <= SIZE_MAX / sizeof(T)) { + // NOLINTNEXTLINE(bugprone-sizeof-expression,cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory) + this->heap_buffer_ = static_cast(malloc(size * sizeof(T))); + } + // Callers write through get() unchecked, so exhaustion aborts like the new[] it replaces + if (this->heap_buffer_ == nullptr) + ESPHOME_ABORT_WITH_REASON("SmallBufferWithHeapFallback: out of memory"); this->buffer_ = this->heap_buffer_; } } - ~SmallBufferWithHeapFallback() { delete[] this->heap_buffer_; } + ~SmallBufferWithHeapFallback() { free(this->heap_buffer_); } // NOLINT(cppcoreguidelines-no-malloc) // Delete copy and move operations to prevent double-delete SmallBufferWithHeapFallback(const SmallBufferWithHeapFallback &) = delete; diff --git a/script/cpp_unit_test.py b/script/cpp_unit_test.py index f8bab39414..8cb18d0875 100755 --- a/script/cpp_unit_test.py +++ b/script/cpp_unit_test.py @@ -36,7 +36,8 @@ PLATFORMIO_OPTIONS = { def run_tests(selected_components: list[str]) -> int: - os.environ["ASAN_OPTIONS"] = "detect_leaks=0" + # allocator_may_return_null: an oversized request must come back empty, not abort the run + os.environ["ASAN_OPTIONS"] = "detect_leaks=0:allocator_may_return_null=1" return build_and_run( selected_components=selected_components, tests_dir=COMPONENTS_TESTS_DIR, diff --git a/tests/components/core/test_helpers.cpp b/tests/components/core/test_helpers.cpp index baf688fc8a..d6b31508d1 100644 --- a/tests/components/core/test_helpers.cpp +++ b/tests/components/core/test_helpers.cpp @@ -348,4 +348,23 @@ TEST(StepToAccuracyDecimals, NonFiniteAndZero) { EXPECT_EQ(step_to_accuracy_decimals(-INFINITY), 0); } +// --- FixedVector::try_init() --- + +// Keeps the block observable, else the compiler may drop the malloc and free pair and fold the check +static void escape(const void *p) { asm volatile("" : : "g"(p) : "memory"); } + +TEST(FixedVectorTryInit, ReportsExhaustionAndStaysEmpty) { + FixedVector v; + const bool ok = v.try_init(SIZE_MAX / sizeof(uint32_t)); + escape(&v); + EXPECT_FALSE(ok); + EXPECT_EQ(v.capacity(), 0u); + EXPECT_FALSE(v.try_init(SIZE_MAX / sizeof(uint32_t) + 1)); // byte count would wrap + EXPECT_EQ(v.capacity(), 0u); + EXPECT_TRUE(v.try_init(0)); + EXPECT_TRUE(v.try_init(4)); + v.push_back(7); + EXPECT_EQ(v.size(), 1u); +} + } // namespace esphome::core::testing From ce8fad14359660e46aa999fa7c872a54fa47dea7 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:44:03 +0000 Subject: [PATCH 127/178] Bump bundled esphome-device-builder to 1.14.9 (#19263) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bdbbe798ce..e00570c8ff 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.8 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.9 RUN \ platformio settings set enable_telemetry No \ From b1bfc512ac1fd41a29570eab641d2d3458de96c2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:49:13 -0500 Subject: [PATCH 128/178] [wifi] Drop a scan instead of aborting when its results cannot be allocated, filter ESP32 scans by SSID in the driver (#19254) --- esphome/components/wifi/__init__.py | 3 + esphome/components/wifi/wifi_component.cpp | 6 +- esphome/components/wifi/wifi_component.h | 16 ++++-- .../wifi/wifi_component_esp8266.cpp | 6 +- .../wifi/wifi_component_esp_idf.cpp | 57 +++++++++++++++---- .../wifi/wifi_component_libretiny.cpp | 6 +- esphome/core/defines.h | 2 + 7 files changed, 74 insertions(+), 22 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 58803a8cdf..1e57c03b7b 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -626,6 +626,9 @@ async def to_code(config): networks = config.get(CONF_NETWORKS, []) if networks: cg.add(var.init_sta(len(networks))) + if len(networks) > 1: + # The ESP32 scan can filter one SSID in the driver; with several the whole list is kept + cg.add_define("USE_WIFI_MULTI_SSID") def add_sta(ap: cg.MockObj, network: dict) -> None: ip_config = network.get(CONF_MANUAL_IP, config.get(CONF_MANUAL_IP)) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index f9e80995e1..5ba3614394 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -1499,8 +1499,8 @@ void WiFiComponent::check_scanning_finished() { return; } this->scan_done_ = false; - this->has_completed_scan_after_captive_portal_start_ = - true; // Track that we've done a scan since captive portal started + // A driver filtered scan saw one SSID; a portal that started during it still needs a full scan + this->has_completed_scan_after_captive_portal_start_ = !this->is_scan_driver_filtered_(); this->retry_hidden_mode_ = RetryHiddenMode::SCAN_BASED; if (this->scan_result_.empty()) { @@ -2416,7 +2416,7 @@ void WiFiComponent::handle_driver_roam_(const bssid_t &bssid, uint8_t channel) { void WiFiComponent::release_scan_results_() { if (!this->keep_scan_results_) { ScanResultsLock lock(this); -#if defined(USE_RP2) || defined(USE_ESP32) +#if defined(USE_RP2) // std::vector - use swap trick since shrink_to_fit is non-binding decltype(this->scan_result_)().swap(this->scan_result_); #else diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 94fdd9bc14..77a4773a27 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -178,12 +178,12 @@ struct EAPAuth { using bssid_t = std::array; -/// Initial reserve size for filtered scan results (typical: 1-3 matching networks per SSID) -static constexpr size_t WIFI_SCAN_RESULT_FILTERED_RESERVE = 8; +// ESP32 with one configured network: the driver filters the scan by its SSID and only this many of +// its BSSIDs are kept, the strongest ones +static constexpr size_t WIFI_SCAN_RESULT_BOUND = 12; -// Use std::vector for RP2040 (callback-based) and ESP32 (destructive scan API) -// Use FixedVector for ESP8266 and LibreTiny where two-pass exact allocation is possible -#if defined(USE_RP2) || defined(USE_ESP32) +// RP2040's callback delivers results one at a time with no count, so it needs a growable vector +#if defined(USE_RP2) template using wifi_scan_vector_t = std::vector; #else template using wifi_scan_vector_t = FixedVector; @@ -954,6 +954,12 @@ class WiFiComponent final : public Component { uint8_t num_ipv6_addresses_{0}; #endif /* USE_NETWORK_IPV6 */ bool error_from_callback_{false}; +#if defined(USE_ESP32) && !defined(USE_WIFI_MULTI_SSID) + bool scan_driver_filtered_{false}; + bool is_scan_driver_filtered_() const { return this->scan_driver_filtered_; } +#else + constexpr bool is_scan_driver_filtered_() const { return false; } +#endif #if defined(USE_ESP8266) || defined(USE_LIBRETINY) // Platform-specific STA state enum, defined in platform cpp file. // On ESP8266, written from SDK system context (wifi_event_callback) — diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 031da1b355..60ec3f9a4d 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -773,7 +773,11 @@ void WiFiComponent::wifi_scan_done_callback_(void *arg, STATUS status) { } } - this->scan_result_.init(count); // Exact allocation + if (!this->scan_result_.try_init(count)) { + ESP_LOGW(TAG, "No memory for %zu scan results", count); + this->scan_done_ = true; + return; + } // Second pass: store matching networks for (bss_info *it = head; it != nullptr; it = STAILQ_NEXT(it, next)) { diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index ce75d21330..24bf64a99c 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -909,7 +909,8 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { ESP_LOGV(TAG, "Scan done: status=%" PRIu32 " number=%u scan_id=%u", it.status, it.number, it.scan_id); uint16_t number = it.number; - bool needs_full = this->needs_full_scan_results_(); + const bool filtered = this->is_scan_driver_filtered_(); + const bool needs_full = this->needs_full_scan_results_(); { // Mutate in place under the lock; blocking a portal request is fine and // avoids scratch buffers @@ -926,8 +927,14 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { return; } - // Smart reserve: full capacity if needed, small reserve otherwise - this->scan_result_.reserve(needs_full ? number : WIFI_SCAN_RESULT_FILTERED_RESERVE); + const size_t wanted = filtered ? std::min(number, WIFI_SCAN_RESULT_BOUND) : number; + // Storage is reused across the scans of one retry cycle and freed on connect; an exhausted + // heap drops this scan and the retry logic scans again + if (this->scan_result_.capacity() < wanted && !this->scan_result_.try_init(wanted)) { + esp_wifi_clear_ap_list(); + ESP_LOGW(TAG, "No memory for %zu scan results", wanted); + return; + } #ifdef USE_ESP32_HOSTED // getting records one at a time fails on P4 with hosted esp32 WiFi coprocessor @@ -955,22 +962,38 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { } #endif // USE_ESP32_HOSTED - // Check C string first - avoid std::string construction for non-matching networks const char *ssid_cstr = reinterpret_cast(record.ssid); - - // Only construct std::string and store if needed - if (needs_full || this->matches_configured_network_(ssid_cstr, record.bssid)) { - bssid_t bssid; - std::copy(record.bssid, record.bssid + 6, bssid.begin()); + if (!needs_full && !this->matches_configured_network_(ssid_cstr, record.bssid)) { + this->log_discarded_scan_result_(ssid_cstr, record.bssid, record.rssi, record.primary); + continue; + } + bssid_t bssid; + std::copy(record.bssid, record.bssid + 6, bssid.begin()); + if (this->scan_result_.size() < wanted) { this->scan_result_.emplace_back(bssid, ssid_cstr, strlen(ssid_cstr), record.primary, record.rssi, record.authmode != WIFI_AUTH_OPEN, ssid_cstr[0] == '\0'); - } else { - this->log_discarded_scan_result_(ssid_cstr, record.bssid, record.rssi, record.primary); + continue; } + // Records arrive in scan order, not by signal, so a bounded store keeps the strongest by + // replacing its weakest entry. Only SSID and signal decide here; a channel or auth constrained + // network hidden behind 12 stronger APs of its own SSID is not a real deployment + WiFiScanResult *weakest = &this->scan_result_[0]; + for (auto &res : this->scan_result_) { + if (res.get_rssi() < weakest->get_rssi()) + weakest = &res; + } + if (record.rssi <= weakest->get_rssi()) { + this->log_discarded_scan_result_(ssid_cstr, record.bssid, record.rssi, record.primary); + continue; + } + // Rebuilt in place rather than assigned; assignment pulls in CompactString's operators, 104 B of flash + weakest->~WiFiScanResult(); + new (weakest) WiFiScanResult(bssid, ssid_cstr, strlen(ssid_cstr), record.primary, record.rssi, + record.authmode != WIFI_AUTH_OPEN, ssid_cstr[0] == '\0'); } } ESP_LOGV(TAG, "Scan complete: %u found, %zu stored%s", number, this->scan_result_.size(), - needs_full ? "" : " (filtered)"); + filtered ? LOG_STR_LITERAL(" (driver filtered)") : LOG_STR_LITERAL("")); #ifdef USE_WIFI_SCAN_RESULTS_LISTENERS this->notify_scan_results_listeners_(); #endif @@ -1047,6 +1070,16 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { wifi_scan_config_t config{}; config.ssid = nullptr; config.bssid = nullptr; +#ifndef USE_WIFI_MULTI_SSID + // One configured network with an SSID: let the driver keep only its APs, so the WiFi library + // holds fewer records during the scan. Full results (portal, provisioning, listeners) and a + // network configured by BSSID alone still scan everything + this->scan_driver_filtered_ = + !this->needs_full_scan_results_() && this->sta_.size() == 1 && !this->sta_[0].get_ssid().empty(); + if (this->scan_driver_filtered_) { + config.ssid = const_cast(reinterpret_cast(this->sta_[0].get_ssid().c_str())); + } +#endif config.channel = 0; config.show_hidden = true; config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE; diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index 63a63e7342..940f2a0783 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -709,7 +709,11 @@ void WiFiComponent::wifi_scan_done_callback_() { } } - this->scan_result_.init(count); // Exact allocation + if (!this->scan_result_.try_init(count)) { + ESP_LOGW(TAG, "No memory for %zu scan results", count); + WiFi.scanDelete(); + return; + } // Second pass: store matching networks for (int i = 0; i < num; i++) { diff --git a/esphome/core/defines.h b/esphome/core/defines.h index fe06cc3418..f6010fd7fa 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -298,6 +298,8 @@ #ifdef USE_ARDUINO #define USE_PROMETHEUS #define USE_WIFI_WPA2_EAP +// Kept in the Arduino block so clang-tidy sees both scan storage paths +#define USE_WIFI_MULTI_SSID #endif // Platforms with native 64-bit time sources (no rollover tracking needed) From eb1ea4aefe9c6d6abacbec52222cb68ad3ed41c2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:49:23 -0500 Subject: [PATCH 129/178] [esp32_ble_tracker] Re-register GATT clients after ble.disable and ble.enable (#19068) --- .../bluetooth_connection_bluedroid.cpp | 37 +++++++++++++------ .../bluetooth_connection_bluedroid.h | 1 + esphome/components/esp32_ble/ble.cpp | 35 +++++++++++------- esphome/components/esp32_ble/ble.h | 13 ++++++- .../esp32_ble_client/ble_client_base.cpp | 35 +++++++++++++++++- .../esp32_ble_client/ble_client_base.h | 12 +++--- .../esp32_ble_tracker/esp32_ble_tracker.cpp | 28 ++++++++++++-- .../esp32_ble_tracker/esp32_ble_tracker.h | 3 ++ 8 files changed, 126 insertions(+), 38 deletions(-) diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp index 15f854239d..986a67c7a8 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp @@ -45,15 +45,7 @@ void BluedroidGattClient::setup() { void BluedroidGattClient::loop() { if (!esp32_ble::global_ble->is_active()) { - // Stack down: no CLOSE_EVT will come. Settle a live link so the consumer - // frees its slot, then re-register the app on the next enable. - auto down_st = this->state(); - if (down_st != ClientState::IDLE && down_st != ClientState::INIT) { - this->release_services(); - this->set_idle_(); - this->listener_->on_connection_state(false, 0, ble_device_base::GATT_ERR_NOT_CONNECTED); - } - this->set_state(ClientState::INIT); + // ble_before_disabled_event_handler() settles the slot. return; } auto st = this->state(); @@ -65,7 +57,7 @@ void BluedroidGattClient::loop() { ESP_LOGE(TAG, "gattc app register failed: app_id=%d code=%d", this->app_id, ret); this->mark_failed(); } - // Do not wait for REG_EVT; a dropped event must not wedge the slot. + // Do not wait for REG_EVT; connect() rejects until it lands. this->set_idle_(); } else if (st == ClientState::DISCONNECTING || this->disconnect_pending()) { // The one teardown safety net: a lost CLOSE_EVT, or a scheduled @@ -78,8 +70,8 @@ void BluedroidGattClient::loop() { this->listener_->on_connection_state(false, 0, ESP_GATT_CONN_TIMEOUT); } } else { - // The loop stays on while a link exists (stack-down watch, pre-started - // search flush); it settles only back at IDLE. + // The loop stays on while a link exists (pre-started search flush); it + // settles only back at IDLE. this->deliver_pending_search_(); if (this->state() == ClientState::IDLE) { this->disable_loop(); @@ -87,6 +79,22 @@ void BluedroidGattClient::loop() { } } +// Stack down: no CLOSE_EVT will come. Settle a live link so the consumer +// frees its slot, then register the app again on the next enable. +void BluedroidGattClient::ble_before_disabled_event_handler() { + auto st = this->state(); + if (st != ClientState::IDLE && st != ClientState::INIT) { + this->release_services(); + this->set_idle_(); + this->listener_->on_connection_state(false, 0, ble_device_base::GATT_ERR_NOT_CONNECTED); + } + // The interface belongs to the torn-down stack. + this->gattc_if_ = ESP_GATT_IF_NONE; + this->set_state(ClientState::INIT); + // An idle slot runs no loop; the INIT branch must run to register again. + this->enable_loop(); +} + void BluedroidGattClient::dump_config() { ESP_LOGCONFIG(TAG, "Bluedroid GATT client %d", this->connection_index_); if (this->is_failed()) { @@ -97,6 +105,11 @@ void BluedroidGattClient::dump_config() { // ---- contract ops ---- int BluedroidGattClient::connect(uint64_t address, uint8_t addr_type) { + if (this->gattc_if_ == ESP_GATT_IF_NONE) { + // Bluedroid drops an open on an unknown interface without any event. + ESP_LOGW(TAG, "[%d] Connect rejected, GATT app not registered", this->connection_index_); + return ble_device_base::GATT_ERR_NOT_CONNECTED; + } // Only from idle: clobbering DISCONNECTING would open a new link the // stale CLOSE_EVT then tears down. if (this->state() != ClientState::IDLE) { diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h index 0d0b4fed5b..f285260e76 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h @@ -56,6 +56,7 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param) override; void connect() override; void disconnect() override; + void ble_before_disabled_event_handler() override; bool wants_parsed_advertisements() override { return false; } void on_scan_end() override {} bool parse_device(const ble_device_base::ESPBTDevice &device) override { return false; } diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index fc95760cf8..81fa328c16 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -83,18 +83,23 @@ void ESP32BLE::setup() { } } -void ESP32BLE::enable() { - if (this->state_ != BLE_COMPONENT_STATE_DISABLED) - return; - - this->state_ = BLE_COMPONENT_STATE_ENABLE; -} - -void ESP32BLE::disable() { - if (this->state_ == BLE_COMPONENT_STATE_DISABLED) - return; - - this->state_ = BLE_COMPONENT_STATE_DISABLE; +// Queue the transition for loop(). A pending transition the other way is +// cancelled instead, since nothing was torn down or brought up yet; any other +// state is already there or on its way. +void ESP32BLE::request_state_(bool enable) { + if (enable) { + if (this->state_ == BLE_COMPONENT_STATE_DISABLED) { + this->state_ = BLE_COMPONENT_STATE_ENABLE; + } else if (this->state_ == BLE_COMPONENT_STATE_DISABLE) { + this->state_ = BLE_COMPONENT_STATE_ACTIVE; + } + } else { + if (this->state_ == BLE_COMPONENT_STATE_ACTIVE) { + this->state_ = BLE_COMPONENT_STATE_DISABLE; + } else if (this->state_ == BLE_COMPONENT_STATE_ENABLE) { + this->state_ = BLE_COMPONENT_STATE_DISABLED; + } + } } #ifdef USE_ESP32_BLE_ADVERTISING @@ -580,7 +585,11 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { this->mark_failed(); return; } - this->state_ = BLE_COMPONENT_STATE_DISABLED; + this->drain_ble_events_(); + // A status callback may have asked for BLE back; the stack is down now, so + // that request becomes a bring-up. + this->state_ = + this->state_ == BLE_COMPONENT_STATE_ACTIVE ? BLE_COMPONENT_STATE_ENABLE : BLE_COMPONENT_STATE_DISABLED; } else if (this->state_ == BLE_COMPONENT_STATE_ENABLE) { ESP_LOGD(TAG, "Enabling"); this->state_ = BLE_COMPONENT_STATE_OFF; diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 7d2d0438a4..fd4fb15ff6 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -102,8 +102,8 @@ class ESP32BLE final : public Component { } uint32_t get_advertising_cycle_time() const { return this->advertising_cycle_time_; } - void enable(); - void disable(); + void enable() { this->request_state_(true); } + void disable() { this->request_state_(false); } ESPHOME_ALWAYS_INLINE bool is_active() { return this->state_ == BLE_COMPONENT_STATE_ACTIVE; } void setup() override; void loop() override; @@ -176,6 +176,15 @@ class ESP32BLE final : public Component { bool ble_setup_(); bool ble_dismantle_(); + void request_state_(bool enable); + // Drop what the old stack queued; the next stack reuses the same interface ids. + void drain_ble_events_() { + BLEEvent *ble_event; + while ((ble_event = this->ble_events_.pop()) != nullptr) { + this->ble_event_pool_.release(ble_event); + } + this->ble_events_.get_and_reset_dropped_count(); + } bool ble_pre_setup_(); #ifdef USE_ESP32_BLE_ADVERTISING void advertising_init_(); diff --git a/esphome/components/esp32_ble_client/ble_client_base.cpp b/esphome/components/esp32_ble_client/ble_client_base.cpp index e6cdde9cda..88454f7bdb 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.cpp +++ b/esphome/components/esp32_ble_client/ble_client_base.cpp @@ -42,7 +42,7 @@ void BLEClientBase::set_state(espbt::ClientState st) { void BLEClientBase::loop() { if (!esp32_ble::global_ble->is_active()) { - this->set_state(espbt::ClientState::INIT); + // ble_before_disabled_event_handler() resets the client. return; } if (this->state() == espbt::ClientState::INIT) { @@ -72,6 +72,21 @@ void BLEClientBase::loop() { float BLEClientBase::get_setup_priority() const { return setup_priority::AFTER_BLUETOOTH; } +void BLEClientBase::ble_before_disabled_event_handler() { + auto st = this->state(); + if (st != espbt::ClientState::IDLE && st != espbt::ClientState::INIT) { + // No CLOSE_EVT will come: free the services and settle the link. + this->release_services(); + this->set_idle_(); + this->on_disconnect_complete(ESP_GATT_CONN_TERMINATE_LOCAL_HOST); + } + // The interface belongs to the torn-down stack. + this->gattc_if_ = ESP_GATT_IF_NONE; + this->set_state(espbt::ClientState::INIT); + // An idle client runs no loop; the INIT branch must run to register again. + this->enable_loop(); +} + void BLEClientBase::dump_config() { ESP_LOGCONFIG(TAG, " Address: %s\n" @@ -93,6 +108,10 @@ bool BLEClientBase::parse_device(const espbt::ESPBTDevice &device) { return false; if (this->state() != espbt::ClientState::IDLE) return false; + // Not registered on this stack yet; promoting now would stop the scan for a + // connect that connect() rejects anyway. + if (this->gattc_if_ == ESP_GATT_IF_NONE) + return false; this->log_event_("Found device"); if (ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_DEBUG) @@ -117,6 +136,15 @@ void BLEClientBase::connect() { this->connection_index_, this->address_str_); return; } + if (this->gattc_if_ == ESP_GATT_IF_NONE) { + // Bluedroid drops an open on an unknown interface without any event. + this->log_warning_("Connect rejected, GATT app not registered"); + // INIT stays so loop() still registers; only a promoted client goes back. + if (this->state() == espbt::ClientState::DISCOVERED) { + this->set_state(espbt::ClientState::IDLE); + } + return; + } ESP_LOGI(TAG, "[%d] [%s] 0x%02x Connecting", this->connection_index_, this->address_str_, this->remote_addr_type_); this->paired_ = false; // A registration whose event never arrived must not block this connection's release. @@ -199,7 +227,10 @@ void BLEClientBase::release_services() { #ifndef CONFIG_BT_GATTC_CACHE_NVS_FLASH // Only the cache clean makes the stack's database unsafe to walk. this->services_released_ = true; - esp_ble_gattc_cache_clean(this->remote_bda_); + // A stack on its way down frees its own cache. + if (esp32_ble::global_ble->is_active()) { + esp_ble_gattc_cache_clean(this->remote_bda_); + } #endif } diff --git a/esphome/components/esp32_ble_client/ble_client_base.h b/esphome/components/esp32_ble_client/ble_client_base.h index e4b9cd5100..fbd405156a 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.h +++ b/esphome/components/esp32_ble_client/ble_client_base.h @@ -41,6 +41,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void connect() override; esp_err_t pair(); void disconnect() override; + void ble_before_disabled_event_handler() override; void unconditional_disconnect(); void release_services(); @@ -114,7 +115,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { #endif // Group 3: 4-byte types - int gattc_if_; + int gattc_if_{ESP_GATT_IF_NONE}; esp_gatt_status_t status_{ESP_GATT_OK}; // Group 4: Arrays @@ -139,7 +140,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { uint8_t pending_notify_regs_{0}; bool auto_connect_{false}; bool paired_{false}; - // Set only when release_services() cleans the stack's GATT cache, which no API may then walk + // Set by release_services() on RAM-cache builds; the stack's GATT database must not be walked after it bool services_released_{false}; // 8 bytes used, no padding @@ -155,10 +156,11 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void log_connection_params_(const char *param_type); void handle_connection_result_(esp_err_t ret); /// Hook called once a connection has been fully torn down (after release_services() and - /// set_idle_()), from both the CLOSE_EVT handler and the DISCONNECTING safety timeout. + /// set_idle_()): CLOSE_EVT, the DISCONNECTING safety timeout, or the BLE stack going down. /// Subclasses with extra per-connection accounting (e.g. bluetooth_proxy slot state) - /// override this to release that state. `reason` is the controller reason code, or - /// ESP_GATT_CONN_TIMEOUT for the safety-timeout path. + /// override this to release that state. `reason` is the controller reason code, + /// ESP_GATT_CONN_TIMEOUT for the safety timeout, or ESP_GATT_CONN_TERMINATE_LOCAL_HOST + /// for the stack going down. virtual void on_disconnect_complete(esp_err_t reason) {} /// Transition to IDLE and reset conn_id — call when the connection is fully dead. void set_idle_() { diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index 5339565a32..b4b793b4d0 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -74,11 +74,11 @@ void ESP32BLETracker::on_ota_global_state(ota::OTAState state, float progress, u void ESP32BLETracker::loop() { if (!this->parent_->is_active()) { - this->ble_was_disabled_ = true; return; - } else if (this->ble_was_disabled_) { + } + if (this->ble_was_disabled_) { this->ble_was_disabled_ = false; - // If the BLE stack was disabled, we need to start the scan again. + // First start after boot or after the stack came back. if (this->scan_continuous_) { this->start_scan(); } @@ -218,7 +218,27 @@ void ESP32BLETracker::stop_scan() { this->stop_scan_(); } -void ESP32BLETracker::ble_before_disabled_event_handler() { this->stop_scan_(); } +void ESP32BLETracker::ble_before_disabled_event_handler() { + // Tell the controller to stop; a scan still starting has nothing to stop yet. + if (this->scanner_state_ == ScannerState::RUNNING || this->scanner_state_ == ScannerState::FAILED) { + this->stop_scan_(); + } +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT + for (auto *client : this->clients_) { + client->ble_before_disabled_event_handler(); + } + this->skip_next_scan_end_ = false; +#endif + // The stop above never completes (stack torn down, events dropped); settle + // here so start_scan_() sees IDLE once the stack is back. + if (this->scanner_state_ != ScannerState::IDLE) { + this->cleanup_scan_state_(true); + } + // A failure latched by the old stack must not be handled against the next. + this->scan_start_failed_ = ESP_BT_STATUS_SUCCESS; + this->scan_set_param_failed_ = ESP_BT_STATUS_SUCCESS; + this->ble_was_disabled_ = true; +} bool ESP32BLETracker::stop_scan_() { if (this->scanner_state_ != ScannerState::RUNNING && this->scanner_state_ != ScannerState::FAILED) { diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h index 618444e626..1a424a4a8e 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h @@ -113,6 +113,9 @@ class ESPBTClient : public ESPBTDeviceListener { virtual void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param) = 0; virtual void connect() = 0; virtual void disconnect() = 0; + /// Called right before the BLE stack is dismantled. Nothing in flight will + /// complete, and the GATT app must register again once the stack is back. + virtual void ble_before_disabled_event_handler() {} bool disconnect_pending() const { return this->want_disconnect_; } void cancel_pending_disconnect() { this->want_disconnect_ = false; } From c51020dbaf91051e03e7bbaa50df66eaab4e1ff3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:56:50 -0500 Subject: [PATCH 130/178] [core] Add RAMAllocator::make_unique for objects whose allocation may fail (#19245) --- esphome/core/helpers.h | 40 ++++++++++++++++++++ tests/components/core/test_helpers.cpp | 52 ++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 987c54a5b0..b1f24b25a3 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,9 +14,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -2123,6 +2126,10 @@ void delay_microseconds_safe(uint32_t us); /// @name Memory management ///@{ +template struct RAMDeleter; +/// unique_ptr over RAMAllocator storage +template using RAMUniquePtr = std::unique_ptr>; + /** An STL allocator that uses SPI or internal RAM. * Returns `nullptr` in case no memory is available. * @@ -2193,6 +2200,26 @@ template class RAMAllocator { free(p); // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) } + /// Value initialize one T; empty on exhaustion. new (std::nothrow) aborts on ESP-IDF instead. + /// Default flags prefer PSRAM; pass PREFER_INTERNAL to keep an object where plain new put it. + template RAMUniquePtr make_unique(Args &&...args) { + static_assert(alignof(T) <= alignof(std::max_align_t), "malloc storage cannot hold an over aligned type"); + T *p = this->allocate(1); + if (p == nullptr) + return {}; + // ::new so a class scoped operator new cannot hide the global placement form + return RAMUniquePtr(::new (p) T(std::forward(args)...)); + } + + /// n elements left uninitialized, as std::make_unique_for_overwrite does; empty on exhaustion, overflow, and n == 0 + RAMUniquePtr make_unique_array_for_overwrite(size_t n) { + static_assert(std::is_trivially_default_constructible_v, "elements are left unconstructed"); + static_assert(alignof(T) <= alignof(std::max_align_t), "malloc storage cannot hold an over aligned type"); + if (n == 0 || n > SIZE_MAX / sizeof(T)) + return {}; + return RAMUniquePtr(this->allocate(n)); + } + /** * Return the total heap space available via this allocator */ @@ -2255,6 +2282,19 @@ template class RAMAllocator { template using ExternalRAMAllocator = RAMAllocator; +/// Destroys and frees RAMAllocator storage. Not convertible: free() needs the address malloc returned +template struct RAMDeleter { + void operator()(T *p) const { + p->~T(); + RAMAllocator().deallocate(p, 1); + } +}; +/// Array form: elements must be trivial, the count is not stored so only the storage is freed +template struct RAMDeleter { + static_assert(std::is_trivially_destructible_v, "RAMUniquePtr is for trivially destructible elements"); + void operator()(T *p) const { RAMAllocator().deallocate(p, 1); } +}; + /** * Functions to constrain the range of arithmetic values. */ diff --git a/tests/components/core/test_helpers.cpp b/tests/components/core/test_helpers.cpp index d6b31508d1..72af605d61 100644 --- a/tests/components/core/test_helpers.cpp +++ b/tests/components/core/test_helpers.cpp @@ -367,4 +367,56 @@ TEST(FixedVectorTryInit, ReportsExhaustionAndStaysEmpty) { EXPECT_EQ(v.size(), 1u); } +// --- RAMAllocator::make_unique() --- + +namespace { +struct Probe { + static inline int live = 0; + int a; + int b; + Probe(int a, int b) : a(a), b(b) { live++; } + ~Probe() { live--; } +}; +} // namespace + +static_assert(sizeof(RAMUniquePtr) == sizeof(Probe *), "the deleter must not add storage"); + +TEST(RAMAllocatorMakeUnique, ForwardsArgsAndDestroysOnce) { + auto p = RAMAllocator().make_unique(3, 4); + ASSERT_NE(p, nullptr); + EXPECT_EQ(p->a, 3); + EXPECT_EQ(p->b, 4); + EXPECT_EQ(Probe::live, 1); + p.reset(); + EXPECT_EQ(Probe::live, 0); +} + +TEST(RAMAllocatorMakeUnique, ValueInitializesLikeMakeUnique) { + struct Plain { + uint32_t words[8]; + }; + // Dirty a block of the same size first so a recycled allocation is not zero by chance + auto dirty = RAMAllocator().make_unique_array_for_overwrite(sizeof(Plain)); + std::memset(dirty.get(), 0xFF, sizeof(Plain)); + dirty.reset(); + auto p = RAMAllocator().make_unique(); + ASSERT_NE(p, nullptr); + // Under ASan fresh blocks are filled with 0xbe, so this holds even when the dirtied block is not reused + EXPECT_TRUE(std::all_of(std::begin(p->words), std::end(p->words), [](uint32_t w) { return w == 0; })); +} + +TEST(RAMAllocatorMakeUnique, ArrayFormRejectsOverflowAndZero) { + EXPECT_EQ(RAMAllocator().make_unique_array_for_overwrite(SIZE_MAX / sizeof(uint32_t) + 1), nullptr); + EXPECT_EQ(RAMAllocator().make_unique_array_for_overwrite(0), nullptr); + EXPECT_NE(RAMAllocator().make_unique_array_for_overwrite(1), nullptr); +} + +TEST(RAMAllocatorMakeUnique, ArrayFormAllocatesElements) { + RAMUniquePtr buf = RAMAllocator().make_unique_array_for_overwrite(256); + ASSERT_NE(buf, nullptr); + std::memset(buf.get(), 0xA5, 256); + EXPECT_EQ(buf[0], 0xA5); + EXPECT_EQ(buf[255], 0xA5); +} + } // namespace esphome::core::testing From 67871bcf55b47355944e35a11fab96386b3f7e78 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sun, 13 Sep 2026 18:01:58 -0400 Subject: [PATCH 131/178] [i2s_audio][router] Loop thread controls all state changes (#19089) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 14 ++++++++----- .../router/speaker/router_speaker.cpp | 21 ++++++++++++++++--- .../router/speaker/router_speaker.h | 3 +++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1382a87046..9feaf39fff 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -53,6 +53,13 @@ void I2SAudioSpeakerBase::dump_config() { void I2SAudioSpeakerBase::loop() { uint32_t event_group_bits = xEventGroupGetBits(this->event_group_); + // A stop that arrives while stopped cancels any start that has not been processed yet + constexpr uint32_t stop_bits = SpeakerEventGroupBits::COMMAND_STOP | SpeakerEventGroupBits::COMMAND_STOP_GRACEFULLY; + if ((event_group_bits & stop_bits) && (this->state_ == speaker::STATE_STOPPED)) { + xEventGroupClearBits(this->event_group_, stop_bits | SpeakerEventGroupBits::COMMAND_START); + event_group_bits &= ~(stop_bits | SpeakerEventGroupBits::COMMAND_START); + } + if ((event_group_bits & SpeakerEventGroupBits::COMMAND_START) && (this->state_ == speaker::STATE_STOPPED)) { this->state_ = speaker::STATE_STARTING; xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); @@ -239,8 +246,6 @@ void I2SAudioSpeakerBase::start() { if ((this->state_ == speaker::STATE_STARTING) || (this->state_ == speaker::STATE_RUNNING)) return; - // Mark STARTING immediately to avoid transient STOPPED observations before loop() processes COMMAND_START. - this->state_ = speaker::STATE_STARTING; xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); } @@ -249,11 +254,10 @@ void I2SAudioSpeakerBase::stop() { this->stop_(false); } void I2SAudioSpeakerBase::finish() { this->stop_(true); } void I2SAudioSpeakerBase::stop_(bool wait_on_empty) { - if (this->is_failed()) - return; - if (this->state_ == speaker::STATE_STOPPED) + if (!this->is_ready() || this->is_failed()) return; + // Always set the bit, even when stopped, so loop() can cancel a start that is still pending if (wait_on_empty) { xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_STOP_GRACEFULLY); } else { diff --git a/esphome/components/router/speaker/router_speaker.cpp b/esphome/components/router/speaker/router_speaker.cpp index f4bf7420ab..dd2428e4df 100644 --- a/esphome/components/router/speaker/router_speaker.cpp +++ b/esphome/components/router/speaker/router_speaker.cpp @@ -2,6 +2,8 @@ #ifdef USE_ESP32 +#include "esphome/core/application.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include "esp_timer.h" @@ -12,6 +14,9 @@ namespace esphome::router { static const char *const TAG = "router.speaker"; +// Maximum time to wait for the active output to report running after start() before giving up +static const uint32_t STATE_TRANSITION_TIMEOUT_MS = 5000; + static inline uint32_t atomic_subtract_clamped(std::atomic &var, uint32_t amount) { uint32_t current = var.load(std::memory_order_acquire); uint32_t subtracted = 0; @@ -72,6 +77,7 @@ void Router::loop() { this->apply_cached_state_to_active_(); this->state_ = speaker::STATE_STARTING; + this->state_start_ms_ = App.get_loop_component_start_time(); active->start(); } return; @@ -86,10 +92,17 @@ void Router::loop() { // set_audio_stream_info() and never reaches the output on its own; if the format // changed while stopped, only start()'s apply_cached_state_to_active_() pushes it // down before the output's play()-side auto-start locks in the stale format. - if (active->is_stopped()) { + // While STARTING, ignore a transient stopped report as speaker running state + // is set asynchronously from start(). Timeout if the speaker never transitions. + if (this->state_ == speaker::STATE_STARTING) { + if (active->is_running()) { + this->state_ = speaker::STATE_RUNNING; + } else if ((App.get_loop_component_start_time() - this->state_start_ms_) > STATE_TRANSITION_TIMEOUT_MS) { + ESP_LOGW(TAG, "Active output did not start; giving up"); + this->state_ = speaker::STATE_STOPPED; + } + } else if (active->is_stopped()) { this->state_ = speaker::STATE_STOPPED; - } else if (this->state_ == speaker::STATE_STARTING && active->is_running()) { - this->state_ = speaker::STATE_RUNNING; } } @@ -133,6 +146,8 @@ void Router::start() { this->frames_in_pipeline_.store(0, std::memory_order_release); this->apply_cached_state_to_active_(); this->state_ = speaker::STATE_STARTING; + // May run on a producer task, so the cached loop timestamp is not usable here + this->state_start_ms_ = millis(); this->get_active_output()->start(); } diff --git a/esphome/components/router/speaker/router_speaker.h b/esphome/components/router/speaker/router_speaker.h index 801d0906ce..31f3f90630 100644 --- a/esphome/components/router/speaker/router_speaker.h +++ b/esphome/components/router/speaker/router_speaker.h @@ -59,6 +59,9 @@ class Router final : public Component, public speaker::Speaker { // frames_in_pipeline_. std::atomic frames_in_pipeline_{0}; + // Set when entering STATE_STARTING; used to time out a start the output never acts on + uint32_t state_start_ms_{0}; + bool cached_pause_{false}; void apply_cached_state_to_active_(); From 16df92212e307096f71ef492b82ba0b16c8daee3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:02:42 -0500 Subject: [PATCH 132/178] [esp32_ble_tracker] Revert coexistence preference to balanced when OTA starts (#19082) --- esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index b4b793b4d0..e25b6f59fa 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -62,6 +62,11 @@ void ESP32BLETracker::on_ota_global_state(ota::OTAState state, float progress, u for (auto *client : this->clients_) { client->disconnect(); } +#ifdef USE_ESP32_BLE_SOFTWARE_COEXISTENCE + // The OTA transfer blocks the main loop, so the revert in loop() cannot run. No + // active-connection gate here: every client was just told to disconnect. + this->update_coex_preference_(false); +#endif #endif } else if ((state == ota::OTA_ERROR || state == ota::OTA_ABORT) && this->scan_continuous_before_ota_) { this->scan_continuous_before_ota_ = false; From ed5a570e1784057770519095d37de0dcec9359eb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:26:14 -0500 Subject: [PATCH 133/178] [nextion] Allocate queue components through RAMAllocator and free entries the way they were allocated (#19246) --- esphome/components/nextion/nextion.cpp | 152 +++++++++--------- esphome/components/nextion/nextion.h | 2 + .../nextion/nextion_component_base.h | 5 +- 3 files changed, 78 insertions(+), 81 deletions(-) diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp index 97910ba3d5..625c915e73 100644 --- a/esphome/components/nextion/nextion.cpp +++ b/esphome/components/nextion/nextion.cpp @@ -13,6 +13,11 @@ namespace esphome::nextion { static const char *const TAG = "nextion"; +// A user entity may be named sleep_wake too; only the internal NO_RESULT command clears the sleeping flag +static bool is_sleep_wake_command(const NextionComponentBase *component) { + return component->get_queue_type() == NextionQueueType::NO_RESULT && component->get_variable_name() == "sleep_wake"; +} + // Nextion command terminator: three consecutive 0xFF bytes (per Nextion Instruction Set v1.1). static constexpr uint8_t COMMAND_DELIMITER[3] = {0xFF, 0xFF, 0xFF}; static constexpr size_t DELIMITER_SIZE = sizeof(COMMAND_DELIMITER); @@ -163,6 +168,17 @@ bool Nextion::check_connect_() { #endif // USE_NEXTION_CONFIG_SKIP_CONNECTION_HANDSHAKE } +// NO_RESULT components are owned by their entry; every other component is a user entity. Entry and +// component storage comes from RAMAllocator, so delete is not valid for either. +void Nextion::release_queue_entry_(NextionQueue *nb) { + if (nb->component != nullptr && nb->component->get_queue_type() == NextionQueueType::NO_RESULT) { + nb->component->~NextionComponentBase(); + RAMAllocator().deallocate(nb->component, 1); + } + nb->~NextionQueue(); + RAMAllocator().deallocate(nb, 1); +} + void Nextion::reset_(bool reset_nextion) { uint8_t d; @@ -170,15 +186,12 @@ void Nextion::reset_(bool reset_nextion) { this->read_byte(&d); } for (auto *entry : this->nextion_queue_) { - if (entry->component != nullptr && entry->component->get_queue_type() == NextionQueueType::NO_RESULT) { - delete entry->component; // NOLINT(cppcoreguidelines-owning-memory) - } - delete entry; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(entry); } this->nextion_queue_.clear(); #ifdef USE_NEXTION_WAVEFORM for (auto *entry : this->waveform_queue_) { - delete entry; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(entry); } this->waveform_queue_.clear(); #endif // USE_NEXTION_WAVEFORM @@ -421,6 +434,9 @@ bool Nextion::remove_from_q_(bool report_empty) { NextionQueue *nb = this->nextion_queue_.front(); if (!nb || !nb->component) { ESP_LOGE(TAG, "Invalid queue"); + if (nb != nullptr) { + this->release_queue_entry_(nb); + } this->nextion_queue_.pop_front(); return false; } @@ -428,13 +444,10 @@ bool Nextion::remove_from_q_(bool report_empty) { ESP_LOGN(TAG, "Removed: %s", component->get_variable_name().c_str()); - if (component->get_queue_type() == NextionQueueType::NO_RESULT) { - if (component->get_variable_name() == "sleep_wake") { - this->is_sleeping_ = false; - } - delete component; // NOLINT(cppcoreguidelines-owning-memory) + if (is_sleep_wake_command(component)) { + this->is_sleeping_ = false; } - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->nextion_queue_.pop_front(); return true; } @@ -544,7 +557,7 @@ void Nextion::process_nextion_commands_() { ESP_LOGW(TAG, "Invalid waveform ID %d/ch %d", component->get_component_id(), component->get_wave_channel_id()); ESP_LOGN(TAG, "Remove waveform ID %d/ch %d", component->get_component_id(), component->get_wave_channel_id()); - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->waveform_queue_.pop(); } #else // USE_NEXTION_WAVEFORM @@ -647,6 +660,9 @@ void Nextion::process_nextion_commands_() { NextionQueue *nb = this->nextion_queue_.front(); if (!nb || !nb->component) { ESP_LOGE(TAG, "Invalid queue entry"); + if (nb != nullptr) { + this->release_queue_entry_(nb); + } this->nextion_queue_.pop_front(); return; } @@ -660,7 +676,7 @@ void Nextion::process_nextion_commands_() { component->set_state_from_string(to_process, true, false); } - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->nextion_queue_.pop_front(); break; @@ -687,6 +703,9 @@ void Nextion::process_nextion_commands_() { NextionQueue *nb = this->nextion_queue_.front(); if (!nb || !nb->component) { ESP_LOGE(TAG, "Invalid queue"); + if (nb != nullptr) { + this->release_queue_entry_(nb); + } this->nextion_queue_.pop_front(); return; } @@ -703,7 +722,7 @@ void Nextion::process_nextion_commands_() { component->set_state_from_int(value, true, false); } - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->nextion_queue_.pop_front(); break; @@ -890,7 +909,7 @@ void Nextion::process_nextion_commands_() { ESP_LOGN(TAG, "Send waveform: component id %d, waveform id %d, size %zu", component->get_component_id(), component->get_wave_channel_id(), buffer_to_send); component->clear_wave_buffer(buffer_to_send); - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->waveform_queue_.pop(); #else // USE_NEXTION_WAVEFORM ESP_LOGW(TAG, "Waveform transmit ready but waveform not enabled"); @@ -920,14 +939,10 @@ void Nextion::purge_stale_queue_entries_() { ESP_LOGV(TAG, "Remove old queue '%s':'%s'", component->get_queue_type_string(), component->get_variable_name().c_str()); - if (component->get_queue_type() == NextionQueueType::NO_RESULT) { - if (component->get_variable_name() == "sleep_wake") { - this->is_sleeping_ = false; - } - delete component; // NOLINT(cppcoreguidelines-owning-memory) + if (is_sleep_wake_command(component)) { + this->is_sleeping_ = false; } - - delete *it; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(*it); it = this->nextion_queue_.erase(it); } else { @@ -1079,6 +1094,34 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool return response.length(); } +// Allocates a queue entry owning a bare NO_RESULT component; nullptr when the queue is full or memory is out +NextionQueue *Nextion::make_no_result_entry_(const std::string &variable_name) { +#ifdef USE_NEXTION_MAX_QUEUE_SIZE + if (this->max_queue_size_ > 0 && this->nextion_queue_.size() >= this->max_queue_size_) { + ESP_LOGW(TAG, "Queue full (%zu), drop: %s", this->nextion_queue_.size(), variable_name.c_str()); + return nullptr; + } +#endif + + auto *nextion_queue = RAMAllocator().allocate(1); + if (nextion_queue == nullptr) { + ESP_LOGW(TAG, "Queue alloc failed"); + return nullptr; + } + new (nextion_queue) nextion::NextionQueue; + + nextion_queue->component = RAMAllocator().allocate(1); + if (nextion_queue->component == nullptr) { + ESP_LOGW(TAG, "Component alloc failed"); + this->release_queue_entry_(nextion_queue); + return nullptr; + } + new (nextion_queue->component) nextion::NextionComponentBase; + nextion_queue->component->set_variable_name(variable_name); + nextion_queue->queue_time = App.get_loop_component_start_time(); + return nextion_queue; +} + /** * @brief Add a command to the Nextion queue that expects no response. * @@ -1090,36 +1133,11 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool * @param variable_name Name of the variable or component associated with the command. */ void Nextion::add_no_result_to_queue_(const std::string &variable_name) { -#ifdef USE_NEXTION_MAX_QUEUE_SIZE - if (this->max_queue_size_ > 0 && this->nextion_queue_.size() >= this->max_queue_size_) { - ESP_LOGW(TAG, "Queue full (%zu), drop: %s", this->nextion_queue_.size(), variable_name.c_str()); + auto *nextion_queue = this->make_no_result_entry_(variable_name); + if (nextion_queue == nullptr) return; - } -#endif - - RAMAllocator allocator; - nextion::NextionQueue *nextion_queue = allocator.allocate(1); - if (nextion_queue == nullptr) { - ESP_LOGW(TAG, "Queue alloc failed"); - return; - } - new (nextion_queue) nextion::NextionQueue(); - - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - nextion_queue->component = new (std::nothrow) nextion::NextionComponentBase; - if (nextion_queue->component == nullptr) { - ESP_LOGW(TAG, "Component alloc failed"); - nextion_queue->~NextionQueue(); - allocator.deallocate(nextion_queue, 1); - return; - } - nextion_queue->component->set_variable_name(variable_name); - - nextion_queue->queue_time = App.get_loop_component_start_time(); - this->nextion_queue_.push_back(nextion_queue); - - ESP_LOGN(TAG, "Queue NORESULT: %s", nextion_queue->component->get_variable_name().c_str()); + ESP_LOGN(TAG, "Queue NORESULT: %s", variable_name.c_str()); } /** @@ -1153,32 +1171,10 @@ void Nextion::add_no_result_to_queue_with_command_(const std::string &variable_n #ifdef USE_NEXTION_COMMAND_SPACING void Nextion::add_no_result_to_queue_with_pending_command_(const std::string &variable_name, const std::string &command) { -#ifdef USE_NEXTION_MAX_QUEUE_SIZE - if (this->max_queue_size_ > 0 && this->nextion_queue_.size() >= this->max_queue_size_) { - ESP_LOGW(TAG, "Queue full (%zu), drop: %s", this->nextion_queue_.size(), variable_name.c_str()); + auto *nextion_queue = this->make_no_result_entry_(variable_name); + if (nextion_queue == nullptr) return; - } -#endif - - RAMAllocator allocator; - nextion::NextionQueue *nextion_queue = allocator.allocate(1); - if (nextion_queue == nullptr) { - ESP_LOGW(TAG, "Queue alloc failed"); - return; - } - new (nextion_queue) nextion::NextionQueue(); - - nextion_queue->component = new (std::nothrow) nextion::NextionComponentBase; - if (nextion_queue->component == nullptr) { - ESP_LOGW(TAG, "Component alloc failed"); - nextion_queue->~NextionQueue(); - allocator.deallocate(nextion_queue, 1); - return; - } - nextion_queue->component->set_variable_name(variable_name); - nextion_queue->queue_time = App.get_loop_component_start_time(); nextion_queue->pending_command = command; // Store command for retry - this->nextion_queue_.push_back(nextion_queue); ESP_LOGVV(TAG, "Queue with pending command: %s", variable_name.c_str()); } @@ -1312,7 +1308,7 @@ void Nextion::add_to_get_queue(NextionComponentBase *component) { ESP_LOGW(TAG, "Queue alloc failed"); return; } - new (nextion_queue) nextion::NextionQueue(); + new (nextion_queue) nextion::NextionQueue; nextion_queue->component = component; nextion_queue->queue_time = App.get_loop_component_start_time(); @@ -1334,7 +1330,7 @@ void Nextion::add_to_get_queue(NextionComponentBase *component) { if (this->send_command_(command)) { this->nextion_queue_.push_back(nextion_queue); } else { - delete nextion_queue; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nextion_queue); } #endif // USE_NEXTION_COMMAND_SPACING } @@ -1355,14 +1351,14 @@ void Nextion::add_addt_command_to_queue(NextionComponentBase *component) { ESP_LOGW(TAG, "Queue alloc failed"); return; } - new (nextion_queue) nextion::NextionQueue(); + new (nextion_queue) nextion::NextionQueue; nextion_queue->component = component; nextion_queue->queue_time = App.get_loop_component_start_time(); if (!this->waveform_queue_.push(nextion_queue)) { ESP_LOGW(TAG, "Waveform queue full, drop"); - delete nextion_queue; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nextion_queue); return; } if (this->waveform_queue_.size() == 1) diff --git a/esphome/components/nextion/nextion.h b/esphome/components/nextion/nextion.h index aa9fe8abb3..6c9c8760f8 100644 --- a/esphome/components/nextion/nextion.h +++ b/esphome/components/nextion/nextion.h @@ -1469,6 +1469,8 @@ class Nextion final : public NextionBase, public PollingComponent, public uart:: void all_components_send_state_(bool force_update = false); uint32_t comok_sent_ = 0; bool remove_from_q_(bool report_empty = true); + void release_queue_entry_(NextionQueue *nb); + NextionQueue *make_no_result_entry_(const std::string &variable_name); /** * @brief Status flags for Nextion display state management diff --git a/esphome/components/nextion/nextion_component_base.h b/esphome/components/nextion/nextion_component_base.h index 6676d01920..5e84291b16 100644 --- a/esphome/components/nextion/nextion_component_base.h +++ b/esphome/components/nextion/nextion_component_base.h @@ -23,8 +23,7 @@ class NextionComponentBase; class NextionQueue { public: - virtual ~NextionQueue() = default; - NextionComponentBase *component; + NextionComponentBase *component{nullptr}; uint32_t queue_time = 0; // Store command for retry if spacing blocked it @@ -105,6 +104,6 @@ class NextionComponentBase { int wave_max_length_ = 255; #endif // USE_NEXTION_WAVEFORM - bool needs_to_send_update_; + bool needs_to_send_update_{false}; }; } // namespace esphome::nextion From bc1841c1b53f33248eb419ff949d469466b4782a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:26:28 -0500 Subject: [PATCH 134/178] [esphome] Allocate the OTA noise session and auth buffer through RAMAllocator (#19249) --- esphome/components/esphome/ota/ota_esphome.cpp | 9 ++++++++- esphome/components/esphome/ota/ota_esphome.h | 4 ++-- esphome/components/esphome/ota/ota_esphome_noise.cpp | 6 ++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index f853ed6a2d..3010df1056 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -842,7 +842,14 @@ bool ESPHomeOTAComponent::handle_auth_send_() { const size_t hex_size = hasher.get_size() * 2; const size_t nonce_len = hasher.get_size() / 4; const size_t auth_buf_size = 1 + 3 * hex_size; - this->auth_buf_ = std::make_unique(auth_buf_size); + // Internal RAM first: 128 of these bytes go straight into the hardware SHA engine + this->auth_buf_ = + RAMAllocator(RAMAllocator::PREFER_INTERNAL).make_unique_array_for_overwrite(auth_buf_size); + if (!this->auth_buf_) { + this->log_auth_warning_(LOG_STR("No memory")); + this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_UNKNOWN); + return false; + } this->auth_buf_pos_ = 0; char *buf = reinterpret_cast(this->auth_buf_.get() + 1); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index c6f710b3fc..68dd0ffb9e 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -145,13 +145,13 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { #ifdef USE_OTA_PASSWORD std::string password_; - std::unique_ptr auth_buf_; + RAMUniquePtr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION #ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; #endif - std::unique_ptr noise_; + RAMUniquePtr noise_; #endif // USE_OTA_ENCRYPTION socket::ListenSocket *server_{nullptr}; diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7401413d6d..65476572a1 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -7,7 +7,6 @@ #include "esphome/core/log.h" #include -#include #ifdef USE_ESP8266 #include @@ -43,9 +42,8 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { // A provisioned key cleared between the offer and here is not guarded: the // session runs on the zero key load_psk fills in and fails the client's MAC. - // Default-init: the frame buffer is written before it is read - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); + // Default placement, PSRAM first where present: the session only lives for one upload + this->noise_ = RAMAllocator().make_unique(); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags From cf398ea8b212c55db7b5c70e219f6e589eb527e5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:27:03 -0500 Subject: [PATCH 135/178] [core] Resolve file paths against the YAML file that declares them (#19259) --- esphome/config_validation.py | 66 +++++++----- tests/unit_tests/test_config_validation.py | 120 ++++++++++++++++++++- 2 files changed, 159 insertions(+), 27 deletions(-) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index a38fb2ed82..1623117a36 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -16,6 +16,7 @@ from ipaddress import ( ip_network, ) import logging +import os from pathlib import Path import re from string import ascii_letters, digits @@ -1999,38 +2000,51 @@ def _remap_bundle_path(value: str) -> Path | None: return remap_bundle_path(value) -def directory(value: object) -> Path: - value = string(value) - path = CORE.relative_config_path(value) +def _declaring_document(value: str) -> Path | None: + """Return the on-disk YAML file *value* was loaded from, absolute, or None.""" + esp_range = getattr(value, "esp_range", None) + if esp_range is None: + return None + document = Path(esp_range.start_mark.document).absolute() + return document if document.is_file() else None - if not path.exists(): - remapped = _remap_bundle_path(value) - if remapped is None: + +def _existing_path(value: str, kind: str, is_kind: Callable[[Path], bool]) -> Path: + """Resolve *value* to a *kind* entry: config dir, then declaring document, then bundle remap.""" + path = CORE.relative_config_path(value) + if is_kind(path): + return path + candidates = [path] + tried_document: Path | None = None + if (document := _declaring_document(value)) is not None: + beside_document = document.parent / Path(value).expanduser() + if os.path.normpath(beside_document) != os.path.normpath(path): + candidates.append(beside_document) + tried_document = document + if (remapped := _remap_bundle_path(value)) is not None: + candidates.append(remapped) + for candidate in candidates: + if is_kind(candidate): + return candidate + for candidate in candidates: + if candidate.exists(): raise Invalid( - f"Could not find directory '{path}'. Please make sure it exists (full path: {path.resolve()})." + f"Path '{candidate}' is not a {kind} (full path: {candidate.resolve()})." ) - path = remapped - if not path.is_dir(): - raise Invalid( - f"Path '{path}' is not a directory (full path: {path.resolve()})." - ) - return path + also = ( + f" Also looked next to {tried_document}." if tried_document is not None else "" + ) + raise Invalid( + f"Could not find {kind} '{path}'. Please make sure it exists (full path: {path.resolve()}).{also}" + ) + + +def directory(value: object) -> Path: + return _existing_path(string(value), "directory", Path.is_dir) def file_(value: object) -> Path: - value = string(value) - path = CORE.relative_config_path(value) - - if not path.exists(): - remapped = _remap_bundle_path(value) - if remapped is None: - raise Invalid( - f"Could not find file '{path}'. Please make sure it exists (full path: {path.resolve()})." - ) - path = remapped - if not path.is_file(): - raise Invalid(f"Path '{path}' is not a file (full path: {path.resolve()}).") - return path + return _existing_path(string(value), "file", Path.is_file) ENTITY_ID_CHARACTERS = "abcdefghijklmnopqrstuvwxyz0123456789_" diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 4092b4c0d5..230a8e1f9e 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -1,4 +1,5 @@ import importlib +import io import json import logging from pathlib import Path @@ -20,6 +21,7 @@ from esphome.components.esp32 import ( VARIANT_ESP32S2, VARIANT_ESP32S3, ) +from esphome.components.substitutions import do_substitution_pass from esphome.config_validation import Invalid from esphome.const import ( CONF_DAY, @@ -65,7 +67,13 @@ from esphome.core import ( ) from esphome.schema_extractors import SCHEMA_EXTRACT from esphome.util import Registry -from esphome.yaml_util import ESPHomeDataBase, SensitiveStr, make_data_base +from esphome.yaml_util import ( + ESPHomeDataBase, + SensitiveStr, + load_yaml, + make_data_base, + parse_yaml, +) def test_check_not_templatable__invalid(): @@ -3174,6 +3182,116 @@ def test_file__existing_relative_path(setup_core: Path) -> None: assert cv.file_("partitions.csv") == setup_core / "partitions.csv" +def _package_value(setup_core: Path, path: str = "assets/ui.js") -> tuple[Path, str]: + """Write a package file next to an ``assets/`` dir; return the dir and its loaded *path* value.""" + package_dir = setup_core / ".esphome" / "packages" / "abc123" / "vendor" + (package_dir / "assets").mkdir(parents=True) + (package_dir / "assets" / "ui.js").write_text("js\n") + (package_dir / "device.yaml").write_text(f"path: {path}\n") + return package_dir, load_yaml(package_dir / "device.yaml")["path"] + + +def test_file__resolves_relative_to_the_declaring_document(setup_core: Path) -> None: + """A package's own asset path resolves against the package file when the config dir lacks it.""" + package_dir, value = _package_value(setup_core) + + assert cv.file_(value) == package_dir / "assets" / "ui.js" + + +def test_file__resolves_a_substituted_path_against_the_use_site( + setup_core: Path, +) -> None: + package_dir, _ = _package_value(setup_core) + (package_dir / "device.yaml").write_text( + "substitutions:\n ui: assets/ui.js\npath: ${ui}\n" + ) + config = do_substitution_pass(load_yaml(package_dir / "device.yaml")) + + assert cv.file_(config["path"]) == package_dir / "assets" / "ui.js" + + +def test_file__result_is_absolute_for_a_relative_document( + setup_core: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A document loaded by a cwd-relative path still yields an absolute result.""" + package_dir, _ = _package_value(setup_core) + monkeypatch.chdir(setup_core) + value = load_yaml(Path(".esphome/packages/abc123/vendor/device.yaml"))["path"] + + result = cv.file_(value) + + assert result.is_absolute() + assert result == package_dir / "assets" / "ui.js" + + +def test_file__config_dir_entry_of_the_wrong_kind_does_not_shadow_the_package( + setup_core: Path, +) -> None: + package_dir, value = _package_value(setup_core) + (setup_core / "assets" / "ui.js").mkdir(parents=True) + + assert cv.file_(value) == package_dir / "assets" / "ui.js" + + +def test_file__miss_names_the_declaring_document(setup_core: Path) -> None: + package_dir, value = _package_value(setup_core, "assets/other.js") + + with pytest.raises(Invalid, match="Could not find file") as excinfo: + cv.file_(value) + + assert f"Also looked next to {package_dir / 'device.yaml'}" in str(excinfo.value) + + +def test_file__document_spelled_through_dotdot_in_the_config_dir_adds_no_hint( + setup_core: Path, +) -> None: + (setup_core / "sub").mkdir() + (setup_core / "device.yaml").write_text("path: assets/other.js\n") + value = load_yaml(setup_core / "sub" / ".." / "device.yaml")["path"] + + with pytest.raises(Invalid) as excinfo: + cv.file_(value) + + assert "Also looked" not in str(excinfo.value) + + +def test_file__wrong_kind_beside_the_document_is_reported(setup_core: Path) -> None: + package_dir, value = _package_value(setup_core, "assets") + + with pytest.raises(Invalid, match="is not a file") as excinfo: + cv.file_(value) + + assert str(package_dir / "assets") in str(excinfo.value) + + +def test_file__config_dir_wins_over_the_declaring_document(setup_core: Path) -> None: + _, value = _package_value(setup_core) + (setup_core / "assets").mkdir() + (setup_core / "assets" / "ui.js").write_text("local\n") + + assert cv.file_(value) == setup_core / "assets" / "ui.js" + + +def test_file__declared_in_an_in_memory_document_is_not_resolved( + setup_core: Path, +) -> None: + """A value whose source document isn't on disk falls through to the config-dir error.""" + value = parse_yaml(Path(""), io.StringIO("path: assets/ui.js\n"))[ + "path" + ] + + with pytest.raises(Invalid, match="Could not find file"): + cv.file_(value) + + +def test_directory_resolves_relative_to_the_declaring_document( + setup_core: Path, +) -> None: + package_dir, value = _package_value(setup_core, "assets") + + assert cv.directory(value) == package_dir / "assets" + + def test_file__missing_raises(setup_core: Path) -> None: with pytest.raises(Invalid, match="Could not find file"): cv.file_("partitions.csv") From e38ee343b407373d1df7849bb1322e66119245f3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:35:38 -0500 Subject: [PATCH 136/178] [ethernet] Keep the W5500 SPI context in a static instance instead of the heap (#19248) --- .../components/ethernet/w5500_custom_spi.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/esphome/components/ethernet/w5500_custom_spi.cpp b/esphome/components/ethernet/w5500_custom_spi.cpp index ed4f149738..9c6b59582a 100644 --- a/esphome/components/ethernet/w5500_custom_spi.cpp +++ b/esphome/components/ethernet/w5500_custom_spi.cpp @@ -6,17 +6,21 @@ #include #include #include -#include namespace esphome::ethernet { namespace { -// Per-device context returned by init() and handed back to read/write/deinit. +// Context returned by init() and handed back to read/write/deinit. There is one W5500 per device, so a +// single static instance replaces a heap allocation that could fail. It is always clear when init() runs: +// esp_eth_mac_new_w5500() calls deinit() on every failure after init() succeeded, and nothing else +// uninstalls the driver struct W5500CustomSpiContext { spi_device_handle_t handle; SemaphoreHandle_t lock; }; +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - intentional mutable state +W5500CustomSpiContext w5500_context{}; // Transfers up to the ESP32 SPI hardware FIFO size (64 bytes) stay on the polling path; larger // transfers (the frame payloads) use the blocking, DMA-backed transmit. @@ -25,23 +29,20 @@ constexpr uint32_t W5500_SPI_LOCK_TIMEOUT_MS = 50; void *w5500_custom_spi_init(const void *spi_config) { const auto *config = static_cast(spi_config); - auto *ctx = new (std::nothrow) W5500CustomSpiContext{}; - if (ctx == nullptr) { - return nullptr; - } + auto *ctx = &w5500_context; // The W5500 SPI frame carries the 16-bit address in the command phase and the 8-bit control // byte in the address phase; mirror what the stock driver configures. spi_device_interface_config_t devcfg = *config->spi_devcfg; devcfg.command_bits = 16; devcfg.address_bits = 8; if (spi_bus_add_device(config->spi_host_id, &devcfg, &ctx->handle) != ESP_OK) { - delete ctx; + ctx->handle = nullptr; return nullptr; } ctx->lock = xSemaphoreCreateMutex(); if (ctx->lock == nullptr) { spi_bus_remove_device(ctx->handle); - delete ctx; + ctx->handle = nullptr; return nullptr; } return ctx; @@ -51,7 +52,7 @@ esp_err_t w5500_custom_spi_deinit(void *spi_ctx) { auto *ctx = static_cast(spi_ctx); spi_bus_remove_device(ctx->handle); vSemaphoreDelete(ctx->lock); - delete ctx; + *ctx = {}; return ESP_OK; } From f5afd141de41203c4a4b31465b295d3121c1db8d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:37:04 -0500 Subject: [PATCH 137/178] [ota] Allocate the signature block through RAMAllocator (#19251) --- esphome/components/ota/ota_signature_esp_idf.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/esphome/components/ota/ota_signature_esp_idf.cpp b/esphome/components/ota/ota_signature_esp_idf.cpp index 501d6ac241..2192a79441 100644 --- a/esphome/components/ota/ota_signature_esp_idf.cpp +++ b/esphome/components/ota/ota_signature_esp_idf.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -235,9 +234,11 @@ bool IDFOTABackend::verify_signed_image_(const esp_partition_t *incoming) { // runs mid-OTA on the loop task, on top of the caller's live 1 KB OTA buffer // and mbedtls's own ~1 KB verify scratch, so keeping it off the stack widens // a thin margin. One short-lived allocation right before reboot is not the - // fragmentation pattern the project guards against. nothrow so an OOM here - // fails closed like every other error path, rather than aborting. - std::unique_ptr block(new (std::nothrow) uint8_t[SIG_BLOCK_SIZE]); + // fragmentation pattern the project guards against. An OOM returns nullptr + // and fails closed like every other error path. Internal RAM first: the + // block is an esp_partition_read target. + auto block = + RAMAllocator(RAMAllocator::PREFER_INTERNAL).make_unique_array_for_overwrite(SIG_BLOCK_SIZE); if (!block) { OTA_IDF_SIG_LOG(ESP_LOGE, "out of memory"); return false; From e431bfcb38f0393521b984de4480dcfa1d8918aa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 19:28:24 -0500 Subject: [PATCH 138/178] [spi] Send ESP8266 writes through transferBytes instead of a heap copy (#19265) --- esphome/components/spi/spi_arduino.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/esphome/components/spi/spi_arduino.cpp b/esphome/components/spi/spi_arduino.cpp index 14428bed62..ae2d2906ed 100644 --- a/esphome/components/spi/spi_arduino.cpp +++ b/esphome/components/spi/spi_arduino.cpp @@ -44,16 +44,8 @@ class SPIDelegateHw : public SPIDelegate { #ifdef USE_RP2 this->channel_->transfer(ptr, nullptr, length); #elif defined(USE_ESP8266) - // ESP8266 SPI library requires the pointer to be word aligned, but the data may not be - // so we need to copy the data to a temporary buffer - if (reinterpret_cast(ptr) & 0x3) { - ESP_LOGVV(TAG, "SPI write buffer not word aligned, copying to temporary buffer"); - auto txbuf = std::vector(length); - memcpy(txbuf.data(), ptr, length); - this->channel_->writeBytes(txbuf.data(), length); - } else { - this->channel_->writeBytes(ptr, length); - } + // writeBytes() needs a word aligned pointer; transferBytes() bounces unaligned chunks through a stack buffer + this->channel_->transferBytes(ptr, nullptr, length); #else this->channel_->writeBytes(ptr, length); #endif From 7801cf4a8ac103a88e64c9855ae2560c2f5b9648 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:55:13 +1200 Subject: [PATCH 139/178] [core] Clear loaded_platforms on CORE.reset() (#19268) --- esphome/core/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 6e3f91af22..5fcad90a81 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -715,6 +715,7 @@ class EsphomeCore: self.defines = set() self.platformio_options = {} self.loaded_integrations = set() + self.loaded_platforms = set() self.component_ids = set() self.platform_counts = defaultdict(int) self.unique_ids = {} From 91b1a82a66aa30ed9a7c6c3f8fc54d9c11266dc9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 20:03:00 -0500 Subject: [PATCH 140/178] [api] Reuse overflow buffer storage instead of allocating per stalled write (#19093) --- esphome/components/api/__init__.py | 5 +- esphome/components/api/api_buffer.cpp | 35 +- esphome/components/api/api_buffer.h | 24 +- esphome/components/api/api_connection.cpp | 5 +- esphome/components/api/api_frame_helper.cpp | 2 +- esphome/components/api/api_frame_helper.h | 3 + .../components/api/api_frame_helper_noise.cpp | 20 +- .../components/api/api_overflow_buffer.cpp | 121 ++--- esphome/components/api/api_overflow_buffer.h | 93 ++-- tests/components/api/__init__.py | 17 + tests/components/api/test_api_buffer.cpp | 65 +++ tests/components/api/test_overflow_buffer.cpp | 510 ++++++++++++++++++ 12 files changed, 755 insertions(+), 145 deletions(-) create mode 100644 tests/components/api/__init__.py create mode 100644 tests/components/api/test_api_buffer.cpp create mode 100644 tests/components/api/test_overflow_buffer.cpp diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 6202e127bf..272b078690 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -350,10 +350,9 @@ CONFIG_SCHEMA = cv.All( ln882x=5, # Moderate RAM nrf52=4, # ~256KB RAM, BSD sockets, Thread (single HA controller) ): cv.int_range(min=1, max=20), - # Maximum queued send buffers per connection before dropping connection - # Each buffer uses ~8-12 bytes overhead plus actual message size + # Max queued messages per connection, and 2 KB of backlog per slot up + # to 64 KB (a lone message is exempt), before the connection is dropped # Platform defaults based on available RAM and typical message rates: - # CONF_MAX_SEND_QUEUE defaults are power of 2 for efficient modulo cv.SplitDefault( CONF_MAX_SEND_QUEUE, esp8266=4, # Limited RAM, need to fail fast diff --git a/esphome/components/api/api_buffer.cpp b/esphome/components/api/api_buffer.cpp index fc45a4e971..62a544b1a4 100644 --- a/esphome/components/api/api_buffer.cpp +++ b/esphome/components/api/api_buffer.cpp @@ -1,20 +1,37 @@ #include "api_buffer.h" -#include +#ifdef ESPHOME_DEBUG_API +#include "esphome/core/log.h" +#endif namespace esphome::api { +#ifdef ESPHOME_DEBUG_API +void APIBuffer::debug_check_drop_(size_t drop) const { + if (drop > this->size_) { + ESP_LOGE("api.buffer", "drop_front: drop=%zu size=%u", drop, this->size_); + abort(); + } +} +#endif + bool APIBuffer::grow_(size_t n) { - // nothrow (no zero-fill) so OOM is reportable; plain new aborts instead - // (NEW_OOM_ABORT on ESP8266 Arduino, exception stub on ESP-IDF). - // RAMAllocator is no fit here: unique_ptr needs delete[]-compatible memory. - std::unique_ptr new_data(new (std::nothrow) uint8_t[n]); - if (new_data == nullptr) + if (n > MAX_SIZE) return false; - if (this->size_) - std::memcpy(new_data.get(), this->data_.get(), this->size_); - this->data_ = std::move(new_data); + // realloc extends in place when it can, avoiding the copy + uint8_t *grown = RAMAllocator().reallocate(this->data_.get(), n); + if (grown == nullptr) + return false; + (void) this->data_.release(); // realloc already freed or reused the old block + this->data_.reset(grown); this->capacity_ = n; return true; } +uint8_t *APIBuffer::append(size_t n) { + const size_t old_size = this->size_; + if (!this->resize(old_size + n)) + return nullptr; + return this->data_.get() + old_size; +} + } // namespace esphome::api diff --git a/esphome/components/api/api_buffer.h b/esphome/components/api/api_buffer.h index 396dadbe58..7caa68aa4d 100644 --- a/esphome/components/api/api_buffer.h +++ b/esphome/components/api/api_buffer.h @@ -25,6 +25,7 @@ namespace esphome::api { /// writes in debug builds. class APIBuffer { public: + static constexpr size_t MAX_SIZE = UINT16_MAX; // API frames carry 16 bit lengths void clear() { this->size_ = 0; } /// Returns false if allocation fails; the buffer is left unchanged. [[nodiscard]] inline bool reserve(size_t n) ESPHOME_ALWAYS_INLINE { return n <= this->capacity_ || this->grow_(n); } @@ -36,9 +37,19 @@ class APIBuffer { [[nodiscard]] inline bool reserve_and_resize(size_t reserve_size, size_t new_size) ESPHOME_ALWAYS_INLINE { if (!this->reserve(std::max(reserve_size, new_size))) return false; - this->size_ = new_size; + this->size_ = static_cast(new_size); return true; } + /// Grow by n bytes; returns the new bytes, or nullptr on allocation failure. + [[nodiscard]] uint8_t *append(size_t n); + /// Drop the first `drop` bytes, sliding the rest down. Precondition: drop <= size(). + void drop_front(size_t drop) { +#ifdef ESPHOME_DEBUG_API + this->debug_check_drop_(drop); +#endif + this->size_ -= drop; + std::memmove(this->data_.get(), this->data_.get() + drop, this->size_); + } uint8_t *data() { return this->data_.get(); } const uint8_t *data() const { return this->data_.get(); } size_t size() const { return this->size_; } @@ -55,9 +66,14 @@ class APIBuffer { protected: bool grow_(size_t n); - std::unique_ptr data_; - size_t size_{0}; - size_t capacity_{0}; +#ifdef ESPHOME_DEBUG_API + void debug_check_drop_(size_t drop) const; +#endif + // RAMAllocator: PSRAM when available, and it reports failure where + // new (std::nothrow) still aborts on ESP-IDF without exceptions + RAMUniquePtr data_; + uint16_t size_{0}; + uint16_t capacity_{0}; }; } // namespace esphome::api diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index d910f6fc67..749eaeb392 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -364,7 +364,10 @@ void APIConnection::check_keepalive_(uint32_t now) { ESP_LOGVV(TAG, "Sending keepalive PING"); PingRequest req; this->flags_.sent_ping = this->send_message(req); - if (!this->flags_.sent_ping) { + if (this->flags_.sent_ping) { + // Quiet for a keepalive period and the ping is on its way: a one-off stall's storage can go + this->helper_->release_overflow_buffer(); + } else { // If we can't send the ping request directly (tx_buffer full), // schedule it at the front of the batch so it will be sent with priority ESP_LOGW(TAG, "Buffer full, ping queued"); diff --git a/esphome/components/api/api_frame_helper.cpp b/esphome/components/api/api_frame_helper.cpp index 38da444a18..41d1230aaa 100644 --- a/esphome/components/api/api_frame_helper.cpp +++ b/esphome/components/api/api_frame_helper.cpp @@ -171,7 +171,7 @@ APIError APIFrameHelper::write_raw_iov_(const struct iovec *iov, int iovcnt, uin return APIError::OK; // Queue unsent data into overflow buffer - if (!this->overflow_buf_.enqueue_iov(iov, iovcnt, total_write_len, static_cast(sent))) { + if (!this->overflow_buf_.enqueue_iov(iov, iovcnt, total_write_len, sent)) { HELPER_LOG("Overflow buffer full or out of memory, dropping connection"); this->state_ = State::FAILED; return APIError::SOCKET_WRITE_FAILED; diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h index ff8aa7834c..a68a0ad0d8 100644 --- a/esphome/components/api/api_frame_helper.h +++ b/esphome/components/api/api_frame_helper.h @@ -219,7 +219,10 @@ class APIFrameHelper { if (this->rx_buf_len_ == 0) { this->rx_buf_.release(); } + this->release_overflow_buffer(); } + // Free the send backlog storage once it has drained + void release_overflow_buffer() { this->overflow_buf_.release(); } protected: // Drain backlogged overflow data to the socket and handle errors. diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 29b2858aee..400cd1d9b8 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -67,15 +67,15 @@ APIError APINoiseFrameHelper::init() { } // init prologue - size_t old_size = prologue_.size(); - if (!prologue_.resize(old_size + PROLOGUE_INIT_LEN)) [[unlikely]] { + uint8_t *dst = prologue_.append(PROLOGUE_INIT_LEN); + if (dst == nullptr) [[unlikely]] { state_ = State::FAILED; return APIError::OUT_OF_MEMORY; } #ifdef USE_ESP8266 - memcpy_P(prologue_.data() + old_size, PROLOGUE_INIT, PROLOGUE_INIT_LEN); + memcpy_P(dst, PROLOGUE_INIT, PROLOGUE_INIT_LEN); #else - std::memcpy(prologue_.data() + old_size, PROLOGUE_INIT, PROLOGUE_INIT_LEN); + std::memcpy(dst, PROLOGUE_INIT, PROLOGUE_INIT_LEN); #endif state_ = State::CLIENT_HELLO; @@ -272,17 +272,17 @@ APIError APINoiseFrameHelper::state_action_client_hello_() { return handle_handshake_frame_error_(aerr); } // ignore contents, may be used in future for flags - // Resize for: existing prologue + 2 size bytes + frame data - size_t old_size = this->prologue_.size(); + // Append 2 size bytes + frame data to the prologue size_t rx_size = this->rx_buf_.size(); - if (!this->prologue_.resize(old_size + 2 + rx_size)) [[unlikely]] { + uint8_t *dst = this->prologue_.append(2 + rx_size); + if (dst == nullptr) [[unlikely]] { state_ = State::FAILED; return APIError::OUT_OF_MEMORY; } - this->prologue_[old_size] = (uint8_t) (rx_size >> 8); - this->prologue_[old_size + 1] = (uint8_t) rx_size; + dst[0] = (uint8_t) (rx_size >> 8); + dst[1] = (uint8_t) rx_size; if (rx_size > 0) { - std::memcpy(this->prologue_.data() + old_size + 2, this->rx_buf_.data(), rx_size); + std::memcpy(dst + 2, this->rx_buf_.data(), rx_size); } state_ = State::SERVER_HELLO; diff --git a/esphome/components/api/api_overflow_buffer.cpp b/esphome/components/api/api_overflow_buffer.cpp index 48d8fe18ba..0b5a874d4b 100644 --- a/esphome/components/api/api_overflow_buffer.cpp +++ b/esphome/components/api/api_overflow_buffer.cpp @@ -1,98 +1,91 @@ #include "api_overflow_buffer.h" #ifdef USE_API #include -#include namespace esphome::api { -APIOverflowBuffer::~APIOverflowBuffer() { - for (auto *entry : this->queue_) { - if (entry != nullptr) - Entry::destroy(entry); - } -} - ssize_t APIOverflowBuffer::try_drain(socket::Socket *socket) { - // socket->write() can re-enter this function: a log message emitted from an - // lwip callback during the write goes out over the API and lands back in the - // frame helper's write/drain path. If a nested drain ran here it would send - // and free the entry the outer drain is still holding, causing a double free. - // Report "no progress" instead; the outer drain keeps draining, and the - // nested send is enqueued behind the existing backlog. + // Nested call from inside socket->write(); see draining_ if (this->draining_) return 0; - // RAII so the flag is cleared on every return path struct DrainGuard { - explicit DrainGuard(bool &flag) : flag_(flag) { flag_ = true; } - ~DrainGuard() { this->flag_ = false; } - bool &flag_; - } guard(this->draining_); + APIOverflowBuffer &owner; + ~DrainGuard() { this->owner.draining_ = false; } + } guard{*this}; + this->draining_ = true; while (this->count_ > 0) { - Entry *front = this->queue_[this->head_]; + uint8_t *msg = this->buf_.data() + this->head_; + size_t len = msg[0] | (msg[1] << 8); - ssize_t sent = socket->write(front->current_data(), front->remaining()); - - if (sent <= 0) { - // -1 = error (caller checks errno for EWOULDBLOCK vs hard error) - // 0 = nothing sent (treat as no progress) + ssize_t sent = socket->write(msg + LEN_PREFIX, len); + if (sent <= 0) + return sent; + if (static_cast(sent) < len) { + // Step past the sent bytes and rewrite the prefix there; it lands on bytes already sent + this->head_ += sent; + len -= sent; + msg += sent; + msg[0] = len; + msg[1] = len >> 8; return sent; } - - if (static_cast(sent) < front->remaining()) { - // Partially sent, update offset and stop - front->offset += static_cast(sent); - return sent; - } - - // Entry fully sent — unlink it before freeing so a freed pointer is never - // reachable from the queue - this->queue_[this->head_] = nullptr; - this->head_ = (this->head_ + 1) % API_MAX_SEND_QUEUE; + this->head_ += LEN_PREFIX + len; this->count_--; - Entry::destroy(front); } - return 0; // All drained + this->head_ = 0; + if (this->release_when_drained_) { + this->release_when_drained_ = false; + this->buf_.release(); + } else { + this->buf_.clear(); + } + return 0; } -bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, uint16_t total_len, uint16_t skip) { +bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, size_t total_len, size_t skip) { if (this->count_ >= API_MAX_SEND_QUEUE) return false; - uint16_t buffer_size = total_len - skip; - // nothrow: a failed allocation returns nullptr so the connection is dropped - // cleanly instead of plain new's crash or abort on OOM - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - auto *data = new (std::nothrow) uint8_t[buffer_size]; - if (data == nullptr) - return false; - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - auto *entry = new (std::nothrow) Entry{data, buffer_size, 0}; - if (entry == nullptr) { - delete[] data; + const size_t new_len = total_len - skip; + const size_t new_bytes = LEN_PREFIX + new_len; + const size_t live = this->buf_.size() - this->head_; + // A lone message is only bound by the buffer; refusing it would just drop the connection + if (live + new_bytes > (this->count_ > 0 ? MAX_BYTES : MAX_LONE_BYTES)) return false; + + if (this->buf_.size() + new_bytes > this->buf_.capacity()) { + // Storage would move under an outer drain's write() + if (this->draining_) + return false; + if (this->head_ > 0) { + // Reclaim the sent prefix before growing + this->buf_.drop_front(this->head_); + this->head_ = 0; + } + if (!this->buf_.reserve(reserve_for(live + new_bytes))) + return false; } - uint16_t to_skip = skip; - uint16_t write_pos = 0; - - for (int i = 0; i < iovcnt; i++) { - if (to_skip >= iov[i].iov_len) { - to_skip -= static_cast(iov[i].iov_len); + uint8_t *dst = this->buf_.append(new_bytes); + if (dst == nullptr) + return false; + dst[0] = new_len; + dst[1] = new_len >> 8; + dst += LEN_PREFIX; + for (const struct iovec *end = iov + iovcnt; iov != end; iov++) { + if (skip >= iov->iov_len) { + skip -= iov->iov_len; } else { - const uint8_t *src = reinterpret_cast(iov[i].iov_base) + to_skip; - uint16_t len = static_cast(iov[i].iov_len) - to_skip; - std::memcpy(entry->data + write_pos, src, len); - write_pos += len; - to_skip = 0; + const size_t len = iov->iov_len - skip; + std::memcpy(dst, static_cast(iov->iov_base) + skip, len); + dst += len; + skip = 0; } } - // Publish only after the copy completes so a half-built entry is never reachable - this->queue_[this->tail_] = entry; - this->tail_ = (this->tail_ + 1) % API_MAX_SEND_QUEUE; this->count_++; return true; } diff --git a/esphome/components/api/api_overflow_buffer.h b/esphome/components/api/api_overflow_buffer.h index 03a334b281..e2e4b9c3c3 100644 --- a/esphome/components/api/api_overflow_buffer.h +++ b/esphome/components/api/api_overflow_buffer.h @@ -1,5 +1,6 @@ #pragma once -#include +#include +#include #include #include @@ -8,71 +9,57 @@ #include "esphome/components/socket/headers.h" #include "esphome/components/socket/socket.h" -#include "esphome/core/helpers.h" +#include "api_buffer.h" namespace esphome::api { -/// Circular queue of heap-allocated byte buffers used as a TCP send backlog. -/// -/// Under normal operation this buffer is **never used** — data goes straight -/// from the frame helper to the socket. It only fills when the LWIP TCP -/// send buffer is full (slow client, congested network, heavy logging). -/// The queue drains automatically on subsequent write/loop calls once the -/// socket becomes writable again. -/// -/// Capacity is compile-time-fixed via API_MAX_SEND_QUEUE (set from Python -/// config). If the queue fills completely the connection is marked failed. +/// TCP send backlog, only used when the socket send buffer is full. +/// One contiguous buffer per connection, allocated on the first stall and +/// kept at its high-water mark so a lossy link does not churn the heap. +/// Messages are stored as a 2 byte length prefix plus payload. +/// API_MAX_SEND_QUEUE bounds queued messages and, at 2 KB per slot, queued +/// bytes; exceeding either fails the connection. class APIOverflowBuffer { public: - /// A single heap-allocated send-backlog entry. - /// Lifetime is manually managed — see destroy(). - struct Entry { - uint8_t *data; - uint16_t size; // Total size of the buffer - uint16_t offset; // Current send offset within the buffer - - uint16_t remaining() const { return this->size - this->offset; } - const uint8_t *current_data() const { return this->data + this->offset; } - - /// Free this entry and its data buffer. - static ESPHOME_ALWAYS_INLINE void destroy(Entry *entry) { - delete[] entry->data; - delete entry; // NOLINT(cppcoreguidelines-owning-memory) - } - }; - - ~APIOverflowBuffer(); - /// True when no backlogged data is waiting. bool empty() const { return this->count_ == 0; } - /// True when the queue has no room for another entry. - bool full() const { return this->count_ >= API_MAX_SEND_QUEUE; } - - /// Number of entries currently queued. - uint8_t count() const { return this->count_; } - - /// Try to drain queued data to the socket. - /// Returns bytes-written > 0 on success/partial, 0 if all drained or no progress, - /// -1 on error (caller must check errno to distinguish EWOULDBLOCK from hard errors). - /// Callers only need to act on -1; 0 and positive values both mean "no error". - /// Frees entries as they are fully sent. + /// Drain queued messages to the socket. + /// Returns bytes written, 0 for a re-entrant call, -1 on error (check errno + /// for EWOULDBLOCK); callers only need to act on -1. ssize_t try_drain(socket::Socket *socket); - /// Enqueue unsent IOV data into the backlog. - /// Copies iov data starting at byte offset `skip` into a new entry. - /// Returns false if the queue is full or allocation fails (caller should fail the connection). - bool enqueue_iov(const struct iovec *iov, int iovcnt, uint16_t total_len, uint16_t skip); + /// Queue iov data from byte offset `skip` as one message. + /// Returns false when a limit is hit, allocation fails, or storage would move + /// during a drain; the caller should fail the connection. + bool enqueue_iov(const struct iovec *iov, int iovcnt, size_t total_len, size_t skip); + + /// Free the retained storage, now if empty, otherwise once it has drained. + void release() { + if (this->count_ == 0) { + this->buf_.release(); + } else { + this->release_when_drained_ = true; + } + } protected: - std::array queue_{}; - uint8_t head_{0}; - uint8_t tail_{0}; + static constexpr size_t LEN_PREFIX = 2; + static constexpr size_t BYTES_PER_SLOT = 2048; + // Reserve in 256 byte steps so a creeping high-water mark settles quickly + static constexpr size_t GROW_QUANTUM = 256; + // Lone message ceiling, rounded down so reserve_for() never exceeds the buffer limit + static constexpr size_t MAX_LONE_BYTES = APIBuffer::MAX_SIZE & ~(GROW_QUANTUM - 1); + static constexpr size_t MAX_BYTES = std::min(API_MAX_SEND_QUEUE * BYTES_PER_SLOT, MAX_LONE_BYTES); + static constexpr size_t reserve_for(size_t want) { return (want + GROW_QUANTUM - 1) & ~(GROW_QUANTUM - 1); } + + APIBuffer buf_; + uint16_t head_{0}; // offset of the front message's length prefix; bytes before it are sent uint8_t count_{0}; - // Guards against re-entrant drains: socket->write() can re-enter the API - // send path (e.g. a log message emitted from an lwip callback), and a nested - // drain would free the entry the outer drain is still holding. - bool draining_{false}; + // socket->write() can re-enter the send path (log from an lwip callback): + // a nested drain makes no progress and a nested enqueue never moves storage + bool draining_ : 1 {false}; + bool release_when_drained_ : 1 {false}; }; } // namespace esphome::api diff --git a/tests/components/api/__init__.py b/tests/components/api/__init__.py new file mode 100644 index 0000000000..2aa558726c --- /dev/null +++ b/tests/components/api/__init__.py @@ -0,0 +1,17 @@ +import esphome.codegen as cg +from esphome.core import CORE +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + # USE_API compiles every api source, so emit what they need. No socket + # override: an __init__.py there makes pytest import its conftest as socket.conftest. + async def to_code_testing(config): + cg.add_define("USE_API") + cg.add_define("USE_API_PLAINTEXT") + cg.add_define("API_MAX_SEND_QUEUE", 8) + cg.add_define("MAX_API_CONNECTIONS", 1) + cg.add_define("USE_SOCKET_IMPL_BSD_SOCKETS") + CORE.register_controller() # api_server registers with the controller registry + + manifest.to_code = to_code_testing diff --git a/tests/components/api/test_api_buffer.cpp b/tests/components/api/test_api_buffer.cpp new file mode 100644 index 0000000000..c54780050e --- /dev/null +++ b/tests/components/api/test_api_buffer.cpp @@ -0,0 +1,65 @@ +#include + +#include +#include + +#include "esphome/components/api/api_buffer.h" + +namespace esphome::api::testing { + +// Pointer plus two 16 bit sizes +static_assert(sizeof(APIBuffer) <= 2 * sizeof(void *)); + +TEST(APIBuffer, RefusesSizesAbove16Bits) { + APIBuffer buf; + ASSERT_TRUE(buf.resize(16)); + EXPECT_FALSE(buf.reserve(UINT16_MAX + 1)); + EXPECT_EQ(buf.size(), 16u); + EXPECT_EQ(buf.capacity(), 16u); + EXPECT_TRUE(buf.reserve(UINT16_MAX)); + EXPECT_EQ(buf.capacity(), UINT16_MAX); +} + +static const uint8_t BYTES[] = {1, 2, 3, 4, 5, 6}; + +TEST(APIBuffer, AppendReturnsTheNewBytes) { + APIBuffer buf; + ASSERT_TRUE(buf.reserve(8)); + uint8_t *first = buf.append(3); + ASSERT_NE(first, nullptr); + std::memcpy(first, BYTES, 3); + EXPECT_EQ(buf.size(), 3u); + EXPECT_EQ(buf.capacity(), 8u); + + // Grows through realloc and keeps what was there + uint8_t *second = buf.append(6); + ASSERT_EQ(second, buf.data() + 3); + std::memcpy(second, BYTES + 3, 3); + EXPECT_EQ(buf.size(), 9u); + EXPECT_EQ(buf.capacity(), 9u); + EXPECT_EQ(std::memcmp(buf.data(), BYTES, 6), 0); +} + +TEST(APIBuffer, DropFrontSlidesTheRestDown) { + APIBuffer buf; + uint8_t *bytes = buf.append(6); + ASSERT_NE(bytes, nullptr); + std::memcpy(bytes, BYTES, 6); + + buf.drop_front(2); + EXPECT_EQ(buf.size(), 4u); + EXPECT_EQ(buf.capacity(), 6u); + EXPECT_EQ(std::memcmp(buf.data(), BYTES + 2, 4), 0); + + // Growing afterwards keeps the slid bytes + ASSERT_TRUE(buf.reserve(64)); + EXPECT_EQ(buf.size(), 4u); + EXPECT_EQ(std::memcmp(buf.data(), BYTES + 2, 4), 0); + + // Dropping everything leaves an empty buffer with its capacity + buf.drop_front(4); + EXPECT_EQ(buf.size(), 0u); + EXPECT_EQ(buf.capacity(), 64u); +} + +} // namespace esphome::api::testing diff --git a/tests/components/api/test_overflow_buffer.cpp b/tests/components/api/test_overflow_buffer.cpp new file mode 100644 index 0000000000..4b27e54496 --- /dev/null +++ b/tests/components/api/test_overflow_buffer.cpp @@ -0,0 +1,510 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "esphome/components/api/api_overflow_buffer.h" + +#ifdef USE_HOST +namespace esphome::api::testing { + +// Idle cost is the buffer plus one word of bookkeeping +static_assert(sizeof(APIOverflowBuffer) <= sizeof(APIBuffer) + sizeof(void *)); + +// Exposes storage so tests can check it is reused, not reallocated +class TestOverflowBuffer : public APIOverflowBuffer { + public: + using APIOverflowBuffer::LEN_PREFIX; + using APIOverflowBuffer::MAX_BYTES; + using APIOverflowBuffer::MAX_LONE_BYTES; + struct Storage { + size_t capacity; + const uint8_t *data; + bool operator==(const Storage &) const = default; + }; + size_t capacity() const { return this->buf_.capacity(); } + Storage storage() const { return {this->buf_.capacity(), this->buf_.data()}; } + uint8_t count() const { return this->count_; } + size_t live() const { return this->buf_.size() - this->head_; } + /// Simulates a socket write inside try_drain() re-entering the send path + void set_draining(bool draining) { this->draining_ = draining; } +}; + +static std::vector make_message(size_t len, uint8_t seed) { + std::vector msg(len); + for (size_t i = 0; i < len; i++) + msg[i] = static_cast(seed + i); + return msg; +} + +static bool enqueue(TestOverflowBuffer &buf, const std::vector &msg, uint16_t skip = 0) { + struct iovec iov = {const_cast(msg.data()), msg.size()}; + return buf.enqueue_iov(&iov, 1, static_cast(msg.size()), skip); +} + +static void append(std::vector &dst, const std::vector &src, size_t skip = 0) { + dst.insert(dst.end(), src.begin() + skip, src.end()); +} + +static std::vector concat(std::initializer_list> parts) { + std::vector out; + for (const auto &part : parts) + append(out, part); + return out; +} + +/// The pipe delivers the filler first, then the drained messages. +static void expect_after_filler(const std::vector &received, size_t filler, + const std::vector &expected) { + ASSERT_EQ(received.size(), filler + expected.size()); + EXPECT_TRUE(std::equal(expected.begin(), expected.end(), received.begin() + filler)); +} + +// Non-blocking socket pair with small buffers, so the writer fills like a stalled TCP connection +class OverflowBufferTest : public ::testing::Test { + protected: + void SetUp() override { + int fds[2]; + ASSERT_EQ(::socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + int size = 4096; + ASSERT_EQ(::setsockopt(fds[0], SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)), 0); + ASSERT_EQ(::setsockopt(fds[1], SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)), 0); + ASSERT_EQ(::fcntl(fds[1], F_SETFL, O_NONBLOCK), 0); + this->reader_ = fds[1]; + this->sock_ = std::make_unique(fds[0]); + ASSERT_EQ(this->sock_->setblocking(false), 0); + } + void TearDown() override { ::close(this->reader_); } + + /// Write filler until the socket refuses; returns the bytes accepted + size_t fill_pipe_() { + uint8_t junk[512]; + std::memset(junk, 0xEE, sizeof(junk)); + size_t total = 0; + for (;;) { + ssize_t written = this->sock_->write(junk, sizeof(junk)); + if (written <= 0) + break; + total += static_cast(written); + } + return total; + } + + /// Append whatever the pipe currently holds. + void read_into_(std::vector &out) { + uint8_t tmp[1024]; + for (;;) { + ssize_t n = ::read(this->reader_, tmp, sizeof(tmp)); + if (n <= 0) + break; + out.insert(out.end(), tmp, tmp + n); + } + } + + /// Drain once; a refusal must be a would-block, never a hard error. + ssize_t drain_(TestOverflowBuffer &buf) { + ssize_t sent = buf.try_drain(this->sock_.get()); + if (sent == -1) { + EXPECT_TRUE(errno == EWOULDBLOCK || errno == EAGAIN); + } + return sent; + } + + /// Read and drain until the backlog is empty; returns all bytes received + std::vector drain_all_(TestOverflowBuffer &buf) { + std::vector received; + for (int i = 0; i < 10000 && !buf.empty(); i++) { + this->read_into_(received); + // A hard socket error would never clear the backlog; stop instead of spinning + if (this->drain_(buf) == -1 && errno != EWOULDBLOCK && errno != EAGAIN) + break; + } + EXPECT_TRUE(buf.empty()); + this->read_into_(received); + return received; + } + + struct Stall { + size_t filler; + std::vector first, second, received; + TestOverflowBuffer::Storage before; + }; + /// Park two messages, then drain the first fully and the second part way + void stall_mid_message_(TestOverflowBuffer &buf, Stall &s) { + s.filler = this->fill_pipe_(); + s.first = make_message(1500, 20); + ASSERT_GT(s.filler, s.first.size()); // the first message must drain in one go + // Larger than the whole pipe, so a drain always stops inside it + s.second = make_message(std::max(s.filler + 1, std::min(s.filler * 3, 12000)), 60); + ASSERT_GT(s.second.size(), s.filler); + ASSERT_TRUE(enqueue(buf, s.first)); + ASSERT_TRUE(enqueue(buf, s.second)); + s.before = buf.storage(); + this->read_into_(s.received); + ASSERT_GT(this->drain_(buf), 0); + ASSERT_EQ(buf.count(), 1); + } + + int reader_{-1}; + std::unique_ptr sock_; +}; + +TEST_F(OverflowBufferTest, IdleBufferOwnsNoStorage) { + TestOverflowBuffer buf; + EXPECT_TRUE(buf.empty()); + EXPECT_EQ(buf.capacity(), 0u); + EXPECT_EQ(buf.storage().data, nullptr); +} + +TEST_F(OverflowBufferTest, StorageIsReusedAcrossStalls) { + TestOverflowBuffer buf; + auto msg = make_message(1000, 1); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + const auto storage = buf.storage(); + EXPECT_GE(storage.capacity, msg.size() + TestOverflowBuffer::LEN_PREFIX); + + for (int stall = 0; stall < 5; stall++) { + expect_after_filler(this->drain_all_(buf), filler, msg); + EXPECT_TRUE(buf.empty()); + // Same allocation every time: no free, no new allocation + EXPECT_EQ(buf.storage(), storage); + + filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + EXPECT_EQ(buf.storage(), storage); + } +} + +TEST_F(OverflowBufferTest, ReleaseWhileQueuedFreesOnceDrained) { + TestOverflowBuffer buf; + auto msg = make_message(1000, 7); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + const size_t capacity = buf.capacity(); + + // Requested while the backlog still holds data: storage must stay until sent + buf.release(); + EXPECT_FALSE(buf.empty()); + EXPECT_EQ(buf.capacity(), capacity); + + expect_after_filler(this->drain_all_(buf), filler, msg); + EXPECT_TRUE(buf.empty()); + EXPECT_EQ(buf.capacity(), 0u); + EXPECT_EQ(buf.storage().data, nullptr); + + // A later stall allocates again and keeps it, since nobody asked for a release + filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + EXPECT_GT(buf.capacity(), 0u); + this->drain_all_(buf); + EXPECT_GT(buf.capacity(), 0u); +} + +TEST_F(OverflowBufferTest, ReleaseWhenEmptyFreesImmediately) { + TestOverflowBuffer buf; + auto msg = make_message(100, 3); + + this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + this->drain_all_(buf); + EXPECT_GT(buf.capacity(), 0u); + + buf.release(); + EXPECT_EQ(buf.capacity(), 0u); + EXPECT_EQ(buf.storage().data, nullptr); +} + +TEST_F(OverflowBufferTest, PreservesOrderAndSkipsSentPrefix) { + TestOverflowBuffer buf; + auto first = make_message(700, 10); + auto second_a = make_message(300, 50); + auto second_b = make_message(400, 90); + auto third = make_message(200, 130); + + size_t filler = this->fill_pipe_(); + // 100 bytes of the first message were already accepted by the socket + ASSERT_TRUE(enqueue(buf, first, 100)); + // Two iovecs with the skip covering all of the first one plus part of the second + struct iovec iov[2] = {{second_a.data(), second_a.size()}, {second_b.data(), second_b.size()}}; + const uint16_t second_skip = static_cast(second_a.size() + 5); + ASSERT_TRUE(buf.enqueue_iov(iov, 2, static_cast(second_a.size() + second_b.size()), second_skip)); + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.count(), 3); + + // Nothing can go out while the pipe is full + EXPECT_EQ(this->drain_(buf), -1); + EXPECT_EQ(buf.count(), 3); + + std::vector expected; + append(expected, first, 100); + append(expected, second_b, 5); + append(expected, third); + expect_after_filler(this->drain_all_(buf), filler, expected); +} + +TEST_F(OverflowBufferTest, RefusesWhenQueueIsFull) { + TestOverflowBuffer buf; + auto msg = make_message(16, 1); + + size_t filler = this->fill_pipe_(); + for (int i = 0; i < API_MAX_SEND_QUEUE; i++) { + ASSERT_TRUE(enqueue(buf, msg)) << "message " << i; + } + EXPECT_FALSE(enqueue(buf, msg)); + EXPECT_EQ(buf.count(), API_MAX_SEND_QUEUE); + + // Draining frees the slots again + std::vector expected; + for (int i = 0; i < API_MAX_SEND_QUEUE; i++) + append(expected, msg); + expect_after_filler(this->drain_all_(buf), filler, expected); + this->fill_pipe_(); + EXPECT_TRUE(enqueue(buf, msg)); + EXPECT_EQ(buf.count(), 1); +} + +TEST_F(OverflowBufferTest, SkipAtIovecBoundary) { + TestOverflowBuffer buf; + auto sent = make_message(300, 50); + auto unsent = make_message(400, 90); + + size_t filler = this->fill_pipe_(); + // The skip covers the first iovec exactly, so only the second is copied + struct iovec iov[2] = {{sent.data(), sent.size()}, {unsent.data(), unsent.size()}}; + ASSERT_TRUE( + buf.enqueue_iov(iov, 2, static_cast(sent.size() + unsent.size()), static_cast(sent.size()))); + EXPECT_EQ(buf.live(), unsent.size() + TestOverflowBuffer::LEN_PREFIX); + expect_after_filler(this->drain_all_(buf), filler, unsent); +} + +TEST_F(OverflowBufferTest, AppendsBehindSentPrefixWhenItFits) { + TestOverflowBuffer buf; + size_t filler = this->fill_pipe_(); + auto first = make_message(200, 20); + // Size the second message so the two land half way into a 256 byte step, + // leaving exactly 128 bytes of slack whatever the pipe accepted + const size_t base = std::max(filler + 1, std::min(filler * 3, 12000)); + const size_t second_len = (base / 256 + 1) * 256 + 128 - first.size() - 2 * TestOverflowBuffer::LEN_PREFIX; + auto second = make_message(second_len, 60); + ASSERT_GT(second.size(), filler); + ASSERT_TRUE(enqueue(buf, first)); + ASSERT_TRUE(enqueue(buf, second)); + const auto storage = buf.storage(); + const size_t slack = storage.capacity - first.size() - second.size() - 2 * TestOverflowBuffer::LEN_PREFIX; + ASSERT_EQ(slack, 128u); + auto third = make_message(slack - TestOverflowBuffer::LEN_PREFIX, 200); + + std::vector received; + this->read_into_(received); + ASSERT_GT(this->drain_(buf), 0); + ASSERT_EQ(buf.count(), 1); + const size_t live = buf.live(); + + // Fits in the tail, so the sent prefix is left alone + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.storage(), storage); + EXPECT_EQ(buf.live(), live + third.size() + TestOverflowBuffer::LEN_PREFIX); + + append(received, this->drain_all_(buf)); + expect_after_filler(received, filler, concat({first, second, third})); +} + +TEST_F(OverflowBufferTest, ReleaseSurvivesFurtherEnqueues) { + TestOverflowBuffer buf; + auto first = make_message(300, 7); + auto second = make_message(300, 70); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, first)); + buf.release(); + ASSERT_TRUE(enqueue(buf, second)); + EXPECT_GT(buf.capacity(), 0u); + + expect_after_filler(this->drain_all_(buf), filler, concat({first, second})); + EXPECT_EQ(buf.capacity(), 0u); +} + +TEST_F(OverflowBufferTest, RefusesWhenByteLimitIsExceeded) { + TestOverflowBuffer buf; + // Two of these fill the byte budget exactly, well before the slot count is reached + static_assert(API_MAX_SEND_QUEUE >= 3); + auto msg = make_message(TestOverflowBuffer::MAX_BYTES / 2 - TestOverflowBuffer::LEN_PREFIX, 1); + + this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + ASSERT_TRUE(enqueue(buf, msg)); + EXPECT_FALSE(enqueue(buf, msg)); + EXPECT_EQ(buf.count(), 2); +} + +TEST_F(OverflowBufferTest, LoneMessageMayExceedByteLimit) { + TestOverflowBuffer buf; + // The oversized message must still fit under the lone message ceiling + static_assert(TestOverflowBuffer::MAX_BYTES + 100 + TestOverflowBuffer::LEN_PREFIX <= + TestOverflowBuffer::MAX_LONE_BYTES); + auto big = make_message(TestOverflowBuffer::MAX_BYTES + 100, 5); + auto small = make_message(16, 9); + + // Refusing the only message would drop the connection for nothing + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, big)); + EXPECT_EQ(buf.count(), 1); + // With a backlog present the byte limit applies again + EXPECT_FALSE(enqueue(buf, small)); + EXPECT_EQ(buf.count(), 1); + + expect_after_filler(this->drain_all_(buf), filler, big); +} + +TEST_F(OverflowBufferTest, LoneMessageAboveOffsetLimitIsRefused) { + TestOverflowBuffer buf; + // Payload plus prefix is past the lone message ceiling + auto msg = make_message(TestOverflowBuffer::MAX_LONE_BYTES, 3); + + this->fill_pipe_(); + EXPECT_FALSE(enqueue(buf, msg)); + EXPECT_TRUE(buf.empty()); + EXPECT_EQ(buf.capacity(), 0u); +} + +TEST_F(OverflowBufferTest, HardSocketErrorLeavesBacklogIntact) { + TestOverflowBuffer buf; + auto msg = make_message(300, 40); + + this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + // A closed socket fails every write outright, unlike a full one + ASSERT_EQ(this->sock_->close(), 0); + + errno = 0; + EXPECT_EQ(buf.try_drain(this->sock_.get()), -1); + EXPECT_NE(errno, EWOULDBLOCK); + EXPECT_NE(errno, EAGAIN); + EXPECT_EQ(buf.count(), 1); + EXPECT_EQ(buf.live(), msg.size() + TestOverflowBuffer::LEN_PREFIX); +} + +TEST_F(OverflowBufferTest, GrowsWhileReclaimingSentPrefix) { + TestOverflowBuffer buf; + Stall s; + ASSERT_NO_FATAL_FAILURE(this->stall_mid_message_(buf, s)); + + // One byte too many to fit even after the sent prefix is reclaimed: grows in one copy + auto third = make_message(s.before.capacity - buf.live() + 1, 200); + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_GT(buf.capacity(), s.before.capacity); + EXPECT_EQ(buf.count(), 2); + + append(s.received, this->drain_all_(buf)); + expect_after_filler(s.received, s.filler, concat({s.first, s.second, third})); +} + +TEST_F(OverflowBufferTest, NestedDrainMakesNoProgress) { + TestOverflowBuffer buf; + auto msg = make_message(300, 40); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + std::vector received; + this->read_into_(received); + + // Room is available, but a nested drain must leave the outer one's message alone + buf.set_draining(true); + EXPECT_EQ(this->drain_(buf), 0); + EXPECT_EQ(buf.count(), 1); + std::vector nothing; + this->read_into_(nothing); + EXPECT_TRUE(nothing.empty()); + + buf.set_draining(false); + append(received, this->drain_all_(buf)); + expect_after_filler(received, filler, msg); +} + +TEST_F(OverflowBufferTest, NestedEnqueueAppendsWithinCapacity) { + TestOverflowBuffer buf; + auto first = make_message(500, 10); + auto second = make_message(4, 90); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, first)); + const auto storage = buf.storage(); + ASSERT_GE(storage.capacity, first.size() + second.size() + 2 * TestOverflowBuffer::LEN_PREFIX); + + buf.set_draining(true); + EXPECT_TRUE(enqueue(buf, second)); + EXPECT_EQ(buf.count(), 2); + EXPECT_EQ(buf.storage(), storage); + buf.set_draining(false); + + expect_after_filler(this->drain_all_(buf), filler, concat({first, second})); +} + +TEST_F(OverflowBufferTest, NestedEnqueueRefusesToGrow) { + TestOverflowBuffer buf; + auto first = make_message(500, 10); + auto second = make_message(100, 90); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, first)); + const auto storage = buf.storage(); + ASSERT_LT(storage.capacity, first.size() + second.size() + 2 * TestOverflowBuffer::LEN_PREFIX); + + // Growing would free the bytes the outer write() is sending from + buf.set_draining(true); + EXPECT_FALSE(enqueue(buf, second)); + EXPECT_EQ(buf.count(), 1); + EXPECT_EQ(buf.storage(), storage); + buf.set_draining(false); + + expect_after_filler(this->drain_all_(buf), filler, first); +} + +TEST_F(OverflowBufferTest, NestedEnqueueRefusesToCompact) { + TestOverflowBuffer buf; + Stall s; + ASSERT_NO_FATAL_FAILURE(this->stall_mid_message_(buf, s)); + auto third = make_message(1000, 200); + + // Sliding the remainder down would move the bytes the outer write() points at + buf.set_draining(true); + EXPECT_FALSE(enqueue(buf, third)); + EXPECT_EQ(buf.count(), 1); + EXPECT_EQ(buf.storage(), s.before); + buf.set_draining(false); + + // Once the drain is over the same enqueue compacts and succeeds + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.storage(), s.before); + append(s.received, this->drain_all_(buf)); + expect_after_filler(s.received, s.filler, concat({s.first, s.second, third})); +} + +TEST_F(OverflowBufferTest, CompactsInsteadOfGrowingAfterPartialDrain) { + TestOverflowBuffer buf; + Stall s; + ASSERT_NO_FATAL_FAILURE(this->stall_mid_message_(buf, s)); + auto third = make_message(1000, 200); + + // The sent first message is reclaimed by sliding the remainder down, not by reallocating + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.storage(), s.before); + + append(s.received, this->drain_all_(buf)); + expect_after_filler(s.received, s.filler, concat({s.first, s.second, third})); +} + +} // namespace esphome::api::testing +#endif // USE_HOST From ccec6e72bfbcb8ff498a3467840e591662885958 Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Wed, 9 Sep 2026 14:22:23 +0200 Subject: [PATCH 141/178] [sendspin] Fix codec enum codegen when codecs is not set (#19055) --- esphome/components/sendspin/__init__.py | 2 +- tests/components/sendspin/common-media_source.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 8ef11a7f90..c1970ab132 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -307,7 +307,7 @@ async def to_code(config: ConfigType) -> None: player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - codecs = player_cfg[CONF_CODECS] + codecs = [CODECS[codec] for codec in player_cfg[CONF_CODECS]] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 0c136fbd43..1977b79c04 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,4 +9,3 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal - codecs: [pcm, opus, flac] From ad4ee1d34e957cb867a291c24fcb20824031471e Mon Sep 17 00:00:00 2001 From: Robin Thoni Date: Thu, 10 Sep 2026 06:06:44 +0200 Subject: [PATCH 142/178] [network] Improve `network::is_connected()` to better handle multiple interfaces (#18999) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/network/util.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/esphome/components/network/util.h b/esphome/components/network/util.h index 65a578c22f..57c5a66833 100644 --- a/esphome/components/network/util.h +++ b/esphome/components/network/util.h @@ -26,30 +26,34 @@ namespace esphome::network { /// Return whether the node is connected to the network (through wifi, eth, ...) ESPHOME_ALWAYS_INLINE inline bool is_connected() { + // With a single interface enabled the checks below collapse to `if (x) return true; return false;`, which + // clang-tidy wants folded into one return. Keep the per-interface form so every enabled interface is checked. + // NOLINTBEGIN(readability-simplify-boolean-expr) #ifdef USE_ETHERNET if (ethernet::global_eth_component != nullptr && ethernet::global_eth_component->is_connected()) return true; #endif #ifdef USE_MODEM - if (modem::global_modem_component != nullptr) - return modem::global_modem_component->is_connected(); + if (modem::global_modem_component != nullptr && modem::global_modem_component->is_connected()) + return true; #endif #ifdef USE_WIFI - if (wifi::global_wifi_component != nullptr) - return wifi::global_wifi_component->is_connected(); + if (wifi::global_wifi_component != nullptr && wifi::global_wifi_component->is_connected()) + return true; #endif #ifdef USE_OPENTHREAD - if (openthread::global_openthread_component != nullptr) - return openthread::global_openthread_component->is_connected(); + if (openthread::global_openthread_component != nullptr && openthread::global_openthread_component->is_connected()) + return true; #endif #ifdef USE_HOST return true; // Assume it's connected #endif return false; + // NOLINTEND(readability-simplify-boolean-expr) } /// Return whether the network is disabled: every configured interface with a From 6bd6603d523160dfa2a7f0ec0b997cc606c498de Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:22:30 +0000 Subject: [PATCH 143/178] Bump bundled esphome-device-builder to 1.14.6 (#19072) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ac84ee4689..cfa47fbdad 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.6 RUN \ platformio settings set enable_telemetry No \ From 3b499ecb3e538c0f5bb20a4f166053004147840c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 11 Sep 2026 03:20:30 -0500 Subject: [PATCH 144/178] [core] Support set_internal() during setup, log error after setup (#19069) --- esphome/core/entity_base.cpp | 9 ++++ esphome/core/entity_base.h | 27 ++++++++---- .../fixtures/set_internal_at_boot.yaml | 34 +++++++++++++++ .../integration/test_set_internal_at_boot.py | 41 +++++++++++++++++++ 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 tests/integration/fixtures/set_internal_at_boot.yaml create mode 100644 tests/integration/test_set_internal_at_boot.py diff --git a/esphome/core/entity_base.cpp b/esphome/core/entity_base.cpp index 21a5fc3706..dc27c1e56a 100644 --- a/esphome/core/entity_base.cpp +++ b/esphome/core/entity_base.cpp @@ -56,6 +56,15 @@ void EntityBase::configure_entity_(const char *name, uint32_t object_id_hash, ui this->flags_.entity_category = (entity_fields >> ENTITY_FIELD_ENTITY_CATEGORY_SHIFT) & 0x3; } +void EntityBase::set_internal(bool internal) { + // Remove the after-setup path in 2027.3.0 and ignore the call instead. + if (App.is_setup_complete()) { + ESP_LOGE(TAG, "'%s': set_internal() after setup is undefined behavior, stops working in 2027.3.0", + this->get_name().c_str()); + } + this->flags_.internal = internal; +} + // Weak default lookup functions — overridden by generated code in main.cpp __attribute__((weak)) const char *entity_device_class_lookup(uint8_t) { return ""; } __attribute__((weak)) const char *entity_uom_lookup(uint8_t) { return ""; } diff --git a/esphome/core/entity_base.h b/esphome/core/entity_base.h index f38e30bf52..8796e9f067 100644 --- a/esphome/core/entity_base.h +++ b/esphome/core/entity_base.h @@ -88,13 +88,26 @@ class EntityBase { // Get whether this Entity should be hidden outside ESPHome bool is_internal() const { return this->flags_.internal; } - // Deprecated: Calling set_internal() at runtime is undefined behavior. Components and clients - // are NOT notified of the change, the flag may have already been read during setup, and there - // is NO guarantee any consumer will observe the new value. Use the 'internal:' YAML key instead. - ESPDEPRECATED("set_internal() is undefined behavior at runtime — components and Home Assistant are NOT " - "notified. Use the 'internal:' YAML key instead. Will be removed in 2027.3.0.", - "2026.3.0") - void set_internal(bool internal) { this->flags_.internal = internal; } + // Set whether this Entity should be hidden outside ESPHome. Prefer the 'internal:' YAML key + // whenever possible: it is guaranteed and has none of the limitations below. Use this only when + // the decision can only be made at boot. Must be called before MQTT and the API read the flag: + // from on_boot at the default priority, or a setup() that runs above setup_priority::AFTER_WIFI. + // If the answer comes from a device handshake, hold setup with can_proceed() until it arrives. + // Calls after setup finishes are undefined behavior: the flag is still written and an error is + // logged, and from 2027.3.0 the call will be ignored. + // + // Known limitations. Not bugs, so no issue reports please; a PR that removes one with no RAM + // or performance cost would be considered. + // - No consumer is notified of a change, so the flag can only be decided once per boot. + // - The guard is coarse: a call from a priority below AFTER_WIFI (an on_boot with a low priority, + // or a setup() at LATE) still passes, but the API camera listener is already registered, MQTT + // (AFTER_CONNECTION) has cached the flag, and an API client that connected while setup was + // stalled on a slow component has already listed the entities, so they keep the old value. + // - Un-hiding an entity declared 'internal: true' in YAML skips the duplicate name check that + // codegen runs for exposed entities, so a name collision can surface at runtime. Entities with + // only an 'id:' are forced internal and use the id as their name. + // - Zigbee codegen skips YAML internal entities entirely, so un-hiding cannot add them to Zigbee. + void set_internal(bool internal); // Check if this object is declared to be disabled by default. // That means that when the device gets added to Home Assistant (or other clients) it should diff --git a/tests/integration/fixtures/set_internal_at_boot.yaml b/tests/integration/fixtures/set_internal_at_boot.yaml new file mode 100644 index 0000000000..b3007e9dbd --- /dev/null +++ b/tests/integration/fixtures/set_internal_at_boot.yaml @@ -0,0 +1,34 @@ +esphome: + name: set-internal-at-boot + on_boot: + then: + - lambda: |- + id(hidden_at_boot).set_internal(true); + id(shown_at_boot).set_internal(false); + +host: + +api: + actions: + - action: set_internal_late + then: + - lambda: id(untouched).set_internal(true); + +logger: + +sensor: + - platform: template + name: "Hidden At Boot" + id: hidden_at_boot + lambda: return 1.0; + + - platform: template + name: "Shown At Boot" + id: shown_at_boot + internal: true + lambda: return 2.0; + + - platform: template + name: "Untouched" + id: untouched + lambda: return 3.0; diff --git a/tests/integration/test_set_internal_at_boot.py b/tests/integration/test_set_internal_at_boot.py new file mode 100644 index 0000000000..68b0bd1080 --- /dev/null +++ b/tests/integration/test_set_internal_at_boot.py @@ -0,0 +1,41 @@ +"""Integration test for set_internal() called during and after setup.""" + +from __future__ import annotations + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_set_internal_at_boot( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """set_internal() in on_boot changes API exposure, later calls log an error.""" + waiter = LineWaiter() + + async with ( + run_compiled(yaml_config, line_callback=waiter.callback), + api_client_connected() as client, + ): + entities, services = await client.list_entities_services() + names = {entity.name for entity in entities} + + assert "Hidden At Boot" not in names + assert "Shown At Boot" in names + assert "Untouched" in names + + late = next(s for s in services if s.name == "set_internal_late") + await client.execute_service(late, {}) + await waiter.wait_for( + "'Untouched'", + "set_internal() after setup is undefined behavior", + timeout=5.0, + ) + + # Still written during the deprecation window, ignored from 2027.3.0 + entities, _ = await client.list_entities_services() + assert "Untouched" not in {entity.name for entity in entities} From 1975b17eac18d1d03778a753835d48c71a58dda9 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:25:41 -0500 Subject: [PATCH 145/178] Bump bundled esphome-device-builder to 1.14.7 (#19096) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index cfa47fbdad..6f500dbe6f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.6 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.7 RUN \ platformio settings set enable_telemetry No \ From 6cc2b9bf1740a1cc004050b5c6e6be48e55ea65a Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:12:08 +1000 Subject: [PATCH 146/178] [lvgl] Fix crash when using lvgl.list.add (#19177) --- esphome/components/lvgl/widgets/lv_list.py | 4 ++++ tests/components/lvgl/lvgl-package.yaml | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/esphome/components/lvgl/widgets/lv_list.py b/esphome/components/lvgl/widgets/lv_list.py index 83cbfb5ef9..7711e8bfe4 100644 --- a/esphome/components/lvgl/widgets/lv_list.py +++ b/esphome/components/lvgl/widgets/lv_list.py @@ -227,6 +227,7 @@ LIST_ID_SCHEMA = cv.Schema({cv.Required(CONF_ID): cv.use_id(lv_list_t)}) ) async def list_add_text_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_add_text(w: Widget): text = await lv_text.process(config[CONF_TEXT]) @@ -370,6 +371,7 @@ async def list_add_to_code(config, action_id, template_arg, args): _register_lv_uses(w_type_name, w_conf) _register_dynamic_widget_style_uses(w_conf) widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_add(w: Widget): index = None @@ -503,6 +505,7 @@ LIST_REMOVE_SCHEMA = LIST_ID_SCHEMA.extend( ) async def list_remove_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_remove(w: Widget): index = await lv_int.process(config[CONF_INDEX]) @@ -536,6 +539,7 @@ async def list_remove_to_code(config, action_id, template_arg, args): ) async def list_clear_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) + await _wait_list_triggers_completed() async def do_clear(w: Widget): await _wait_list_triggers_completed() diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 07c492db35..bd2e77ee8c 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -30,6 +30,18 @@ binary_sensor: widget: button_button state: pressed +globals: + - id: counter + type: int + +script: + - id: add_row + then: + - lvgl.list.add: + id: test_list_id + label: + text: row + lvgl: id: lvgl_id rotation: 90 @@ -1291,7 +1303,7 @@ lvgl: then: - logger.log: format: "table selected row %u col %u" - args: [row, column] + args: [(unsigned)row, (unsigned)column] on_click: then: - lvgl.table.cell.update: @@ -1347,10 +1359,12 @@ lvgl: - logger.log: format: "list entry added at %d" args: [list_index] + - lambda: "id(counter)++;" on_remove: - logger.log: format: "list entry removed at %d" args: [list_index] + - lambda: "id(counter)--;" on_click: - lvgl.list.add_text: id: test_list_id From 3eda3060b8f938e231cc1b2a63e7b8e07f6f381f Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 08:06:35 -0500 Subject: [PATCH 147/178] Bump bundled esphome-device-builder to 1.14.8 (#19250) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f500dbe6f..bdbbe798ce 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.7 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.8 RUN \ platformio settings set enable_telemetry No \ From 9814966fe7a8f48b29dac6d3f2a50bb681abe534 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:14:26 -0500 Subject: [PATCH 148/178] [noise] Bump noise-c to 0.1.30 and libsodium to 1.10021.11 (#19062) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index d17ebf235e..6067fde164 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.26") + cg.add_library("esphome/noise-c", "0.1.30") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.8") + cg.add_library("esphome/libsodium", "1.10021.11") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 738773d1b5..0e334ac5b4 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.26 ; noise (api, ota) + esphome/noise-c@0.1.30 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.26 ; noise (api, ota) + esphome/noise-c@0.1.30 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.26 ; used by noise (api, ota) + esphome/noise-c@0.1.30 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 00f22ca138..0dce00785b 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 1.0") == "noise-c" + assert mod.spec_key("esphome/noise-c@1.0") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.26\n" + " esphome/noise-c @ 1.0\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.26\n" + " esphome/noise-c @ 1.0\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.26"] + assert libs == ["esphome/noise-c @ 1.0"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 1.0", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.26", - "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 1.0", + "esphome/noise-c @ 1.0", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.26"] + assert cls.calls == ["esphome/noise-c @ 1.0"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.26": [ + "esphome/noise-c @ 1.0": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 1.0", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.26"] is None + assert compats["esphome/noise-c @ 1.0"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.26": [ + "esphome/noise-c @ 1.0": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) + mod.parallel_install(cls, ["esphome/noise-c @ 1.0"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 1.0"}) cls.deps = { - "esphome/noise-c @ 0.1.26": [ + "esphome/noise-c @ 1.0": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) + mod.parallel_install(cls, ["esphome/noise-c @ 1.0"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index b03bff19a2..774493ecf4 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@1.0", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@1.0", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 0803d7b37ce8c0e52960ca43180cb2ad30d4c7a0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:16:37 -0500 Subject: [PATCH 149/178] [core] Add FixedVector::try_init so callers can handle an exhausted heap (#19253) --- esphome/core/helpers.h | 52 ++++++++++++++++++++------ script/cpp_unit_test.py | 3 +- tests/components/core/test_helpers.cpp | 19 ++++++++++ 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index a0afb03124..987c54a5b0 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #endif #ifdef USE_ESP32 +#include #include #endif @@ -539,7 +541,15 @@ template inline void init_array_from(std::array &des } } -/// Fixed-capacity vector - allocates once at runtime, never reallocates +// Abort with a reason that reaches the panic output on ESP32. Elsewhere the literal is dropped +// before it can land in rodata, which is RAM on ESP8266 +#ifdef USE_ESP32 +#define ESPHOME_ABORT_WITH_REASON(reason) esp_system_abort(reason) +#else +#define ESPHOME_ABORT_WITH_REASON(reason) abort() +#endif + +/// Fixed-capacity vector - sized once through init() or try_init(); push_back never reallocates /// This avoids std::vector template overhead (_M_realloc_insert, _M_default_append) /// when size is known at initialization but not at compile time template class FixedVector { @@ -562,8 +572,7 @@ template class FixedVector { void cleanup_() { if (data_ != nullptr) { destroy_elements_(); - // Free raw memory - ::operator delete(data_); + free(data_); // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) } } @@ -632,16 +641,27 @@ template class FixedVector { // Allocate capacity - can be called multiple times to reinit // IMPORTANT: After calling init(), you MUST use push_back() to add elements. // Direct assignment via operator[] does NOT update the size counter. + // Aborts on exhaustion; use try_init() to handle failure. void init(size_t n) { + if (!try_init(n)) + ESPHOME_ABORT_WITH_REASON("FixedVector: out of memory"); + } + + // Same as init(), but returns false when memory is exhausted; the previous storage is freed either way + bool try_init(size_t n) { cleanup_(); reset_(); - if (n > 0) { - // Allocate raw memory without calling constructors - // sizeof(T) is correct here for any type T (value types, pointers, etc.) - // NOLINTNEXTLINE(bugprone-sizeof-expression) - data_ = static_cast(::operator new(n * sizeof(T))); - capacity_ = n; - } + if (n == 0) + return true; + if (n > SIZE_MAX / sizeof(T)) + return false; // the byte count would wrap into a small block + // sizeof(T) is correct here for any type T (value types, pointers, etc.) + // NOLINTNEXTLINE(bugprone-sizeof-expression,cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory) + data_ = static_cast(malloc(n * sizeof(T))); + if (data_ == nullptr) + return false; + capacity_ = n; + return true; } // Clear the vector (destroy all elements, reset size to 0, keep capacity) @@ -738,14 +758,22 @@ template class FixedVector { template class SmallBufferWithHeapFallback { public: explicit SmallBufferWithHeapFallback(size_t size) { + static_assert(std::is_trivially_default_constructible_v && std::is_trivially_destructible_v, + "the heap fallback leaves elements unconstructed"); if (size <= STACK_SIZE) { this->buffer_ = this->stack_buffer_; } else { - this->heap_buffer_ = new T[size]; + if (size <= SIZE_MAX / sizeof(T)) { + // NOLINTNEXTLINE(bugprone-sizeof-expression,cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory) + this->heap_buffer_ = static_cast(malloc(size * sizeof(T))); + } + // Callers write through get() unchecked, so exhaustion aborts like the new[] it replaces + if (this->heap_buffer_ == nullptr) + ESPHOME_ABORT_WITH_REASON("SmallBufferWithHeapFallback: out of memory"); this->buffer_ = this->heap_buffer_; } } - ~SmallBufferWithHeapFallback() { delete[] this->heap_buffer_; } + ~SmallBufferWithHeapFallback() { free(this->heap_buffer_); } // NOLINT(cppcoreguidelines-no-malloc) // Delete copy and move operations to prevent double-delete SmallBufferWithHeapFallback(const SmallBufferWithHeapFallback &) = delete; diff --git a/script/cpp_unit_test.py b/script/cpp_unit_test.py index f8bab39414..8cb18d0875 100755 --- a/script/cpp_unit_test.py +++ b/script/cpp_unit_test.py @@ -36,7 +36,8 @@ PLATFORMIO_OPTIONS = { def run_tests(selected_components: list[str]) -> int: - os.environ["ASAN_OPTIONS"] = "detect_leaks=0" + # allocator_may_return_null: an oversized request must come back empty, not abort the run + os.environ["ASAN_OPTIONS"] = "detect_leaks=0:allocator_may_return_null=1" return build_and_run( selected_components=selected_components, tests_dir=COMPONENTS_TESTS_DIR, diff --git a/tests/components/core/test_helpers.cpp b/tests/components/core/test_helpers.cpp index baf688fc8a..d6b31508d1 100644 --- a/tests/components/core/test_helpers.cpp +++ b/tests/components/core/test_helpers.cpp @@ -348,4 +348,23 @@ TEST(StepToAccuracyDecimals, NonFiniteAndZero) { EXPECT_EQ(step_to_accuracy_decimals(-INFINITY), 0); } +// --- FixedVector::try_init() --- + +// Keeps the block observable, else the compiler may drop the malloc and free pair and fold the check +static void escape(const void *p) { asm volatile("" : : "g"(p) : "memory"); } + +TEST(FixedVectorTryInit, ReportsExhaustionAndStaysEmpty) { + FixedVector v; + const bool ok = v.try_init(SIZE_MAX / sizeof(uint32_t)); + escape(&v); + EXPECT_FALSE(ok); + EXPECT_EQ(v.capacity(), 0u); + EXPECT_FALSE(v.try_init(SIZE_MAX / sizeof(uint32_t) + 1)); // byte count would wrap + EXPECT_EQ(v.capacity(), 0u); + EXPECT_TRUE(v.try_init(0)); + EXPECT_TRUE(v.try_init(4)); + v.push_back(7); + EXPECT_EQ(v.size(), 1u); +} + } // namespace esphome::core::testing From 9064bfcc85fc99cd2403dc109c308c9782295784 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:44:03 +0000 Subject: [PATCH 150/178] Bump bundled esphome-device-builder to 1.14.9 (#19263) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bdbbe798ce..e00570c8ff 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.8 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.9 RUN \ platformio settings set enable_telemetry No \ From b17cd89469498a698cf68f441dc5dcc09c3fce15 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:49:13 -0500 Subject: [PATCH 151/178] [wifi] Drop a scan instead of aborting when its results cannot be allocated, filter ESP32 scans by SSID in the driver (#19254) --- esphome/components/wifi/__init__.py | 3 + esphome/components/wifi/wifi_component.cpp | 6 +- esphome/components/wifi/wifi_component.h | 16 ++++-- .../wifi/wifi_component_esp8266.cpp | 6 +- .../wifi/wifi_component_esp_idf.cpp | 57 +++++++++++++++---- .../wifi/wifi_component_libretiny.cpp | 6 +- esphome/core/defines.h | 2 + 7 files changed, 74 insertions(+), 22 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index b8c6d774ac..d4b39c029b 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -623,6 +623,9 @@ async def to_code(config): networks = config.get(CONF_NETWORKS, []) if networks: cg.add(var.init_sta(len(networks))) + if len(networks) > 1: + # The ESP32 scan can filter one SSID in the driver; with several the whole list is kept + cg.add_define("USE_WIFI_MULTI_SSID") def add_sta(ap: cg.MockObj, network: dict) -> None: ip_config = network.get(CONF_MANUAL_IP, config.get(CONF_MANUAL_IP)) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 694e616476..f290832a18 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -1498,8 +1498,8 @@ void WiFiComponent::check_scanning_finished() { return; } this->scan_done_ = false; - this->has_completed_scan_after_captive_portal_start_ = - true; // Track that we've done a scan since captive portal started + // A driver filtered scan saw one SSID; a portal that started during it still needs a full scan + this->has_completed_scan_after_captive_portal_start_ = !this->is_scan_driver_filtered_(); this->retry_hidden_mode_ = RetryHiddenMode::SCAN_BASED; if (this->scan_result_.empty()) { @@ -2415,7 +2415,7 @@ void WiFiComponent::handle_driver_roam_(const bssid_t &bssid, uint8_t channel) { void WiFiComponent::release_scan_results_() { if (!this->keep_scan_results_) { ScanResultsLock lock(this); -#if defined(USE_RP2) || defined(USE_ESP32) +#if defined(USE_RP2) // std::vector - use swap trick since shrink_to_fit is non-binding decltype(this->scan_result_)().swap(this->scan_result_); #else diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 63df9fbfa5..16b62a5bb0 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -178,12 +178,12 @@ struct EAPAuth { using bssid_t = std::array; -/// Initial reserve size for filtered scan results (typical: 1-3 matching networks per SSID) -static constexpr size_t WIFI_SCAN_RESULT_FILTERED_RESERVE = 8; +// ESP32 with one configured network: the driver filters the scan by its SSID and only this many of +// its BSSIDs are kept, the strongest ones +static constexpr size_t WIFI_SCAN_RESULT_BOUND = 12; -// Use std::vector for RP2040 (callback-based) and ESP32 (destructive scan API) -// Use FixedVector for ESP8266 and LibreTiny where two-pass exact allocation is possible -#if defined(USE_RP2) || defined(USE_ESP32) +// RP2040's callback delivers results one at a time with no count, so it needs a growable vector +#if defined(USE_RP2) template using wifi_scan_vector_t = std::vector; #else template using wifi_scan_vector_t = FixedVector; @@ -948,6 +948,12 @@ class WiFiComponent final : public Component { uint8_t num_ipv6_addresses_{0}; #endif /* USE_NETWORK_IPV6 */ bool error_from_callback_{false}; +#if defined(USE_ESP32) && !defined(USE_WIFI_MULTI_SSID) + bool scan_driver_filtered_{false}; + bool is_scan_driver_filtered_() const { return this->scan_driver_filtered_; } +#else + constexpr bool is_scan_driver_filtered_() const { return false; } +#endif #if defined(USE_ESP8266) || defined(USE_LIBRETINY) // Platform-specific STA state enum, defined in platform cpp file. // On ESP8266, written from SDK system context (wifi_event_callback) — diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 031da1b355..60ec3f9a4d 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -773,7 +773,11 @@ void WiFiComponent::wifi_scan_done_callback_(void *arg, STATUS status) { } } - this->scan_result_.init(count); // Exact allocation + if (!this->scan_result_.try_init(count)) { + ESP_LOGW(TAG, "No memory for %zu scan results", count); + this->scan_done_ = true; + return; + } // Second pass: store matching networks for (bss_info *it = head; it != nullptr; it = STAILQ_NEXT(it, next)) { diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index ce75d21330..24bf64a99c 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -909,7 +909,8 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { ESP_LOGV(TAG, "Scan done: status=%" PRIu32 " number=%u scan_id=%u", it.status, it.number, it.scan_id); uint16_t number = it.number; - bool needs_full = this->needs_full_scan_results_(); + const bool filtered = this->is_scan_driver_filtered_(); + const bool needs_full = this->needs_full_scan_results_(); { // Mutate in place under the lock; blocking a portal request is fine and // avoids scratch buffers @@ -926,8 +927,14 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { return; } - // Smart reserve: full capacity if needed, small reserve otherwise - this->scan_result_.reserve(needs_full ? number : WIFI_SCAN_RESULT_FILTERED_RESERVE); + const size_t wanted = filtered ? std::min(number, WIFI_SCAN_RESULT_BOUND) : number; + // Storage is reused across the scans of one retry cycle and freed on connect; an exhausted + // heap drops this scan and the retry logic scans again + if (this->scan_result_.capacity() < wanted && !this->scan_result_.try_init(wanted)) { + esp_wifi_clear_ap_list(); + ESP_LOGW(TAG, "No memory for %zu scan results", wanted); + return; + } #ifdef USE_ESP32_HOSTED // getting records one at a time fails on P4 with hosted esp32 WiFi coprocessor @@ -955,22 +962,38 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { } #endif // USE_ESP32_HOSTED - // Check C string first - avoid std::string construction for non-matching networks const char *ssid_cstr = reinterpret_cast(record.ssid); - - // Only construct std::string and store if needed - if (needs_full || this->matches_configured_network_(ssid_cstr, record.bssid)) { - bssid_t bssid; - std::copy(record.bssid, record.bssid + 6, bssid.begin()); + if (!needs_full && !this->matches_configured_network_(ssid_cstr, record.bssid)) { + this->log_discarded_scan_result_(ssid_cstr, record.bssid, record.rssi, record.primary); + continue; + } + bssid_t bssid; + std::copy(record.bssid, record.bssid + 6, bssid.begin()); + if (this->scan_result_.size() < wanted) { this->scan_result_.emplace_back(bssid, ssid_cstr, strlen(ssid_cstr), record.primary, record.rssi, record.authmode != WIFI_AUTH_OPEN, ssid_cstr[0] == '\0'); - } else { - this->log_discarded_scan_result_(ssid_cstr, record.bssid, record.rssi, record.primary); + continue; } + // Records arrive in scan order, not by signal, so a bounded store keeps the strongest by + // replacing its weakest entry. Only SSID and signal decide here; a channel or auth constrained + // network hidden behind 12 stronger APs of its own SSID is not a real deployment + WiFiScanResult *weakest = &this->scan_result_[0]; + for (auto &res : this->scan_result_) { + if (res.get_rssi() < weakest->get_rssi()) + weakest = &res; + } + if (record.rssi <= weakest->get_rssi()) { + this->log_discarded_scan_result_(ssid_cstr, record.bssid, record.rssi, record.primary); + continue; + } + // Rebuilt in place rather than assigned; assignment pulls in CompactString's operators, 104 B of flash + weakest->~WiFiScanResult(); + new (weakest) WiFiScanResult(bssid, ssid_cstr, strlen(ssid_cstr), record.primary, record.rssi, + record.authmode != WIFI_AUTH_OPEN, ssid_cstr[0] == '\0'); } } ESP_LOGV(TAG, "Scan complete: %u found, %zu stored%s", number, this->scan_result_.size(), - needs_full ? "" : " (filtered)"); + filtered ? LOG_STR_LITERAL(" (driver filtered)") : LOG_STR_LITERAL("")); #ifdef USE_WIFI_SCAN_RESULTS_LISTENERS this->notify_scan_results_listeners_(); #endif @@ -1047,6 +1070,16 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { wifi_scan_config_t config{}; config.ssid = nullptr; config.bssid = nullptr; +#ifndef USE_WIFI_MULTI_SSID + // One configured network with an SSID: let the driver keep only its APs, so the WiFi library + // holds fewer records during the scan. Full results (portal, provisioning, listeners) and a + // network configured by BSSID alone still scan everything + this->scan_driver_filtered_ = + !this->needs_full_scan_results_() && this->sta_.size() == 1 && !this->sta_[0].get_ssid().empty(); + if (this->scan_driver_filtered_) { + config.ssid = const_cast(reinterpret_cast(this->sta_[0].get_ssid().c_str())); + } +#endif config.channel = 0; config.show_hidden = true; config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE; diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index 63a63e7342..940f2a0783 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -709,7 +709,11 @@ void WiFiComponent::wifi_scan_done_callback_() { } } - this->scan_result_.init(count); // Exact allocation + if (!this->scan_result_.try_init(count)) { + ESP_LOGW(TAG, "No memory for %zu scan results", count); + WiFi.scanDelete(); + return; + } // Second pass: store matching networks for (int i = 0; i < num; i++) { diff --git a/esphome/core/defines.h b/esphome/core/defines.h index eaece6d5ff..b78516c6ef 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -260,6 +260,8 @@ #ifdef USE_ARDUINO #define USE_PROMETHEUS #define USE_WIFI_WPA2_EAP +// Kept in the Arduino block so clang-tidy sees both scan storage paths +#define USE_WIFI_MULTI_SSID #endif // Platforms with native 64-bit time sources (no rollover tracking needed) From 1a555d58489a4c4e8e1bb30e13a5e8e30285e59f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:49:23 -0500 Subject: [PATCH 152/178] [esp32_ble_tracker] Re-register GATT clients after ble.disable and ble.enable (#19068) --- .../bluetooth_connection_bluedroid.cpp | 37 +++++++++++++------ .../bluetooth_connection_bluedroid.h | 1 + esphome/components/esp32_ble/ble.cpp | 35 +++++++++++------- esphome/components/esp32_ble/ble.h | 13 ++++++- .../esp32_ble_client/ble_client_base.cpp | 35 +++++++++++++++++- .../esp32_ble_client/ble_client_base.h | 12 +++--- .../esp32_ble_tracker/esp32_ble_tracker.cpp | 28 ++++++++++++-- .../esp32_ble_tracker/esp32_ble_tracker.h | 3 ++ 8 files changed, 126 insertions(+), 38 deletions(-) diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp index 15f854239d..986a67c7a8 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp @@ -45,15 +45,7 @@ void BluedroidGattClient::setup() { void BluedroidGattClient::loop() { if (!esp32_ble::global_ble->is_active()) { - // Stack down: no CLOSE_EVT will come. Settle a live link so the consumer - // frees its slot, then re-register the app on the next enable. - auto down_st = this->state(); - if (down_st != ClientState::IDLE && down_st != ClientState::INIT) { - this->release_services(); - this->set_idle_(); - this->listener_->on_connection_state(false, 0, ble_device_base::GATT_ERR_NOT_CONNECTED); - } - this->set_state(ClientState::INIT); + // ble_before_disabled_event_handler() settles the slot. return; } auto st = this->state(); @@ -65,7 +57,7 @@ void BluedroidGattClient::loop() { ESP_LOGE(TAG, "gattc app register failed: app_id=%d code=%d", this->app_id, ret); this->mark_failed(); } - // Do not wait for REG_EVT; a dropped event must not wedge the slot. + // Do not wait for REG_EVT; connect() rejects until it lands. this->set_idle_(); } else if (st == ClientState::DISCONNECTING || this->disconnect_pending()) { // The one teardown safety net: a lost CLOSE_EVT, or a scheduled @@ -78,8 +70,8 @@ void BluedroidGattClient::loop() { this->listener_->on_connection_state(false, 0, ESP_GATT_CONN_TIMEOUT); } } else { - // The loop stays on while a link exists (stack-down watch, pre-started - // search flush); it settles only back at IDLE. + // The loop stays on while a link exists (pre-started search flush); it + // settles only back at IDLE. this->deliver_pending_search_(); if (this->state() == ClientState::IDLE) { this->disable_loop(); @@ -87,6 +79,22 @@ void BluedroidGattClient::loop() { } } +// Stack down: no CLOSE_EVT will come. Settle a live link so the consumer +// frees its slot, then register the app again on the next enable. +void BluedroidGattClient::ble_before_disabled_event_handler() { + auto st = this->state(); + if (st != ClientState::IDLE && st != ClientState::INIT) { + this->release_services(); + this->set_idle_(); + this->listener_->on_connection_state(false, 0, ble_device_base::GATT_ERR_NOT_CONNECTED); + } + // The interface belongs to the torn-down stack. + this->gattc_if_ = ESP_GATT_IF_NONE; + this->set_state(ClientState::INIT); + // An idle slot runs no loop; the INIT branch must run to register again. + this->enable_loop(); +} + void BluedroidGattClient::dump_config() { ESP_LOGCONFIG(TAG, "Bluedroid GATT client %d", this->connection_index_); if (this->is_failed()) { @@ -97,6 +105,11 @@ void BluedroidGattClient::dump_config() { // ---- contract ops ---- int BluedroidGattClient::connect(uint64_t address, uint8_t addr_type) { + if (this->gattc_if_ == ESP_GATT_IF_NONE) { + // Bluedroid drops an open on an unknown interface without any event. + ESP_LOGW(TAG, "[%d] Connect rejected, GATT app not registered", this->connection_index_); + return ble_device_base::GATT_ERR_NOT_CONNECTED; + } // Only from idle: clobbering DISCONNECTING would open a new link the // stale CLOSE_EVT then tears down. if (this->state() != ClientState::IDLE) { diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h index 0d0b4fed5b..f285260e76 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h @@ -56,6 +56,7 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param) override; void connect() override; void disconnect() override; + void ble_before_disabled_event_handler() override; bool wants_parsed_advertisements() override { return false; } void on_scan_end() override {} bool parse_device(const ble_device_base::ESPBTDevice &device) override { return false; } diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index fc95760cf8..81fa328c16 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -83,18 +83,23 @@ void ESP32BLE::setup() { } } -void ESP32BLE::enable() { - if (this->state_ != BLE_COMPONENT_STATE_DISABLED) - return; - - this->state_ = BLE_COMPONENT_STATE_ENABLE; -} - -void ESP32BLE::disable() { - if (this->state_ == BLE_COMPONENT_STATE_DISABLED) - return; - - this->state_ = BLE_COMPONENT_STATE_DISABLE; +// Queue the transition for loop(). A pending transition the other way is +// cancelled instead, since nothing was torn down or brought up yet; any other +// state is already there or on its way. +void ESP32BLE::request_state_(bool enable) { + if (enable) { + if (this->state_ == BLE_COMPONENT_STATE_DISABLED) { + this->state_ = BLE_COMPONENT_STATE_ENABLE; + } else if (this->state_ == BLE_COMPONENT_STATE_DISABLE) { + this->state_ = BLE_COMPONENT_STATE_ACTIVE; + } + } else { + if (this->state_ == BLE_COMPONENT_STATE_ACTIVE) { + this->state_ = BLE_COMPONENT_STATE_DISABLE; + } else if (this->state_ == BLE_COMPONENT_STATE_ENABLE) { + this->state_ = BLE_COMPONENT_STATE_DISABLED; + } + } } #ifdef USE_ESP32_BLE_ADVERTISING @@ -580,7 +585,11 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { this->mark_failed(); return; } - this->state_ = BLE_COMPONENT_STATE_DISABLED; + this->drain_ble_events_(); + // A status callback may have asked for BLE back; the stack is down now, so + // that request becomes a bring-up. + this->state_ = + this->state_ == BLE_COMPONENT_STATE_ACTIVE ? BLE_COMPONENT_STATE_ENABLE : BLE_COMPONENT_STATE_DISABLED; } else if (this->state_ == BLE_COMPONENT_STATE_ENABLE) { ESP_LOGD(TAG, "Enabling"); this->state_ = BLE_COMPONENT_STATE_OFF; diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 7d2d0438a4..fd4fb15ff6 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -102,8 +102,8 @@ class ESP32BLE final : public Component { } uint32_t get_advertising_cycle_time() const { return this->advertising_cycle_time_; } - void enable(); - void disable(); + void enable() { this->request_state_(true); } + void disable() { this->request_state_(false); } ESPHOME_ALWAYS_INLINE bool is_active() { return this->state_ == BLE_COMPONENT_STATE_ACTIVE; } void setup() override; void loop() override; @@ -176,6 +176,15 @@ class ESP32BLE final : public Component { bool ble_setup_(); bool ble_dismantle_(); + void request_state_(bool enable); + // Drop what the old stack queued; the next stack reuses the same interface ids. + void drain_ble_events_() { + BLEEvent *ble_event; + while ((ble_event = this->ble_events_.pop()) != nullptr) { + this->ble_event_pool_.release(ble_event); + } + this->ble_events_.get_and_reset_dropped_count(); + } bool ble_pre_setup_(); #ifdef USE_ESP32_BLE_ADVERTISING void advertising_init_(); diff --git a/esphome/components/esp32_ble_client/ble_client_base.cpp b/esphome/components/esp32_ble_client/ble_client_base.cpp index e6cdde9cda..88454f7bdb 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.cpp +++ b/esphome/components/esp32_ble_client/ble_client_base.cpp @@ -42,7 +42,7 @@ void BLEClientBase::set_state(espbt::ClientState st) { void BLEClientBase::loop() { if (!esp32_ble::global_ble->is_active()) { - this->set_state(espbt::ClientState::INIT); + // ble_before_disabled_event_handler() resets the client. return; } if (this->state() == espbt::ClientState::INIT) { @@ -72,6 +72,21 @@ void BLEClientBase::loop() { float BLEClientBase::get_setup_priority() const { return setup_priority::AFTER_BLUETOOTH; } +void BLEClientBase::ble_before_disabled_event_handler() { + auto st = this->state(); + if (st != espbt::ClientState::IDLE && st != espbt::ClientState::INIT) { + // No CLOSE_EVT will come: free the services and settle the link. + this->release_services(); + this->set_idle_(); + this->on_disconnect_complete(ESP_GATT_CONN_TERMINATE_LOCAL_HOST); + } + // The interface belongs to the torn-down stack. + this->gattc_if_ = ESP_GATT_IF_NONE; + this->set_state(espbt::ClientState::INIT); + // An idle client runs no loop; the INIT branch must run to register again. + this->enable_loop(); +} + void BLEClientBase::dump_config() { ESP_LOGCONFIG(TAG, " Address: %s\n" @@ -93,6 +108,10 @@ bool BLEClientBase::parse_device(const espbt::ESPBTDevice &device) { return false; if (this->state() != espbt::ClientState::IDLE) return false; + // Not registered on this stack yet; promoting now would stop the scan for a + // connect that connect() rejects anyway. + if (this->gattc_if_ == ESP_GATT_IF_NONE) + return false; this->log_event_("Found device"); if (ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_DEBUG) @@ -117,6 +136,15 @@ void BLEClientBase::connect() { this->connection_index_, this->address_str_); return; } + if (this->gattc_if_ == ESP_GATT_IF_NONE) { + // Bluedroid drops an open on an unknown interface without any event. + this->log_warning_("Connect rejected, GATT app not registered"); + // INIT stays so loop() still registers; only a promoted client goes back. + if (this->state() == espbt::ClientState::DISCOVERED) { + this->set_state(espbt::ClientState::IDLE); + } + return; + } ESP_LOGI(TAG, "[%d] [%s] 0x%02x Connecting", this->connection_index_, this->address_str_, this->remote_addr_type_); this->paired_ = false; // A registration whose event never arrived must not block this connection's release. @@ -199,7 +227,10 @@ void BLEClientBase::release_services() { #ifndef CONFIG_BT_GATTC_CACHE_NVS_FLASH // Only the cache clean makes the stack's database unsafe to walk. this->services_released_ = true; - esp_ble_gattc_cache_clean(this->remote_bda_); + // A stack on its way down frees its own cache. + if (esp32_ble::global_ble->is_active()) { + esp_ble_gattc_cache_clean(this->remote_bda_); + } #endif } diff --git a/esphome/components/esp32_ble_client/ble_client_base.h b/esphome/components/esp32_ble_client/ble_client_base.h index e4b9cd5100..fbd405156a 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.h +++ b/esphome/components/esp32_ble_client/ble_client_base.h @@ -41,6 +41,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void connect() override; esp_err_t pair(); void disconnect() override; + void ble_before_disabled_event_handler() override; void unconditional_disconnect(); void release_services(); @@ -114,7 +115,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { #endif // Group 3: 4-byte types - int gattc_if_; + int gattc_if_{ESP_GATT_IF_NONE}; esp_gatt_status_t status_{ESP_GATT_OK}; // Group 4: Arrays @@ -139,7 +140,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { uint8_t pending_notify_regs_{0}; bool auto_connect_{false}; bool paired_{false}; - // Set only when release_services() cleans the stack's GATT cache, which no API may then walk + // Set by release_services() on RAM-cache builds; the stack's GATT database must not be walked after it bool services_released_{false}; // 8 bytes used, no padding @@ -155,10 +156,11 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void log_connection_params_(const char *param_type); void handle_connection_result_(esp_err_t ret); /// Hook called once a connection has been fully torn down (after release_services() and - /// set_idle_()), from both the CLOSE_EVT handler and the DISCONNECTING safety timeout. + /// set_idle_()): CLOSE_EVT, the DISCONNECTING safety timeout, or the BLE stack going down. /// Subclasses with extra per-connection accounting (e.g. bluetooth_proxy slot state) - /// override this to release that state. `reason` is the controller reason code, or - /// ESP_GATT_CONN_TIMEOUT for the safety-timeout path. + /// override this to release that state. `reason` is the controller reason code, + /// ESP_GATT_CONN_TIMEOUT for the safety timeout, or ESP_GATT_CONN_TERMINATE_LOCAL_HOST + /// for the stack going down. virtual void on_disconnect_complete(esp_err_t reason) {} /// Transition to IDLE and reset conn_id — call when the connection is fully dead. void set_idle_() { diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index 5339565a32..b4b793b4d0 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -74,11 +74,11 @@ void ESP32BLETracker::on_ota_global_state(ota::OTAState state, float progress, u void ESP32BLETracker::loop() { if (!this->parent_->is_active()) { - this->ble_was_disabled_ = true; return; - } else if (this->ble_was_disabled_) { + } + if (this->ble_was_disabled_) { this->ble_was_disabled_ = false; - // If the BLE stack was disabled, we need to start the scan again. + // First start after boot or after the stack came back. if (this->scan_continuous_) { this->start_scan(); } @@ -218,7 +218,27 @@ void ESP32BLETracker::stop_scan() { this->stop_scan_(); } -void ESP32BLETracker::ble_before_disabled_event_handler() { this->stop_scan_(); } +void ESP32BLETracker::ble_before_disabled_event_handler() { + // Tell the controller to stop; a scan still starting has nothing to stop yet. + if (this->scanner_state_ == ScannerState::RUNNING || this->scanner_state_ == ScannerState::FAILED) { + this->stop_scan_(); + } +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT + for (auto *client : this->clients_) { + client->ble_before_disabled_event_handler(); + } + this->skip_next_scan_end_ = false; +#endif + // The stop above never completes (stack torn down, events dropped); settle + // here so start_scan_() sees IDLE once the stack is back. + if (this->scanner_state_ != ScannerState::IDLE) { + this->cleanup_scan_state_(true); + } + // A failure latched by the old stack must not be handled against the next. + this->scan_start_failed_ = ESP_BT_STATUS_SUCCESS; + this->scan_set_param_failed_ = ESP_BT_STATUS_SUCCESS; + this->ble_was_disabled_ = true; +} bool ESP32BLETracker::stop_scan_() { if (this->scanner_state_ != ScannerState::RUNNING && this->scanner_state_ != ScannerState::FAILED) { diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h index 618444e626..1a424a4a8e 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h @@ -113,6 +113,9 @@ class ESPBTClient : public ESPBTDeviceListener { virtual void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param) = 0; virtual void connect() = 0; virtual void disconnect() = 0; + /// Called right before the BLE stack is dismantled. Nothing in flight will + /// complete, and the GATT app must register again once the stack is back. + virtual void ble_before_disabled_event_handler() {} bool disconnect_pending() const { return this->want_disconnect_; } void cancel_pending_disconnect() { this->want_disconnect_ = false; } From cd5d4ff25422e7a0142e73095f50c5a29f1a03fe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 16:56:50 -0500 Subject: [PATCH 153/178] [core] Add RAMAllocator::make_unique for objects whose allocation may fail (#19245) --- esphome/core/helpers.h | 40 ++++++++++++++++++++ tests/components/core/test_helpers.cpp | 52 ++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 987c54a5b0..b1f24b25a3 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,9 +14,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -2123,6 +2126,10 @@ void delay_microseconds_safe(uint32_t us); /// @name Memory management ///@{ +template struct RAMDeleter; +/// unique_ptr over RAMAllocator storage +template using RAMUniquePtr = std::unique_ptr>; + /** An STL allocator that uses SPI or internal RAM. * Returns `nullptr` in case no memory is available. * @@ -2193,6 +2200,26 @@ template class RAMAllocator { free(p); // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc) } + /// Value initialize one T; empty on exhaustion. new (std::nothrow) aborts on ESP-IDF instead. + /// Default flags prefer PSRAM; pass PREFER_INTERNAL to keep an object where plain new put it. + template RAMUniquePtr make_unique(Args &&...args) { + static_assert(alignof(T) <= alignof(std::max_align_t), "malloc storage cannot hold an over aligned type"); + T *p = this->allocate(1); + if (p == nullptr) + return {}; + // ::new so a class scoped operator new cannot hide the global placement form + return RAMUniquePtr(::new (p) T(std::forward(args)...)); + } + + /// n elements left uninitialized, as std::make_unique_for_overwrite does; empty on exhaustion, overflow, and n == 0 + RAMUniquePtr make_unique_array_for_overwrite(size_t n) { + static_assert(std::is_trivially_default_constructible_v, "elements are left unconstructed"); + static_assert(alignof(T) <= alignof(std::max_align_t), "malloc storage cannot hold an over aligned type"); + if (n == 0 || n > SIZE_MAX / sizeof(T)) + return {}; + return RAMUniquePtr(this->allocate(n)); + } + /** * Return the total heap space available via this allocator */ @@ -2255,6 +2282,19 @@ template class RAMAllocator { template using ExternalRAMAllocator = RAMAllocator; +/// Destroys and frees RAMAllocator storage. Not convertible: free() needs the address malloc returned +template struct RAMDeleter { + void operator()(T *p) const { + p->~T(); + RAMAllocator().deallocate(p, 1); + } +}; +/// Array form: elements must be trivial, the count is not stored so only the storage is freed +template struct RAMDeleter { + static_assert(std::is_trivially_destructible_v, "RAMUniquePtr is for trivially destructible elements"); + void operator()(T *p) const { RAMAllocator().deallocate(p, 1); } +}; + /** * Functions to constrain the range of arithmetic values. */ diff --git a/tests/components/core/test_helpers.cpp b/tests/components/core/test_helpers.cpp index d6b31508d1..72af605d61 100644 --- a/tests/components/core/test_helpers.cpp +++ b/tests/components/core/test_helpers.cpp @@ -367,4 +367,56 @@ TEST(FixedVectorTryInit, ReportsExhaustionAndStaysEmpty) { EXPECT_EQ(v.size(), 1u); } +// --- RAMAllocator::make_unique() --- + +namespace { +struct Probe { + static inline int live = 0; + int a; + int b; + Probe(int a, int b) : a(a), b(b) { live++; } + ~Probe() { live--; } +}; +} // namespace + +static_assert(sizeof(RAMUniquePtr) == sizeof(Probe *), "the deleter must not add storage"); + +TEST(RAMAllocatorMakeUnique, ForwardsArgsAndDestroysOnce) { + auto p = RAMAllocator().make_unique(3, 4); + ASSERT_NE(p, nullptr); + EXPECT_EQ(p->a, 3); + EXPECT_EQ(p->b, 4); + EXPECT_EQ(Probe::live, 1); + p.reset(); + EXPECT_EQ(Probe::live, 0); +} + +TEST(RAMAllocatorMakeUnique, ValueInitializesLikeMakeUnique) { + struct Plain { + uint32_t words[8]; + }; + // Dirty a block of the same size first so a recycled allocation is not zero by chance + auto dirty = RAMAllocator().make_unique_array_for_overwrite(sizeof(Plain)); + std::memset(dirty.get(), 0xFF, sizeof(Plain)); + dirty.reset(); + auto p = RAMAllocator().make_unique(); + ASSERT_NE(p, nullptr); + // Under ASan fresh blocks are filled with 0xbe, so this holds even when the dirtied block is not reused + EXPECT_TRUE(std::all_of(std::begin(p->words), std::end(p->words), [](uint32_t w) { return w == 0; })); +} + +TEST(RAMAllocatorMakeUnique, ArrayFormRejectsOverflowAndZero) { + EXPECT_EQ(RAMAllocator().make_unique_array_for_overwrite(SIZE_MAX / sizeof(uint32_t) + 1), nullptr); + EXPECT_EQ(RAMAllocator().make_unique_array_for_overwrite(0), nullptr); + EXPECT_NE(RAMAllocator().make_unique_array_for_overwrite(1), nullptr); +} + +TEST(RAMAllocatorMakeUnique, ArrayFormAllocatesElements) { + RAMUniquePtr buf = RAMAllocator().make_unique_array_for_overwrite(256); + ASSERT_NE(buf, nullptr); + std::memset(buf.get(), 0xA5, 256); + EXPECT_EQ(buf[0], 0xA5); + EXPECT_EQ(buf[255], 0xA5); +} + } // namespace esphome::core::testing From 64fd87f43670a0112a8b1b21eed346f8d315fd72 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sun, 13 Sep 2026 18:01:58 -0400 Subject: [PATCH 154/178] [i2s_audio][router] Loop thread controls all state changes (#19089) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 14 ++++++++----- .../router/speaker/router_speaker.cpp | 21 ++++++++++++++++--- .../router/speaker/router_speaker.h | 3 +++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1382a87046..9feaf39fff 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -53,6 +53,13 @@ void I2SAudioSpeakerBase::dump_config() { void I2SAudioSpeakerBase::loop() { uint32_t event_group_bits = xEventGroupGetBits(this->event_group_); + // A stop that arrives while stopped cancels any start that has not been processed yet + constexpr uint32_t stop_bits = SpeakerEventGroupBits::COMMAND_STOP | SpeakerEventGroupBits::COMMAND_STOP_GRACEFULLY; + if ((event_group_bits & stop_bits) && (this->state_ == speaker::STATE_STOPPED)) { + xEventGroupClearBits(this->event_group_, stop_bits | SpeakerEventGroupBits::COMMAND_START); + event_group_bits &= ~(stop_bits | SpeakerEventGroupBits::COMMAND_START); + } + if ((event_group_bits & SpeakerEventGroupBits::COMMAND_START) && (this->state_ == speaker::STATE_STOPPED)) { this->state_ = speaker::STATE_STARTING; xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); @@ -239,8 +246,6 @@ void I2SAudioSpeakerBase::start() { if ((this->state_ == speaker::STATE_STARTING) || (this->state_ == speaker::STATE_RUNNING)) return; - // Mark STARTING immediately to avoid transient STOPPED observations before loop() processes COMMAND_START. - this->state_ = speaker::STATE_STARTING; xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); } @@ -249,11 +254,10 @@ void I2SAudioSpeakerBase::stop() { this->stop_(false); } void I2SAudioSpeakerBase::finish() { this->stop_(true); } void I2SAudioSpeakerBase::stop_(bool wait_on_empty) { - if (this->is_failed()) - return; - if (this->state_ == speaker::STATE_STOPPED) + if (!this->is_ready() || this->is_failed()) return; + // Always set the bit, even when stopped, so loop() can cancel a start that is still pending if (wait_on_empty) { xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_STOP_GRACEFULLY); } else { diff --git a/esphome/components/router/speaker/router_speaker.cpp b/esphome/components/router/speaker/router_speaker.cpp index f4bf7420ab..dd2428e4df 100644 --- a/esphome/components/router/speaker/router_speaker.cpp +++ b/esphome/components/router/speaker/router_speaker.cpp @@ -2,6 +2,8 @@ #ifdef USE_ESP32 +#include "esphome/core/application.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include "esp_timer.h" @@ -12,6 +14,9 @@ namespace esphome::router { static const char *const TAG = "router.speaker"; +// Maximum time to wait for the active output to report running after start() before giving up +static const uint32_t STATE_TRANSITION_TIMEOUT_MS = 5000; + static inline uint32_t atomic_subtract_clamped(std::atomic &var, uint32_t amount) { uint32_t current = var.load(std::memory_order_acquire); uint32_t subtracted = 0; @@ -72,6 +77,7 @@ void Router::loop() { this->apply_cached_state_to_active_(); this->state_ = speaker::STATE_STARTING; + this->state_start_ms_ = App.get_loop_component_start_time(); active->start(); } return; @@ -86,10 +92,17 @@ void Router::loop() { // set_audio_stream_info() and never reaches the output on its own; if the format // changed while stopped, only start()'s apply_cached_state_to_active_() pushes it // down before the output's play()-side auto-start locks in the stale format. - if (active->is_stopped()) { + // While STARTING, ignore a transient stopped report as speaker running state + // is set asynchronously from start(). Timeout if the speaker never transitions. + if (this->state_ == speaker::STATE_STARTING) { + if (active->is_running()) { + this->state_ = speaker::STATE_RUNNING; + } else if ((App.get_loop_component_start_time() - this->state_start_ms_) > STATE_TRANSITION_TIMEOUT_MS) { + ESP_LOGW(TAG, "Active output did not start; giving up"); + this->state_ = speaker::STATE_STOPPED; + } + } else if (active->is_stopped()) { this->state_ = speaker::STATE_STOPPED; - } else if (this->state_ == speaker::STATE_STARTING && active->is_running()) { - this->state_ = speaker::STATE_RUNNING; } } @@ -133,6 +146,8 @@ void Router::start() { this->frames_in_pipeline_.store(0, std::memory_order_release); this->apply_cached_state_to_active_(); this->state_ = speaker::STATE_STARTING; + // May run on a producer task, so the cached loop timestamp is not usable here + this->state_start_ms_ = millis(); this->get_active_output()->start(); } diff --git a/esphome/components/router/speaker/router_speaker.h b/esphome/components/router/speaker/router_speaker.h index 801d0906ce..31f3f90630 100644 --- a/esphome/components/router/speaker/router_speaker.h +++ b/esphome/components/router/speaker/router_speaker.h @@ -59,6 +59,9 @@ class Router final : public Component, public speaker::Speaker { // frames_in_pipeline_. std::atomic frames_in_pipeline_{0}; + // Set when entering STATE_STARTING; used to time out a start the output never acts on + uint32_t state_start_ms_{0}; + bool cached_pause_{false}; void apply_cached_state_to_active_(); From be2c3dda94ea17d596e80b3332f920d783921503 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:02:42 -0500 Subject: [PATCH 155/178] [esp32_ble_tracker] Revert coexistence preference to balanced when OTA starts (#19082) --- esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index b4b793b4d0..e25b6f59fa 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -62,6 +62,11 @@ void ESP32BLETracker::on_ota_global_state(ota::OTAState state, float progress, u for (auto *client : this->clients_) { client->disconnect(); } +#ifdef USE_ESP32_BLE_SOFTWARE_COEXISTENCE + // The OTA transfer blocks the main loop, so the revert in loop() cannot run. No + // active-connection gate here: every client was just told to disconnect. + this->update_coex_preference_(false); +#endif #endif } else if ((state == ota::OTA_ERROR || state == ota::OTA_ABORT) && this->scan_continuous_before_ota_) { this->scan_continuous_before_ota_ = false; From b0b75f705a6cda23e12ed33fac7c6f1e307e3eca Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:26:14 -0500 Subject: [PATCH 156/178] [nextion] Allocate queue components through RAMAllocator and free entries the way they were allocated (#19246) --- esphome/components/nextion/nextion.cpp | 152 +++++++++--------- esphome/components/nextion/nextion.h | 2 + .../nextion/nextion_component_base.h | 5 +- 3 files changed, 78 insertions(+), 81 deletions(-) diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp index 97910ba3d5..625c915e73 100644 --- a/esphome/components/nextion/nextion.cpp +++ b/esphome/components/nextion/nextion.cpp @@ -13,6 +13,11 @@ namespace esphome::nextion { static const char *const TAG = "nextion"; +// A user entity may be named sleep_wake too; only the internal NO_RESULT command clears the sleeping flag +static bool is_sleep_wake_command(const NextionComponentBase *component) { + return component->get_queue_type() == NextionQueueType::NO_RESULT && component->get_variable_name() == "sleep_wake"; +} + // Nextion command terminator: three consecutive 0xFF bytes (per Nextion Instruction Set v1.1). static constexpr uint8_t COMMAND_DELIMITER[3] = {0xFF, 0xFF, 0xFF}; static constexpr size_t DELIMITER_SIZE = sizeof(COMMAND_DELIMITER); @@ -163,6 +168,17 @@ bool Nextion::check_connect_() { #endif // USE_NEXTION_CONFIG_SKIP_CONNECTION_HANDSHAKE } +// NO_RESULT components are owned by their entry; every other component is a user entity. Entry and +// component storage comes from RAMAllocator, so delete is not valid for either. +void Nextion::release_queue_entry_(NextionQueue *nb) { + if (nb->component != nullptr && nb->component->get_queue_type() == NextionQueueType::NO_RESULT) { + nb->component->~NextionComponentBase(); + RAMAllocator().deallocate(nb->component, 1); + } + nb->~NextionQueue(); + RAMAllocator().deallocate(nb, 1); +} + void Nextion::reset_(bool reset_nextion) { uint8_t d; @@ -170,15 +186,12 @@ void Nextion::reset_(bool reset_nextion) { this->read_byte(&d); } for (auto *entry : this->nextion_queue_) { - if (entry->component != nullptr && entry->component->get_queue_type() == NextionQueueType::NO_RESULT) { - delete entry->component; // NOLINT(cppcoreguidelines-owning-memory) - } - delete entry; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(entry); } this->nextion_queue_.clear(); #ifdef USE_NEXTION_WAVEFORM for (auto *entry : this->waveform_queue_) { - delete entry; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(entry); } this->waveform_queue_.clear(); #endif // USE_NEXTION_WAVEFORM @@ -421,6 +434,9 @@ bool Nextion::remove_from_q_(bool report_empty) { NextionQueue *nb = this->nextion_queue_.front(); if (!nb || !nb->component) { ESP_LOGE(TAG, "Invalid queue"); + if (nb != nullptr) { + this->release_queue_entry_(nb); + } this->nextion_queue_.pop_front(); return false; } @@ -428,13 +444,10 @@ bool Nextion::remove_from_q_(bool report_empty) { ESP_LOGN(TAG, "Removed: %s", component->get_variable_name().c_str()); - if (component->get_queue_type() == NextionQueueType::NO_RESULT) { - if (component->get_variable_name() == "sleep_wake") { - this->is_sleeping_ = false; - } - delete component; // NOLINT(cppcoreguidelines-owning-memory) + if (is_sleep_wake_command(component)) { + this->is_sleeping_ = false; } - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->nextion_queue_.pop_front(); return true; } @@ -544,7 +557,7 @@ void Nextion::process_nextion_commands_() { ESP_LOGW(TAG, "Invalid waveform ID %d/ch %d", component->get_component_id(), component->get_wave_channel_id()); ESP_LOGN(TAG, "Remove waveform ID %d/ch %d", component->get_component_id(), component->get_wave_channel_id()); - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->waveform_queue_.pop(); } #else // USE_NEXTION_WAVEFORM @@ -647,6 +660,9 @@ void Nextion::process_nextion_commands_() { NextionQueue *nb = this->nextion_queue_.front(); if (!nb || !nb->component) { ESP_LOGE(TAG, "Invalid queue entry"); + if (nb != nullptr) { + this->release_queue_entry_(nb); + } this->nextion_queue_.pop_front(); return; } @@ -660,7 +676,7 @@ void Nextion::process_nextion_commands_() { component->set_state_from_string(to_process, true, false); } - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->nextion_queue_.pop_front(); break; @@ -687,6 +703,9 @@ void Nextion::process_nextion_commands_() { NextionQueue *nb = this->nextion_queue_.front(); if (!nb || !nb->component) { ESP_LOGE(TAG, "Invalid queue"); + if (nb != nullptr) { + this->release_queue_entry_(nb); + } this->nextion_queue_.pop_front(); return; } @@ -703,7 +722,7 @@ void Nextion::process_nextion_commands_() { component->set_state_from_int(value, true, false); } - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->nextion_queue_.pop_front(); break; @@ -890,7 +909,7 @@ void Nextion::process_nextion_commands_() { ESP_LOGN(TAG, "Send waveform: component id %d, waveform id %d, size %zu", component->get_component_id(), component->get_wave_channel_id(), buffer_to_send); component->clear_wave_buffer(buffer_to_send); - delete nb; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nb); this->waveform_queue_.pop(); #else // USE_NEXTION_WAVEFORM ESP_LOGW(TAG, "Waveform transmit ready but waveform not enabled"); @@ -920,14 +939,10 @@ void Nextion::purge_stale_queue_entries_() { ESP_LOGV(TAG, "Remove old queue '%s':'%s'", component->get_queue_type_string(), component->get_variable_name().c_str()); - if (component->get_queue_type() == NextionQueueType::NO_RESULT) { - if (component->get_variable_name() == "sleep_wake") { - this->is_sleeping_ = false; - } - delete component; // NOLINT(cppcoreguidelines-owning-memory) + if (is_sleep_wake_command(component)) { + this->is_sleeping_ = false; } - - delete *it; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(*it); it = this->nextion_queue_.erase(it); } else { @@ -1079,6 +1094,34 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool return response.length(); } +// Allocates a queue entry owning a bare NO_RESULT component; nullptr when the queue is full or memory is out +NextionQueue *Nextion::make_no_result_entry_(const std::string &variable_name) { +#ifdef USE_NEXTION_MAX_QUEUE_SIZE + if (this->max_queue_size_ > 0 && this->nextion_queue_.size() >= this->max_queue_size_) { + ESP_LOGW(TAG, "Queue full (%zu), drop: %s", this->nextion_queue_.size(), variable_name.c_str()); + return nullptr; + } +#endif + + auto *nextion_queue = RAMAllocator().allocate(1); + if (nextion_queue == nullptr) { + ESP_LOGW(TAG, "Queue alloc failed"); + return nullptr; + } + new (nextion_queue) nextion::NextionQueue; + + nextion_queue->component = RAMAllocator().allocate(1); + if (nextion_queue->component == nullptr) { + ESP_LOGW(TAG, "Component alloc failed"); + this->release_queue_entry_(nextion_queue); + return nullptr; + } + new (nextion_queue->component) nextion::NextionComponentBase; + nextion_queue->component->set_variable_name(variable_name); + nextion_queue->queue_time = App.get_loop_component_start_time(); + return nextion_queue; +} + /** * @brief Add a command to the Nextion queue that expects no response. * @@ -1090,36 +1133,11 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool * @param variable_name Name of the variable or component associated with the command. */ void Nextion::add_no_result_to_queue_(const std::string &variable_name) { -#ifdef USE_NEXTION_MAX_QUEUE_SIZE - if (this->max_queue_size_ > 0 && this->nextion_queue_.size() >= this->max_queue_size_) { - ESP_LOGW(TAG, "Queue full (%zu), drop: %s", this->nextion_queue_.size(), variable_name.c_str()); + auto *nextion_queue = this->make_no_result_entry_(variable_name); + if (nextion_queue == nullptr) return; - } -#endif - - RAMAllocator allocator; - nextion::NextionQueue *nextion_queue = allocator.allocate(1); - if (nextion_queue == nullptr) { - ESP_LOGW(TAG, "Queue alloc failed"); - return; - } - new (nextion_queue) nextion::NextionQueue(); - - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - nextion_queue->component = new (std::nothrow) nextion::NextionComponentBase; - if (nextion_queue->component == nullptr) { - ESP_LOGW(TAG, "Component alloc failed"); - nextion_queue->~NextionQueue(); - allocator.deallocate(nextion_queue, 1); - return; - } - nextion_queue->component->set_variable_name(variable_name); - - nextion_queue->queue_time = App.get_loop_component_start_time(); - this->nextion_queue_.push_back(nextion_queue); - - ESP_LOGN(TAG, "Queue NORESULT: %s", nextion_queue->component->get_variable_name().c_str()); + ESP_LOGN(TAG, "Queue NORESULT: %s", variable_name.c_str()); } /** @@ -1153,32 +1171,10 @@ void Nextion::add_no_result_to_queue_with_command_(const std::string &variable_n #ifdef USE_NEXTION_COMMAND_SPACING void Nextion::add_no_result_to_queue_with_pending_command_(const std::string &variable_name, const std::string &command) { -#ifdef USE_NEXTION_MAX_QUEUE_SIZE - if (this->max_queue_size_ > 0 && this->nextion_queue_.size() >= this->max_queue_size_) { - ESP_LOGW(TAG, "Queue full (%zu), drop: %s", this->nextion_queue_.size(), variable_name.c_str()); + auto *nextion_queue = this->make_no_result_entry_(variable_name); + if (nextion_queue == nullptr) return; - } -#endif - - RAMAllocator allocator; - nextion::NextionQueue *nextion_queue = allocator.allocate(1); - if (nextion_queue == nullptr) { - ESP_LOGW(TAG, "Queue alloc failed"); - return; - } - new (nextion_queue) nextion::NextionQueue(); - - nextion_queue->component = new (std::nothrow) nextion::NextionComponentBase; - if (nextion_queue->component == nullptr) { - ESP_LOGW(TAG, "Component alloc failed"); - nextion_queue->~NextionQueue(); - allocator.deallocate(nextion_queue, 1); - return; - } - nextion_queue->component->set_variable_name(variable_name); - nextion_queue->queue_time = App.get_loop_component_start_time(); nextion_queue->pending_command = command; // Store command for retry - this->nextion_queue_.push_back(nextion_queue); ESP_LOGVV(TAG, "Queue with pending command: %s", variable_name.c_str()); } @@ -1312,7 +1308,7 @@ void Nextion::add_to_get_queue(NextionComponentBase *component) { ESP_LOGW(TAG, "Queue alloc failed"); return; } - new (nextion_queue) nextion::NextionQueue(); + new (nextion_queue) nextion::NextionQueue; nextion_queue->component = component; nextion_queue->queue_time = App.get_loop_component_start_time(); @@ -1334,7 +1330,7 @@ void Nextion::add_to_get_queue(NextionComponentBase *component) { if (this->send_command_(command)) { this->nextion_queue_.push_back(nextion_queue); } else { - delete nextion_queue; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nextion_queue); } #endif // USE_NEXTION_COMMAND_SPACING } @@ -1355,14 +1351,14 @@ void Nextion::add_addt_command_to_queue(NextionComponentBase *component) { ESP_LOGW(TAG, "Queue alloc failed"); return; } - new (nextion_queue) nextion::NextionQueue(); + new (nextion_queue) nextion::NextionQueue; nextion_queue->component = component; nextion_queue->queue_time = App.get_loop_component_start_time(); if (!this->waveform_queue_.push(nextion_queue)) { ESP_LOGW(TAG, "Waveform queue full, drop"); - delete nextion_queue; // NOLINT(cppcoreguidelines-owning-memory) + this->release_queue_entry_(nextion_queue); return; } if (this->waveform_queue_.size() == 1) diff --git a/esphome/components/nextion/nextion.h b/esphome/components/nextion/nextion.h index aa9fe8abb3..6c9c8760f8 100644 --- a/esphome/components/nextion/nextion.h +++ b/esphome/components/nextion/nextion.h @@ -1469,6 +1469,8 @@ class Nextion final : public NextionBase, public PollingComponent, public uart:: void all_components_send_state_(bool force_update = false); uint32_t comok_sent_ = 0; bool remove_from_q_(bool report_empty = true); + void release_queue_entry_(NextionQueue *nb); + NextionQueue *make_no_result_entry_(const std::string &variable_name); /** * @brief Status flags for Nextion display state management diff --git a/esphome/components/nextion/nextion_component_base.h b/esphome/components/nextion/nextion_component_base.h index 6676d01920..5e84291b16 100644 --- a/esphome/components/nextion/nextion_component_base.h +++ b/esphome/components/nextion/nextion_component_base.h @@ -23,8 +23,7 @@ class NextionComponentBase; class NextionQueue { public: - virtual ~NextionQueue() = default; - NextionComponentBase *component; + NextionComponentBase *component{nullptr}; uint32_t queue_time = 0; // Store command for retry if spacing blocked it @@ -105,6 +104,6 @@ class NextionComponentBase { int wave_max_length_ = 255; #endif // USE_NEXTION_WAVEFORM - bool needs_to_send_update_; + bool needs_to_send_update_{false}; }; } // namespace esphome::nextion From 977542061d285694ab94967757d70f0a698a2269 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:26:28 -0500 Subject: [PATCH 157/178] [esphome] Allocate the OTA noise session and auth buffer through RAMAllocator (#19249) --- esphome/components/esphome/ota/ota_esphome.cpp | 9 ++++++++- esphome/components/esphome/ota/ota_esphome.h | 4 ++-- esphome/components/esphome/ota/ota_esphome_noise.cpp | 6 ++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index f853ed6a2d..3010df1056 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -842,7 +842,14 @@ bool ESPHomeOTAComponent::handle_auth_send_() { const size_t hex_size = hasher.get_size() * 2; const size_t nonce_len = hasher.get_size() / 4; const size_t auth_buf_size = 1 + 3 * hex_size; - this->auth_buf_ = std::make_unique(auth_buf_size); + // Internal RAM first: 128 of these bytes go straight into the hardware SHA engine + this->auth_buf_ = + RAMAllocator(RAMAllocator::PREFER_INTERNAL).make_unique_array_for_overwrite(auth_buf_size); + if (!this->auth_buf_) { + this->log_auth_warning_(LOG_STR("No memory")); + this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_UNKNOWN); + return false; + } this->auth_buf_pos_ = 0; char *buf = reinterpret_cast(this->auth_buf_.get() + 1); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index c6f710b3fc..68dd0ffb9e 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -145,13 +145,13 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { #ifdef USE_OTA_PASSWORD std::string password_; - std::unique_ptr auth_buf_; + RAMUniquePtr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION #ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; #endif - std::unique_ptr noise_; + RAMUniquePtr noise_; #endif // USE_OTA_ENCRYPTION socket::ListenSocket *server_{nullptr}; diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7401413d6d..65476572a1 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -7,7 +7,6 @@ #include "esphome/core/log.h" #include -#include #ifdef USE_ESP8266 #include @@ -43,9 +42,8 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { // A provisioned key cleared between the offer and here is not guarded: the // session runs on the zero key load_psk fills in and fails the client's MAC. - // Default-init: the frame buffer is written before it is read - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); + // Default placement, PSRAM first where present: the session only lives for one upload + this->noise_ = RAMAllocator().make_unique(); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags From ebe72c3cefaaad5e664a0f9f400d88da5b11262d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:27:03 -0500 Subject: [PATCH 158/178] [core] Resolve file paths against the YAML file that declares them (#19259) --- esphome/config_validation.py | 66 +++++++----- tests/unit_tests/test_config_validation.py | 120 ++++++++++++++++++++- 2 files changed, 159 insertions(+), 27 deletions(-) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 685a9d04b3..2346c28cce 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -15,6 +15,7 @@ from ipaddress import ( ip_network, ) import logging +import os from pathlib import Path import re from string import ascii_letters, digits @@ -1967,38 +1968,51 @@ def _remap_bundle_path(value: str) -> Path | None: return remap_bundle_path(value) -def directory(value: object) -> Path: - value = string(value) - path = CORE.relative_config_path(value) +def _declaring_document(value: str) -> Path | None: + """Return the on-disk YAML file *value* was loaded from, absolute, or None.""" + esp_range = getattr(value, "esp_range", None) + if esp_range is None: + return None + document = Path(esp_range.start_mark.document).absolute() + return document if document.is_file() else None - if not path.exists(): - remapped = _remap_bundle_path(value) - if remapped is None: + +def _existing_path(value: str, kind: str, is_kind: Callable[[Path], bool]) -> Path: + """Resolve *value* to a *kind* entry: config dir, then declaring document, then bundle remap.""" + path = CORE.relative_config_path(value) + if is_kind(path): + return path + candidates = [path] + tried_document: Path | None = None + if (document := _declaring_document(value)) is not None: + beside_document = document.parent / Path(value).expanduser() + if os.path.normpath(beside_document) != os.path.normpath(path): + candidates.append(beside_document) + tried_document = document + if (remapped := _remap_bundle_path(value)) is not None: + candidates.append(remapped) + for candidate in candidates: + if is_kind(candidate): + return candidate + for candidate in candidates: + if candidate.exists(): raise Invalid( - f"Could not find directory '{path}'. Please make sure it exists (full path: {path.resolve()})." + f"Path '{candidate}' is not a {kind} (full path: {candidate.resolve()})." ) - path = remapped - if not path.is_dir(): - raise Invalid( - f"Path '{path}' is not a directory (full path: {path.resolve()})." - ) - return path + also = ( + f" Also looked next to {tried_document}." if tried_document is not None else "" + ) + raise Invalid( + f"Could not find {kind} '{path}'. Please make sure it exists (full path: {path.resolve()}).{also}" + ) + + +def directory(value: object) -> Path: + return _existing_path(string(value), "directory", Path.is_dir) def file_(value: object) -> Path: - value = string(value) - path = CORE.relative_config_path(value) - - if not path.exists(): - remapped = _remap_bundle_path(value) - if remapped is None: - raise Invalid( - f"Could not find file '{path}'. Please make sure it exists (full path: {path.resolve()})." - ) - path = remapped - if not path.is_file(): - raise Invalid(f"Path '{path}' is not a file (full path: {path.resolve()}).") - return path + return _existing_path(string(value), "file", Path.is_file) ENTITY_ID_CHARACTERS = "abcdefghijklmnopqrstuvwxyz0123456789_" diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 457b9d017b..52070e7aba 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -1,4 +1,5 @@ import importlib +import io import json import logging from pathlib import Path @@ -20,6 +21,7 @@ from esphome.components.esp32 import ( VARIANT_ESP32S2, VARIANT_ESP32S3, ) +from esphome.components.substitutions import do_substitution_pass from esphome.config_validation import Invalid from esphome.const import ( CONF_DAY, @@ -65,7 +67,13 @@ from esphome.core import ( ) from esphome.schema_extractors import SCHEMA_EXTRACT from esphome.util import Registry -from esphome.yaml_util import ESPHomeDataBase, SensitiveStr, make_data_base +from esphome.yaml_util import ( + ESPHomeDataBase, + SensitiveStr, + load_yaml, + make_data_base, + parse_yaml, +) def test_check_not_templatable__invalid(): @@ -3145,6 +3153,116 @@ def test_file__existing_relative_path(setup_core: Path) -> None: assert cv.file_("partitions.csv") == setup_core / "partitions.csv" +def _package_value(setup_core: Path, path: str = "assets/ui.js") -> tuple[Path, str]: + """Write a package file next to an ``assets/`` dir; return the dir and its loaded *path* value.""" + package_dir = setup_core / ".esphome" / "packages" / "abc123" / "vendor" + (package_dir / "assets").mkdir(parents=True) + (package_dir / "assets" / "ui.js").write_text("js\n") + (package_dir / "device.yaml").write_text(f"path: {path}\n") + return package_dir, load_yaml(package_dir / "device.yaml")["path"] + + +def test_file__resolves_relative_to_the_declaring_document(setup_core: Path) -> None: + """A package's own asset path resolves against the package file when the config dir lacks it.""" + package_dir, value = _package_value(setup_core) + + assert cv.file_(value) == package_dir / "assets" / "ui.js" + + +def test_file__resolves_a_substituted_path_against_the_use_site( + setup_core: Path, +) -> None: + package_dir, _ = _package_value(setup_core) + (package_dir / "device.yaml").write_text( + "substitutions:\n ui: assets/ui.js\npath: ${ui}\n" + ) + config = do_substitution_pass(load_yaml(package_dir / "device.yaml")) + + assert cv.file_(config["path"]) == package_dir / "assets" / "ui.js" + + +def test_file__result_is_absolute_for_a_relative_document( + setup_core: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A document loaded by a cwd-relative path still yields an absolute result.""" + package_dir, _ = _package_value(setup_core) + monkeypatch.chdir(setup_core) + value = load_yaml(Path(".esphome/packages/abc123/vendor/device.yaml"))["path"] + + result = cv.file_(value) + + assert result.is_absolute() + assert result == package_dir / "assets" / "ui.js" + + +def test_file__config_dir_entry_of_the_wrong_kind_does_not_shadow_the_package( + setup_core: Path, +) -> None: + package_dir, value = _package_value(setup_core) + (setup_core / "assets" / "ui.js").mkdir(parents=True) + + assert cv.file_(value) == package_dir / "assets" / "ui.js" + + +def test_file__miss_names_the_declaring_document(setup_core: Path) -> None: + package_dir, value = _package_value(setup_core, "assets/other.js") + + with pytest.raises(Invalid, match="Could not find file") as excinfo: + cv.file_(value) + + assert f"Also looked next to {package_dir / 'device.yaml'}" in str(excinfo.value) + + +def test_file__document_spelled_through_dotdot_in_the_config_dir_adds_no_hint( + setup_core: Path, +) -> None: + (setup_core / "sub").mkdir() + (setup_core / "device.yaml").write_text("path: assets/other.js\n") + value = load_yaml(setup_core / "sub" / ".." / "device.yaml")["path"] + + with pytest.raises(Invalid) as excinfo: + cv.file_(value) + + assert "Also looked" not in str(excinfo.value) + + +def test_file__wrong_kind_beside_the_document_is_reported(setup_core: Path) -> None: + package_dir, value = _package_value(setup_core, "assets") + + with pytest.raises(Invalid, match="is not a file") as excinfo: + cv.file_(value) + + assert str(package_dir / "assets") in str(excinfo.value) + + +def test_file__config_dir_wins_over_the_declaring_document(setup_core: Path) -> None: + _, value = _package_value(setup_core) + (setup_core / "assets").mkdir() + (setup_core / "assets" / "ui.js").write_text("local\n") + + assert cv.file_(value) == setup_core / "assets" / "ui.js" + + +def test_file__declared_in_an_in_memory_document_is_not_resolved( + setup_core: Path, +) -> None: + """A value whose source document isn't on disk falls through to the config-dir error.""" + value = parse_yaml(Path(""), io.StringIO("path: assets/ui.js\n"))[ + "path" + ] + + with pytest.raises(Invalid, match="Could not find file"): + cv.file_(value) + + +def test_directory_resolves_relative_to_the_declaring_document( + setup_core: Path, +) -> None: + package_dir, value = _package_value(setup_core, "assets") + + assert cv.directory(value) == package_dir / "assets" + + def test_file__missing_raises(setup_core: Path) -> None: with pytest.raises(Invalid, match="Could not find file"): cv.file_("partitions.csv") From e28b4eb2a0584792102702af9f1a47fc2b5d751f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:35:38 -0500 Subject: [PATCH 159/178] [ethernet] Keep the W5500 SPI context in a static instance instead of the heap (#19248) --- .../components/ethernet/w5500_custom_spi.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/esphome/components/ethernet/w5500_custom_spi.cpp b/esphome/components/ethernet/w5500_custom_spi.cpp index ed4f149738..9c6b59582a 100644 --- a/esphome/components/ethernet/w5500_custom_spi.cpp +++ b/esphome/components/ethernet/w5500_custom_spi.cpp @@ -6,17 +6,21 @@ #include #include #include -#include namespace esphome::ethernet { namespace { -// Per-device context returned by init() and handed back to read/write/deinit. +// Context returned by init() and handed back to read/write/deinit. There is one W5500 per device, so a +// single static instance replaces a heap allocation that could fail. It is always clear when init() runs: +// esp_eth_mac_new_w5500() calls deinit() on every failure after init() succeeded, and nothing else +// uninstalls the driver struct W5500CustomSpiContext { spi_device_handle_t handle; SemaphoreHandle_t lock; }; +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - intentional mutable state +W5500CustomSpiContext w5500_context{}; // Transfers up to the ESP32 SPI hardware FIFO size (64 bytes) stay on the polling path; larger // transfers (the frame payloads) use the blocking, DMA-backed transmit. @@ -25,23 +29,20 @@ constexpr uint32_t W5500_SPI_LOCK_TIMEOUT_MS = 50; void *w5500_custom_spi_init(const void *spi_config) { const auto *config = static_cast(spi_config); - auto *ctx = new (std::nothrow) W5500CustomSpiContext{}; - if (ctx == nullptr) { - return nullptr; - } + auto *ctx = &w5500_context; // The W5500 SPI frame carries the 16-bit address in the command phase and the 8-bit control // byte in the address phase; mirror what the stock driver configures. spi_device_interface_config_t devcfg = *config->spi_devcfg; devcfg.command_bits = 16; devcfg.address_bits = 8; if (spi_bus_add_device(config->spi_host_id, &devcfg, &ctx->handle) != ESP_OK) { - delete ctx; + ctx->handle = nullptr; return nullptr; } ctx->lock = xSemaphoreCreateMutex(); if (ctx->lock == nullptr) { spi_bus_remove_device(ctx->handle); - delete ctx; + ctx->handle = nullptr; return nullptr; } return ctx; @@ -51,7 +52,7 @@ esp_err_t w5500_custom_spi_deinit(void *spi_ctx) { auto *ctx = static_cast(spi_ctx); spi_bus_remove_device(ctx->handle); vSemaphoreDelete(ctx->lock); - delete ctx; + *ctx = {}; return ESP_OK; } From 81a54ea9dbbd7c5482057f9993ce8bda1a4f7047 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 17:37:04 -0500 Subject: [PATCH 160/178] [ota] Allocate the signature block through RAMAllocator (#19251) --- esphome/components/ota/ota_signature_esp_idf.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/esphome/components/ota/ota_signature_esp_idf.cpp b/esphome/components/ota/ota_signature_esp_idf.cpp index 501d6ac241..2192a79441 100644 --- a/esphome/components/ota/ota_signature_esp_idf.cpp +++ b/esphome/components/ota/ota_signature_esp_idf.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -235,9 +234,11 @@ bool IDFOTABackend::verify_signed_image_(const esp_partition_t *incoming) { // runs mid-OTA on the loop task, on top of the caller's live 1 KB OTA buffer // and mbedtls's own ~1 KB verify scratch, so keeping it off the stack widens // a thin margin. One short-lived allocation right before reboot is not the - // fragmentation pattern the project guards against. nothrow so an OOM here - // fails closed like every other error path, rather than aborting. - std::unique_ptr block(new (std::nothrow) uint8_t[SIG_BLOCK_SIZE]); + // fragmentation pattern the project guards against. An OOM returns nullptr + // and fails closed like every other error path. Internal RAM first: the + // block is an esp_partition_read target. + auto block = + RAMAllocator(RAMAllocator::PREFER_INTERNAL).make_unique_array_for_overwrite(SIG_BLOCK_SIZE); if (!block) { OTA_IDF_SIG_LOG(ESP_LOGE, "out of memory"); return false; From 501009073d1f6da6245c4100fc2de1a44a202922 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:55:13 +1200 Subject: [PATCH 161/178] [core] Clear loaded_platforms on CORE.reset() (#19268) --- esphome/core/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 6e3f91af22..5fcad90a81 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -715,6 +715,7 @@ class EsphomeCore: self.defines = set() self.platformio_options = {} self.loaded_integrations = set() + self.loaded_platforms = set() self.component_ids = set() self.platform_counts = defaultdict(int) self.unique_ids = {} From 93fa95c8335585fc5d6492ca01039b252ae3820e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 20:03:00 -0500 Subject: [PATCH 162/178] [api] Reuse overflow buffer storage instead of allocating per stalled write (#19093) --- esphome/components/api/__init__.py | 5 +- esphome/components/api/api_buffer.cpp | 35 +- esphome/components/api/api_buffer.h | 24 +- esphome/components/api/api_connection.cpp | 5 +- esphome/components/api/api_frame_helper.cpp | 2 +- esphome/components/api/api_frame_helper.h | 3 + .../components/api/api_frame_helper_noise.cpp | 20 +- .../components/api/api_overflow_buffer.cpp | 121 ++--- esphome/components/api/api_overflow_buffer.h | 93 ++-- tests/components/api/__init__.py | 17 + tests/components/api/test_api_buffer.cpp | 65 +++ tests/components/api/test_overflow_buffer.cpp | 510 ++++++++++++++++++ 12 files changed, 755 insertions(+), 145 deletions(-) create mode 100644 tests/components/api/__init__.py create mode 100644 tests/components/api/test_api_buffer.cpp create mode 100644 tests/components/api/test_overflow_buffer.cpp diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 6202e127bf..272b078690 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -350,10 +350,9 @@ CONFIG_SCHEMA = cv.All( ln882x=5, # Moderate RAM nrf52=4, # ~256KB RAM, BSD sockets, Thread (single HA controller) ): cv.int_range(min=1, max=20), - # Maximum queued send buffers per connection before dropping connection - # Each buffer uses ~8-12 bytes overhead plus actual message size + # Max queued messages per connection, and 2 KB of backlog per slot up + # to 64 KB (a lone message is exempt), before the connection is dropped # Platform defaults based on available RAM and typical message rates: - # CONF_MAX_SEND_QUEUE defaults are power of 2 for efficient modulo cv.SplitDefault( CONF_MAX_SEND_QUEUE, esp8266=4, # Limited RAM, need to fail fast diff --git a/esphome/components/api/api_buffer.cpp b/esphome/components/api/api_buffer.cpp index fc45a4e971..62a544b1a4 100644 --- a/esphome/components/api/api_buffer.cpp +++ b/esphome/components/api/api_buffer.cpp @@ -1,20 +1,37 @@ #include "api_buffer.h" -#include +#ifdef ESPHOME_DEBUG_API +#include "esphome/core/log.h" +#endif namespace esphome::api { +#ifdef ESPHOME_DEBUG_API +void APIBuffer::debug_check_drop_(size_t drop) const { + if (drop > this->size_) { + ESP_LOGE("api.buffer", "drop_front: drop=%zu size=%u", drop, this->size_); + abort(); + } +} +#endif + bool APIBuffer::grow_(size_t n) { - // nothrow (no zero-fill) so OOM is reportable; plain new aborts instead - // (NEW_OOM_ABORT on ESP8266 Arduino, exception stub on ESP-IDF). - // RAMAllocator is no fit here: unique_ptr needs delete[]-compatible memory. - std::unique_ptr new_data(new (std::nothrow) uint8_t[n]); - if (new_data == nullptr) + if (n > MAX_SIZE) return false; - if (this->size_) - std::memcpy(new_data.get(), this->data_.get(), this->size_); - this->data_ = std::move(new_data); + // realloc extends in place when it can, avoiding the copy + uint8_t *grown = RAMAllocator().reallocate(this->data_.get(), n); + if (grown == nullptr) + return false; + (void) this->data_.release(); // realloc already freed or reused the old block + this->data_.reset(grown); this->capacity_ = n; return true; } +uint8_t *APIBuffer::append(size_t n) { + const size_t old_size = this->size_; + if (!this->resize(old_size + n)) + return nullptr; + return this->data_.get() + old_size; +} + } // namespace esphome::api diff --git a/esphome/components/api/api_buffer.h b/esphome/components/api/api_buffer.h index 396dadbe58..7caa68aa4d 100644 --- a/esphome/components/api/api_buffer.h +++ b/esphome/components/api/api_buffer.h @@ -25,6 +25,7 @@ namespace esphome::api { /// writes in debug builds. class APIBuffer { public: + static constexpr size_t MAX_SIZE = UINT16_MAX; // API frames carry 16 bit lengths void clear() { this->size_ = 0; } /// Returns false if allocation fails; the buffer is left unchanged. [[nodiscard]] inline bool reserve(size_t n) ESPHOME_ALWAYS_INLINE { return n <= this->capacity_ || this->grow_(n); } @@ -36,9 +37,19 @@ class APIBuffer { [[nodiscard]] inline bool reserve_and_resize(size_t reserve_size, size_t new_size) ESPHOME_ALWAYS_INLINE { if (!this->reserve(std::max(reserve_size, new_size))) return false; - this->size_ = new_size; + this->size_ = static_cast(new_size); return true; } + /// Grow by n bytes; returns the new bytes, or nullptr on allocation failure. + [[nodiscard]] uint8_t *append(size_t n); + /// Drop the first `drop` bytes, sliding the rest down. Precondition: drop <= size(). + void drop_front(size_t drop) { +#ifdef ESPHOME_DEBUG_API + this->debug_check_drop_(drop); +#endif + this->size_ -= drop; + std::memmove(this->data_.get(), this->data_.get() + drop, this->size_); + } uint8_t *data() { return this->data_.get(); } const uint8_t *data() const { return this->data_.get(); } size_t size() const { return this->size_; } @@ -55,9 +66,14 @@ class APIBuffer { protected: bool grow_(size_t n); - std::unique_ptr data_; - size_t size_{0}; - size_t capacity_{0}; +#ifdef ESPHOME_DEBUG_API + void debug_check_drop_(size_t drop) const; +#endif + // RAMAllocator: PSRAM when available, and it reports failure where + // new (std::nothrow) still aborts on ESP-IDF without exceptions + RAMUniquePtr data_; + uint16_t size_{0}; + uint16_t capacity_{0}; }; } // namespace esphome::api diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index da4b7d7702..cc0543a690 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -364,7 +364,10 @@ void APIConnection::check_keepalive_(uint32_t now) { ESP_LOGVV(TAG, "Sending keepalive PING"); PingRequest req; this->flags_.sent_ping = this->send_message(req); - if (!this->flags_.sent_ping) { + if (this->flags_.sent_ping) { + // Quiet for a keepalive period and the ping is on its way: a one-off stall's storage can go + this->helper_->release_overflow_buffer(); + } else { // If we can't send the ping request directly (tx_buffer full), // schedule it at the front of the batch so it will be sent with priority ESP_LOGW(TAG, "Buffer full, ping queued"); diff --git a/esphome/components/api/api_frame_helper.cpp b/esphome/components/api/api_frame_helper.cpp index 38da444a18..41d1230aaa 100644 --- a/esphome/components/api/api_frame_helper.cpp +++ b/esphome/components/api/api_frame_helper.cpp @@ -171,7 +171,7 @@ APIError APIFrameHelper::write_raw_iov_(const struct iovec *iov, int iovcnt, uin return APIError::OK; // Queue unsent data into overflow buffer - if (!this->overflow_buf_.enqueue_iov(iov, iovcnt, total_write_len, static_cast(sent))) { + if (!this->overflow_buf_.enqueue_iov(iov, iovcnt, total_write_len, sent)) { HELPER_LOG("Overflow buffer full or out of memory, dropping connection"); this->state_ = State::FAILED; return APIError::SOCKET_WRITE_FAILED; diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h index ff8aa7834c..a68a0ad0d8 100644 --- a/esphome/components/api/api_frame_helper.h +++ b/esphome/components/api/api_frame_helper.h @@ -219,7 +219,10 @@ class APIFrameHelper { if (this->rx_buf_len_ == 0) { this->rx_buf_.release(); } + this->release_overflow_buffer(); } + // Free the send backlog storage once it has drained + void release_overflow_buffer() { this->overflow_buf_.release(); } protected: // Drain backlogged overflow data to the socket and handle errors. diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 29b2858aee..400cd1d9b8 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -67,15 +67,15 @@ APIError APINoiseFrameHelper::init() { } // init prologue - size_t old_size = prologue_.size(); - if (!prologue_.resize(old_size + PROLOGUE_INIT_LEN)) [[unlikely]] { + uint8_t *dst = prologue_.append(PROLOGUE_INIT_LEN); + if (dst == nullptr) [[unlikely]] { state_ = State::FAILED; return APIError::OUT_OF_MEMORY; } #ifdef USE_ESP8266 - memcpy_P(prologue_.data() + old_size, PROLOGUE_INIT, PROLOGUE_INIT_LEN); + memcpy_P(dst, PROLOGUE_INIT, PROLOGUE_INIT_LEN); #else - std::memcpy(prologue_.data() + old_size, PROLOGUE_INIT, PROLOGUE_INIT_LEN); + std::memcpy(dst, PROLOGUE_INIT, PROLOGUE_INIT_LEN); #endif state_ = State::CLIENT_HELLO; @@ -272,17 +272,17 @@ APIError APINoiseFrameHelper::state_action_client_hello_() { return handle_handshake_frame_error_(aerr); } // ignore contents, may be used in future for flags - // Resize for: existing prologue + 2 size bytes + frame data - size_t old_size = this->prologue_.size(); + // Append 2 size bytes + frame data to the prologue size_t rx_size = this->rx_buf_.size(); - if (!this->prologue_.resize(old_size + 2 + rx_size)) [[unlikely]] { + uint8_t *dst = this->prologue_.append(2 + rx_size); + if (dst == nullptr) [[unlikely]] { state_ = State::FAILED; return APIError::OUT_OF_MEMORY; } - this->prologue_[old_size] = (uint8_t) (rx_size >> 8); - this->prologue_[old_size + 1] = (uint8_t) rx_size; + dst[0] = (uint8_t) (rx_size >> 8); + dst[1] = (uint8_t) rx_size; if (rx_size > 0) { - std::memcpy(this->prologue_.data() + old_size + 2, this->rx_buf_.data(), rx_size); + std::memcpy(dst + 2, this->rx_buf_.data(), rx_size); } state_ = State::SERVER_HELLO; diff --git a/esphome/components/api/api_overflow_buffer.cpp b/esphome/components/api/api_overflow_buffer.cpp index 48d8fe18ba..0b5a874d4b 100644 --- a/esphome/components/api/api_overflow_buffer.cpp +++ b/esphome/components/api/api_overflow_buffer.cpp @@ -1,98 +1,91 @@ #include "api_overflow_buffer.h" #ifdef USE_API #include -#include namespace esphome::api { -APIOverflowBuffer::~APIOverflowBuffer() { - for (auto *entry : this->queue_) { - if (entry != nullptr) - Entry::destroy(entry); - } -} - ssize_t APIOverflowBuffer::try_drain(socket::Socket *socket) { - // socket->write() can re-enter this function: a log message emitted from an - // lwip callback during the write goes out over the API and lands back in the - // frame helper's write/drain path. If a nested drain ran here it would send - // and free the entry the outer drain is still holding, causing a double free. - // Report "no progress" instead; the outer drain keeps draining, and the - // nested send is enqueued behind the existing backlog. + // Nested call from inside socket->write(); see draining_ if (this->draining_) return 0; - // RAII so the flag is cleared on every return path struct DrainGuard { - explicit DrainGuard(bool &flag) : flag_(flag) { flag_ = true; } - ~DrainGuard() { this->flag_ = false; } - bool &flag_; - } guard(this->draining_); + APIOverflowBuffer &owner; + ~DrainGuard() { this->owner.draining_ = false; } + } guard{*this}; + this->draining_ = true; while (this->count_ > 0) { - Entry *front = this->queue_[this->head_]; + uint8_t *msg = this->buf_.data() + this->head_; + size_t len = msg[0] | (msg[1] << 8); - ssize_t sent = socket->write(front->current_data(), front->remaining()); - - if (sent <= 0) { - // -1 = error (caller checks errno for EWOULDBLOCK vs hard error) - // 0 = nothing sent (treat as no progress) + ssize_t sent = socket->write(msg + LEN_PREFIX, len); + if (sent <= 0) + return sent; + if (static_cast(sent) < len) { + // Step past the sent bytes and rewrite the prefix there; it lands on bytes already sent + this->head_ += sent; + len -= sent; + msg += sent; + msg[0] = len; + msg[1] = len >> 8; return sent; } - - if (static_cast(sent) < front->remaining()) { - // Partially sent, update offset and stop - front->offset += static_cast(sent); - return sent; - } - - // Entry fully sent — unlink it before freeing so a freed pointer is never - // reachable from the queue - this->queue_[this->head_] = nullptr; - this->head_ = (this->head_ + 1) % API_MAX_SEND_QUEUE; + this->head_ += LEN_PREFIX + len; this->count_--; - Entry::destroy(front); } - return 0; // All drained + this->head_ = 0; + if (this->release_when_drained_) { + this->release_when_drained_ = false; + this->buf_.release(); + } else { + this->buf_.clear(); + } + return 0; } -bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, uint16_t total_len, uint16_t skip) { +bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, size_t total_len, size_t skip) { if (this->count_ >= API_MAX_SEND_QUEUE) return false; - uint16_t buffer_size = total_len - skip; - // nothrow: a failed allocation returns nullptr so the connection is dropped - // cleanly instead of plain new's crash or abort on OOM - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - auto *data = new (std::nothrow) uint8_t[buffer_size]; - if (data == nullptr) - return false; - // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - auto *entry = new (std::nothrow) Entry{data, buffer_size, 0}; - if (entry == nullptr) { - delete[] data; + const size_t new_len = total_len - skip; + const size_t new_bytes = LEN_PREFIX + new_len; + const size_t live = this->buf_.size() - this->head_; + // A lone message is only bound by the buffer; refusing it would just drop the connection + if (live + new_bytes > (this->count_ > 0 ? MAX_BYTES : MAX_LONE_BYTES)) return false; + + if (this->buf_.size() + new_bytes > this->buf_.capacity()) { + // Storage would move under an outer drain's write() + if (this->draining_) + return false; + if (this->head_ > 0) { + // Reclaim the sent prefix before growing + this->buf_.drop_front(this->head_); + this->head_ = 0; + } + if (!this->buf_.reserve(reserve_for(live + new_bytes))) + return false; } - uint16_t to_skip = skip; - uint16_t write_pos = 0; - - for (int i = 0; i < iovcnt; i++) { - if (to_skip >= iov[i].iov_len) { - to_skip -= static_cast(iov[i].iov_len); + uint8_t *dst = this->buf_.append(new_bytes); + if (dst == nullptr) + return false; + dst[0] = new_len; + dst[1] = new_len >> 8; + dst += LEN_PREFIX; + for (const struct iovec *end = iov + iovcnt; iov != end; iov++) { + if (skip >= iov->iov_len) { + skip -= iov->iov_len; } else { - const uint8_t *src = reinterpret_cast(iov[i].iov_base) + to_skip; - uint16_t len = static_cast(iov[i].iov_len) - to_skip; - std::memcpy(entry->data + write_pos, src, len); - write_pos += len; - to_skip = 0; + const size_t len = iov->iov_len - skip; + std::memcpy(dst, static_cast(iov->iov_base) + skip, len); + dst += len; + skip = 0; } } - // Publish only after the copy completes so a half-built entry is never reachable - this->queue_[this->tail_] = entry; - this->tail_ = (this->tail_ + 1) % API_MAX_SEND_QUEUE; this->count_++; return true; } diff --git a/esphome/components/api/api_overflow_buffer.h b/esphome/components/api/api_overflow_buffer.h index 03a334b281..e2e4b9c3c3 100644 --- a/esphome/components/api/api_overflow_buffer.h +++ b/esphome/components/api/api_overflow_buffer.h @@ -1,5 +1,6 @@ #pragma once -#include +#include +#include #include #include @@ -8,71 +9,57 @@ #include "esphome/components/socket/headers.h" #include "esphome/components/socket/socket.h" -#include "esphome/core/helpers.h" +#include "api_buffer.h" namespace esphome::api { -/// Circular queue of heap-allocated byte buffers used as a TCP send backlog. -/// -/// Under normal operation this buffer is **never used** — data goes straight -/// from the frame helper to the socket. It only fills when the LWIP TCP -/// send buffer is full (slow client, congested network, heavy logging). -/// The queue drains automatically on subsequent write/loop calls once the -/// socket becomes writable again. -/// -/// Capacity is compile-time-fixed via API_MAX_SEND_QUEUE (set from Python -/// config). If the queue fills completely the connection is marked failed. +/// TCP send backlog, only used when the socket send buffer is full. +/// One contiguous buffer per connection, allocated on the first stall and +/// kept at its high-water mark so a lossy link does not churn the heap. +/// Messages are stored as a 2 byte length prefix plus payload. +/// API_MAX_SEND_QUEUE bounds queued messages and, at 2 KB per slot, queued +/// bytes; exceeding either fails the connection. class APIOverflowBuffer { public: - /// A single heap-allocated send-backlog entry. - /// Lifetime is manually managed — see destroy(). - struct Entry { - uint8_t *data; - uint16_t size; // Total size of the buffer - uint16_t offset; // Current send offset within the buffer - - uint16_t remaining() const { return this->size - this->offset; } - const uint8_t *current_data() const { return this->data + this->offset; } - - /// Free this entry and its data buffer. - static ESPHOME_ALWAYS_INLINE void destroy(Entry *entry) { - delete[] entry->data; - delete entry; // NOLINT(cppcoreguidelines-owning-memory) - } - }; - - ~APIOverflowBuffer(); - /// True when no backlogged data is waiting. bool empty() const { return this->count_ == 0; } - /// True when the queue has no room for another entry. - bool full() const { return this->count_ >= API_MAX_SEND_QUEUE; } - - /// Number of entries currently queued. - uint8_t count() const { return this->count_; } - - /// Try to drain queued data to the socket. - /// Returns bytes-written > 0 on success/partial, 0 if all drained or no progress, - /// -1 on error (caller must check errno to distinguish EWOULDBLOCK from hard errors). - /// Callers only need to act on -1; 0 and positive values both mean "no error". - /// Frees entries as they are fully sent. + /// Drain queued messages to the socket. + /// Returns bytes written, 0 for a re-entrant call, -1 on error (check errno + /// for EWOULDBLOCK); callers only need to act on -1. ssize_t try_drain(socket::Socket *socket); - /// Enqueue unsent IOV data into the backlog. - /// Copies iov data starting at byte offset `skip` into a new entry. - /// Returns false if the queue is full or allocation fails (caller should fail the connection). - bool enqueue_iov(const struct iovec *iov, int iovcnt, uint16_t total_len, uint16_t skip); + /// Queue iov data from byte offset `skip` as one message. + /// Returns false when a limit is hit, allocation fails, or storage would move + /// during a drain; the caller should fail the connection. + bool enqueue_iov(const struct iovec *iov, int iovcnt, size_t total_len, size_t skip); + + /// Free the retained storage, now if empty, otherwise once it has drained. + void release() { + if (this->count_ == 0) { + this->buf_.release(); + } else { + this->release_when_drained_ = true; + } + } protected: - std::array queue_{}; - uint8_t head_{0}; - uint8_t tail_{0}; + static constexpr size_t LEN_PREFIX = 2; + static constexpr size_t BYTES_PER_SLOT = 2048; + // Reserve in 256 byte steps so a creeping high-water mark settles quickly + static constexpr size_t GROW_QUANTUM = 256; + // Lone message ceiling, rounded down so reserve_for() never exceeds the buffer limit + static constexpr size_t MAX_LONE_BYTES = APIBuffer::MAX_SIZE & ~(GROW_QUANTUM - 1); + static constexpr size_t MAX_BYTES = std::min(API_MAX_SEND_QUEUE * BYTES_PER_SLOT, MAX_LONE_BYTES); + static constexpr size_t reserve_for(size_t want) { return (want + GROW_QUANTUM - 1) & ~(GROW_QUANTUM - 1); } + + APIBuffer buf_; + uint16_t head_{0}; // offset of the front message's length prefix; bytes before it are sent uint8_t count_{0}; - // Guards against re-entrant drains: socket->write() can re-enter the API - // send path (e.g. a log message emitted from an lwip callback), and a nested - // drain would free the entry the outer drain is still holding. - bool draining_{false}; + // socket->write() can re-enter the send path (log from an lwip callback): + // a nested drain makes no progress and a nested enqueue never moves storage + bool draining_ : 1 {false}; + bool release_when_drained_ : 1 {false}; }; } // namespace esphome::api diff --git a/tests/components/api/__init__.py b/tests/components/api/__init__.py new file mode 100644 index 0000000000..2aa558726c --- /dev/null +++ b/tests/components/api/__init__.py @@ -0,0 +1,17 @@ +import esphome.codegen as cg +from esphome.core import CORE +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + # USE_API compiles every api source, so emit what they need. No socket + # override: an __init__.py there makes pytest import its conftest as socket.conftest. + async def to_code_testing(config): + cg.add_define("USE_API") + cg.add_define("USE_API_PLAINTEXT") + cg.add_define("API_MAX_SEND_QUEUE", 8) + cg.add_define("MAX_API_CONNECTIONS", 1) + cg.add_define("USE_SOCKET_IMPL_BSD_SOCKETS") + CORE.register_controller() # api_server registers with the controller registry + + manifest.to_code = to_code_testing diff --git a/tests/components/api/test_api_buffer.cpp b/tests/components/api/test_api_buffer.cpp new file mode 100644 index 0000000000..c54780050e --- /dev/null +++ b/tests/components/api/test_api_buffer.cpp @@ -0,0 +1,65 @@ +#include + +#include +#include + +#include "esphome/components/api/api_buffer.h" + +namespace esphome::api::testing { + +// Pointer plus two 16 bit sizes +static_assert(sizeof(APIBuffer) <= 2 * sizeof(void *)); + +TEST(APIBuffer, RefusesSizesAbove16Bits) { + APIBuffer buf; + ASSERT_TRUE(buf.resize(16)); + EXPECT_FALSE(buf.reserve(UINT16_MAX + 1)); + EXPECT_EQ(buf.size(), 16u); + EXPECT_EQ(buf.capacity(), 16u); + EXPECT_TRUE(buf.reserve(UINT16_MAX)); + EXPECT_EQ(buf.capacity(), UINT16_MAX); +} + +static const uint8_t BYTES[] = {1, 2, 3, 4, 5, 6}; + +TEST(APIBuffer, AppendReturnsTheNewBytes) { + APIBuffer buf; + ASSERT_TRUE(buf.reserve(8)); + uint8_t *first = buf.append(3); + ASSERT_NE(first, nullptr); + std::memcpy(first, BYTES, 3); + EXPECT_EQ(buf.size(), 3u); + EXPECT_EQ(buf.capacity(), 8u); + + // Grows through realloc and keeps what was there + uint8_t *second = buf.append(6); + ASSERT_EQ(second, buf.data() + 3); + std::memcpy(second, BYTES + 3, 3); + EXPECT_EQ(buf.size(), 9u); + EXPECT_EQ(buf.capacity(), 9u); + EXPECT_EQ(std::memcmp(buf.data(), BYTES, 6), 0); +} + +TEST(APIBuffer, DropFrontSlidesTheRestDown) { + APIBuffer buf; + uint8_t *bytes = buf.append(6); + ASSERT_NE(bytes, nullptr); + std::memcpy(bytes, BYTES, 6); + + buf.drop_front(2); + EXPECT_EQ(buf.size(), 4u); + EXPECT_EQ(buf.capacity(), 6u); + EXPECT_EQ(std::memcmp(buf.data(), BYTES + 2, 4), 0); + + // Growing afterwards keeps the slid bytes + ASSERT_TRUE(buf.reserve(64)); + EXPECT_EQ(buf.size(), 4u); + EXPECT_EQ(std::memcmp(buf.data(), BYTES + 2, 4), 0); + + // Dropping everything leaves an empty buffer with its capacity + buf.drop_front(4); + EXPECT_EQ(buf.size(), 0u); + EXPECT_EQ(buf.capacity(), 64u); +} + +} // namespace esphome::api::testing diff --git a/tests/components/api/test_overflow_buffer.cpp b/tests/components/api/test_overflow_buffer.cpp new file mode 100644 index 0000000000..4b27e54496 --- /dev/null +++ b/tests/components/api/test_overflow_buffer.cpp @@ -0,0 +1,510 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "esphome/components/api/api_overflow_buffer.h" + +#ifdef USE_HOST +namespace esphome::api::testing { + +// Idle cost is the buffer plus one word of bookkeeping +static_assert(sizeof(APIOverflowBuffer) <= sizeof(APIBuffer) + sizeof(void *)); + +// Exposes storage so tests can check it is reused, not reallocated +class TestOverflowBuffer : public APIOverflowBuffer { + public: + using APIOverflowBuffer::LEN_PREFIX; + using APIOverflowBuffer::MAX_BYTES; + using APIOverflowBuffer::MAX_LONE_BYTES; + struct Storage { + size_t capacity; + const uint8_t *data; + bool operator==(const Storage &) const = default; + }; + size_t capacity() const { return this->buf_.capacity(); } + Storage storage() const { return {this->buf_.capacity(), this->buf_.data()}; } + uint8_t count() const { return this->count_; } + size_t live() const { return this->buf_.size() - this->head_; } + /// Simulates a socket write inside try_drain() re-entering the send path + void set_draining(bool draining) { this->draining_ = draining; } +}; + +static std::vector make_message(size_t len, uint8_t seed) { + std::vector msg(len); + for (size_t i = 0; i < len; i++) + msg[i] = static_cast(seed + i); + return msg; +} + +static bool enqueue(TestOverflowBuffer &buf, const std::vector &msg, uint16_t skip = 0) { + struct iovec iov = {const_cast(msg.data()), msg.size()}; + return buf.enqueue_iov(&iov, 1, static_cast(msg.size()), skip); +} + +static void append(std::vector &dst, const std::vector &src, size_t skip = 0) { + dst.insert(dst.end(), src.begin() + skip, src.end()); +} + +static std::vector concat(std::initializer_list> parts) { + std::vector out; + for (const auto &part : parts) + append(out, part); + return out; +} + +/// The pipe delivers the filler first, then the drained messages. +static void expect_after_filler(const std::vector &received, size_t filler, + const std::vector &expected) { + ASSERT_EQ(received.size(), filler + expected.size()); + EXPECT_TRUE(std::equal(expected.begin(), expected.end(), received.begin() + filler)); +} + +// Non-blocking socket pair with small buffers, so the writer fills like a stalled TCP connection +class OverflowBufferTest : public ::testing::Test { + protected: + void SetUp() override { + int fds[2]; + ASSERT_EQ(::socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + int size = 4096; + ASSERT_EQ(::setsockopt(fds[0], SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)), 0); + ASSERT_EQ(::setsockopt(fds[1], SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)), 0); + ASSERT_EQ(::fcntl(fds[1], F_SETFL, O_NONBLOCK), 0); + this->reader_ = fds[1]; + this->sock_ = std::make_unique(fds[0]); + ASSERT_EQ(this->sock_->setblocking(false), 0); + } + void TearDown() override { ::close(this->reader_); } + + /// Write filler until the socket refuses; returns the bytes accepted + size_t fill_pipe_() { + uint8_t junk[512]; + std::memset(junk, 0xEE, sizeof(junk)); + size_t total = 0; + for (;;) { + ssize_t written = this->sock_->write(junk, sizeof(junk)); + if (written <= 0) + break; + total += static_cast(written); + } + return total; + } + + /// Append whatever the pipe currently holds. + void read_into_(std::vector &out) { + uint8_t tmp[1024]; + for (;;) { + ssize_t n = ::read(this->reader_, tmp, sizeof(tmp)); + if (n <= 0) + break; + out.insert(out.end(), tmp, tmp + n); + } + } + + /// Drain once; a refusal must be a would-block, never a hard error. + ssize_t drain_(TestOverflowBuffer &buf) { + ssize_t sent = buf.try_drain(this->sock_.get()); + if (sent == -1) { + EXPECT_TRUE(errno == EWOULDBLOCK || errno == EAGAIN); + } + return sent; + } + + /// Read and drain until the backlog is empty; returns all bytes received + std::vector drain_all_(TestOverflowBuffer &buf) { + std::vector received; + for (int i = 0; i < 10000 && !buf.empty(); i++) { + this->read_into_(received); + // A hard socket error would never clear the backlog; stop instead of spinning + if (this->drain_(buf) == -1 && errno != EWOULDBLOCK && errno != EAGAIN) + break; + } + EXPECT_TRUE(buf.empty()); + this->read_into_(received); + return received; + } + + struct Stall { + size_t filler; + std::vector first, second, received; + TestOverflowBuffer::Storage before; + }; + /// Park two messages, then drain the first fully and the second part way + void stall_mid_message_(TestOverflowBuffer &buf, Stall &s) { + s.filler = this->fill_pipe_(); + s.first = make_message(1500, 20); + ASSERT_GT(s.filler, s.first.size()); // the first message must drain in one go + // Larger than the whole pipe, so a drain always stops inside it + s.second = make_message(std::max(s.filler + 1, std::min(s.filler * 3, 12000)), 60); + ASSERT_GT(s.second.size(), s.filler); + ASSERT_TRUE(enqueue(buf, s.first)); + ASSERT_TRUE(enqueue(buf, s.second)); + s.before = buf.storage(); + this->read_into_(s.received); + ASSERT_GT(this->drain_(buf), 0); + ASSERT_EQ(buf.count(), 1); + } + + int reader_{-1}; + std::unique_ptr sock_; +}; + +TEST_F(OverflowBufferTest, IdleBufferOwnsNoStorage) { + TestOverflowBuffer buf; + EXPECT_TRUE(buf.empty()); + EXPECT_EQ(buf.capacity(), 0u); + EXPECT_EQ(buf.storage().data, nullptr); +} + +TEST_F(OverflowBufferTest, StorageIsReusedAcrossStalls) { + TestOverflowBuffer buf; + auto msg = make_message(1000, 1); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + const auto storage = buf.storage(); + EXPECT_GE(storage.capacity, msg.size() + TestOverflowBuffer::LEN_PREFIX); + + for (int stall = 0; stall < 5; stall++) { + expect_after_filler(this->drain_all_(buf), filler, msg); + EXPECT_TRUE(buf.empty()); + // Same allocation every time: no free, no new allocation + EXPECT_EQ(buf.storage(), storage); + + filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + EXPECT_EQ(buf.storage(), storage); + } +} + +TEST_F(OverflowBufferTest, ReleaseWhileQueuedFreesOnceDrained) { + TestOverflowBuffer buf; + auto msg = make_message(1000, 7); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + const size_t capacity = buf.capacity(); + + // Requested while the backlog still holds data: storage must stay until sent + buf.release(); + EXPECT_FALSE(buf.empty()); + EXPECT_EQ(buf.capacity(), capacity); + + expect_after_filler(this->drain_all_(buf), filler, msg); + EXPECT_TRUE(buf.empty()); + EXPECT_EQ(buf.capacity(), 0u); + EXPECT_EQ(buf.storage().data, nullptr); + + // A later stall allocates again and keeps it, since nobody asked for a release + filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + EXPECT_GT(buf.capacity(), 0u); + this->drain_all_(buf); + EXPECT_GT(buf.capacity(), 0u); +} + +TEST_F(OverflowBufferTest, ReleaseWhenEmptyFreesImmediately) { + TestOverflowBuffer buf; + auto msg = make_message(100, 3); + + this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + this->drain_all_(buf); + EXPECT_GT(buf.capacity(), 0u); + + buf.release(); + EXPECT_EQ(buf.capacity(), 0u); + EXPECT_EQ(buf.storage().data, nullptr); +} + +TEST_F(OverflowBufferTest, PreservesOrderAndSkipsSentPrefix) { + TestOverflowBuffer buf; + auto first = make_message(700, 10); + auto second_a = make_message(300, 50); + auto second_b = make_message(400, 90); + auto third = make_message(200, 130); + + size_t filler = this->fill_pipe_(); + // 100 bytes of the first message were already accepted by the socket + ASSERT_TRUE(enqueue(buf, first, 100)); + // Two iovecs with the skip covering all of the first one plus part of the second + struct iovec iov[2] = {{second_a.data(), second_a.size()}, {second_b.data(), second_b.size()}}; + const uint16_t second_skip = static_cast(second_a.size() + 5); + ASSERT_TRUE(buf.enqueue_iov(iov, 2, static_cast(second_a.size() + second_b.size()), second_skip)); + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.count(), 3); + + // Nothing can go out while the pipe is full + EXPECT_EQ(this->drain_(buf), -1); + EXPECT_EQ(buf.count(), 3); + + std::vector expected; + append(expected, first, 100); + append(expected, second_b, 5); + append(expected, third); + expect_after_filler(this->drain_all_(buf), filler, expected); +} + +TEST_F(OverflowBufferTest, RefusesWhenQueueIsFull) { + TestOverflowBuffer buf; + auto msg = make_message(16, 1); + + size_t filler = this->fill_pipe_(); + for (int i = 0; i < API_MAX_SEND_QUEUE; i++) { + ASSERT_TRUE(enqueue(buf, msg)) << "message " << i; + } + EXPECT_FALSE(enqueue(buf, msg)); + EXPECT_EQ(buf.count(), API_MAX_SEND_QUEUE); + + // Draining frees the slots again + std::vector expected; + for (int i = 0; i < API_MAX_SEND_QUEUE; i++) + append(expected, msg); + expect_after_filler(this->drain_all_(buf), filler, expected); + this->fill_pipe_(); + EXPECT_TRUE(enqueue(buf, msg)); + EXPECT_EQ(buf.count(), 1); +} + +TEST_F(OverflowBufferTest, SkipAtIovecBoundary) { + TestOverflowBuffer buf; + auto sent = make_message(300, 50); + auto unsent = make_message(400, 90); + + size_t filler = this->fill_pipe_(); + // The skip covers the first iovec exactly, so only the second is copied + struct iovec iov[2] = {{sent.data(), sent.size()}, {unsent.data(), unsent.size()}}; + ASSERT_TRUE( + buf.enqueue_iov(iov, 2, static_cast(sent.size() + unsent.size()), static_cast(sent.size()))); + EXPECT_EQ(buf.live(), unsent.size() + TestOverflowBuffer::LEN_PREFIX); + expect_after_filler(this->drain_all_(buf), filler, unsent); +} + +TEST_F(OverflowBufferTest, AppendsBehindSentPrefixWhenItFits) { + TestOverflowBuffer buf; + size_t filler = this->fill_pipe_(); + auto first = make_message(200, 20); + // Size the second message so the two land half way into a 256 byte step, + // leaving exactly 128 bytes of slack whatever the pipe accepted + const size_t base = std::max(filler + 1, std::min(filler * 3, 12000)); + const size_t second_len = (base / 256 + 1) * 256 + 128 - first.size() - 2 * TestOverflowBuffer::LEN_PREFIX; + auto second = make_message(second_len, 60); + ASSERT_GT(second.size(), filler); + ASSERT_TRUE(enqueue(buf, first)); + ASSERT_TRUE(enqueue(buf, second)); + const auto storage = buf.storage(); + const size_t slack = storage.capacity - first.size() - second.size() - 2 * TestOverflowBuffer::LEN_PREFIX; + ASSERT_EQ(slack, 128u); + auto third = make_message(slack - TestOverflowBuffer::LEN_PREFIX, 200); + + std::vector received; + this->read_into_(received); + ASSERT_GT(this->drain_(buf), 0); + ASSERT_EQ(buf.count(), 1); + const size_t live = buf.live(); + + // Fits in the tail, so the sent prefix is left alone + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.storage(), storage); + EXPECT_EQ(buf.live(), live + third.size() + TestOverflowBuffer::LEN_PREFIX); + + append(received, this->drain_all_(buf)); + expect_after_filler(received, filler, concat({first, second, third})); +} + +TEST_F(OverflowBufferTest, ReleaseSurvivesFurtherEnqueues) { + TestOverflowBuffer buf; + auto first = make_message(300, 7); + auto second = make_message(300, 70); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, first)); + buf.release(); + ASSERT_TRUE(enqueue(buf, second)); + EXPECT_GT(buf.capacity(), 0u); + + expect_after_filler(this->drain_all_(buf), filler, concat({first, second})); + EXPECT_EQ(buf.capacity(), 0u); +} + +TEST_F(OverflowBufferTest, RefusesWhenByteLimitIsExceeded) { + TestOverflowBuffer buf; + // Two of these fill the byte budget exactly, well before the slot count is reached + static_assert(API_MAX_SEND_QUEUE >= 3); + auto msg = make_message(TestOverflowBuffer::MAX_BYTES / 2 - TestOverflowBuffer::LEN_PREFIX, 1); + + this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + ASSERT_TRUE(enqueue(buf, msg)); + EXPECT_FALSE(enqueue(buf, msg)); + EXPECT_EQ(buf.count(), 2); +} + +TEST_F(OverflowBufferTest, LoneMessageMayExceedByteLimit) { + TestOverflowBuffer buf; + // The oversized message must still fit under the lone message ceiling + static_assert(TestOverflowBuffer::MAX_BYTES + 100 + TestOverflowBuffer::LEN_PREFIX <= + TestOverflowBuffer::MAX_LONE_BYTES); + auto big = make_message(TestOverflowBuffer::MAX_BYTES + 100, 5); + auto small = make_message(16, 9); + + // Refusing the only message would drop the connection for nothing + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, big)); + EXPECT_EQ(buf.count(), 1); + // With a backlog present the byte limit applies again + EXPECT_FALSE(enqueue(buf, small)); + EXPECT_EQ(buf.count(), 1); + + expect_after_filler(this->drain_all_(buf), filler, big); +} + +TEST_F(OverflowBufferTest, LoneMessageAboveOffsetLimitIsRefused) { + TestOverflowBuffer buf; + // Payload plus prefix is past the lone message ceiling + auto msg = make_message(TestOverflowBuffer::MAX_LONE_BYTES, 3); + + this->fill_pipe_(); + EXPECT_FALSE(enqueue(buf, msg)); + EXPECT_TRUE(buf.empty()); + EXPECT_EQ(buf.capacity(), 0u); +} + +TEST_F(OverflowBufferTest, HardSocketErrorLeavesBacklogIntact) { + TestOverflowBuffer buf; + auto msg = make_message(300, 40); + + this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + // A closed socket fails every write outright, unlike a full one + ASSERT_EQ(this->sock_->close(), 0); + + errno = 0; + EXPECT_EQ(buf.try_drain(this->sock_.get()), -1); + EXPECT_NE(errno, EWOULDBLOCK); + EXPECT_NE(errno, EAGAIN); + EXPECT_EQ(buf.count(), 1); + EXPECT_EQ(buf.live(), msg.size() + TestOverflowBuffer::LEN_PREFIX); +} + +TEST_F(OverflowBufferTest, GrowsWhileReclaimingSentPrefix) { + TestOverflowBuffer buf; + Stall s; + ASSERT_NO_FATAL_FAILURE(this->stall_mid_message_(buf, s)); + + // One byte too many to fit even after the sent prefix is reclaimed: grows in one copy + auto third = make_message(s.before.capacity - buf.live() + 1, 200); + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_GT(buf.capacity(), s.before.capacity); + EXPECT_EQ(buf.count(), 2); + + append(s.received, this->drain_all_(buf)); + expect_after_filler(s.received, s.filler, concat({s.first, s.second, third})); +} + +TEST_F(OverflowBufferTest, NestedDrainMakesNoProgress) { + TestOverflowBuffer buf; + auto msg = make_message(300, 40); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, msg)); + std::vector received; + this->read_into_(received); + + // Room is available, but a nested drain must leave the outer one's message alone + buf.set_draining(true); + EXPECT_EQ(this->drain_(buf), 0); + EXPECT_EQ(buf.count(), 1); + std::vector nothing; + this->read_into_(nothing); + EXPECT_TRUE(nothing.empty()); + + buf.set_draining(false); + append(received, this->drain_all_(buf)); + expect_after_filler(received, filler, msg); +} + +TEST_F(OverflowBufferTest, NestedEnqueueAppendsWithinCapacity) { + TestOverflowBuffer buf; + auto first = make_message(500, 10); + auto second = make_message(4, 90); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, first)); + const auto storage = buf.storage(); + ASSERT_GE(storage.capacity, first.size() + second.size() + 2 * TestOverflowBuffer::LEN_PREFIX); + + buf.set_draining(true); + EXPECT_TRUE(enqueue(buf, second)); + EXPECT_EQ(buf.count(), 2); + EXPECT_EQ(buf.storage(), storage); + buf.set_draining(false); + + expect_after_filler(this->drain_all_(buf), filler, concat({first, second})); +} + +TEST_F(OverflowBufferTest, NestedEnqueueRefusesToGrow) { + TestOverflowBuffer buf; + auto first = make_message(500, 10); + auto second = make_message(100, 90); + + size_t filler = this->fill_pipe_(); + ASSERT_TRUE(enqueue(buf, first)); + const auto storage = buf.storage(); + ASSERT_LT(storage.capacity, first.size() + second.size() + 2 * TestOverflowBuffer::LEN_PREFIX); + + // Growing would free the bytes the outer write() is sending from + buf.set_draining(true); + EXPECT_FALSE(enqueue(buf, second)); + EXPECT_EQ(buf.count(), 1); + EXPECT_EQ(buf.storage(), storage); + buf.set_draining(false); + + expect_after_filler(this->drain_all_(buf), filler, first); +} + +TEST_F(OverflowBufferTest, NestedEnqueueRefusesToCompact) { + TestOverflowBuffer buf; + Stall s; + ASSERT_NO_FATAL_FAILURE(this->stall_mid_message_(buf, s)); + auto third = make_message(1000, 200); + + // Sliding the remainder down would move the bytes the outer write() points at + buf.set_draining(true); + EXPECT_FALSE(enqueue(buf, third)); + EXPECT_EQ(buf.count(), 1); + EXPECT_EQ(buf.storage(), s.before); + buf.set_draining(false); + + // Once the drain is over the same enqueue compacts and succeeds + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.storage(), s.before); + append(s.received, this->drain_all_(buf)); + expect_after_filler(s.received, s.filler, concat({s.first, s.second, third})); +} + +TEST_F(OverflowBufferTest, CompactsInsteadOfGrowingAfterPartialDrain) { + TestOverflowBuffer buf; + Stall s; + ASSERT_NO_FATAL_FAILURE(this->stall_mid_message_(buf, s)); + auto third = make_message(1000, 200); + + // The sent first message is reclaimed by sliding the remainder down, not by reallocating + ASSERT_TRUE(enqueue(buf, third)); + EXPECT_EQ(buf.storage(), s.before); + + append(s.received, this->drain_all_(buf)); + expect_after_filler(s.received, s.filler, concat({s.first, s.second, third})); +} + +} // namespace esphome::api::testing +#endif // USE_HOST From 7fe0689fb8e1b0b94ce2f0e3284533ecb61d9981 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:12:27 +1200 Subject: [PATCH 163/178] Bump version to 2026.9.0b4 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 97ce92240c..331d2f7984 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0b3 +PROJECT_NUMBER = 2026.9.0b4 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index b013098f33..5696125355 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0b3" +__version__ = "2026.9.0b4" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From a0821c225af41045f426b41e851972e07c6ddd9f Mon Sep 17 00:00:00 2001 From: Jeff Brown Date: Sun, 13 Sep 2026 18:18:52 -0700 Subject: [PATCH 164/178] [pmsa003i] Fix read from uninitialized stack memory (#19053) --- esphome/components/pmsa003i/pmsa003i.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/esphome/components/pmsa003i/pmsa003i.cpp b/esphome/components/pmsa003i/pmsa003i.cpp index 15f5d3e879..0b5c72a94d 100644 --- a/esphome/components/pmsa003i/pmsa003i.cpp +++ b/esphome/components/pmsa003i/pmsa003i.cpp @@ -88,7 +88,11 @@ void PMSA003IComponent::update() { bool PMSA003IComponent::read_data_(PM25AQIData *data) { uint8_t buffer[COUNT_DATA_BYTES]; - this->read_bytes_raw(buffer, COUNT_DATA_BYTES); + const i2c::ErrorCode error = this->read(buffer, COUNT_DATA_BYTES); + if (error != i2c::ERROR_OK) { + ESP_LOGW(TAG, "I2C error %d", error); + return false; + } // https://github.com/adafruit/Adafruit_PM25AQI From 41e34c19eb433ca620c80bcef842d447c1d5f491 Mon Sep 17 00:00:00 2001 From: tronikos Date: Sun, 13 Sep 2026 18:36:39 -0700 Subject: [PATCH 165/178] [template] Stop water heater republishing when a temperature is unknown (#19013) --- .../water_heater/template_water_heater.cpp | 10 ++++-- ...r_heater_template_unknown_temperature.yaml | 16 +++++++++ .../integration/test_water_heater_template.py | 33 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/integration/fixtures/water_heater_template_unknown_temperature.yaml diff --git a/esphome/components/template/water_heater/template_water_heater.cpp b/esphome/components/template/water_heater/template_water_heater.cpp index 092df6fdca..9d6a3523d2 100644 --- a/esphome/components/template/water_heater/template_water_heater.cpp +++ b/esphome/components/template/water_heater/template_water_heater.cpp @@ -1,6 +1,8 @@ #include "template_water_heater.h" #include "esphome/core/log.h" +#include + namespace esphome::template_ { static const char *const TAG = "template.water_heater"; @@ -45,9 +47,12 @@ water_heater::WaterHeaterTraits TemplateWaterHeater::traits() { void TemplateWaterHeater::loop() { bool changed = false; + // NAN is passed through so a source that has no value yet shows as unknown, but NAN never + // equals NAN, so an already-NAN value must not count as a change or it would republish forever. auto curr_temp = this->current_temperature_f_.call(); if (curr_temp.has_value()) { - if (*curr_temp != this->current_temperature_) { + if (*curr_temp != this->current_temperature_ && + !(std::isnan(*curr_temp) && std::isnan(this->current_temperature_))) { this->current_temperature_ = *curr_temp; changed = true; } @@ -55,7 +60,8 @@ void TemplateWaterHeater::loop() { auto target_temp = this->target_temperature_f_.call(); if (target_temp.has_value()) { - if (*target_temp != this->target_temperature_) { + if (*target_temp != this->target_temperature_ && + !(std::isnan(*target_temp) && std::isnan(this->target_temperature_))) { this->target_temperature_ = *target_temp; changed = true; } diff --git a/tests/integration/fixtures/water_heater_template_unknown_temperature.yaml b/tests/integration/fixtures/water_heater_template_unknown_temperature.yaml new file mode 100644 index 0000000000..a70ed25bd7 --- /dev/null +++ b/tests/integration/fixtures/water_heater_template_unknown_temperature.yaml @@ -0,0 +1,16 @@ +esphome: + name: wh-template-unknown-test +host: +api: +logger: + +water_heater: + - platform: template + id: unknown_boiler + name: Unknown Boiler + # Both temperatures stay unknown, as they do before an upstream component reports a value. + current_temperature: !lambda "return NAN;" + target_temperature: !lambda "return NAN;" + supported_modes: + - "off" + - eco diff --git a/tests/integration/test_water_heater_template.py b/tests/integration/test_water_heater_template.py index d63d1d6984..3d7f885160 100644 --- a/tests/integration/test_water_heater_template.py +++ b/tests/integration/test_water_heater_template.py @@ -155,3 +155,36 @@ async def test_water_heater_template( client.water_heater_command(test_water_heater.key, mode=WaterHeaterMode.ECO) eco_state = await wait_for_state() assert eco_state.mode == WaterHeaterMode.ECO + + +@pytest.mark.asyncio +async def test_water_heater_template_unknown_temperature( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test a template water heater whose temperature lambdas stay unknown. + + NAN never compares equal to itself, so a lambda that keeps returning NAN must not be + mistaken for a changed value and republish the state on every loop iteration. + """ + async with run_compiled(yaml_config), api_client_connected() as client: + state_count = 0 + + def on_state(state: aioesphomeapi.EntityState) -> None: + nonlocal state_count + if isinstance(state, WaterHeaterState): + state_count += 1 + + entities, _ = await client.list_entities_services() + water_heater_infos = [e for e in entities if isinstance(e, WaterHeaterInfo)] + assert len(water_heater_infos) == 1 + + client.subscribe_states(on_state) + + # Let the device run for a while; only the single initial state may arrive. + await asyncio.sleep(1.0) + assert state_count <= 1, ( + f"Expected at most 1 state publish, got {state_count} - " + "an unknown (NAN) temperature is republishing every loop" + ) From 02648d3547401d6478149c9c9f322a5b99a467fa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 21:17:58 -0500 Subject: [PATCH 166/178] [bluetooth_connection] Keep USE_BLUETOOTH_PROXY out of the shared host test binary (#19271) --- tests/components/bluetooth_connection/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/components/bluetooth_connection/__init__.py b/tests/components/bluetooth_connection/__init__.py index 9c1ad4e74d..45bf77b4e8 100644 --- a/tests/components/bluetooth_connection/__init__.py +++ b/tests/components/bluetooth_connection/__init__.py @@ -6,15 +6,14 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # close_service_batch compiles only under USE_BLUETOOTH_PROXY_CONNECTIONS; # emit the backend define so the host build exercises it. async def to_code_testing(config): - # These defines are global to the merged host test binary; safe - # because no co-compiled test observes them. + # These defines are global to the merged host test binary. The api sources are + # compiled in it too (the api tests define USE_API), and USE_BLUETOOTH_PROXY would make + # them include and call bluetooth_proxy, which has no host build without a BLE hub. cg.add_define("USE_BLE_GATT_CLIENT") cg.add_define("USE_BLE_GATT_CLIENT_STUB_BACKEND") - cg.add_define("USE_BLUETOOTH_PROXY") # Gates the connection half of the API surface, which is what # close_service_batch and the GATT response types live behind. cg.add_define("USE_BLUETOOTH_PROXY_CONNECTIONS") - cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", 1) manifest.to_code = to_code_testing From 82b608706cd49a231017190de0bbb8120ccc2fbf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 21:17:58 -0500 Subject: [PATCH 167/178] [bluetooth_connection] Keep USE_BLUETOOTH_PROXY out of the shared host test binary (#19271) --- tests/components/bluetooth_connection/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/components/bluetooth_connection/__init__.py b/tests/components/bluetooth_connection/__init__.py index 9c1ad4e74d..45bf77b4e8 100644 --- a/tests/components/bluetooth_connection/__init__.py +++ b/tests/components/bluetooth_connection/__init__.py @@ -6,15 +6,14 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # close_service_batch compiles only under USE_BLUETOOTH_PROXY_CONNECTIONS; # emit the backend define so the host build exercises it. async def to_code_testing(config): - # These defines are global to the merged host test binary; safe - # because no co-compiled test observes them. + # These defines are global to the merged host test binary. The api sources are + # compiled in it too (the api tests define USE_API), and USE_BLUETOOTH_PROXY would make + # them include and call bluetooth_proxy, which has no host build without a BLE hub. cg.add_define("USE_BLE_GATT_CLIENT") cg.add_define("USE_BLE_GATT_CLIENT_STUB_BACKEND") - cg.add_define("USE_BLUETOOTH_PROXY") # Gates the connection half of the API surface, which is what # close_service_batch and the GATT response types live behind. cg.add_define("USE_BLUETOOTH_PROXY_CONNECTIONS") - cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", 1) manifest.to_code = to_code_testing From abadfbfd20eb16d9272ef225f160e55adad2824b Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:32:31 +1200 Subject: [PATCH 168/178] [core] Mark filters, manual_ip and interlock as advanced (#19272) --- esphome/components/binary_sensor/__init__.py | 4 +- esphome/components/ethernet/__init__.py | 4 +- esphome/components/gpio/switch/__init__.py | 8 ++- esphome/components/sensor/__init__.py | 4 +- esphome/components/text_sensor/__init__.py | 4 +- esphome/components/wifi/__init__.py | 8 ++- .../test_advanced_visibility.py | 53 +++++++++++++++++++ 7 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 tests/component_tests/config_validation/test_advanced_visibility.py diff --git a/esphome/components/binary_sensor/__init__.py b/esphome/components/binary_sensor/__init__.py index 1ab6f7103f..9ef7efc96a 100644 --- a/esphome/components/binary_sensor/__init__.py +++ b/esphome/components/binary_sensor/__init__.py @@ -452,7 +452,9 @@ _BINARY_SENSOR_SCHEMA = ( cv.Optional( CONF_DEVICE_CLASS, visibility=cv.Visibility.ADVANCED ): validate_device_class, - cv.Optional(CONF_FILTERS): validate_filters, + cv.Optional( + CONF_FILTERS, visibility=cv.Visibility.ADVANCED + ): validate_filters, cv.Optional(CONF_ON_PRESS): automation.validate_automation({}), cv.Optional(CONF_ON_RELEASE): automation.validate_automation({}), cv.Optional(CONF_ON_CLICK): cv.All( diff --git a/esphome/components/ethernet/__init__.py b/esphome/components/ethernet/__init__.py index 0454440f14..3e7d345805 100644 --- a/esphome/components/ethernet/__init__.py +++ b/esphome/components/ethernet/__init__.py @@ -420,7 +420,9 @@ def _validate(config: ConfigType) -> ConfigType: BASE_SCHEMA = cv.Schema( { cv.GenerateID(): cv.declare_id(EthernetComponent), - cv.Optional(CONF_MANUAL_IP): MANUAL_IP_SCHEMA, + cv.Optional( + CONF_MANUAL_IP, visibility=cv.Visibility.ADVANCED + ): MANUAL_IP_SCHEMA, cv.Optional(CONF_DOMAIN, default=".local"): cv.domain_name, cv.Optional(CONF_USE_ADDRESS): cv.string_strict, cv.Optional(CONF_MAC_ADDRESS): cv.mac_address, diff --git a/esphome/components/gpio/switch/__init__.py b/esphome/components/gpio/switch/__init__.py index 2e0b0969bc..766cdc4afb 100644 --- a/esphome/components/gpio/switch/__init__.py +++ b/esphome/components/gpio/switch/__init__.py @@ -15,9 +15,13 @@ CONFIG_SCHEMA = ( .extend( { cv.Required(CONF_PIN): pins.gpio_output_pin_schema, - cv.Optional(CONF_INTERLOCK): cv.ensure_list(cv.use_id(switch.Switch)), cv.Optional( - CONF_INTERLOCK_WAIT_TIME, default="0ms" + CONF_INTERLOCK, visibility=cv.Visibility.ADVANCED + ): cv.ensure_list(cv.use_id(switch.Switch)), + cv.Optional( + CONF_INTERLOCK_WAIT_TIME, + default="0ms", + visibility=cv.Visibility.ADVANCED, ): cv.positive_time_period_milliseconds, } ) diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 79d4ce5e0c..3b632a1847 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -344,7 +344,9 @@ _SENSOR_SCHEMA = ( cv.requires_component("mqtt"), cv.Any(None, cv.positive_time_period_milliseconds), ), - cv.Optional(CONF_FILTERS): validate_filters, + cv.Optional( + CONF_FILTERS, visibility=cv.Visibility.ADVANCED + ): validate_filters, cv.Optional(CONF_ON_VALUE): automation.validate_automation({}), cv.Optional(CONF_ON_RAW_VALUE): automation.validate_automation({}), cv.Optional(CONF_ON_VALUE_RANGE): automation.validate_automation( diff --git a/esphome/components/text_sensor/__init__.py b/esphome/components/text_sensor/__init__.py index 29399a51b7..5c8d71696f 100644 --- a/esphome/components/text_sensor/__init__.py +++ b/esphome/components/text_sensor/__init__.py @@ -148,7 +148,9 @@ _TEXT_SENSOR_SCHEMA = ( cv.Optional( CONF_DEVICE_CLASS, visibility=cv.Visibility.ADVANCED ): validate_device_class, - cv.Optional(CONF_FILTERS): validate_filters, + cv.Optional( + CONF_FILTERS, visibility=cv.Visibility.ADVANCED + ): validate_filters, cv.Optional(CONF_ON_VALUE): automation.validate_automation({}), cv.Optional(CONF_ON_RAW_VALUE): automation.validate_automation({}), } diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 1e57c03b7b..95f627596d 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -288,7 +288,9 @@ WIFI_NETWORK_BASE = cv.Schema( cv.Optional(CONF_SSID): cv.sensitive(cv.ssid), cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), cv.Optional(CONF_CHANNEL): validate_channel, - cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, + cv.Optional( + CONF_MANUAL_IP, visibility=cv.Visibility.ADVANCED + ): STA_MANUAL_IP_SCHEMA, } ) @@ -487,7 +489,9 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_SSID): cv.sensitive(cv.ssid), cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), - cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, + cv.Optional( + CONF_MANUAL_IP, visibility=cv.Visibility.ADVANCED + ): STA_MANUAL_IP_SCHEMA, cv.Optional(CONF_EAP): EAP_AUTH_SCHEMA, cv.Optional(CONF_AP): wifi_network_ap, cv.Optional(CONF_DOMAIN, default=".local"): cv.domain_name, diff --git a/tests/component_tests/config_validation/test_advanced_visibility.py b/tests/component_tests/config_validation/test_advanced_visibility.py new file mode 100644 index 0000000000..f7e0374319 --- /dev/null +++ b/tests/component_tests/config_validation/test_advanced_visibility.py @@ -0,0 +1,53 @@ +"""Power-user fields are marked as advanced on the shared schemas. + +``filters``, ``manual_ip`` and the GPIO switch interlock options are knobs +whose defaults suit nearly every user, so a schema-aware editor should keep +them behind its "advanced settings" disclosure rather than on the main form. +""" + +from __future__ import annotations + +import importlib + +import pytest + +from esphome.components import binary_sensor, ethernet, sensor, text_sensor, wifi +import esphome.config_validation as cv + + +def _markers(schema: cv.Schema) -> dict[str, object]: + s = schema + if hasattr(s, "validators"): + # cv.All -> the schema is the first validator. + s = s.validators[0] + return {str(k): k for k in s.schema} + + +def _gpio_switch_schema() -> cv.Schema: + return importlib.import_module("esphome.components.gpio.switch").CONFIG_SCHEMA + + +@pytest.mark.parametrize( + ("label", "schema_factory", "fields"), + [ + ("sensor", sensor.sensor_schema, ["filters"]), + ("binary_sensor", binary_sensor.binary_sensor_schema, ["filters"]), + ("text_sensor", text_sensor.text_sensor_schema, ["filters"]), + ("wifi_network", lambda: wifi.WIFI_NETWORK_BASE, ["manual_ip"]), + ("wifi", lambda: wifi.CONFIG_SCHEMA, ["manual_ip"]), + ("ethernet", lambda: ethernet.BASE_SCHEMA, ["manual_ip"]), + ("gpio_switch", _gpio_switch_schema, ["interlock", "interlock_wait_time"]), + ], +) +def test_power_user_fields_are_advanced( + label: str, schema_factory, fields: list[str] +) -> None: + markers = _markers(schema_factory()) + for field in fields: + assert markers[field].visibility is cv.Visibility.ADVANCED, f"{label}.{field}" + + +def test_interlock_wait_time_keeps_its_default() -> None: + """Marking the field advanced must not drop its default.""" + markers = _markers(_gpio_switch_schema()) + assert markers["interlock_wait_time"].default() == "0ms" From 282be54d1eb4c4a1ab00288229c247a06ce7975f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 23:44:36 -0500 Subject: [PATCH 169/178] [number] Fix the default mode check so mode auto is no longer emitted (#19231) --- esphome/components/number/__init__.py | 14 ++++++---- esphome/components/number/number_traits.h | 2 +- tests/component_tests/number/__init__.py | 0 tests/component_tests/number/config/mode.yaml | 28 +++++++++++++++++++ tests/component_tests/number/test_number.py | 16 +++++++++++ 5 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 tests/component_tests/number/__init__.py create mode 100644 tests/component_tests/number/config/mode.yaml create mode 100644 tests/component_tests/number/test_number.py diff --git a/esphome/components/number/__init__.py b/esphome/components/number/__init__.py index ea0c2d77f6..fc0893323b 100644 --- a/esphome/components/number/__init__.py +++ b/esphome/components/number/__init__.py @@ -174,6 +174,10 @@ NumberInRangeCondition = number_ns.class_( NumberMode = number_ns.enum("NumberMode") +# Schema default that also matches the C++ initializer in number_traits.h; codegen +# skips the setter when the config equals it. +DEFAULT_MODE = "AUTO" + NUMBER_MODES = { "AUTO": NumberMode.NUMBER_MODE_AUTO, "BOX": NumberMode.NUMBER_MODE_BOX, @@ -216,7 +220,7 @@ _NUMBER_SCHEMA = ( CONF_UNIT_OF_MEASUREMENT, visibility=cv.Visibility.ADVANCED ): validate_unit_of_measurement, cv.Optional( - CONF_MODE, default="AUTO", visibility=cv.Visibility.ADVANCED + CONF_MODE, default=DEFAULT_MODE, visibility=cv.Visibility.ADVANCED ): cv.enum(NUMBER_MODES, upper=True), cv.Optional( CONF_DEVICE_CLASS, visibility=cv.Visibility.ADVANCED @@ -286,10 +290,10 @@ async def setup_number_core_( cg.add(var.traits.set_max_value(max_value)) cg.add(var.traits.set_step(step)) - # Only set if non-default to avoid bloating setup() function - # (mode_ is initialized to NUMBER_MODE_AUTO in the header) - if config[CONF_MODE] != NumberMode.NUMBER_MODE_AUTO: - cg.add(var.traits.set_mode(config[CONF_MODE])) + # Skip the setter when the config matches the C++ initializer (DEFAULT_MODE). + # The validated value is the enum key string, not the C++ enum expression. + if (mode := config[CONF_MODE]) != DEFAULT_MODE: + cg.add(var.traits.set_mode(mode)) CORE.add_job(_build_number_automations, var, config) diff --git a/esphome/components/number/number_traits.h b/esphome/components/number/number_traits.h index f855813c9b..3c7942b9a3 100644 --- a/esphome/components/number/number_traits.h +++ b/esphome/components/number/number_traits.h @@ -31,7 +31,7 @@ class NumberTraits { float min_value_ = NAN; float max_value_ = NAN; float step_ = NAN; - NumberMode mode_{NUMBER_MODE_AUTO}; + NumberMode mode_{NUMBER_MODE_AUTO}; // Keep in sync with DEFAULT_MODE in __init__.py }; } // namespace esphome::number diff --git a/tests/component_tests/number/__init__.py b/tests/component_tests/number/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/number/config/mode.yaml b/tests/component_tests/number/config/mode.yaml new file mode 100644 index 0000000000..b3eae34436 --- /dev/null +++ b/tests/component_tests/number/config/mode.yaml @@ -0,0 +1,28 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +number: + - platform: template + id: auto_number + min_value: 0 + max_value: 10 + step: 1 + optimistic: true + - platform: template + id: box_number + min_value: 0 + max_value: 10 + step: 1 + mode: box + optimistic: true + - platform: template + id: explicit_auto_number + min_value: 0 + max_value: 10 + step: 1 + mode: auto + optimistic: true diff --git a/tests/component_tests/number/test_number.py b/tests/component_tests/number/test_number.py new file mode 100644 index 0000000000..b33508602a --- /dev/null +++ b/tests/component_tests/number/test_number.py @@ -0,0 +1,16 @@ +"""Tests for the number component codegen.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_default_mode_is_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Mode auto is the C++ initializer, so only a non default mode is set.""" + main_cpp = generate_main(component_config_path("mode.yaml")) + + assert "auto_number->traits.set_mode(" not in main_cpp + assert "explicit_auto_number->traits.set_mode(" not in main_cpp + assert "box_number->traits.set_mode(number::NUMBER_MODE_BOX);" in main_cpp From 1e22861d11ddcd27096239b2d7d4ea130b83883c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 23:50:00 -0500 Subject: [PATCH 170/178] [web_server] Skip setters that pass the default port, log and include internal values (#19226) --- esphome/components/web_server/__init__.py | 20 ++++++++--- .../web_server_base/web_server_base.h | 2 +- .../web_server/config/bare.yaml | 12 +++++++ .../web_server/config/custom.yaml | 15 ++++++++ .../web_server/config/defaults.yaml | 15 ++++++++ .../web_server/test_default_setters.py | 35 +++++++++++++++++++ 6 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 tests/component_tests/web_server/config/bare.yaml create mode 100644 tests/component_tests/web_server/config/custom.yaml create mode 100644 tests/component_tests/web_server/config/defaults.yaml create mode 100644 tests/component_tests/web_server/test_default_setters.py diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index a50c14a2f7..2459163786 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -56,6 +56,10 @@ CONF_SORTING_GROUPS = "sorting_groups" CONF_SORTING_WEIGHT = "sorting_weight" CONF_ALLOWED_ORIGINS = "allowed_origins" +# Schema default that also matches the C++ initializer in web_server_base.h; codegen +# skips the setter when the config equals it. +DEFAULT_PORT = 80 + web_server_ns = cg.esphome_ns.namespace("web_server") WebServer = web_server_ns.class_("WebServer", cg.Component, cg.Controller) @@ -251,7 +255,7 @@ CONFIG_SCHEMA = cv.All( cv.Schema( { cv.GenerateID(): cv.declare_id(WebServer), - cv.Optional(CONF_PORT, default=80): cv.port, + cv.Optional(CONF_PORT, default=DEFAULT_PORT): cv.port, cv.Optional(CONF_VERSION, default=2): cv.one_of(1, 2, 3, int=True), cv.Optional(CONF_CSS_URL): cv.string, cv.Optional(CONF_CSS_INCLUDE): cv.file_, @@ -379,9 +383,11 @@ async def to_code(config: ConfigType) -> None: version = config[CONF_VERSION] - cg.add(paren.set_port(config[CONF_PORT])) + # Skip the setter when the config matches the C++ initializer (DEFAULT_PORT). + if (port := config[CONF_PORT]) != DEFAULT_PORT: + cg.add(paren.set_port(port)) cg.add_define("USE_WEBSERVER") - cg.add_define("USE_WEBSERVER_PORT", config[CONF_PORT]) + cg.add_define("USE_WEBSERVER_PORT", port) cg.add_define("USE_WEBSERVER_VERSION", version) if version >= 2: # Don't compress the index HTML as the data sizes are almost the same. @@ -395,9 +401,11 @@ async def to_code(config: ConfigType) -> None: # Captive portal will still be able to perform OTA updates even when this is set if config.get(CONF_OTA) is False: cg.add_define("USE_WEBSERVER_OTA_DISABLED") - cg.add(var.set_expose_log(config[CONF_LOG])) + # expose_log_ is true in C++; only emit the setter to turn it off. if config[CONF_LOG]: request_log_listener() # Request a log listener slot for web server log streaming + else: + cg.add(var.set_expose_log(False)) if config[CONF_ENABLE_PRIVATE_NETWORK_ACCESS]: cg.add_define("USE_WEBSERVER_PRIVATE_NETWORK_ACCESS") if (allowed_origins := config.get(CONF_ALLOWED_ORIGINS)) is not None: @@ -433,7 +441,9 @@ async def to_code(config: ConfigType) -> None: path = CORE.relative_config_path(config[CONF_JS_INCLUDE]) with path.open(encoding="utf-8") as js_file: add_resource_as_progmem("JS_INCLUDE", js_file.read()) - cg.add(var.set_include_internal(config[CONF_INCLUDE_INTERNAL])) + # include_internal_ is false in C++; only emit the setter to turn it on. + if config[CONF_INCLUDE_INTERNAL]: + cg.add(var.set_include_internal(True)) if CONF_LOCAL in config and config[CONF_LOCAL]: cg.add_define("USE_WEBSERVER_LOCAL") if config[CONF_COMPRESSION] == "gzip": diff --git a/esphome/components/web_server_base/web_server_base.h b/esphome/components/web_server_base/web_server_base.h index 94579de70f..72d3bf75b1 100644 --- a/esphome/components/web_server_base/web_server_base.h +++ b/esphome/components/web_server_base/web_server_base.h @@ -170,7 +170,7 @@ class WebServerBase final { protected: uint8_t initialized_{0}; - uint16_t port_{80}; + uint16_t port_{80}; // Keep in sync with DEFAULT_PORT in web_server/__init__.py AsyncWebServer *server_{nullptr}; std::vector handlers_; #ifdef USE_WEBSERVER_AUTH diff --git a/tests/component_tests/web_server/config/bare.yaml b/tests/component_tests/web_server/config/bare.yaml new file mode 100644 index 0000000000..dae1c48883 --- /dev/null +++ b/tests/component_tests/web_server/config/bare.yaml @@ -0,0 +1,12 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +wifi: + ssid: test + password: testtest + +web_server: diff --git a/tests/component_tests/web_server/config/custom.yaml b/tests/component_tests/web_server/config/custom.yaml new file mode 100644 index 0000000000..2d37d7ae19 --- /dev/null +++ b/tests/component_tests/web_server/config/custom.yaml @@ -0,0 +1,15 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +wifi: + ssid: test + password: testtest + +web_server: + port: 8080 + log: false + include_internal: true diff --git a/tests/component_tests/web_server/config/defaults.yaml b/tests/component_tests/web_server/config/defaults.yaml new file mode 100644 index 0000000000..3c34da43ac --- /dev/null +++ b/tests/component_tests/web_server/config/defaults.yaml @@ -0,0 +1,15 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +wifi: + ssid: test + password: testtest + +web_server: + port: 80 + log: true + include_internal: false diff --git a/tests/component_tests/web_server/test_default_setters.py b/tests/component_tests/web_server/test_default_setters.py new file mode 100644 index 0000000000..2b13ed966b --- /dev/null +++ b/tests/component_tests/web_server/test_default_setters.py @@ -0,0 +1,35 @@ +"""Tests that web_server only emits setters for non default values.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize("config_file", ["bare.yaml", "defaults.yaml"]) +def test_default_values_are_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, +) -> None: + """Port 80, log on and include_internal off already live in the C++ initializers. + + Both the schema defaults and the same values written explicitly take the skip path. + """ + main_cpp = generate_main(component_config_path(config_file)) + + assert "set_port(" not in main_cpp + assert "set_expose_log(" not in main_cpp + assert "set_include_internal(" not in main_cpp + + +def test_custom_values_are_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Non default values still reach the C++ setters.""" + main_cpp = generate_main(component_config_path("custom.yaml")) + + assert "set_port(8080);" in main_cpp + assert "set_expose_log(false);" in main_cpp + assert "set_include_internal(true);" in main_cpp From 4067f572cc953f5f98d1f2770222a4245ab21dcc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 23:50:56 -0500 Subject: [PATCH 171/178] [output] Skip the power limit setters when they match the defaults (#19225) --- esphome/components/output/__init__.py | 13 +++++--- esphome/components/output/float_output.h | 1 + tests/component_tests/output/__init__.py | 0 .../config/ac_dimmer_min_power_zero.yaml | 13 ++++++++ .../output/config/power_limits.yaml | 18 +++++++++++ tests/component_tests/output/test_output.py | 31 +++++++++++++++++++ tests/components/ac_dimmer/common.yaml | 1 + 7 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/output/__init__.py create mode 100644 tests/component_tests/output/config/ac_dimmer_min_power_zero.yaml create mode 100644 tests/component_tests/output/config/power_limits.yaml create mode 100644 tests/component_tests/output/test_output.py diff --git a/esphome/components/output/__init__.py b/esphome/components/output/__init__.py index 4f6c8943f5..10d5e5eb59 100644 --- a/esphome/components/output/__init__.py +++ b/esphome/components/output/__init__.py @@ -53,12 +53,17 @@ async def setup_output_platform_(obj, config): if CONF_POWER_SUPPLY in config: power_supply_ = await cg.get_variable(config[CONF_POWER_SUPPLY]) cg.add(obj.set_power_supply(power_supply_)) - if CONF_MAX_POWER in config: + # The C++ initializers are max_power 1.0 and min_power 0.0; skip the setter when + # the config matches them. The define stays whenever the key is present because + # platforms such as ac_dimmer read the scaling fields directly. + if (max_power := config.get(CONF_MAX_POWER)) is not None: cg.add_define("USE_OUTPUT_FLOAT_POWER_SCALING") - cg.add(obj.set_max_power(config[CONF_MAX_POWER])) - if CONF_MIN_POWER in config: + if max_power != 1.0: + cg.add(obj.set_max_power(max_power)) + if (min_power := config.get(CONF_MIN_POWER)) is not None: cg.add_define("USE_OUTPUT_FLOAT_POWER_SCALING") - cg.add(obj.set_min_power(config[CONF_MIN_POWER])) + if min_power != 0.0: + cg.add(obj.set_min_power(min_power)) # Only emit when zero_means_zero is actually enabled. The schema defaults to False # so this key is always present; emitting unconditionally would force # USE_OUTPUT_FLOAT_POWER_SCALING on for every output, defeating the gate. diff --git a/esphome/components/output/float_output.h b/esphome/components/output/float_output.h index 673f423572..57c8c553f6 100644 --- a/esphome/components/output/float_output.h +++ b/esphome/components/output/float_output.h @@ -123,6 +123,7 @@ class FloatOutput : public BinaryOutput { virtual void write_state(float state) = 0; #ifdef USE_OUTPUT_FLOAT_POWER_SCALING + // Codegen skips the setters for these values; keep in sync with output/__init__.py float max_power_{1.0f}; float min_power_{0.0f}; bool zero_means_zero_{false}; diff --git a/tests/component_tests/output/__init__.py b/tests/component_tests/output/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/output/config/ac_dimmer_min_power_zero.yaml b/tests/component_tests/output/config/ac_dimmer_min_power_zero.yaml new file mode 100644 index 0000000000..84c5eafc5a --- /dev/null +++ b/tests/component_tests/output/config/ac_dimmer_min_power_zero.yaml @@ -0,0 +1,13 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +output: + - platform: ac_dimmer + id: dimmer + gate_pin: GPIO4 + zero_cross_pin: GPIO5 + min_power: 0% diff --git a/tests/component_tests/output/config/power_limits.yaml b/tests/component_tests/output/config/power_limits.yaml new file mode 100644 index 0000000000..682ae9de51 --- /dev/null +++ b/tests/component_tests/output/config/power_limits.yaml @@ -0,0 +1,18 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +output: + - platform: ledc + id: default_power + pin: GPIO4 + max_power: 100% + min_power: 0% + - platform: ledc + id: custom_power + pin: GPIO5 + max_power: 90% + min_power: 1% diff --git a/tests/component_tests/output/test_output.py b/tests/component_tests/output/test_output.py new file mode 100644 index 0000000000..172715aef0 --- /dev/null +++ b/tests/component_tests/output/test_output.py @@ -0,0 +1,31 @@ +"""Tests for the output platform codegen.""" + +from collections.abc import Callable +from pathlib import Path + +from esphome.core import CORE + + +def test_default_power_limits_are_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """max_power 100% and min_power 0% already live in the C++ initializers.""" + main_cpp = generate_main(component_config_path("power_limits.yaml")) + + assert "default_power->set_max_power(" not in main_cpp + assert "default_power->set_min_power(" not in main_cpp + assert "custom_power->set_max_power(0.9f);" in main_cpp + assert "custom_power->set_min_power(0.01f);" in main_cpp + assert "USE_OUTPUT_FLOAT_POWER_SCALING" in {d.name for d in CORE.defines} + + +def test_default_min_power_keeps_scaling_fields( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """ac_dimmer reads min_power_ directly, so the define must stay on for min_power 0%.""" + main_cpp = generate_main(component_config_path("ac_dimmer_min_power_zero.yaml")) + + assert "dimmer->set_min_power(" not in main_cpp + assert "USE_OUTPUT_FLOAT_POWER_SCALING" in {d.name for d in CORE.defines} diff --git a/tests/components/ac_dimmer/common.yaml b/tests/components/ac_dimmer/common.yaml index c16e2e834a..8fa62c0636 100644 --- a/tests/components/ac_dimmer/common.yaml +++ b/tests/components/ac_dimmer/common.yaml @@ -4,3 +4,4 @@ output: gate_pin: ${gate_pin} zero_cross_pin: ${zero_cross_pin} zero_cross_interrupt_type: ANY + min_power: 0% From 8109aa96f628711482501d2bd5de1406eb764bb8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 23:51:42 -0500 Subject: [PATCH 172/178] [light] Skip the flash transition setter and the empty effect list (#19228) --- esphome/components/light/__init__.py | 14 +++++++-- esphome/components/light/light_state.h | 2 +- .../light/config/transitions.yaml | 29 +++++++++++++++++++ .../light/test_default_setters.py | 19 ++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/light/config/transitions.yaml create mode 100644 tests/component_tests/light/test_default_setters.py diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index dbcc28d64a..ab9624c364 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -340,6 +340,10 @@ RESTORE_MODES = { "RESTORE_AND_ON": LightRestoreMode.LIGHT_RESTORE_AND_ON, } +# Schema default that also matches the C++ initializer in light_state.h; codegen +# skips the setter when the config equals it. +DEFAULT_FLASH_TRANSITION_LENGTH = "0s" + LIGHT_SCHEMA = ( cv.ENTITY_BASE_SCHEMA.extend(web_server.WEBSERVER_SORTING_SCHEMA) .extend(cv.MQTT_COMMAND_COMPONENT_SCHEMA) @@ -387,7 +391,7 @@ BRIGHTNESS_ONLY_LIGHT_SCHEMA = LIGHT_SCHEMA.extend( CONF_DEFAULT_TRANSITION_LENGTH, default="1s" ): cv.positive_time_period_milliseconds, cv.Optional( - CONF_FLASH_TRANSITION_LENGTH, default="0s" + CONF_FLASH_TRANSITION_LENGTH, default=DEFAULT_FLASH_TRANSITION_LENGTH ): cv.positive_time_period_milliseconds, cv.Optional(CONF_EFFECTS): validate_effects(MONOCHROMATIC_EFFECTS), } @@ -502,9 +506,12 @@ async def setup_light_core_(light_var, config, output_var): default_transition_length := config.get(CONF_DEFAULT_TRANSITION_LENGTH) ) is not None: cg.add(light_var.set_default_transition_length(default_transition_length)) + # Skip the setter when the config matches the C++ initializer. if ( flash_transition_length := config.get(CONF_FLASH_TRANSITION_LENGTH) - ) is not None: + ) is not None and flash_transition_length != cv.time_period( + DEFAULT_FLASH_TRANSITION_LENGTH + ): cg.add(light_var.set_flash_transition_length(flash_transition_length)) if (gamma_correct := config.get(CONF_GAMMA_CORRECT)) is not None: cg.add(light_var.set_gamma_correct(gamma_correct)) @@ -514,7 +521,8 @@ async def setup_light_core_(light_var, config, output_var): effects = await cg.build_registry_list( EFFECTS_REGISTRY, config.get(CONF_EFFECTS, []) ) - cg.add(light_var.add_effects(effects)) + if effects: + cg.add(light_var.add_effects(effects)) for conf in config.get(CONF_ON_TURN_ON, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], light_var) diff --git a/esphome/components/light/light_state.h b/esphome/components/light/light_state.h index 3a3f8fc368..eafa161f51 100644 --- a/esphome/components/light/light_state.h +++ b/esphome/components/light/light_state.h @@ -356,7 +356,7 @@ class LightState : public EntityBase, public Component { /// Default transition length for all transitions in ms. uint32_t default_transition_length_{}; /// Transition length to use for flash transitions. - uint32_t flash_transition_length_{}; + uint32_t flash_transition_length_{}; // Keep in sync with DEFAULT_FLASH_TRANSITION_LENGTH in __init__.py /// Gamma correction factor for the light. float gamma_correct_{}; #ifdef USE_LIGHT_GAMMA_LUT diff --git a/tests/component_tests/light/config/transitions.yaml b/tests/component_tests/light/config/transitions.yaml new file mode 100644 index 0000000000..ecb33b0ea8 --- /dev/null +++ b/tests/component_tests/light/config/transitions.yaml @@ -0,0 +1,29 @@ +--- +esphome: + name: test + +esp32: + board: esp32dev + +output: + - platform: ledc + id: out_a + pin: GPIO4 + - platform: ledc + id: out_b + pin: GPIO5 + +light: + - platform: monochromatic + id: plain_light + output: out_a + flash_transition_length: 0s + - platform: monochromatic + id: fancy_light + output: out_b + flash_transition_length: 500ms + effects: + - pulse: + - platform: monochromatic + id: bare_light + output: out_a diff --git a/tests/component_tests/light/test_default_setters.py b/tests/component_tests/light/test_default_setters.py new file mode 100644 index 0000000000..a4fc24a7cb --- /dev/null +++ b/tests/component_tests/light/test_default_setters.py @@ -0,0 +1,19 @@ +"""Tests that light codegen skips setters for default values.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_default_flash_length_and_empty_effects_are_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A 0 ms flash transition and an empty effect list match the C++ defaults.""" + main_cpp = generate_main(component_config_path("transitions.yaml")) + + assert "plain_light->set_flash_transition_length(" not in main_cpp + assert "plain_light->add_effects(" not in main_cpp + assert "bare_light->set_flash_transition_length(" not in main_cpp + assert "bare_light->add_effects(" not in main_cpp + assert "fancy_light->set_flash_transition_length(500);" in main_cpp + assert "fancy_light->add_effects({" in main_cpp From cba4f5bc05dfad191c27131a7b731caf7a1df53a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 23:52:57 -0500 Subject: [PATCH 173/178] [wifi] Skip setters that pass the default priority, timeouts, power save and auth mode (#19229) --- esphome/components/wifi/__init__.py | 28 +++++++++---- esphome/components/wifi/wifi_component.h | 4 +- tests/component_tests/wifi/__init__.py | 0 tests/component_tests/wifi/config/bare.yaml | 12 ++++++ tests/component_tests/wifi/config/custom.yaml | 18 +++++++++ .../component_tests/wifi/config/defaults.yaml | 18 +++++++++ .../wifi/test_default_setters.py | 39 +++++++++++++++++++ 7 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 tests/component_tests/wifi/__init__.py create mode 100644 tests/component_tests/wifi/config/bare.yaml create mode 100644 tests/component_tests/wifi/config/custom.yaml create mode 100644 tests/component_tests/wifi/config/defaults.yaml create mode 100644 tests/component_tests/wifi/test_default_setters.py diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 95f627596d..81b90766b9 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -169,6 +169,9 @@ MAX_WIFI_NETWORKS = 127 # get best-effort connection attempts. Longer timeout ensures we exhaust all options # before falling back to AP mode. Aligned with improv wifi_timeout default. DEFAULT_AP_TIMEOUT = "90s" +DEFAULT_REBOOT_TIMEOUT = "15min" +# Both defaults also match the C++ initializers in wifi_component.h; codegen skips +# the setter when the config equals them. wifi_ns = cg.esphome_ns.namespace("wifi") EAPAuth = wifi_ns.struct("EAPAuth") @@ -496,7 +499,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_AP): wifi_network_ap, cv.Optional(CONF_DOMAIN, default=".local"): cv.domain_name, cv.Optional( - CONF_REBOOT_TIMEOUT, default="15min" + CONF_REBOOT_TIMEOUT, default=DEFAULT_REBOOT_TIMEOUT ): cv.positive_time_period_milliseconds, cv.SplitDefault( CONF_POWER_SAVE_MODE, @@ -606,7 +609,8 @@ def wifi_network(config, ap, static_ip): cg.add(ap.set_channel(config[CONF_CHANNEL])) if static_ip is not None: cg.add(ap.set_manual_ip(manual_ip(static_ip))) - if CONF_PRIORITY in config: + # priority_ is 0 in C++; skip the setter when the config matches it. + if config.get(CONF_PRIORITY, 0) != 0: cg.add(ap.set_priority(config[CONF_PRIORITY])) return ap @@ -655,7 +659,9 @@ async def to_code(config): WiFiAP(), lambda ap: cg.add(var.set_ap(wifi_network(conf, ap, ip_config))), ) - cg.add(var.set_ap_timeout(conf[CONF_AP_TIMEOUT])) + # Skip the setter when the config matches the C++ initializer. + if (ap_timeout := conf[CONF_AP_TIMEOUT]) != cv.time_period(DEFAULT_AP_TIMEOUT): + cg.add(var.set_ap_timeout(ap_timeout)) cg.add_define("USE_WIFI_AP") # ESP32: register the WiFi stack with the esp32 sdkconfig reconciler, which @@ -677,10 +683,18 @@ async def to_code(config): if has_manual_ip: cg.add_define("USE_WIFI_MANUAL_IP") - cg.add(var.set_reboot_timeout(config[CONF_REBOOT_TIMEOUT])) - cg.add(var.set_power_save_mode(config[CONF_POWER_SAVE_MODE])) - if CONF_MIN_AUTH_MODE in config: - cg.add(var.set_min_auth_mode(config[CONF_MIN_AUTH_MODE])) + # The C++ initializers are DEFAULT_REBOOT_TIMEOUT, power save NONE and minimum + # auth WPA2; skip the setters when the config matches them. + if (reboot_timeout := config[CONF_REBOOT_TIMEOUT]) != cv.time_period( + DEFAULT_REBOOT_TIMEOUT + ): + cg.add(var.set_reboot_timeout(reboot_timeout)) + if (power_save_mode := config[CONF_POWER_SAVE_MODE]) != "NONE": + cg.add(var.set_power_save_mode(power_save_mode)) + if ( + min_auth_mode := config.get(CONF_MIN_AUTH_MODE) + ) is not None and min_auth_mode != "WPA2": + cg.add(var.set_min_auth_mode(min_auth_mode)) fast_connect = config[CONF_FAST_CONNECT] if fast_connect[CONF_ENABLED]: cg.add_define("USE_WIFI_FAST_CONNECT") diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 77a4773a27..a0983545fb 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -919,11 +919,11 @@ class WiFiComponent final : public Component { float output_power_{NAN}; uint32_t action_started_; uint32_t last_connected_{0}; - uint32_t reboot_timeout_{}; + uint32_t reboot_timeout_{900000}; // Keep in sync with DEFAULT_REBOOT_TIMEOUT in __init__.py uint32_t roaming_last_check_{0}; uint32_t roaming_scan_end_{0}; // Timestamp when last roaming scan completed #ifdef USE_WIFI_AP - uint32_t ap_timeout_{}; + uint32_t ap_timeout_{90000}; // Keep in sync with DEFAULT_AP_TIMEOUT in __init__.py #endif // 1-byte enums and integers diff --git a/tests/component_tests/wifi/__init__.py b/tests/component_tests/wifi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/wifi/config/bare.yaml b/tests/component_tests/wifi/config/bare.yaml new file mode 100644 index 0000000000..94e5de47a0 --- /dev/null +++ b/tests/component_tests/wifi/config/bare.yaml @@ -0,0 +1,12 @@ +--- +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: test + password: testtest + ap: + ssid: fallback diff --git a/tests/component_tests/wifi/config/custom.yaml b/tests/component_tests/wifi/config/custom.yaml new file mode 100644 index 0000000000..068479a540 --- /dev/null +++ b/tests/component_tests/wifi/config/custom.yaml @@ -0,0 +1,18 @@ +--- +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + networks: + - ssid: test + password: testtest + priority: 5 + ap: + ssid: fallback + ap_timeout: 2min + reboot_timeout: 0s + power_save_mode: light + min_auth_mode: wpa diff --git a/tests/component_tests/wifi/config/defaults.yaml b/tests/component_tests/wifi/config/defaults.yaml new file mode 100644 index 0000000000..1b5e7d7dba --- /dev/null +++ b/tests/component_tests/wifi/config/defaults.yaml @@ -0,0 +1,18 @@ +--- +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + networks: + - ssid: test + password: testtest + priority: 0 + ap: + ssid: fallback + ap_timeout: 90s + reboot_timeout: 15min + power_save_mode: none + min_auth_mode: wpa2 diff --git a/tests/component_tests/wifi/test_default_setters.py b/tests/component_tests/wifi/test_default_setters.py new file mode 100644 index 0000000000..b326f3eaee --- /dev/null +++ b/tests/component_tests/wifi/test_default_setters.py @@ -0,0 +1,39 @@ +"""Tests that wifi codegen skips setters for default values.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize("config_file", ["bare.yaml", "defaults.yaml"]) +def test_default_values_are_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, +) -> None: + """Priority 0, 90 s AP timeout, 15 min reboot, power save none, WPA2 are C++ defaults. + + Both the schema defaults and the same values written explicitly take the skip path. + """ + main_cpp = generate_main(component_config_path(config_file)) + + assert "set_priority(" not in main_cpp + assert "set_ap_timeout(" not in main_cpp + assert "set_reboot_timeout(" not in main_cpp + assert "set_power_save_mode(" not in main_cpp + assert "set_min_auth_mode(" not in main_cpp + + +def test_custom_values_are_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Non default values still reach the C++ setters.""" + main_cpp = generate_main(component_config_path("custom.yaml")) + + assert "set_priority(5);" in main_cpp + assert "set_ap_timeout(120000);" in main_cpp + assert "set_reboot_timeout(0);" in main_cpp + assert "set_power_save_mode(wifi::WIFI_POWER_SAVE_LIGHT);" in main_cpp + assert "set_min_auth_mode(wifi::WIFI_MIN_AUTH_MODE_WPA);" in main_cpp From c378ea13001066844f84b13fd8a86c016525536b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 13 Sep 2026 23:53:41 -0500 Subject: [PATCH 174/178] [logger] Skip the hardware UART setter when it matches the default (#19230) --- esphome/components/logger/__init__.py | 13 ++++---- esphome/components/logger/logger.h | 4 +-- tests/component_tests/logger/test_logger.py | 32 +++++++++++++++++++ .../logger/test_logger_libretiny_default.yaml | 8 +++++ .../logger/test_logger_libretiny_uart0.yaml | 9 ++++++ .../logger/test_logger_uart1.yaml | 9 ++++++ 6 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 tests/component_tests/logger/test_logger_libretiny_default.yaml create mode 100644 tests/component_tests/logger/test_logger_libretiny_uart0.yaml create mode 100644 tests/component_tests/logger/test_logger_uart1.yaml diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index 07b8b03084..138db75ad1 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -362,12 +362,13 @@ async def to_code(config: ConfigType) -> None: # pre_setup() switches on uart_ to decide which hardware to initialize # (e.g. UART0 vs USB_SERIAL_JTAG). Without this, uart_ is still the # default UART_SELECTION_UART0 and the wrong hardware gets initialized. - if CONF_HARDWARE_UART in config: - cg.add( - log.set_uart_selection( - HARDWARE_UART_TO_UART_SELECTION[config[CONF_HARDWARE_UART]] - ) - ) + # uart_ is UART0 in C++ except on LibreTiny where it is DEFAULT; skip the + # setter when the config matches it. + cpp_default_uart = DEFAULT if CORE.is_libretiny else UART0 + if ( + hardware_uart := config.get(CONF_HARDWARE_UART) + ) is not None and hardware_uart != cpp_default_uart: + cg.add(log.set_uart_selection(HARDWARE_UART_TO_UART_SELECTION[hardware_uart])) # pre_setup() sets global_logger and must run before any other code # that may call ESP_LOG* (e.g. setup_preferences contains ESP_LOGVV). cg.add(log.pre_setup()) diff --git a/esphome/components/logger/logger.h b/esphome/components/logger/logger.h index 9c26814f7e..ae55f4145a 100644 --- a/esphome/components/logger/logger.h +++ b/esphome/components/logger/logger.h @@ -352,10 +352,10 @@ class Logger final : public Component { // Group smaller types together at the end uint8_t current_level_{ESPHOME_LOG_LEVEL_VERY_VERBOSE}; #if defined(USE_ESP32) || defined(USE_ESP8266) || defined(USE_RP2) || defined(USE_ZEPHYR) - UARTSelection uart_{UART_SELECTION_UART0}; + UARTSelection uart_{UART_SELECTION_UART0}; // Must match cpp_default_uart in __init__.py #endif #ifdef USE_LIBRETINY - UARTSelection uart_{UART_SELECTION_DEFAULT}; + UARTSelection uart_{UART_SELECTION_DEFAULT}; // Must match cpp_default_uart in __init__.py #endif #if defined(USE_ESP32) || defined(USE_HOST) || defined(USE_LIBRETINY) || defined(USE_ZEPHYR) bool main_task_recursion_guard_{false}; diff --git a/tests/component_tests/logger/test_logger.py b/tests/component_tests/logger/test_logger.py index 94a6f7ac7b..4ce30afb94 100644 --- a/tests/component_tests/logger/test_logger.py +++ b/tests/component_tests/logger/test_logger.py @@ -52,3 +52,35 @@ def test_logger_pre_setup_before_other_components(generate_main): f"Component allocation '{alloc.group()}' at position {alloc.start()} " f"appears before logger pre_setup() at position {logger_pre_setup.start()}" ) + + +def test_default_uart_selection_is_not_emitted(generate_main): + """UART0 is the C++ initializer on ESP8266, so the setter is skipped.""" + main_cpp = generate_main("tests/component_tests/logger/test_logger.yaml") + + assert "set_uart_selection(" not in main_cpp + + +def test_custom_uart_selection_is_emitted(generate_main): + """A non default UART still reaches the setter before pre_setup().""" + main_cpp = generate_main("tests/component_tests/logger/test_logger_uart1.yaml") + + assert "set_uart_selection(logger::UART_SELECTION_UART1);" in main_cpp + + +def test_libretiny_default_uart_selection_is_not_emitted(generate_main): + """DEFAULT is the C++ initializer on LibreTiny, so the setter is skipped.""" + main_cpp = generate_main( + "tests/component_tests/logger/test_logger_libretiny_default.yaml" + ) + + assert "set_uart_selection(" not in main_cpp + + +def test_libretiny_uart0_is_emitted(generate_main): + """UART0 is not the LibreTiny initializer, so it must still be set.""" + main_cpp = generate_main( + "tests/component_tests/logger/test_logger_libretiny_uart0.yaml" + ) + + assert "set_uart_selection(logger::UART_SELECTION_UART0);" in main_cpp diff --git a/tests/component_tests/logger/test_logger_libretiny_default.yaml b/tests/component_tests/logger/test_logger_libretiny_default.yaml new file mode 100644 index 0000000000..1f11ea4580 --- /dev/null +++ b/tests/component_tests/logger/test_logger_libretiny_default.yaml @@ -0,0 +1,8 @@ +--- +esphome: + name: test + +rtl87xx: + board: generic-rtl8710bn-2mb-788k + +logger: diff --git a/tests/component_tests/logger/test_logger_libretiny_uart0.yaml b/tests/component_tests/logger/test_logger_libretiny_uart0.yaml new file mode 100644 index 0000000000..dc25fe99ce --- /dev/null +++ b/tests/component_tests/logger/test_logger_libretiny_uart0.yaml @@ -0,0 +1,9 @@ +--- +esphome: + name: test + +rtl87xx: + board: generic-rtl8710bn-2mb-788k + +logger: + hardware_uart: UART0 diff --git a/tests/component_tests/logger/test_logger_uart1.yaml b/tests/component_tests/logger/test_logger_uart1.yaml new file mode 100644 index 0000000000..ce45a6ae3f --- /dev/null +++ b/tests/component_tests/logger/test_logger_uart1.yaml @@ -0,0 +1,9 @@ +--- +esphome: + name: test + +esp8266: + board: d1_mini_lite + +logger: + hardware_uart: UART1 From e5eb577b49ed824d3fd1a82b633f52e94c51f7cd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 14 Sep 2026 01:05:27 -0500 Subject: [PATCH 175/178] [esp8266_pwm] Skip the frequency setter when it matches the default (#19224) --- esphome/components/esp8266_pwm/esp8266_pwm.h | 2 +- esphome/components/esp8266_pwm/output.py | 10 ++++++++-- tests/component_tests/esp8266_pwm/__init__.py | 0 .../esp8266_pwm/config/frequency.yaml | 19 +++++++++++++++++++ .../esp8266_pwm/test_esp8266_pwm.py | 16 ++++++++++++++++ 5 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/esp8266_pwm/__init__.py create mode 100644 tests/component_tests/esp8266_pwm/config/frequency.yaml create mode 100644 tests/component_tests/esp8266_pwm/test_esp8266_pwm.py diff --git a/esphome/components/esp8266_pwm/esp8266_pwm.h b/esphome/components/esp8266_pwm/esp8266_pwm.h index be58a098b6..79c2e50984 100644 --- a/esphome/components/esp8266_pwm/esp8266_pwm.h +++ b/esphome/components/esp8266_pwm/esp8266_pwm.h @@ -29,7 +29,7 @@ class ESP8266PWM final : public output::FloatOutput, public Component { void write_state(float state) override; InternalGPIOPin *pin_; - float frequency_{1000.0}; + float frequency_{1000.0}; // Keep in sync with DEFAULT_FREQUENCY in output.py /// Cache last output level for dynamic frequency updating float last_output_{0.0}; }; diff --git a/esphome/components/esp8266_pwm/output.py b/esphome/components/esp8266_pwm/output.py index dd151a3e04..be6e63b154 100644 --- a/esphome/components/esp8266_pwm/output.py +++ b/esphome/components/esp8266_pwm/output.py @@ -22,6 +22,10 @@ ESP8266PWM = esp8266_pwm_ns.class_("ESP8266PWM", output.FloatOutput, cg.Componen SetFrequencyAction = esp8266_pwm_ns.class_("SetFrequencyAction", automation.Action) validate_frequency = cv.All(cv.frequency, cv.float_range(min=1.0e-6)) +# Schema default that also matches the C++ initializer in esp8266_pwm.h; codegen +# skips the setter when the config equals it. +DEFAULT_FREQUENCY = 1000.0 + CONFIG_SCHEMA = cv.All( output.FLOAT_OUTPUT_SCHEMA.extend( { @@ -29,7 +33,7 @@ CONFIG_SCHEMA = cv.All( cv.Required(CONF_PIN): cv.All( pins.internal_gpio_output_pin_schema, valid_pwm_pin ), - cv.Optional(CONF_FREQUENCY, default="1kHz"): validate_frequency, + cv.Optional(CONF_FREQUENCY, default=DEFAULT_FREQUENCY): validate_frequency, } ).extend(cv.COMPONENT_SCHEMA), cv.require_framework_version( @@ -48,7 +52,9 @@ async def to_code(config: ConfigType) -> None: pin = await cg.gpio_pin_expression(config[CONF_PIN]) cg.add(var.set_pin(pin)) - cg.add(var.set_frequency(config[CONF_FREQUENCY])) + # Skip the setter when the config matches the C++ initializer (DEFAULT_FREQUENCY). + if (frequency := config[CONF_FREQUENCY]) != DEFAULT_FREQUENCY: + cg.add(var.set_frequency(frequency)) @automation.register_action( diff --git a/tests/component_tests/esp8266_pwm/__init__.py b/tests/component_tests/esp8266_pwm/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/esp8266_pwm/config/frequency.yaml b/tests/component_tests/esp8266_pwm/config/frequency.yaml new file mode 100644 index 0000000000..9ffc8af736 --- /dev/null +++ b/tests/component_tests/esp8266_pwm/config/frequency.yaml @@ -0,0 +1,19 @@ +--- +esphome: + name: test + +esp8266: + board: d1_mini + +output: + - platform: esp8266_pwm + id: default_frequency + pin: GPIO4 + frequency: 1kHz + - platform: esp8266_pwm + id: custom_frequency + pin: GPIO5 + frequency: 2kHz + - platform: esp8266_pwm + id: schema_default_frequency + pin: GPIO12 diff --git a/tests/component_tests/esp8266_pwm/test_esp8266_pwm.py b/tests/component_tests/esp8266_pwm/test_esp8266_pwm.py new file mode 100644 index 0000000000..771e513345 --- /dev/null +++ b/tests/component_tests/esp8266_pwm/test_esp8266_pwm.py @@ -0,0 +1,16 @@ +"""Tests for the esp8266_pwm output codegen.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_default_frequency_is_not_emitted( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The 1 kHz default already lives in the C++ initializer.""" + main_cpp = generate_main(component_config_path("frequency.yaml")) + + assert "default_frequency->set_frequency(" not in main_cpp + assert "schema_default_frequency->set_frequency(" not in main_cpp + assert "custom_frequency->set_frequency(2000.0f);" in main_cpp From 1e627a31f1027478206e15c3fe27d2334026e034 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:36:30 -0500 Subject: [PATCH 176/178] [ci] Refresh integration test durations (#19275) --- .../integration_test_durations.json | 305 +++++++++--------- 1 file changed, 153 insertions(+), 152 deletions(-) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json index b4a7f4e1ae..16748b4537 100644 --- a/tests/integration/integration_test_durations.json +++ b/tests/integration/integration_test_durations.json @@ -1,154 +1,155 @@ { - "tests/integration/test_action_concurrent_reentry.py": 30.48, - "tests/integration/test_addressable_light_transition.py": 42.1, - "tests/integration/test_alarm_control_panel_state_transitions.py": 35.76, - "tests/integration/test_api_action_metadata.py": 22.35, - "tests/integration/test_api_action_responses.py": 30.31, - "tests/integration/test_api_action_timeout.py": 34.73, - "tests/integration/test_api_conditional_memory.py": 18.35, - "tests/integration/test_api_custom_services.py": 15.99, - "tests/integration/test_api_get_time_response_timezone.py": 24.21, - "tests/integration/test_api_homeassistant.py": 33.77, - "tests/integration/test_api_homeassistant_action_no_subscriber.py": 20.8, - "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 23.55, - "tests/integration/test_api_list_entities_backpressure.py": 23.04, - "tests/integration/test_api_message_size_batching.py": 27.31, - "tests/integration/test_api_reboot_timeout.py": 29.32, - "tests/integration/test_api_string_lambda.py": 14.9, - "tests/integration/test_api_vv_logging.py": 26.25, - "tests/integration/test_api_zero_psk_provisioning.py": 38.19, - "tests/integration/test_areas_and_devices.py": 27.52, - "tests/integration/test_automation_wait_actions.py": 24.25, - "tests/integration/test_automations.py": 36.02, - "tests/integration/test_batch_delay_zero_rapid_transitions.py": 18.46, - "tests/integration/test_binary_sensor_autorepeat_filter.py": 17.47, - "tests/integration/test_binary_sensor_invalidate_state.py": 14.79, - "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 21.52, - "tests/integration/test_build_info.py": 21.42, - "tests/integration/test_camera_mock.py": 17.02, - "tests/integration/test_climate_control_action.py": 26.56, - "tests/integration/test_climate_custom_modes.py": 18.82, - "tests/integration/test_continuation_actions.py": 20.39, - "tests/integration/test_cover_control_action.py": 19.91, - "tests/integration/test_crc8_helper.py": 16.73, - "tests/integration/test_device_id_in_state.py": 58.41, - "tests/integration/test_duplicate_entities.py": 30.76, - "tests/integration/test_entity_icon.py": 25.34, - "tests/integration/test_fan_turn_on_action.py": 23.64, - "tests/integration/test_fnv1_hash_object_id.py": 25.44, - "tests/integration/test_fnv1a_hash.py": 20.85, - "tests/integration/test_gpio_expander_cache.py": 14.42, - "tests/integration/test_host_logger_thread_safety.py": 21.31, - "tests/integration/test_host_mode_basic.py": 2.65, - "tests/integration/test_host_mode_batch_delay.py": 22.21, - "tests/integration/test_host_mode_climate_basic_state.py": 27.12, - "tests/integration/test_host_mode_climate_control.py": 21.57, - "tests/integration/test_host_mode_empty_string_options.py": 27.17, - "tests/integration/test_host_mode_entity_fields.py": 30.1, - "tests/integration/test_host_mode_fan_preset.py": 17.55, - "tests/integration/test_host_mode_many_entities.py": 38.98, - "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.82, - "tests/integration/test_host_mode_noise_encryption.py": 39.84, - "tests/integration/test_host_mode_reconnect.py": 13.1, - "tests/integration/test_host_mode_sensor.py": 22.17, - "tests/integration/test_host_ota.py": 92.05, - "tests/integration/test_host_preferences.py": 20.29, - "tests/integration/test_host_preferences_suspend_resume.py": 15.02, - "tests/integration/test_improv_serial_uart.py": 30.15, - "tests/integration/test_large_message_batching.py": 25.84, - "tests/integration/test_legacy_area.py": 21.24, - "tests/integration/test_legacy_climate_compat.py": 17.34, - "tests/integration/test_legacy_fan_compat.py": 22.6, - "tests/integration/test_light_automations.py": 29.13, - "tests/integration/test_light_binary_effect_off_phase.py": 33.99, - "tests/integration/test_light_calls.py": 26.81, - "tests/integration/test_light_constant_brightness.py": 25.0, - "tests/integration/test_light_control_action.py": 25.57, - "tests/integration/test_light_dim_relative_action.py": 21.4, - "tests/integration/test_light_effect_zero_brightness.py": 19.65, - "tests/integration/test_light_initial_state.py": 17.58, - "tests/integration/test_light_toggle_action.py": 28.28, - "tests/integration/test_lock_automations.py": 23.3, - "tests/integration/test_logger_buffered_recursion_guard.py": 22.96, - "tests/integration/test_loop_disable_enable.py": 16.18, - "tests/integration/test_loop_interval_decoupling.py": 25.19, - "tests/integration/test_loop_interval_default_not_pulled_forward.py": 20.59, - "tests/integration/test_lvgl_headless_render.py": 87.78, - "tests/integration/test_micros_to_millis.py": 18.73, - "tests/integration/test_multi_click_trigger.py": 24.2, - "tests/integration/test_multi_device_preferences.py": 20.52, - "tests/integration/test_noise_encryption_key_protection.py": 19.1, - "tests/integration/test_object_id_api_verification.py": 26.24, - "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 14.88, - "tests/integration/test_object_id_no_friendly_name.py": 61.27, - "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 82.32, - "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 46.03, - "tests/integration/test_online_image_bmp.py": 34.21, - "tests/integration/test_oversized_payloads.py": 62.75, - "tests/integration/test_preference_key_stability.py": 26.8, - "tests/integration/test_runtime_stats.py": 28.26, - "tests/integration/test_safe_mode_loop_runs.py": 18.14, - "tests/integration/test_scheduler_blocking_warning.py": 28.7, - "tests/integration/test_scheduler_bulk_cleanup.py": 20.73, - "tests/integration/test_scheduler_defer_cancel.py": 22.99, - "tests/integration/test_scheduler_defer_cancel_regular.py": 21.97, - "tests/integration/test_scheduler_defer_fifo_simple.py": 24.15, - "tests/integration/test_scheduler_defer_stress.py": 23.11, - "tests/integration/test_scheduler_heap_stress.py": 20.2, - "tests/integration/test_scheduler_internal_id_no_collision.py": 23.75, - "tests/integration/test_scheduler_interval_reschedule.py": 15.32, - "tests/integration/test_scheduler_interval_zero_coerced.py": 20.1, - "tests/integration/test_scheduler_null_name.py": 17.43, - "tests/integration/test_scheduler_numeric_id_test.py": 25.51, - "tests/integration/test_scheduler_pool.py": 24.22, - "tests/integration/test_scheduler_rapid_cancellation.py": 24.01, - "tests/integration/test_scheduler_recursive_timeout.py": 22.94, - "tests/integration/test_scheduler_removed_item_race.py": 23.07, - "tests/integration/test_scheduler_self_keyed.py": 18.43, - "tests/integration/test_scheduler_simultaneous_callbacks.py": 21.99, - "tests/integration/test_scheduler_string_test.py": 17.27, - "tests/integration/test_script_array_params.py": 4.59, - "tests/integration/test_script_delay_params.py": 22.46, - "tests/integration/test_script_queued.py": 25.24, - "tests/integration/test_script_queued_idle_loop.py": 5.04, - "tests/integration/test_script_wait_on_boot.py": 21.77, - "tests/integration/test_sdl_headless_screenshot.py": 19.23, - "tests/integration/test_select_stringref_trigger.py": 19.31, - "tests/integration/test_sensor_filters_delta.py": 25.92, - "tests/integration/test_sensor_filters_ring_buffer.py": 22.39, - "tests/integration/test_sensor_filters_sliding_window.py": 57.93, - "tests/integration/test_sensor_filters_value_list.py": 20.32, - "tests/integration/test_sensor_timeout_filter.py": 25.35, - "tests/integration/test_snapshot_display.py": 19.7, - "tests/integration/test_socket_wake_gate_tcp.py": 14.5, - "tests/integration/test_status_flags.py": 33.83, - "tests/integration/test_strftime_to.py": 17.64, - "tests/integration/test_syslog.py": 24.49, - "tests/integration/test_template_alarm_control_panel_many_sensors.py": 24.81, - "tests/integration/test_template_climate_basic.py": 15.28, - "tests/integration/test_template_climate_custom_modes.py": 25.07, - "tests/integration/test_template_climate_nonoptimistic.py": 24.25, - "tests/integration/test_template_climate_on_control_ordering.py": 24.09, - "tests/integration/test_template_climate_publish_all_fields.py": 17.78, - "tests/integration/test_template_climate_sensor_push.py": 17.42, - "tests/integration/test_template_climate_set_actions.py": 23.63, - "tests/integration/test_template_climate_two_point_temperature.py": 25.19, - "tests/integration/test_template_text_save.py": 17.88, - "tests/integration/test_text_command.py": 22.71, - "tests/integration/test_text_sensor_raw_state.py": 25.17, - "tests/integration/test_uart_mock_ld2410.py": 58.15, - "tests/integration/test_uart_mock_ld2412.py": 61.14, - "tests/integration/test_uart_mock_ld2420.py": 33.87, - "tests/integration/test_uart_mock_ld2450.py": 26.06, - "tests/integration/test_uart_mock_modbus.py": 391.79, - "tests/integration/test_udp.py": 7.38, - "tests/integration/test_use_address_runtime.py": 24.09, - "tests/integration/test_valve_control_action.py": 23.22, - "tests/integration/test_varint_five_byte_device_id.py": 17.93, - "tests/integration/test_wait_until_mid_loop_timing.py": 22.26, - "tests/integration/test_wait_until_on_boot.py": 17.46, - "tests/integration/test_wait_until_ordering.py": 11.89, - "tests/integration/test_wait_until_reentrant_restart.py": 22.88, - "tests/integration/test_wake_loop_forces_phase_b.py": 16.6, - "tests/integration/test_water_heater_template.py": 19.66 + "tests/integration/test_action_concurrent_reentry.py": 34.72, + "tests/integration/test_addressable_light_transition.py": 33.71, + "tests/integration/test_alarm_control_panel_state_transitions.py": 38.96, + "tests/integration/test_api_action_metadata.py": 33.36, + "tests/integration/test_api_action_responses.py": 26.22, + "tests/integration/test_api_action_timeout.py": 25.41, + "tests/integration/test_api_conditional_memory.py": 20.74, + "tests/integration/test_api_custom_services.py": 23.21, + "tests/integration/test_api_get_time_response_timezone.py": 25.04, + "tests/integration/test_api_homeassistant.py": 24.18, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 23.47, + "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 22.86, + "tests/integration/test_api_list_entities_backpressure.py": 18.8, + "tests/integration/test_api_message_size_batching.py": 28.8, + "tests/integration/test_api_reboot_timeout.py": 9.47, + "tests/integration/test_api_string_lambda.py": 16.88, + "tests/integration/test_api_vv_logging.py": 17.99, + "tests/integration/test_api_zero_psk_provisioning.py": 47.07, + "tests/integration/test_areas_and_devices.py": 20.77, + "tests/integration/test_automation_wait_actions.py": 20.07, + "tests/integration/test_automations.py": 27.41, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 20.5, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 26.54, + "tests/integration/test_binary_sensor_invalidate_state.py": 16.09, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 14.7, + "tests/integration/test_build_info.py": 18.07, + "tests/integration/test_camera_mock.py": 20.44, + "tests/integration/test_climate_control_action.py": 27.97, + "tests/integration/test_climate_custom_modes.py": 26.77, + "tests/integration/test_continuation_actions.py": 12.09, + "tests/integration/test_cover_control_action.py": 19.77, + "tests/integration/test_crc8_helper.py": 12.64, + "tests/integration/test_device_id_in_state.py": 63.19, + "tests/integration/test_duplicate_entities.py": 29.26, + "tests/integration/test_entity_icon.py": 34.95, + "tests/integration/test_fan_turn_on_action.py": 25.98, + "tests/integration/test_fnv1_hash_object_id.py": 4.85, + "tests/integration/test_fnv1a_hash.py": 5.14, + "tests/integration/test_gpio_expander_cache.py": 21.0, + "tests/integration/test_host_logger_thread_safety.py": 17.51, + "tests/integration/test_host_mode_basic.py": 21.2, + "tests/integration/test_host_mode_batch_delay.py": 26.68, + "tests/integration/test_host_mode_climate_basic_state.py": 18.04, + "tests/integration/test_host_mode_climate_control.py": 29.64, + "tests/integration/test_host_mode_empty_string_options.py": 28.8, + "tests/integration/test_host_mode_entity_fields.py": 28.68, + "tests/integration/test_host_mode_fan_preset.py": 16.98, + "tests/integration/test_host_mode_many_entities.py": 39.8, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 33.12, + "tests/integration/test_host_mode_noise_encryption.py": 52.53, + "tests/integration/test_host_mode_reconnect.py": 14.48, + "tests/integration/test_host_mode_sensor.py": 17.38, + "tests/integration/test_host_ota.py": 94.96, + "tests/integration/test_host_preferences.py": 27.11, + "tests/integration/test_host_preferences_suspend_resume.py": 21.48, + "tests/integration/test_improv_serial_uart.py": 19.43, + "tests/integration/test_large_message_batching.py": 25.67, + "tests/integration/test_legacy_area.py": 15.59, + "tests/integration/test_legacy_climate_compat.py": 18.59, + "tests/integration/test_legacy_fan_compat.py": 18.34, + "tests/integration/test_light_automations.py": 16.74, + "tests/integration/test_light_binary_effect_off_phase.py": 57.7, + "tests/integration/test_light_calls.py": 25.88, + "tests/integration/test_light_constant_brightness.py": 22.32, + "tests/integration/test_light_control_action.py": 18.15, + "tests/integration/test_light_dim_relative_action.py": 30.35, + "tests/integration/test_light_effect_zero_brightness.py": 18.38, + "tests/integration/test_light_initial_state.py": 23.93, + "tests/integration/test_light_toggle_action.py": 26.16, + "tests/integration/test_lock_automations.py": 34.13, + "tests/integration/test_logger_buffered_recursion_guard.py": 25.77, + "tests/integration/test_loop_disable_enable.py": 14.71, + "tests/integration/test_loop_interval_decoupling.py": 26.16, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 28.47, + "tests/integration/test_lvgl_headless_render.py": 96.36, + "tests/integration/test_micros_to_millis.py": 28.76, + "tests/integration/test_multi_click_trigger.py": 19.8, + "tests/integration/test_multi_device_preferences.py": 38.85, + "tests/integration/test_noise_encryption_key_protection.py": 25.81, + "tests/integration/test_object_id_api_verification.py": 28.46, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 16.14, + "tests/integration/test_object_id_no_friendly_name.py": 18.82, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 30.16, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 40.67, + "tests/integration/test_online_image_bmp.py": 7.41, + "tests/integration/test_oversized_payloads.py": 59.52, + "tests/integration/test_preference_key_stability.py": 27.24, + "tests/integration/test_runtime_stats.py": 20.53, + "tests/integration/test_safe_mode_loop_runs.py": 10.17, + "tests/integration/test_scheduler_blocking_warning.py": 51.45, + "tests/integration/test_scheduler_bulk_cleanup.py": 22.59, + "tests/integration/test_scheduler_defer_cancel.py": 17.64, + "tests/integration/test_scheduler_defer_cancel_regular.py": 21.61, + "tests/integration/test_scheduler_defer_fifo_simple.py": 24.73, + "tests/integration/test_scheduler_defer_stress.py": 23.91, + "tests/integration/test_scheduler_heap_stress.py": 25.77, + "tests/integration/test_scheduler_internal_id_no_collision.py": 19.83, + "tests/integration/test_scheduler_interval_reschedule.py": 23.4, + "tests/integration/test_scheduler_interval_zero_coerced.py": 5.11, + "tests/integration/test_scheduler_null_name.py": 17.36, + "tests/integration/test_scheduler_numeric_id_test.py": 20.81, + "tests/integration/test_scheduler_pool.py": 17.42, + "tests/integration/test_scheduler_rapid_cancellation.py": 25.28, + "tests/integration/test_scheduler_recursive_timeout.py": 16.48, + "tests/integration/test_scheduler_removed_item_race.py": 16.14, + "tests/integration/test_scheduler_self_keyed.py": 26.19, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 23.26, + "tests/integration/test_scheduler_string_test.py": 27.09, + "tests/integration/test_script_array_params.py": 3.6, + "tests/integration/test_script_delay_params.py": 24.74, + "tests/integration/test_script_queued.py": 17.21, + "tests/integration/test_script_queued_idle_loop.py": 3.4, + "tests/integration/test_script_wait_on_boot.py": 23.7, + "tests/integration/test_sdl_headless_screenshot.py": 19.53, + "tests/integration/test_select_stringref_trigger.py": 18.93, + "tests/integration/test_sensor_filters_delta.py": 20.85, + "tests/integration/test_sensor_filters_ring_buffer.py": 16.82, + "tests/integration/test_sensor_filters_sliding_window.py": 54.78, + "tests/integration/test_sensor_filters_value_list.py": 19.46, + "tests/integration/test_sensor_timeout_filter.py": 18.39, + "tests/integration/test_set_internal_at_boot.py": 21.69, + "tests/integration/test_snapshot_display.py": 12.64, + "tests/integration/test_socket_wake_gate_tcp.py": 13.08, + "tests/integration/test_status_flags.py": 29.54, + "tests/integration/test_strftime_to.py": 25.62, + "tests/integration/test_syslog.py": 16.25, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 27.18, + "tests/integration/test_template_climate_basic.py": 20.51, + "tests/integration/test_template_climate_custom_modes.py": 27.73, + "tests/integration/test_template_climate_nonoptimistic.py": 26.35, + "tests/integration/test_template_climate_on_control_ordering.py": 26.55, + "tests/integration/test_template_climate_publish_all_fields.py": 17.59, + "tests/integration/test_template_climate_sensor_push.py": 22.04, + "tests/integration/test_template_climate_set_actions.py": 16.82, + "tests/integration/test_template_climate_two_point_temperature.py": 25.13, + "tests/integration/test_template_text_save.py": 25.36, + "tests/integration/test_text_command.py": 18.79, + "tests/integration/test_text_sensor_raw_state.py": 17.07, + "tests/integration/test_uart_mock_ld2410.py": 59.58, + "tests/integration/test_uart_mock_ld2412.py": 59.4, + "tests/integration/test_uart_mock_ld2420.py": 45.27, + "tests/integration/test_uart_mock_ld2450.py": 27.96, + "tests/integration/test_uart_mock_modbus.py": 562.45, + "tests/integration/test_udp.py": 7.48, + "tests/integration/test_use_address_runtime.py": 17.27, + "tests/integration/test_valve_control_action.py": 18.33, + "tests/integration/test_varint_five_byte_device_id.py": 17.59, + "tests/integration/test_wait_until_mid_loop_timing.py": 16.93, + "tests/integration/test_wait_until_on_boot.py": 19.96, + "tests/integration/test_wait_until_ordering.py": 16.19, + "tests/integration/test_wait_until_reentrant_restart.py": 23.67, + "tests/integration/test_wake_loop_forces_phase_b.py": 17.58, + "tests/integration/test_water_heater_template.py": 21.96 } From 651863323b0b67626c91b798dacf51c7e29ac8bd Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:40:00 +1000 Subject: [PATCH 177/178] [issues] Add AI usage guidance to the bug report template (#19122) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- .github/ISSUE_TEMPLATE/bug_report.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 44722ec85c..2244963a79 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -9,6 +9,29 @@ body: If you have a feature request or enhancement, please [request them here instead][fr]. [fr]: https://github.com/orgs/esphome/discussions + - type: markdown + attributes: + value: | + ## Use of AI in bug reports + + AI tools are good at carrying out well-defined tasks, but they are not good at troubleshooting. + Please do NOT paste an AI-generated wall of text into the issue template - if the AI hasn't solved + your problem, its wild guesses are not likely to help. + + Please DO include your own words and observations, compile/boot logs, and + especially a minimal reproducible example of your YAML configuration that demonstrates the problem. + + It is however quite acceptable to use AI to translate your *own* report, + if you aren't a competent English speaker. + + If you really think it will be useful to include an AI's analysis, preferably wrap it in a `
` block which will be collapsed by default. + + If you are using AI to help solve a problem, rather than asking it to speculate about what the problem is, + it can be more useful to ask it to create a step-by-step troubleshooting procedure. + AI is also useful for generating boilerplate code, such as a minimal reproducible example of your YAML + configuration that demonstrates the problem. + + Used properly, AI can be a useful tool to help you solve your problem, but don't let it get in the way. - type: textarea validations: required: true From 54b8e2e6dc7078657da75414fc8bd5b5967a8c59 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 14 Sep 2026 09:56:14 -0400 Subject: [PATCH 178/178] [audio] Update esp-audio-libs to 4.0.0 (#19300) --- esphome/components/audio/__init__.py | 5 +---- esphome/components/mixer/speaker/mixer_speaker.cpp | 8 ++++---- esphome/components/mixer/speaker/mixer_speaker.h | 4 ++-- esphome/idf_component.yml | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 2a5304be77..14a0818894 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -339,10 +339,7 @@ async def to_code(config: ConfigType) -> None: # HTTPS streams verify the server against the root certificate bundle require_certificate_bundle() - add_idf_component( - name="esphome/esp-audio-libs", - ref="3.2.1", - ) + add_idf_component(name="esphome/esp-audio-libs", ref="4.0.0") data = _get_data() diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index ef21da65c5..7d33b6c49f 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -306,9 +306,9 @@ size_t SourceSpeaker::process_data_from_source(std::shared_ptraudio_stream_info_.bytes_to_samples(bytes_read); if (samples_to_duck > 0) { - esp_audio_libs::ducking::apply(audio_source->mutable_data(), - static_cast(this->audio_stream_info_.get_bits_per_sample() / 8), - samples_to_duck, this->ducking_state_); + this->ducking_ramp_.process(audio_source->mutable_data(), + static_cast(this->audio_stream_info_.get_bits_per_sample() / 8), + samples_to_duck); } return bytes_read; @@ -316,7 +316,7 @@ size_t SourceSpeaker::process_data_from_source(std::shared_ptr 0 ? this->audio_stream_info_.ms_to_samples(duration) : 0; - esp_audio_libs::ducking::set_target(this->ducking_state_, decibel_reduction, transition_samples); + this->ducking_ramp_.set_target_db_reduction_over(decibel_reduction, transition_samples); } void SourceSpeaker::enter_stopping_state_() { diff --git a/esphome/components/mixer/speaker/mixer_speaker.h b/esphome/components/mixer/speaker/mixer_speaker.h index 00e89d1782..494443d695 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.h +++ b/esphome/components/mixer/speaker/mixer_speaker.h @@ -11,7 +11,7 @@ #include "esphome/core/helpers.h" #include "esphome/core/static_task.h" -#include // esp-audio-libs +#include // esp-audio-libs #include @@ -108,7 +108,7 @@ class SourceSpeaker final : public speaker::Speaker, public Component { bool pause_state_{false}; - esp_audio_libs::ducking::DuckingState ducking_state_{}; + esp_audio_libs::gain::GainRamp ducking_ramp_; std::atomic pending_playback_frames_{0}; std::atomic playback_delay_frames_{0}; // Frames in output pipeline when this source started contributing diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index e817a253d9..b3cd5ee09b 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -4,7 +4,7 @@ dependencies: esphome/dlms_parser: version: 1.1.0 esphome/esp-audio-libs: - version: 3.2.1 + version: 4.0.0 esphome/esp-micro-speech-features: version: 1.2.3 esphome/micro-decoder: