From be99553fd4aad7afdbb5d9a49037eb30656008d0 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 23 May 2026 13:56:53 +0930 Subject: [PATCH 01/96] [ci] Fix flash memory overflow on tests (#16587) --- tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml | 2 ++ .../build_components_base.esp32-c6-idf.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml index 6c27bd35d0..df5b0123b5 100644 --- a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml +++ b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml @@ -1,6 +1,8 @@ <<: !include common.yaml esp32_ble_tracker: + +esp32_ble: max_connections: 9 bluetooth_proxy: diff --git a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml index 9dbc465ca2..4105481dc5 100644 --- a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml +++ b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml @@ -4,6 +4,7 @@ esphome: esp32: board: esp32-c6-devkitc-1 + flash_size: 8MB framework: type: esp-idf From 188ff7ebfd70bc1f3fb417af5dbc41a486bcf99b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 23 May 2026 14:30:12 -0500 Subject: [PATCH 02/96] [bluetooth_proxy] Recover slot stuck in DISCONNECTING when CLOSE_EVT is dropped (#16588) --- .../bluetooth_proxy/bluetooth_connection.cpp | 26 ++++++++++++------- .../bluetooth_proxy/bluetooth_connection.h | 2 ++ .../esp32_ble_client/ble_client_base.cpp | 2 ++ .../esp32_ble_client/ble_client_base.h | 10 +++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 21573f0184..7ba9e61e19 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -135,12 +135,26 @@ void BluetoothConnection::loop() { // - For V3_WITH_CACHE: Services are never sent, disable after INIT state // - For V3_WITHOUT_CACHE: Disable only after service discovery is complete // (send_service_ == DONE_SENDING_SERVICES, which is only set after services are sent) - if (this->state() != espbt::ClientState::INIT && (this->connection_type_ == espbt::ConnectionType::V3_WITH_CACHE || - this->send_service_ == DONE_SENDING_SERVICES)) { + // Never disable while DISCONNECTING — BLEClientBase::loop() needs to keep running so the + // 10s safety timeout can force IDLE if CLOSE_EVT is never delivered. + if (this->state() != espbt::ClientState::INIT && this->state() != espbt::ClientState::DISCONNECTING && + (this->connection_type_ == espbt::ConnectionType::V3_WITH_CACHE || + this->send_service_ == DONE_SENDING_SERVICES)) { this->disable_loop(); } } +void BluetoothConnection::on_disconnect_complete(esp_err_t reason) { + // Called from both the CLOSE_EVT handler and the DISCONNECTING safety timeout in the + // base class. Free the proxy slot, notify the API client, and reset send_service_. + // address_ may already be 0 if reset_connection_ ran earlier on this teardown. + if (this->address_ == 0) { + return; + } + ESP_LOGD(TAG, "[%d] [%s] Close, reason=0x%02x, freeing slot", this->connection_index_, this->address_str_, reason); + this->reset_connection_(reason); +} + void BluetoothConnection::reset_connection_(esp_err_t reason) { // Send disconnection notification this->proxy_->send_device_connection(this->address_, false, 0, reason); @@ -372,14 +386,6 @@ bool BluetoothConnection::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga this->proxy_->send_device_connection(this->address_, false, 0, param->disconnect.reason); break; } - case ESP_GATTC_CLOSE_EVT: { - ESP_LOGD(TAG, "[%d] [%s] Close, reason=0x%02x, freeing slot", this->connection_index_, this->address_str_, - param->close.reason); - // Now the GATT connection is fully closed and controller resources are freed - // Safe to mark the connection slot as available - this->reset_connection_(param->close.reason); - break; - } case ESP_GATTC_OPEN_EVT: { if (param->open.status != ESP_GATT_OK && param->open.status != ESP_GATT_ALREADY_OPEN) { this->reset_connection_(param->open.status); diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.h b/esphome/components/bluetooth_proxy/bluetooth_connection.h index b50ea2d6a2..e5600f6af4 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.h +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.h @@ -33,6 +33,8 @@ class BluetoothConnection final : public esp32_ble_client::BLEClientBase { protected: friend class BluetoothProxy; + void on_disconnect_complete(esp_err_t reason) override; + bool supports_efficient_uuids_() const; void send_service_for_discovery_(); void reset_connection_(esp_err_t reason); diff --git a/esphome/components/esp32_ble_client/ble_client_base.cpp b/esphome/components/esp32_ble_client/ble_client_base.cpp index 7f0f2c624d..3fb9632e9a 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.cpp +++ b/esphome/components/esp32_ble_client/ble_client_base.cpp @@ -72,6 +72,7 @@ void BLEClientBase::loop() { // never delivered CLOSE_EVT/DISCONNECT_EVT, services would leak without this call. this->release_services(); this->set_idle_(); + this->on_disconnect_complete(ESP_GATT_CONN_TIMEOUT); } } @@ -418,6 +419,7 @@ bool BLEClientBase::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_ this->log_gattc_lifecycle_event_("CLOSE"); this->release_services(); this->set_idle_(); + this->on_disconnect_complete(param->close.reason); break; } case ESP_GATTC_SEARCH_RES_EVT: { diff --git a/esphome/components/esp32_ble_client/ble_client_base.h b/esphome/components/esp32_ble_client/ble_client_base.h index 4e0b22cc29..0291a4b993 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.h +++ b/esphome/components/esp32_ble_client/ble_client_base.h @@ -140,6 +140,12 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void log_gattc_warning_(const char *operation, esp_err_t err); void log_connection_params_(const char *param_type); void handle_connection_result_(esp_err_t ret); + /// Hook called once a connection has been fully torn down (after release_services() and + /// set_idle_()), from both the CLOSE_EVT handler and the DISCONNECTING safety timeout. + /// Subclasses with extra per-connection accounting (e.g. bluetooth_proxy slot state) + /// override this to release that state. `reason` is the controller reason code, or + /// ESP_GATT_CONN_TIMEOUT for the safety-timeout path. + virtual void on_disconnect_complete(esp_err_t reason) {} /// Transition to IDLE and reset conn_id — call when the connection is fully dead. void set_idle_() { this->set_state(espbt::ClientState::IDLE); @@ -149,6 +155,10 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void set_disconnecting_() { this->disconnecting_started_ = millis(); this->set_state(espbt::ClientState::DISCONNECTING); + // BluetoothConnection::loop() disables the component loop after service discovery + // completes, so the DISCONNECTING timeout check in loop() would never run if CLOSE_EVT + // gets lost. Re-enable the loop so the 10s safety timeout can force IDLE. + this->enable_loop(); } // Compact error logging helpers to reduce flash usage void log_error_(const char *message); From f61610362182d077af8f7e0e80ebca3fc3a77d87 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Sat, 23 May 2026 15:44:25 -0400 Subject: [PATCH 03/96] [esp32] Replace per-class -Wno-error=X demotes with blanket -Wno-error for ESP-IDF toolchain (#16599) --- esphome/components/esp32/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 24312d64ad..1864c3b544 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1816,12 +1816,9 @@ async def to_code(config): Path(__file__).parent / "iram_fix.py.script", ) else: - cg.add_build_flag("-Wno-error=format") - cg.add_build_flag("-Wno-error=maybe-uninitialized") - cg.add_build_flag("-Wno-error=overloaded-virtual") - cg.add_build_flag("-Wno-error=reorder") - cg.add_build_flag("-Wno-error=volatile") - cg.add_build_flag("-Wno-error=cpp") + # Undo IDF's blanket -Werror so third-party libraries and user + # lambdas don't need a -Wno-error= entry per warning class. + cg.add_build_flag("-Wno-error") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") From 58931f2610f65a5ca6c8c6204ed5253b18ebf378 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sat, 23 May 2026 17:37:59 -0400 Subject: [PATCH 04/96] [audio] Add `clear_buffered_data` method to RingBufferAudioSource (#16594) --- .../components/audio/audio_transfer_buffer.cpp | 16 ++++++++++++++++ esphome/components/audio/audio_transfer_buffer.h | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index d9ce8060e2..a611549e58 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -252,6 +252,22 @@ void RingBufferAudioSource::consume(size_t bytes) { } } +void RingBufferAudioSource::clear_buffered_data() { + // Release the held item before reset() so the source no longer references memory the reset will reclaim. + if (this->acquired_item_ != nullptr) { + this->ring_buffer_->receive_release(this->acquired_item_); + this->acquired_item_ = nullptr; + } + this->current_data_ = nullptr; + this->current_available_ = 0; + this->queued_data_ = nullptr; + this->queued_length_ = 0; + this->item_trailing_ptr_ = nullptr; + this->item_trailing_length_ = 0; + this->splice_length_ = 0; + this->ring_buffer_->reset(); +} + bool RingBufferAudioSource::has_buffered_data() const { // splice_length_ is deliberately not considered here. It holds an incomplete frame whose completion // bytes must still arrive through the ring buffer, which ring_buffer_->available() already reports. diff --git a/esphome/components/audio/audio_transfer_buffer.h b/esphome/components/audio/audio_transfer_buffer.h index b713326141..074684f068 100644 --- a/esphome/components/audio/audio_transfer_buffer.h +++ b/esphome/components/audio/audio_transfer_buffer.h @@ -250,6 +250,10 @@ class RingBufferAudioSource : public AudioReadableBuffer { /// exposure stays in place and fill() returns 0 until it is fully consumed. size_t fill(TickType_t ticks_to_wait, bool pre_shift) override; + /// @brief Discards all buffered audio: releases any held ring buffer item, clears the source's in-flight + /// state, and resets the underlying ring buffer. Must be invoked from the ring buffer's consumer thread. + void clear_buffered_data(); + /// @brief Returns a mutable pointer to the currently exposed audio data. /// The pointer may reference the ring buffer's internal storage or, when exposing a stitched frame /// across a wrap boundary, an internal splice buffer. In either case mutations are safe but data From 74001ccf05646c1a1c0dfe80df0724db822613e4 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sat, 23 May 2026 17:39:20 -0400 Subject: [PATCH 05/96] [wifi] Wake main loop when requesting high performance mode (#16598) --- esphome/components/wifi/wifi_component.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index edfb93bba2..72832d7ac8 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -2193,7 +2193,15 @@ bool WiFiComponent::request_high_performance() { } // Give the semaphore (non-blocking). This increments the count. - return xSemaphoreGive(this->high_performance_semaphore_) == pdTRUE; + bool success = xSemaphoreGive(this->high_performance_semaphore_) == pdTRUE; + + // Wake the main loop so the switch to high-performance mode is applied on the + // next tick instead of waiting up to loop_interval. + if (success) { + App.wake_loop_threadsafe(); + } + + return success; } bool WiFiComponent::release_high_performance() { From 5cb145a8c3869e568adbb4628d99d10181e0b473 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sat, 23 May 2026 17:53:53 -0400 Subject: [PATCH 06/96] [ethernet] Offload W5500 bulk SPI transfers from the busy-wait path (#16596) --- .../ethernet/ethernet_component_esp32.cpp | 5 + .../components/ethernet/w5500_custom_spi.cpp | 118 ++++++++++++++++++ .../components/ethernet/w5500_custom_spi.h | 35 ++++++ 3 files changed, 158 insertions(+) create mode 100644 esphome/components/ethernet/w5500_custom_spi.cpp create mode 100644 esphome/components/ethernet/w5500_custom_spi.h diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index d4585bf100..46e2bb4ec1 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -5,6 +5,7 @@ #include "esphome/core/application.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include "w5500_custom_spi.h" #include #include @@ -207,6 +208,10 @@ void EthernetComponent::setup() { #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT w5500_config.poll_period_ms = this->polling_interval_; #endif + // Install the custom SPI driver that offloads the bulk RX/TX frame transfers off the busy-wait + // path. w5500_config (and the devcfg it references) outlives esp_eth_mac_new_w5500() below, which + // runs the driver's init(). + install_w5500_async_spi(w5500_config); #elif defined(USE_ETHERNET_DM9051) dm9051_config.int_gpio_num = this->interrupt_pin_; #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT diff --git a/esphome/components/ethernet/w5500_custom_spi.cpp b/esphome/components/ethernet/w5500_custom_spi.cpp new file mode 100644 index 0000000000..ed4f149738 --- /dev/null +++ b/esphome/components/ethernet/w5500_custom_spi.cpp @@ -0,0 +1,118 @@ +#include "w5500_custom_spi.h" + +#if defined(USE_ESP32) && defined(USE_ETHERNET_W5500) + +#include +#include +#include +#include +#include + +namespace esphome::ethernet { + +namespace { + +// Per-device context returned by init() and handed back to read/write/deinit. +struct W5500CustomSpiContext { + spi_device_handle_t handle; + SemaphoreHandle_t lock; +}; + +// Transfers up to the ESP32 SPI hardware FIFO size (64 bytes) stay on the polling path; larger +// transfers (the frame payloads) use the blocking, DMA-backed transmit. +constexpr uint32_t W5500_SPI_BULK_THRESHOLD = 64; +constexpr uint32_t W5500_SPI_LOCK_TIMEOUT_MS = 50; + +void *w5500_custom_spi_init(const void *spi_config) { + const auto *config = static_cast(spi_config); + auto *ctx = new (std::nothrow) W5500CustomSpiContext{}; + if (ctx == nullptr) { + return nullptr; + } + // The W5500 SPI frame carries the 16-bit address in the command phase and the 8-bit control + // byte in the address phase; mirror what the stock driver configures. + spi_device_interface_config_t devcfg = *config->spi_devcfg; + devcfg.command_bits = 16; + devcfg.address_bits = 8; + if (spi_bus_add_device(config->spi_host_id, &devcfg, &ctx->handle) != ESP_OK) { + delete ctx; + return nullptr; + } + ctx->lock = xSemaphoreCreateMutex(); + if (ctx->lock == nullptr) { + spi_bus_remove_device(ctx->handle); + delete ctx; + return nullptr; + } + return ctx; +} + +esp_err_t w5500_custom_spi_deinit(void *spi_ctx) { + auto *ctx = static_cast(spi_ctx); + spi_bus_remove_device(ctx->handle); + vSemaphoreDelete(ctx->lock); + delete ctx; + return ESP_OK; +} + +// Runs one transaction under the device lock, choosing the polling vs blocking transmit by size. +// Bulk payloads (> FIFO size) block so the calling task sleeps while DMA runs; small register +// accesses stay on the cheaper polling path. Used by both read and write. +esp_err_t w5500_custom_spi_transfer(W5500CustomSpiContext *ctx, spi_transaction_t *trans, uint32_t len) { + if (xSemaphoreTake(ctx->lock, pdMS_TO_TICKS(W5500_SPI_LOCK_TIMEOUT_MS)) != pdTRUE) { + return ESP_ERR_TIMEOUT; + } + esp_err_t ret; + if (len > W5500_SPI_BULK_THRESHOLD) { + ret = spi_device_transmit(ctx->handle, trans); + } else { + ret = spi_device_polling_transmit(ctx->handle, trans); + } + xSemaphoreGive(ctx->lock); + return ret; +} + +esp_err_t w5500_custom_spi_write(void *spi_ctx, uint32_t cmd, uint32_t addr, const void *data, uint32_t len) { + auto *ctx = static_cast(spi_ctx); + spi_transaction_t trans = {}; + trans.cmd = static_cast(cmd); + trans.addr = addr; + trans.length = 8 * len; + trans.tx_buffer = data; + return w5500_custom_spi_transfer(ctx, &trans, len); +} + +esp_err_t w5500_custom_spi_read(void *spi_ctx, uint32_t cmd, uint32_t addr, void *data, uint32_t len) { + auto *ctx = static_cast(spi_ctx); + spi_transaction_t trans = {}; + // Reads of <= 4 bytes use the transaction's inline RX buffer to avoid 4-byte boundary + // overwrites of adjacent registers (same guard the stock driver uses). + const bool use_rxdata = len <= 4; + trans.flags = use_rxdata ? SPI_TRANS_USE_RXDATA : 0; + trans.cmd = static_cast(cmd); + trans.addr = addr; + trans.length = 8 * len; + trans.rx_buffer = data; + esp_err_t ret = w5500_custom_spi_transfer(ctx, &trans, len); + if (use_rxdata && (ret == ESP_OK)) { + memcpy(data, trans.rx_data, len); + } + return ret; +} + +} // namespace + +void install_w5500_async_spi(eth_w5500_config_t &config) { + // Point the custom driver's config at the W5500 config itself; init() reads spi_host_id and + // spi_devcfg back out of it. The self-reference is valid because both the config and the + // spi_devcfg it points at outlive the esp_eth_mac_new_w5500() call that runs init(). + config.custom_spi_driver.config = &config; + config.custom_spi_driver.init = w5500_custom_spi_init; + config.custom_spi_driver.deinit = w5500_custom_spi_deinit; + config.custom_spi_driver.read = w5500_custom_spi_read; + config.custom_spi_driver.write = w5500_custom_spi_write; +} + +} // namespace esphome::ethernet + +#endif // USE_ESP32 && USE_ETHERNET_W5500 diff --git a/esphome/components/ethernet/w5500_custom_spi.h b/esphome/components/ethernet/w5500_custom_spi.h new file mode 100644 index 0000000000..8756a149af --- /dev/null +++ b/esphome/components/ethernet/w5500_custom_spi.h @@ -0,0 +1,35 @@ +#pragma once + +#include "esphome/core/defines.h" + +#if defined(USE_ESP32) && defined(USE_ETHERNET_W5500) + +#include +// IDF 6.0 moved the per-chip SPI MAC drivers to the Espressif Component Registry; eth_w5500_config_t +// is no longer reachable through esp_eth.h and needs the explicit header. +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) +#include +#else +#include +#endif + +namespace esphome::ethernet { + +// Installs a custom W5500 SPI driver that offloads the bulk frame transfers off the busy-wait path. +// +// The stock W5500 driver runs every SPI transfer through spi_device_polling_transmit(), which +// busy-waits the CPU for the whole transfer. The frame payload (one large read per received frame, +// one large write per transmitted frame) is by far the biggest transfer, so the RX task and the TX +// caller each spin for hundreds of microseconds per frame. This driver sends payload transfers +// through the blocking, interrupt-driven spi_device_transmit() instead, so the calling task sleeps +// while DMA moves the bytes. Small register accesses stay on the polling path, where the busy-wait +// is cheaper than an interrupt round-trip. +// +// Must be called before esp_eth_mac_new_w5500(). The driver reads spi_host_id and spi_devcfg back +// out of `config` in its init() callback, so `config` (and the spi_devcfg it points at) must stay +// alive until esp_eth_mac_new_w5500() returns. +void install_w5500_async_spi(eth_w5500_config_t &config); + +} // namespace esphome::ethernet + +#endif // USE_ESP32 && USE_ETHERNET_W5500 From c951881eea1a4066eef6d5ab1d872ccac0362bb4 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sat, 23 May 2026 23:05:18 -0500 Subject: [PATCH 07/96] [api] Fix uint32_t/int32_t format strings for stricter GCC toolchain (#16603) --- esphome/components/api/api_server.cpp | 7 ++++--- tests/components/api/common-base.yaml | 28 +++++++++++++-------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index c30bd2e612..031fa342c1 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -1,6 +1,7 @@ #include "api_server.h" #ifdef USE_API #include +#include #include "api_connection.h" #include "esphome/components/network/util.h" #include "esphome/core/application.h" @@ -677,7 +678,7 @@ uint32_t APIServer::register_active_action_call(uint32_t client_call_id, APIConn // Schedule automatic cleanup after timeout (client will have given up by then) // Uses numeric ID overload to avoid heap allocation from str_sprintf this->set_timeout(action_call_id, USE_API_ACTION_CALL_TIMEOUT_MS, [this, action_call_id]() { - ESP_LOGD(TAG, "Action call %u timed out", action_call_id); + ESP_LOGD(TAG, "Action call %" PRIu32 " timed out", action_call_id); this->unregister_active_action_call(action_call_id); }); @@ -721,7 +722,7 @@ void APIServer::send_action_response(uint32_t action_call_id, bool success, Stri return; } } - ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %u", action_call_id); + ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %" PRIu32, action_call_id); } #ifdef USE_API_USER_DEFINED_ACTION_RESPONSES_JSON void APIServer::send_action_response(uint32_t action_call_id, bool success, StringRef error_message, @@ -733,7 +734,7 @@ void APIServer::send_action_response(uint32_t action_call_id, bool success, Stri return; } } - ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %u", action_call_id); + ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %" PRIu32, action_call_id); } #endif // USE_API_USER_DEFINED_ACTION_RESPONSES_JSON #endif // USE_API_USER_DEFINED_ACTION_RESPONSES diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index 504c52a57b..ca86445777 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -120,12 +120,12 @@ api: lambda: 'return condition;' then: - logger.log: - format: "Condition true, value: %d" - args: ['value'] + format: "Condition true, value: %ld" + args: ['(long) value'] else: - logger.log: - format: "Condition false, value: %d" - args: ['value'] + format: "Condition false, value: %ld" + args: ['(long) value'] - logger.log: "After if/else" # Test nested IfAction (multiple ContinuationAction instances) - action: test_nested_if @@ -171,8 +171,8 @@ api: count: !lambda 'return count;' then: - logger.log: - format: "Repeat iteration: %d" - args: ['iteration'] + format: "Repeat iteration: %lu" + args: ['(unsigned long) iteration'] - logger.log: "After repeat" # Test combined continuations (if + while + repeat) - action: test_combined_continuations @@ -193,8 +193,8 @@ api: lambda: 'return id(api_continuation_test_counter) > 0;' then: - logger.log: - format: "Combined: repeat=%d, while=%d" - args: ['iteration', 'id(api_continuation_test_counter)'] + format: "Combined: repeat=%lu, while=%d" + args: ['(unsigned long) iteration', 'id(api_continuation_test_counter)'] - lambda: 'id(api_continuation_test_counter)--;' else: - logger.log: "Skipped loops" @@ -208,8 +208,8 @@ api: - api.respond: success: true - logger.log: - format: "Status response sent (call_id=%d)" - args: [call_id] + format: "Status response sent (call_id=%lu)" + args: ['(unsigned long) call_id'] - action: test_respond_status_error variables: @@ -229,8 +229,8 @@ api: value: float then: - logger.log: - format: "Optional response (call_id=%d, return_response=%d)" - args: [call_id, return_response] + format: "Optional response (call_id=%lu, return_response=%lu)" + args: ['(unsigned long) call_id', '(unsigned long) return_response'] - api.respond: data: !lambda |- root["sensor"] = sensor_name; @@ -264,8 +264,8 @@ api: input: string then: - logger.log: - format: "Only response (call_id=%d)" - args: [call_id] + format: "Only response (call_id=%lu)" + args: ['(unsigned long) call_id'] - api.respond: data: !lambda |- root["input"] = input; From 5f860ff5bdabfb574554e617ed5379dbd847fd04 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Sun, 24 May 2026 00:19:07 -0400 Subject: [PATCH 08/96] [esp32] Disable IDF's COMPILER_DISABLE_DEFAULT_ERRORS so -Wno-error actually undoes -Werror (#16604) --- esphome/components/esp32/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 1864c3b544..a06ae89c3e 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1816,8 +1816,11 @@ async def to_code(config): Path(__file__).parent / "iram_fix.py.script", ) else: - # Undo IDF's blanket -Werror so third-party libraries and user - # lambdas don't need a -Wno-error= entry per warning class. + # Demote IDF's blanket -Werror to warnings so third-party libs + # and user lambdas don't need a -Wno-error= per warning. + # The sdkconfig knob disables IDF's rewrite to -Werror=all (which + # can't be globally undone); -Wno-error then handles the demotion. + add_idf_sdkconfig_option("CONFIG_COMPILER_DISABLE_DEFAULT_ERRORS", False) cg.add_build_flag("-Wno-error") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") From a37f27ee7f2d7ed74f666aa1b53d394b89b8d36f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Mart=C3=ADn?= Date: Sun, 24 May 2026 07:27:31 +0200 Subject: [PATCH 09/96] [espnow, ethernet, network, openthread, wifi] centralize network initialization for ESP32 (#14012) Co-authored-by: kbx81 Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- esphome/components/espnow/__init__.py | 2 +- .../components/espnow/espnow_component.cpp | 8 ----- .../ethernet/ethernet_component_esp32.cpp | 6 +--- esphome/components/network/__init__.py | 17 +++++++++- .../components/network/network_component.cpp | 33 +++++++++++++++++++ .../components/network/network_component.h | 14 ++++++++ .../components/openthread/openthread_esp.cpp | 3 +- esphome/components/wifi/wifi_component.cpp | 3 -- .../wifi/wifi_component_esp_idf.cpp | 14 ++------ 9 files changed, 69 insertions(+), 31 deletions(-) create mode 100644 esphome/components/network/network_component.cpp create mode 100644 esphome/components/network/network_component.h diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 7861c0affa..13f278d3bc 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -17,7 +17,7 @@ from esphome.core import HexInt from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] - +AUTO_LOAD = ["network"] byte_vector = cg.std_vector.template(cg.uint8) peer_address_t = cg.std_ns.class_("array").template(cg.uint8, 6) diff --git a/esphome/components/espnow/espnow_component.cpp b/esphome/components/espnow/espnow_component.cpp index 91d44394e8..403e6f4944 100644 --- a/esphome/components/espnow/espnow_component.cpp +++ b/esphome/components/espnow/espnow_component.cpp @@ -149,12 +149,6 @@ bool ESPNowComponent::is_wifi_enabled() { } void ESPNowComponent::setup() { -#ifndef USE_WIFI - // Initialize LwIP stack for wake_loop_threadsafe() socket support - // When WiFi component is present, it handles esp_netif_init() - ESP_ERROR_CHECK(esp_netif_init()); -#endif - if (this->enable_on_boot_) { this->enable_(); } else { @@ -174,8 +168,6 @@ void ESPNowComponent::enable() { void ESPNowComponent::enable_() { if (!this->is_wifi_enabled()) { - esp_event_loop_create_default(); - wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); ESP_ERROR_CHECK(esp_wifi_init(&cfg)); diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index 46e2bb4ec1..6481c8c1f4 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -164,11 +164,7 @@ void EthernetComponent::setup() { err = spi_bus_initialize(host, &buscfg, SPI_DMA_CH_AUTO); ESPHL_ERROR_CHECK(err, "SPI bus initialize error"); #endif - - err = esp_netif_init(); - ESPHL_ERROR_CHECK(err, "ETH netif init error"); - err = esp_event_loop_create_default(); - ESPHL_ERROR_CHECK(err, "ETH event loop error"); + // Network interface setup handled by network component esp_netif_config_t cfg = ESP_NETIF_DEFAULT_ETH(); this->eth_netif_ = esp_netif_new(&cfg); diff --git a/esphome/components/network/__init__.py b/esphome/components/network/__init__.py index 811e7c875a..2818b8c93e 100644 --- a/esphome/components/network/__init__.py +++ b/esphome/components/network/__init__.py @@ -5,8 +5,9 @@ import esphome.codegen as cg from esphome.components.esp32 import add_idf_sdkconfig_option from esphome.components.psram import is_guaranteed as psram_is_guaranteed import esphome.config_validation as cv -from esphome.const import CONF_ENABLE_IPV6, CONF_MIN_IPV6_ADDR_COUNT +from esphome.const import CONF_ENABLE_IPV6, CONF_ID, CONF_MIN_IPV6_ADDR_COUNT from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] AUTO_LOAD = ["mdns"] @@ -19,6 +20,7 @@ KEY_HIGH_PERFORMANCE_NETWORKING = "high_performance_networking" CONF_ENABLE_HIGH_PERFORMANCE = "enable_high_performance" network_ns = cg.esphome_ns.namespace("network") +NetworkComponent = network_ns.class_("NetworkComponent", cg.Component) IPAddress = network_ns.class_("IPAddress") @@ -107,6 +109,7 @@ def has_high_performance_networking() -> bool: CONFIG_SCHEMA = cv.Schema( { + cv.GenerateID(): cv.declare_id(NetworkComponent), cv.SplitDefault( CONF_ENABLE_IPV6, bk72xx=False, @@ -224,3 +227,15 @@ async def to_code(config): cg.add_build_flag("-DPIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_LOW_MEMORY") if CORE.is_rp2040: cg.add_build_flag("-DPIO_FRAMEWORK_ARDUINO_ENABLE_IPV6") + # Pvariable creation lives in a separate coroutine at NETWORK_SERVICES so it + # emits after wifi/ethernet at COMMUNICATION. This keeps compile-time config + # (above) separate from C++ object lifecycle and allows wiring in interface + # pointers via get_variable(). + if CORE.is_esp32: + CORE.add_job(network_component_to_code, config) + + +@coroutine_with_priority(CoroPriority.NETWORK_SERVICES) +async def network_component_to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) diff --git a/esphome/components/network/network_component.cpp b/esphome/components/network/network_component.cpp new file mode 100644 index 0000000000..40cf64906c --- /dev/null +++ b/esphome/components/network/network_component.cpp @@ -0,0 +1,33 @@ +#include "network_component.h" + +#include "esphome/core/defines.h" +#if defined(USE_NETWORK) && defined(USE_ESP32) +#include "esphome/core/log.h" +#include "esp_err.h" +#include "esp_netif.h" +#include "esp_event.h" +namespace esphome::network { + +static const char *const TAG = "network"; + +void NetworkComponent::setup() { + // Initialize ESP-IDF network interfaces and ensure the default event loop exists + esp_err_t err; + err = esp_netif_init(); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_netif_init failed: (%d) %s", err, esp_err_to_name(err)); + this->mark_failed(); + return; + } + err = esp_event_loop_create_default(); + // ESP_ERR_INVALID_STATE is returned if the default loop already exists, + // which is fine since we just want to make sure it exists + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + ESP_LOGE(TAG, "esp_event_loop_create_default failed: (%d) %s", err, esp_err_to_name(err)); + this->mark_failed(); + return; + } +} + +} // namespace esphome::network +#endif diff --git a/esphome/components/network/network_component.h b/esphome/components/network/network_component.h new file mode 100644 index 0000000000..dde15940e4 --- /dev/null +++ b/esphome/components/network/network_component.h @@ -0,0 +1,14 @@ +#pragma once +#include "esphome/core/defines.h" +#if defined(USE_NETWORK) && defined(USE_ESP32) +#include "esphome/core/component.h" + +namespace esphome::network { +class NetworkComponent : public Component { + public: + void setup() override; + // AFTER_BLUETOOTH: BLE controller must initialize before esp_netif_init per IDF guidance. + float get_setup_priority() const override { return setup_priority::AFTER_BLUETOOTH; } +}; +} // namespace esphome::network +#endif diff --git a/esphome/components/openthread/openthread_esp.cpp b/esphome/components/openthread/openthread_esp.cpp index 27712bd86a..787f2f5de8 100644 --- a/esphome/components/openthread/openthread_esp.cpp +++ b/esphome/components/openthread/openthread_esp.cpp @@ -35,9 +35,8 @@ void OpenThreadComponent::setup() { esp_vfs_eventfd_config_t eventfd_config = { .max_fds = 3, }; + // Network interface setup handled by network component ESP_ERROR_CHECK(nvs_flash_init()); - ESP_ERROR_CHECK(esp_event_loop_create_default()); - ESP_ERROR_CHECK(esp_netif_init()); ESP_ERROR_CHECK(esp_vfs_eventfd_register(&eventfd_config)); xTaskCreate( diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 72832d7ac8..fdbd70bc61 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -634,9 +634,6 @@ void WiFiComponent::setup() { if (this->enable_on_boot_) { this->start(); } else { -#ifdef USE_ESP32 - esp_netif_init(); -#endif this->state_ = WIFI_COMPONENT_STATE_DISABLED; } } diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 4f39a3a4b1..11b39b5000 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -145,23 +145,15 @@ void WiFiComponent::wifi_pre_setup_() { get_mac_address_raw(mac); set_mac_address(mac); } - esp_err_t err = esp_netif_init(); - if (err != ERR_OK) { - ESP_LOGE(TAG, "esp_netif_init failed: %s", esp_err_to_name(err)); - return; - } + // Network interface setup handled by network component s_wifi_event_group = xEventGroupCreate(); if (s_wifi_event_group == nullptr) { ESP_LOGE(TAG, "xEventGroupCreate failed"); return; } - err = esp_event_loop_create_default(); - if (err != ERR_OK) { - ESP_LOGE(TAG, "esp_event_loop_create_default failed: %s", esp_err_to_name(err)); - return; - } esp_event_handler_instance_t instance_wifi_id, instance_ip_id; - err = esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, nullptr, &instance_wifi_id); + esp_err_t err = + esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, nullptr, &instance_wifi_id); if (err != ERR_OK) { ESP_LOGE(TAG, "esp_event_handler_instance_register failed: %s", esp_err_to_name(err)); return; From 750d52741a8c5136e855c4f1ad5992f64d973630 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sun, 24 May 2026 08:36:53 -0400 Subject: [PATCH 10/96] [voice_assistant] Use RingBufferAudioSource (#16597) --- .../components/voice_assistant/__init__.py | 2 +- .../voice_assistant/voice_assistant.cpp | 129 ++++++++---------- .../voice_assistant/voice_assistant.h | 13 +- 3 files changed, 63 insertions(+), 81 deletions(-) diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py index 958d1cbf91..f41adfd8de 100644 --- a/esphome/components/voice_assistant/__init__.py +++ b/esphome/components/voice_assistant/__init__.py @@ -15,7 +15,7 @@ from esphome.const import ( CONF_SPEAKER, ) -AUTO_LOAD = ["ring_buffer", "socket"] +AUTO_LOAD = ["audio", "ring_buffer", "socket"] DEPENDENCIES = ["api", "microphone"] CODEOWNERS = ["@jesserockz", "@kahrendt"] diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 286e6645d2..af1b98da02 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -30,7 +30,7 @@ VoiceAssistant::VoiceAssistant() { global_voice_assistant = this; } void VoiceAssistant::setup() { this->mic_source_->add_data_callback([this](const std::vector &data) { - std::shared_ptr temp_ring_buffer = this->ring_buffer_; + std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); if (temp_ring_buffer != nullptr) { temp_ring_buffer->write((void *) data.data(), data.size()); } @@ -39,7 +39,7 @@ void VoiceAssistant::setup() { // Second microphone channel if (this->mic_source2_ != nullptr) { this->mic_source2_->add_data_callback([this](const std::vector &data) { - std::shared_ptr temp_ring_buffer = this->ring_buffer2_; + std::shared_ptr temp_ring_buffer = this->ring_buffer2_.lock(); if (temp_ring_buffer != nullptr) { temp_ring_buffer->write((void *) data.data(), data.size()); } @@ -125,62 +125,47 @@ bool VoiceAssistant::allocate_buffers_() { } #endif - if (this->ring_buffer_ == nullptr) { - this->ring_buffer_ = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); - if (this->ring_buffer_ == nullptr) { + if (this->audio_source_ == nullptr) { + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); + if (temp_ring_buffer == nullptr) { ESP_LOGE(TAG, "Could not allocate ring buffer"); return false; } - } - - if (this->send_buffer_ == nullptr) { - RAMAllocator send_allocator; - this->send_buffer_ = send_allocator.allocate(SEND_BUFFER_SIZE); - if (send_buffer_ == nullptr) { - ESP_LOGW(TAG, "Could not allocate send buffer"); + // Zero-copy source that reads directly from the ring buffer; frame-aligned to never split an int16 sample. + this->audio_source_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t)); + if (this->audio_source_ == nullptr) { + ESP_LOGE(TAG, "Could not allocate audio source"); return false; } + this->ring_buffer_ = temp_ring_buffer; } // Second microphone channel - if (this->mic_source2_ != nullptr) { - if (this->ring_buffer2_ == nullptr) { - this->ring_buffer2_ = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); - if (this->ring_buffer2_ == nullptr) { - ESP_LOGE(TAG, "Could not allocate second ring buffer"); - return false; - } + if ((this->mic_source2_ != nullptr) && (this->audio_source2_ == nullptr)) { + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); + if (temp_ring_buffer == nullptr) { + ESP_LOGE(TAG, "Could not allocate second ring buffer"); + return false; } - - if (this->send_buffer2_ == nullptr) { - RAMAllocator send_allocator; - this->send_buffer2_ = send_allocator.allocate(SEND_BUFFER_SIZE); - if (this->send_buffer2_ == nullptr) { - ESP_LOGW(TAG, "Could not allocate second send buffer"); - return false; - } + this->audio_source2_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t)); + if (this->audio_source2_ == nullptr) { + ESP_LOGE(TAG, "Could not allocate second audio source"); + return false; } + this->ring_buffer2_ = temp_ring_buffer; } return true; } void VoiceAssistant::clear_buffers_() { - if (this->send_buffer_ != nullptr) { - memset(this->send_buffer_, 0, SEND_BUFFER_SIZE); - } - - if (this->ring_buffer_ != nullptr) { - this->ring_buffer_->reset(); + if (this->audio_source_ != nullptr) { + this->audio_source_->clear_buffered_data(); } // Second microphone channel - if (this->send_buffer2_ != nullptr) { - memset(this->send_buffer2_, 0, SEND_BUFFER_SIZE); - } - - if (this->ring_buffer2_ != nullptr) { - this->ring_buffer2_->reset(); + if (this->audio_source2_ != nullptr) { + this->audio_source2_->clear_buffered_data(); } #ifdef USE_SPEAKER @@ -195,22 +180,11 @@ void VoiceAssistant::clear_buffers_() { } void VoiceAssistant::deallocate_buffers_() { - if (this->send_buffer_ != nullptr) { - RAMAllocator send_deallocator; - send_deallocator.deallocate(this->send_buffer_, SEND_BUFFER_SIZE); - this->send_buffer_ = nullptr; - } - - this->ring_buffer_.reset(); + // Destroying each source releases its ring buffer; the matching weak_ptr then expires automatically. + this->audio_source_.reset(); // Second microphone channel - if (this->send_buffer2_ != nullptr) { - RAMAllocator send_deallocator; - send_deallocator.deallocate(this->send_buffer2_, SEND_BUFFER_SIZE); - this->send_buffer2_ = nullptr; - } - - this->ring_buffer2_.reset(); + this->audio_source2_.reset(); #ifdef USE_SPEAKER if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) { @@ -316,52 +290,57 @@ void VoiceAssistant::loop() { break; // State changed when udp server port received } case State::STREAMING_MICROPHONE: { + // pre_shift is ignored by RingBufferAudioSource (no intermediate transfer buffer to compact). if (this->audio_mode_ == AUDIO_MODE_API) { // API audio // Both microphone channels are sent, if configured - bool is_available = this->ring_buffer_->available() >= SEND_BUFFER_SIZE; - bool is_available2 = false; - if (this->mic_source2_) { - is_available2 = this->ring_buffer2_->available() >= SEND_BUFFER_SIZE; + size_t available = this->audio_source_->fill(0, false); + size_t available2 = 0; + if (this->audio_source2_ != nullptr) { + available2 = this->audio_source2_->fill(0, false); } - while (is_available || is_available2) { + while (available > 0 || available2 > 0) { api::VoiceAssistantAudio msg; - if (is_available) { - size_t read_bytes = this->ring_buffer_->read((void *) this->send_buffer_, SEND_BUFFER_SIZE, 0); - msg.data = this->send_buffer_; - msg.data_len = read_bytes; + if (available > 0) { + // Zero-copy: send_message() copies the data out before we consume it + msg.data = this->audio_source_->data(); + msg.data_len = available; } // Second microphone channel - if (is_available2) { - size_t read_bytes = this->ring_buffer2_->read((void *) this->send_buffer2_, SEND_BUFFER_SIZE, 0); - msg.data2 = this->send_buffer2_; - msg.data2_len = read_bytes; + if (available2 > 0) { + msg.data2 = this->audio_source2_->data(); + msg.data2_len = available2; } this->api_client_->send_message(msg); - is_available = this->ring_buffer_->available() >= SEND_BUFFER_SIZE; - if (this->mic_source2_) { - is_available2 = this->ring_buffer2_->available() >= SEND_BUFFER_SIZE; - } else { - is_available2 = false; + + if (available > 0) { + this->audio_source_->consume(available); + } + available = this->audio_source_->fill(0, false); + if (available2 > 0) { + this->audio_source2_->consume(available2); + } + if (this->audio_source2_ != nullptr) { + available2 = this->audio_source2_->fill(0, false); } } } else { // UDP (will eventually be deprecated) // Only the primary microphone channel is used - while (this->ring_buffer_->available() >= SEND_BUFFER_SIZE) { - size_t read_bytes = this->ring_buffer_->read((void *) this->send_buffer_, SEND_BUFFER_SIZE, 0); + while (this->audio_source_->fill(0, false) > 0) { if (!this->udp_socket_running_) { if (!this->start_udp_socket_()) { this->set_state_(State::STOP_MICROPHONE, State::IDLE); break; } } - this->socket_->sendto(this->send_buffer_, read_bytes, 0, (struct sockaddr *) &this->dest_addr_, - sizeof(this->dest_addr_)); + this->socket_->sendto(this->audio_source_->data(), this->audio_source_->available(), 0, + (struct sockaddr *) &this->dest_addr_, sizeof(this->dest_addr_)); + this->audio_source_->consume(this->audio_source_->available()); } } // audio mode break; diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index c4fa7eb615..f3ea669e15 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -9,6 +9,7 @@ #include "esphome/core/helpers.h" #include "esphome/components/api/api_connection.h" +#include "esphome/components/audio/audio_transfer_buffer.h" #include "esphome/components/ring_buffer/ring_buffer.h" #include "esphome/components/api/api_pb2.h" #include "esphome/components/microphone/microphone_source.h" @@ -306,8 +307,13 @@ class VoiceAssistant : public Component { std::string wake_word_; - std::shared_ptr ring_buffer_; - std::shared_ptr ring_buffer2_; + // Zero-copy sources that read directly from each microphone channel's ring buffer internal storage. + // Each source owns its ring buffer; the matching ``ring_buffer_``/``ring_buffer2_`` weak_ptr is used by + // the microphone callback (a different thread) to write into it. + std::unique_ptr audio_source_; + std::unique_ptr audio_source2_; + std::weak_ptr ring_buffer_; + std::weak_ptr ring_buffer2_; bool use_wake_word_; uint8_t noise_suppression_level_; @@ -315,9 +321,6 @@ class VoiceAssistant : public Component { float volume_multiplier_; uint32_t conversation_timeout_; - uint8_t *send_buffer_{nullptr}; - uint8_t *send_buffer2_{nullptr}; - bool continuous_{false}; bool silence_detection_; From c17c4478ac17f375c79d561701cdc92a7152ae91 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sun, 24 May 2026 15:32:43 -0400 Subject: [PATCH 11/96] [mixer] Support any bit depth audio (#16524) --- esphome/components/mixer/speaker/__init__.py | 29 +-- .../mixer/speaker/mixer_speaker.cpp | 210 +++--------------- .../components/mixer/speaker/mixer_speaker.h | 58 +---- tests/components/mixer/common.yaml | 4 + 4 files changed, 65 insertions(+), 236 deletions(-) diff --git a/esphome/components/mixer/speaker/__init__.py b/esphome/components/mixer/speaker/__init__.py index 59a80d9297..8501843d3f 100644 --- a/esphome/components/mixer/speaker/__init__.py +++ b/esphome/components/mixer/speaker/__init__.py @@ -44,20 +44,10 @@ SOURCE_SPEAKER_SCHEMA = speaker.SPEAKER_SCHEMA.extend( cv.positive_time_period_milliseconds, cv.one_of(CONF_NEVER, lower=True), ), - cv.Optional(CONF_BITS_PER_SAMPLE, default=16): cv.int_range(16, 16), } ) -def _set_stream_limits(config): - audio.set_stream_limits( - min_bits_per_sample=16, - max_bits_per_sample=16, - )(config) - - return config - - def _validate_source_speaker(config): fconf = fv.full_config.get() @@ -67,15 +57,25 @@ def _validate_source_speaker(config): output_speaker_id = fconf.get_config_for_path(path) config[CONF_OUTPUT_SPEAKER] = output_speaker_id + inherit_property_from(CONF_BITS_PER_SAMPLE, CONF_OUTPUT_SPEAKER)(config) inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER)(config) inherit_property_from(CONF_SAMPLE_RATE, CONF_OUTPUT_SPEAKER)(config) + audio.final_validate_audio_schema( + "mixer", + audio_device=CONF_OUTPUT_SPEAKER, + sample_rate=config.get(CONF_SAMPLE_RATE), + )(config) + + return config + + +def _validate_output_speaker(config): audio.final_validate_audio_schema( "mixer", audio_device=CONF_OUTPUT_SPEAKER, bits_per_sample=config.get(CONF_BITS_PER_SAMPLE), channels=config.get(CONF_NUM_CHANNELS), - sample_rate=config.get(CONF_SAMPLE_RATE), )(config) return config @@ -89,8 +89,8 @@ CONFIG_SCHEMA = cv.All( cv.Required(CONF_SOURCE_SPEAKERS): cv.All( cv.ensure_list(SOURCE_SPEAKER_SCHEMA), cv.Length(min=2, max=8), - [_set_stream_limits], ), + cv.Optional(CONF_BITS_PER_SAMPLE): cv.one_of(8, 16, 24, 32, int=True), cv.Optional(CONF_NUM_CHANNELS): cv.int_range(min=1, max=2), cv.Optional(CONF_QUEUE_MODE, default=False): cv.boolean, cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, @@ -100,13 +100,15 @@ CONFIG_SCHEMA = cv.All( ) FINAL_VALIDATE_SCHEMA = cv.All( + inherit_property_from(CONF_BITS_PER_SAMPLE, CONF_OUTPUT_SPEAKER), + inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER), cv.Schema( { cv.Optional(CONF_SOURCE_SPEAKERS): [_validate_source_speaker], }, extra=cv.ALLOW_EXTRA, ), - inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER), + _validate_output_speaker, ) @@ -116,6 +118,7 @@ async def to_code(config): spkr = await cg.get_variable(config[CONF_OUTPUT_SPEAKER]) + cg.add(var.set_output_bits_per_sample(config[CONF_BITS_PER_SAMPLE])) cg.add(var.set_output_channels(config[CONF_NUM_CHANNELS])) cg.add(var.set_output_speaker(spkr)) cg.add(var.set_queue_mode(config[CONF_QUEUE_MODE])) diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 1a995a6edf..6128dc3767 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -7,8 +7,10 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include // esp-audio-libs +#include // esp-audio-libs + #include -#include #include namespace esphome::mixer_speaker { @@ -22,19 +24,8 @@ static const uint32_t MIXER_AUTO_STOP_DEBOUNCE_MS = 200; static const size_t TASK_STACK_SIZE = 4096; -static const int16_t MAX_AUDIO_SAMPLE_VALUE = INT16_MAX; -static const int16_t MIN_AUDIO_SAMPLE_VALUE = INT16_MIN; - static const char *const TAG = "speaker_mixer"; -// Gives the Q15 fixed point scaling factor to reduce by 0 dB, 1dB, ..., 50 dB -// dB to PCM scaling factor formula: floating_point_scale_factor = 2^(-db/6.014) -// float to Q15 fixed point formula: q15_scale_factor = floating_point_scale_factor * 2^(15) -static const std::array DECIBEL_REDUCTION_TABLE = { - 32767, 29201, 26022, 23189, 20665, 18415, 16410, 14624, 13032, 11613, 10349, 9222, 8218, 7324, 6527, 5816, 5183, - 4619, 4116, 3668, 3269, 2913, 2596, 2313, 2061, 1837, 1637, 1459, 1300, 1158, 1032, 920, 820, 731, - 651, 580, 517, 461, 411, 366, 326, 291, 259, 231, 206, 183, 163, 146, 130, 116, 103}; - // Event bits for SourceSpeaker command processing enum SourceSpeakerEventBits : uint32_t { SOURCE_SPEAKER_COMMAND_START = (1 << 0), @@ -315,97 +306,17 @@ size_t SourceSpeaker::process_data_from_source(std::shared_ptraudio_stream_info_.bytes_to_samples(bytes_read); if (samples_to_duck > 0) { - int16_t *current_buffer = reinterpret_cast(audio_source->mutable_data()); - - duck_samples(current_buffer, samples_to_duck, &this->current_ducking_db_reduction_, - &this->ducking_transition_samples_remaining_, this->samples_per_ducking_step_, - this->db_change_per_ducking_step_); + esp_audio_libs::ducking::apply(audio_source->mutable_data(), + static_cast(this->audio_stream_info_.get_bits_per_sample() / 8), + samples_to_duck, this->ducking_state_); } return bytes_read; } void SourceSpeaker::apply_ducking(uint8_t decibel_reduction, uint32_t duration) { - if (this->target_ducking_db_reduction_ != decibel_reduction) { - // Start transition from the previous target (which becomes the new current level) - this->current_ducking_db_reduction_ = this->target_ducking_db_reduction_; - - this->target_ducking_db_reduction_ = decibel_reduction; - - // Calculate the number of intermediate dB steps for the transition timing. - // Subtract 1 because the first step is taken immediately after this calculation. - uint8_t total_ducking_steps = 0; - if (this->target_ducking_db_reduction_ > this->current_ducking_db_reduction_) { - // The dB reduction level is increasing (which results in quieter audio) - total_ducking_steps = this->target_ducking_db_reduction_ - this->current_ducking_db_reduction_ - 1; - this->db_change_per_ducking_step_ = 1; - } else { - // The dB reduction level is decreasing (which results in louder audio) - total_ducking_steps = this->current_ducking_db_reduction_ - this->target_ducking_db_reduction_ - 1; - this->db_change_per_ducking_step_ = -1; - } - if ((duration > 0) && (total_ducking_steps > 0)) { - this->ducking_transition_samples_remaining_ = this->audio_stream_info_.ms_to_samples(duration); - - this->samples_per_ducking_step_ = this->ducking_transition_samples_remaining_ / total_ducking_steps; - this->ducking_transition_samples_remaining_ = - this->samples_per_ducking_step_ * total_ducking_steps; // adjust for integer division rounding - - this->current_ducking_db_reduction_ += this->db_change_per_ducking_step_; - } else { - this->ducking_transition_samples_remaining_ = 0; - this->current_ducking_db_reduction_ = this->target_ducking_db_reduction_; - } - } -} - -void SourceSpeaker::duck_samples(int16_t *input_buffer, uint32_t input_samples_to_duck, - int8_t *current_ducking_db_reduction, uint32_t *ducking_transition_samples_remaining, - uint32_t samples_per_ducking_step, int8_t db_change_per_ducking_step) { - if (*ducking_transition_samples_remaining > 0) { - // Ducking level is still transitioning - - // Takes the ceiling of input_samples_to_duck/samples_per_ducking_step - uint32_t ducking_steps_in_batch = - input_samples_to_duck / samples_per_ducking_step + (input_samples_to_duck % samples_per_ducking_step != 0); - - for (uint32_t i = 0; i < ducking_steps_in_batch; ++i) { - uint32_t samples_left_in_step = *ducking_transition_samples_remaining % samples_per_ducking_step; - - if (samples_left_in_step == 0) { - samples_left_in_step = samples_per_ducking_step; - } - - uint32_t samples_to_duck = std::min(input_samples_to_duck, samples_left_in_step); - samples_to_duck = std::min(samples_to_duck, *ducking_transition_samples_remaining); - - // Ensure we only point to valid index in the Q15 scaling factor table - uint8_t safe_db_reduction_index = - clamp(*current_ducking_db_reduction, 0, DECIBEL_REDUCTION_TABLE.size() - 1); - int16_t q15_scale_factor = DECIBEL_REDUCTION_TABLE[safe_db_reduction_index]; - - audio::scale_audio_samples(input_buffer, input_buffer, q15_scale_factor, samples_to_duck); - - if (samples_left_in_step - samples_to_duck == 0) { - // After scaling the current samples, we are ready to transition to the next step - *current_ducking_db_reduction += db_change_per_ducking_step; - } - - input_buffer += samples_to_duck; - *ducking_transition_samples_remaining -= samples_to_duck; - input_samples_to_duck -= samples_to_duck; - } - } - - if ((*current_ducking_db_reduction > 0) && (input_samples_to_duck > 0)) { - // Audio is ducked, but its not in the middle of a transition step - - uint8_t safe_db_reduction_index = - clamp(*current_ducking_db_reduction, 0, DECIBEL_REDUCTION_TABLE.size() - 1); - int16_t q15_scale_factor = DECIBEL_REDUCTION_TABLE[safe_db_reduction_index]; - - audio::scale_audio_samples(input_buffer, input_buffer, q15_scale_factor, input_samples_to_duck); - } + const uint32_t transition_samples = duration > 0 ? this->audio_stream_info_.ms_to_samples(duration) : 0; + esp_audio_libs::ducking::set_target(this->ducking_state_, decibel_reduction, transition_samples); } void SourceSpeaker::enter_stopping_state_() { @@ -417,8 +328,9 @@ void SourceSpeaker::enter_stopping_state_() { void MixerSpeaker::dump_config() { ESP_LOGCONFIG(TAG, "Speaker Mixer:\n" - " Number of output channels: %u", - this->output_channels_); + " Number of output channels: %" PRIu8 "\n" + " Output bits per sample: %" PRIu8, + this->output_channels_, this->output_bits_per_sample_); } void MixerSpeaker::setup() { @@ -512,13 +424,8 @@ void MixerSpeaker::loop() { esp_err_t MixerSpeaker::start(audio::AudioStreamInfo &stream_info) { if (!this->audio_stream_info_.has_value()) { - if (stream_info.get_bits_per_sample() != 16) { - // Audio streams that don't have 16 bits per sample are not supported - return ESP_ERR_NOT_SUPPORTED; - } - - this->audio_stream_info_ = audio::AudioStreamInfo(stream_info.get_bits_per_sample(), this->output_channels_, - stream_info.get_sample_rate()); + this->audio_stream_info_ = + audio::AudioStreamInfo(this->output_bits_per_sample_, this->output_channels_, stream_info.get_sample_rate()); this->output_speaker_->set_audio_stream_info(this->audio_stream_info_.value()); } else { if (!this->queue_mode_ && (stream_info.get_sample_rate() != this->audio_stream_info_.value().get_sample_rate())) { @@ -542,57 +449,6 @@ esp_err_t MixerSpeaker::start(audio::AudioStreamInfo &stream_info) { return ESP_OK; } -void MixerSpeaker::copy_frames(const int16_t *input_buffer, audio::AudioStreamInfo input_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_transfer) { - uint8_t input_channels = input_stream_info.get_channels(); - uint8_t output_channels = output_stream_info.get_channels(); - const uint8_t max_input_channel_index = input_channels - 1; - - if (input_channels == output_channels) { - size_t bytes_to_copy = input_stream_info.frames_to_bytes(frames_to_transfer); - memcpy(output_buffer, input_buffer, bytes_to_copy); - - return; - } - - for (uint32_t frame_index = 0; frame_index < frames_to_transfer; ++frame_index) { - for (uint8_t output_channel_index = 0; output_channel_index < output_channels; ++output_channel_index) { - uint8_t input_channel_index = std::min(output_channel_index, max_input_channel_index); - output_buffer[output_channels * frame_index + output_channel_index] = - input_buffer[input_channels * frame_index + input_channel_index]; - } - } -} - -void MixerSpeaker::mix_audio_samples(const int16_t *primary_buffer, audio::AudioStreamInfo primary_stream_info, - const int16_t *secondary_buffer, audio::AudioStreamInfo secondary_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_mix) { - const uint8_t primary_channels = primary_stream_info.get_channels(); - const uint8_t secondary_channels = secondary_stream_info.get_channels(); - const uint8_t output_channels = output_stream_info.get_channels(); - - const uint8_t max_primary_channel_index = primary_channels - 1; - const uint8_t max_secondary_channel_index = secondary_channels - 1; - - for (uint32_t frames_index = 0; frames_index < frames_to_mix; ++frames_index) { - for (uint8_t output_channel_index = 0; output_channel_index < output_channels; ++output_channel_index) { - const uint32_t secondary_channel_index = std::min(output_channel_index, max_secondary_channel_index); - const int32_t secondary_sample = secondary_buffer[frames_index * secondary_channels + secondary_channel_index]; - - const uint32_t primary_channel_index = std::min(output_channel_index, max_primary_channel_index); - const int32_t primary_sample = - static_cast(primary_buffer[frames_index * primary_channels + primary_channel_index]); - - const int32_t added_sample = secondary_sample + primary_sample; - - output_buffer[frames_index * output_channels + output_channel_index] = - static_cast(clamp(added_sample, MIN_AUDIO_SAMPLE_VALUE, MAX_AUDIO_SAMPLE_VALUE)); - } - } -} - // NOLINTBEGIN(bugprone-unchecked-optional-access) -- audio_stream_info_ always set before this task is created void MixerSpeaker::audio_mixer_task(void *params) { MixerSpeaker *this_mixer = static_cast(params); @@ -662,6 +518,10 @@ void MixerSpeaker::audio_mixer_task(void *params) { uint32_t frames_to_mix = output_frames_free; + const audio::AudioStreamInfo &output_info = this_mixer->audio_stream_info_.value(); + const uint8_t output_bps = output_info.get_bits_per_sample() / 8; + const uint8_t output_channels = output_info.get_channels(); + if ((audio_sources_with_data.size() == 1) || this_mixer->queue_mode_) { // Only one speaker has audio data, just copy samples over @@ -669,14 +529,15 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (active_stream_info.get_sample_rate() == this_mixer->output_speaker_->get_audio_stream_info().get_sample_rate()) { - // Speaker's sample rate matches the output speaker's, copy directly + // Speaker's sample rate matches the output speaker's, convert directly into the output buffer const uint32_t frames_available_in_buffer = active_stream_info.bytes_to_frames(audio_sources_with_data[0]->available()); frames_to_mix = std::min(frames_to_mix, frames_available_in_buffer); - copy_frames(reinterpret_cast(audio_sources_with_data[0]->data()), active_stream_info, - reinterpret_cast(output_transfer_buffer->get_buffer_end()), - this_mixer->audio_stream_info_.value(), frames_to_mix); + esp_audio_libs::pcm_convert::copy_frames( + audio_sources_with_data[0]->data(), output_transfer_buffer->get_buffer_end(), + static_cast(active_stream_info.get_bits_per_sample() / 8), active_stream_info.get_channels(), + output_bps, output_channels, frames_to_mix); // Set playback delay for newly contributing source if (!speakers_with_data[0]->has_contributed_.load(std::memory_order_acquire)) { @@ -690,8 +551,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { audio_sources_with_data[0]->consume(active_stream_info.frames_to_bytes(frames_to_mix)); // Update output transfer buffer length and pipeline frame count - output_transfer_buffer->increase_buffer_length( - this_mixer->audio_stream_info_.value().frames_to_bytes(frames_to_mix)); + output_transfer_buffer->increase_buffer_length(output_info.frames_to_bytes(frames_to_mix)); this_mixer->frames_in_pipeline_.fetch_add(frames_to_mix, std::memory_order_release); } else { // Speaker's stream info doesn't match the output speaker's, so it's a new source speaker @@ -703,7 +563,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { } else { // Speaker has finished writing the current audio, update the stream information and restart the speaker this_mixer->audio_stream_info_ = - audio::AudioStreamInfo(active_stream_info.get_bits_per_sample(), this_mixer->output_channels_, + audio::AudioStreamInfo(this_mixer->output_bits_per_sample_, this_mixer->output_channels_, active_stream_info.get_sample_rate()); this_mixer->output_speaker_->set_audio_stream_info(this_mixer->audio_stream_info_.value()); this_mixer->output_speaker_->start(); @@ -719,21 +579,22 @@ void MixerSpeaker::audio_mixer_task(void *params) { speakers_with_data[i]->get_audio_stream_info().bytes_to_frames(audio_sources_with_data[i]->available()); frames_to_mix = std::min(frames_to_mix, frames_available_in_buffer); } - const int16_t *primary_buffer = reinterpret_cast(audio_sources_with_data[0]->data()); + const uint8_t *primary_buffer = audio_sources_with_data[0]->data(); audio::AudioStreamInfo primary_stream_info = speakers_with_data[0]->get_audio_stream_info(); - // Mix two streams together + // Mix two streams together at a time, accumulating into the output buffer. for (size_t i = 1; i < audio_sources_with_data.size(); ++i) { - mix_audio_samples(primary_buffer, primary_stream_info, - reinterpret_cast(audio_sources_with_data[i]->data()), - speakers_with_data[i]->get_audio_stream_info(), - reinterpret_cast(output_transfer_buffer->get_buffer_end()), - this_mixer->audio_stream_info_.value(), frames_to_mix); + esp_audio_libs::mixer::mix_frames( + primary_buffer, static_cast(primary_stream_info.get_bits_per_sample() / 8), + primary_stream_info.get_channels(), audio_sources_with_data[i]->data(), + static_cast(speakers_with_data[i]->get_audio_stream_info().get_bits_per_sample() / 8), + speakers_with_data[i]->get_audio_stream_info().get_channels(), output_transfer_buffer->get_buffer_end(), + output_bps, output_channels, frames_to_mix); if (i != audio_sources_with_data.size() - 1) { // Need to mix more streams together, point primary buffer and stream info to the already mixed output - primary_buffer = reinterpret_cast(output_transfer_buffer->get_buffer_end()); - primary_stream_info = this_mixer->audio_stream_info_.value(); + primary_buffer = output_transfer_buffer->get_buffer_end(); + primary_stream_info = output_info; } } @@ -754,8 +615,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { } // Update output transfer buffer length and pipeline frame count (once, not per source) - output_transfer_buffer->increase_buffer_length( - this_mixer->audio_stream_info_.value().frames_to_bytes(frames_to_mix)); + output_transfer_buffer->increase_buffer_length(output_info.frames_to_bytes(frames_to_mix)); this_mixer->frames_in_pipeline_.fetch_add(frames_to_mix, std::memory_order_release); } } diff --git a/esphome/components/mixer/speaker/mixer_speaker.h b/esphome/components/mixer/speaker/mixer_speaker.h index f57bead679..f1ae919b50 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.h +++ b/esphome/components/mixer/speaker/mixer_speaker.h @@ -11,6 +11,8 @@ #include "esphome/core/helpers.h" #include "esphome/core/static_task.h" +#include // esp-audio-libs + #include #include @@ -22,7 +24,8 @@ namespace esphome::mixer_speaker { * - Source speaker commands are signaled via event group bits and processed in its loop function to ensure thread * safety * - Directly handles pausing at the SourceSpeaker level; pause state is not passed through to the output speaker. - * - Audio sent to the SourceSpeaker must have 16 bits per sample. + * - Audio sent to the SourceSpeaker can have 8, 16, 24, or 32 bits per sample. Each source is converted to the output + * speaker's bit depth as it is mixed (or copied) into the output buffer. * - Audio sent to the SourceSpeaker can have any number of channels. They are duplicated or ignored as needed to match * the number of channels required for the output speaker. * - In queue mode, the audio sent to the SourceSpeakers can have different sample rates. @@ -93,19 +96,6 @@ class SourceSpeaker : public speaker::Speaker, public Component { void enter_stopping_state_(); void send_command_(uint32_t command_bit, bool wake_loop = false); - /// @brief Ducks audio samples by a specified amount. When changing the ducking amount, it can transition gradually - /// over a specified amount of samples. - /// @param input_buffer buffer with audio samples to be ducked in place - /// @param input_samples_to_duck number of samples to process in ``input_buffer`` - /// @param current_ducking_db_reduction pointer to the current dB reduction - /// @param ducking_transition_samples_remaining pointer to the total number of samples left before the - /// transition is finished - /// @param samples_per_ducking_step total number of samples per ducking step for the transition - /// @param db_change_per_ducking_step the change in dB reduction per step - static void duck_samples(int16_t *input_buffer, uint32_t input_samples_to_duck, int8_t *current_ducking_db_reduction, - uint32_t *ducking_transition_samples_remaining, uint32_t samples_per_ducking_step, - int8_t db_change_per_ducking_step); - MixerSpeaker *parent_; std::shared_ptr audio_source_; @@ -118,11 +108,7 @@ class SourceSpeaker : public speaker::Speaker, public Component { bool pause_state_{false}; - int8_t target_ducking_db_reduction_{0}; - int8_t current_ducking_db_reduction_{0}; - int8_t db_change_per_ducking_step_{1}; - uint32_t ducking_transition_samples_remaining_{0}; - uint32_t samples_per_ducking_step_{0}; + esp_audio_libs::ducking::DuckingState ducking_state_{}; std::atomic pending_playback_frames_{0}; std::atomic playback_delay_frames_{0}; // Frames in output pipeline when this source started contributing @@ -143,12 +129,14 @@ class MixerSpeaker : public Component { /// @brief Starts the mixer task. Called by a source speaker giving the current audio stream information /// @param stream_info The calling source speaker's audio stream information - /// @return ESP_ERR_NOT_SUPPORTED if the incoming stream is incompatible due to unsupported bits per sample - /// ESP_ERR_INVALID_ARG if the incoming stream is incompatible to be mixed with the other input audio stream + /// @return ESP_ERR_INVALID_ARG if the incoming stream is incompatible to be mixed with the other input audio stream /// ESP_OK if the incoming stream is compatible and the mixer task starts esp_err_t start(audio::AudioStreamInfo &stream_info); void set_output_channels(uint8_t output_channels) { this->output_channels_ = output_channels; } + void set_output_bits_per_sample(uint8_t output_bits_per_sample) { + this->output_bits_per_sample_ = output_bits_per_sample; + } void set_output_speaker(speaker::Speaker *speaker) { this->output_speaker_ = speaker; } void set_queue_mode(bool queue_mode) { this->queue_mode_ = queue_mode; } void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } @@ -159,33 +147,6 @@ class MixerSpeaker : public Component { uint32_t get_frames_in_pipeline() const { return this->frames_in_pipeline_.load(std::memory_order_acquire); } protected: - /// @brief Copies audio frames from the input buffer to the output buffer taking into account the number of channels - /// in each stream. If the output stream has more channels, the input samples are duplicated. If the output stream has - /// less channels, the extra channel input samples are dropped. - /// @param input_buffer - /// @param input_stream_info - /// @param output_buffer - /// @param output_stream_info - /// @param frames_to_transfer number of frames (consisting of a sample for each channel) to copy from the input buffer - static void copy_frames(const int16_t *input_buffer, audio::AudioStreamInfo input_stream_info, int16_t *output_buffer, - audio::AudioStreamInfo output_stream_info, uint32_t frames_to_transfer); - - /// @brief Mixes the primary and secondary streams taking into account the number of channels in each stream. Primary - /// and secondary samples are duplicated or dropped as necessary to ensure the output stream has the configured number - /// of channels. Output samples are clamped to the corresponding int16 min or max values if the mixed sample - /// overflows. - /// @param primary_buffer samples buffer for the primary stream - /// @param primary_stream_info stream info for the primary stream - /// @param secondary_buffer samples buffer for secondary stream - /// @param secondary_stream_info stream info for the secondary stream - /// @param output_buffer buffer for the mixed samples - /// @param output_stream_info stream info for the output buffer - /// @param frames_to_mix number of frames in the primary and secondary buffers to mix together - static void mix_audio_samples(const int16_t *primary_buffer, audio::AudioStreamInfo primary_stream_info, - const int16_t *secondary_buffer, audio::AudioStreamInfo secondary_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_mix); - static void audio_mixer_task(void *params); EventGroupHandle_t event_group_{nullptr}; @@ -193,6 +154,7 @@ class MixerSpeaker : public Component { FixedVector source_speakers_; speaker::Speaker *output_speaker_{nullptr}; + uint8_t output_bits_per_sample_; uint8_t output_channels_; bool queue_mode_; bool task_stack_in_psram_{false}; diff --git a/tests/components/mixer/common.yaml b/tests/components/mixer/common.yaml index e171b9499c..ef613b82bc 100644 --- a/tests/components/mixer/common.yaml +++ b/tests/components/mixer/common.yaml @@ -16,8 +16,12 @@ speaker: id: speaker_id dac_type: external i2s_dout_pin: ${dout_pin} + bits_per_sample: 32bit + channel: stereo - platform: mixer output_speaker: speaker_id + bits_per_sample: 32 + num_channels: 2 source_speakers: - id: source_speaker_1_id - id: source_speaker_2_id From 5cb7e622415ca036f8b33a139ea5db871eabc27d Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sun, 24 May 2026 15:33:32 -0400 Subject: [PATCH 12/96] [audio] Use RingBufferAudioSource for decoding (#16564) --- esphome/components/audio/audio_decoder.cpp | 83 +++++++++++----------- esphome/components/audio/audio_decoder.h | 9 +-- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index d4ff59fc36..f709c23fb6 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -9,9 +9,12 @@ namespace esphome::audio { static const char *const TAG = "audio.decoder"; -static const uint32_t DECODING_TIMEOUT_MS = 50; // The decode function will yield after this duration static const uint32_t READ_WRITE_TIMEOUT_MS = 20; // Timeout for transferring audio data +// Max consecutive decode iterations that consume input but produce no output; e.g., skipping a large metadata block, +// before yielding and returning. +static const uint8_t MAX_NO_OUTPUT_ITERATIONS = 32; + static const uint32_t MAX_POTENTIALLY_FAILED_COUNT = 10; AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) @@ -20,11 +23,13 @@ AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) } esp_err_t AudioDecoder::add_source(std::weak_ptr &input_ring_buffer) { - auto source = AudioSourceTransferBuffer::create(this->input_buffer_size_); + // Zero-copy source reading directly from the ring buffer's internal storage. Raw file data is byte + // aligned, so no frame alignment is required. + auto source = RingBufferAudioSource::create(input_ring_buffer.lock(), this->input_buffer_size_); if (source == nullptr) { - return ESP_ERR_NO_MEM; + // create() only returns nullptr for invalid arguments (expired ring buffer or zero buffer size) + return ESP_ERR_INVALID_ARG; } - source->set_source(input_ring_buffer); this->input_buffer_ = std::move(source); return ESP_OK; } @@ -141,13 +146,7 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } FileDecoderState state = FileDecoderState::MORE_TO_PROCESS; - - uint32_t decoding_start = millis(); - - bool first_loop_iteration = true; - - size_t bytes_processed = 0; - size_t bytes_available_before_processing = 0; + uint8_t no_output_iterations = 0; while (state == FileDecoderState::MORE_TO_PROCESS) { // Transfer decoded out @@ -161,45 +160,39 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { this->playback_ms_ += this->audio_stream_info_.value().frames_to_milliseconds_with_remainder(&this->accumulated_frames_written_); } + + if ((bytes_written > 0) && (this->output_transfer_buffer_->available() == 0)) { + // All decoded audio has been flushed to the sink; return so the caller can react to stop/pause before + // decoding the next batch + return AudioDecoderState::DECODING; + } } else { // If paused, block to avoid wasting CPU resources delay(READ_WRITE_TIMEOUT_MS); } - // Verify there is enough space to store more decoded audio and that the function hasn't been running too long - if ((this->output_transfer_buffer_->free() < this->free_buffer_required_) || - (millis() - decoding_start > DECODING_TIMEOUT_MS)) { + if (this->output_transfer_buffer_->available() > 0) { + // Output transfer buffer indicates backpressure, return so caller can handle other events; + // e.g., stop/pause, before trying again return AudioDecoderState::DECODING; } - // Decode more audio - - // Never shift the input buffer; every decoder buffers internally and consumes only what it processed. - size_t bytes_read = this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - - if (!first_loop_iteration && (this->input_buffer_->available() < bytes_processed)) { - // Less data is available than what was processed in last iteration, so don't attempt to decode. - // This attempts to avoid the decoder from consistently trying to decode an incomplete frame. The transfer buffer - // will shift the remaining data to the start and copy more from the source the next time the decode function is - // called - break; + // Reaching here means no decoded output is pending (any would have returned above). Bounds long no-output + // stretches; e.g., skipping a large metadata block, so a source that keeps the ring buffer full can't spin this + // loop without yielding and trip the watchdog. The delay yields allowing other tasks to feed the watchdog and + // the return keeps stop/pause responsive. + if (++no_output_iterations >= MAX_NO_OUTPUT_ITERATIONS) { + delay(1); + return AudioDecoderState::DECODING; } - bytes_available_before_processing = this->input_buffer_->available(); + // Expose the next chunk of file data. Every decoder buffers internally and consumes only what it + // processed, so the source does not need to accumulate or stitch chunks across fill() calls. + this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - if ((this->potentially_failed_count_ > 0) && (bytes_read == 0)) { - // Failed to decode in last attempt and there is no new data + const size_t available_before_decode = this->input_buffer_->available(); - if ((this->input_buffer_->free() == 0) && first_loop_iteration) { - // The input buffer is full (or read-only, e.g. const flash source). Since it previously failed on the exact - // same data, we can never recover. For const sources this is correct: the entire file is already available, so - // a decode failure is genuine, not a transient out-of-data condition. - state = FileDecoderState::FAILED; - } else { - // Attempt to get more data next time - state = FileDecoderState::IDLE; - } - } else if (this->input_buffer_->available() == 0) { + if (available_before_decode == 0) { // No data to decode, attempt to get more data next time state = FileDecoderState::IDLE; } else { @@ -231,9 +224,6 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } } - first_loop_iteration = false; - bytes_processed = bytes_available_before_processing - this->input_buffer_->available(); - if (state == FileDecoderState::POTENTIALLY_FAILED) { ++this->potentially_failed_count_; } else if (state == FileDecoderState::END_OF_FILE) { @@ -241,7 +231,16 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } else if (state == FileDecoderState::FAILED) { return AudioDecoderState::FAILED; } else if (state == FileDecoderState::MORE_TO_PROCESS) { - this->potentially_failed_count_ = 0; + // Reset the failsafe only when the iteration made forward progress: input was consumed or output was + // produced (output_transfer_buffer_ is drained empty above, so any available bytes are new). A + // MORE_TO_PROCESS that neither consumes input nor produces output means the decoder is stalled; count it + // toward the failsafe so a stuck stream eventually surfaces as FAILED instead of looping forever. + if ((this->input_buffer_->available() < available_before_decode) || + (this->output_transfer_buffer_->available() > 0)) { + this->potentially_failed_count_ = 0; + } else { + ++this->potentially_failed_count_; + } } } return AudioDecoderState::DECODING; diff --git a/esphome/components/audio/audio_decoder.h b/esphome/components/audio/audio_decoder.h index c34ebbc613..e772b7eb5f 100644 --- a/esphome/components/audio/audio_decoder.h +++ b/esphome/components/audio/audio_decoder.h @@ -61,15 +61,16 @@ class AudioDecoder { */ public: /// @brief Allocates the output transfer buffer and stores the input buffer size for later use by add_source() - /// @param input_buffer_size Size of the input transfer buffer in bytes. + /// @param input_buffer_size Soft cap on the bytes a ring buffer source exposes per fill, in bytes. /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioDecoder(size_t input_buffer_size, size_t output_buffer_size); ~AudioDecoder() = default; - /// @brief Adds a source ring buffer for raw file data. Takes ownership of the ring buffer in a shared_ptr. - /// @param input_ring_buffer weak_ptr of a shared_ptr of the sink ring buffer to transfer ownership - /// @return ESP_OK if successsful, ESP_ERR_NO_MEM if the transfer buffer wasn't allocated + /// @brief Adds a source ring buffer for raw file data. Shares ownership of the ring buffer via a shared_ptr. + /// The decoder reads directly from the ring buffer's internal storage with a zero-copy RingBufferAudioSource. + /// @param input_ring_buffer weak_ptr of the source ring buffer to read from + /// @return ESP_OK if successful, ESP_ERR_INVALID_ARG if the ring buffer is expired or the buffer size is zero esp_err_t add_source(std::weak_ptr &input_ring_buffer); /// @brief Adds a sink ring buffer for decoded audio. Takes ownership of the ring buffer in a shared_ptr. From 747787ae98f3a19819aa1ecbbecafca558435267 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sun, 24 May 2026 15:34:15 -0400 Subject: [PATCH 13/96] [audio] Use RingBufferAudioSource for resampling (#16560) --- esphome/components/audio/audio_resampler.cpp | 61 +++++++++++++++----- esphome/components/audio/audio_resampler.h | 17 +++--- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/esphome/components/audio/audio_resampler.cpp b/esphome/components/audio/audio_resampler.cpp index c04cc881f5..bef62ce190 100644 --- a/esphome/components/audio/audio_resampler.cpp +++ b/esphome/components/audio/audio_resampler.cpp @@ -12,16 +12,17 @@ static const uint32_t READ_WRITE_TIMEOUT_MS = 20; AudioResampler::AudioResampler(size_t input_buffer_size, size_t output_buffer_size) : input_buffer_size_(input_buffer_size), output_buffer_size_(output_buffer_size) { - this->input_transfer_buffer_ = AudioSourceTransferBuffer::create(input_buffer_size); this->output_transfer_buffer_ = AudioSinkTransferBuffer::create(output_buffer_size); } esp_err_t AudioResampler::add_source(std::weak_ptr &input_ring_buffer) { - if (this->input_transfer_buffer_ != nullptr) { - this->input_transfer_buffer_->set_source(input_ring_buffer); - return ESP_OK; + // The zero-copy RingBufferAudioSource is created lazily on the first resample() call, once both the ring + // buffer (stored here) and the input stream info (set by start()) are available, in either order. + this->source_ring_buffer_ = input_ring_buffer.lock(); + if (this->source_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; } - return ESP_ERR_NO_MEM; + return ESP_OK; } esp_err_t AudioResampler::add_sink(std::weak_ptr &output_ring_buffer) { @@ -47,7 +48,7 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI this->input_stream_info_ = input_stream_info; this->output_stream_info_ = output_stream_info; - if ((this->input_transfer_buffer_ == nullptr) || (this->output_transfer_buffer_ == nullptr)) { + if (this->output_transfer_buffer_ == nullptr) { return ESP_ERR_NO_MEM; } @@ -56,6 +57,13 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI return ESP_ERR_NOT_SUPPORTED; } + // Reject frame sizes that can't be used as the zero-copy source's alignment up front, where the caller checks + // the return code. The lazy create() in resample() keeps its own guard since it runs before the uint8_t cast. + const size_t bytes_per_frame = this->input_stream_info_.frames_to_bytes(1); + if ((bytes_per_frame == 0) || (bytes_per_frame > RingBufferAudioSource::MAX_ALIGNMENT_BYTES)) { + return ESP_ERR_NOT_SUPPORTED; + } + if ((input_stream_info.get_sample_rate() != output_stream_info.get_sample_rate()) || (input_stream_info.get_bits_per_sample() != output_stream_info.get_bits_per_sample())) { this->resampler_ = make_unique( @@ -87,8 +95,27 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI } AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_differential) { + if (this->audio_source_ == nullptr) { + // Lazily create the zero-copy source on first use. Frame-aligned reads ensure multi-channel frames are + // never split across the ring buffer's wrap boundary. + const size_t bytes_per_frame = this->input_stream_info_.frames_to_bytes(1); + if ((bytes_per_frame == 0) || (bytes_per_frame > RingBufferAudioSource::MAX_ALIGNMENT_BYTES)) { + // Stream info is unset or the frame is too large to use as an alignment; the uint8_t cast below would + // truncate it and could yield a source that tears frames. + return AudioResamplerState::FAILED; + } + // Pass the shared_ptr by copy so a failed create() leaves source_ring_buffer_ intact; release our + // reference only after the source has taken ownership. + this->audio_source_ = RingBufferAudioSource::create(this->source_ring_buffer_, this->input_buffer_size_, + static_cast(bytes_per_frame)); + if (this->audio_source_ == nullptr) { + return AudioResamplerState::FAILED; + } + this->source_ring_buffer_.reset(); + } + if (stop_gracefully) { - if (!this->input_transfer_buffer_->has_buffered_data() && (this->output_transfer_buffer_->available() == 0)) { + if (!this->audio_source_->has_buffered_data() && (this->output_transfer_buffer_->available() == 0)) { return AudioResamplerState::FINISHED; } } @@ -102,9 +129,11 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d delay(READ_WRITE_TIMEOUT_MS); } - this->input_transfer_buffer_->transfer_data_from_source(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS)); + // Expose a chunk of the ring buffer's internal storage. pre_shift is ignored by RingBufferAudioSource + // (there is no intermediate transfer buffer to compact). + this->audio_source_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - if (this->input_transfer_buffer_->available() == 0) { + if (this->audio_source_->available() == 0) { // No samples available to process return AudioResamplerState::RESAMPLING; } @@ -112,17 +141,17 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d const size_t bytes_free = this->output_transfer_buffer_->free(); const uint32_t frames_free = this->output_stream_info_.bytes_to_frames(bytes_free); - const size_t bytes_available = this->input_transfer_buffer_->available(); + const size_t bytes_available = this->audio_source_->available(); const uint32_t frames_available = this->input_stream_info_.bytes_to_frames(bytes_available); if ((this->input_stream_info_.get_sample_rate() != this->output_stream_info_.get_sample_rate()) || (this->input_stream_info_.get_bits_per_sample() != this->output_stream_info_.get_bits_per_sample())) { // Adjust gain by -3 dB to avoid clipping due to the resampling process esp_audio_libs::resampler::ResamplerResults results = - this->resampler_->resample(this->input_transfer_buffer_->get_buffer_start(), - this->output_transfer_buffer_->get_buffer_end(), frames_available, frames_free, -3); + this->resampler_->resample(this->audio_source_->data(), this->output_transfer_buffer_->get_buffer_end(), + frames_available, frames_free, -3); - this->input_transfer_buffer_->decrease_buffer_length(this->input_stream_info_.frames_to_bytes(results.frames_used)); + this->audio_source_->consume(this->input_stream_info_.frames_to_bytes(results.frames_used)); this->output_transfer_buffer_->increase_buffer_length( this->output_stream_info_.frames_to_bytes(results.frames_generated)); @@ -146,10 +175,10 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d const size_t bytes_to_transfer = std::min(this->output_stream_info_.frames_to_bytes(frames_free), this->input_stream_info_.frames_to_bytes(frames_available)); - std::memcpy((void *) this->output_transfer_buffer_->get_buffer_end(), - (void *) this->input_transfer_buffer_->get_buffer_start(), bytes_to_transfer); + std::memcpy((void *) this->output_transfer_buffer_->get_buffer_end(), (const void *) this->audio_source_->data(), + bytes_to_transfer); - this->input_transfer_buffer_->decrease_buffer_length(bytes_to_transfer); + this->audio_source_->consume(bytes_to_transfer); this->output_transfer_buffer_->increase_buffer_length(bytes_to_transfer); } diff --git a/esphome/components/audio/audio_resampler.h b/esphome/components/audio/audio_resampler.h index 575ad13692..c09070c0ce 100644 --- a/esphome/components/audio/audio_resampler.h +++ b/esphome/components/audio/audio_resampler.h @@ -22,7 +22,7 @@ namespace esphome::audio { enum class AudioResamplerState : uint8_t { RESAMPLING, // More data is available to resample FINISHED, // All file data has been resampled and transferred - FAILED, // Unused state included for consistency among Audio classes + FAILED, // Failed to allocate the audio source }; class AudioResampler { @@ -32,14 +32,16 @@ class AudioResampler { * component). Also supports converting bits per sample. */ public: - /// @brief Allocates the input and output transfer buffers - /// @param input_buffer_size Size of the input transfer buffer in bytes. + /// @brief Allocates the output transfer buffer. The input source is created later in resample(). + /// @param input_buffer_size Max bytes exposed per fill() call on the zero-copy input source. /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioResampler(size_t input_buffer_size, size_t output_buffer_size); - /// @brief Adds a source ring buffer for audio data. Takes ownership of the ring buffer in a shared_ptr. - /// @param input_ring_buffer weak_ptr of a shared_ptr of the sink ring buffer to transfer ownership - /// @return ESP_OK if successsful, ESP_ERR_NO_MEM if the transfer buffer wasn't allocated + /// @brief Sets the ring buffer the audio is read from and takes shared ownership of it. The zero-copy + /// RingBufferAudioSource that reads directly from its internal storage is created lazily on the first + /// resample() call, so add_source() and start() may be called in any order. + /// @param input_ring_buffer weak_ptr of a shared_ptr of the source ring buffer to transfer ownership + /// @return ESP_OK if successful, ESP_ERR_INVALID_STATE if the ring buffer is no longer alive esp_err_t add_source(std::weak_ptr &input_ring_buffer); /// @brief Adds a sink ring buffer for resampled audio. Takes ownership of the ring buffer in a shared_ptr. @@ -78,7 +80,8 @@ class AudioResampler { void set_pause_output_state(bool pause_state) { this->pause_output_ = pause_state; } protected: - std::unique_ptr input_transfer_buffer_; + std::shared_ptr source_ring_buffer_; + std::unique_ptr audio_source_; std::unique_ptr output_transfer_buffer_; size_t input_buffer_size_; From 9fcb638f33fcd8814cb4aa5fb2817b5480b73be8 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Sun, 24 May 2026 15:34:51 -0400 Subject: [PATCH 14/96] [micro_wake_word] Use RingBufferAudioSource (#16595) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../micro_wake_word/micro_wake_word.cpp | 108 ++++++++++-------- .../micro_wake_word/micro_wake_word.h | 17 +-- 2 files changed, 71 insertions(+), 54 deletions(-) diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 6877e9e5df..739d64dc28 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -33,7 +33,8 @@ static const uint32_t INFERENCE_TASK_STACK_SIZE = 3072; static const UBaseType_t INFERENCE_TASK_PRIORITY = 3; enum EventGroupBits : uint32_t { - COMMAND_STOP = (1 << 0), // Signals the inference task should stop + COMMAND_STOP = (1 << 0), // Signals the inference task should stop + COMMAND_RESET_RING_BUFFER = (1 << 1), // Signals the inference task to discard buffered audio TASK_STARTING = (1 << 3), TASK_RUNNING = (1 << 4), @@ -114,13 +115,13 @@ void MicroWakeWord::setup() { } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); if (this->ring_buffer_.use_count() > 1) { - size_t bytes_free = temp_ring_buffer->free(); - - if (bytes_free < data.size()) { - xEventGroupSetBits(this->event_group_, EventGroupBits::WARNING_FULL_RING_BUFFER); - temp_ring_buffer->reset(); + // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task + // to drain it - reset() is a consumer operation and must run on the inference task's thread. + // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. + if (temp_ring_buffer->write_without_replacement(data.data(), data.size(), 0, false) == 0) { + xEventGroupSetBits(this->event_group_, + EventGroupBits::WARNING_FULL_RING_BUFFER | EventGroupBits::COMMAND_RESET_RING_BUFFER); } - temp_ring_buffer->write((void *) data.data(), data.size()); } }); @@ -146,56 +147,65 @@ void MicroWakeWord::inference_task(void *params) { { // Ensures any C++ objects fall out of scope to deallocate before deleting the task - const size_t new_bytes_to_process = - this_mww->microphone_source_->get_audio_stream_info().ms_to_bytes(this_mww->features_step_size_); - std::unique_ptr audio_buffer; + const auto &stream_info = this_mww->microphone_source_->get_audio_stream_info(); + const size_t bytes_per_frame = stream_info.frames_to_bytes(1); + const size_t max_fill_bytes = stream_info.ms_to_bytes(this_mww->features_step_size_); + std::unique_ptr audio_source; int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]; if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { - // Allocate audio transfer buffer - audio_buffer = audio::AudioSourceTransferBuffer::create(new_bytes_to_process); - - if (audio_buffer == nullptr) { + // Round ring buffer size down to a frame multiple so the wrap boundary never splits an int16 sample. + const size_t ring_buffer_size = + (stream_info.ms_to_bytes(RING_BUFFER_DURATION_MS) / bytes_per_frame) * bytes_per_frame; + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); + if (temp_ring_buffer == nullptr) { xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); + } else { + audio_source = audio::RingBufferAudioSource::create(temp_ring_buffer, max_fill_bytes, + static_cast(bytes_per_frame)); + if (audio_source == nullptr) { + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); + } else { + this_mww->ring_buffer_ = temp_ring_buffer; + } } } - if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { - // Allocate ring buffer - std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( - this_mww->microphone_source_->get_audio_stream_info().ms_to_bytes(RING_BUFFER_DURATION_MS)); - if (temp_ring_buffer.use_count() == 0) { - xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); - } - audio_buffer->set_source(temp_ring_buffer); - this_mww->ring_buffer_ = temp_ring_buffer; - } - if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { this_mww->microphone_source_->start(); xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_RUNNING); - while (!(xEventGroupGetBits(this_mww->event_group_) & COMMAND_STOP)) { - audio_buffer->transfer_data_from_source(pdMS_TO_TICKS(DATA_TIMEOUT_MS)); - - if (audio_buffer->available() < new_bytes_to_process) { - // Insufficient data to generate new spectrogram features, read more next iteration - continue; + while (!(xEventGroupGetBits(this_mww->event_group_) & (COMMAND_STOP | ERROR_BITS))) { + if (xEventGroupGetBits(this_mww->event_group_) & EventGroupBits::COMMAND_RESET_RING_BUFFER) { + // Producer asked us to drain; run the consumer-side reset from this thread. + audio_source->clear_buffered_data(); + xEventGroupClearBits(this_mww->event_group_, EventGroupBits::COMMAND_RESET_RING_BUFFER); } - // Generate new spectrogram features - uint32_t processed_samples = this_mww->generate_features_( - (int16_t *) audio_buffer->get_buffer_start(), audio_buffer->available() / sizeof(int16_t), features_buffer); - audio_buffer->decrease_buffer_length(processed_samples * sizeof(int16_t)); + audio_source->fill(pdMS_TO_TICKS(DATA_TIMEOUT_MS), false); - // Run inference using the new spectorgram features - if (!this_mww->update_model_probabilities_(features_buffer)) { - xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_INFERENCE); - break; + // The frontend buffers samples internally and only emits a feature once it has a full window, so we can + // hand it whatever the source exposes. The frontend consumes at least one sample per call, so available() + // strictly decreases and this loop always terminates. + while (audio_source->available() >= sizeof(int16_t)) { + const size_t samples_available = audio_source->available() / sizeof(int16_t); + const int16_t *audio_data = reinterpret_cast(audio_source->data()); + + size_t processed_samples = 0; + const bool feature_generated = + this_mww->generate_features_(audio_data, samples_available, features_buffer, &processed_samples); + audio_source->consume(processed_samples * sizeof(int16_t)); + + if (feature_generated) { + if (!this_mww->update_model_probabilities_(features_buffer)) { + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_INFERENCE); + break; + } + + // Process each model's probabilities and possibly send a Detection Event to the queue + this_mww->process_probabilities_(); + } } - - // Process each model's probabilities and possibly send a Detection Event to the queue - this_mww->process_probabilities_(); } } } @@ -386,11 +396,15 @@ void MicroWakeWord::set_state_(State state) { } } -size_t MicroWakeWord::generate_features_(int16_t *audio_buffer, size_t samples_available, - int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]) { - size_t processed_samples = 0; +bool MicroWakeWord::generate_features_(const int16_t *audio_buffer, size_t samples_available, + int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE], size_t *processed_samples) { + *processed_samples = 0; struct FrontendOutput frontend_output = - FrontendProcessSamples(&this->frontend_state_, audio_buffer, samples_available, &processed_samples); + FrontendProcessSamples(&this->frontend_state_, audio_buffer, samples_available, processed_samples); + + if (frontend_output.size == 0) { + return false; + } for (size_t i = 0; i < frontend_output.size; ++i) { // These scaling values are set to match the TFLite audio frontend int8 output. @@ -415,7 +429,7 @@ size_t MicroWakeWord::generate_features_(int16_t *audio_buffer, size_t samples_a features_buffer[i] = static_cast(clamp(value, INT8_MIN, INT8_MAX)); } - return processed_samples; + return true; } void MicroWakeWord::process_probabilities_() { diff --git a/esphome/components/micro_wake_word/micro_wake_word.h b/esphome/components/micro_wake_word/micro_wake_word.h index 5c0c056ac0..ef440b5d37 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.h +++ b/esphome/components/micro_wake_word/micro_wake_word.h @@ -115,13 +115,16 @@ class MicroWakeWord : public Component void set_state_(State state); - /// @brief Generates spectrogram features from an input buffer of audio samples - /// @param audio_buffer (int16_t *) Buffer containing input audio samples - /// @param samples_available (size_t) Number of samples avaiable in the input buffer - /// @param features_buffer (int8_t *) Buffer to store generated features - /// @return (size_t) Number of samples processed from the input buffer - size_t generate_features_(int16_t *audio_buffer, size_t samples_available, - int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]); + /// @brief Generates a spectrogram feature from an input buffer of audio samples. The frontend buffers samples + /// internally, so callers may stream arbitrary-sized chunks; a feature is only emitted once enough samples have + /// accumulated to fill a full analysis window. + /// @param audio_buffer (const int16_t *) Buffer containing input audio samples + /// @param samples_available (size_t) Number of samples available in the input buffer + /// @param features_buffer (int8_t *) Buffer to store the generated feature, valid only when the return value is true + /// @param processed_samples (size_t *) Set to the number of samples consumed from the input buffer + /// @return True if a new feature was generated; false if more samples are required + bool generate_features_(const int16_t *audio_buffer, size_t samples_available, + int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE], size_t *processed_samples); /// @brief Processes any new probabilities for each model. If any wake word is detected, it will send a DetectionEvent /// to the detection_queue_. From 16b6509a0348eecc22c51066049052a3c9383fc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 08:28:09 -0500 Subject: [PATCH 15/96] Bump zeroconf from 0.149.12 to 0.149.13 (#16520) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4338063387..bd775d16a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 aioesphomeapi==45.0.4 -zeroconf==0.149.12 +zeroconf==0.149.13 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From 79539cb85d937238ce977372c0953fb6bbde8ff5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 09:55:28 -0500 Subject: [PATCH 16/96] Bump zeroconf from 0.149.13 to 0.149.16 (#16533) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd775d16a1..1174f957a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 aioesphomeapi==45.0.4 -zeroconf==0.149.13 +zeroconf==0.149.16 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From cc88456ce7f779c2e9b2413ae9b02bbe0bfec1d9 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 11:35:51 -0400 Subject: [PATCH 17/96] [espidf] Filter noisy 'git rev-parse' errors when .git is stripped (#16521) --- esphome/espidf/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index 65df37c7b2..da3f77cdd3 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -66,6 +66,12 @@ FILTER_IDF_LINES: list[str] = [ # Drop the blank line rich emits after the note so the build log # doesn't end with an orphan gap before ESPHome's own status lines. r"\s*$", + # ESP-IDF shells out to ``git rev-parse`` to embed a commit hash; + # esphome-libs strips ``.git`` from the tarball so those probes fail + # noisily without affecting the build. + r"-- git rev-parse returned ", + r"fatal: not a git repository", + r"Stopping at filesystem boundary", ] From 32fa856bf0597f7cecfdc80b838de6add3c15730 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 11:36:27 -0400 Subject: [PATCH 18/96] [espidf] Fix tarfile extract crashing on Python 3.11 with None mode (#16530) --- esphome/espidf/framework.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index aa97c65227..0e841af7d7 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -546,11 +546,11 @@ def _tar_extract_all( if not (mode & stat.S_IXUSR): mode &= ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) mode |= stat.S_IRUSR | stat.S_IWUSR - elif member.isdir() or member.issym(): - # Ignore mode for directories & symlinks - mode = None - else: - # Block special files + elif not (member.isdir() or member.issym()): + # Block special files. Directories and symlinks keep + # their masked-original mode — passing None here would + # crash tarfile.extract on Python <3.12 (its chmod + # path calls os.chmod unconditionally). continue member.mode = mode From e92a4c9472c8bf726e51b3213ae43a7b55010090 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 11:37:10 -0400 Subject: [PATCH 19/96] [espidf] Write version.txt after extract so bootloader shows the real version (#16532) --- esphome/espidf/framework.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 0e841af7d7..6710c632c4 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -780,6 +780,34 @@ def download_from_mirrors( return None +def _write_idf_version_txt(framework_path: Path, version: str) -> None: + """Write /version.txt if missing. + + IDF's build.cmake picks the version it embeds in the firmware (and + stamps onto the bootloader) in this order: ``${IDF_PATH}/version.txt`` + if present, else ``git describe`` against IDF_PATH, else the + ``IDF_VERSION_MAJOR/MINOR/PATCH`` triplet from ``tools/cmake/version.cmake``. + On a clean esphome-libs tarball ``.git`` is fully stripped, so + git_describe returns ``HEAD-HASH-NOTFOUND`` (falsy) and the triplet + wins -- correct by luck. But a *partial* ``.git`` (e.g. a custom + framework.source pointed at a real git URL where build artifacts + mark the tree dirty) makes git_describe return ``-dirty``, + which is what then gets baked into the bootloader. Dropping + version.txt forces the right answer regardless. + """ + version_txt = framework_path / "version.txt" + if version_txt.exists(): + return + try: + version_txt.write_text(f"v{version}\n", encoding="utf-8") + except OSError as e: + _LOGGER.warning( + "Could not write %s (%s); bootloader version string may be incorrect.", + version_txt, + e, + ) + + def _check_esphome_idf_framework_install( version: str, targets: list[str], @@ -861,6 +889,11 @@ def _check_esphome_idf_framework_install( archive_extract_all(tmp.file, framework_path, progress_header="Extracting") extracted_marker.touch() + # Idempotent post-extract patch: written every invocation so a build + # dir extracted before this fix gets the file too, without forcing a + # clean. Skips when version.txt already exists. + _write_idf_version_txt(framework_path, version) + # 3. Check if the framework tools are the same and correctly installed if not install: install = True From 615d5aa827560675c5531e77f4d76ec9e4d5971c Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 11:37:46 -0400 Subject: [PATCH 20/96] [core] Persist & restore CORE.toolchain through StorageJSON (#16531) --- esphome/storage_json.py | 20 ++++++++ tests/unit_tests/test_storage_json.py | 73 ++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/esphome/storage_json.py b/esphome/storage_json.py index e481827080..7f8885ba5f 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -14,6 +14,7 @@ from esphome.const import ( KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + Toolchain, ) from esphome.core import CORE from esphome.helpers import write_file_if_changed @@ -98,6 +99,7 @@ class StorageJSON: no_mdns: bool, framework: str | None = None, core_platform: str | None = None, + toolchain: str | None = None, ) -> None: # Version of the storage JSON schema assert storage_version is None or isinstance(storage_version, int) @@ -134,6 +136,8 @@ class StorageJSON: self.framework = framework # The core platform of this firmware. Like "esp32", "rp2040", "host" etc. self.core_platform = core_platform + # The toolchain used for the build ("platformio" / "esp-idf") + self.toolchain = toolchain def as_dict(self): return { @@ -153,6 +157,7 @@ class StorageJSON: "no_mdns": self.no_mdns, "framework": self.framework, "core_platform": self.core_platform, + "toolchain": self.toolchain, } def to_json(self): @@ -189,6 +194,7 @@ class StorageJSON: ), framework=esph.target_framework, core_platform=esph.target_platform, + toolchain=esph.toolchain.value if esph.toolchain is not None else None, ) @staticmethod @@ -236,6 +242,7 @@ class StorageJSON: no_mdns = storage.get("no_mdns", False) framework = storage.get("framework") core_platform = storage.get("core_platform") + toolchain = storage.get("toolchain") return StorageJSON( storage_version, name, @@ -253,6 +260,7 @@ class StorageJSON: no_mdns, framework, core_platform, + toolchain, ) @staticmethod @@ -273,6 +281,18 @@ class StorageJSON: """ CORE.name = self.name CORE.build_path = self.build_path + # Restore toolchain so upload/logs picks the right firmware_bin path. + # An unknown value (corrupt sidecar, or written by a newer ESPHome) + # just leaves CORE.toolchain None — the fallback then picks PlatformIO. + if self.toolchain and CORE.toolchain is None: + try: + CORE.toolchain = Toolchain(self.toolchain) + except ValueError: + _LOGGER.debug( + "Ignoring unknown toolchain %r from %s", + self.toolchain, + storage_path(), + ) target_platform = self.core_platform or self.target_platform.lower() CORE.data[KEY_CORE] = { KEY_TARGET_PLATFORM: target_platform, diff --git a/tests/unit_tests/test_storage_json.py b/tests/unit_tests/test_storage_json.py index a3a38960e7..ea37492cf4 100644 --- a/tests/unit_tests/test_storage_json.py +++ b/tests/unit_tests/test_storage_json.py @@ -9,7 +9,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest from esphome import storage_json -from esphome.const import CONF_DISABLED, CONF_MDNS +from esphome.const import CONF_DISABLED, CONF_MDNS, Toolchain from esphome.core import CORE @@ -308,6 +308,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: mock_core.loaded_platforms = {"sensor"} mock_core.config = {CONF_MDNS: {CONF_DISABLED: True}} mock_core.target_framework = "esp-idf" + mock_core.toolchain = Toolchain.ESP_IDF with patch("esphome.components.esp32.get_esp32_variant") as mock_variant: mock_variant.return_value = "ESP32-C3" @@ -327,6 +328,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: assert result.no_mdns is True assert result.framework == "esp-idf" assert result.core_platform == "esp32" + assert result.toolchain == "esp-idf" def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None: @@ -345,10 +347,12 @@ def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None: mock_core.loaded_platforms = set() mock_core.config = {} # No MDNS config means enabled mock_core.target_framework = "arduino" + mock_core.toolchain = None result = storage_json.StorageJSON.from_esphome_core(mock_core, old=None) assert result.no_mdns is False + assert result.toolchain is None def test_storage_json_load_valid_file(tmp_path: Path) -> None: @@ -470,6 +474,73 @@ def test_storage_json_equality() -> None: assert storage1 != "not a storage object" +def _make_storage_with_toolchain( + toolchain: str | None, +) -> storage_json.StorageJSON: + return storage_json.StorageJSON( + storage_version=1, + name="dev", + friendly_name=None, + comment=None, + esphome_version="2024.1.0", + src_version=1, + address="dev.local", + web_port=None, + target_platform="ESP32", + build_path=Path("/build"), + firmware_bin_path=Path("/build/firmware.bin"), + loaded_integrations=set(), + loaded_platforms=set(), + no_mdns=False, + framework="esp-idf", + core_platform="esp32", + toolchain=toolchain, + ) + + +def test_storage_json_toolchain_round_trip(setup_core: Path) -> None: + """Sidecar toolchain survives save -> load -> apply_to_core.""" + storage = _make_storage_with_toolchain("esp-idf") + path = setup_core / "storage.json" + path.write_text(storage.to_json()) + + # Serialization key is stable -- device-builder relies on it. + assert json.loads(path.read_text())["toolchain"] == "esp-idf" + + loaded = storage_json.StorageJSON.load(path) + assert loaded is not None + assert loaded.toolchain == "esp-idf" + + CORE.toolchain = None + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain == Toolchain.ESP_IDF + + +def test_storage_json_apply_to_core_preserves_cli_toolchain( + setup_core: Path, +) -> None: + """A CLI-set CORE.toolchain wins over the sidecar value.""" + loaded = _make_storage_with_toolchain("esp-idf") + + CORE.toolchain = Toolchain.PLATFORMIO + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain == Toolchain.PLATFORMIO + + +def test_storage_json_apply_to_core_ignores_unknown_toolchain( + setup_core: Path, +) -> None: + """Unknown enum values (corrupt sidecar / newer ESPHome) fall through to None.""" + loaded = _make_storage_with_toolchain("gcc") + + CORE.toolchain = None + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain is None + + def test_esphome_storage_json_as_dict() -> None: """Test EsphomeStorageJSON.as_dict returns correct dictionary.""" storage = storage_json.EsphomeStorageJSON( From 522541634738b95ac91d8266be67f2dd509e97c5 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 12:05:27 -0400 Subject: [PATCH 21/96] [espidf] Backport ninja linux-arm64 entry into tools.json on aarch64 hosts (#16527) --- esphome/espidf/framework.py | 76 ++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 6710c632c4..ee612c4c7e 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -7,6 +7,7 @@ import json import logging import os from pathlib import Path +import platform import shutil import subprocess import sys @@ -17,7 +18,7 @@ import requests from esphome.config_validation import Version from esphome.core import CORE -from esphome.helpers import ProgressBar, get_str_env, rmtree +from esphome.helpers import ProgressBar, get_str_env, rmtree, write_file_if_changed PathType = str | os.PathLike @@ -808,6 +809,74 @@ def _write_idf_version_txt(framework_path: Path, version: str) -> None: ) +# Backport of espressif/esp-idf#18272: every ESPHome-supported IDF release +# through v6.0 ships a tools.json whose ninja 1.12.1 entry has no +# ``linux-arm64`` source. ``idf_tools.py`` then either fails to find a +# matching binary or grabs the x86_64 one, which can't execute on +# aarch64. cmake is already populated across the same release range; we +# only need to inject ninja. Values lifted verbatim from the IDF v6.0.1 +# tools.json where the fix landed natively. +_NINJA_ARM64_BACKPORT: dict[str, dict[str, str | int]] = { + "1.12.1": { + "rename_dist": "ninja-linux-arm64-v1.12.1.zip", + "sha256": "5c25c6570b0155e95fce5918cb95f1ad9870df5768653afe128db822301a05a1", + "size": 121787, + "url": "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux-aarch64.zip", + }, +} + + +def _patch_tools_json_for_linux_arm64(framework_path: Path) -> None: + """Inject ninja linux-arm64 entries into the framework's tools.json on aarch64. + + Idempotent: a tools.json that already has the entry, or a host that + isn't aarch64, is a no-op. Applied unconditionally on every install + check so a build dir extracted before the backport got fixed up + without forcing a clean. + """ + if platform.machine() != "aarch64": + return + + tools_json = framework_path / "tools" / "tools.json" + if not tools_json.is_file(): + return + + try: + with open(tools_json, encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + _LOGGER.warning( + "Could not parse %s for linux-arm64 backport (%s); " + "skipping. A clean reinstall of the framework directory " + "may be needed.", + tools_json, + e, + ) + return + + changed = False + for tool in data.get("tools", []): + if tool.get("name") != "ninja": + continue + for ver in tool.get("versions", []): + entry = _NINJA_ARM64_BACKPORT.get(ver.get("name")) + if entry is None or ver.get("linux-arm64"): + continue + ver["linux-arm64"] = entry + changed = True + + if changed: + # write_file_if_changed stages a tempfile in the destination dir + # and atomically replaces — safe against mid-write interruption + # and concurrent invocations. + write_file_if_changed(tools_json, json.dumps(data, indent=2) + "\n") + _LOGGER.info( + "Patched %s to add ninja linux-arm64 download " + "(espressif/esp-idf#18272 backport).", + tools_json, + ) + + def _check_esphome_idf_framework_install( version: str, targets: list[str], @@ -894,6 +963,11 @@ def _check_esphome_idf_framework_install( # clean. Skips when version.txt already exists. _write_idf_version_txt(framework_path, version) + # Apply the ninja linux-arm64 backport on every invocation, not just on + # fresh extracts — idempotent and cheap, and lets a build dir carrying + # a pre-patch tools.json get fixed up without forcing a clean. + _patch_tools_json_for_linux_arm64(framework_path) + # 3. Check if the framework tools are the same and correctly installed if not install: install = True From 858cfd5b94f48e8120a40c17bb31dab5c3408cdb Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 14:01:19 -0400 Subject: [PATCH 22/96] [espidf] Default to remote HEAD when cg.add_library URL has no #ref (#16535) --- esphome/espidf/component.py | 15 ++++++++------- tests/unit_tests/test_espidf_component.py | 16 +++++++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index b1352f7791..7d9874ad5f 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -93,7 +93,7 @@ class URLSource(Source): class GitSource(Source): - def __init__(self, url: str, ref: str): + def __init__(self, url: str, ref: str | None): self.url = url self.ref = ref @@ -109,7 +109,7 @@ class GitSource(Source): return path def __str__(self): - return f"{self.url}#{self.ref}" + return f"{self.url}#{self.ref}" if self.ref else self.url class InvalidIDFComponent(Exception): @@ -352,7 +352,6 @@ def _convert_library_to_component(library: Library) -> IDFComponent: IDFComponent: The resolved component with name, version, and URL Raises: - ValueError: If a repository URL is missing a reference (#) RuntimeError: If no artifact can be found for the library """ name = None @@ -362,10 +361,11 @@ def _convert_library_to_component(library: Library) -> IDFComponent: # Repository is provided directly if library.repository: # Parse repository URL: path becomes the component name, fragment - # becomes the git ref stored on GitSource. + # (if any) becomes the git ref stored on GitSource. A missing + # fragment is fine -- clone_or_update leaves the depth-1 clone on + # the remote's default branch, matching PIO's lib_deps behavior + # and external_components handling. split_result = urlsplit(library.repository) - if not split_result.fragment.strip(): - raise ValueError(f"Missing ref in URL {library.repository}") # Sanitize name name = str(split_result.path).strip("/") @@ -377,7 +377,8 @@ def _convert_library_to_component(library: Library) -> IDFComponent: version = "*" repository = urlunsplit(split_result._replace(fragment="")) - source = GitSource(str(repository), split_result.fragment) + ref = split_result.fragment.strip() or None + source = GitSource(str(repository), ref) # Version is provided - resolve using PlatformIO registry elif library.version: diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 373432f7d2..c4a419d1a2 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -436,11 +436,21 @@ def test_convert_library_with_branch_ref(): assert result.source.ref == "some-branch" -def test_convert_library_missing_ref(): +def test_convert_library_missing_ref_uses_default_branch(): + """A bare URL with no #ref clones the remote's default branch. + + Matches PIO's lib_deps behavior and external_components handling -- + git.clone_or_update with ref=None leaves the depth-1 clone on + whatever branch the remote HEAD points at. + """ lib = Library("name", None, "https://github.com/foo/bar.git") - with pytest.raises(ValueError): - _convert_library_to_component(lib) + result = _convert_library_to_component(lib) + + assert result.name == "foo/bar" + assert result.version == "*" + assert isinstance(result.source, GitSource) + assert result.source.ref is None def test_convert_library_registry(monkeypatch): From 878027ff50ea1760ff08c92189247ec28f8f5d64 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 14:01:54 -0400 Subject: [PATCH 23/96] [espidf] Honor the dict shorthand for library.json dependencies (#16537) --- esphome/espidf/component.py | 20 ++++ tests/unit_tests/test_espidf_component.py | 110 ++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 7d9874ad5f..a452a3f34a 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -656,6 +656,26 @@ def _process_dependencies(component: IDFComponent): if not dependencies: return + # PIO's library.json accepts both the list-of-dicts form and the + # shorthand dict form ``{"owner/Name": "version_spec"}``. Normalize + # the dict form so the loop below sees a uniform list. Iterating a + # dict gives string keys, which would silently fail the + # ``"name" in dependency`` substring check and skip every entry. + if isinstance(dependencies, dict): + normalized = [] + for raw_name, spec in dependencies.items(): + if "/" in raw_name: + owner, pkgname = raw_name.split("/", 1) + else: + owner, pkgname = None, raw_name + entry = {"name": pkgname, "owner": owner} + if isinstance(spec, dict): + entry.update(spec) + else: + entry["version"] = spec + normalized.append(entry) + dependencies = normalized + _LOGGER.info("Processing %s@%s component dependencies...", name, version) for dependency in dependencies: # Validate dependency structure diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index c4a419d1a2..7d6c861ffd 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -505,3 +505,113 @@ def test_process_dependencies_skips_invalid(tmp_component): _process_dependencies(tmp_component) assert tmp_component.dependencies == [] + + +def test_process_dependencies_dict_form(tmp_component, monkeypatch): + """PIO library.json shorthand ``{"owner/Name": "version"}`` is honored. + + Iterating a dict gives string keys, which would silently fail the + ``"name" in dependency`` substring check. Normalize to list-of-dicts + first so the dict form (used by e.g. tesla-ble for its nanopb dep) + is treated the same as the verbose list form. + """ + captured: list[Library] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent( + library.name, library.version, source=URLSource("http://dummy.com") + ) + + tmp_component.data = { + "dependencies": { + "nanopb/Nanopb": "^0.4.91", + "BareName": "1.2.3", + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) + + _process_dependencies(tmp_component) + + assert len(tmp_component.dependencies) == 2 + names = sorted(lib.name for lib in captured) + versions = sorted(lib.version for lib in captured) + assert names == ["BareName", "nanopb/Nanopb"] + assert versions == ["1.2.3", "^0.4.91"] + + +def test_process_dependencies_dict_form_with_url_value(tmp_component, monkeypatch): + """A dict-value that's a URL gets routed to ``repository`` like the list form.""" + captured: list[Library] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent(library.name, "*", source=URLSource("http://dummy.com")) + + tmp_component.data = { + "dependencies": { + "foo/Bar": "https://github.com/foo/bar.git#main", + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) + + _process_dependencies(tmp_component) + + assert len(captured) == 1 + assert captured[0].name == "foo/Bar" + assert captured[0].version is None + assert captured[0].repository == "https://github.com/foo/bar.git#main" + + +def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypatch): + """A dict-value that's itself a dict is merged into the entry. + + PIO's library.json allows ``{"owner/Name": {"version": "...", ...}}`` + for entries that need fields beyond just a version (platforms, + frameworks, etc.). The extra fields flow into _check_library_data + via the entry merge. + """ + captured: list[Library] = [] + checked: list[dict] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent( + library.name, library.version, source=URLSource("http://dummy.com") + ) + + tmp_component.data = { + "dependencies": { + "nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}, + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr( + esphome.espidf.component, + "_check_library_data", + checked.append, + ) + + _process_dependencies(tmp_component) + + assert len(captured) == 1 + assert captured[0].name == "nanopb/Nanopb" + assert captured[0].version == "^0.4.91" + # Extra spec fields reach _check_library_data so platform/framework + # gating still applies. + assert checked == [ + { + "name": "Nanopb", + "owner": "nanopb", + "version": "^0.4.91", + "platforms": "espidf", + } + ] From e6ed27574625d7d6706fab7243e3f6ee2b8999b6 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 14:03:55 -0400 Subject: [PATCH 24/96] [esp32] Defer esp_panic_handler wrap so arduino-esp32 IDF component skips it (#16538) --- esphome/components/esp32/__init__.py | 35 +++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 1db97f95eb..2fdaa991ae 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -113,6 +113,7 @@ ARDUINO_FRAMEWORK_NAME = "framework-arduinoespressif32" ARDUINO_FRAMEWORK_PKG = f"pioarduino/{ARDUINO_FRAMEWORK_NAME}" ARDUINO_LIBS_NAME = f"{ARDUINO_FRAMEWORK_NAME}-libs" ARDUINO_LIBS_PKG = f"pioarduino/{ARDUINO_LIBS_NAME}" +ARDUINO_ESP32_COMPONENT_NAME = "espressif/arduino-esp32" LOG_LEVELS_IDF = [ "NONE", @@ -1743,6 +1744,31 @@ async def _add_yaml_idf_components(components: list[ConfigType]): ) +@coroutine_with_priority(CoroPriority.FINAL - 1) +async def _finalize_arduino_aware_flags(): + """Build flags that depend on whether arduino-esp32 is linked in. + + Scheduler runs lower priority values later, so ``FINAL - 1`` fires + after every ``FINAL`` job (incl. ``_add_yaml_idf_components``) -- + by then ``KEY_COMPONENTS`` is fully populated. + + - Skip our esp_panic_handler wrap when Arduino is linked; Arduino + wraps the same symbol and the linker errors on the duplicate. + - Define USE_ARDUINO in the hybrid esp-idf+arduino-esp32-component + case so ESPHome's ``#ifdef USE_ARDUINO`` paths light up. The + framework=arduino branch already adds it inline in to_code. + """ + arduino_linked = ( + CORE.using_arduino + or ARDUINO_ESP32_COMPONENT_NAME in CORE.data[KEY_ESP32][KEY_COMPONENTS] + ) + if not arduino_linked: + cg.add_build_flag("-Wl,--wrap=esp_panic_handler") + cg.add_define("USE_ESP32_CRASH_HANDLER") + elif not CORE.using_arduino: + cg.add_build_flag("-DUSE_ARDUINO") + + async def to_code(config): framework_ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] conf = config[CONF_FRAMEWORK] @@ -1802,11 +1828,8 @@ async def to_code(config): cg.add_build_flag("-DUSE_ESP32") cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_build_flag("-Wl,-z,noexecstack") - # Arduino already wraps esp_panic_handler for its own backtrace handler, - # so only add our wrap when using ESP-IDF framework to avoid linker conflicts. - if conf[CONF_TYPE] == FRAMEWORK_ESP_IDF: - cg.add_build_flag("-Wl,--wrap=esp_panic_handler") - cg.add_define("USE_ESP32_CRASH_HANDLER") + # Deferred so KEY_COMPONENTS is fully populated -- see the coroutine. + CORE.add_job(_finalize_arduino_aware_flags) cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) variant = config[CONF_VARIANT] cg.add_build_flag(f"-DUSE_ESP32_VARIANT_{variant}") @@ -2566,7 +2589,7 @@ def _write_idf_component_yml(): if CORE.using_toolchain_esp_idf: add_idf_component( - name="espressif/arduino-esp32", + name=ARDUINO_ESP32_COMPONENT_NAME, ref=str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]), ) From ae2e372762235e1dbf5cfe12164367940a8fc4b4 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 14:08:09 -0400 Subject: [PATCH 25/96] [tuya] Restore null guard on status_pin lost in #16353 (#16539) --- esphome/components/tuya/tuya.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index fd14844908..3058d82cc4 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -206,15 +206,17 @@ void Tuya::handle_command_(uint8_t command, uint8_t version, const uint8_t *buff if (this->status_pin_reported_ != -1) { this->init_state_ = TuyaInitState::INIT_DATAPOINT; this->send_empty_command_(TuyaCommandType::DATAPOINT_QUERY); - bool is_pin_equals = - this->status_pin_ != nullptr && this->status_pin_->get_pin() == this->status_pin_reported_; - // Configure status pin toggling (if reported and configured) or WIFI_STATE periodic send - if (!is_pin_equals) { - ESP_LOGW(TAG, "Supplied status_pin does not equals the reported pin %i. Using supplied pin anyway.", + if (this->status_pin_ != nullptr) { + if (this->status_pin_->get_pin() != this->status_pin_reported_) { + ESP_LOGW(TAG, "Supplied status_pin does not equal the reported pin %i. Using supplied pin anyway.", + this->status_pin_reported_); + } + ESP_LOGV(TAG, "Configured status pin %i", this->status_pin_->get_pin()); + this->set_interval("wifi", 1000, [this] { this->set_status_pin_(); }); + } else { + ESP_LOGW(TAG, "MCU reported status_pin %i but no status_pin was configured; running in limited mode.", this->status_pin_reported_); } - ESP_LOGV(TAG, "Configured status pin %i", this->status_pin_->get_pin()); - this->set_interval("wifi", 1000, [this] { this->set_status_pin_(); }); } else { this->init_state_ = TuyaInitState::INIT_WIFI; ESP_LOGV(TAG, "Configured WIFI_STATE periodic send"); From 0c94a173b664ab2db05d841f622763545c41ff29 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 21 May 2026 15:42:57 -0500 Subject: [PATCH 26/96] [api] Break api_connection/api_server include cycle to drop custom unique_ptr deleter (#16542) --- esphome/components/api/api_connection.cpp | 1 + esphome/components/api/api_connection.h | 51 ++++-------------- .../components/api/api_connection_buffer.h | 54 +++++++++++++++++++ esphome/components/api/api_server.cpp | 5 -- esphome/components/api/api_server.h | 14 ++--- .../bluetooth_proxy/bluetooth_proxy.cpp | 1 + 6 files changed, 71 insertions(+), 55 deletions(-) create mode 100644 esphome/components/api/api_connection_buffer.h diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index b6f4aa2141..f2bf3752fa 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1,5 +1,6 @@ #include "api_connection.h" #ifdef USE_API +#include "api_connection_buffer.h" // for encode_to_buffer / get_batch_delay_ms_ inlines #ifdef USE_API_NOISE #include "api_frame_helper_noise.h" #endif diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 4165b7f3a2..804cd9ddd1 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -11,7 +11,8 @@ #endif #include "api_pb2.h" #include "api_pb2_service.h" -#include "api_server.h" +#include "list_entities.h" +#include "subscribe_state.h" #include "esphome/core/application.h" #include "esphome/core/component.h" #ifdef USE_ESP32_CRASH_HANDLER @@ -36,6 +37,9 @@ class ComponentIterator; namespace esphome::api { +// Forward-declared to break the api_server.h cycle; full-type inlines are in api_connection_buffer.h. +class APIServer; + // Keepalive timeout in milliseconds static constexpr uint32_t KEEPALIVE_TIMEOUT_MS = 60000; // Maximum number of entities to process in a single batch during initial state/info sending @@ -411,44 +415,10 @@ class APIConnection final : public APIServerConnectionBase { // Non-template buffer management for send_message bool send_message_(uint32_t payload_size, uint8_t message_type, MessageEncodeFn encode_fn, const void *msg); - // Core batch encoding logic. Computes header size, checks fit, resizes buffer, encodes. - // ALWAYS_INLINE so the compiler can devirtualize encode_fn at hot call sites. - static inline uint16_t ESPHOME_ALWAYS_INLINE encode_to_buffer(uint32_t calculated_size, MessageEncodeFn encode_fn, - const void *msg, APIConnection *conn, - uint32_t remaining_size) { -#ifdef HAS_PROTO_MESSAGE_DUMP - if (conn->flags_.log_only_mode) { - auto *proto_msg = static_cast(msg); - DumpBuffer dump_buf; - conn->log_send_message_(proto_msg->message_name(), proto_msg->dump_to(dump_buf)); - return 1; - } -#endif - const uint8_t footer_size = conn->helper_->frame_footer_size(); - - // First message uses max padding (already in buffer), subsequent use exact header size - size_t to_add; - if (conn->flags_.batch_first_message) { - conn->flags_.batch_first_message = false; - conn->batch_header_size_ = conn->helper_->frame_header_padding(); - to_add = calculated_size; - } else { - conn->batch_header_size_ = conn->helper_->frame_header_size(calculated_size, conn->batch_message_type_); - to_add = calculated_size + conn->batch_header_size_ + footer_size; - } - - // Check if it fits (using actual header size, not max padding) - uint16_t total_calculated_size = calculated_size + conn->batch_header_size_ + footer_size; - if (total_calculated_size > remaining_size) - return 0; - - auto &shared_buf = conn->parent_->get_shared_buffer_ref(); - shared_buf.resize(shared_buf.size() + to_add); - ProtoWriteBuffer buffer{&shared_buf, shared_buf.size() - calculated_size}; - encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); - - return total_calculated_size; - } + // Core batch encoding logic. ALWAYS_INLINE so encode_fn devirtualizes at hot call sites. + // Defined in api_connection_buffer.h (needs APIServer complete). + static uint16_t ESPHOME_ALWAYS_INLINE encode_to_buffer(uint32_t calculated_size, MessageEncodeFn encode_fn, + const void *msg, APIConnection *conn, uint32_t remaining_size); // Noinline version of encode_to_buffer for cold paths (entity info, zero-payload messages). // All cold callers share this single copy instead of each getting an ALWAYS_INLINE expansion. @@ -792,7 +762,8 @@ class APIConnection final : public APIServerConnectionBase { // Read by process_batch_multi_ to pass into MessageInfo. uint8_t batch_header_size_{0}; - uint32_t get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } + // Defined in api_connection_buffer.h (needs APIServer complete). + uint32_t get_batch_delay_ms_() const; // Message will use 8 more bytes than the minimum size, and typical // MTU is 1500. Sometimes users will see as low as 1460 MTU. // If its IPv6 the header is 40 bytes, and if its IPv4 diff --git a/esphome/components/api/api_connection_buffer.h b/esphome/components/api/api_connection_buffer.h new file mode 100644 index 0000000000..1dd8a162e4 --- /dev/null +++ b/esphome/components/api/api_connection_buffer.h @@ -0,0 +1,54 @@ +#pragma once + +#include "esphome/core/defines.h" +#ifdef USE_API + +// Inline APIConnection methods that need APIServer complete. Include this +// instead of api_connection.h when calling encode_to_buffer or get_batch_delay_ms_. + +#include "api_connection.h" +#include "api_server.h" + +namespace esphome::api { + +inline uint16_t ESPHOME_ALWAYS_INLINE APIConnection::encode_to_buffer(uint32_t calculated_size, + MessageEncodeFn encode_fn, const void *msg, + APIConnection *conn, uint32_t remaining_size) { +#ifdef HAS_PROTO_MESSAGE_DUMP + if (conn->flags_.log_only_mode) { + auto *proto_msg = static_cast(msg); + DumpBuffer dump_buf; + conn->log_send_message_(proto_msg->message_name(), proto_msg->dump_to(dump_buf)); + return 1; + } +#endif + const uint8_t footer_size = conn->helper_->frame_footer_size(); + + // First message uses max padding (already in buffer), subsequent use exact header size + size_t to_add; + if (conn->flags_.batch_first_message) { + conn->flags_.batch_first_message = false; + conn->batch_header_size_ = conn->helper_->frame_header_padding(); + to_add = calculated_size; + } else { + conn->batch_header_size_ = conn->helper_->frame_header_size(calculated_size, conn->batch_message_type_); + to_add = calculated_size + conn->batch_header_size_ + footer_size; + } + + // Check if it fits (using actual header size, not max padding) + uint16_t total_calculated_size = calculated_size + conn->batch_header_size_ + footer_size; + if (total_calculated_size > remaining_size) + return 0; + + auto &shared_buf = conn->parent_->get_shared_buffer_ref(); + shared_buf.resize(shared_buf.size() + to_add); + ProtoWriteBuffer buffer{&shared_buf, shared_buf.size() - calculated_size}; + encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); + + return total_calculated_size; +} + +inline uint32_t APIConnection::get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } + +} // namespace esphome::api +#endif diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 6c26c4e187..c30bd2e612 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -30,11 +30,6 @@ APIServer *global_api_server = nullptr; // NOLINT(cppcoreguidelines-avoid-non-c APIServer::APIServer() { global_api_server = this; } -// Custom deleter defined here so `delete` sees the complete APIConnection type. -// This prevents libc++ from emitting an "incomplete type" error when other -// translation units only have the forward declaration of APIConnection. -void APIServer::APIConnectionDeleter::operator()(APIConnection *p) const { delete p; } - void APIServer::socket_failed_(const LogString *msg) { ESP_LOGW(TAG, "Socket %s: errno %d", LOG_STR_ARG(msg), errno); this->destroy_socket_(); diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 6b575e536d..fbc8115091 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -3,6 +3,8 @@ #include "esphome/core/defines.h" #ifdef USE_API #include "api_buffer.h" +// Must precede clients_ so APIConnection is complete for default_delete (libc++). +#include "api_connection.h" #include "api_noise_context.h" #include "api_pb2.h" #include "api_pb2_service.h" @@ -12,8 +14,6 @@ #include "esphome/core/controller.h" #include "esphome/core/log.h" #include "esphome/core/string_ref.h" -#include "list_entities.h" -#include "subscribe_state.h" #ifdef USE_LOGGER #include "esphome/components/logger/logger.h" #endif @@ -191,15 +191,9 @@ class APIServer final : public Component, bool is_connected_with_state_subscription() const; // Range-for view over the populated slice [0, api_connection_count_). Read-only with respect - // to ownership — callers get `const unique_ptr&` so they can invoke non-const methods on the + // to ownership; callers get `const unique_ptr&` so they can invoke non-const methods on the // APIConnection but cannot reset/move the slot and break the count invariant. - // Custom deleter is defined out-of-line in api_server.cpp so libc++ does not - // eagerly instantiate `delete static_cast(p)` here, where - // only the forward declaration of APIConnection is visible (incomplete type). - struct APIConnectionDeleter { - void operator()(APIConnection *p) const; - }; - using APIConnectionPtr = std::unique_ptr; + using APIConnectionPtr = std::unique_ptr; class ActiveClientsView { const APIConnectionPtr *begin_; const APIConnectionPtr *end_; diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index c3461f9c51..ca30aab943 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -1,5 +1,6 @@ #include "bluetooth_proxy.h" +#include "esphome/components/api/api_server.h" #include "esphome/core/log.h" #include "esphome/core/macros.h" #include "esphome/core/application.h" From 27d53ec1171fe257400a0e214420c4cd77efafbb Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 21 May 2026 18:03:07 -0400 Subject: [PATCH 27/96] [sx126x] Assert NSS before wait_busy so commands wake the chip from sleep (#16546) --- esphome/components/sx126x/sx126x.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/components/sx126x/sx126x.cpp b/esphome/components/sx126x/sx126x.cpp index 6e6857fadb..83afeac50a 100644 --- a/esphome/components/sx126x/sx126x.cpp +++ b/esphome/components/sx126x/sx126x.cpp @@ -30,8 +30,8 @@ static constexpr uint8_t OCP_140MA = 0x38; // 140 mA max current static constexpr float LOW_DATA_RATE_OPTIMIZE_THRESHOLD = 16.38f; // 16.38 ms uint8_t SX126x::read_fifo_(uint8_t offset, std::vector &packet) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(RADIO_READ_BUFFER); this->transfer_byte(offset); uint8_t status = this->transfer_byte(0x00); @@ -43,8 +43,8 @@ uint8_t SX126x::read_fifo_(uint8_t offset, std::vector &packet) { } void SX126x::write_fifo_(uint8_t offset, const std::vector &packet) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(RADIO_WRITE_BUFFER); this->transfer_byte(offset); for (const uint8_t &byte : packet) { @@ -55,8 +55,8 @@ void SX126x::write_fifo_(uint8_t offset, const std::vector &packet) { } uint8_t SX126x::read_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(opcode); uint8_t status = this->transfer_byte(0x00); for (int32_t i = 0; i < size; i++) { @@ -67,8 +67,8 @@ uint8_t SX126x::read_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { } void SX126x::write_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(opcode); for (int32_t i = 0; i < size; i++) { this->transfer_byte(data[i]); @@ -78,8 +78,8 @@ void SX126x::write_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { } void SX126x::read_register_(uint16_t reg, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->write_byte(RADIO_READ_REGISTER); this->write_byte((reg >> 8) & 0xFF); this->write_byte((reg >> 0) & 0xFF); @@ -91,8 +91,8 @@ void SX126x::read_register_(uint16_t reg, uint8_t *data, uint8_t size) { } void SX126x::write_register_(uint16_t reg, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->write_byte(RADIO_WRITE_REGISTER); this->write_byte((reg >> 8) & 0xFF); this->write_byte((reg >> 0) & 0xFF); From f247def4acdeee5d707c71927cdf45c851c9452d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 21 May 2026 17:05:17 -0500 Subject: [PATCH 28/96] [core] Refresh compiled config cache after upload/logs fallback (#16548) --- esphome/__main__.py | 15 +++- tests/unit_tests/test_compiled_config.py | 100 +++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 16a05ad552..07bbd89358 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2449,7 +2449,10 @@ def run_esphome(argv): # Skipped when -s overrides are passed, since the cache was written # against the previous substitution set. config: ConfigType | None = None - if args.command in ("upload", "logs") and not command_line_substitutions: + cache_eligible = ( + args.command in ("upload", "logs") and not command_line_substitutions + ) + if cache_eligible: from esphome.compiled_config import load_compiled_config config = load_compiled_config(conf_path) @@ -2464,6 +2467,16 @@ def run_esphome(argv): command_line_substitutions, skip_external_update=skip_external, ) + # Refresh the cache so the next upload/logs hits the fast path + # instead of re-running read_config. Skip when the storage + # sidecar is absent (no compile has run): the cache would + # never be loaded back, so writing secrets to disk is wasted. + if cache_eligible and config is not None: + from esphome.compiled_config import save_compiled_config + from esphome.storage_json import ext_storage_path + + if ext_storage_path(conf_path.name).exists(): + save_compiled_config(config) if config is None: return 2 CORE.config = config diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 8c9cfa8101..e12107152b 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -253,6 +253,106 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache( mock_read.assert_called_once() +def test_run_esphome_upload_does_not_refresh_cache_without_sidecar( + tmp_path: Path, +) -> None: + """Without a StorageJSON sidecar (no compile has run), the fallback + skips the cache write -- load_compiled_config requires the sidecar, + so writing the rendered (secret-resolved) YAML would be inert and + leak secrets to disk for nothing.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + with ( + patch( + "esphome.__main__.read_config", + return_value={"esphome": {"name": "lite_test"}}, + ), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "upload", str(yaml_path)]) + + mock_save.assert_not_called() + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_upload_and_logs_refresh_cache_on_fallback( + tmp_path: Path, command: str +) -> None: + """A stale-cache fallback rewrites the cache so the next call hits + the fast path. Without this, every upload/logs after a YAML edit + pays for read_config() until the next compile rewrites the cache.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json") + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache, yaml_path, offset=-60) # stale + + fresh_config = {"esphome": {"name": "lite_test"}, "logger": {}} + + with ( + patch("esphome.__main__.read_config", return_value=fresh_config), + patch( + "esphome.compiled_config.save_compiled_config", wraps=save_compiled_config + ) as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {command: lambda args, config: 0}, + ), + ): + assert run_esphome(["esphome", command, str(yaml_path)]) == 0 + + mock_save.assert_called_once_with(fresh_config) + # mtime is now newer than the source YAML, so a follow-up call hits + # the fast path instead of repeating read_config. + assert cache.stat().st_mtime >= yaml_path.stat().st_mtime + + +def test_run_esphome_upload_with_substitution_does_not_refresh_cache( + fresh_cache_files: Path, +) -> None: + """`-s` substitutions skip the cache on both read and write -- saving + here would clobber the cache with a substitution-specific config.""" + with ( + patch("esphome.__main__.read_config", return_value={"esphome": {}}), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "-s", "var", "val", "upload", str(fresh_cache_files)]) + + mock_save.assert_not_called() + + +def test_run_esphome_compile_does_not_refresh_cache_via_fallback( + fresh_cache_files: Path, +) -> None: + """Compile writes the cache through update_storage_json, not via the + upload/logs fallback path -- the fallback save would skip the + storage_should_clean check.""" + with ( + patch("esphome.__main__.read_config", return_value={"esphome": {}}), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"compile": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "compile", str(fresh_cache_files)]) + + mock_save.assert_not_called() + + def test_run_esphome_upload_with_substitution_skips_cache( fresh_cache_files: Path, ) -> None: From 7ae55664727da6073c7672fc0e28aba1bf67126a Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Fri, 22 May 2026 06:43:08 -0400 Subject: [PATCH 29/96] [sendspin] Bump sendspin-cpp to v0.6.1 (#16553) --- esphome/components/sendspin/__init__.py | 2 +- esphome/idf_component.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 36f13f7d07..b670bd3c4d 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -206,7 +206,7 @@ async def to_code(config: ConfigType) -> None: ) # sendspin-cpp library - esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.6.0") + esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.6.1") cg.add_define("USE_SENDSPIN", True) # for MDNS diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 42d0d5de6b..8dafde1111 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -100,6 +100,6 @@ dependencies: esp32async/asynctcp: version: 3.4.91 sendspin/sendspin-cpp: - version: 0.6.0 + version: 0.6.1 lvgl/lvgl: version: 9.5.0 From 59db9a4673e1e159476a83f8d6654d669344231a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 22 May 2026 08:49:03 -0500 Subject: [PATCH 30/96] [dashboard] Fix flaky test_websocket_refresh_command on Windows CI (#16565) --- tests/dashboard/test_web_server.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/dashboard/test_web_server.py b/tests/dashboard/test_web_server.py index 626aea0216..0ee841e68c 100644 --- a/tests/dashboard/test_web_server.py +++ b/tests/dashboard/test_web_server.py @@ -1503,13 +1503,18 @@ async def test_websocket_refresh_command( ) -> None: """Test WebSocket refresh command triggers dashboard update.""" with patch("esphome.dashboard.web_server.DASHBOARD_SUBSCRIBER") as mock_subscriber: - mock_subscriber.request_refresh = Mock() + # Signal an asyncio.Event when request_refresh is invoked so the + # test can deterministically wait for the server-side handler to run + # instead of relying on a fixed sleep (flaky on Windows CI under load). + called = asyncio.Event() + mock_subscriber.request_refresh = Mock(side_effect=called.set) # Send refresh command await websocket_client.write_message(json.dumps({"event": "refresh"})) - # Give it a moment to process - await asyncio.sleep(0.01) + # Wait for the server to process the message and invoke request_refresh + async with asyncio.timeout(5): + await called.wait() # Verify request_refresh was called mock_subscriber.request_refresh.assert_called_once() From 1f4a0615721382e36c0fecfe158a8f4eac33c080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edvard=20Filistovi=C4=8D?= Date: Fri, 22 May 2026 22:09:32 +0300 Subject: [PATCH 31/96] [libretiny] Fix LN882H IRAM_ATTR injection point in patch_linker.py (#16570) --- esphome/components/libretiny/patch_linker.py.script | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/esphome/components/libretiny/patch_linker.py.script b/esphome/components/libretiny/patch_linker.py.script index 282a31d3f2..3a8a4787ed 100644 --- a/esphome/components/libretiny/patch_linker.py.script +++ b/esphome/components/libretiny/patch_linker.py.script @@ -13,7 +13,9 @@ import subprocess # - RTL8710B: hal.h uses section(".image2.ram.text"); stock linker consumes it. # - RTL8720C: hal.h uses section(".sram.text"); stock linker consumes it. # - LN882H: stock linker has no glob for ".sram.text", so we inject -# KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH). +# KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH) +# immediately after KEEP(*(.vectors)), so the vector table stays at +# __copysection_ram0_start (0x20000000) for correct Cortex-M4 VTOR alignment. # # All families also get a post-link summary showing where IRAM_ATTR landed. @@ -27,7 +29,11 @@ _KEEP_LINE = ( "__esphome_sram_text_end = .; " + _MARKER + "\n" ) -_LN_COPY = re.compile(r"(\.flash_copysection\s*:\s*\{\s*\n)") +# Inject after KEEP(*(.vectors)) so the vector table stays at +# __copysection_ram0_start (0x20000000). Cortex-M4 VTOR requires a 512-byte- +# aligned address; injecting before the vectors would push them to an +# unaligned offset and mis-route every IRQ handler. +_LN_COPY = re.compile(r"(KEEP\(\*\(\.vectors\)\)[^\n]*\n)") def _detect(env): @@ -56,7 +62,7 @@ KNOWN_VARIANTS = frozenset({ def _inject_keep(host_section): - """Return a patcher that injects _KEEP_LINE at the top of `host_section`.""" + """Return a patcher that injects _KEEP_LINE after `host_section` match.""" def patch(content): if _MARKER in content: return content From 4e7bc92061d5cd6a962c5512d2e2c91e6e491ac9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 22 May 2026 14:30:28 -0500 Subject: [PATCH 32/96] [esp8266] Use os_timer-based esp_delay() in delay() (#16563) --- esphome/components/esp8266/hal.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/esphome/components/esp8266/hal.cpp b/esphome/components/esp8266/hal.cpp index e8f472dc8a..3501c51859 100644 --- a/esphome/components/esp8266/hal.cpp +++ b/esphome/components/esp8266/hal.cpp @@ -5,6 +5,7 @@ #include #include +#include extern "C" { #include @@ -71,23 +72,22 @@ uint32_t IRAM_ATTR HOT millis() { return result; } -// Poll-based delay that avoids ::delay() — Arduino's __delay has an intra-object -// call to the original millis() that --wrap can't intercept, so calling ::delay() -// would keep the slow Arduino millis body alive in IRAM. optimistic_yield still -// enters esp_schedule()/esp_suspend_within_cont() via yield(), so SDK tasks and -// WiFi run correctly. Theoretically less power-efficient than Arduino's -// os_timer-based delay() for long waits, but nearly all ESPHome delays are short -// (sensor/I²C/SPI settling in the 1–100 ms range) where the difference is -// negligible. +// Delegate to Arduino's 1-arg esp_delay(), which uses os_timer + esp_suspend to +// suspend the cont task for `ms` milliseconds without polling millis(). This +// matches pre-2026.5.0 behavior (when esphome::delay() forwarded to ::delay()) +// and lets the SDK run freely while we wait, which timing-sensitive +// interrupt-driven code (e.g. ESP8266 software-serial RX in components like +// fingerprint_grow) depends on. The poll-based busy-wait that this replaced +// rarely yielded inside short waits like delay(1), starving WiFi/SDK tasks and +// extending interrupt latency. Unlike ::delay(), esp_delay()'s 1-arg form does +// not call millis(), so the slow Arduino millis() body is not pulled into IRAM +// by this path (the --wrap=millis goal of #15662 is preserved). void HOT delay(uint32_t ms) { if (ms == 0) { optimistic_yield(1000); return; } - uint32_t start = millis(); - while (millis() - start < ms) { - optimistic_yield(1000); - } + esp_delay(ms); } void arch_restart() { From 8f6ea62628965cdb47fa3f30d7f461be3b3c1f89 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 22 May 2026 14:30:43 -0500 Subject: [PATCH 33/96] [uart] Wake main loop on ESP8266 software serial RX (#16562) --- esphome/components/uart/__init__.py | 9 +++++---- .../components/uart/uart_component_esp8266.cpp | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/esphome/components/uart/__init__.py b/esphome/components/uart/__init__.py index 7075228743..4ea32e26a3 100644 --- a/esphome/components/uart/__init__.py +++ b/esphome/components/uart/__init__.py @@ -513,10 +513,11 @@ async def uart_write_to_code(config, action_id, template_arg, args): @coroutine_with_priority(CoroPriority.FINAL) async def final_step(): """Final code generation step to configure optional UART features.""" - if CORE.is_esp32 and CORE.has_networking: - # Wake-on-RX is essentially free on ESP32 (just an ISR function pointer - # registration) — enable by default to reduce RX buffer overflow risk - # by waking the main loop immediately when data arrives. + if (CORE.is_esp32 or CORE.is_esp8266) and CORE.has_networking: + # Wake-on-RX is essentially free (just an ISR function pointer + # registration on ESP32, an inline flag set on ESP8266 software + # serial) — enable by default to reduce RX buffer overflow risk by + # waking the main loop immediately when data arrives. cg.add_define("USE_UART_WAKE_LOOP_ON_RX") diff --git a/esphome/components/uart/uart_component_esp8266.cpp b/esphome/components/uart/uart_component_esp8266.cpp index 0ea7930760..fc1509f737 100644 --- a/esphome/components/uart/uart_component_esp8266.cpp +++ b/esphome/components/uart/uart_component_esp8266.cpp @@ -4,6 +4,9 @@ #include "esphome/core/defines.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#ifdef USE_UART_WAKE_LOOP_ON_RX +#include "esphome/core/wake.h" +#endif #ifdef USE_LOGGER #include "esphome/components/logger/logger.h" @@ -149,7 +152,11 @@ void ESP8266UartComponent::dump_config() { if (this->hw_serial_ != nullptr) { ESP_LOGCONFIG(TAG, " Using hardware serial interface."); } else { - ESP_LOGCONFIG(TAG, " Using software serial"); + ESP_LOGCONFIG(TAG, " Using software serial" +#ifdef USE_UART_WAKE_LOOP_ON_RX + "\n Wake on data RX: ENABLED" +#endif + ); } this->check_logger_conflict(); } @@ -266,6 +273,12 @@ void IRAM_ATTR ESP8266SoftwareSerial::gpio_intr(ESP8266SoftwareSerial *arg) { arg->rx_in_pos_ = (arg->rx_in_pos_ + 1) % arg->rx_buffer_size_; // Clear RX pin so that the interrupt doesn't re-trigger right away again. arg->rx_pin_.clear_interrupt(); +#ifdef USE_UART_WAKE_LOOP_ON_RX + // Wake the main loop so the consuming component drains the byte promptly + // instead of waiting for the next loop_interval_ tick. Important for timing + // sensitive setups that poll read() in a tight loop (e.g. fingerprint_grow). + wake_loop_isrsafe(); +#endif } void IRAM_ATTR HOT ESP8266SoftwareSerial::write_byte(uint8_t data) { if (this->gpio_tx_pin_ == nullptr) { From adde7681e8a33884d47b11211b9a079bf12135de Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Fri, 22 May 2026 20:30:25 -0400 Subject: [PATCH 34/96] [esp32] Demote IDF #warning deprecations from error under ESP-IDF toolchain (#16584) --- esphome/components/esp32/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 2fdaa991ae..367ec32578 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1821,6 +1821,7 @@ async def to_code(config): cg.add_build_flag("-Wno-error=overloaded-virtual") cg.add_build_flag("-Wno-error=reorder") cg.add_build_flag("-Wno-error=volatile") + cg.add_build_flag("-Wno-error=cpp") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") From 0babc52472c2cc617c36dd5d9e872882d5e2577e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 23 May 2026 14:30:12 -0500 Subject: [PATCH 35/96] [bluetooth_proxy] Recover slot stuck in DISCONNECTING when CLOSE_EVT is dropped (#16588) --- .../bluetooth_proxy/bluetooth_connection.cpp | 26 ++++++++++++------- .../bluetooth_proxy/bluetooth_connection.h | 2 ++ .../esp32_ble_client/ble_client_base.cpp | 2 ++ .../esp32_ble_client/ble_client_base.h | 10 +++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 21573f0184..7ba9e61e19 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -135,12 +135,26 @@ void BluetoothConnection::loop() { // - For V3_WITH_CACHE: Services are never sent, disable after INIT state // - For V3_WITHOUT_CACHE: Disable only after service discovery is complete // (send_service_ == DONE_SENDING_SERVICES, which is only set after services are sent) - if (this->state() != espbt::ClientState::INIT && (this->connection_type_ == espbt::ConnectionType::V3_WITH_CACHE || - this->send_service_ == DONE_SENDING_SERVICES)) { + // Never disable while DISCONNECTING — BLEClientBase::loop() needs to keep running so the + // 10s safety timeout can force IDLE if CLOSE_EVT is never delivered. + if (this->state() != espbt::ClientState::INIT && this->state() != espbt::ClientState::DISCONNECTING && + (this->connection_type_ == espbt::ConnectionType::V3_WITH_CACHE || + this->send_service_ == DONE_SENDING_SERVICES)) { this->disable_loop(); } } +void BluetoothConnection::on_disconnect_complete(esp_err_t reason) { + // Called from both the CLOSE_EVT handler and the DISCONNECTING safety timeout in the + // base class. Free the proxy slot, notify the API client, and reset send_service_. + // address_ may already be 0 if reset_connection_ ran earlier on this teardown. + if (this->address_ == 0) { + return; + } + ESP_LOGD(TAG, "[%d] [%s] Close, reason=0x%02x, freeing slot", this->connection_index_, this->address_str_, reason); + this->reset_connection_(reason); +} + void BluetoothConnection::reset_connection_(esp_err_t reason) { // Send disconnection notification this->proxy_->send_device_connection(this->address_, false, 0, reason); @@ -372,14 +386,6 @@ bool BluetoothConnection::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga this->proxy_->send_device_connection(this->address_, false, 0, param->disconnect.reason); break; } - case ESP_GATTC_CLOSE_EVT: { - ESP_LOGD(TAG, "[%d] [%s] Close, reason=0x%02x, freeing slot", this->connection_index_, this->address_str_, - param->close.reason); - // Now the GATT connection is fully closed and controller resources are freed - // Safe to mark the connection slot as available - this->reset_connection_(param->close.reason); - break; - } case ESP_GATTC_OPEN_EVT: { if (param->open.status != ESP_GATT_OK && param->open.status != ESP_GATT_ALREADY_OPEN) { this->reset_connection_(param->open.status); diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.h b/esphome/components/bluetooth_proxy/bluetooth_connection.h index b50ea2d6a2..e5600f6af4 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.h +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.h @@ -33,6 +33,8 @@ class BluetoothConnection final : public esp32_ble_client::BLEClientBase { protected: friend class BluetoothProxy; + void on_disconnect_complete(esp_err_t reason) override; + bool supports_efficient_uuids_() const; void send_service_for_discovery_(); void reset_connection_(esp_err_t reason); diff --git a/esphome/components/esp32_ble_client/ble_client_base.cpp b/esphome/components/esp32_ble_client/ble_client_base.cpp index 7f0f2c624d..3fb9632e9a 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.cpp +++ b/esphome/components/esp32_ble_client/ble_client_base.cpp @@ -72,6 +72,7 @@ void BLEClientBase::loop() { // never delivered CLOSE_EVT/DISCONNECT_EVT, services would leak without this call. this->release_services(); this->set_idle_(); + this->on_disconnect_complete(ESP_GATT_CONN_TIMEOUT); } } @@ -418,6 +419,7 @@ bool BLEClientBase::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_ this->log_gattc_lifecycle_event_("CLOSE"); this->release_services(); this->set_idle_(); + this->on_disconnect_complete(param->close.reason); break; } case ESP_GATTC_SEARCH_RES_EVT: { diff --git a/esphome/components/esp32_ble_client/ble_client_base.h b/esphome/components/esp32_ble_client/ble_client_base.h index 4e0b22cc29..0291a4b993 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.h +++ b/esphome/components/esp32_ble_client/ble_client_base.h @@ -140,6 +140,12 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void log_gattc_warning_(const char *operation, esp_err_t err); void log_connection_params_(const char *param_type); void handle_connection_result_(esp_err_t ret); + /// Hook called once a connection has been fully torn down (after release_services() and + /// set_idle_()), from both the CLOSE_EVT handler and the DISCONNECTING safety timeout. + /// Subclasses with extra per-connection accounting (e.g. bluetooth_proxy slot state) + /// override this to release that state. `reason` is the controller reason code, or + /// ESP_GATT_CONN_TIMEOUT for the safety-timeout path. + virtual void on_disconnect_complete(esp_err_t reason) {} /// Transition to IDLE and reset conn_id — call when the connection is fully dead. void set_idle_() { this->set_state(espbt::ClientState::IDLE); @@ -149,6 +155,10 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void set_disconnecting_() { this->disconnecting_started_ = millis(); this->set_state(espbt::ClientState::DISCONNECTING); + // BluetoothConnection::loop() disables the component loop after service discovery + // completes, so the DISCONNECTING timeout check in loop() would never run if CLOSE_EVT + // gets lost. Re-enable the loop so the 10s safety timeout can force IDLE. + this->enable_loop(); } // Compact error logging helpers to reduce flash usage void log_error_(const char *message); From 9a34a6aabb8691447126e9775cb8fef90e80e932 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Sat, 23 May 2026 15:44:25 -0400 Subject: [PATCH 36/96] [esp32] Replace per-class -Wno-error=X demotes with blanket -Wno-error for ESP-IDF toolchain (#16599) --- esphome/components/esp32/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 367ec32578..5aeff91830 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1816,12 +1816,9 @@ async def to_code(config): Path(__file__).parent / "iram_fix.py.script", ) else: - cg.add_build_flag("-Wno-error=format") - cg.add_build_flag("-Wno-error=maybe-uninitialized") - cg.add_build_flag("-Wno-error=overloaded-virtual") - cg.add_build_flag("-Wno-error=reorder") - cg.add_build_flag("-Wno-error=volatile") - cg.add_build_flag("-Wno-error=cpp") + # Undo IDF's blanket -Werror so third-party libraries and user + # lambdas don't need a -Wno-error= entry per warning class. + cg.add_build_flag("-Wno-error") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") From ddd353d10560c46a97abb0b2dd7bce31b55a8bfe Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Sun, 24 May 2026 00:19:07 -0400 Subject: [PATCH 37/96] [esp32] Disable IDF's COMPILER_DISABLE_DEFAULT_ERRORS so -Wno-error actually undoes -Werror (#16604) --- esphome/components/esp32/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 5aeff91830..4f77258b2c 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1816,8 +1816,11 @@ async def to_code(config): Path(__file__).parent / "iram_fix.py.script", ) else: - # Undo IDF's blanket -Werror so third-party libraries and user - # lambdas don't need a -Wno-error= entry per warning class. + # Demote IDF's blanket -Werror to warnings so third-party libs + # and user lambdas don't need a -Wno-error= per warning. + # The sdkconfig knob disables IDF's rewrite to -Werror=all (which + # can't be globally undone); -Wno-error then handles the demotion. + add_idf_sdkconfig_option("CONFIG_COMPILER_DISABLE_DEFAULT_ERRORS", False) cg.add_build_flag("-Wno-error") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") From 03e2eb4b4a938b8600954afba8c74bdcbbbd7aee Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 25 May 2026 09:28:49 +1200 Subject: [PATCH 38/96] Bump version to 2026.5.1 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 206a181ffd..30ae42ea2c 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.5.0 +PROJECT_NUMBER = 2026.5.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 96554b12da..39c5c6b60e 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.5.0" +__version__ = "2026.5.1" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 090f5a486a63418c54a0f01b0e9a78ae61ec25ee Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 24 May 2026 16:32:47 -0500 Subject: [PATCH 39/96] Lift dependabot pip open PR limit (#16609) --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 528e69c478..e87939f824 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,7 @@ updates: directory: "/" schedule: interval: daily + open-pull-requests-limit: 10 ignore: # Hypotehsis is only used for testing and is updated quite often - dependency-name: hypothesis From 917ffc379795d9f42c5d92e46834811d056e3774 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 21:49:52 +0000 Subject: [PATCH 40/96] Bump aioesphomeapi from 45.0.4 to 45.2.2 (#16611) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 178e05497f..45401a7995 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 -aioesphomeapi==45.0.4 +aioesphomeapi==45.2.2 zeroconf==0.149.16 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import From 62b0a93e5e032d1ad4573cb055df1e50e6a5af56 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 25 May 2026 10:43:39 +1200 Subject: [PATCH 41/96] [rp2040] Add variant config option for RP2040/RP2350 (#16602) --- esphome/components/rp2040/__init__.py | 109 +++++++++++++++++- esphome/components/rp2040/const.py | 26 +++++ esphome/core/defines.h | 1 + tests/components/rp2040/test.rp2040-ard.yaml | 1 + .../rp2040/test.rp2040-pico2-ard.yaml | 6 + tests/unit_tests/components/test_rp2040.py | 67 ++++++++++- 6 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 tests/components/rp2040/test.rp2040-pico2-ard.yaml diff --git a/esphome/components/rp2040/__init__.py b/esphome/components/rp2040/__init__.py index 862d532645..830c961476 100644 --- a/esphome/components/rp2040/__init__.py +++ b/esphome/components/rp2040/__init__.py @@ -1,8 +1,10 @@ +from collections.abc import Callable import logging from pathlib import Path import re from string import ascii_letters, digits import subprocess +from typing import Any import esphome.codegen as cg import esphome.config_validation as cv @@ -12,6 +14,7 @@ from esphome.const import ( CONF_FRAMEWORK, CONF_PLATFORM_VERSION, CONF_SOURCE, + CONF_VARIANT, CONF_VERSION, CONF_WATCHDOG_TIMEOUT, KEY_CORE, @@ -21,12 +24,30 @@ from esphome.const import ( PLATFORM_RP2040, ThreadModel, ) -from esphome.core import CORE, CoroPriority, EsphomeError, coroutine_with_priority +from esphome.core import ( + CORE, + CoroPriority, + EsphomeCore, + EsphomeError, + coroutine_with_priority, +) from esphome.core.config import BOARD_MAX_LENGTH from esphome.helpers import copy_file_if_changed, read_file, write_file_if_changed +from esphome.types import ConfigType from . import boards -from .const import KEY_BOARD, KEY_LWIP_OPTS, KEY_PIO_FILES, KEY_RP2040, rp2040_ns +from .const import ( + KEY_BOARD, + KEY_LWIP_OPTS, + KEY_PIO_FILES, + KEY_RP2040, + KEY_VARIANT, + MCU_TO_VARIANT, + STANDARD_BOARDS, + VARIANT_FRIENDLY, + VARIANTS, + rp2040_ns, +) # force import gpio to register pin schema from .gpio import rp2040_pin_to_code # noqa @@ -68,7 +89,7 @@ def board_id_has_wifi(board_id: str) -> bool: return board_info.get("wifi", False) -def set_core_data(config): +def set_core_data(config: ConfigType) -> ConfigType: CORE.data[KEY_RP2040] = {} CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = PLATFORM_RP2040 CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "arduino" @@ -76,12 +97,46 @@ def set_core_data(config): config[CONF_FRAMEWORK][CONF_VERSION] ) CORE.data[KEY_RP2040][KEY_BOARD] = config[CONF_BOARD] + CORE.data[KEY_RP2040][KEY_VARIANT] = config[CONF_VARIANT] CORE.data[KEY_RP2040][KEY_PIO_FILES] = {} return config +def get_rp2040_variant(core_obj: EsphomeCore | None = None) -> str: + return (core_obj or CORE).data[KEY_RP2040][KEY_VARIANT] + + +def only_on_variant( + *, + supported: str | list[str] | None = None, + unsupported: str | list[str] | None = None, + msg_prefix: str = "This feature", +) -> Callable[[Any], Any]: + """Config validator for features only available on some RP2040 variants.""" + if supported is not None and not isinstance(supported, list): + supported = [supported] + if unsupported is not None and not isinstance(unsupported, list): + unsupported = [unsupported] + + def validator_(obj: Any) -> Any: + if not CORE.is_rp2040: + raise cv.Invalid(f"{msg_prefix} is only available on RP2040") + variant = get_rp2040_variant() + if supported is not None and variant not in supported: + raise cv.Invalid( + f"{msg_prefix} is only available on {', '.join(supported)}" + ) + if unsupported is not None and variant in unsupported: + raise cv.Invalid( + f"{msg_prefix} is not available on {', '.join(unsupported)}" + ) + return obj + + return validator_ + + def get_download_types(storage_json): """Binary-download entries for a built RP2040 firmware. @@ -192,12 +247,52 @@ ARDUINO_FRAMEWORK_SCHEMA = cv.All( _arduino_check_versions, ) + +def _detect_variant(value: ConfigType) -> ConfigType: + value = value.copy() + board: str | None = value.get(CONF_BOARD) + variant: str | None = value.get(CONF_VARIANT) + + if board is None: + # `cv.has_at_least_one_key` guarantees variant is set here. + board = STANDARD_BOARDS[variant] + value[CONF_BOARD] = board + + board_info = boards.BOARDS.get(board) + if board_info is None: + if variant is None: + raise cv.Invalid( + "This board is unknown; please specify the chip variant using " + f"the '{CONF_VARIANT}' option.", + path=[CONF_BOARD], + ) + _LOGGER.warning( + "This board is unknown; the specified variant '%s' will be used " + "but this may not work as expected.", + variant, + ) + else: + board_variant = MCU_TO_VARIANT[board_info["mcu"]] + if variant is None: + variant = board_variant + elif variant != board_variant: + raise cv.Invalid( + f"Option '{CONF_VARIANT}' ({variant}) does not match the " + f"selected board '{board}' ({board_variant}).", + path=[CONF_VARIANT], + ) + + value[CONF_VARIANT] = variant + return value + + CONFIG_SCHEMA = cv.All( cv.Schema( { - cv.Required(CONF_BOARD): cv.All( + cv.Optional(CONF_BOARD): cv.All( cv.string_strict, cv.ByteLength(max=BOARD_MAX_LENGTH) ), + cv.Optional(CONF_VARIANT): cv.one_of(*VARIANTS, upper=True), cv.Optional(CONF_FRAMEWORK, default={}): ARDUINO_FRAMEWORK_SCHEMA, cv.Optional(CONF_WATCHDOG_TIMEOUT, default="8388ms"): cv.All( cv.positive_time_period_milliseconds, @@ -206,6 +301,8 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_FULL_PRINTF, default=False): cv.boolean, } ), + cv.has_at_least_one_key(CONF_BOARD, CONF_VARIANT), + _detect_variant, set_core_data, ) @@ -223,7 +320,9 @@ async def to_code(config): cg.add_define("USE_NATIVE_64BIT_TIME") cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) - cg.add_define("ESPHOME_VARIANT", "RP2040") + variant = config[CONF_VARIANT] + cg.add_build_flag(f"-DUSE_RP2040_VARIANT_{variant}") + cg.add_define("ESPHOME_VARIANT", VARIANT_FRIENDLY[variant]) cg.add_define(ThreadModel.SINGLE) cg.add_platformio_option("extra_scripts", ["post:post_build.py"]) diff --git a/esphome/components/rp2040/const.py b/esphome/components/rp2040/const.py index e381d0482d..959753d95b 100644 --- a/esphome/components/rp2040/const.py +++ b/esphome/components/rp2040/const.py @@ -4,5 +4,31 @@ KEY_BOARD = "board" KEY_LWIP_OPTS = "lwip_opts" KEY_RP2040 = "rp2040" KEY_PIO_FILES = "pio_files" +KEY_VARIANT = "variant" + +VARIANT_RP2040 = "RP2040" +VARIANT_RP2350 = "RP2350" +VARIANTS = [ + VARIANT_RP2040, + VARIANT_RP2350, +] + +VARIANT_FRIENDLY = { + VARIANT_RP2040: "RP2040", + VARIANT_RP2350: "RP2350", +} + +# Map BOARDS[board]["mcu"] (lowercase) to canonical variant constant +MCU_TO_VARIANT = { + "rp2040": VARIANT_RP2040, + "rp2350": VARIANT_RP2350, +} + +# Default board chosen when only `variant` is specified — the Raspberry Pi +# Foundation reference boards (Pico W / Pico 2 W). +STANDARD_BOARDS = { + VARIANT_RP2040: "rpipicow", + VARIANT_RP2350: "rpipico2w", +} rp2040_ns = cg.esphome_ns.namespace("rp2040") diff --git a/esphome/core/defines.h b/esphome/core/defines.h index ee8e89de8b..3cb92616bb 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -401,6 +401,7 @@ #define USE_LOGGER_USB_CDC #define USE_SOCKET_IMPL_LWIP_TCP #define USE_RP2040_BLE +#define USE_RP2040_VARIANT_RP2040 #define USE_SPI #ifndef USE_ETHERNET #define USE_ETHERNET diff --git a/tests/components/rp2040/test.rp2040-ard.yaml b/tests/components/rp2040/test.rp2040-ard.yaml index 1eb315a3b4..09531f914e 100644 --- a/tests/components/rp2040/test.rp2040-ard.yaml +++ b/tests/components/rp2040/test.rp2040-ard.yaml @@ -1,4 +1,5 @@ rp2040: + variant: rp2040 enable_full_printf: false logger: diff --git a/tests/components/rp2040/test.rp2040-pico2-ard.yaml b/tests/components/rp2040/test.rp2040-pico2-ard.yaml new file mode 100644 index 0000000000..c9d795840d --- /dev/null +++ b/tests/components/rp2040/test.rp2040-pico2-ard.yaml @@ -0,0 +1,6 @@ +rp2040: + variant: rp2350 + enable_full_printf: false + +logger: + level: VERBOSE diff --git a/tests/unit_tests/components/test_rp2040.py b/tests/unit_tests/components/test_rp2040.py index 25a9ade567..8e726933ed 100644 --- a/tests/unit_tests/components/test_rp2040.py +++ b/tests/unit_tests/components/test_rp2040.py @@ -1,6 +1,11 @@ -"""Tests for RP2040 component public helpers.""" +"""Tests for RP2040 component public helpers and variant detection.""" -from esphome.components.rp2040 import board_id_has_wifi +import pytest + +from esphome.components.rp2040 import _detect_variant, board_id_has_wifi +from esphome.components.rp2040.const import VARIANT_RP2040, VARIANT_RP2350 +import esphome.config_validation as cv +from esphome.const import CONF_BOARD, CONF_VARIANT def test_board_id_has_wifi_for_known_wifi_board() -> None: @@ -27,3 +32,61 @@ def test_board_id_has_wifi_for_unknown_board_returns_true() -> None: "no CYW43" guard at compile time. """ assert board_id_has_wifi("not-a-real-board-id") is True + + +def test_detect_variant_derives_variant_from_board() -> None: + """Board alone resolves to the matching variant.""" + result = _detect_variant({CONF_BOARD: "rpipicow"}) + assert result[CONF_BOARD] == "rpipicow" + assert result[CONF_VARIANT] == VARIANT_RP2040 + + +def test_detect_variant_derives_variant_from_rp2350_board() -> None: + """An RP2350 board resolves to ``RP2350``.""" + result = _detect_variant({CONF_BOARD: "rpipico2"}) + assert result[CONF_BOARD] == "rpipico2" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_only_picks_default_board_rp2040() -> None: + """Variant alone picks Pico W as the canonical RP2040 board.""" + result = _detect_variant({CONF_VARIANT: VARIANT_RP2040}) + assert result[CONF_BOARD] == "rpipicow" + assert result[CONF_VARIANT] == VARIANT_RP2040 + + +def test_detect_variant_only_picks_default_board_rp2350() -> None: + """Variant alone picks Pico 2 W as the canonical RP2350 board.""" + result = _detect_variant({CONF_VARIANT: VARIANT_RP2350}) + assert result[CONF_BOARD] == "rpipico2w" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_matching_explicit_variant_passes() -> None: + """Specifying both a board and the matching variant is allowed.""" + result = _detect_variant({CONF_BOARD: "rpipico2", CONF_VARIANT: VARIANT_RP2350}) + assert result[CONF_BOARD] == "rpipico2" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_mismatched_variant_raises() -> None: + """Board/variant mismatch must be rejected and name the offending board.""" + with pytest.raises( + cv.Invalid, match=r"does not match the selected board 'rpipicow'" + ): + _detect_variant({CONF_BOARD: "rpipicow", CONF_VARIANT: VARIANT_RP2350}) + + +def test_detect_variant_unknown_board_without_variant_raises() -> None: + """Unknown board with no variant tells the user how to recover.""" + with pytest.raises(cv.Invalid, match="please specify the chip variant"): + _detect_variant({CONF_BOARD: "not-a-real-board"}) + + +def test_detect_variant_unknown_board_with_variant_passes() -> None: + """Unknown board + explicit variant is accepted (with a warning).""" + result = _detect_variant( + {CONF_BOARD: "not-a-real-board", CONF_VARIANT: VARIANT_RP2040} + ) + assert result[CONF_BOARD] == "not-a-real-board" + assert result[CONF_VARIANT] == VARIANT_RP2040 From e0167e9bdff94e5a0aace72fbcb010ea1d9022d6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 24 May 2026 20:17:51 -0500 Subject: [PATCH 42/96] [lvgl] Memoize obj_schema by widget_type (#16615) --- esphome/components/lvgl/schemas.py | 14 +++- .../lvgl/test_obj_schema_cache.py | 67 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tests/component_tests/lvgl/test_obj_schema_cache.py diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index 58ef88d6a8..7436581fb4 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -462,13 +462,23 @@ def base_update_schema(widget_type: WidgetType | LvType, parts): return schema +# Widget types are module-level singletons populated at import time, so we +# can cache compiled obj_schemas by widget_type identity for the lifetime of +# the process. The strong reference in the value keeps the key (an id() +# target) from being recycled. +_OBJ_SCHEMA_CACHE: dict[int, tuple[WidgetType, cv.Schema]] = {} + + def obj_schema(widget_type: WidgetType): """ Create a schema for a widget type itself i.e. no allowance for children :param widget_type: :return: """ - return ( + cached = _OBJ_SCHEMA_CACHE.get(id(widget_type)) + if cached is not None and cached[0] is widget_type: + return cached[1] + schema = ( part_schema(widget_type.parts) .extend(ALIGN_TO_SCHEMA) .extend(automation_schema(widget_type.w_type)) @@ -479,6 +489,8 @@ def obj_schema(widget_type: WidgetType): } ) ) + _OBJ_SCHEMA_CACHE[id(widget_type)] = (widget_type, schema) + return schema ALIGN_TO_SCHEMA = { diff --git a/tests/component_tests/lvgl/test_obj_schema_cache.py b/tests/component_tests/lvgl/test_obj_schema_cache.py new file mode 100644 index 0000000000..860ee211dd --- /dev/null +++ b/tests/component_tests/lvgl/test_obj_schema_cache.py @@ -0,0 +1,67 @@ +"""Tests for obj_schema() memoization.""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest + +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import WIDGET_TYPES, obj_schema + + +@pytest.fixture(autouse=True) +def _clear_obj_schema_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_OBJ_SCHEMA_CACHE", None) + if cache is not None: + cache.clear() + yield + if cache is not None: + cache.clear() + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_same_widget_type_returns_same_schema() -> None: + wt = _widget_type("obj") + assert obj_schema(wt) is obj_schema(wt) + + +def test_different_widget_types_return_different_schemas() -> None: + assert obj_schema(_widget_type("obj")) is not obj_schema(_widget_type("label")) + + +def test_cache_is_populated_after_first_call() -> None: + wt = _widget_type("obj") + assert id(wt) not in lvgl_schemas._OBJ_SCHEMA_CACHE + obj_schema(wt) + assert id(wt) in lvgl_schemas._OBJ_SCHEMA_CACHE + + +def test_cached_schema_produces_equivalent_output() -> None: + wt = _widget_type("obj") + cached_result = obj_schema(wt)({}) + lvgl_schemas._OBJ_SCHEMA_CACHE.clear() + fresh_result = obj_schema(wt)({}) + assert cached_result == fresh_result + + +def test_id_recycling_is_caught_by_identity_guard() -> None: + wt = _widget_type("obj") + real_schema = obj_schema(wt) + + cached_widget_type, _ = lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] + sentinel_schema = object() + lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] = (cached_widget_type, sentinel_schema) + assert obj_schema(wt) is sentinel_schema + + other = _widget_type("label") + lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] = (other, sentinel_schema) + rebuilt = obj_schema(wt) + assert rebuilt is not sentinel_schema + assert rebuilt is not real_schema From e7ab78366d184a912f5f76d717f6aa387b7cd21c Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 25 May 2026 10:03:38 -0400 Subject: [PATCH 43/96] [core] Add esphome.build_flags option for IDF + PlatformIO (#16629) --- esphome/const.py | 1 + esphome/core/config.py | 11 +++++++++++ script/ci-custom.py | 2 +- tests/components/esphome/common.yaml | 2 ++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/esphome/const.py b/esphome/const.py index 9dd77a7cb8..07f6bad771 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -199,6 +199,7 @@ CONF_BROKER = "broker" CONF_BSSID = "bssid" CONF_BUFFER_DURATION = "buffer_duration" CONF_BUFFER_SIZE = "buffer_size" +CONF_BUILD_FLAGS = "build_flags" CONF_BUILD_PATH = "build_path" CONF_BUS_VOLTAGE = "bus_voltage" CONF_BUSY_PIN = "busy_pin" diff --git a/esphome/core/config.py b/esphome/core/config.py index 5a98b94781..6125c4ecc9 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -13,6 +13,7 @@ from esphome.const import ( CONF_AREA, CONF_AREA_ID, CONF_AREAS, + CONF_BUILD_FLAGS, CONF_BUILD_PATH, CONF_COMMENT, CONF_COMPILE_PROCESS_LIMIT, @@ -288,6 +289,7 @@ CONFIG_SCHEMA = cv.All( cv.string_strict: cv.Any([cv.string], cv.string), } ), + cv.Optional(CONF_BUILD_FLAGS, default=[]): cv.ensure_list(cv.string_strict), cv.Optional(CONF_ENVIRONMENT_VARIABLES, default={}): cv.Schema( { cv.string_strict: cv.string, @@ -510,6 +512,12 @@ async def _add_platformio_options(pio_options): cg.add_platformio_option(key, val) +@coroutine_with_priority(CoroPriority.FINAL) +async def _add_build_flags(flags: list[str]) -> None: + for flag in flags: + cg.add_build_flag(flag) + + @coroutine_with_priority(CoroPriority.FINAL) async def _add_environment_variables(env_vars: dict[str, str]) -> None: # Set environment variables for the build process @@ -705,6 +713,9 @@ async def to_code(config: ConfigType) -> None: if config[CONF_PLATFORMIO_OPTIONS]: CORE.add_job(_add_platformio_options, config[CONF_PLATFORMIO_OPTIONS]) + if config[CONF_BUILD_FLAGS]: + CORE.add_job(_add_build_flags, config[CONF_BUILD_FLAGS]) + if config[CONF_ENVIRONMENT_VARIABLES]: CORE.add_job(_add_environment_variables, config[CONF_ENVIRONMENT_VARIABLES]) diff --git a/script/ci-custom.py b/script/ci-custom.py index 56ca0d0355..51fea97874 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -562,7 +562,7 @@ def lint_constants_usage(): # Maximum allowed CONF_ constants in esphome/const.py. # This file is frozen — new constants go in esphome/components/const/__init__.py. # Decrease this number when constants are moved out of const.py. -CONST_PY_MAX_CONF = 1012 +CONST_PY_MAX_CONF = 1013 @lint_content_check(include=["esphome/const.py"]) diff --git a/tests/components/esphome/common.yaml b/tests/components/esphome/common.yaml index db75b08b38..93f82824e6 100644 --- a/tests/components/esphome/common.yaml +++ b/tests/components/esphome/common.yaml @@ -2,6 +2,8 @@ esphome: debug_scheduler: true platformio_options: board_build.flash_mode: dio + build_flags: + - "-DESPHOME_TEST_BUILD_FLAG" environment_variables: TEST_ENV_VAR: "test_value" BUILD_NUMBER: "12345" From 98e72133872cd7b198df16e6468dc874224f6f60 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 25 May 2026 10:08:16 -0400 Subject: [PATCH 44/96] [espidf] Warn instead of skipping libraries with framework mismatch (#16630) --- esphome/espidf/component.py | 23 +++++++++++++++++++---- tests/unit_tests/test_espidf_component.py | 11 ++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index a452a3f34a..3534ac82f5 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -610,11 +610,17 @@ def _check_library_data(data: dict): """ Check if a library data is compatible with the ESP-IDF framework. + A platform mismatch (e.g. an AVR-only library on ESP32) raises + ``InvalidIDFComponent`` so the caller skips the library. A framework + mismatch only logs a warning — PIO manifests often understate the + frameworks they actually compile under, and IDF (unlike PIO's + ``lib_compat_mode``) has no opt-out, so we include the library anyway. + Args: - component: IDFComponent object being processed + data: PIO library manifest dict being processed. Raises: - ValueError: If library has unsupported platforms or frameworks + InvalidIDFComponent: If the library does not support the ESP32 platform. """ platforms = data.get("platforms", "*") if isinstance(platforms, str): @@ -632,12 +638,21 @@ def _check_library_data(data: dict): frameworks = [a.strip() for a in frameworks.split(",")] frameworks = _ensure_list(frameworks) - # Check if library supports ESP-IDF framework + # Check if library declares the active framework. PIO library manifests + # often list only "arduino" even when the library actually compiles fine + # under ESP-IDF, and IDF (unlike PIO with `lib_compat_mode`) has no way to + # opt out of the check. Warn instead of failing so the user isn't forced to + # fork the library to fix the manifest. framework = "arduino" if CORE.using_arduino else "espidf" valid_framework = "*" in frameworks or framework in frameworks if not valid_framework: - raise InvalidIDFComponent(f"Unsupported library frameworks: {frameworks}") + _LOGGER.warning( + "Library %s declares frameworks %s that do not include '%s'; including anyway", + data.get("name", ""), + frameworks, + framework, + ) def _process_dependencies(component: IDFComponent): diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 7d6c861ffd..f50f5317de 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -261,9 +261,14 @@ def test_check_library_data_invalid_platform(esp32_idf_core): _check_library_data({"platforms": ["other"], "frameworks": "*"}) -def test_check_library_data_invalid_framework(esp32_idf_core): - with pytest.raises(InvalidIDFComponent): - _check_library_data({"platforms": "*", "frameworks": ["other"]}) +def test_check_library_data_invalid_framework( + esp32_idf_core: None, caplog: pytest.LogCaptureFixture +) -> None: + # Framework mismatch is a warning, not a hard skip: the library is still + # included so that PIO manifests that only list "arduino" (but actually + # compile under IDF) can be used without forking them. + _check_library_data({"name": "lib", "platforms": "*", "frameworks": ["other"]}) + assert "do not include 'espidf'" in caplog.text def test_extra_script_captures_libpath_libs_and_defines(tmp_path): From cde52ef75e3c9f633883719f1c15b79041ca9efa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 09:09:54 -0500 Subject: [PATCH 45/96] [lvgl] Merge dict-extend chains to speed up schema construction (#16614) --- esphome/components/lvgl/__init__.py | 27 +- esphome/components/lvgl/schemas.py | 77 ++++-- .../lvgl/test_schema_dict_helpers.py | 236 ++++++++++++++++++ 3 files changed, 319 insertions(+), 21 deletions(-) create mode 100644 tests/component_tests/lvgl/test_schema_dict_helpers.py diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index 4277c14dd7..44bcda9ba9 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -1,3 +1,4 @@ +import functools import importlib from pathlib import Path import pkgutil @@ -79,7 +80,7 @@ from .schemas import ( WIDGET_TYPES, any_widget_schema, container_schema, - obj_schema, + obj_dict, ) from .styles import styles_to_code, theme_to_code from .touchscreens import touchscreen_schema, touchscreens_to_code @@ -518,16 +519,32 @@ def add_hello_world(config): return config -def _theme_schema(value): +@functools.cache +def _build_theme_schema( + widget_types: tuple[tuple[str, widgets.WidgetType], ...], +) -> cv.Schema: + # The theme schema is value-independent: it depends only on the set of + # registered widget types. Key the cache on a snapshot of WIDGET_TYPES so + # that an external component registering a new widget after the first + # validation (legal per any_widget_schema's lazy-evaluation contract) + # produces a fresh tuple, a cache miss, and a rebuilt schema -- the cache + # self-heals instead of stale-rejecting valid themes. See obj_dict() in + # schemas.py for why chained .extend() is avoided here. return cv.Schema( { cv.Optional(df.CONF_DARK_MODE, default=False): cv.boolean, **{ - cv.Optional(name): obj_schema(w).extend(FULL_STYLE_SCHEMA) - for name, w in WIDGET_TYPES.items() + cv.Optional(name): cv.Schema( + {**obj_dict(w), **FULL_STYLE_SCHEMA.schema} + ) + for name, w in widget_types }, } - )(value) + ) + + +def _theme_schema(value: dict) -> dict: + return _build_theme_schema(tuple(WIDGET_TYPES.items()))(value) FINAL_VALIDATE_SCHEMA = final_validation diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index 7436581fb4..b901eb4b53 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -378,15 +378,33 @@ TRIGGER_EVENT_MAP = { } -def part_schema(parts): +def part_dict(parts: tuple[str, ...] | list[str]) -> dict[Any, Any]: + """ + Return the raw mapping used by part_schema, so callers can merge it into a + larger dict and avoid chained .extend() calls (each .extend() recompiles the + whole mapping, turning the build into O(N^2)). + + Invariant: the source schemas spread here (STATE_SCHEMA, FLAG_SCHEMA, the + nested STATE_SCHEMA values) must use the default extra=PREVENT_EXTRA and + required=False and must not register any add_extra/prepend_extra + validators. Reaching into .schema and rebuilding via cv.Schema(...) keeps + only the mapping; non-default extra/required and any _extra_schemas would + be silently dropped. + """ + return { + **STATE_SCHEMA.schema, + **FLAG_SCHEMA.schema, + **{cv.Optional(part): STATE_SCHEMA for part in parts}, + } + + +def part_schema(parts: tuple[str, ...] | list[str]) -> cv.Schema: """ Generate a schema for the various parts (e.g. main:, indicator:) of a widget type :param parts: The parts to include :return: The schema """ - return STATE_SCHEMA.extend(FLAG_SCHEMA).extend( - {cv.Optional(part): STATE_SCHEMA for part in parts} - ) + return cv.Schema(part_dict(parts)) def automation_schema(typ: LvType): @@ -462,6 +480,43 @@ def base_update_schema(widget_type: WidgetType | LvType, parts): return schema +# Memoize obj_dict() the same way _OBJ_SCHEMA_CACHE memoizes obj_schema(). +# automation_schema(w.w_type) builds fresh Trigger.template(...) objects on +# every call, so without this cache _theme_schema pays that cost per widget +# per validation. Callers must treat the returned dict as immutable. The +# _theme_schema caller spreads it into a fresh dict, which is safe; the +# obj_schema caller passes it directly to cv.Schema(...) -- voluptuous stores +# the mapping by reference but never mutates it (.extend() copies first), so +# the alias is also safe today. Adding in-place mutation of obj_schema(w).schema +# would corrupt this cache. +_OBJ_DICT_CACHE: dict[int, tuple[WidgetType, dict[Any, Any]]] = {} + + +def obj_dict(widget_type: WidgetType) -> dict[Any, Any]: + """ + Return the raw mapping used by obj_schema, so callers can merge it into a + larger dict and avoid chained .extend() calls. + + Inherits the same source-schema invariant documented on part_dict: any + schema spread into this mapping must use the default extra=PREVENT_EXTRA + and required=False and must carry no add_extra/prepend_extra validators. + + The returned mapping is cached and must be treated as immutable by callers. + """ + cached = _OBJ_DICT_CACHE.get(id(widget_type)) + if cached is not None and cached[0] is widget_type: + return cached[1] + built = { + **part_dict(widget_type.parts), + **ALIGN_TO_SCHEMA, + **automation_schema(widget_type.w_type), + cv.Optional(CONF_STATE): SET_STATE_SCHEMA, + cv.Optional(CONF_GROUP): cv.use_id(lv_group_t), + } + _OBJ_DICT_CACHE[id(widget_type)] = (widget_type, built) + return built + + # Widget types are module-level singletons populated at import time, so we # can cache compiled obj_schemas by widget_type identity for the lifetime of # the process. The strong reference in the value keeps the key (an id() @@ -469,7 +524,7 @@ def base_update_schema(widget_type: WidgetType | LvType, parts): _OBJ_SCHEMA_CACHE: dict[int, tuple[WidgetType, cv.Schema]] = {} -def obj_schema(widget_type: WidgetType): +def obj_schema(widget_type: WidgetType) -> cv.Schema: """ Create a schema for a widget type itself i.e. no allowance for children :param widget_type: @@ -478,17 +533,7 @@ def obj_schema(widget_type: WidgetType): cached = _OBJ_SCHEMA_CACHE.get(id(widget_type)) if cached is not None and cached[0] is widget_type: return cached[1] - schema = ( - part_schema(widget_type.parts) - .extend(ALIGN_TO_SCHEMA) - .extend(automation_schema(widget_type.w_type)) - .extend( - { - cv.Optional(CONF_STATE): SET_STATE_SCHEMA, - cv.Optional(CONF_GROUP): cv.use_id(lv_group_t), - } - ) - ) + schema = cv.Schema(obj_dict(widget_type)) _OBJ_SCHEMA_CACHE[id(widget_type)] = (widget_type, schema) return schema diff --git a/tests/component_tests/lvgl/test_schema_dict_helpers.py b/tests/component_tests/lvgl/test_schema_dict_helpers.py new file mode 100644 index 0000000000..16714f54d7 --- /dev/null +++ b/tests/component_tests/lvgl/test_schema_dict_helpers.py @@ -0,0 +1,236 @@ +"""Tests for part_dict / obj_dict / part_schema / obj_schema mapping contracts. + +These guard the dict-merge refactor: the dict helpers must keep returning the +same logical mapping as the chained-extend version produced, and the +corresponding Schema(...) wrappers must accept and reject the same configs. +""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest +import voluptuous as vol + +from esphome import config_validation as cv +import esphome.components.lvgl +from esphome.components.lvgl import ( + _theme_schema, + defines as df, + schemas as lvgl_schemas, +) +from esphome.components.lvgl.schemas import ( + ALIGN_TO_SCHEMA, + FLAG_SCHEMA, + FULL_STYLE_SCHEMA, + STATE_SCHEMA, + STYLE_SCHEMA, + WIDGET_TYPES, + automation_schema, + obj_dict, + obj_schema, + part_dict, + part_schema, +) +from esphome.components.lvgl.types import LvType +from esphome.components.lvgl.widgets import WidgetType + + +@pytest.fixture(autouse=True) +def _clear_obj_dict_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_OBJ_DICT_CACHE", None) + if cache is not None: + cache.clear() + # The lazily-built theme schema is cached on _build_theme_schema; clear it + # too so each test starts from a clean slate. + build_theme = getattr(esphome.components.lvgl, "_build_theme_schema", None) + if build_theme is not None and hasattr(build_theme, "cache_clear"): + build_theme.cache_clear() + yield + if cache is not None: + cache.clear() + if build_theme is not None and hasattr(build_theme, "cache_clear"): + build_theme.cache_clear() + + +def _marker_names(mapping) -> set[str]: + """Return the underlying string names of every voluptuous Marker key.""" + names: set[str] = set() + for key in mapping: + if isinstance(key, vol.Marker): + schema = key.schema + if isinstance(schema, str): + names.add(schema) + return names + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_part_dict_includes_state_flag_and_part_keys() -> None: + parts = ("indicator", "knob") + keys = _marker_names(part_dict(parts)) + + assert {"indicator", "knob"} <= keys + assert _marker_names(STATE_SCHEMA.schema) <= keys + assert _marker_names(FLAG_SCHEMA.schema) <= keys + + +def test_obj_dict_extends_part_dict_with_align_automation_state_group() -> None: + wt = _widget_type("obj") + part_keys = _marker_names(part_dict(wt.parts)) + obj_keys = _marker_names(obj_dict(wt)) + + assert part_keys <= obj_keys + assert _marker_names(ALIGN_TO_SCHEMA) <= obj_keys + assert _marker_names(automation_schema(wt.w_type)) <= obj_keys + assert {"state", "group"} <= obj_keys + + +def test_obj_dict_is_memoized_by_widget_type() -> None: + wt = _widget_type("obj") + first = obj_dict(wt) + second = obj_dict(wt) + assert first is second + # Different widget type, different dict. + assert obj_dict(_widget_type("label")) is not first + + +def test_part_schema_round_trips_known_state_and_part_settings() -> None: + schema = part_schema(("indicator",)) + out = schema( + { + "bg_color": 0x112233, + "checked": {"bg_color": 0x445566}, + "indicator": {"bg_color": 0x778899}, + } + ) + assert out["bg_color"] == 0x112233 + assert out["checked"]["bg_color"] == 0x445566 + assert out["indicator"]["bg_color"] == 0x778899 + + +def test_part_schema_rejects_unknown_part() -> None: + schema = part_schema(("indicator",)) + with pytest.raises(vol.Invalid): + schema({"definitely_not_a_part": {}}) + + +@pytest.mark.parametrize("name", sorted(WIDGET_TYPES)) +def test_obj_schema_accepts_empty_config_for_every_widget_type(name: str) -> None: + obj_schema(_widget_type(name))({}) + + +def test_obj_schema_accepts_align_to_and_state_group() -> None: + schema = obj_schema(_widget_type("obj")) + out = schema( + { + df.CONF_ALIGN_TO: { + "id": "some_other_widget", + df.CONF_ALIGN: "TOP_LEFT", + }, + "state": {"checked": True}, + } + ) + assert out[df.CONF_ALIGN_TO][df.CONF_ALIGN] == "LV_ALIGN_TOP_LEFT" + assert out["state"]["checked"] is True + + +def test_obj_schema_rejects_unknown_top_level_key() -> None: + with pytest.raises(vol.Invalid): + obj_schema(_widget_type("obj"))({"definitely_not_a_real_key": 1}) + + +def test_part_schema_returns_cv_schema_for_extend_callers() -> None: + schema = part_schema(("indicator",)) + extended = schema.extend({cv.Optional("extra_key"): cv.string}) + out = extended({"extra_key": "value", "bg_color": 0xAABBCC}) + assert out["extra_key"] == "value" + assert out["bg_color"] == 0xAABBCC + + +def test_obj_schema_returns_cv_schema_for_extend_callers() -> None: + schema = obj_schema(_widget_type("obj")) + extended = schema.extend({cv.Optional("extra_key"): cv.string}) + extended({"extra_key": "value"}) + + +@pytest.mark.parametrize( + "schema", + [STATE_SCHEMA, FLAG_SCHEMA, STYLE_SCHEMA, FULL_STYLE_SCHEMA], +) +def test_spread_sources_carry_no_extra_schemas(schema: cv.Schema) -> None: + # part_dict / obj_dict reach into .schema and rebuild via cv.Schema(...), + # which silently drops _extra_schemas and any non-default extra/required. + # Lock the invariant so a future add_extra() on these sources fails CI + # instead of quietly removing validation from part/obj/theme schemas. + assert not schema._extra_schemas + assert schema.extra is vol.PREVENT_EXTRA + assert schema.required is False + + +def test_theme_schema_merges_obj_dict_and_full_style_props() -> None: + # _theme_schema is the riskiest merge: obj_dict(w) and FULL_STYLE_SCHEMA.schema + # share many STYLE_SCHEMA marker instances. Exercise the merged schema + # end-to-end with one key from each side (a STATE_SCHEMA part from obj_dict + # and a FULL_STYLE-only property) to lock the behaviour against future + # regressions in either source. + out = _theme_schema( + { + df.CONF_DARK_MODE: True, + "obj": { + "bg_color": 0x112233, + "checked": {"bg_color": 0x445566}, + df.CONF_PAD_ROW: 4, + df.CONF_GRID_CELL_X_ALIGN: "CENTER", + }, + } + ) + assert out[df.CONF_DARK_MODE] is True + obj_out = out["obj"] + assert obj_out["bg_color"] == 0x112233 + assert obj_out["checked"]["bg_color"] == 0x445566 + assert obj_out[df.CONF_PAD_ROW] == 4 + assert obj_out[df.CONF_GRID_CELL_X_ALIGN] == "LV_GRID_ALIGN_CENTER" + + +def test_theme_schema_self_heals_when_a_widget_type_is_registered_later() -> None: + # _build_theme_schema is functools.cached on a snapshot of WIDGET_TYPES. + # any_widget_schema explicitly supports external components registering + # widgets lazily, and the device builder revalidates in-process, so a + # widget registered after first use must invalidate the cached snapshot. + _theme_schema({df.CONF_DARK_MODE: True}) # populate the cache + + name = "test_self_heal_widget" + assert name not in WIDGET_TYPES + # is_mock=True skips registration side-effects; insert into WIDGET_TYPES + # manually so the next theme call sees the new entry. + WIDGET_TYPES[name] = WidgetType(name, LvType("test_fake_t"), (), is_mock=True) + try: + out = _theme_schema({df.CONF_DARK_MODE: False, name: {"bg_color": 0x010203}}) + assert out[name]["bg_color"] == 0x010203 + finally: + WIDGET_TYPES.pop(name, None) + + +@pytest.mark.parametrize( + "schema", + [STATE_SCHEMA, FLAG_SCHEMA, STYLE_SCHEMA, FULL_STYLE_SCHEMA], +) +def test_spread_sources_have_no_top_level_marker_defaults(schema: cv.Schema) -> None: + # _theme_schema merges obj_dict(w) with FULL_STYLE_SCHEMA.schema; on a key + # collision, dict-spread keeps the first source's marker (and its default) + # but the last source's value, whereas .extend() would take both from the + # later source. The two are equivalent today because the overlapping + # markers are the same instances (both derive from STYLE_SCHEMA) and none + # carry a top-level default. Lock that so a future divergent default would + # fail CI rather than silently drift the merged validation. + offenders = [ + marker.schema + for marker in schema.schema + if isinstance(marker, vol.Optional) and marker.default is not vol.UNDEFINED + ] + assert not offenders, f"top-level Optional with default: {offenders}" From cf1fabe6d4d4e14936ce496fc4445a04ca62654f Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 25 May 2026 10:11:31 -0400 Subject: [PATCH 46/96] [esp32_hosted] Bump esp_hosted to 2.12.8 and add use_psram option (#16627) --- esphome/components/esp32_hosted/__init__.py | 10 +++++++++- esphome/idf_component.yml | 2 +- tests/components/esp32_hosted/common.yaml | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index 71d1fd3ac1..94e20ea6c9 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -3,6 +3,7 @@ from pathlib import Path from esphome import pins from esphome.components import esp32 +from esphome.components.const import CONF_USE_PSRAM import esphome.config_validation as cv from esphome.const import ( CONF_CLK_PIN, @@ -39,6 +40,7 @@ BASE_SCHEMA = cv.Schema( cv.Required(CONF_VARIANT): cv.one_of(*esp32.VARIANTS, upper=True), cv.Required(CONF_ACTIVE_HIGH): cv.boolean, cv.Required(CONF_RESET_PIN): pins.internal_gpio_output_pin_number, + cv.Optional(CONF_USE_PSRAM, default=False): cv.boolean, } ) @@ -242,6 +244,12 @@ async def to_code(config): else: _configure_spi(config) + # Place the transport mempool in PSRAM. Required on memory-tight host + # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM + # mempool allocation fails at boot with `sdio_mempool_create` assert. + if config[CONF_USE_PSRAM]: + esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_MEMPOOL_PREFER_SPIRAM", True) + # Library versions idf_ver = esp32.idf_version() os.environ["ESP_IDF_VERSION"] = f"{idf_ver.major}.{idf_ver.minor}" @@ -249,7 +257,7 @@ async def to_code(config): esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.5.1") esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.2") esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5") - esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.7") + esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.8") else: esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="0.13.0") esp32.add_idf_component(name="espressif/eppp_link", ref="0.2.0") diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 6bc166ff44..5af25fc351 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -36,7 +36,7 @@ dependencies: rules: - if: "target in [esp32h2, esp32p4]" espressif/esp_hosted: - version: 2.12.7 + version: 2.12.8 rules: - if: "target in [esp32h2, esp32p4]" zorxx/multipart-parser: diff --git a/tests/components/esp32_hosted/common.yaml b/tests/components/esp32_hosted/common.yaml index ab029e5064..332fe5b070 100644 --- a/tests/components/esp32_hosted/common.yaml +++ b/tests/components/esp32_hosted/common.yaml @@ -3,6 +3,7 @@ esp32_hosted: slot: 1 active_high: true reset_pin: GPIO15 + use_psram: true cmd_pin: GPIO13 clk_pin: GPIO12 d0_pin: GPIO11 From 7c494fd3efcc9fc1c1dce0d6fc09d47a44affdfd Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 25 May 2026 10:15:51 -0400 Subject: [PATCH 47/96] [psram] Consolidate task stack in PSRAM handling (#16628) --- .../audio_file/media_source/__init__.py | 16 +++----------- esphome/components/audio_http/media_source.py | 18 +++------------ esphome/components/mixer/speaker/__init__.py | 13 +++++------ esphome/components/psram/__init__.py | 22 +++++++++++++++++++ .../components/resampler/speaker/__init__.py | 8 +++---- esphome/components/sendspin/__init__.py | 17 +++----------- .../sendspin/media_source/__init__.py | 5 ++--- .../speaker/media_player/__init__.py | 9 ++------ .../audio_file/validate.esp32-idf.yaml | 11 ++++++++++ 9 files changed, 54 insertions(+), 65 deletions(-) create mode 100644 tests/components/audio_file/validate.esp32-idf.yaml diff --git a/esphome/components/audio_file/media_source/__init__.py b/esphome/components/audio_file/media_source/__init__.py index 635a51b610..0710582813 100644 --- a/esphome/components/audio_file/media_source/__init__.py +++ b/esphome/components/audio_file/media_source/__init__.py @@ -1,7 +1,5 @@ -from typing import Any - import esphome.codegen as cg -from esphome.components import audio, esp32, media_source, psram +from esphome.components import audio, media_source, psram import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_TASK_STACK_IN_PSRAM from esphome.types import ConfigType @@ -21,19 +19,13 @@ def _request_micro_decoder(config: ConfigType) -> ConfigType: return config -def _validate_task_stack_in_psram(value: Any) -> bool: - if value := cv.boolean(value): - return cv.requires_component(psram.DOMAIN)(value) - return value - - CONFIG_SCHEMA = cv.All( media_source.media_source_schema( AudioFileMediaSource, ) .extend( { - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ) .extend(cv.COMPONENT_SCHEMA), @@ -49,6 +41,4 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() diff --git a/esphome/components/audio_http/media_source.py b/esphome/components/audio_http/media_source.py index 519d8df698..e8acbc81af 100644 --- a/esphome/components/audio_http/media_source.py +++ b/esphome/components/audio_http/media_source.py @@ -1,7 +1,5 @@ -from typing import Any - import esphome.codegen as cg -from esphome.components import audio, esp32, media_source, psram +from esphome.components import audio, media_source, psram import esphome.config_validation as cv from esphome.const import CONF_BUFFER_SIZE, CONF_ID, CONF_TASK_STACK_IN_PSRAM from esphome.types import ConfigType @@ -20,14 +18,6 @@ def _request_micro_decoder(config: ConfigType) -> ConfigType: return config -def _validate_task_stack_in_psram(value: Any) -> bool: - # Only require the psram component when actually enabling PSRAM stacks; validating - # the boolean first means `false` doesn't trigger the requires_component check. - if value := cv.boolean(value): - return cv.requires_component(psram.DOMAIN)(value) - return value - - CONFIG_SCHEMA = cv.All( media_source.media_source_schema( AudioHTTPMediaSource, @@ -37,7 +27,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_BUFFER_SIZE, default=50000): cv.int_range( min=5000, max=1000000 ), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ) .extend(cv.COMPONENT_SCHEMA), @@ -53,7 +43,5 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) diff --git a/esphome/components/mixer/speaker/__init__.py b/esphome/components/mixer/speaker/__init__.py index 8501843d3f..47164a9997 100644 --- a/esphome/components/mixer/speaker/__init__.py +++ b/esphome/components/mixer/speaker/__init__.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import audio, esp32, speaker +from esphome.components import audio, psram, speaker import esphome.config_validation as cv from esphome.const import ( CONF_BITS_PER_SAMPLE, @@ -93,7 +93,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_BITS_PER_SAMPLE): cv.one_of(8, 16, 24, 32, int=True), cv.Optional(CONF_NUM_CHANNELS): cv.int_range(min=1, max=2), cv.Optional(CONF_QUEUE_MODE, default=False): cv.boolean, - cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ), cv.only_on([PLATFORM_ESP32]), @@ -123,12 +123,9 @@ async def to_code(config): cg.add(var.set_output_speaker(spkr)) cg.add(var.set_queue_mode(config[CONF_QUEUE_MODE])) - if task_stack_in_psram := config.get(CONF_TASK_STACK_IN_PSRAM): - cg.add(var.set_task_stack_in_psram(task_stack_in_psram)) - if task_stack_in_psram and config[CONF_TASK_STACK_IN_PSRAM]: - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + if config.get(CONF_TASK_STACK_IN_PSRAM): + cg.add(var.set_task_stack_in_psram(True)) + psram.request_external_task_stack() # Initialize FixedVector with exact count of source speakers cg.add(var.init_source_speakers(len(config[CONF_SOURCE_SPEAKERS]))) diff --git a/esphome/components/psram/__init__.py b/esphome/components/psram/__init__.py index 86c17ce9ca..d36d900997 100644 --- a/esphome/components/psram/__init__.py +++ b/esphome/components/psram/__init__.py @@ -1,5 +1,6 @@ import logging import textwrap +from typing import Any import esphome.codegen as cg from esphome.components.const import CONF_IGNORE_NOT_FOUND @@ -94,6 +95,27 @@ def is_guaranteed() -> bool: return CORE.data.get(KEY_PSRAM_GUARANTEED, False) +def request_external_task_stack() -> None: + """Allow FreeRTOS task stacks to be allocated in external RAM (PSRAM). + + Components that expose a ``task_stack_in_psram`` option should call this from their + ``to_code`` when the option is enabled. The sdkconfig option only permits external + stacks; it does not move any stack into PSRAM on its own, so it stays opt-in per task. + """ + add_idf_sdkconfig_option("CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True) + + +def validate_task_stack_in_psram(value: Any) -> bool: + """Validate a ``task_stack_in_psram`` boolean, requiring the psram component only when enabled. + + Validating the boolean first means an explicit ``false`` does not pull in the psram + requirement, so the option can still be set to false on devices without PSRAM. + """ + if value := cv.boolean(value): + return cv.requires_component(DOMAIN)(value) + return value + + def validate_psram_mode(config): esp32_config = fv.full_config.get()[PLATFORM_ESP32] if config[CONF_SPEED] == "120MHZ": diff --git a/esphome/components/resampler/speaker/__init__.py b/esphome/components/resampler/speaker/__init__.py index 3134cf7646..8a13110631 100644 --- a/esphome/components/resampler/speaker/__init__.py +++ b/esphome/components/resampler/speaker/__init__.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import audio, esp32, speaker +from esphome.components import audio, psram, speaker import esphome.config_validation as cv from esphome.const import ( CONF_BITS_PER_SAMPLE, @@ -63,7 +63,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional( CONF_BUFFER_DURATION, default="100ms" ): cv.positive_time_period_milliseconds, - cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_FILTERS, default=16): cv.int_range(min=2, max=1024), cv.Optional(CONF_TAPS, default=16): _validate_taps, } @@ -88,9 +88,7 @@ async def to_code(config): if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_target_bits_per_sample(config[CONF_BITS_PER_SAMPLE])) cg.add(var.set_target_sample_rate(config[CONF_SAMPLE_RATE])) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index b670bd3c4d..e8c643f9b9 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -121,13 +121,6 @@ def register_player_config(config: ConfigType) -> None: data.player_config = config -def _validate_task_stack_in_psram(value): - value = cv.boolean(value) - if value: - return cv.requires_component(psram.DOMAIN)(value) - return value - - def _request_high_performance_networking(config: ConfigType) -> ConfigType: """Request high performance networking for Sendspin streaming. @@ -152,7 +145,7 @@ CONFIG_SCHEMA = cv.All( cv.Schema( { cv.GenerateID(): cv.declare_id(SendspinHub), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ), cv.only_on_esp32, @@ -201,9 +194,7 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() # sendspin-cpp library esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.6.1") @@ -261,9 +252,7 @@ async def to_code(config: ConfigType) -> None: psram_stack = player_cfg.get(CONF_TASK_STACK_IN_PSRAM, False) if psram_stack: - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() # Library defaults: priority 18 (one above httpd_priority 17 so the decoder is not # starved by the HTTP server during the initial encoded-audio burst at stream start), diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index f689ab01cb..6af244d41f 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import media_source +from esphome.components import media_source, psram import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, @@ -19,7 +19,6 @@ from .. import ( CONF_SENDSPIN_ID, MEMORY_LOCATIONS, SendspinHub, - _validate_task_stack_in_psram, register_player_config, request_controller_support, sendspin_ns, @@ -71,7 +70,7 @@ CONFIG_SCHEMA = cv.All( ).extend( { cv.GenerateID(CONF_SENDSPIN_ID): cv.use_id(SendspinHub), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_BUFFER_SIZE, default=1000000): cv.int_range(min=25000), cv.Optional(CONF_INITIAL_STATIC_DELAY, default="0ms"): cv.All( cv.positive_time_period_milliseconds, diff --git a/esphome/components/speaker/media_player/__init__.py b/esphome/components/speaker/media_player/__init__.py index 094043c292..90eb19d73d 100644 --- a/esphome/components/speaker/media_player/__init__.py +++ b/esphome/components/speaker/media_player/__init__.py @@ -7,7 +7,6 @@ import esphome.codegen as cg from esphome.components import ( audio, audio_file, - esp32, media_player, network, ota, @@ -155,9 +154,7 @@ CONFIG_SCHEMA = cv.All( # Remove before 2026.10.0 cv.Optional(CONF_CODEC_SUPPORT_ENABLED): cv.Any(cv.boolean, cv.string), cv.Optional(CONF_FILES): audio_file.audio_files_schema(), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): cv.All( - cv.boolean, cv.requires_component(psram.DOMAIN) - ), + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_VOLUME_INCREMENT, default=0.05): cv.percentage, cv.Optional(CONF_VOLUME_INITIAL, default=0.5): cv.percentage, cv.Optional(CONF_VOLUME_MAX, default=1.0): cv.percentage, @@ -198,9 +195,7 @@ async def to_code(config): if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_volume_increment(config[CONF_VOLUME_INCREMENT])) cg.add(var.set_volume_initial(config[CONF_VOLUME_INITIAL])) diff --git a/tests/components/audio_file/validate.esp32-idf.yaml b/tests/components/audio_file/validate.esp32-idf.yaml new file mode 100644 index 0000000000..085f853c8e --- /dev/null +++ b/tests/components/audio_file/validate.esp32-idf.yaml @@ -0,0 +1,11 @@ +audio_file: + - id: test_audio + file: + type: local + path: $component_dir/test.wav + +media_source: + - platform: audio_file + id: audio_file_source + # task_stack_in_psram: false must validate without a psram: component + task_stack_in_psram: false From 684bce8b9a588a0a939673298158b3b9380dc633 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 25 May 2026 10:36:41 -0400 Subject: [PATCH 48/96] [esp32] Decode crash PCs via IDF toolchain on IDF builds (#16626) --- esphome/components/esp32/__init__.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index a06ae89c3e..e3bff8f934 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -46,7 +46,7 @@ from esphome.const import ( Toolchain, __version__, ) -from esphome.core import CORE, HexInt, Library +from esphome.core import CORE, EsphomeError, HexInt, Library from esphome.core.config import BOARD_MAX_LENGTH from esphome.coroutine import CoroPriority, coroutine_with_priority from esphome.espidf.component import generate_idf_component @@ -2658,13 +2658,29 @@ def copy_files(): def _decode_pc(config, addr): - from esphome.platformio import toolchain + # _decode_pc runs from the api log processor's asyncio callback, which + # only catches EsphomeError. Any other exception escaping here tears down + # the protocol and triggers an infinite reconnect/replay loop. Convert + # toolchain-resolution errors (e.g. missing build dir / cmake cache) into + # EsphomeError so the caller can disable decoding cleanly. + if CORE.using_toolchain_esp_idf: + from esphome.espidf import toolchain as idf_toolchain - idedata = toolchain.get_idedata(config) - if not idedata.addr2line_path or not idedata.firmware_elf_path: + try: + addr2line_path = idf_toolchain.get_addr2line_path() + firmware_elf_path = idf_toolchain.get_elf_path() + except RuntimeError as err: + raise EsphomeError(f"ESP-IDF toolchain not available: {err}") from err + else: + from esphome.platformio import toolchain + + idedata = toolchain.get_idedata(config) + addr2line_path = idedata.addr2line_path + firmware_elf_path = idedata.firmware_elf_path + if not addr2line_path or not firmware_elf_path: _LOGGER.debug("decode_pc no addr2line") return - command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + command = [str(addr2line_path), "-pfiaC", "-e", str(firmware_elf_path), addr] try: translation = subprocess.check_output(command, close_fds=False).decode().strip() except Exception: # pylint: disable=broad-except From 1c7ae96e424cadc00bbc9b778fbb2327b8103020 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 25 May 2026 11:04:26 -0400 Subject: [PATCH 49/96] [micro_wake_word] Allow task stack to be allocated in PSRAM (#16632) --- .../components/micro_wake_word/__init__.py | 8 ++++++- .../micro_wake_word/micro_wake_word.cpp | 24 +++++++------------ .../micro_wake_word/micro_wake_word.h | 8 ++++++- tests/components/micro_wake_word/common.yaml | 4 ++++ 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/esphome/components/micro_wake_word/__init__.py b/esphome/components/micro_wake_word/__init__.py index 38926fce99..61d296cbba 100644 --- a/esphome/components/micro_wake_word/__init__.py +++ b/esphome/components/micro_wake_word/__init__.py @@ -7,7 +7,7 @@ from urllib.parse import urljoin from esphome import automation, external_files, git from esphome.automation import register_action, register_condition import esphome.codegen as cg -from esphome.components import esp32, microphone, ota +from esphome.components import esp32, microphone, ota, psram import esphome.config_validation as cv from esphome.const import ( CONF_FILE, @@ -20,6 +20,7 @@ from esphome.const import ( CONF_RAW_DATA_ID, CONF_REF, CONF_REFRESH, + CONF_TASK_STACK_IN_PSRAM, CONF_TYPE, CONF_URL, CONF_USERNAME, @@ -358,6 +359,7 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_VAD): _maybe_empty_vad_schema, cv.Optional(CONF_STOP_AFTER_DETECTION, default=True): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_MODEL): cv.invalid( f"The {CONF_MODEL} parameter has moved to be a list element under the {CONF_MODELS} parameter." ), @@ -451,6 +453,10 @@ async def to_code(config): cg.add_define("USE_MICRO_WAKE_WORD") ota.request_ota_state_listeners() + if config.get(CONF_TASK_STACK_IN_PSRAM): + cg.add(var.set_task_stack_in_psram(True)) + psram.request_external_task_stack() + esp32.add_idf_component(name="espressif/esp-tflite-micro", ref="1.3.3~1") # Pin esp-nn for stable future builds (esp-tflite-micro depends on esp-nn) esp32.add_idf_component(name="espressif/esp-nn", ref="1.1.2") diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 739d64dc28..237d72229d 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -217,10 +217,7 @@ void MicroWakeWord::inference_task(void *params) { FrontendFreeStateContents(&this_mww->frontend_state_); xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_STOPPED); - while (true) { - // Continuously delay until the main loop deletes the task - delay(10); - } + vTaskSuspend(nullptr); // Suspend this task indefinitely until the loop method deletes it } std::vector MicroWakeWord::get_wake_words() { @@ -243,14 +240,14 @@ void MicroWakeWord::add_vad_model(const uint8_t *model_start, uint8_t probabilit #endif void MicroWakeWord::suspend_task_() { - if (this->inference_task_handle_ != nullptr) { - vTaskSuspend(this->inference_task_handle_); + if (this->inference_task_.is_created()) { + vTaskSuspend(this->inference_task_.get_handle()); } } void MicroWakeWord::resume_task_() { - if (this->inference_task_handle_ != nullptr) { - vTaskResume(this->inference_task_handle_); + if (this->inference_task_.is_created()) { + vTaskResume(this->inference_task_.get_handle()); } } @@ -292,8 +289,7 @@ void MicroWakeWord::loop() { if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - vTaskDelete(this->inference_task_handle_); - this->inference_task_handle_ = nullptr; + this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); @@ -311,7 +307,7 @@ void MicroWakeWord::loop() { switch (this->state_) { case State::STARTING: - if ((this->inference_task_handle_ == nullptr) && !this->status_has_error()) { + if (!this->inference_task_.is_created() && !this->status_has_error()) { // Setup preprocesor feature generator. If done in the task, it would lock the task to its initial core, as it // uses floating point operations. if (!FrontendPopulateState(&this->frontend_config_, &this->frontend_state_, @@ -320,10 +316,8 @@ void MicroWakeWord::loop() { return; } - xTaskCreate(MicroWakeWord::inference_task, "mww", INFERENCE_TASK_STACK_SIZE, (void *) this, - INFERENCE_TASK_PRIORITY, &this->inference_task_handle_); - - if (this->inference_task_handle_ == nullptr) { + if (!this->inference_task_.create(MicroWakeWord::inference_task, "mww", INFERENCE_TASK_STACK_SIZE, + (void *) this, INFERENCE_TASK_PRIORITY, this->task_stack_in_psram_)) { FrontendFreeStateContents(&this->frontend_state_); // Deallocate frontend state this->status_momentary_error("task_start", 1000); } diff --git a/esphome/components/micro_wake_word/micro_wake_word.h b/esphome/components/micro_wake_word/micro_wake_word.h index ef440b5d37..e4c590a423 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.h +++ b/esphome/components/micro_wake_word/micro_wake_word.h @@ -11,6 +11,7 @@ #include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/defines.h" +#include "esphome/core/static_task.h" #ifdef USE_OTA_STATE_LISTENER #include "esphome/components/ota/ota_backend.h" @@ -59,6 +60,8 @@ class MicroWakeWord : public Component void set_stop_after_detection(bool stop_after_detection) { this->stop_after_detection_ = stop_after_detection; } + void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } + Trigger *get_wake_word_detected_trigger() { return &this->wake_word_detected_trigger_; } void add_wake_word_model(WakeWordModel *model); @@ -93,6 +96,8 @@ class MicroWakeWord : public Component bool stop_after_detection_; + bool task_stack_in_psram_{false}; + uint8_t features_step_size_; // Audio frontend handles generating spectrogram features @@ -105,8 +110,9 @@ class MicroWakeWord : public Component // Used to send messages about the models' states to the main loop QueueHandle_t detection_queue_; + StaticTask inference_task_; + static void inference_task(void *params); - TaskHandle_t inference_task_handle_{nullptr}; /// @brief Suspends the inference task void suspend_task_(); diff --git a/tests/components/micro_wake_word/common.yaml b/tests/components/micro_wake_word/common.yaml index c051c8dd57..cd060c176e 100644 --- a/tests/components/micro_wake_word/common.yaml +++ b/tests/components/micro_wake_word/common.yaml @@ -1,3 +1,6 @@ +psram: + mode: quad + i2s_audio: i2s_lrclk_pin: GPIO18 i2s_bclk_pin: GPIO19 @@ -12,6 +15,7 @@ microphone: micro_wake_word: microphone: echo_microphone + task_stack_in_psram: true on_wake_word_detected: - logger.log: "Wake word detected" - micro_wake_word.stop: From 892e116680a0c8063e7d1ead63b74ed7748c89f9 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 25 May 2026 12:42:49 -0400 Subject: [PATCH 50/96] [router] Add a router speaker component to runtime choose output speaker (#16592) --- CODEOWNERS | 1 + esphome/components/router/__init__.py | 0 esphome/components/router/speaker/__init__.py | 123 +++++++++ .../router/speaker/router_speaker.cpp | 236 ++++++++++++++++++ .../router/speaker/router_speaker.h | 92 +++++++ tests/components/router/common.yaml | 44 ++++ tests/components/router/test.esp32-idf.yaml | 7 + 7 files changed, 503 insertions(+) create mode 100644 esphome/components/router/__init__.py create mode 100644 esphome/components/router/speaker/__init__.py create mode 100644 esphome/components/router/speaker/router_speaker.cpp create mode 100644 esphome/components/router/speaker/router_speaker.h create mode 100644 tests/components/router/common.yaml create mode 100644 tests/components/router/test.esp32-idf.yaml diff --git a/CODEOWNERS b/CODEOWNERS index f8cdfdc6c6..3c3e502058 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -417,6 +417,7 @@ esphome/components/restart/* @esphome/core esphome/components/rf_bridge/* @jesserockz esphome/components/rgbct/* @jesserockz esphome/components/ring_buffer/* @kahrendt +esphome/components/router/speaker/* @kahrendt esphome/components/rp2040/* @jesserockz esphome/components/rp2040_ble/* @bdraco esphome/components/rp2040_pio_led_strip/* @Papa-DMan diff --git a/esphome/components/router/__init__.py b/esphome/components/router/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/router/speaker/__init__.py b/esphome/components/router/speaker/__init__.py new file mode 100644 index 0000000000..2b2dc56433 --- /dev/null +++ b/esphome/components/router/speaker/__init__.py @@ -0,0 +1,123 @@ +from esphome import automation, core +import esphome.codegen as cg +from esphome.components import audio, speaker +import esphome.config_validation as cv +from esphome.const import ( + CONF_BITS_PER_SAMPLE, + CONF_ID, + CONF_NUM_CHANNELS, + CONF_OUTPUT_SPEAKER, + CONF_SAMPLE_RATE, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj +from esphome.types import ConfigType, TemplateArgsType + +CODEOWNERS = ["@kahrendt"] + +CONF_OUTPUT_SPEAKERS = "output_speakers" +CONF_TARGET_SPEAKER = "target_speaker" + +router_ns = cg.esphome_ns.namespace("router") +Router = router_ns.class_("Router", cg.Component, speaker.Speaker) +SwitchOutputAction = router_ns.class_("SwitchOutputAction", automation.Action) + +SpeakerPtr = speaker.Speaker.operator("ptr") + + +def _set_stream_limits(config: ConfigType) -> ConfigType: + # Lock the router's stream limits to the user-declared format. Limits are set + # at CONFIG_SCHEMA time so they're visible to other components' FINAL_VALIDATE + # (which has no guaranteed ordering vs. ours). + audio.set_stream_limits( + min_bits_per_sample=config[CONF_BITS_PER_SAMPLE], + max_bits_per_sample=config[CONF_BITS_PER_SAMPLE], + min_channels=config[CONF_NUM_CHANNELS], + max_channels=config[CONF_NUM_CHANNELS], + min_sample_rate=config[CONF_SAMPLE_RATE], + max_sample_rate=config[CONF_SAMPLE_RATE], + )(config) + return config + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(Router), + cv.Required(CONF_OUTPUT_SPEAKERS): cv.All( + cv.ensure_list(cv.use_id(speaker.Speaker)), + cv.Length(min=2, max=8), + ), + # All outputs must agree on a single format so the producer can keep + # streaming through a switch without reconfiguring. These are required + # rather than inherited because downstream components (e.g. mixer) + # read them from the router's declaration during FINAL_VALIDATE, + # which can't depend on our FINAL_VALIDATE running first. + cv.Required(CONF_BITS_PER_SAMPLE): cv.int_range(8, 32), + cv.Required(CONF_NUM_CHANNELS): cv.int_range(1, 2), + cv.Required(CONF_SAMPLE_RATE): cv.int_range(8000, 96000), + } + ).extend(cv.COMPONENT_SCHEMA), + cv.only_on_esp32, + _set_stream_limits, +) + + +def _final_validate(config: ConfigType) -> ConfigType: + # Validate every configured output speaker can accept the router's format. + # Switching to an output that can't reproduce the format the producer is + # already sending would otherwise fail silently at runtime. + for spk_id in config[CONF_OUTPUT_SPEAKERS]: + proxy = {**config, CONF_OUTPUT_SPEAKER: spk_id} + audio.final_validate_audio_schema( + "router", + audio_device=CONF_OUTPUT_SPEAKER, + bits_per_sample=config[CONF_BITS_PER_SAMPLE], + channels=config[CONF_NUM_CHANNELS], + sample_rate=config[CONF_SAMPLE_RATE], + )(proxy) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + # The first configured output is the default active output on boot. + speakers = config[CONF_OUTPUT_SPEAKERS] + cg.add(var.set_output_count(len(speakers))) + for spk_id in speakers: + spk = await cg.get_variable(spk_id) + cg.add(var.add_output(spk)) + + +@automation.register_action( + "router.speaker.switch_output", + SwitchOutputAction, + cv.Schema( + { + cv.GenerateID(CONF_ID): cv.use_id(Router), + cv.Required(CONF_TARGET_SPEAKER): cv.templatable( + cv.use_id(speaker.Speaker) + ), + } + ), + synchronous=True, +) +async def switch_output_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + parent = await cg.get_variable(config[CONF_ID]) + var = cg.new_Pvariable(action_id, template_arg, parent) + target = config[CONF_TARGET_SPEAKER] + if not isinstance(target, core.Lambda): + target = await cg.get_variable(target) + template_ = await cg.templatable(target, args, SpeakerPtr) + cg.add(var.set_target(template_)) + return var diff --git a/esphome/components/router/speaker/router_speaker.cpp b/esphome/components/router/speaker/router_speaker.cpp new file mode 100644 index 0000000000..f4bf7420ab --- /dev/null +++ b/esphome/components/router/speaker/router_speaker.cpp @@ -0,0 +1,236 @@ +#include "router_speaker.h" + +#ifdef USE_ESP32 + +#include "esphome/core/log.h" + +#include "esp_timer.h" + +#include + +namespace esphome::router { + +static const char *const TAG = "router.speaker"; + +static inline uint32_t atomic_subtract_clamped(std::atomic &var, uint32_t amount) { + uint32_t current = var.load(std::memory_order_acquire); + uint32_t subtracted = 0; + if (current > 0) { + uint32_t new_value; + do { + subtracted = std::min(amount, current); + new_value = current - subtracted; + } while (!var.compare_exchange_weak(current, new_value, std::memory_order_release, std::memory_order_acquire)); + } + return subtracted; +} + +void Router::setup() { + // Register a callback on every configured output. Each lambda captures its own + // index and only forwards when that output is the active one. This is required + // because CallbackManager has no remove() API. + for (size_t i = 0; i < this->outputs_.size(); i++) { + this->outputs_[i]->add_audio_output_callback([this, i](uint32_t frames, int64_t timestamp_us) { + // Always suppress the draining previous output during a switch, even if it's + // also the reselected active output (switching back to the bus holder). + // loop() fires one synthetic credit for its in-flight frames instead. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) == static_cast(i)) { + return; + } + if (this->active_output_idx_.load(std::memory_order_relaxed) != static_cast(i)) { + return; + } + atomic_subtract_clamped(this->frames_in_pipeline_, frames); + this->audio_output_callback_.call(frames, timestamp_us); + }); + } +} + +void Router::loop() { + speaker::Speaker *active = this->get_active_output(); + + // Mid-switch: the new output's start() is deferred until the previous output + // fully releases shared hardware (e.g. a single i2s_audio bus driving two + // speakers). Starting earlier produces "Parent bus is busy" retries. The + // synthetic-credit callback is also deferred until prev is fully stopped, so + // that once its task has drained no natural callbacks can race ours. + const int8_t pending_prev_idx = this->pending_start_prev_idx_.load(std::memory_order_relaxed); + if (pending_prev_idx >= 0) { + speaker::Speaker *prev = this->outputs_[pending_prev_idx]; + if (prev->is_stopped()) { + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + + // Credit any frames left in prev's ring buffer / DMA so producer frame + // accounting (SpeakerSourceMediaPlayer pending_frames, sendspin/AEC + // clocks) clears cleanly. The leftover audio is intentionally dropped and + // the producer is told it played "now", giving a clean discontinuity that + // keeps frame accounting consistent across the switch. + const uint32_t in_flight = this->frames_in_pipeline_.exchange(0, std::memory_order_acq_rel); + if (in_flight > 0) { + this->audio_output_callback_.call(in_flight, esp_timer_get_time()); + } + + this->apply_cached_state_to_active_(); + this->state_ = speaker::STATE_STARTING; + active->start(); + } + return; + } + + // Mirror the active output's running/stopped state into our own state_ so that + // is_running() / is_stopped() stay accurate from the producer's perspective. + // Also catch the active output self-stopping (e.g. i2s_audio silence timeout): + // without this, our state_ would stay RUNNING forever and the next play() would + // skip start(). The output retains its own volume/mute across a restart (and we + // forward those live regardless), but stream info arrives via the non-virtual + // set_audio_stream_info() and never reaches the output on its own; if the format + // changed while stopped, only start()'s apply_cached_state_to_active_() pushes it + // down before the output's play()-side auto-start locks in the stale format. + if (active->is_stopped()) { + this->state_ = speaker::STATE_STOPPED; + } else if (this->state_ == speaker::STATE_STARTING && active->is_running()) { + this->state_ = speaker::STATE_RUNNING; + } +} + +void Router::dump_config() { + ESP_LOGCONFIG(TAG, + "Router Speaker:\n" + " Outputs: %u", + static_cast(this->outputs_.size())); +} + +size_t Router::play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) { + speaker::Speaker *active = this->get_active_output(); + + // Drop frames during a mid-switch until the old output releases shared hardware; + // forwarding now would trigger the new output's play()-side auto-start while + // the bus is still busy. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) >= 0) { + vTaskDelay(ticks_to_wait); + return 0; + } + + // Producers (e.g. mixer) set stream info on us and then drive play() from a + // task without ever calling our start(). i2s_audio's play() auto-starts the + // underlying driver, so we must push our cached stream info to the active + // output before that auto-start, or it locks to its default (16k mono). + if (this->state_ == speaker::STATE_STOPPED) { + this->start(); + vTaskDelay(ticks_to_wait); + ticks_to_wait = 0; + } + + size_t written = active->play(data, length, ticks_to_wait); + if (written > 0) { + const uint32_t frames = this->audio_stream_info_.bytes_to_frames(written); + this->frames_in_pipeline_.fetch_add(frames, std::memory_order_release); + } + return written; +} + +void Router::start() { + this->frames_in_pipeline_.store(0, std::memory_order_release); + this->apply_cached_state_to_active_(); + this->state_ = speaker::STATE_STARTING; + this->get_active_output()->start(); +} + +void Router::stop() { + // Cancel any pending mid-switch start; the producer wants us stopped. + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + this->state_ = speaker::STATE_STOPPING; + this->get_active_output()->stop(); +} + +void Router::finish() { + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + this->state_ = speaker::STATE_STOPPING; + this->get_active_output()->finish(); +} + +bool Router::has_buffered_data() const { return this->get_active_output()->has_buffered_data(); } + +void Router::set_pause_state(bool pause_state) { + this->cached_pause_ = pause_state; + this->get_active_output()->set_pause_state(pause_state); +} + +void Router::set_volume(float volume) { + this->volume_ = volume; + this->get_active_output()->set_volume(volume); +} + +void Router::set_mute_state(bool mute_state) { + this->mute_state_ = mute_state; + this->get_active_output()->set_mute_state(mute_state); +} + +bool Router::switch_to_output(speaker::Speaker *target) { + if (target == nullptr) { + return false; + } + + int8_t new_idx = -1; + for (size_t i = 0; i < this->outputs_.size(); i++) { + if (this->outputs_[i] == target) { + new_idx = static_cast(i); + break; + } + } + if (new_idx < 0) { + ESP_LOGW(TAG, "Switch target is not a configured output"); + return false; + } + if (new_idx == this->active_output_idx_.load(std::memory_order_relaxed)) { + return true; + } + + // A switch is already in flight: pending_start_prev_idx_ is still releasing the + // shared bus and the current active output's start() is still deferred (it never + // started). Just redirect which output we start once the bus frees. Leave the bus + // holder (pending_start_prev_idx_), the in-flight frame counter (loop() still owes one + // synthetic credit for the bus holder's in-flight frames), and state_ alone, and + // don't stop the current active output, which never started. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) >= 0) { + this->active_output_idx_.store(new_idx, std::memory_order_relaxed); + return true; + } + + const bool was_active = (this->state_ == speaker::STATE_STARTING || this->state_ == speaker::STATE_RUNNING); + const int8_t old_idx = this->active_output_idx_.load(std::memory_order_relaxed); + + if (was_active) { + this->outputs_[old_idx]->stop(); + } + + this->active_output_idx_.store(new_idx, std::memory_order_relaxed); + + if (was_active) { + // Defer start and the synthetic-credit callback until the old output's + // task is fully stopped; loop() handles both. Firing the synthetic credit + // here would race the old task's still-in-flight natural callbacks, + // dispatching audio_output_callback_ concurrently from two threads, which + // some consumers (e.g. sendspin's progress sync) aren't reentrant-safe for. + // STATE_STOPPING keeps producers from observing a transient stopped state + // and lets our play() short-circuit so the new output's play() doesn't + // auto-start it while the shared bus is still being released. + this->state_ = speaker::STATE_STOPPING; + this->pending_start_prev_idx_.store(old_idx, std::memory_order_relaxed); + } else { + this->frames_in_pipeline_.store(0, std::memory_order_release); + } + return true; +} + +void Router::apply_cached_state_to_active_() { + speaker::Speaker *active = this->get_active_output(); + active->set_audio_stream_info(this->audio_stream_info_); + active->set_volume(this->volume_); + active->set_mute_state(this->mute_state_); + active->set_pause_state(this->cached_pause_); +} + +} // namespace esphome::router + +#endif // USE_ESP32 diff --git a/esphome/components/router/speaker/router_speaker.h b/esphome/components/router/speaker/router_speaker.h new file mode 100644 index 0000000000..13b58a1c72 --- /dev/null +++ b/esphome/components/router/speaker/router_speaker.h @@ -0,0 +1,92 @@ +#pragma once + +#ifdef USE_ESP32 + +#include "esphome/components/speaker/speaker.h" +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +#include + +#include + +namespace esphome::router { + +class Router : public Component, public speaker::Speaker { + public: + float get_setup_priority() const override { return setup_priority::DATA; } + + void setup() override; + void loop() override; + void dump_config() override; + + size_t play(const uint8_t *data, size_t length) override { return this->play(data, length, 0); } + size_t play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) override; + + void start() override; + void stop() override; + void finish() override; + + bool has_buffered_data() const override; + + void set_pause_state(bool pause_state) override; + bool get_pause_state() const override { return this->cached_pause_; } + + void set_volume(float volume) override; + float get_volume() override { return this->volume_; } + + void set_mute_state(bool mute_state) override; + bool get_mute_state() override { return this->mute_state_; } + + // Allocate the output list to its final size. Must be called before add_output(). + void set_output_count(size_t count) { this->outputs_.init(count); } + void add_output(speaker::Speaker *spk) { this->outputs_.push_back(spk); } + + /// Switch the active output to the given speaker. Must be one of the configured outputs. + /// Returns false if `target` is not in the output list. + bool switch_to_output(speaker::Speaker *target); + + // Always valid: active_output_idx_ stays within [0, outputs_.size()) and at least + // two outputs are required (validated in Python), so this never returns null. + speaker::Speaker *get_active_output() const { + return this->outputs_[this->active_output_idx_.load(std::memory_order_relaxed)]; + } + + protected: + // Frames written to the active output but not yet played: incremented in play() and decremented + // (clamped at zero) by the active output's audio_output_callback. Mirrors mixer_speaker's + // frames_in_pipeline_. + std::atomic frames_in_pipeline_{0}; + + bool cached_pause_{false}; + + void apply_cached_state_to_active_(); + + // Index of the previously-active output we're waiting on to fully stop before + // starting the new one. -1 means no pending start. Set by switch_to_output() + // when switching mid-playback; cleared by loop() once the old output reports + // is_stopped(). Required because shared-bus drivers (e.g. two i2s_audio + // speakers on one i2s_bus) reject start() until the previous user releases. + std::atomic pending_start_prev_idx_{-1}; + + private: + FixedVector outputs_; + // Index into outputs_, always within [0, outputs_.size()). Defaults to the first + // configured output; updated by switch_to_output(). + std::atomic active_output_idx_{0}; +}; + +template class SwitchOutputAction : public Action { + public: + explicit SwitchOutputAction(Router *parent) : parent_(parent) {} + TEMPLATABLE_VALUE(speaker::Speaker *, target) + void play(const Ts &...x) override { this->parent_->switch_to_output(this->target_.value(x...)); } + + protected: + Router *parent_; +}; + +} // namespace esphome::router + +#endif // USE_ESP32 diff --git a/tests/components/router/common.yaml b/tests/components/router/common.yaml new file mode 100644 index 0000000000..360c6daaee --- /dev/null +++ b/tests/components/router/common.yaml @@ -0,0 +1,44 @@ +esphome: + on_boot: + then: + - router.speaker.switch_output: + id: router_id + target_speaker: speaker_b_id + # id omitted: auto-resolved since there's a single router instance + - router.speaker.switch_output: + target_speaker: !lambda return id(speaker_a_id); + +i2s_audio: + - id: i2s_a + i2s_lrclk_pin: ${a_lrclk_pin} + i2s_bclk_pin: ${a_bclk_pin} + - id: i2s_b + +speaker: + - platform: i2s_audio + id: speaker_a_id + i2s_audio_id: i2s_a + dac_type: external + i2s_dout_pin: ${a_dout_pin} + sample_rate: 48000 + bits_per_sample: 16bit + channel: stereo + - platform: i2s_audio + id: speaker_b_id + i2s_audio_id: i2s_b + dac_type: external + i2s_dout_pin: ${b_dout_pin} + spdif_mode: true + use_apll: true + sample_rate: 48000 + bits_per_sample: 16bit + channel: stereo + i2s_mode: primary + - platform: router + id: router_id + output_speakers: + - speaker_a_id + - speaker_b_id + sample_rate: 48000 + bits_per_sample: 16 + num_channels: 2 diff --git a/tests/components/router/test.esp32-idf.yaml b/tests/components/router/test.esp32-idf.yaml new file mode 100644 index 0000000000..241a9a8903 --- /dev/null +++ b/tests/components/router/test.esp32-idf.yaml @@ -0,0 +1,7 @@ +substitutions: + a_lrclk_pin: GPIO4 + a_bclk_pin: GPIO5 + a_dout_pin: GPIO14 + b_dout_pin: GPIO19 + +<<: !include common.yaml From dcc30f865105587c26a85fc2cf29a4212726c86e Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 25 May 2026 15:39:54 -0400 Subject: [PATCH 51/96] [router] Share a single I2S bus in test (#16637) --- tests/components/router/common.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/components/router/common.yaml b/tests/components/router/common.yaml index 360c6daaee..f1239de3cb 100644 --- a/tests/components/router/common.yaml +++ b/tests/components/router/common.yaml @@ -9,15 +9,12 @@ esphome: target_speaker: !lambda return id(speaker_a_id); i2s_audio: - - id: i2s_a - i2s_lrclk_pin: ${a_lrclk_pin} - i2s_bclk_pin: ${a_bclk_pin} - - id: i2s_b + i2s_lrclk_pin: ${a_lrclk_pin} + i2s_bclk_pin: ${a_bclk_pin} speaker: - platform: i2s_audio id: speaker_a_id - i2s_audio_id: i2s_a dac_type: external i2s_dout_pin: ${a_dout_pin} sample_rate: 48000 @@ -25,7 +22,6 @@ speaker: channel: stereo - platform: i2s_audio id: speaker_b_id - i2s_audio_id: i2s_b dac_type: external i2s_dout_pin: ${b_dout_pin} spdif_mode: true From 0b780f1fd29a70279c41e6e376d9a19f8f6f7e58 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 26 May 2026 06:21:15 +0930 Subject: [PATCH 52/96] [time][homeassistant] Fix timezone handling (#16583) --- esphome/components/api/api_connection.cpp | 2 +- esphome/components/api/client.py | 1 + .../components/homeassistant/time/__init__.py | 4 +- esphome/components/time/__init__.py | 56 ++- esphome/core/defines.h | 1 + tests/component_tests/time/__init__.py | 1 + tests/component_tests/time/test_init.py | 369 ++++++++++++++++++ 7 files changed, 414 insertions(+), 20 deletions(-) create mode 100644 tests/component_tests/time/__init__.py create mode 100644 tests/component_tests/time/test_init.py diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index f2bf3752fa..c880e036cb 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1169,7 +1169,7 @@ void APIConnection::on_camera_image_request(const CameraImageRequest &msg) { void APIConnection::on_get_time_response(const GetTimeResponse &value) { if (homeassistant::global_homeassistant_time != nullptr) { homeassistant::global_homeassistant_time->set_epoch_time(value.epoch_seconds); -#ifdef USE_TIME_TIMEZONE +#if defined(USE_HOMEASSISTANT_TIMEZONE) && defined(USE_TIME_TIMEZONE) if (!value.timezone.empty()) { // Check if the sender provided pre-parsed timezone data. // If std_offset is non-zero or DST rules are present, the parsed data was populated. diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index d6150fbd29..7fba091730 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -101,6 +101,7 @@ async def async_run_logs( client_info=f"ESPHome Logs {__version__}", noise_psk=noise_psk, addresses=addresses, # Pass all addresses for automatic retry + provide_time=False, ) # Try platform-specific stacktrace handler first, fall back to generic diff --git a/esphome/components/homeassistant/time/__init__.py b/esphome/components/homeassistant/time/__init__.py index 62cb96a25a..05ca86a26e 100644 --- a/esphome/components/homeassistant/time/__init__.py +++ b/esphome/components/homeassistant/time/__init__.py @@ -1,7 +1,7 @@ import esphome.codegen as cg from esphome.components import time as time_ import esphome.config_validation as cv -from esphome.const import CONF_ID +from esphome.const import CONF_ID, CONF_TIMEZONE from .. import homeassistant_ns @@ -21,3 +21,5 @@ async def to_code(config): await time_.register_time(var, config) await cg.register_component(var, config) cg.add_define("USE_HOMEASSISTANT_TIME") + if CONF_TIMEZONE not in config: + cg.add_define("USE_HOMEASSISTANT_TIMEZONE") diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index 29bb01b499..8839a988a1 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -30,13 +30,21 @@ from esphome.const import ( CONF_SECONDS, CONF_TIMEZONE, CONF_TRIGGER_ID, + PLATFORM_BK72XX, + PLATFORM_ESP32, + PLATFORM_ESP8266, + PLATFORM_HOST, + PLATFORM_LN882X, + PLATFORM_RP2040, + PLATFORM_RTL87XX, ) -from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.core import CORE, CoroPriority, EsphomeError, coroutine_with_priority _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@esphome/core"] IS_PLATFORM_COMPONENT = True +DOMAIN = "time" time_ns = cg.esphome_ns.namespace("time") RealTimeClock = time_ns.class_("RealTimeClock", cg.PollingComponent) @@ -92,20 +100,34 @@ def _extract_tz_string(tzfile: bytes) -> str: raise -def detect_tz() -> str: +def detect_tz() -> str | None: + if CORE.target_platform not in { + PLATFORM_ESP8266, + PLATFORM_ESP32, + PLATFORM_RP2040, + PLATFORM_BK72XX, + PLATFORM_RTL87XX, + PLATFORM_LN882X, + PLATFORM_HOST, + }: + return None + # Avoids duplicate logger messages when multiple time components are configured + if cached := CORE.data.setdefault(DOMAIN, {}).get(CONF_TIMEZONE): + return cached iana_key = tzlocal.get_localzone_name() if iana_key is None: - raise cv.Invalid( + raise EsphomeError( "Could not automatically determine timezone, please set timezone manually." ) - _LOGGER.info("Detected timezone '%s'", iana_key) tzfile = _load_tzdata(iana_key) if tzfile is None: - raise cv.Invalid( + raise EsphomeError( "Could not automatically determine timezone, please set timezone manually." ) ret = _extract_tz_string(tzfile) + _LOGGER.info("Detected timezone '%s'", iana_key) _LOGGER.debug(" -> TZ string %s", ret) + CORE.data.setdefault(DOMAIN, {})[CONF_TIMEZONE] = ret return ret @@ -312,16 +334,7 @@ def validate_tz(value: str) -> str: TIME_SCHEMA = cv.Schema( { - cv.SplitDefault( - CONF_TIMEZONE, - esp8266=detect_tz, - esp32=detect_tz, - rp2040=detect_tz, - bk72xx=detect_tz, - rtl87xx=detect_tz, - ln882x=detect_tz, - host=detect_tz, - ): cv.All( + cv.Optional(CONF_TIMEZONE): cv.All( cv.only_with_framework(["arduino", "esp-idf", "host"]), validate_tz, ), @@ -384,7 +397,11 @@ def _emit_parsed_timezone_fields(parsed): async def setup_time_core_(time_var, config): - if timezone := config.get(CONF_TIMEZONE): + timezone = config.get(CONF_TIMEZONE) + # an empty timezone is treated as disabling timezones completely as before + if timezone is None: + timezone = detect_tz() + if timezone: cg.add_define("USE_TIME_TIMEZONE") if CORE.is_host: @@ -392,8 +409,11 @@ async def setup_time_core_(time_var, config): cg.add(time_var.set_timezone(timezone)) else: # Embedded: pre-parse at codegen time, emit struct directly - parsed = parse_posix_tz_python(timezone) - _emit_parsed_timezone_fields(parsed) + try: + parsed = parse_posix_tz_python(timezone) + _emit_parsed_timezone_fields(parsed) + except ValueError as e: + raise EsphomeError(f"Invalid timezone: {timezone}") from e for conf in config.get(CONF_ON_TIME, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], time_var) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 3cb92616bb..0229bc14fa 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_GRAPH #define USE_GRAPHICAL_DISPLAY_MENU #define USE_HOMEASSISTANT_TIME +#define USE_HOMEASSISTANT_TIMEZONE #define USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT 8000 // NOLINT #define USE_I2S_AUDIO_SPDIF_MODE #define USE_IMAGE diff --git a/tests/component_tests/time/__init__.py b/tests/component_tests/time/__init__.py new file mode 100644 index 0000000000..dc24f4e532 --- /dev/null +++ b/tests/component_tests/time/__init__.py @@ -0,0 +1 @@ +"""Tests for the time component.""" diff --git a/tests/component_tests/time/test_init.py b/tests/component_tests/time/test_init.py new file mode 100644 index 0000000000..44469cfe28 --- /dev/null +++ b/tests/component_tests/time/test_init.py @@ -0,0 +1,369 @@ +"""Tests for time component – ha-timezone branch changes. + +Covers: +- detect_tz() platform guard (returns None for unsupported platforms) +- detect_tz() result caching (avoids duplicate log messages) +- detect_tz() error paths (tzlocal None, tzdata missing) +- validate_tz() accepts/rejects POSIX timezone strings and IANA keys +- TIME_SCHEMA: timezone is now truly optional (was SplitDefault) +- homeassistant/time: USE_HOMEASSISTANT_TIMEZONE define emitted iff + CONF_TIMEZONE is absent from the config +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from esphome.components.time import DOMAIN, TIME_SCHEMA, detect_tz, validate_tz +import esphome.config_validation as cv +from esphome.const import ( + CONF_ID, + CONF_TIMEZONE, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Platform, + PlatformFramework, +) +from esphome.core import CORE, EsphomeError +from tests.component_tests.types import SetCoreConfigCallable + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# A minimal TZif v2/v3 file that encodes "EST5EDT" as the footer line. +# The binary content is not validated at this level – what matters is that +# _extract_tz_string() picks up the last-but-one newline-terminated line. +_FAKE_TZFILE = b"\x00" * 44 + b"TZif2\x00" * 1 + b"\n" + b"EST5EDT,M3.2.0,M11.1.0\n" + + +def _set_platform(platform: Platform) -> None: + """Set CORE.data so that CORE.target_platform returns *platform*.""" + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: platform, + KEY_TARGET_FRAMEWORK: "arduino", + } + + +# --------------------------------------------------------------------------- +# detect_tz – platform guard +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "platform_framework", + [ + PlatformFramework.NRF52_ZEPHYR, + ], +) +def test_detect_tz_returns_none_for_unsupported_platform( + platform_framework: PlatformFramework, + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must return None for platforms that do not support TZ auto-detection.""" + set_core_config(platform_framework) + result = detect_tz() + assert result is None + + +@pytest.mark.parametrize( + "platform_framework", + [ + PlatformFramework.ESP32_IDF, + PlatformFramework.ESP32_ARDUINO, + PlatformFramework.ESP8266_ARDUINO, + PlatformFramework.RP2040_ARDUINO, + PlatformFramework.BK72XX_ARDUINO, + PlatformFramework.RTL87XX_ARDUINO, + PlatformFramework.LN882X_ARDUINO, + PlatformFramework.HOST_NATIVE, + ], +) +def test_detect_tz_calls_tzlocal_for_supported_platform( + platform_framework: PlatformFramework, + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must call tzlocal for every supported platform.""" + set_core_config(platform_framework) + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="America/New_York", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ), + ): + result = detect_tz() + assert result is not None + assert isinstance(result, str) + assert len(result) > 0 + + +# --------------------------------------------------------------------------- +# detect_tz – caching +# --------------------------------------------------------------------------- + + +def test_detect_tz_caches_result( + set_core_config: SetCoreConfigCallable, + caplog: pytest.LogCaptureFixture, +) -> None: + """detect_tz() must cache the TZ string after the first call so that + subsequent invocations (e.g. when multiple time platforms are configured) + skip tzlocal and avoid duplicate INFO messages.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="America/New_York", + ) as mock_tz, + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ) as mock_load, + ): + first = detect_tz() + second = detect_tz() + + assert first == second + # tzlocal and _load_tzdata must be called exactly once despite two detect_tz() calls + mock_tz.assert_called_once() + mock_load.assert_called_once() + + +def test_detect_tz_cache_stored_in_core_data( + set_core_config: SetCoreConfigCallable, +) -> None: + """The cached TZ string should be stored under CORE.data[DOMAIN][CONF_TIMEZONE].""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="Europe/London", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ), + ): + result = detect_tz() + + assert CORE.data.get(DOMAIN, {}).get(CONF_TIMEZONE) == result + + +def test_detect_tz_returns_pre_seeded_cache( + set_core_config: SetCoreConfigCallable, +) -> None: + """If CORE.data already has a cached TZ string, detect_tz() must return it + without calling tzlocal at all.""" + set_core_config(PlatformFramework.ESP32_IDF) + CORE.data[DOMAIN] = {CONF_TIMEZONE: "CET-1CEST,M3.5.0,M10.5.0/3"} + + with mock.patch("esphome.components.time.tzlocal.get_localzone_name") as mock_tz: + result = detect_tz() + + assert result == "CET-1CEST,M3.5.0,M10.5.0/3" + mock_tz.assert_not_called() + + +# --------------------------------------------------------------------------- +# detect_tz – error paths +# --------------------------------------------------------------------------- + + +def test_detect_tz_raises_when_tzlocal_returns_none( + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must raise EsphomeError when the local timezone cannot be determined.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value=None, + ), + pytest.raises(EsphomeError, match="Could not automatically determine timezone"), + ): + detect_tz() + + +def test_detect_tz_raises_when_tzdata_not_found( + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must raise EsphomeError when tzdata has no entry for the IANA key.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="Antarctica/Troll", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=None, + ), + pytest.raises(EsphomeError, match="Could not automatically determine timezone"), + ): + detect_tz() + + +# --------------------------------------------------------------------------- +# validate_tz +# --------------------------------------------------------------------------- + + +def test_validate_tz_accepts_valid_posix_string() -> None: + """validate_tz() must accept a syntactically valid POSIX TZ string.""" + result = validate_tz("UTC0") + assert result == "UTC0" + + +def test_validate_tz_accepts_posix_string_with_dst() -> None: + """validate_tz() must accept a full POSIX TZ string with DST rules.""" + tz = "EST5EDT,M3.2.0,M11.1.0" + result = validate_tz(tz) + assert result == tz + + +def test_validate_tz_accepts_iana_key_and_converts() -> None: + """validate_tz() must accept an IANA timezone key and return the POSIX string.""" + with mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ): + result = validate_tz("America/New_York") + + # Should have been converted from IANA to POSIX via _extract_tz_string + assert result == "EST5EDT,M3.2.0,M11.1.0" + + +def test_validate_tz_rejects_invalid_posix_string() -> None: + """validate_tz() must raise cv.Invalid for a malformed POSIX TZ string.""" + with pytest.raises(cv.Invalid, match="Invalid POSIX timezone string"): + validate_tz("NOTAVALIDTZ!!!") + + +def test_validate_tz_accepts_empty_string() -> None: + """An empty string is accepted by validate_tz() and signals 'disable timezone'.""" + result = validate_tz("") + assert result == "" + + +# --------------------------------------------------------------------------- +# TIME_SCHEMA – timezone is now cv.Optional (no SplitDefault) +# --------------------------------------------------------------------------- + + +def test_time_schema_timezone_is_optional( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must accept a config with no timezone key on a supported platform.""" + set_core_config(PlatformFramework.ESP32_IDF) + # Should not raise + config = TIME_SCHEMA({}) + assert CONF_TIMEZONE not in config + + +def test_time_schema_explicit_timezone_accepted( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must accept an explicit valid POSIX timezone on Arduino/IDF.""" + set_core_config(PlatformFramework.ESP32_IDF) + config = TIME_SCHEMA({CONF_TIMEZONE: "UTC0"}) + assert config[CONF_TIMEZONE] == "UTC0" + + +def test_time_schema_explicit_empty_timezone_accepted( + set_core_config: SetCoreConfigCallable, +) -> None: + """An empty timezone string (timezone-disable sentinel) must pass TIME_SCHEMA.""" + set_core_config(PlatformFramework.ESP32_IDF) + config = TIME_SCHEMA({CONF_TIMEZONE: ""}) + assert config[CONF_TIMEZONE] == "" + + +def test_time_schema_timezone_rejected_on_zephyr( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must reject a timezone value on Zephyr with the framework error. + + The platform check (cv.only_with_framework) must run BEFORE validate_tz so + that users receive an actionable "unsupported framework" message rather than a + confusing TZ-parsing error. + """ + set_core_config(PlatformFramework.NRF52_ZEPHYR) + with pytest.raises(cv.Invalid, match="only available with framework"): + TIME_SCHEMA({CONF_TIMEZONE: "UTC0"}) + + +def test_time_schema_invalid_tz_on_zephyr_gives_framework_error( + set_core_config: SetCoreConfigCallable, +) -> None: + """Even a syntactically invalid TZ string must produce the framework error on Zephyr. + + This specifically tests that cv.only_with_framework is evaluated before + validate_tz: if the order were reversed, an invalid POSIX string would + generate a misleading TZ-parsing error instead. + """ + set_core_config(PlatformFramework.NRF52_ZEPHYR) + with pytest.raises(cv.Invalid, match="only available with framework"): + TIME_SCHEMA({CONF_TIMEZONE: "NOTAVALIDTZ!!!"}) + + +# --------------------------------------------------------------------------- +# homeassistant/time: USE_HOMEASSISTANT_TIMEZONE define +# --------------------------------------------------------------------------- + + +@pytest.fixture +def mock_ha_cg(): + """Mock codegen functions used by homeassistant/time to_code.""" + with ( + mock.patch( + "esphome.components.homeassistant.time.cg.new_Pvariable", + return_value=mock.MagicMock(), + ), + mock.patch( + "esphome.components.homeassistant.time.cg.add_define", + ) as mock_add_define, + mock.patch( + "esphome.components.homeassistant.time.cg.register_component", + new_callable=mock.AsyncMock, + ), + mock.patch( + "esphome.components.homeassistant.time.time_.register_time", + new_callable=mock.AsyncMock, + ), + ): + yield mock_add_define + + +@pytest.mark.asyncio +async def test_ha_time_defines_ha_timezone_when_no_explicit_tz(mock_ha_cg) -> None: + """When CONF_TIMEZONE is absent from the config, to_code() must call + cg.add_define('USE_HOMEASSISTANT_TIMEZONE').""" + from esphome.components.homeassistant.time import to_code + + await to_code({CONF_ID: mock.MagicMock()}) + + mock_ha_cg.assert_any_call("USE_HOMEASSISTANT_TIMEZONE") + + +@pytest.mark.asyncio +async def test_ha_time_no_ha_timezone_define_when_explicit_tz(mock_ha_cg) -> None: + """When CONF_TIMEZONE is present in the config, to_code() must NOT call + cg.add_define('USE_HOMEASSISTANT_TIMEZONE').""" + from esphome.components.homeassistant.time import to_code + + await to_code({CONF_ID: mock.MagicMock(), CONF_TIMEZONE: "UTC0"}) + + define_calls = [call.args[0] for call in mock_ha_cg.call_args_list] + assert "USE_HOMEASSISTANT_TIME" in define_calls + assert "USE_HOMEASSISTANT_TIMEZONE" not in define_calls From fc0a4e22011fc4e852dc05b1433fbbc398434db7 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 25 May 2026 17:07:35 -0400 Subject: [PATCH 53/96] [espidf] Support github:// and https://github.com/.../.git framework sources (#16639) --- esphome/espidf/framework.py | 115 +++++++++++++--- esphome/git.py | 5 +- tests/unit_tests/test_espidf_framework.py | 156 ++++++++++++++++++++++ 3 files changed, 256 insertions(+), 20 deletions(-) create mode 100644 tests/unit_tests/test_espidf_framework.py diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 079c97cc98..fb53066edb 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -8,6 +8,7 @@ import logging import os from pathlib import Path import platform +import re import shutil import subprocess import sys @@ -784,6 +785,77 @@ def download_from_mirrors( return None +_GITHUB_SHORTHAND_RE = re.compile( + r"^github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+?)(?:@([a-zA-Z0-9\-_.\./]+))?$" +) +_GITHUB_HTTPS_RE = re.compile( + r"^(https://github\.com/[a-zA-Z0-9\-]+/[a-zA-Z0-9\-\._]+?\.git)(?:@([a-zA-Z0-9\-_.\./]+))?$" +) + + +def _parse_git_source(source_url: str) -> tuple[str, str | None] | None: + """Return ``(url, ref)`` for ``github://owner/repo[@ref]`` or + ``https://github.com/owner/repo.git[@ref]``, else ``None``.""" + if m := _GITHUB_SHORTHAND_RE.match(source_url): + owner, repo, ref = m.group(1), m.group(2), m.group(3) + # Tolerate a trailing ".git" on the shorthand repo so the + # github://owner/repo.git form doesn't silently become repo.git.git. + repo = repo.removesuffix(".git") + return f"https://github.com/{owner}/{repo}.git", ref + if m := _GITHUB_HTTPS_RE.match(source_url): + return m.group(1), m.group(2) + return None + + +def _clone_idf_with_submodules( + framework_path: Path, git_url: str, ref: str | None +) -> None: + """Shallow-clone ESP-IDF with submodules into ``framework_path``. + + GitHub's archive zip strips submodules, so vendored components + (mbedtls, openthread, esptool, ...) come down empty and CMake fails. + + Uses clone + ``fetch FETCH_HEAD`` + ``reset --hard`` instead of + ``--branch``: ``--branch`` only accepts branch or tag names, but a + user can also point at a commit SHA. The fetch-then-reset pattern + handles branches, tags, and SHAs uniformly (mirrors the approach in + ``esphome.git.clone_or_update``). + """ + from esphome.git import run_git_command + + _LOGGER.info("Cloning ESP-IDF from %s%s", git_url, f"@{ref}" if ref else "") + run_git_command(["git", "clone", "--depth=1", "--", git_url, str(framework_path)]) + if ref: + run_git_command( + ["git", "fetch", "--depth=1", "--", "origin", ref], + git_dir=framework_path, + ) + run_git_command( + ["git", "reset", "--hard", "FETCH_HEAD"], + git_dir=framework_path, + ) + run_git_command( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + "--depth=1", + ], + git_dir=framework_path, + ) + + # Sanity-check the resulting tree. run_git_command only raises when + # stderr is non-empty, so a clone that silently produces no working + # tree would otherwise be marked extracted and stuck until + # ``esphome clean``. + if not (framework_path / "tools" / "idf_tools.py").is_file(): + raise RuntimeError( + f"Clone of {git_url} produced no usable ESP-IDF tree at {framework_path}" + ) + + def _write_idf_version_txt(framework_path: Path, version: str) -> None: """Write /version.txt if missing. @@ -939,27 +1011,34 @@ def _check_esphome_idf_framework_install( if install: rmdir(framework_path, msg=f"Clean up ESP-IDF {version} framework") - # Download in temporary file - with tempfile.NamedTemporaryFile() as tmp: - _LOGGER.info("Downloading ESP-IDF %s framework ...", version) + git_source = _parse_git_source(source_url) if source_url else None + if git_source is not None: + git_url, ref = git_source + _clone_idf_with_submodules(framework_path, git_url, ref) + else: + # Download in temporary file + with tempfile.NamedTemporaryFile() as tmp: + _LOGGER.info("Downloading ESP-IDF %s framework ...", version) - # Create substitutions for the URLs - substitutions = {"VERSION": version} - try: - ver = Version.parse(version) - substitutions["MAJOR"] = str(ver.major) - substitutions["MINOR"] = str(ver.minor) - substitutions["PATCH"] = str(ver.patch) - substitutions["EXTRA"] = ver.extra - except ValueError: - pass + # Create substitutions for the URLs + substitutions = {"VERSION": version} + try: + ver = Version.parse(version) + substitutions["MAJOR"] = str(ver.major) + substitutions["MINOR"] = str(ver.minor) + substitutions["PATCH"] = str(ver.patch) + substitutions["EXTRA"] = ver.extra + except ValueError: + pass - mirrors = [source_url] if source_url else ESPHOME_IDF_FRAMEWORK_MIRRORS - download_from_mirrors(mirrors, substitutions, tmp.file) + mirrors = [source_url] if source_url else ESPHOME_IDF_FRAMEWORK_MIRRORS + download_from_mirrors(mirrors, substitutions, tmp.file) - _LOGGER.info("Extracting ESP-IDF %s framework ...", version) - archive_extract_all(tmp.file, framework_path, progress_header="Extracting") - extracted_marker.touch() + _LOGGER.info("Extracting ESP-IDF %s framework ...", version) + archive_extract_all( + tmp.file, framework_path, progress_header="Extracting" + ) + extracted_marker.touch() # Idempotent post-extract patch: written every invocation so a build # dir extracted before this fix gets the file too, without forcing a diff --git a/esphome/git.py b/esphome/git.py index 0106f24845..f36bd559ef 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -72,8 +72,9 @@ def run_git_command(cmd: list[str], git_dir: Path | None = None) -> str: ) except FileNotFoundError as err: raise GitNotInstalledError( - "git is not installed but required for external_components.\n" - "Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git" + "git is not installed. See " + "https://git-scm.com/book/en/v2/Getting-Started-Installing-Git " + "for installation instructions." ) from err if ret.returncode != 0 and ret.stderr: diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py new file mode 100644 index 0000000000..9f4e4fcca8 --- /dev/null +++ b/tests/unit_tests/test_espidf_framework.py @@ -0,0 +1,156 @@ +"""Tests for esphome.espidf.framework helpers.""" + +# pylint: disable=protected-access + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.espidf.framework import _clone_idf_with_submodules, _parse_git_source + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + # github:// shorthand + ( + "github://espressif/esp-idf", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "github://espressif/esp-idf@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ( + "github://espressif/esp-idf@release/v6.0", + ("https://github.com/espressif/esp-idf.git", "release/v6.0"), + ), + # explicit https://github.com/...git URL + ( + "https://github.com/espressif/esp-idf.git", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "https://github.com/espressif/esp-idf.git@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ( + "https://github.com/espressif/esp-idf.git@v6.0.1", + ("https://github.com/espressif/esp-idf.git", "v6.0.1"), + ), + # Tolerate a trailing ".git" on the shorthand so the user doesn't + # silently end up with a doubled "...esp-idf.git.git" URL. + ( + "github://espressif/esp-idf.git", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "github://espressif/esp-idf.git@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ], +) +def test_parse_git_source_recognized( + source: str, expected: tuple[str, str | None] +) -> None: + assert _parse_git_source(source) == expected + + +@pytest.mark.parametrize( + "source", + [ + # archive URLs fall through to the existing download path + "https://github.com/espressif/esp-idf/archive/refs/heads/master.zip", + "https://dl.espressif.com/dl/esp-idf/v6.0.1/esp-idf-v6.0.1.zip", + "https://github.com/esphome-libs/esp-idf/releases/download/v5.5.4/esp-idf-v5.5.4.tar.xz", + # SSH and other git protocols are intentionally rejected — match + # external_components, which only recognizes github:// + structured + # dicts for these. + "git@github.com:espressif/esp-idf.git", + "ssh://git@github.com/espressif/esp-idf.git", + "git://github.com/espressif/esp-idf.git", + # non-GitHub .git URLs are intentionally rejected for the same reason + "https://gitlab.com/foo/bar.git", + "https://github.example.com/foo/bar.git", + ], +) +def test_parse_git_source_rejected(source: str) -> None: + assert _parse_git_source(source) is None + + +def _make_idf_tree(framework_path: Path) -> None: + """Create the minimum tree _clone_idf_with_submodules sanity-checks for.""" + (framework_path / "tools").mkdir(parents=True) + (framework_path / "tools" / "idf_tools.py").write_text("# stub\n") + + +def test_clone_idf_with_submodules_without_ref(tmp_path: Path) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + _make_idf_tree(framework_path) + + with patch("esphome.git.run_git_command", return_value="") as run_git_command_mock: + _clone_idf_with_submodules( + framework_path, "https://github.com/espressif/esp-idf.git", None + ) + + # No ref -> just clone + submodule update, no fetch/reset. + calls = [c.args[0] for c in run_git_command_mock.call_args_list] + assert calls[0] == [ + "git", + "clone", + "--depth=1", + "--", + "https://github.com/espressif/esp-idf.git", + str(framework_path), + ] + assert calls[-1][:5] == ["git", "submodule", "update", "--init", "--recursive"] + assert not any(c[1] == "fetch" for c in calls) + assert not any(c[1] == "reset" for c in calls) + + +def test_clone_idf_with_submodules_with_ref(tmp_path: Path) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + _make_idf_tree(framework_path) + + with patch("esphome.git.run_git_command", return_value="") as run_git_command_mock: + _clone_idf_with_submodules( + framework_path, + "https://github.com/espressif/esp-idf.git", + "master", + ) + + calls = [c.args[0] for c in run_git_command_mock.call_args_list] + # clone, fetch ref, reset hard, submodule update + assert calls[0][:2] == ["git", "clone"] + assert calls[1] == [ + "git", + "fetch", + "--depth=1", + "--", + "origin", + "master", + ] + assert calls[2] == ["git", "reset", "--hard", "FETCH_HEAD"] + assert calls[3][:5] == ["git", "submodule", "update", "--init", "--recursive"] + + +def test_clone_idf_with_submodules_raises_when_tree_missing( + tmp_path: Path, +) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + # Deliberately do NOT call _make_idf_tree — simulate a clone that + # returned 0 but produced no tools/idf_tools.py. + + with ( + patch("esphome.git.run_git_command", return_value=""), + pytest.raises(RuntimeError, match="no usable ESP-IDF tree"), + ): + _clone_idf_with_submodules( + framework_path, + "https://github.com/espressif/esp-idf.git", + None, + ) From 61e8830a3ce38a67647d2b17b863049b296e7aaf Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 25 May 2026 17:40:38 -0400 Subject: [PATCH 54/96] [espidf] Keep cmake output filter working when IDF writes raw bytes (#16642) --- esphome/espidf/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index da3f77cdd3..857d16c674 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -164,6 +164,12 @@ def main() -> int: self._line_buffer = "" def __getattr__(self, name: str): + # Hide ``buffer`` so consumers that use either + # ``getattr(stream, 'buffer', None)`` or + # ``hasattr(stream, 'buffer')`` see this as a text-only stream + # and skip writing raw bytes (which would bypass the filter). + if name == "buffer": + raise AttributeError(name) return getattr(self._stream, name) def isatty(self) -> bool: From a257edba626ab1455f882f58a8a3f5787745a231 Mon Sep 17 00:00:00 2001 From: Boris Krivonog Date: Mon, 25 May 2026 23:46:33 +0200 Subject: [PATCH 55/96] [mitsubishi_cn105] Add basic swing support (#15653) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../components/mitsubishi_cn105/climate.py | 12 +- .../mitsubishi_cn105/mitsubishi_cn105.h | 1 + .../mitsubishi_cn105_climate.cpp | 90 ++++++++++ .../mitsubishi_cn105_climate.h | 5 + .../mitsubishi_cn105_climate_tests.cpp | 165 ++++++++++++++++++ tests/components/mitsubishi_cn105/common.h | 11 ++ tests/components/mitsubishi_cn105/common.yaml | 3 + 7 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp diff --git a/esphome/components/mitsubishi_cn105/climate.py b/esphome/components/mitsubishi_cn105/climate.py index cc44494d89..522b9218fc 100644 --- a/esphome/components/mitsubishi_cn105/climate.py +++ b/esphome/components/mitsubishi_cn105/climate.py @@ -1,8 +1,14 @@ from esphome import automation import esphome.codegen as cg from esphome.components import climate, uart +from esphome.components.climate import validate_climate_swing_mode import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_TEMPERATURE, CONF_UPDATE_INTERVAL +from esphome.const import ( + CONF_ID, + CONF_SUPPORTED_SWING_MODES, + CONF_TEMPERATURE, + CONF_UPDATE_INTERVAL, +) from esphome.core import ID from esphome.cpp_generator import MockObj from esphome.types import ConfigType, TemplateArgsType @@ -43,6 +49,9 @@ CONFIG_SCHEMA = ( cv.Optional( CONF_CURRENT_TEMPERATURE_MIN_INTERVAL, default="60s" ): cv.update_interval, + cv.Optional( + CONF_SUPPORTED_SWING_MODES, default="OFF" + ): validate_climate_swing_mode, } ) ) @@ -63,6 +72,7 @@ async def to_code(config: ConfigType) -> None: var = await climate.new_climate(config) await cg.register_component(var, config) await uart.register_uart_device(var, config) + cg.add(var.set_supported_swing_mode(config[CONF_SUPPORTED_SWING_MODES])) cg.add( var.set_current_temperature_min_interval( config[CONF_CURRENT_TEMPERATURE_MIN_INTERVAL] diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h index dbeb43068e..742d8e18a9 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "esphome/components/uart/uart.h" #include "esphome/core/finite_set_mask.h" diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp index 67a561397a..afffe7ea5e 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp @@ -84,6 +84,8 @@ climate::ClimateTraits MitsubishiCN105Climate::traits() { traits.add_supported_fan_mode(p.second); } + traits.set_supported_swing_modes(this->supported_swing_modes_); + traits.set_visual_min_temperature(16.0f); traits.set_visual_max_temperature(31.0f); traits.set_visual_temperature_step(1.0f); @@ -114,6 +116,37 @@ void MitsubishiCN105Climate::control(const climate::ClimateCall &call) { this->hp_.set_fan_mode(*fan_mode); } + if (const auto swing_mode = call.get_swing_mode()) { + auto vane = this->last_non_swing_vane_mode_; + auto wide = this->last_non_swing_wide_vane_mode_; + + switch (*swing_mode) { + case climate::CLIMATE_SWING_BOTH: + vane = MitsubishiCN105::VaneMode::SWING; + wide = MitsubishiCN105::WideVaneMode::SWING; + break; + + case climate::CLIMATE_SWING_VERTICAL: + vane = MitsubishiCN105::VaneMode::SWING; + break; + + case climate::CLIMATE_SWING_HORIZONTAL: + wide = MitsubishiCN105::WideVaneMode::SWING; + break; + + case climate::CLIMATE_SWING_OFF: + default: + break; + } + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_VERTICAL)) { + this->hp_.set_vane_mode(vane); + } + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_HORIZONTAL)) { + this->hp_.set_wide_vane_mode(wide); + } + } + if (this->hp_.is_status_initialized()) { this->apply_values_(); } @@ -143,7 +176,64 @@ void MitsubishiCN105Climate::apply_values_() { ESP_LOGD(TAG, "Unable to map fan mode"); } + if (!this->supported_swing_modes_.empty()) { + bool vertical_swinging = false; + bool horizontal_swinging = false; + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_VERTICAL)) { + if (status.vane_mode == MitsubishiCN105::VaneMode::SWING) { + vertical_swinging = true; + } else if (status.vane_mode != MitsubishiCN105::VaneMode::UNKNOWN) { + this->last_non_swing_vane_mode_ = status.vane_mode; + } + } + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_HORIZONTAL)) { + if (status.wide_vane_mode == MitsubishiCN105::WideVaneMode::SWING) { + horizontal_swinging = true; + } else if (status.wide_vane_mode != MitsubishiCN105::WideVaneMode::UNKNOWN) { + this->last_non_swing_wide_vane_mode_ = status.wide_vane_mode; + } + } + + if (vertical_swinging && horizontal_swinging) { + this->swing_mode = climate::CLIMATE_SWING_BOTH; + } else if (vertical_swinging) { + this->swing_mode = climate::CLIMATE_SWING_VERTICAL; + } else if (horizontal_swinging) { + this->swing_mode = climate::CLIMATE_SWING_HORIZONTAL; + } else { + this->swing_mode = climate::CLIMATE_SWING_OFF; + } + } + this->publish_state(); } +void MitsubishiCN105Climate::set_supported_swing_mode(climate::ClimateSwingMode mode) { + this->supported_swing_modes_.clear(); + switch (mode) { + case climate::CLIMATE_SWING_VERTICAL: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_VERTICAL); + break; + + case climate::CLIMATE_SWING_HORIZONTAL: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_HORIZONTAL); + break; + + case climate::CLIMATE_SWING_BOTH: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_VERTICAL); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_HORIZONTAL); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_BOTH); + break; + + case climate::CLIMATE_SWING_OFF: + default: + break; + } +} + } // namespace esphome::mitsubishi_cn105 diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h index e09158bfcf..c83a5519c1 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h @@ -25,10 +25,15 @@ class MitsubishiCN105Climate : public climate::Climate, public Component, public void set_remote_temperature(float temperature) { this->hp_.set_remote_temperature(temperature); } void clear_remote_temperature() { this->hp_.clear_remote_temperature(); } + void set_supported_swing_mode(climate::ClimateSwingMode mode); + protected: void apply_values_(); MitsubishiCN105 hp_; + climate::ClimateSwingModeMask supported_swing_modes_{}; + MitsubishiCN105::VaneMode last_non_swing_vane_mode_{MitsubishiCN105::VaneMode::AUTO}; + MitsubishiCN105::WideVaneMode last_non_swing_wide_vane_mode_{MitsubishiCN105::WideVaneMode::CENTER}; }; template diff --git a/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp new file mode 100644 index 0000000000..36e0fc90b4 --- /dev/null +++ b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp @@ -0,0 +1,165 @@ +#include "../common.h" + +namespace esphome::mitsubishi_cn105::testing { + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeOffLeavesTraitsEmpty) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_OFF); + + EXPECT_FALSE(sut.traits().get_supports_swing_modes()); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeVerticalExposesOffAndVertical) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeHorizontalExposesOffAndHorizontal) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeBothExposesAllExpectedModes) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsVerticalSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_VERTICAL); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsHorizontalSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::AUTO; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_HORIZONTAL); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsBothSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_BOTH); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsSwingOffWhenNoSwingActive) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::POSITION_3; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesRemembersLastNonSwingPositions) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::POSITION_4; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::RIGHT; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_4); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::RIGHT); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_4); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::RIGHT); + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_BOTH); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesDoesNotOverwriteRememberedPositionWithUnknownValues) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.last_non_swing_vane_mode_ = MitsubishiCN105::VaneMode::POSITION_2; + sut.last_non_swing_wide_vane_mode_ = MitsubishiCN105::WideVaneMode::LEFT; + + sut.status().vane_mode = MitsubishiCN105::VaneMode::UNKNOWN; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::UNKNOWN; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_2); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::LEFT); + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesIgnoresUnsupportedVerticalSwingState) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesIgnoresUnsupportedHorizontalSwingState) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::AUTO; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +} // namespace esphome::mitsubishi_cn105::testing diff --git a/tests/components/mitsubishi_cn105/common.h b/tests/components/mitsubishi_cn105/common.h index 59b6203732..798f7283f6 100644 --- a/tests/components/mitsubishi_cn105/common.h +++ b/tests/components/mitsubishi_cn105/common.h @@ -8,6 +8,7 @@ #include #include "esphome/components/uart/uart_component.h" #include "esphome/components/mitsubishi_cn105/mitsubishi_cn105.h" +#include "esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h" namespace esphome::mitsubishi_cn105::testing { @@ -44,6 +45,7 @@ class TestableMitsubishiCN105 : public MitsubishiCN105 { using MitsubishiCN105::State; using MitsubishiCN105::UpdateFlag; using MitsubishiCN105::state_; + using MitsubishiCN105::status_; using MitsubishiCN105::operation_start_ms_; using MitsubishiCN105::use_temperature_encoding_b_; using MitsubishiCN105::set_wide_vane_high_bit_; @@ -58,4 +60,13 @@ class TestableMitsubishiCN105 : public MitsubishiCN105 { void set_current_time(uint32_t ms) { test_loop_time_ms = ms; } }; +class TestableMitsubishiCN105Climate : public MitsubishiCN105Climate { + public: + using MitsubishiCN105Climate::apply_values_; + using MitsubishiCN105Climate::last_non_swing_vane_mode_; + using MitsubishiCN105Climate::last_non_swing_wide_vane_mode_; + + MitsubishiCN105::Status &status() { return static_cast(this->hp_).status_; } +}; + } // namespace esphome::mitsubishi_cn105::testing diff --git a/tests/components/mitsubishi_cn105/common.yaml b/tests/components/mitsubishi_cn105/common.yaml index 4b64f51261..5b9c3aaaf6 100644 --- a/tests/components/mitsubishi_cn105/common.yaml +++ b/tests/components/mitsubishi_cn105/common.yaml @@ -3,6 +3,9 @@ climate: id: ac name: "AC Test" uart_id: uart_bus + update_interval: 30s + current_temperature_min_interval: 120s + supported_swing_modes: BOTH esphome: on_boot: From 8645f3672d142417e5b3c0351e3946b9c2d7ebb0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 18:11:40 -0500 Subject: [PATCH 56/96] [core] Enable additional zero-violation ruff lint families (#16645) --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d16bf2b625..94cd6d21b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,14 +113,22 @@ exclude = ['generated'] select = [ "E", # pycodestyle "F", # pyflakes/autoflake + "FA", # flake8-future-annotations "FLY", # flynt: convert string formatting to f-strings "FURB", # refurb "I", # isort + "ICN", # flake8-import-conventions + "LOG", # flake8-logging + "NPY", # numpy-specific rules "PERF", # performance "PL", # pylint + "Q", # flake8-quotes "SIM", # flake8-simplify "RET", # flake8-ret + "T10", # flake8-debugger "UP", # pyupgrade + "W", # pycodestyle warnings + "YTT", # flake8-2020 ] ignore = [ From 97267105e1917a0195d51b0267dc9b1073a47d88 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:05:51 -0500 Subject: [PATCH 57/96] [core] Enable ruff EXE (flake8-executable) lint family (#16648) --- pyproject.toml | 1 + script/ci_helpers.py | 0 2 files changed, 1 insertion(+) mode change 100755 => 100644 script/ci_helpers.py diff --git a/pyproject.toml b/pyproject.toml index 94cd6d21b8..5de8775713 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,7 @@ exclude = ['generated'] [tool.ruff.lint] select = [ "E", # pycodestyle + "EXE", # flake8-executable "F", # pyflakes/autoflake "FA", # flake8-future-annotations "FLY", # flynt: convert string formatting to f-strings diff --git a/script/ci_helpers.py b/script/ci_helpers.py old mode 100755 new mode 100644 From 51722279312ab8c9e0475766592a0c39a581c2a8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:06:01 -0500 Subject: [PATCH 58/96] [core] Enable ruff SLOT (flake8-slots) lint family (#16647) --- pyproject.toml | 1 + tests/unit_tests/test_yaml_util.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5de8775713..a48bae660f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,7 @@ select = [ "PL", # pylint "Q", # flake8-quotes "SIM", # flake8-simplify + "SLOT", # flake8-slots "RET", # flake8-ret "T10", # flake8-debugger "UP", # pyupgrade diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index de70a5307d..d6fb5b81f2 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -907,7 +907,7 @@ def test_format_path_current_obj_without_location_falls_back_to_key(): """An ESPHomeDataBase current_obj with no esp_range falls back to the key's location.""" class _NoRange(ESPHomeDataBase, str): - pass + __slots__ = () obj = _NoRange.__new__(_NoRange, "value") str.__init__(obj) From f1839489dd6c7ec61020fb0fa7e7e1fc5c62bda9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:06:18 -0500 Subject: [PATCH 59/96] [core] Enable ruff ISC (flake8-implicit-str-concat) lint family (#16646) --- pyproject.toml | 1 + script/api_protobuf/api_protobuf.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a48bae660f..8916abd6b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,6 +119,7 @@ select = [ "FURB", # refurb "I", # isort "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat "LOG", # flake8-logging "NPY", # numpy-specific rules "PERF", # performance diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index bf672d0567..91aec91637 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1283,11 +1283,11 @@ class PackedBufferTypeInfo(TypeInfo): """Dump shows buffer info but not decoded values.""" return ( f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n' - + 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n' - + f"append_uint(out, this->{self.field_name}_count_);\n" - + 'out.append_p(ESPHOME_PSTR(" values, "));\n' - + f"append_uint(out, this->{self.field_name}_length_);\n" - + 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));' + 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n' + f"append_uint(out, this->{self.field_name}_count_);\n" + 'out.append_p(ESPHOME_PSTR(" values, "));\n' + f"append_uint(out, this->{self.field_name}_length_);\n" + 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));' ) def dump(self, name: str) -> str: From bbc24ab5469ad15075ecd4ee52a60e6285bdd7c3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:06:34 -0500 Subject: [PATCH 60/96] [core] Enable ruff RSE (flake8-raise) lint family (#16649) --- esphome/components/external_components/__init__.py | 2 +- esphome/components/waveshare_epaper/display.py | 2 +- esphome/config_validation.py | 2 +- esphome/cpp_generator.py | 2 +- esphome/espidf/component.py | 2 +- pyproject.toml | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/esphome/components/external_components/__init__.py b/esphome/components/external_components/__init__.py index 6eb577e5ad..c892ec1112 100644 --- a/esphome/components/external_components/__init__.py +++ b/esphome/components/external_components/__init__.py @@ -81,7 +81,7 @@ def _process_single_config(config: dict[str, Any]) -> None: elif conf[CONF_TYPE] == TYPE_LOCAL: components_dir = Path(CORE.relative_config_path(conf[CONF_PATH])) else: - raise NotImplementedError() + raise NotImplementedError if config[CONF_COMPONENTS] == "all": num_components = len(list(components_dir.glob("*/__init__.py"))) diff --git a/esphome/components/waveshare_epaper/display.py b/esphome/components/waveshare_epaper/display.py index 5db7a1fc3d..7ecc3b4a87 100644 --- a/esphome/components/waveshare_epaper/display.py +++ b/esphome/components/waveshare_epaper/display.py @@ -236,7 +236,7 @@ async def to_code(config): rhs = model.new() var = cg.Pvariable(config[CONF_ID], rhs, model) else: - raise NotImplementedError() + raise NotImplementedError await display.register_display(var, config) await spi.register_spi_device(var, config, write_only=True) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index c993c1dcc5..ca1fd8f5d4 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1862,7 +1862,7 @@ def extract_keys(schema): elif isinstance(skey, vol.Marker) and isinstance(skey.schema, str): keys.append(skey.schema) else: - raise ValueError() + raise ValueError keys.sort() return keys diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index c622207dac..c5e398b2d7 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -893,7 +893,7 @@ class MockObj(Expression): def __getattr__(self, attr: str) -> "MockObj": # prevent python dunder methods being replaced by mock objects if attr.startswith("__"): - raise AttributeError() + raise AttributeError next_op = "." if attr.startswith("P") and self.op not in ["::", ""]: attr = attr[1:] diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 3534ac82f5..81f2cd9632 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -55,7 +55,7 @@ ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE" class Source: def download(self, dir_suffix: str, force: bool = False) -> Path: - raise NotImplementedError() + raise NotImplementedError class URLSource(Source): diff --git a/pyproject.toml b/pyproject.toml index 8916abd6b9..0e7bba82e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,7 @@ select = [ "PERF", # performance "PL", # pylint "Q", # flake8-quotes + "RSE", # flake8-raise "SIM", # flake8-simplify "SLOT", # flake8-slots "RET", # flake8-ret From b39b34bfe1f866c774719638552117ba5a2d3cc9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:14:26 -0500 Subject: [PATCH 61/96] [core] Enable ruff C4 (flake8-comprehensions) lint family (#16653) --- esphome/components/font/__init__.py | 4 ++-- esphome/components/libretiny/__init__.py | 14 +++++++------- esphome/components/lvgl/__init__.py | 2 +- esphome/components/lvgl/defines.py | 2 +- esphome/components/opentherm/generate.py | 10 +++------- esphome/components/time/__init__.py | 2 +- esphome/espidf/framework.py | 2 +- esphome/pins.py | 4 +--- pyproject.toml | 1 + script/ci-custom.py | 2 +- script/extract_automations.py | 6 +++--- 11 files changed, 22 insertions(+), 27 deletions(-) diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index a10c45a9d7..cb4b1d3a60 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -563,13 +563,13 @@ async def to_code(config): point_set.update(flatten(config[CONF_GLYPHS])) # Create the codepoint to font file map base_font = FONT_CACHE[config[CONF_FILE]] - point_font_map: dict[str, Face] = {c: base_font for c in point_set} + point_font_map: dict[str, Face] = dict.fromkeys(point_set, base_font) # process extras, updating the map and extending the codepoint list for extra in config[CONF_EXTRAS]: extra_points = flatten(extra[CONF_GLYPHS]) point_set.update(extra_points) extra_font = FONT_CACHE[extra[CONF_FILE]] - point_font_map.update({c: extra_font for c in extra_points}) + point_font_map.update(dict.fromkeys(extra_points, extra_font)) codepoints = list(point_set) codepoints.sort(key=functools.cmp_to_key(glyph_comparator)) diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index 40fb773784..d1f1042501 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -513,13 +513,13 @@ async def component_to_code(config): # apply LibreTiny options from framework: block # setup LT logger to work nicely with ESPHome logger - lt_options = dict( - LT_LOGLEVEL="LT_LEVEL_" + framework[CONF_LOGLEVEL], - LT_LOGGER_CALLER=0, - LT_LOGGER_TASK=0, - LT_LOGGER_COLOR=1, - LT_USE_TIME=1, - ) + lt_options = { + "LT_LOGLEVEL": "LT_LEVEL_" + framework[CONF_LOGLEVEL], + "LT_LOGGER_CALLER": 0, + "LT_LOGGER_TASK": 0, + "LT_LOGGER_COLOR": 1, + "LT_USE_TIME": 1, + } # enable/disable per-module debugging for module in framework[CONF_DEBUG]: if module == "NONE": diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index 44bcda9ba9..6e005f897e 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -174,7 +174,7 @@ def generate_lv_conf_h(): if clashes: LOGGER.warning( "Some defines are set both by ESPHome build flags and by LVGL configuration which may lead to unexpected behavior: %s", - sorted(list(clashes)), + sorted(clashes), ) unused_defines = all_defines - lv_defines.keys() - defines_from_flags diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 15a24f1ad2..d9be881a7f 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -335,7 +335,7 @@ TYPE_NONE = "none" DIRECTIONS = LvConstant("LV_DIR_", "LEFT", "RIGHT", "BOTTOM", "TOP") -LV_FONTS = list(f"montserrat_{s}" for s in range(8, 50, 2)) + [ +LV_FONTS = [f"montserrat_{s}" for s in range(8, 50, 2)] + [ "dejavu_16_persian_hebrew", "simsun_16_cjk", "unscii_8", diff --git a/esphome/components/opentherm/generate.py b/esphome/components/opentherm/generate.py index 0b39895798..1c0de329e5 100644 --- a/esphome/components/opentherm/generate.py +++ b/esphome/components/opentherm/generate.py @@ -16,7 +16,7 @@ def define_has_component(component_type: str, keys: list[str]) -> None: cg.add_define( f"OPENTHERM_{component_type.upper()}_LIST(F, sep)", cg.RawExpression( - " sep ".join(map(lambda key: f"F({key}_{component_type.lower()})", keys)) + " sep ".join(f"F({key}_{component_type.lower()})" for key in keys) ), ) for key in keys: @@ -30,12 +30,8 @@ def define_has_settings(keys: list[str], schemas: dict[str, SettingSchema]) -> N "OPENTHERM_SETTING_LIST(F, sep)", cg.RawExpression( " sep ".join( - map( - lambda key: ( - f"F({schemas[key].backing_type}, {key}_setting, {schemas[key].default_value})" - ), - keys, - ) + f"F({schemas[key].backing_type}, {key}_setting, {schemas[key].default_value})" + for key in keys ) ), ) diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index 8839a988a1..e9f6cd77e5 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -204,7 +204,7 @@ def cron_expression_validator(name, min_value, max_value, special_mapping=None): raise cv.Invalid( f"{name} {v} is out of range (min={min_value} max={max_value})." ) - return list(sorted(value)) + return sorted(value) value = cv.string(value) values = set() for part in value.split(","): diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index fb53066edb..f393600732 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -40,7 +40,7 @@ def _str_to_lst_of_str(a: str | list[str]) -> list[str]: """ if isinstance(a, list): return a - return list(f.strip() for f in a.split(";") if f.strip()) + return [f.strip() for f in a.split(";") if f.strip()] ESPHOME_STAMP_FILE = ".esphome.stamp.json" diff --git a/esphome/pins.py b/esphome/pins.py index bdaa0e28ab..3e7848949f 100644 --- a/esphome/pins.py +++ b/esphome/pins.py @@ -313,9 +313,7 @@ def gpio_base_schema( :return: A schema for the pin """ mode_default = len(modes) == 1 - mode_dict = dict( - map(lambda m: (cv.Optional(m, default=mode_default), cv.boolean), modes) - ) + mode_dict = {cv.Optional(m, default=mode_default): cv.boolean for m in modes} def _number_validator(value): if isinstance(value, str) and value.upper().startswith("GPIOX"): diff --git a/pyproject.toml b/pyproject.toml index 0e7bba82e9..2b4a7272f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ exclude = ['generated'] [tool.ruff.lint] select = [ + "C4", # flake8-comprehensions "E", # pycodestyle "EXE", # flake8-executable "F", # pyflakes/autoflake diff --git a/script/ci-custom.py b/script/ci-custom.py index 51fea97874..c241343a1b 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -341,7 +341,7 @@ def lint_const_ordered(fname, content): matching = [ (i + 1, line) for i, line in enumerate(lines) if line.startswith(start) ] - ordered = list(sorted(matching, key=lambda x: x[1].replace("_", " "))) + ordered = sorted(matching, key=lambda x: x[1].replace("_", " ")) ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered)] for (mi, mline), (_, ol) in zip(matching, ordered): if mline == ol: diff --git a/script/extract_automations.py b/script/extract_automations.py index 4e650ce25f..3cdfb5d32c 100755 --- a/script/extract_automations.py +++ b/script/extract_automations.py @@ -12,9 +12,9 @@ if __name__ == "__main__": components = get_components_with_dependencies(files, True) dump = { - "actions": sorted(list(ACTION_REGISTRY.keys())), - "conditions": sorted(list(CONDITION_REGISTRY.keys())), - "pin_providers": sorted(list(PIN_SCHEMA_REGISTRY.keys())), + "actions": sorted(ACTION_REGISTRY.keys()), + "conditions": sorted(CONDITION_REGISTRY.keys()), + "pin_providers": sorted(PIN_SCHEMA_REGISTRY.keys()), } print(json.dumps(dump, indent=2)) From e492f8f8b6ba03f0cbe994d666e33215385800e1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:14:36 -0500 Subject: [PATCH 62/96] [tests] Disable hypothesis deadline on flaky IP address test (#16652) --- tests/unit_tests/test_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index bb00a15bee..efc2d8e42a 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -7,7 +7,7 @@ import stat from unittest.mock import MagicMock, patch from aioesphomeapi.host_resolver import AddrInfo, IPv4Sockaddr, IPv6Sockaddr -from hypothesis import given +from hypothesis import given, settings from hypothesis.strategies import ip_addresses import pytest @@ -151,6 +151,7 @@ def test_is_ip_address__invalid(host): assert actual is False +@settings(deadline=None) @given(value=ip_addresses(v=4).map(str)) def test_is_ip_address__valid(value): actual = helpers.is_ip_address(value) From dd0028c1b5a38b157143b267cd883fc0db277003 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:36:49 -0500 Subject: [PATCH 63/96] [core] Enable ruff G (flake8-logging-format) lint family (#16650) --- esphome/components/time/__init__.py | 2 +- esphome/loader.py | 4 ++-- esphome/pins.py | 3 ++- esphome/platformio/toolchain.py | 2 +- pyproject.toml | 1 + 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index e9f6cd77e5..f687df26c2 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -96,7 +96,7 @@ def _extract_tz_string(tzfile: bytes) -> str: return tzfile.split(b"\n")[-2].decode() except (IndexError, UnicodeDecodeError): _LOGGER.error("Could not determine TZ string. Please report this issue.") - _LOGGER.error("tzfile contents: %s", tzfile, exc_info=True) + _LOGGER.exception("tzfile contents: %s", tzfile) raise diff --git a/esphome/loader.py b/esphome/loader.py index d50554f8c9..c57c09274e 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -239,12 +239,12 @@ def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None: "Unable to import component %s: %s", domain, str(e), exc_info=False ) else: - _LOGGER.error("Unable to import component %s:", domain, exc_info=True) + _LOGGER.exception("Unable to import component %s:", domain) return None except Exception: # pylint: disable=broad-except if exception: raise - _LOGGER.error("Unable to load component %s:", domain, exc_info=True) + _LOGGER.exception("Unable to load component %s:", domain) return None manif = ComponentManifest(module) diff --git a/esphome/pins.py b/esphome/pins.py index 3e7848949f..d6393508ab 100644 --- a/esphome/pins.py +++ b/esphome/pins.py @@ -272,9 +272,10 @@ def check_strapping_pin(conf, strapping_pin_list: set[int], logger: Logger): num = conf[CONF_NUMBER] if num in strapping_pin_list and not conf.get(CONF_IGNORE_STRAPPING_WARNING): logger.warning( - f"GPIO{num} is a strapping PIN and should only be used for I/O with care.\n" + "GPIO%s is a strapping PIN and should only be used for I/O with care.\n" "Attaching external pullup/down resistors to strapping pins can cause unexpected failures.\n" "See https://esphome.io/guides/faq/#why-am-i-getting-a-warning-about-strapping-pins", + num, ) # mitigate undisciplined use of strapping: if num not in strapping_pin_list and conf.get(CONF_IGNORE_STRAPPING_WARNING): diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 073e134ac4..c81420e6ca 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -96,7 +96,7 @@ def _run_idedata(config): try: return json.loads(match.group()) except ValueError: - _LOGGER.error("Could not parse idedata", exc_info=True) + _LOGGER.exception("Could not parse idedata") _LOGGER.error("Stdout: %s", stdout) raise diff --git a/pyproject.toml b/pyproject.toml index 2b4a7272f2..c8f4d88351 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,7 @@ select = [ "FA", # flake8-future-annotations "FLY", # flynt: convert string formatting to f-strings "FURB", # refurb + "G", # flake8-logging-format "I", # isort "ICN", # flake8-import-conventions "ISC", # flake8-implicit-str-concat From 489cf483d0f7edd2373ada70efa284dc58e182ce Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 20:55:35 -0500 Subject: [PATCH 64/96] [core] Enable ruff PYI (flake8-pyi) lint family (#16654) --- esphome/cpp_generator.py | 22 ++++++++++++---------- esphome/mqtt.py | 2 +- esphome/yaml_util.py | 2 +- pyproject.toml | 1 + tests/unit_tests/test_main.py | 6 +++--- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index c5e398b2d7..151018baa4 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -1077,43 +1077,45 @@ class MockObj(Expression): op = BinOpExpression(other, "|", self) return MockObj(op) - def __iadd__(self, other: SafeExpType) -> "MockObj": + # MockObj operator overloads build a new C++ expression rather than mutating self, + # so the PYI034 "augmented assignment returns self" assumption does not apply. + def __iadd__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "+=", other) return MockObj(op) - def __isub__(self, other: SafeExpType) -> "MockObj": + def __isub__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "-=", other) return MockObj(op) - def __imul__(self, other: SafeExpType) -> "MockObj": + def __imul__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "*=", other) return MockObj(op) - def __itruediv__(self, other: SafeExpType) -> "MockObj": + def __itruediv__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "/=", other) return MockObj(op) - def __imod__(self, other: SafeExpType) -> "MockObj": + def __imod__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "%=", other) return MockObj(op) - def __ilshift__(self, other: SafeExpType) -> "MockObj": + def __ilshift__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "<<=", other) return MockObj(op) - def __irshift__(self, other: SafeExpType) -> "MockObj": + def __irshift__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, ">>=", other) return MockObj(op) - def __iand__(self, other: SafeExpType) -> "MockObj": + def __iand__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "&=", other) return MockObj(op) - def __ixor__(self, other: SafeExpType) -> "MockObj": + def __ixor__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "^=", other) return MockObj(op) - def __ior__(self, other: SafeExpType) -> "MockObj": + def __ior__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "|=", other) return MockObj(op) diff --git a/esphome/mqtt.py b/esphome/mqtt.py index ccacbaea54..098292f599 100644 --- a/esphome/mqtt.py +++ b/esphome/mqtt.py @@ -159,7 +159,7 @@ def get_esphome_device_ip( username: str | None = None, password: str | None = None, client_id: str | None = None, - timeout: int | float = 25, + timeout: float = 25, ) -> list[str]: if CONF_MQTT not in config: raise EsphomeError( diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 9a36ad089c..28f72ab831 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -763,7 +763,7 @@ def parse_yaml(file_name: Path, file_handle: TextIOWrapper, yaml_loader=None) -> def _load_yaml_internal_with_type( - loader_type: type[ESPHomeLoader] | type[ESPHomePurePythonLoader], + loader_type: type[ESPHomeLoader | ESPHomePurePythonLoader], fname: Path, content: TextIOWrapper, yaml_loader: Callable[[Path], dict[str, Any]], diff --git a/pyproject.toml b/pyproject.toml index c8f4d88351..a094b05efe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,7 @@ select = [ "NPY", # numpy-specific rules "PERF", # performance "PL", # pylint + "PYI", # flake8-pyi "Q", # flake8-quotes "RSE", # flake8-raise "SIM", # flake8-simplify diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 6ec0069b3a..f6b6d0b05f 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -11,7 +11,7 @@ from pathlib import Path import re import sys import time -from typing import Any +from typing import Any, Self from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -5110,11 +5110,11 @@ class MockSerial: self.timeout = 0.1 self._is_open = False - def __enter__(self) -> MockSerial: + def __enter__(self) -> Self: self._is_open = True return self - def __exit__(self, *args: Any) -> None: + def __exit__(self, *args: object) -> None: self._is_open = False @property From ae814cff5c51314f2daf43e89caeb71a832f636c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 25 May 2026 21:28:14 -0500 Subject: [PATCH 65/96] [core] Enable ruff B (flake8-bugbear) lint family (#16655) --- esphome/__main__.py | 6 +++--- esphome/analyze_memory/cli.py | 6 +++--- esphome/analyze_memory/demangle.py | 2 +- esphome/components/api/client.py | 2 +- esphome/components/font/__init__.py | 4 +++- esphome/components/lvgl/lv_validation.py | 2 +- esphome/components/lvgl/widgets/tabview.py | 2 +- esphome/components/msa3xx/binary_sensor.py | 2 +- esphome/components/opentherm/output/__init__.py | 2 +- esphome/components/sensor/__init__.py | 9 ++++++--- esphome/core/__init__.py | 2 +- esphome/dashboard/status/mdns.py | 2 +- esphome/dashboard/status/ping.py | 6 +++--- esphome/dashboard/web_server.py | 9 ++++++--- esphome/writer.py | 2 +- esphome/zeroconf.py | 2 +- pyproject.toml | 1 + script/api_protobuf/api_protobuf.py | 4 ++-- script/build_language_schema.py | 2 +- script/ci-custom.py | 4 ++-- script/determine-jobs.py | 2 +- script/split_components_for_ci.py | 2 +- script/test_build_components.py | 4 ++-- tests/component_tests/display/test_display_metadata.py | 7 +++---- tests/component_tests/packages/test_packages.py | 10 ++-------- tests/integration/conftest.py | 6 ++++-- tests/unit_tests/test_config_normalization.py | 2 +- 27 files changed, 54 insertions(+), 50 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 07bbd89358..268164acf6 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -608,7 +608,7 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: try: module = importlib.import_module("esphome.components." + CORE.target_platform) - process_stacktrace = getattr(module, "process_stacktrace") + process_stacktrace = module.process_stacktrace except (AttributeError, ImportError): _LOGGER.info( 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', @@ -1101,7 +1101,7 @@ def upload_program( host = devices[0] try: module = importlib.import_module("esphome.components." + CORE.target_platform) - if getattr(module, "upload_program")(config, args, host): + if module.upload_program(config, args, host): return 0, host except AttributeError: pass @@ -1353,7 +1353,7 @@ def _validate_bootloader_binary(binary: Path) -> None: def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int | None: try: module = importlib.import_module("esphome.components." + CORE.target_platform) - if getattr(module, "show_logs")(config, args, devices): + if module.show_logs(config, args, devices): return 0 except AttributeError: pass diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 8f1f39e1d6..a856e2988d 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -509,7 +509,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f"{_COMPONENT_CORE} Symbols > {self.SYMBOL_SIZE_THRESHOLD} B ({len(large_core_symbols)} symbols):" ) - for i, (symbol, demangled, size) in enumerate(large_core_symbols): + for i, (_symbol, demangled, size) in enumerate(large_core_symbols): # Core symbols only track (symbol, demangled, size) without section info, # so we don't show section labels here lines.append( @@ -601,7 +601,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f"{comp_name} Symbols > {self.SYMBOL_SIZE_THRESHOLD} B & storage ({len(large_symbols)} symbols):" ) - for i, (symbol, demangled, size, section) in enumerate(large_symbols): + for i, (_symbol, demangled, size, section) in enumerate(large_symbols): lines.append( f"{i + 1}. {self._format_symbol_with_section(demangled, size, section)}" ) @@ -640,7 +640,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f" Symbols > {self.RAM_SYMBOL_SIZE_THRESHOLD} B ({len(large_ram_syms)}):" ) - for symbol, demangled, size, section in large_ram_syms[:10]: + for _symbol, demangled, size, section in large_ram_syms[:10]: # Format section label consistently by stripping leading dot section_label = section.lstrip(".") if section else "" display_name = _format_pstorage_name(demangled) diff --git a/esphome/analyze_memory/demangle.py b/esphome/analyze_memory/demangle.py index 8999108b51..7dbd6d4f63 100644 --- a/esphome/analyze_memory/demangle.py +++ b/esphome/analyze_memory/demangle.py @@ -154,7 +154,7 @@ def batch_demangle( failed_count = 0 for original, stripped, prefix, demangled in zip( - symbols, symbols_stripped, symbols_prefixes, demangled_lines + symbols, symbols_stripped, symbols_prefixes, demangled_lines, strict=True ): # Add back any prefix that was removed demangled = _restore_symbol_prefix(prefix, stripped, demangled) diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index 7fba091730..327973a605 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -108,7 +108,7 @@ async def async_run_logs( platform_process_stacktrace = None try: module = importlib.import_module("esphome.components." + CORE.target_platform) - platform_process_stacktrace = getattr(module, "process_stacktrace") + platform_process_stacktrace = module.process_stacktrace except (AttributeError, ImportError): _LOGGER.info( 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index cb4b1d3a60..4ea6267275 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -594,7 +594,9 @@ async def to_code(config): x.height, ] for (x, y) in zip( - glyph_args, list(accumulate([len(x.bitmap_data) for x in glyph_args])) + glyph_args, + list(accumulate([len(x.bitmap_data) for x in glyph_args])), + strict=True, ) ] diff --git a/esphome/components/lvgl/lv_validation.py b/esphome/components/lvgl/lv_validation.py index a1b75182eb..27cbfff694 100644 --- a/esphome/components/lvgl/lv_validation.py +++ b/esphome/components/lvgl/lv_validation.py @@ -239,7 +239,7 @@ def color_retmapper(value): else: r, g, b, _ = from_rgbw(cval) return literal(f"lv_color_make({r}, {g}, {b})") - assert False + raise AssertionError(f"Unhandled lv_color value: {value!r}") def option_string(value): diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index 5e9e0494dd..ee252ecf0b 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -97,7 +97,7 @@ class TabviewType(WidgetType): tab_bar = Widget(bar_obj, obj_spec) await set_obj_properties(tab_bar, tab_style) if tab_items_style: - for index, tab_conf in enumerate(config[CONF_TABS]): + for index, _tab_conf in enumerate(config[CONF_TABS]): await set_obj_properties( Widget(lv_obj.get_child(bar_obj, index), button_spec), tab_items_style, diff --git a/esphome/components/msa3xx/binary_sensor.py b/esphome/components/msa3xx/binary_sensor.py index 793d5190af..732a0ed291 100644 --- a/esphome/components/msa3xx/binary_sensor.py +++ b/esphome/components/msa3xx/binary_sensor.py @@ -26,7 +26,7 @@ CONFIG_SCHEMA = MSA_SENSOR_SCHEMA.extend( ), key=CONF_NAME, ) - for event, icon in zip(EVENT_SENSORS, ICONS) + for event, icon in zip(EVENT_SENSORS, ICONS, strict=True) } ) diff --git a/esphome/components/opentherm/output/__init__.py b/esphome/components/opentherm/output/__init__.py index 87307eb051..68977b9e34 100644 --- a/esphome/components/opentherm/output/__init__.py +++ b/esphome/components/opentherm/output/__init__.py @@ -21,7 +21,7 @@ async def new_openthermoutput( var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) await output.register_output(var, config) - cg.add(getattr(var, "set_id")(cg.RawExpression(f'"{key}_{config[CONF_ID]}"'))) + cg.add(var.set_id(cg.RawExpression(f'"{key}_{config[CONF_ID]}"'))) input.generate_setters(var, config) return var diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 6bbab76363..5a2ebf03c0 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -1192,7 +1192,7 @@ def _std(x): def _correlation_coeff(x, y): m_x, m_y = _mean(x), _mean(y) - s_xy = sum((x_ - m_x) * (y_ - m_y) for x_, y_ in zip(x, y)) + s_xy = sum((x_ - m_x) * (y_ - m_y) for x_, y_ in zip(x, y, strict=True)) s_sq_x = sum((x_ - m_x) ** 2 for x_ in x) s_sq_y = sum((y_ - m_y) ** 2 for y_ in y) return s_xy / math.sqrt(s_sq_x * s_sq_y) @@ -1228,7 +1228,7 @@ def _mat_copy(m): def _mat_transpose(m): - return _mat_copy(zip(*m)) + return _mat_copy(zip(*m, strict=True)) def _mat_identity(n): @@ -1237,7 +1237,10 @@ def _mat_identity(n): def _mat_dot(a, b): b_t = _mat_transpose(b) - return [[sum(x * y for x, y in zip(row_a, col_b)) for col_b in b_t] for row_a in a] + return [ + [sum(x * y for x, y in zip(row_a, col_b, strict=True)) for col_b in b_t] + for row_a in a + ] def _mat_inverse(m): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 580d7f6477..182be38b18 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -1081,7 +1081,7 @@ class EnumValue: @enum_value.setter def enum_value(self, value): - setattr(self, "_enum_value", value) + self._enum_value = value CORE = EsphomeCore() diff --git a/esphome/dashboard/status/mdns.py b/esphome/dashboard/status/mdns.py index 881340ab24..9da9bb8f01 100644 --- a/esphome/dashboard/status/mdns.py +++ b/esphome/dashboard/status/mdns.py @@ -115,7 +115,7 @@ class MDNSStatus: results = await asyncio.gather( *(self.aiozc.async_resolve_host(name) for name in poll_names) ) - for name, address_list in zip(poll_names, results): + for name, address_list in zip(poll_names, results, strict=True): result = bool(address_list) host_mdns_state[name] = result for entry in poll_names[name]: diff --git a/esphome/dashboard/status/ping.py b/esphome/dashboard/status/ping.py index b4f106d21a..eb69fbb9b3 100644 --- a/esphome/dashboard/status/ping.py +++ b/esphome/dashboard/status/ping.py @@ -83,7 +83,7 @@ class PingStatus: return_exceptions=True, ) - for entry, result in zip(ping_group, dns_results): + for entry, result in zip(ping_group, dns_results, strict=True): if isinstance(result, Exception): # Only update state if its unknown or from ping # so we don't mark it as offline if we have a state @@ -106,7 +106,7 @@ class PingStatus: return_exceptions=True, ) - for entry_addresses, result in zip(entry_addresses, results): + for entry_address, result in zip(entry_addresses, results, strict=True): if isinstance(result, Exception): ping_result = False elif isinstance(result, BaseException): @@ -114,7 +114,7 @@ class PingStatus: else: host: Host = result ping_result = host.is_alive - entry: DashboardEntry = entry_addresses[0] + entry: DashboardEntry = entry_address[0] # If we can reach it via ping, we always set it # online, however if we can't reach it via ping # we only set it to offline if the state is unknown diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py index 916e937a53..88b454e5cf 100644 --- a/esphome/dashboard/web_server.py +++ b/esphome/dashboard/web_server.py @@ -1030,7 +1030,7 @@ class DownloadListRequestHandler(BaseHandler): try: module = importlib.import_module(f"esphome.components.{platform}") - get_download_types = getattr(module, "get_download_types") + get_download_types = module.get_download_types except AttributeError as exc: raise ValueError(f"Unknown platform {platform}") from exc downloads = get_download_types(storage_json) @@ -1146,7 +1146,7 @@ class MainRequestHandler(BaseHandler): begin = bool(self.get_argument("begin", False)) if settings.using_password: # Simply accessing the xsrf_token sets the cookie for us - self.xsrf_token # pylint: disable=pointless-statement + self.xsrf_token # pylint: disable=pointless-statement # noqa: B018 else: self.clear_cookie("_xsrf") @@ -1519,7 +1519,10 @@ def get_static_file_url(name: str) -> str: return f"{base}?hash={hash_}" -def make_app(debug=get_bool_env(ENV_DEV)) -> tornado.web.Application: +def make_app(debug: bool | None = None) -> tornado.web.Application: + if debug is None: + debug = get_bool_env(ENV_DEV) + def log_function(handler: tornado.web.RequestHandler) -> None: if handler.get_status() < 400: log_method = access_log.info diff --git a/esphome/writer.py b/esphome/writer.py index ad3877465d..ab014c5daa 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -358,7 +358,7 @@ def copy_src_tree(): platform = "esphome.components." + CORE.target_platform try: module = importlib.import_module(platform) - copy_files = getattr(module, "copy_files") + copy_files = module.copy_files copy_files() except AttributeError: pass diff --git a/esphome/zeroconf.py b/esphome/zeroconf.py index 5d922ea911..a4f4f46097 100644 --- a/esphome/zeroconf.py +++ b/esphome/zeroconf.py @@ -249,7 +249,7 @@ async def async_resolve_hosts( ), return_exceptions=True, ) - for host, result in zip(pending, results): + for host, result in zip(pending, results, strict=True): if isinstance(result, BaseException): _LOGGER.debug("Failed to resolve %s: %s", host, result) diff --git a/pyproject.toml b/pyproject.toml index a094b05efe..c6d96560d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ exclude = ['generated'] [tool.ruff.lint] select = [ + "B", # flake8-bugbear "C4", # flake8-comprehensions "E", # pycodestyle "EXE", # flake8-executable diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 91aec91637..1cc8e1ec98 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -3551,7 +3551,7 @@ static const char *const TAG = "api.service"; if id_ is not None and not mt.options.deprecated: id_to_msg_name[id_] = mt.name - for id_, (_, _, case_label) in cases: + for id_, (_, _, _case_label) in cases: msg_name = id_to_msg_name.get(id_, "") if msg_name in message_auth_map: needs_auth = message_auth_map[msg_name] @@ -3614,7 +3614,7 @@ static const char *const TAG = "api.service"; # Dispatch switch out += " switch (msg_type) {\n" - for i, (case, ifdef, case_label) in cases: + for _i, (case, ifdef, case_label) in cases: if ifdef is not None: out += _make_ifdef_line(ifdef) + "\n" diff --git a/script/build_language_schema.py b/script/build_language_schema.py index 921ee9d3d7..9dff70af3c 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -972,7 +972,7 @@ def convert(schema, config_var, path): } elif schema_type == "use_id": if inspect.ismodule(data): - m_attr_obj = getattr(data, "CONFIG_SCHEMA") + m_attr_obj = data.CONFIG_SCHEMA use_schema = known_schemas.get(repr(m_attr_obj)) if use_schema: [output_module, output_name] = use_schema[0][1].split(".") diff --git a/script/ci-custom.py b/script/ci-custom.py index c241343a1b..f2a9681be5 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -342,8 +342,8 @@ def lint_const_ordered(fname, content): (i + 1, line) for i, line in enumerate(lines) if line.startswith(start) ] ordered = sorted(matching, key=lambda x: x[1].replace("_", " ")) - ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered)] - for (mi, mline), (_, ol) in zip(matching, ordered): + ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered, strict=True)] + for (mi, mline), (_, ol) in zip(matching, ordered, strict=True): if mline == ol: continue target = next(i for i, line in ordered if line == mline) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index ef2175eb79..01b8623813 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -1047,7 +1047,7 @@ def detect_memory_impact_config( # Find common platforms supported by ALL components # This ensures we can build all components together in a merged config common_platforms = set(MEMORY_IMPACT_PLATFORM_PREFERENCE) - for component, platforms in component_platforms_map.items(): + for platforms in component_platforms_map.values(): common_platforms &= platforms # Select the most preferred platform from the common set diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index 0d10246bb4..7f06f50f48 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -295,7 +295,7 @@ def main() -> int: # Sort groups by signature for readability groupable_groups = [] isolated_groups = [] - for (platform, signature), group_comps in sorted(signature_groups.items()): + for (_platform, signature), group_comps in sorted(signature_groups.items()): if signature.startswith(ISOLATED_SIGNATURE_PREFIX): isolated_groups.append((signature, group_comps)) else: diff --git a/script/test_build_components.py b/script/test_build_components.py index 43b71004eb..51f3758291 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -890,7 +890,7 @@ def run_grouped_component_tests( print("=" * 80 + "\n") # Execute grouped tests - for (platform, signature), components in grouped_components.items(): + for (platform, _signature), components in grouped_components.items(): # Only group if we have multiple components with same signature if len(components) <= 1: continue @@ -1055,7 +1055,7 @@ def test_components( # Create empty test files for each platform (or filtered platform) reference_tests: list[Path] = [] - for platform_name, base_file in platform_bases.items(): + for platform_name in platform_bases: if platform_filter and not platform_name.startswith(platform_filter): continue # Create an empty test file named to match the platform diff --git a/tests/component_tests/display/test_display_metadata.py b/tests/component_tests/display/test_display_metadata.py index e569754494..ef3f12cb73 100644 --- a/tests/component_tests/display/test_display_metadata.py +++ b/tests/component_tests/display/test_display_metadata.py @@ -2,6 +2,8 @@ from unittest.mock import patch +import pytest + from esphome.components.display import ( DisplayMetaData, add_metadata, @@ -74,8 +76,5 @@ def test_add_metadata_overwrites_existing(): def test_metadata_is_frozen(): """Test that DisplayMetaData instances are immutable (frozen dataclass).""" meta = DisplayMetaData(320, 240, True, False) - try: + with pytest.raises(AttributeError): meta.width = 640 - assert False, "Expected FrozenInstanceError" - except AttributeError: - pass diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 8c809c5e91..66f946a5bd 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -510,15 +510,9 @@ def test_package_merge_by_missing_id() -> None: ], } - error_raised = False - try: + with pytest.raises(cv.Invalid) as exc_info: packages_pass(config) - assert False, "Expected validation error for missing ID" - except cv.Invalid as err: - error_raised = True - assert err.path == [CONF_SENSOR, 2] - - assert error_raised + assert exc_info.value.path == [CONF_SENSOR, 2] def test_package_list_remove_by_id() -> None: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fb025ce427..e593929583 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -407,8 +407,10 @@ async def wait_and_connect_api_client( # Wait for connection with timeout try: await asyncio.wait_for(connected_future, timeout=timeout) - except TimeoutError: - raise TimeoutError(f"Failed to connect to API after {timeout} seconds") + except TimeoutError as err: + raise TimeoutError( + f"Failed to connect to API after {timeout} seconds" + ) from err if return_disconnect_event: yield client, disconnect_event diff --git a/tests/unit_tests/test_config_normalization.py b/tests/unit_tests/test_config_normalization.py index d70f3c24e0..4ec17b3c7c 100644 --- a/tests/unit_tests/test_config_normalization.py +++ b/tests/unit_tests/test_config_normalization.py @@ -67,7 +67,7 @@ def test_iter_component_configs_with_multi_conf(mock_get_component: Mock) -> Non configs = list(config.iter_component_configs(test_config)) assert len(configs) == 2 - for domain, component, conf in configs: + for domain, _component, conf in configs: assert domain == "switch" assert "name" in conf From ae74920b814ad555a15002d12675d057b040e0e5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 00:14:42 -0500 Subject: [PATCH 66/96] [core] Enable ruff PTH (flake8-use-pathlib) lint family (#16661) --- esphome/__main__.py | 4 +- esphome/analyze_memory/cli.py | 6 +-- esphome/analyze_memory/toolchain.py | 3 +- esphome/build_gen/espidf.py | 2 +- esphome/bundle.py | 2 +- esphome/components/audio_file/__init__.py | 2 +- esphome/components/bme68x_bsec2/__init__.py | 2 +- .../esp32_hosted/update/__init__.py | 4 +- esphome/components/http_request/__init__.py | 4 +- esphome/components/image/__init__.py | 2 +- .../components/micro_wake_word/__init__.py | 4 +- esphome/components/nrf52/ota.py | 2 +- esphome/components/rp2040/generate_boards.py | 4 +- esphome/components/web_server/__init__.py | 4 +- esphome/components/zigbee/zigbee_esp32.py | 5 +-- esphome/dashboard/dashboard.py | 3 +- esphome/dashboard/web_server.py | 6 +-- esphome/espidf/component.py | 18 ++++---- esphome/espidf/extra_script.py | 2 +- esphome/espidf/framework.py | 44 +++++++++++-------- esphome/espidf/get_idf_tool_paths.py | 3 +- esphome/espidf/runner.py | 3 +- esphome/espidf/toolchain.py | 19 ++++---- esphome/espota2.py | 2 +- esphome/helpers.py | 4 +- esphome/mqtt.py | 6 +-- esphome/web_server_ota.py | 2 +- pyproject.toml | 1 + script/api_protobuf/api_protobuf.py | 12 ++--- script/build_helpers.py | 2 +- script/bump-version.py | 5 ++- script/ci-custom.py | 2 +- script/ci_add_metadata_to_json.py | 4 +- script/ci_helpers.py | 3 +- script/ci_memory_impact_comment.py | 2 +- script/ci_memory_impact_extract.py | 4 +- script/clang-format | 3 +- script/clang-tidy | 9 ++-- script/clang_tidy_hash.py | 6 +-- script/determine-jobs.py | 2 +- script/helpers.py | 19 ++++---- script/lint-python | 6 ++- script/sync-device_class.py | 5 ++- script/test_build_components.py | 2 +- tests/dashboard/test_web_server_paths.py | 10 ++--- tests/integration/conftest.py | 2 +- tests/script/test_check_import_time.py | 7 +-- tests/script/test_determine_jobs.py | 7 +-- tests/script/test_helpers.py | 4 +- tests/script/test_test_helpers.py | 5 +-- tests/unit_tests/core/test_config.py | 20 ++++----- tests/unit_tests/test_espidf_component.py | 2 +- tests/unit_tests/test_substitutions.py | 3 +- tests/unit_tests/test_writer.py | 8 ++-- 54 files changed, 162 insertions(+), 155 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 268164acf6..5f281ce832 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -794,7 +794,7 @@ def _check_and_emit_build_info() -> None: # Read build_info from JSON try: - with open(build_info_json_path, encoding="utf-8") as f: + with build_info_json_path.open(encoding="utf-8") as f: build_info = json.load(f) except (OSError, json.JSONDecodeError) as e: _LOGGER.debug("Failed to read build_info: %s", e) @@ -1056,7 +1056,7 @@ def _wait_for_serial_port( def _port_found() -> bool: if port is not None: if os.name == "posix": - return os.path.exists(port) + return Path(port).exists() return any(p.path == port for p in get_serial_ports()) ports = get_serial_ports() if known_ports is not None: diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index a856e2988d..4fbceb7e5e 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -6,6 +6,7 @@ from collections import defaultdict from collections.abc import Callable import heapq from operator import itemgetter +from pathlib import Path import sys from typing import TYPE_CHECKING @@ -699,7 +700,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): content = "\n".join(lines) if output_file: - with open(output_file, "w", encoding="utf-8") as f: + with Path(output_file).open("w", encoding="utf-8") as f: f.write(content) else: print(content) @@ -737,7 +738,6 @@ def main(): # Load build directory import json - from pathlib import Path from esphome.platformio.toolchain import IDEData @@ -785,7 +785,7 @@ def main(): if not idedata_path.exists(): continue try: - with open(idedata_path, encoding="utf-8") as f: + with idedata_path.open(encoding="utf-8") as f: raw_data = json.load(f) idedata = IDEData(raw_data) print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) diff --git a/esphome/analyze_memory/toolchain.py b/esphome/analyze_memory/toolchain.py index 3a8a5f7be4..a724d52f25 100644 --- a/esphome/analyze_memory/toolchain.py +++ b/esphome/analyze_memory/toolchain.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import os from pathlib import Path import subprocess from typing import TYPE_CHECKING @@ -37,7 +36,7 @@ def _find_in_platformio_packages(tool_name: str) -> str | None: Full path to the tool or None if not found """ # Get PlatformIO packages directory - platformio_home = Path(os.path.expanduser("~/.platformio/packages")) + platformio_home = Path("~/.platformio/packages").expanduser() if not platformio_home.exists(): return None diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 96f84ebbd1..0b50f72382 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -24,7 +24,7 @@ def get_available_components() -> list[str] | None: return None try: - with open(project_desc, encoding="utf-8") as f: + with project_desc.open(encoding="utf-8") as f: data = json.load(f) component_info = data.get("build_component_info", {}) diff --git a/esphome/bundle.py b/esphome/bundle.py index 4537cbce9d..d38f68ebfd 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -412,7 +412,7 @@ class ConfigBundleCreator: @staticmethod def _add_to_tar(tar: tarfile.TarFile, bf: BundleFile) -> None: """Add a BundleFile to the tar archive with deterministic metadata.""" - with open(bf.source, "rb") as f: + with bf.source.open("rb") as f: _add_bytes_to_tar(tar, bf.path, f.read()) diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py index 23c90e9b76..8dc546cec1 100644 --- a/esphome/components/audio_file/__init__.py +++ b/esphome/components/audio_file/__init__.py @@ -98,7 +98,7 @@ def read_audio_file_and_type(file_config: ConfigType) -> tuple[bytes, MockObj]: else: raise cv.Invalid("Unsupported file source") - with open(path, "rb") as f: + with path.open("rb") as f: data = f.read() try: diff --git a/esphome/components/bme68x_bsec2/__init__.py b/esphome/components/bme68x_bsec2/__init__.py index 5083d283ef..62cd9e2e36 100644 --- a/esphome/components/bme68x_bsec2/__init__.py +++ b/esphome/components/bme68x_bsec2/__init__.py @@ -169,7 +169,7 @@ async def to_code_base(config): path = _compute_local_file_path(_compute_url(config)) try: - with open(path, encoding="utf-8") as f: + with path.open(encoding="utf-8") as f: bsec2_iaq_config = f.read() except Exception as e: raise core.EsphomeError( diff --git a/esphome/components/esp32_hosted/update/__init__.py b/esphome/components/esp32_hosted/update/__init__.py index b258a26b08..202df21ab5 100644 --- a/esphome/components/esp32_hosted/update/__init__.py +++ b/esphome/components/esp32_hosted/update/__init__.py @@ -75,7 +75,7 @@ def _validate_firmware(config: dict[str, Any]) -> None: return path = CORE.relative_config_path(config[CONF_PATH]) - with open(path, "rb") as f: + with path.open("rb") as f: firmware_data = f.read() calculated = hashlib.sha256(firmware_data).hexdigest() expected = config[CONF_SHA256].lower() @@ -93,7 +93,7 @@ async def to_code(config: dict[str, Any]) -> None: if config[CONF_TYPE] == TYPE_EMBEDDED: path = config[CONF_PATH] - with open(CORE.relative_config_path(path), "rb") as f: + with CORE.relative_config_path(path).open("rb") as f: firmware_data = f.read() rhs = [HexInt(x) for x in firmware_data] arr_id = ID(f"{config[CONF_ID]}_data", is_declaration=True, type=cg.uint8) diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 90879c459e..2617951f0d 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -1,3 +1,5 @@ +from pathlib import Path + from esphome import automation import esphome.codegen as cg from esphome.components import esp32 @@ -174,7 +176,7 @@ async def to_code(config): if config.get(CONF_VERIFY_SSL): if ca_cert_path := config.get(CONF_CA_CERTIFICATE_PATH): - with open(ca_cert_path, encoding="utf-8") as f: + with Path(ca_cert_path).open(encoding="utf-8") as f: ca_cert_content = f.read() cg.add(var.set_ca_certificate(ca_cert_content)) else: diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 365554f7d2..2fefbdcd58 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -395,7 +395,7 @@ def download_image(value): def is_svg_file(file): if not file: return False - with open(file, "rb") as f: + with Path(file).open("rb") as f: return " tuple[dict, dict]: for json_file in sorted(json_dir.glob("*.json")): board_name = json_file.stem - with open(json_file, encoding="utf-8") as f: + with json_file.open(encoding="utf-8") as f: data = json.load(f) build = data.get("build", {}) @@ -136,7 +136,7 @@ def _get_variant(json_file: Path) -> str | None: """Get variant name from a board JSON file.""" if not json_file.exists(): return None - with open(json_file, encoding="utf-8") as f: + with json_file.open(encoding="utf-8") as f: data = json.load(f) return data.get("build", {}).get("variant") diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 84910b6f90..99a9b7518c 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -326,12 +326,12 @@ async def to_code(config): if CONF_CSS_INCLUDE in config: cg.add_define("USE_WEBSERVER_CSS_INCLUDE") path = CORE.relative_config_path(config[CONF_CSS_INCLUDE]) - with open(file=path, encoding="utf-8") as css_file: + with path.open(encoding="utf-8") as css_file: add_resource_as_progmem("CSS_INCLUDE", css_file.read()) if CONF_JS_INCLUDE in config: cg.add_define("USE_WEBSERVER_JS_INCLUDE") path = CORE.relative_config_path(config[CONF_JS_INCLUDE]) - with open(file=path, encoding="utf-8") as js_file: + with path.open(encoding="utf-8") as js_file: add_resource_as_progmem("JS_INCLUDE", js_file.read()) cg.add(var.set_include_internal(config[CONF_INCLUDE_INTERNAL])) if CONF_LOCAL in config and config[CONF_LOCAL]: diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index 89efd583ab..a0fadbce8b 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -129,9 +129,8 @@ def final_validate_esp32(config: ConfigType) -> ConfigType: if CONF_PARTITIONS in fv.full_config.get() and not isinstance( fv.full_config.get()[CONF_PARTITIONS], list ): - with open( - CORE.relative_config_path(fv.full_config.get()[CONF_PARTITIONS]), - encoding="utf8", + with CORE.relative_config_path(fv.full_config.get()[CONF_PARTITIONS]).open( + encoding="utf8" ) as f: partitions_tab = f.read() for partition, types in [ diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py index 81c10763e7..7fc21f8a44 100644 --- a/esphome/dashboard/dashboard.py +++ b/esphome/dashboard/dashboard.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor import contextlib import logging import os +from pathlib import Path import socket import threading from time import monotonic @@ -149,4 +150,4 @@ async def async_start(args) -> None: await dashboard.async_run() finally: if sock: - os.remove(sock) + Path(sock).unlink() diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py index 88b454e5cf..97d6639c1f 100644 --- a/esphome/dashboard/web_server.py +++ b/esphome/dashboard/web_server.py @@ -1040,7 +1040,7 @@ class DownloadListRequestHandler(BaseHandler): class DownloadBinaryRequestHandler(BaseHandler): def _load_file(self, path: str, compressed: bool) -> bytes: """Load a file from disk and compress it if requested.""" - with open(path, "rb") as f: + with Path(path).open("rb") as f: data = f.read() if compressed: return gzip.compress(data, 9) @@ -1292,7 +1292,7 @@ class EditRequestHandler(BaseHandler): def _read_file(self, filename: str, configuration: str) -> bytes | None: """Read a file and return the content as bytes.""" try: - with open(file=filename, encoding="utf-8") as f: + with Path(filename).open(encoding="utf-8") as f: return f.read() except FileNotFoundError: if configuration in const.SECRETS_FILES: @@ -1493,7 +1493,7 @@ def get_base_frontend_path() -> Path: static_path += "/" # This path can be relative, so resolve against the root or else templates don't work - path = Path(os.getcwd()) / static_path / "esphome_dashboard" + path = Path.cwd() / static_path / "esphome_dashboard" return path.resolve() diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 81f2cd9632..050002d9e2 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -317,24 +317,26 @@ def _collect_filtered_files(src_dir: PathType, src_filters: list[str]) -> list[s if pattern.endswith("/"): pattern = pattern.rstrip("/") + "/**" - full_pattern = os.path.join(glob.escape(str(src_dir)), pattern) + # glob.escape has no pathlib equivalent and the matcher works on raw + # path strings, so PTH118/PTH207 don't apply here. + full_pattern = os.path.join(glob.escape(str(src_dir)), pattern) # noqa: PTH118 matched = [] - for item in glob.glob(full_pattern, recursive=True): - if not os.path.isdir(item): + for item in glob.glob(full_pattern, recursive=True): # noqa: PTH207 + if not Path(item).is_dir(): matched.append(item) else: # PlatformIO quirk: a directory matched with "*" should include all its # nested files and subdirectories, not just the directory itself. for root, _, files in os.walk(item): - matched.extend([os.path.join(root, f) for f in files]) + matched.extend([str(Path(root) / f) for f in files]) if sign == "+": selected.update(matched) elif sign == "-": selected.difference_update(matched) - return [r for r in selected if os.path.isfile(r)] + return [r for r in selected if Path(r).is_file()] def _convert_library_to_component(library: Library) -> IDFComponent: @@ -486,7 +488,7 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: # Only keep sources build_src_files = [os.path.relpath(p, component.path) for p in build_src_files] build_src_files = [ - f for f in build_src_files if os.path.splitext(f)[1] in SRC_FILE_EXTENSIONS + f for f in build_src_files if Path(f).suffix in SRC_FILE_EXTENSIONS ] # Handle build flags @@ -740,7 +742,7 @@ def _parse_library_json(library_json_path: PathType): Returns: dict: Parsed JSON content as a Python dictionary. """ - with open(library_json_path, encoding="utf8") as fp: + with Path(library_json_path).open(encoding="utf8") as fp: return json.load(fp) @@ -754,7 +756,7 @@ def _parse_library_properties(library_properties_path: PathType): Returns: dict[str, str]: Mapping of parsed property keys to values. """ - with open(library_properties_path, encoding="utf8") as fp: + with Path(library_properties_path).open(encoding="utf8") as fp: data = {} for line in fp.read().splitlines(): line = line.strip() diff --git a/esphome/espidf/extra_script.py b/esphome/espidf/extra_script.py index 2f22f23c10..bead63ca21 100644 --- a/esphome/espidf/extra_script.py +++ b/esphome/espidf/extra_script.py @@ -108,7 +108,7 @@ def run_extra_script( """ env = _FakeSConsEnv(board_mcu=idf_target, pio_env=f"esphome_{idf_target}") code = compile(script_path.read_text(), str(script_path), "exec") - old_cwd = os.getcwd() + old_cwd = Path.cwd() try: os.chdir(library_dir) exec( # noqa: S102 pylint: disable=exec-used diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index f393600732..331c2f84b0 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -138,7 +138,7 @@ def rmdir(directory: PathType, msg: str | None = None): Raises: RuntimeError: If directory removal fails """ - if os.path.isdir(directory): + if Path(directory).is_dir(): try: if msg: _LOGGER.debug(msg) @@ -192,7 +192,7 @@ def _check_stamp(file: PathType, data: dict[str, str]) -> bool: return False try: - with open(file, encoding="utf-8") as f: + with Path(file).open(encoding="utf-8") as f: return json.load(f) == data except (json.JSONDecodeError, OSError): return False @@ -206,7 +206,7 @@ def _write_stamp(file: PathType, data: dict[str, str]): file: Path to the stamp file to write data: Dictionary containing data to write """ - with open(file, "w", encoding="utf8") as fp: + with Path(file).open("w", encoding="utf8") as fp: json.dump(data, fp) @@ -471,8 +471,12 @@ def _tar_extract_all( import stat import tarfile + # Tar extraction safety: os.path.realpath / commonpath / normpath have no + # pathlib equivalents and Path.resolve() would follow symlinks unsafely. + # Use os.path for the security-sensitive parts; the simple checks move to + # Path. extract_dir = os.fspath(extract_dir) - abs_dest = os.path.abspath(extract_dir) + abs_dest = os.path.abspath(extract_dir) # noqa: PTH100 with tarfile.open(fileobj=data, mode="r") as tar_ref: all_members = tar_ref.getmembers() @@ -491,8 +495,8 @@ def _tar_extract_all( name = name.lstrip("/" + os.sep) # 2. Reject absolute paths (incl. Windows drive) - if os.path.isabs(name) or ( - os.name == "nt" and ":" in name.split(os.sep)[0] + if Path(name).is_absolute() or ( + os.name == "nt" and ":" in name.split(os.sep)[0] # noqa: PTH206 ): continue @@ -506,7 +510,7 @@ def _tar_extract_all( name = norm[len(strip_prefix) :] # 4. Compute final path - target_path = os.path.realpath(os.path.join(abs_dest, name)) + target_path = os.path.realpath(os.path.join(abs_dest, name)) # noqa: PTH118 if os.path.commonpath([abs_dest, target_path]) != abs_dest: continue @@ -515,18 +519,20 @@ def _tar_extract_all( linkname = member.linkname # Reject absolute link targets - if os.path.isabs(linkname): + if Path(linkname).is_absolute(): continue # Strip leading slashes linkname = os.path.normpath(linkname) if member.issym(): - link_target = os.path.join( - abs_dest, os.path.dirname(name), linkname + link_target = os.path.join( # noqa: PTH118 + abs_dest, + os.path.dirname(name), # noqa: PTH120 + linkname, ) else: - link_target = os.path.join(abs_dest, linkname) + link_target = os.path.join(abs_dest, linkname) # noqa: PTH118 link_target = os.path.realpath(link_target) if os.path.commonpath([abs_dest, link_target]) != abs_dest: @@ -598,7 +604,9 @@ def _zip_extract_all( """ import zipfile - extract_dir = os.path.abspath(extract_dir) + # See note in archive_extract_all_tar: os.path is used intentionally for + # the security-sensitive abspath/commonpath checks below. + extract_dir = os.path.abspath(extract_dir) # noqa: PTH100 with zipfile.ZipFile(data, "r") as zip_ref: all_members = zip_ref.infolist() @@ -618,8 +626,8 @@ def _zip_extract_all( name = member.filename.lstrip("/\\") # 2. Reject absolute paths / Windows drives - if os.path.isabs(name) or ( - os.name == "nt" and ":" in name.split(os.sep)[0] + if Path(name).is_absolute() or ( + os.name == "nt" and ":" in name.split(os.sep)[0] # noqa: PTH206 ): continue @@ -633,7 +641,7 @@ def _zip_extract_all( name = norm[len(strip_prefix) :] # 4. Compute safe target path - target_path = os.path.abspath(os.path.join(extract_dir, name)) + target_path = os.path.abspath(os.path.join(extract_dir, name)) # noqa: PTH100, PTH118 if os.path.commonpath([extract_dir, target_path]) != extract_dir: raise ValueError(f"Unsafe path detected: {member.filename}") @@ -680,7 +688,7 @@ def archive_extract_all( with ExitStack() as stack: archive_ref: io.BufferedIOBase if isinstance(archive, (str, os.PathLike)): - archive_ref = stack.enter_context(open(archive, "rb")) + archive_ref = stack.enter_context(Path(archive).open("rb")) elif isinstance(archive, (io.BufferedReader, io.BufferedRandom)): archive_ref = archive elif isinstance(archive, io.RawIOBase): @@ -727,7 +735,7 @@ def download_from_mirrors( # 1. Open target file for writing if path given with ExitStack() as stack: if isinstance(target, (str, os.PathLike)): - f = stack.enter_context(open(target, "wb")) + f = stack.enter_context(Path(target).open("wb")) elif isinstance(target, (io.RawIOBase, io.IOBase)): f = target else: @@ -917,7 +925,7 @@ def _patch_tools_json_for_linux_arm64(framework_path: Path) -> None: return try: - with open(tools_json, encoding="utf-8") as f: + with tools_json.open(encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, OSError) as e: _LOGGER.warning( diff --git a/esphome/espidf/get_idf_tool_paths.py b/esphome/espidf/get_idf_tool_paths.py index 2e8859631d..7d99e629b1 100644 --- a/esphome/espidf/get_idf_tool_paths.py +++ b/esphome/espidf/get_idf_tool_paths.py @@ -10,6 +10,7 @@ not installed. import json import os +from pathlib import Path import sys from types import SimpleNamespace @@ -25,7 +26,7 @@ from idf_tools import ( g.idf_path = sys.argv[1] g.idf_tools_path = os.environ.get("IDF_TOOLS_PATH") -g.tools_json = os.path.join(g.idf_path, TOOLS_FILE) +g.tools_json = str(Path(g.idf_path) / TOOLS_FILE) tools_info = filter_tools_info(IDFEnv.get_idf_env(), load_tools_info()) args = SimpleNamespace(prefer_system=False) diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index 857d16c674..7c568db7be 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -91,6 +91,7 @@ def main() -> int: # ---- end sys.path fix-up ----------------------------------------------- import os + from pathlib import Path import re import runpy @@ -229,7 +230,7 @@ def main() -> int: # runpy.run_path does not do this automatically, but idf.py relies # on it to import its sibling modules (python_version_checker, # idf_py_actions, ...). - script_dir = os.path.dirname(os.path.abspath(script_path)) + script_dir = str(Path(script_path).resolve().parent) if script_dir not in sys.path: sys.path.insert(0, script_dir) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index ef28575caa..752f582e74 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -241,20 +241,21 @@ def has_outdated_files(): dependency_lock_path = CORE.relative_build_path("dependencies.lock") build_ninja_path = CORE.relative_build_path("build/build.ninja") - if not os.path.isdir(build_config_path) or not os.listdir(build_config_path): + if not build_config_path.is_dir() or not any(build_config_path.iterdir()): return True - if not os.path.isfile(cmakecache_txt_path): + if not cmakecache_txt_path.is_file(): return True - if not os.path.isfile(build_ninja_path): + if not build_ninja_path.is_file(): return True - if os.path.isfile(dependency_lock_path) and os.path.getmtime( - dependency_lock_path - ) > os.path.getmtime(build_ninja_path): + if ( + dependency_lock_path.is_file() + and dependency_lock_path.stat().st_mtime > build_ninja_path.stat().st_mtime + ): return True - cmakecache_txt_mtime = os.path.getmtime(cmakecache_txt_path) + cmakecache_txt_mtime = cmakecache_txt_path.stat().st_mtime return any( - os.path.getmtime(f) > cmakecache_txt_mtime + f.stat().st_mtime > cmakecache_txt_mtime for f in [sdkconfig_internal_path, idf_component_yml_path] if f.exists() ) @@ -452,7 +453,7 @@ def create_factory_bin() -> bool: return False try: - with open(flasher_args_path, encoding="utf-8") as f: + with flasher_args_path.open(encoding="utf-8") as f: flash_data = json.load(f) except (json.JSONDecodeError, OSError) as e: _LOGGER.error("Failed to read flasher_args.json: %s", e) diff --git a/esphome/espota2.py b/esphome/espota2.py index 701a125bcd..266702c142 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -517,7 +517,7 @@ def run_ota_impl_( continue _LOGGER.info("Connected to %s", sa[0]) - with open(filename, "rb") as file_handle: + with Path(filename).open("rb") as file_handle: try: perform_ota(sock, password, file_handle, filename, ota_type) except OTAError as err: diff --git a/esphome/helpers.py b/esphome/helpers.py index d7ddb5c416..733474c9c9 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -385,7 +385,7 @@ def rmtree(path: Path | str) -> None: def _onerror(func, path, exc_info): if os.access(path, os.W_OK): raise exc_info[1].with_traceback(exc_info[2]) - os.chmod(path, stat.S_IWUSR | stat.S_IRUSR) + Path(path).chmod(stat.S_IWUSR | stat.S_IRUSR) func(path) # ``onerror`` is deprecated in 3.12 in favour of ``onexc`` (different @@ -512,7 +512,7 @@ def copy_file_if_changed(src: Path, dst: Path) -> bool: # -> delete file (it would be overwritten anyway), and try again # if that fails, use normal error handler with suppress(OSError): - os.unlink(dst) + Path(dst).unlink() shutil.copyfile(src, dst) return True diff --git a/esphome/mqtt.py b/esphome/mqtt.py index 098292f599..d6bde0cbfd 100644 --- a/esphome/mqtt.py +++ b/esphome/mqtt.py @@ -2,7 +2,7 @@ import contextlib from datetime import datetime import json import logging -import os +from pathlib import Path import ssl import tempfile import time @@ -120,8 +120,8 @@ def prepare( key_file.close() context.load_cert_chain(cert_file.name, key_file.name) finally: - os.unlink(cert_file.name) - os.unlink(key_file.name) + Path(cert_file.name).unlink() + Path(key_file.name).unlink() client.tls_set_context(context) try: diff --git a/esphome/web_server_ota.py b/esphome/web_server_ota.py index 7c31c1b123..8d0fdeecff 100644 --- a/esphome/web_server_ota.py +++ b/esphome/web_server_ota.py @@ -126,7 +126,7 @@ def _try_upload( _LOGGER.info("Connecting to %s port %s...", ip, port) try: - with open(filename, "rb") as fh: + with filename.open("rb") as fh: streamer = _MultipartStreamer(fh, file_size, filename.name) try: response = requests.post( diff --git a/pyproject.toml b/pyproject.toml index c6d96560d5..ae1bd34f60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,7 @@ select = [ "NPY", # numpy-specific rules "PERF", # performance "PL", # pylint + "PTH", # flake8-use-pathlib "PYI", # flake8-pyi "Q", # flake8-quotes "RSE", # flake8-raise diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 1cc8e1ec98..240ee7890f 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -3163,7 +3163,7 @@ def main() -> None: defines_content += "\n" defines_content += "\nnamespace esphome::api {} // namespace esphome::api\n" - with open(root / "api_pb2_defines.h", "w", encoding="utf-8") as f: + with (root / "api_pb2_defines.h").open("w", encoding="utf-8") as f: f.write(defines_content) content = FILE_HEADER @@ -3448,13 +3448,13 @@ static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint #endif // HAS_PROTO_MESSAGE_DUMP """ - with open(root / "api_pb2.h", "w", encoding="utf-8") as f: + with (root / "api_pb2.h").open("w", encoding="utf-8") as f: f.write(content) - with open(root / "api_pb2.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2.cpp").open("w", encoding="utf-8") as f: f.write(cpp) - with open(root / "api_pb2_dump.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2_dump.cpp").open("w", encoding="utf-8") as f: f.write(dump_cpp) hpp = FILE_HEADER @@ -3641,10 +3641,10 @@ static const char *const TAG = "api.service"; } // namespace esphome::api """ - with open(root / "api_pb2_service.h", "w", encoding="utf-8") as f: + with (root / "api_pb2_service.h").open("w", encoding="utf-8") as f: f.write(hpp) - with open(root / "api_pb2_service.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2_service.cpp").open("w", encoding="utf-8") as f: f.write(cpp) prot_file.unlink() diff --git a/script/build_helpers.py b/script/build_helpers.py index fa722aa099..52f7ee317e 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -195,7 +195,7 @@ def load_component_yaml_configs(components: list[str], tests_dir: Path) -> dict: yaml_path = tests_dir / component / BENCHMARK_YAML_FILENAME if not yaml_path.is_file(): continue - with open(yaml_path) as f: + with yaml_path.open() as f: component_config = yaml.safe_load(f) if component_config and isinstance(component_config, dict): for key, value in component_config.items(): diff --git a/script/bump-version.py b/script/bump-version.py index ed927cb991..e09fc87c60 100755 --- a/script/bump-version.py +++ b/script/bump-version.py @@ -2,6 +2,7 @@ import argparse from dataclasses import dataclass +from pathlib import Path import re import sys @@ -39,12 +40,12 @@ class Version: def sub(path, pattern, repl, expected_count=1): - with open(path, encoding="utf-8") as fh: + with Path(path).open(encoding="utf-8") as fh: content = fh.read() content, count = re.subn(pattern, repl, content, flags=re.MULTILINE) if expected_count is not None: assert count == expected_count, f"Pattern {pattern} replacement failed!" - with open(path, "w", encoding="utf-8") as fh: + with Path(path).open("w", encoding="utf-8") as fh: fh.write(content) diff --git a/script/ci-custom.py b/script/ci-custom.py index f2a9681be5..1ac13e18f7 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -14,7 +14,7 @@ import time import colorama from helpers import filter_changed, git_ls_files, print_error_for_file, styled -sys.path.append(os.path.dirname(__file__)) +sys.path.append(str(Path(__file__).parent)) def find_all(a_str, sub): diff --git a/script/ci_add_metadata_to_json.py b/script/ci_add_metadata_to_json.py index 687b5131c0..e884e9a64c 100755 --- a/script/ci_add_metadata_to_json.py +++ b/script/ci_add_metadata_to_json.py @@ -44,7 +44,7 @@ def main() -> int: return 1 try: - with open(json_path, encoding="utf-8") as f: + with Path(json_path).open(encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, OSError) as e: print(f"Error loading JSON: {e}", file=sys.stderr) @@ -74,7 +74,7 @@ def main() -> int: # Write back try: - with open(json_path, "w", encoding="utf-8") as f: + with Path(json_path).open("w", encoding="utf-8") as f: json.dump(data, f, indent=2) print(f"Added metadata to {args.json_file}", file=sys.stderr) except OSError as e: diff --git a/script/ci_helpers.py b/script/ci_helpers.py index 48b0e4bbfe..a51a857ada 100644 --- a/script/ci_helpers.py +++ b/script/ci_helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from pathlib import Path def write_github_output(outputs: dict[str, str | int]) -> None: @@ -16,7 +17,7 @@ def write_github_output(outputs: dict[str, str | int]) -> None: """ github_output = os.environ.get("GITHUB_OUTPUT") if github_output: - with open(github_output, "a", encoding="utf-8") as f: + with Path(github_output).open("a", encoding="utf-8") as f: f.writelines(f"{key}={value}\n" for key, value in outputs.items()) else: for key, value in outputs.items(): diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 01316da27f..0908b99595 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -91,7 +91,7 @@ def load_analysis_json(json_path: str) -> dict | None: return None try: - with open(json_file, encoding="utf-8") as f: + with Path(json_file).open(encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as e: print(f"Failed to load analysis JSON: {e}", file=sys.stderr) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 2aa7394b11..feacc2b1af 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -127,7 +127,7 @@ def run_detailed_analysis(build_dir: str) -> dict | None: if not idedata_path.exists(): continue try: - with open(idedata_path, encoding="utf-8") as f: + with idedata_path.open(encoding="utf-8") as f: raw_data = json.load(f) idedata = IDEData(raw_data) print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) @@ -264,7 +264,7 @@ def main() -> int: output_path = Path(args.output_json) output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: + with output_path.open("w", encoding="utf-8") as f: json.dump(output_data, f, indent=2) print(f"Saved analysis to {args.output_json}", file=sys.stderr) diff --git a/script/clang-format b/script/clang-format index 028d752c55..df45798a30 100755 --- a/script/clang-format +++ b/script/clang-format @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import queue import re import subprocess @@ -70,7 +71,7 @@ def main(): ) args = parser.parse_args() - cwd = os.getcwd() + cwd = Path.cwd() files = [ os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp", "*.h", "*.tcc"]) ] diff --git a/script/clang-tidy b/script/clang-tidy index 1c413ffa23..56c0a9db71 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import queue import re import shutil @@ -32,7 +33,7 @@ def clang_options(idedata): cmd = [] # extract target architecture from triplet in g++ filename - triplet = os.path.basename(idedata["cxx_path"])[:-4] + triplet = Path(idedata["cxx_path"]).name[:-4] if triplet.startswith("xtensa-"): # clang doesn't support Xtensa (yet?), so compile in 32-bit mode and pretend we're the Xtensa compiler cmd.append("-m32") @@ -153,8 +154,8 @@ def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): if sys.stdout.isatty(): invocation.append("--use-color") - invocation.append(f"--header-filter={os.path.abspath(basepath)}/.*") - invocation.append(os.path.abspath(path)) + invocation.append(f"--header-filter={Path(basepath).resolve()}/.*") + invocation.append(str(Path(path).resolve())) invocation.append("--") invocation.extend(options) @@ -229,7 +230,7 @@ def main(): ) args = parser.parse_args() - cwd = os.getcwd() + cwd = Path.cwd() files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])] # Exclude benchmark files — they require google benchmark headers not # available in the ESP32 toolchain and use different naming conventions. diff --git a/script/clang_tidy_hash.py b/script/clang_tidy_hash.py index d0d8438437..f478535567 100755 --- a/script/clang_tidy_hash.py +++ b/script/clang_tidy_hash.py @@ -16,7 +16,7 @@ sys.path.insert(0, str(script_dir)) def read_file_lines(path: Path) -> list[str]: """Read lines from a file.""" - with open(path) as f: + with path.open() as f: return f.readlines() @@ -65,7 +65,7 @@ def get_clang_tidy_version_from_requirements(repo_root: Path | None = None) -> s def read_file_bytes(path: Path) -> bytes: """Read bytes from a file.""" - with open(path, "rb") as f: + with path.open("rb") as f: return f.read() @@ -120,7 +120,7 @@ def read_stored_hash(repo_root: Path | None = None) -> str | None: def write_file_content(path: Path, content: str) -> None: """Write content to a file.""" - with open(path, "w") as f: + with path.open("w") as f: f.write(content) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 01b8623813..417716cd77 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -306,7 +306,7 @@ def _is_clang_tidy_full_scan() -> bool: """ try: result = subprocess.run( - [os.path.join(root_path, "script", "clang_tidy_hash.py"), "--check"], + [str(Path(root_path) / "script" / "clang_tidy_hash.py"), "--check"], capture_output=True, check=False, ) diff --git a/script/helpers.py b/script/helpers.py index cf82a89f93..c56a434edf 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -17,10 +17,10 @@ from typing import Any import colorama -root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, "..", ".."))) -basepath = os.path.join(root_path, "esphome") -temp_folder = os.path.join(root_path, ".temp") -temp_header_file = os.path.join(temp_folder, "all-include.cpp") +root_path = str(Path(__file__).resolve().parent.parent) +basepath = str(Path(root_path) / "esphome") +temp_folder = str(Path(root_path) / ".temp") +temp_header_file = str(Path(temp_folder) / "all-include.cpp") # C++ file extensions used for clang-tidy and clang-format checks CPP_FILE_EXTENSIONS = (".cpp", ".h", ".hpp", ".cc", ".cxx", ".c", ".tcc") @@ -339,8 +339,8 @@ def _get_github_event_data() -> dict | None: Parsed event data dictionary, or None if not available """ github_event_path = os.environ.get("GITHUB_EVENT_PATH") - if github_event_path and os.path.exists(github_event_path): - with open(github_event_path) as f: + if github_event_path and Path(github_event_path).exists(): + with Path(github_event_path).open() as f: return json.load(f) return None @@ -464,7 +464,8 @@ def _get_changed_files_from_command(command: list[str]) -> list[str]: raise Exception(f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}") changed_files = splitlines_no_ends(proc.stdout) - changed_files = [os.path.relpath(f, os.getcwd()) for f in changed_files if f] + cwd = Path.cwd() + changed_files = [os.path.relpath(f, cwd) for f in changed_files if f] # noqa: PTH109 changed_files.sort() return changed_files @@ -499,7 +500,7 @@ def get_changed_components() -> list[str] | None: return None # Use list-components.py to get changed components - script_path = os.path.join(root_path, "script", "list-components.py") + script_path = str(Path(root_path) / "script" / "list-components.py") cmd = [script_path, "--changed"] try: @@ -619,7 +620,7 @@ def filter_changed(files: list[str]) -> list[str]: def filter_grep(files: list[str], value: list[str]) -> list[str]: matched = [] for file in files: - with open(file, encoding="utf-8") as handle: + with Path(file).open(encoding="utf-8") as handle: contents = handle.read() if any(v in contents for v in value): matched.append(file) diff --git a/script/lint-python b/script/lint-python index 18281c711e..e4b3314d2a 100755 --- a/script/lint-python +++ b/script/lint-python @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import re import sys @@ -66,11 +67,12 @@ def main(): args = parser.parse_args() files = [] + cwd = Path.cwd() for path in git_ls_files(): filetypes = (".py",) - ext = os.path.splitext(path)[1] + ext = Path(path).suffix if ext in filetypes and path.startswith("esphome"): - path = os.path.relpath(path, os.getcwd()) + path = os.path.relpath(path, cwd) files.append(path) # Match against re file_name_re = re.compile("|".join(args.files)) diff --git a/script/sync-device_class.py b/script/sync-device_class.py index 121c89b8f9..660142195a 100755 --- a/script/sync-device_class.py +++ b/script/sync-device_class.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +from pathlib import Path import re # pylint: disable=import-error @@ -34,10 +35,10 @@ DOMAINS = { def sub(path, pattern, repl): - with open(path, encoding="utf-8") as handle: + with Path(path).open(encoding="utf-8") as handle: content = handle.read() content = re.sub(pattern, repl, content, flags=re.MULTILINE) - with open(path, "w", encoding="utf-8") as handle: + with Path(path).open("w", encoding="utf-8") as handle: handle.write(content) diff --git a/script/test_build_components.py b/script/test_build_components.py index 51f3758291..767b55c94b 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -297,7 +297,7 @@ def write_github_summary( test_results: List of all test results """ summary_content = format_github_summary(test_results, toolchain) - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as f: f.write(summary_content) diff --git a/tests/dashboard/test_web_server_paths.py b/tests/dashboard/test_web_server_paths.py index b596ebb581..efeafbf3b5 100644 --- a/tests/dashboard/test_web_server_paths.py +++ b/tests/dashboard/test_web_server_paths.py @@ -34,9 +34,7 @@ def test_get_base_frontend_path_dev_mode() -> None: # The function uses Path.resolve() which resolves symlinks # The actual function adds "/" to the path, so we simulate that test_path_with_slash = test_path if test_path.endswith("/") else test_path + "/" - expected = ( - Path(os.getcwd()) / test_path_with_slash / "esphome_dashboard" - ).resolve() + expected = (Path.cwd() / test_path_with_slash / "esphome_dashboard").resolve() assert result == expected @@ -62,9 +60,7 @@ def test_get_base_frontend_path_dev_mode_relative_path() -> None: # The function uses Path.resolve() which resolves symlinks # The actual function adds "/" to the path, so we simulate that test_path_with_slash = test_path if test_path.endswith("/") else test_path + "/" - expected = ( - Path(os.getcwd()) / test_path_with_slash / "esphome_dashboard" - ).resolve() + expected = (Path.cwd() / test_path_with_slash / "esphome_dashboard").resolve() assert result == expected assert result.is_absolute() @@ -157,7 +153,7 @@ def test_load_file_path(tmp_path: Path) -> None: test_file = tmp_path / "test.txt" test_file.write_bytes(b"test content") - with open(test_file, "rb") as f: + with test_file.open("rb") as f: content = f.read() assert content == b"test content" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e593929583..a9c9e0686f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -79,7 +79,7 @@ def shared_platformio_cache() -> Generator[Path]: lock_file = Path.home() / ".esphome-integration-tests-init.lock" # Always acquire the lock to ensure cache is ready before proceeding - with open(lock_file, "w") as lock_fd: + with lock_file.open("w") as lock_fd: fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX) # Check if the native platform is installed (the actual indicator of a populated cache) diff --git a/tests/script/test_check_import_time.py b/tests/script/test_check_import_time.py index 223c58002c..528ca0701c 100644 --- a/tests/script/test_check_import_time.py +++ b/tests/script/test_check_import_time.py @@ -4,7 +4,6 @@ from __future__ import annotations import importlib.util import json -import os from pathlib import Path import sys from unittest.mock import patch @@ -13,12 +12,10 @@ import pytest # Load the script-under-test as `check_import_time` (it's a hyphenated path # inside `script/` that mirrors the existing `determine_jobs` pattern). -script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "script") -) +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) sys.path.insert(0, script_dir) spec = importlib.util.spec_from_file_location( - "check_import_time", os.path.join(script_dir, "check_import_time.py") + "check_import_time", str(Path(script_dir) / "check_import_time.py") ) check_import_time = importlib.util.module_from_spec(spec) spec.loader.exec_module(check_import_time) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 7bb9fe2543..ac3c6424bf 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -3,7 +3,6 @@ from collections.abc import Generator import importlib.util import json -import os from pathlib import Path import sys from unittest.mock import Mock, call, patch @@ -11,9 +10,7 @@ from unittest.mock import Mock, call, patch import pytest # Add the script directory to Python path so we can import the module -script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "script") -) +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) sys.path.insert(0, script_dir) # Import helpers module for patching @@ -22,7 +19,7 @@ import helpers # noqa: E402 import script.helpers # noqa: E402 spec = importlib.util.spec_from_file_location( - "determine_jobs", os.path.join(script_dir, "determine-jobs.py") + "determine_jobs", str(Path(script_dir) / "determine-jobs.py") ) determine_jobs = importlib.util.module_from_spec(spec) spec.loader.exec_module(determine_jobs) diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 10f258aa83..82ff5e1411 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -12,9 +12,7 @@ import pytest from pytest import MonkeyPatch # Add the script directory to Python path so we can import helpers -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "script")) -) +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) import helpers # noqa: E402 diff --git a/tests/script/test_test_helpers.py b/tests/script/test_test_helpers.py index 3149712563..a8100252da 100644 --- a/tests/script/test_test_helpers.py +++ b/tests/script/test_test_helpers.py @@ -1,6 +1,5 @@ """Unit tests for script/build_helpers.py manifest override and build helpers.""" -import os from pathlib import Path import sys import textwrap @@ -9,9 +8,7 @@ from unittest.mock import MagicMock, patch import pytest # Add the script directory to Python path so we can import build_helpers -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "script")) -) +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) import build_helpers # noqa: E402 diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 4ce862315d..b5b35b5172 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -486,7 +486,7 @@ def test_preload_core_config_basic(setup_core: Path) -> None: assert CONF_BUILD_PATH in config[CONF_ESPHOME] # Verify default build path is "build/" build_path = config[CONF_ESPHOME][CONF_BUILD_PATH] - assert build_path.endswith(os.path.join("build", "test_device")) + assert build_path.endswith(str(Path("build") / "test_device")) def test_preload_core_config_with_build_path(setup_core: Path) -> None: @@ -523,7 +523,7 @@ def test_preload_core_config_env_build_path(setup_core: Path) -> None: assert "test_device" in config[CONF_ESPHOME][CONF_BUILD_PATH] # Verify it uses the env var path with device name appended build_path = config[CONF_ESPHOME][CONF_BUILD_PATH] - expected_path = os.path.join("/env/build", "test_device") + expected_path = str(Path("/env/build") / "test_device") assert build_path == expected_path or build_path == expected_path.replace( "/", os.sep ) @@ -739,7 +739,7 @@ async def test_add_includes_with_single_file( """Test add_includes copies a single header file to build directory.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include file include_file = tmp_path / "my_header.h" @@ -769,7 +769,7 @@ async def test_add_includes_with_directory_unix( """Test add_includes copies all files from a directory on Unix.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include directory with files include_dir = tmp_path / "includes" @@ -814,7 +814,7 @@ async def test_add_includes_with_directory_windows( """Test add_includes copies all files from a directory on Windows.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include directory with files include_dir = tmp_path / "includes" @@ -856,7 +856,7 @@ async def test_add_includes_with_multiple_sources( """Test add_includes with multiple files and directories.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create various include sources single_file = tmp_path / "single.h" @@ -884,7 +884,7 @@ async def test_add_includes_empty_directory( """Test add_includes with an empty directory doesn't fail.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create empty directory empty_dir = tmp_path / "empty" @@ -906,7 +906,7 @@ async def test_add_includes_preserves_directory_structure_unix( """Test that add_includes preserves relative directory structure on Unix.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create nested directory structure lib_dir = tmp_path / "lib" @@ -940,7 +940,7 @@ async def test_add_includes_preserves_directory_structure_windows( """Test that add_includes preserves relative directory structure on Windows.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create nested directory structure lib_dir = tmp_path / "lib" @@ -973,7 +973,7 @@ async def test_add_includes_overwrites_existing_files( """Test that add_includes overwrites existing files in build directory.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include file include_file = tmp_path / "header.h" diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index f50f5317de..4f0a71053d 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -293,7 +293,7 @@ def test_extra_script_captures_libpath_libs_and_defines(tmp_path): result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32") - assert result.libpath == [os.path.join("src", "esp32")] + assert result.libpath == [str(Path("src") / "esp32")] assert result.libs == ["algobsec"] assert ("BAR", "1") in result.cppdefines assert "FOO" in result.cppdefines diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 4783112578..c71be2fbab 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -1,4 +1,3 @@ -import glob import logging from pathlib import Path from typing import Any @@ -106,7 +105,7 @@ REMOTES = { # Collect all input YAML files for test_substitutions_fixtures parametrized tests: HERE = Path(__file__).parent BASE_DIR = HERE / "fixtures" / "substitutions" -SOURCES = sorted(glob.glob(str(BASE_DIR / "*.input.yaml"))) +SOURCES = sorted(str(p) for p in BASE_DIR.glob("*.input.yaml")) assert SOURCES, f"test_substitutions_fixtures: No input YAML files found in {BASE_DIR}" diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 91b4bd8e87..fc49f03067 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -1358,7 +1358,7 @@ def test_clean_build_handles_readonly_files( # Create a read-only file (simulating git pack files on Windows) readonly_file = git_dir / "pack-abc123.pack" readonly_file.write_text("pack data") - os.chmod(readonly_file, stat.S_IRUSR) # Read-only + readonly_file.chmod(stat.S_IRUSR) # Read-only # Setup mocks mock_core.relative_pioenvs_path.return_value = pioenvs_dir @@ -1393,7 +1393,7 @@ def test_clean_all_handles_readonly_files( subdir.mkdir() readonly_file = subdir / "readonly.txt" readonly_file.write_text("content") - os.chmod(readonly_file, stat.S_IRUSR) # Read-only + readonly_file.chmod(stat.S_IRUSR) # Read-only # Verify file is read-only assert not os.access(readonly_file, os.W_OK) @@ -1422,7 +1422,7 @@ def test_clean_build_reraises_for_other_errors( test_file.write_text("content") # Make subdir read-only so files inside can't be deleted - os.chmod(subdir, stat.S_IRUSR | stat.S_IXUSR) + subdir.chmod(stat.S_IRUSR | stat.S_IXUSR) # Setup mocks mock_core.relative_pioenvs_path.return_value = pioenvs_dir @@ -1440,7 +1440,7 @@ def test_clean_build_reraises_for_other_errors( clean_build() finally: # Cleanup - restore write permission so tmp_path cleanup works - os.chmod(subdir, stat.S_IRWXU) + subdir.chmod(stat.S_IRWXU) # Tests for get_build_info() From 423b60c90ce030f02f0bfa560464d4016301da59 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 26 May 2026 19:56:44 +1200 Subject: [PATCH 67/96] [packages] Resolve git symlinks on Windows when materialized as text (#16657) --- esphome/components/packages/__init__.py | 42 +++- esphome/git.py | 87 +++++++ tests/unit_tests/test_git.py | 303 +++++++++++++++++++++++- tests/unit_tests/test_substitutions.py | 83 +++++++ 4 files changed, 507 insertions(+), 8 deletions(-) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 06a64208b6..f3e0e0db8f 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -215,7 +215,7 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: If loading fails after cloning, attempts a revert and retry in case a prior cached checkout is stale. """ - repo_dir, revert = git.clone_or_update( + repo_root, revert = git.clone_or_update( url=config[CONF_URL], ref=config.get(CONF_REF), refresh=config[CONF_REFRESH], @@ -225,6 +225,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: ) files: list[dict[str, Any]] = [] + # ``repo_root`` is the directory containing ``.git`` and must be passed + # to git for symlink-stub resolution. ``repo_dir`` may be narrowed to a + # subdirectory via the user's CONF_PATH and is used for file lookups. + repo_dir = repo_root if base_path := config.get(CONF_PATH): repo_dir = repo_dir / base_path @@ -236,13 +240,37 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: def _load_package_yaml(yaml_file: Path, filename: str) -> dict: """Load a YAML file from a remote package, validating min_version.""" - try: - new_yaml = yaml_util.load_yaml(yaml_file) - except EsphomeError as e: + + def _load(path: Path) -> dict | str | None: + try: + return yaml_util.load_yaml(path) + except EsphomeError as e: + raise cv.Invalid( + f"{filename} is not a valid YAML file." + f" Please check the file contents.\n{e}" + ) from e + + new_yaml = _load(yaml_file) + if not isinstance(new_yaml, dict): + # On Windows, git defaults to core.symlinks=false unless the user + # has Developer Mode enabled or is running elevated. Files stored + # in the repo as symlinks (tree mode 120000) are then checked out + # as plain text files containing the symlink target path, so + # parsing them as YAML yields a bare scalar instead of a mapping. + # Best-effort: follow the symlink target ourselves and re-load. + target = git.resolve_symlink_stub(repo_root, yaml_file) + if target is not None: + new_yaml = _load(target) + if not isinstance(new_yaml, dict): raise cv.Invalid( - f"{filename} is not a valid YAML file." - f" Please check the file contents.\n{e}" - ) from e + f"{filename} does not contain a YAML mapping at the top level " + f"(got {type(new_yaml).__name__}). " + f"If this file is a git symlink in the source repository, it " + f"may not have been materialized correctly on your platform " + f"(this is a known issue with git on Windows without Developer " + f"Mode enabled). Try pointing your package at the real file " + f"path instead." + ) esphome_config = new_yaml.get(CONF_ESPHOME) or {} min_version = esphome_config.get(CONF_MIN_VERSION) if min_version is not None and cv.Version.parse(min_version) > cv.Version.parse( diff --git a/esphome/git.py b/esphome/git.py index f36bd559ef..094a6dae19 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -6,6 +6,7 @@ import logging from pathlib import Path import re import subprocess +import sys import urllib.parse import esphome.config_validation as cv @@ -94,6 +95,92 @@ def _compute_destination_path(key: str, domain: str) -> Path: return base_dir / h.hexdigest()[:8] +def resolve_symlink_stub(repo_dir: Path, file_path: Path) -> Path | None: + """Return the symlink target if ``file_path`` is a Windows-checked-out symlink stub. + + On Windows, when ``core.symlinks=false`` (the default unless the user has + SeCreateSymbolicLinkPrivilege — i.e. Developer Mode or running elevated), + git materializes files with tree mode ``120000`` as plain text files + whose content is the literal symlink target path. Opening such a file + yields the target path string instead of the target's content. + + If ``file_path`` is one of those stubs, return the resolved target Path + inside ``repo_dir``. Otherwise return ``None`` and the caller should use + ``file_path`` as-is. + + Designed to be called *only* when normal access has already produced an + unexpected result (e.g. YAML parsed as a top-level scalar), so the + per-file ``git ls-files`` subprocess cost is paid only on the failure + path. Returns ``None`` on any error or check failure — it's purely a + best-effort recovery, never raises. + """ + # On non-Windows, git creates real symlinks; ordinary file access already + # transparently follows them. + if sys.platform != "win32": + return None + if file_path.is_symlink(): + return None + if not file_path.is_file(): + return None + + try: + rel = file_path.relative_to(repo_dir) + except ValueError: + return None + + try: + # ``git ls-files -s `` prints " \t" + # for that single entry, or empty if untracked. + out = run_git_command( + ["git", "ls-files", "-s", "--", rel.as_posix()], + git_dir=repo_dir, + ) + except GitException: + return None + + parts = out.split() + if not parts or parts[0] != "120000": + return None + + # Stubs are short ASCII relative paths. Decode defensively, and only + # strip the trailing newline git's checkout may append — preserving any + # whitespace that could be part of a valid target name. + try: + raw = file_path.read_bytes() + except OSError: + return None + try: + target_str = raw.decode("utf-8").rstrip("\r\n") + except UnicodeDecodeError: + return None + + # ``Path()`` and ``Path.resolve()`` can raise on malformed inputs (e.g. + # embedded NUL bytes from a hostile symlink blob, paths too long for the + # OS, or temporary I/O errors). Catch broadly — this helper is purely a + # best-effort recovery and must never raise. + try: + target_path = (file_path.parent / target_str).resolve() + repo_root_resolved = repo_dir.resolve() + except (OSError, ValueError, RuntimeError): + return None + + # ``Path.resolve()`` follows ``..``; re-verify containment afterwards. + try: + target_path.relative_to(repo_root_resolved) + except ValueError: + _LOGGER.warning( + "Refusing to follow symlink %s -> %s (escapes repository)", + file_path, + target_str, + ) + return None + + if not target_path.is_file(): + return None + + return target_path + + def clone_or_update( *, url: str, diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index eab6bfc2cb..690c47c183 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta import os from pathlib import Path from typing import Any -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest @@ -1001,3 +1001,304 @@ def test_refresh_picks_up_new_remote_commits( "--hard", "old_sha", ] + + +def test_resolve_symlink_stub_returns_none_on_non_windows( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """On non-Windows, resolve_symlink_stub returns None without calling git.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + stub = repo_dir / "file.yaml" + stub.write_text("static/file.yaml") + + with patch("esphome.git.sys.platform", "linux"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_target_for_mode_120000( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A mode-120000 file is recognised as a stub; its target Path is returned.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "real.yaml" + target.write_text("esphome:\n name: real\n") + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\treal.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + # Stub file itself was not modified — only inspected. + assert stub.read_text() == "static/real.yaml" + + +def test_resolve_symlink_stub_resolves_relative_parent_paths( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Symlink targets with ``..`` segments resolve correctly within the repo.""" + repo_dir = tmp_path / "repo" + (repo_dir / "subdir").mkdir(parents=True) + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "shared.yaml" + target.write_text("shared content") + + stub = repo_dir / "subdir" / "shared.yaml" + stub.write_text("../static/shared.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tsubdir/shared.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_refuses_escape_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing outside the repository is not followed.""" + outside = tmp_path / "outside.yaml" + outside.write_text("sensitive") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "escape.yaml" + stub.write_text("../outside.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tescape.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_real_symlink( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A real symlink already opens transparently, so the helper short-circuits. + + Skipped on Windows where symlink creation requires + SeCreateSymbolicLinkPrivilege. + """ + if os.name == "nt": + pytest.skip("Requires symlink-creation privilege on Windows") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target = repo_dir / "real.yaml" + target.write_text("real content") + + real_link = repo_dir / "link.yaml" + real_link.symlink_to("real.yaml") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, real_link) + + assert result is None + # No git call needed for real symlinks. + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_for_regular_file( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A regular file (mode 100644) whose content looks path-shaped is not + followed.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + regular = repo_dir / "looks_like_path.txt" + regular.write_text("static/something.yaml") + + mock_run_git_command.return_value = "100644 abc123 0\tlooks_like_path.txt" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, regular) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_git_fails( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """If ``git ls-files`` fails (e.g. not a repo), the helper returns None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.side_effect = GitCommandError("ls-files exploded") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_non_utf8_content( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file whose bytes are not valid UTF-8 must not raise — return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "binary.bin" + stub.write_bytes(b"\xff\xfe\x00\xff") + + mock_run_git_command.return_value = "120000 abc123 0\tbinary.bin" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_preserves_whitespace_in_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Only trailing CR/LF is stripped — internal whitespace is preserved.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target_dir = repo_dir / "dir with spaces" + target_dir.mkdir() + target = target_dir / "real.yaml" + target.write_text("hello") + + stub = repo_dir / "link.yaml" + # Trailing newline (as git's checkout may append) is stripped, but + # whitespace inside the target path itself must survive. + stub.write_bytes(b"dir with spaces/real.yaml\n") + + mock_run_git_command.return_value = "120000 abc123 0\tlink.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_returns_none_for_directory_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing at a directory has no file content to load.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "dir_target").mkdir() + + stub = repo_dir / "link_to_dir" + stub.write_text("dir_target") + + mock_run_git_command.return_value = "120000 abc123 0\tlink_to_dir" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_resolve_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Path.resolve() raising (e.g. on a malformed target) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "broken.yaml" + stub.write_text("ignored") + + mock_run_git_command.return_value = "120000 abc123 0\tbroken.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "resolve", side_effect=OSError("bad path")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_file_missing( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that doesn't exist is rejected before git is consulted.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + missing = repo_dir / "ghost.yaml" # not created + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, missing) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_path_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that isn't under repo_dir is rejected (ValueError from relative_to).""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + outside = tmp_path / "stray.yaml" + outside.write_text("something") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, outside) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_untracked( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Empty `git ls-files` output (untracked file) makes the helper return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "untracked.yaml" + stub.write_text("static/foo.yaml") + + mock_run_git_command.return_value = "" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_read_bytes_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """An OSError from read_bytes() (e.g. file vanished mid-call) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "racy.yaml" + stub.write_text("static/racy.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tracy.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "read_bytes", side_effect=OSError("vanished")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index c71be2fbab..b5816f742e 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -837,3 +837,86 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None: assert isinstance(result["value"], Lambda) assert result["value"].value == 'return "bar";' + + +@patch("esphome.git.resolve_symlink_stub") +@patch("esphome.git.clone_or_update") +def test_remote_package_symlink_stub_is_followed( + mock_clone_or_update: MagicMock, + mock_resolve_symlink_stub: MagicMock, + tmp_path: Path, +) -> None: + """When a package YAML is a scalar (symlink stub) and resolve_symlink_stub + returns a target, the loader follows the target and uses its content.""" + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + # Stub file: content is the target path string (simulating Windows behavior). + stub = repo_dir / "file1.yaml" + stub.write_text("static/file1.yaml") + + # Real target with valid YAML mapping. + target = repo_dir / "static" / "file1.yaml" + target.write_text("substitutions:\n hello: world\n") + + mock_clone_or_update.return_value = (repo_dir, None) + mock_resolve_symlink_stub.return_value = target + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + # Must succeed (does not raise the helpful cv.Invalid) because the stub + # was followed and a valid mapping was loaded from the target. + do_packages_pass(config) + assert mock_resolve_symlink_stub.called + + +@patch("esphome.git.clone_or_update") +def test_remote_package_scalar_yaml_raises_helpful_error( + mock_clone_or_update: MagicMock, tmp_path: Path +) -> None: + """A remote package YAML that is a top-level scalar (e.g. an unmaterialized + git symlink on Windows) raises a clear cv.Invalid, not AttributeError. + + Regression test for the case where a repo containing a YAML symlink, + checked out on Windows without symlink privilege, lands as a short text + file containing the symlink target path. PyYAML parses that as a bare + string scalar; the package loader must reject it with a human-readable + error instead of dying inside ``.get()``. + """ + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + # Simulate the broken-symlink state: a YAML file whose entire content is + # the symlink target string. PyYAML parses this as a top-level scalar. + (repo_dir / "file1.yaml").write_text("static/file1.yaml") + + mock_clone_or_update.return_value = (repo_dir, None) + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + with pytest.raises(cv.Invalid) as exc_info: + do_packages_pass(config) + + msg = str(exc_info.value) + assert "mapping at the top level" in msg + assert "file1.yaml" in msg From 8b62cfded7aec52d5d74b7f12751502b9ad2a059 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 02:57:44 -0500 Subject: [PATCH 68/96] [libretiny] Fix RTL8710B IRAM_ATTR section being dropped from flashed image (#16616) --- esphome/components/libretiny/hal.h | 24 +++++---- .../libretiny/patch_linker.py.script | 54 +++++++++++++++---- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/esphome/components/libretiny/hal.h b/esphome/components/libretiny/hal.h index 9c512504b7..01a7b5450b 100644 --- a/esphome/components/libretiny/hal.h +++ b/esphome/components/libretiny/hal.h @@ -11,11 +11,19 @@ #include "esphome/core/time_64.h" // IRAM_ATTR places a function in executable RAM so it is callable from an -// ISR even while flash is busy (XIP stall, OTA, logger flash write). -// Each family uses a section its stock linker already routes to RAM: -// RTL8710B → .image2.ram.text, RTL8720C → .sram.text. LN882H is the -// exception: its stock linker has no matching glob, so patch_linker.py -// injects KEEP(*(.sram.text*)) into .flash_copysection at pre-link. +// ISR even while flash is busy (XIP stall, OTA, logger flash write). All +// LibreTiny families that need it share the same .sram.text input section +// name; how that section is routed into RAM differs per family: +// RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. +// RTL8710B: patch_linker.py.script injects KEEP(*(.sram.text*)) at the +// top of .ram_image2.data (which IS in ltchiptool's +// sections_ram). The stock linker has KEEP(*(.image2.ram.text*)) +// in .ram_image2.text but that output section is NOT in +// ltchiptool's AmebaZ elf2bin sections_ram list, so code routed +// there is dropped from the flashed binary. +// LN882H: patch_linker.py.script injects KEEP(*(.sram.text*)) into +// .flash_copysection (> RAM0 AT> FLASH), after KEEP(*(.vectors)) +// so the Cortex-M4 vector table stays 512-byte-aligned for VTOR. // // BK72xx (all variants) are left as a no-op: their SDK wraps flash // operations in GLOBAL_INT_DISABLE() which masks FIQ + IRQ at the CPU for @@ -26,13 +34,7 @@ // layer. #if defined(USE_BK72XX) #define IRAM_ATTR -#elif defined(USE_LIBRETINY_VARIANT_RTL8710B) -// Stock linker consumes *(.image2.ram.text*) into .ram_image2.text (> BD_RAM). -#define IRAM_ATTR __attribute__((noinline, section(".image2.ram.text"))) #else -// RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. -// LN882H: patch_linker.py.script injects *(.sram.text*) into -// .flash_copysection (> RAM0 AT> FLASH). #define IRAM_ATTR __attribute__((noinline, section(".sram.text"))) #endif #define PROGMEM diff --git a/esphome/components/libretiny/patch_linker.py.script b/esphome/components/libretiny/patch_linker.py.script index 3a8a4787ed..dfeaaa57d1 100644 --- a/esphome/components/libretiny/patch_linker.py.script +++ b/esphome/components/libretiny/patch_linker.py.script @@ -6,12 +6,18 @@ import re import subprocess # ESPHome marks ISR code IRAM_ATTR, which on LibreTiny maps to a per-family -# section routed into RAM-executable memory (see esphome/core/hal.h). +# section routed into RAM-executable memory (see esphome/core/hal.h). The +# input section name is always .sram.text; only the output section it lands +# in differs per family. # # This script is NOT loaded on BK72xx (IRAM_ATTR is a no-op there; the SDK # masks FIQ+IRQ around flash writes). On the remaining families: -# - RTL8710B: hal.h uses section(".image2.ram.text"); stock linker consumes it. -# - RTL8720C: hal.h uses section(".sram.text"); stock linker consumes it. +# - RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. +# - RTL8710B: stock linker has KEEP(*(.image2.ram.text*)) in .ram_image2.text, +# but ltchiptool's AmebaZ elf2bin (soc/ambz/binary.py) does NOT list +# .ram_image2.text in sections_ram, so code there is silently dropped from +# the flashed image. Inject KEEP(*(.sram.text*)) at the top of +# .ram_image2.data (which IS extracted) instead. # - LN882H: stock linker has no glob for ".sram.text", so we inject # KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH) # immediately after KEEP(*(.vectors)), so the vector table stays at @@ -34,6 +40,20 @@ _KEEP_LINE = ( # aligned address; injecting before the vectors would push them to an # unaligned offset and mis-route every IRQ handler. _LN_COPY = re.compile(r"(KEEP\(\*\(\.vectors\)\)[^\n]*\n)") +# Inject at the top of .ram_image2.data, before __data_start__ so our code +# does not fall inside the data range markers. .ram_image2.data is one of the +# sections ltchiptool's AmebaZ elf2bin extracts; BD_RAM is rwx so the code is +# executable. AmbZ has no C runtime .data copy loop (the bootloader loads +# image2 into BD_RAM whole) so the inline code is not clobbered after boot. +# +# The regex is intentionally strict (no attribute / ALIGN between the section +# name and the opening brace, brace on its own line). If a future AmbZ SDK +# linker template changes this format, _pre_link raises RuntimeError on the +# unpatched .ld file(s), and the RTL8710B CI compile job in +# tests/test_build_components fails on the PR, surfacing the mismatch loudly +# rather than silently shipping a binary with IRAM_ATTR code dropped from +# one or both OTA slots. +_AMBZ_DATA = re.compile(r"(\.ram_image2\.data\s*:\s*\n?\s*\{\s*\n)") def _detect(env): @@ -71,12 +91,11 @@ def _inject_keep(host_section): # Variants not listed here intentionally have no .ld patcher: -# - RTL8710B: hal.h uses section(".image2.ram.text") which the stock linker -# already routes into .ram_image2.text (> BD_RAM). -# - RTL8720C: stock linker already consumes *(.sram.text*). +# - RTL8720C: stock linker already consumes *(.sram.text*) into .ram.code_text. # - BK72xx (all): SDK masks FIQ+IRQ around flash writes, IRAM_ATTR is no-op. _PATCHERS_BY_VARIANT = { "LN882H": (_inject_keep(_LN_COPY),), + "RTL8710B": (_inject_keep(_AMBZ_DATA),), } @@ -87,13 +106,14 @@ def _patchers_for(variant): def _pre_link(target, source, env): build_dir = env.subst("$BUILD_DIR") ld_files = [f for f in os.listdir(build_dir) if f.endswith(".ld")] - patched = 0 + patched = [] + unpatched = [] for name in ld_files: path = os.path.join(build_dir, name) with open(path, "r", encoding="utf-8") as fh: original = fh.read() if _MARKER in original: - patched += 1 + patched.append(name) continue content = original for fn in _patchers: @@ -102,7 +122,9 @@ def _pre_link(target, source, env): with open(path, "w", encoding="utf-8") as fh: fh.write(content) print("ESPHome: patched {} for IRAM_ATTR placement".format(name)) - patched += 1 + patched.append(name) + else: + unpatched.append(name) if not patched: raise RuntimeError( "ESPHome: no .ld in {} was patched for IRAM_ATTR. Update the " @@ -110,6 +132,20 @@ def _pre_link(target, source, env): build_dir ) ) + # Every .ld in the build must be patched. RTL8710B generates one .ld per + # OTA slot (xip1, xip2); if only one matches, the unpatched slot would + # ship with IRAM_ATTR code dropped to zeros and brick the device on the + # boot after an OTA into that slot. + if unpatched: + raise RuntimeError( + "ESPHome: {} of {} .ld file(s) in {} were not patched for " + "IRAM_ATTR: {}. The regex in patch_linker.py.script " + "(_PATCHERS_BY_VARIANT[{!r}]) matched the others but not " + "these. Update the regex to cover all linker scripts.".format( + len(unpatched), len(ld_files), build_dir, + ", ".join(unpatched), _variant, + ) + ) # Substrings matched against demangled names as a fallback on RTL8720C, From ceb9d406e172919020c174e7931d69916a0ceeea Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 06:46:44 -0500 Subject: [PATCH 69/96] [core] Enable ruff PIE (flake8-pie) lint family (#16658) --- esphome/components/as5600/__init__.py | 2 +- esphome/components/audio_file/__init__.py | 2 +- esphome/components/font/__init__.py | 2 +- esphome/components/http_request/__init__.py | 2 +- esphome/components/image/__init__.py | 2 +- esphome/components/packages/__init__.py | 2 +- esphome/components/time/__init__.py | 6 +++--- esphome/log.py | 6 ++++-- esphome/upload_targets.py | 2 +- pyproject.toml | 1 + script/api_protobuf/api_protobuf.py | 7 +------ script/determine-jobs.py | 10 +++------- script/helpers.py | 6 ++---- tests/integration/test_gpio_expander_cache.py | 4 ++-- tests/unit_tests/components/test_time.py | 4 ++-- 15 files changed, 25 insertions(+), 33 deletions(-) diff --git a/esphome/components/as5600/__init__.py b/esphome/components/as5600/__init__.py index 444306cec3..c05e556376 100644 --- a/esphome/components/as5600/__init__.py +++ b/esphome/components/as5600/__init__.py @@ -100,7 +100,7 @@ def position(min=-MAX_POSITION, max=MAX_POSITION): if isinstance(value, str) and value.endswith("%"): value = percent_to_position(value) - if isinstance(value, str) and (value.endswith("°") or value.endswith("deg")): + if isinstance(value, str) and value.endswith(("°", "deg")): return angle_to_position( value, min=round(min * POSITION_TO_ANGLE), diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py index 8dc546cec1..53193c8008 100644 --- a/esphome/components/audio_file/__init__.py +++ b/esphome/components/audio_file/__init__.py @@ -72,7 +72,7 @@ def _file_schema(value: ConfigType | str) -> ConfigType: def _validate_file_shorthand(value: str) -> ConfigType: value = cv.string_strict(value) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return _file_schema( { CONF_TYPE: TYPE_WEB, diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index 4ea6267275..7510f2f8b6 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -401,7 +401,7 @@ def validate_file_shorthand(value): data[CONF_WEIGHT] = weight[1:] return font_file_schema(data) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return font_file_schema( { CONF_TYPE: TYPE_WEB, diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 2617951f0d..fd033dac7f 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -65,7 +65,7 @@ CONF_JSON = "json" def validate_url(value): value = cv.url(value) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return value raise cv.Invalid("URL must start with 'http://' or 'https://'") diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 2fefbdcd58..5f8e5ca132 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -408,7 +408,7 @@ def validate_file_shorthand(value): raise cv.Invalid(f"Could not parse mdi icon name from '{value}'.") return download_gh_svg(parts[1], parts[0]) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return download_image(value) value = cv.file_(value) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index f3e0e0db8f..c1c5bd2ae3 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -112,7 +112,7 @@ def expand_file_to_files(config: dict): def validate_yaml_filename(value): value = cv.string(value) - if not (value.endswith(".yaml") or value.endswith(".yml")): + if not value.endswith((".yaml", ".yml")): raise cv.Invalid("Only YAML (.yaml / .yml) files are supported.") return value diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index f687df26c2..b3bf2d44d7 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -418,11 +418,11 @@ async def setup_time_core_(time_var, config): for conf in config.get(CONF_ON_TIME, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], time_var) - seconds = conf.get(CONF_SECONDS, list(range(0, 61))) + seconds = conf.get(CONF_SECONDS, list(range(61))) cg.add(trigger.add_seconds(seconds)) - minutes = conf.get(CONF_MINUTES, list(range(0, 60))) + minutes = conf.get(CONF_MINUTES, list(range(60))) cg.add(trigger.add_minutes(minutes)) - hours = conf.get(CONF_HOURS, list(range(0, 24))) + hours = conf.get(CONF_HOURS, list(range(24))) cg.add(trigger.add_hours(hours)) days_of_month = conf.get(CONF_DAYS_OF_MONTH, list(range(1, 32))) cg.add(trigger.add_days_of_month(days_of_month)) diff --git a/esphome/log.py b/esphome/log.py index bfd1875b55..b120c930d0 100644 --- a/esphome/log.py +++ b/esphome/log.py @@ -28,10 +28,12 @@ class AnsiFore(Enum): class AnsiStyle(Enum): + # BOLD/BRIGHT and THIN/DIM are intentional ANSI synonyms; Enum treats the + # second name in each pair as an alias of the first. BRIGHT = "\033[1m" - BOLD = "\033[1m" + BOLD = "\033[1m" # noqa: PIE796 DIM = "\033[2m" - THIN = "\033[2m" + THIN = "\033[2m" # noqa: PIE796 NORMAL = "\033[22m" RESET_ALL = "\033[0m" diff --git a/esphome/upload_targets.py b/esphome/upload_targets.py index 302ecf7301..d9d9713fc1 100644 --- a/esphome/upload_targets.py +++ b/esphome/upload_targets.py @@ -57,7 +57,7 @@ def get_port_type(port: str) -> PortType: """ if port == "BOOTSEL": return PortType.BOOTSEL - if port.startswith("/") or port.startswith("COM"): + if port.startswith(("/", "COM")): return PortType.SERIAL if port == "MQTT": return PortType.MQTT diff --git a/pyproject.toml b/pyproject.toml index ae1bd34f60..6572078746 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,7 @@ select = [ "LOG", # flake8-logging "NPY", # numpy-specific rules "PERF", # performance + "PIE", # flake8-pie "PL", # pylint "PTH", # flake8-use-pathlib "PYI", # flake8-pyi diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 240ee7890f..451cd9ac1f 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -84,12 +84,7 @@ def indent_list(text: str, padding: str = " ") -> list[str]: """Indent each line of the given text with the specified padding.""" lines = [] for line in text.splitlines(): - if ( - line == "" - or line.startswith("#ifdef") - or line.startswith("#if ") - or line.startswith("#endif") - ): + if line == "" or line.startswith(("#ifdef", "#if ", "#endif")): p = "" else: p = padding diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 417716cd77..d91936952e 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -483,9 +483,7 @@ def should_run_device_builder(branch: str | None = None) -> bool: True if the device-builder downstream tests should run, False otherwise. """ target_branch = get_target_branch() - if target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") - ): + if target_branch and (target_branch.startswith(("release", "beta"))): return False for file in changed_files(branch): @@ -955,9 +953,7 @@ def detect_memory_impact_config( # all components at once would produce nonsensical memory impact results. # Memory impact analysis is most useful for focused PRs targeting dev. target_branch = get_target_branch() - if target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") - ): + if target_branch and (target_branch.startswith(("release", "beta"))): print( f"Memory impact: Skipping analysis for target branch {target_branch} " f"(would try to build all components at once, giving nonsensical results)", @@ -1311,7 +1307,7 @@ def main() -> None: # (no isolation, all components are groupable) target_branch = get_target_branch() is_release_branch = target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") + target_branch.startswith(("release", "beta")) ) if is_release_branch: diff --git a/script/helpers.py b/script/helpers.py index c56a434edf..9839e766e2 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -103,9 +103,7 @@ def get_component_from_path(file_path: str) -> str | None: Returns: Component name if path is in components or tests directory, None otherwise """ - if file_path.startswith(ESPHOME_COMPONENTS_PATH) or file_path.startswith( - ESPHOME_TESTS_COMPONENTS_PATH - ): + if file_path.startswith((ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH)): parts = file_path.split("/") if len(parts) >= 3 and parts[2]: # Verify that parts[2] is actually a component directory, not a file @@ -160,7 +158,7 @@ def is_validate_only_file(test_file: Path) -> bool: ``esphome config`` only and skipped during compile. """ name = test_file.name - return name.startswith("validate.") or name.startswith("validate-") + return name.startswith(("validate.", "validate-")) @dataclass(frozen=True) diff --git a/tests/integration/test_gpio_expander_cache.py b/tests/integration/test_gpio_expander_cache.py index e5f0f2818f..1d36ca3446 100644 --- a/tests/integration/test_gpio_expander_cache.py +++ b/tests/integration/test_gpio_expander_cache.py @@ -43,7 +43,7 @@ async def test_gpio_expander_cache( # ensure logs are in the expected order log_order = [ (digital_read_hw_pattern, 0), - [(digital_read_cache_pattern, i) for i in range(0, 8)], + [(digital_read_cache_pattern, i) for i in range(8)], (digital_read_hw_pattern, 8), [(digital_read_cache_pattern, i) for i in range(8, 16)], (digital_read_hw_pattern, 16), @@ -68,7 +68,7 @@ async def test_gpio_expander_cache( # uint16_t component tests (single bank of 16 pins) (uint16_read_hw_pattern, 0), # First pin triggers hw read [ - (uint16_read_cache_pattern, i) for i in range(0, 16) + (uint16_read_cache_pattern, i) for i in range(16) ], # All 16 pins return via cache # After cache reset (uint16_read_hw_pattern, 5), # First read after reset triggers hw diff --git a/tests/unit_tests/components/test_time.py b/tests/unit_tests/components/test_time.py index 6325bfbe75..5ae9d787d6 100644 --- a/tests/unit_tests/components/test_time.py +++ b/tests/unit_tests/components/test_time.py @@ -70,11 +70,11 @@ def test_numeric_offset_slash() -> None: def test_star() -> None: - assert _parse_cron_part("*", 0, 59, {}) == set(range(0, 60)) + assert _parse_cron_part("*", 0, 59, {}) == set(range(60)) def test_question() -> None: - assert _parse_cron_part("?", 0, 59, {}) == set(range(0, 60)) + assert _parse_cron_part("?", 0, 59, {}) == set(range(60)) def test_range() -> None: From 88b12a1c457f171ef877b033dcf8f4231e74bf13 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 08:41:54 -0500 Subject: [PATCH 70/96] [lvgl] Build automation_schema event validators lazily (#16633) --- esphome/components/lvgl/schemas.py | 34 ++++++++- .../lvgl/test_automation_schema_lazy.py | 71 +++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/test_automation_schema_lazy.py diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index b901eb4b53..bdaa91f15c 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -22,6 +22,7 @@ from esphome.const import ( ) from esphome.core import TimePeriod from esphome.core.config import StartupTrigger +from esphome.schema_extractors import EnableSchemaExtraction from . import defines as df, lv_validation as lvalid from .defines import ( @@ -407,7 +408,34 @@ def part_schema(parts: tuple[str, ...] | list[str]) -> cv.Schema: return cv.Schema(part_dict(parts)) -def automation_schema(typ: LvType): +def _lazy_validate_automation(extra_schema: dict) -> Callable[[Any], Any]: + """Return a validator that defers building the validate_automation schema. + + validate_automation() runs AUTOMATION_SCHEMA.extend(extra_schema), which + voluptuous compiles eagerly. automation_schema() builds ~60 of these per + widget type, and the vast majority of slots are never invoked by a given + user config. Deferring the build to first use removes that work from + schema-construction time. + + When EnableSchemaExtraction is set (build_language_schema.py), fall back + to eager construction so the @schema_extractor("automation") decoration + inside validate_automation is registered. + """ + if EnableSchemaExtraction: + return validate_automation(extra_schema) + + cached: Callable[[Any], Any] | None = None + + def validator(value: Any) -> Any: + nonlocal cached + if cached is None: + cached = validate_automation(extra_schema) + return cached(value) + + return validator + + +def automation_schema(typ: LvType) -> dict[Any, Any]: events = df.LV_EVENT_TRIGGERS + df.SWIPE_TRIGGERS if typ.has_on_value: events = events + (CONF_ON_VALUE, CONF_ON_UPDATE) @@ -422,7 +450,7 @@ def automation_schema(typ: LvType): return { **{ - cv.Optional(event): validate_automation( + cv.Optional(event): _lazy_validate_automation( { cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id( Trigger.template(*get_trigger_args(event)) @@ -431,7 +459,7 @@ def automation_schema(typ: LvType): ) for event in events }, - cv.Optional(CONF_ON_BOOT): validate_automation( + cv.Optional(CONF_ON_BOOT): _lazy_validate_automation( {cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(StartupTrigger)} ), } diff --git a/tests/component_tests/lvgl/test_automation_schema_lazy.py b/tests/component_tests/lvgl/test_automation_schema_lazy.py new file mode 100644 index 0000000000..46430824f6 --- /dev/null +++ b/tests/component_tests/lvgl/test_automation_schema_lazy.py @@ -0,0 +1,71 @@ +"""Tests for lvgl automation_schema lazy validate_automation build.""" + +from __future__ import annotations + +from unittest.mock import patch + +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import ( + WIDGET_TYPES, + _lazy_validate_automation, + automation_schema, +) +from esphome.components.lvgl.widgets import WidgetType +from esphome.config_validation import GenerateID, declare_id +from esphome.const import CONF_TRIGGER_ID +from esphome.core.config import StartupTrigger + + +def _widget_type(name: str = "obj") -> WidgetType: + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def _trigger_extra_schema() -> dict: + return {GenerateID(CONF_TRIGGER_ID): declare_id(StartupTrigger)} + + +def test_lazy_validator_defers_build_until_first_call() -> None: + with patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock: + validator = _lazy_validate_automation(_trigger_extra_schema()) + assert va_mock.call_count == 0 + validator({"then": []}) + assert va_mock.call_count == 1 + validator({"then": []}) + assert va_mock.call_count == 1 + + +def test_eager_build_when_schema_extraction_enabled() -> None: + with ( + patch("esphome.components.lvgl.schemas.EnableSchemaExtraction", True), + patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock, + ): + _lazy_validate_automation(_trigger_extra_schema()) + assert va_mock.call_count == 1 + + +def test_lazy_and_eager_produce_equivalent_validation() -> None: + extra = _trigger_extra_schema() + with patch("esphome.components.lvgl.schemas.EnableSchemaExtraction", True): + eager = _lazy_validate_automation(extra) + lazy = _lazy_validate_automation(_trigger_extra_schema()) + sample = {"then": []} + assert lazy(sample) == eager(sample) + + +def test_automation_schema_uses_lazy_validators() -> None: + wt = _widget_type("obj") + with patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock: + automation_schema(wt.w_type) + assert va_mock.call_count == 0 From 722cbfe843cad4aa729ef848016261fdb4bb884d Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 26 May 2026 14:05:57 -0400 Subject: [PATCH 71/96] [voice_assistant] Never send zero-length audio to Home Assistant (#16634) --- .../voice_assistant/voice_assistant.cpp | 136 ++++++++++++------ .../voice_assistant/voice_assistant.h | 13 ++ 2 files changed, 107 insertions(+), 42 deletions(-) diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index af1b98da02..f13ea39fa2 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -4,6 +4,7 @@ #ifdef USE_VOICE_ASSISTANT #include "esphome/components/socket/socket.h" +#include "esphome/core/application.h" #include "esphome/core/log.h" #include @@ -26,6 +27,11 @@ static const size_t SEND_BUFFER_SIZE = SEND_BUFFER_SAMPLES * sizeof(int16_t); static const size_t RECEIVE_SIZE = 1024; static const size_t SPEAKER_BUFFER_SIZE = 16 * RECEIVE_SIZE; +// If one microphone channel keeps producing audio while another configured channel produces none for this +// long, treat the silent channel as failed and stop the stream. A working microphone exposes a chunk every +// SEND_BUFFER_SAMPLES (32 ms), so this is far longer than any legitimate gap between chunks. +static const uint32_t AUDIO_CHANNEL_STALL_TIMEOUT_MS = 2000; + VoiceAssistant::VoiceAssistant() { global_voice_assistant = this; } void VoiceAssistant::setup() { @@ -168,6 +174,9 @@ void VoiceAssistant::clear_buffers_() { this->audio_source2_->clear_buffered_data(); } + // Reset the multi-channel stall watchdog (see audio_channel_stall_start_). + this->audio_channel_stall_start_ = 0; + #ifdef USE_SPEAKER if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) { memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE); @@ -200,6 +209,79 @@ void VoiceAssistant::reset_conversation_id() { ESP_LOGD(TAG, "reset conversation ID"); } +void VoiceAssistant::stream_api_audio_() { + // Both microphone channels are sent together, if configured. Home Assistant feeds one of the + // channels to its speech-to-text stream and treats an empty payload on that channel as + // end-of-stream, and the device cannot know which channel it picked, so only send once every + // configured channel has audio exposed, and always send them together. We don't target any + // particular message size: Home Assistant re-chunks the audio, and each fill() exposes at most + // SEND_BUFFER_SIZE bytes. + while (true) { + // fill() exposes a new chunk, or returns 0 if a previous chunk is still exposed; available() + // reports the currently exposed bytes either way. + this->audio_source_->fill(0, false); + size_t available = this->audio_source_->available(); + size_t available2 = 0; + if (this->audio_source2_ != nullptr) { + this->audio_source2_->fill(0, false); + available2 = this->audio_source2_->available(); + } + + const bool channel_empty = (available == 0); + const bool channel2_empty = (this->audio_source2_ != nullptr) && (available2 == 0); + if (channel_empty || channel2_empty) { + // A configured channel has no audio yet, so keep any chunk exposed on the other channel for the + // next pass rather than sending an empty payload. + this->handle_channel_stall_(available, available2); + break; + } + + // Both channels have audio exposed; clear any in-progress stall timer. + this->audio_channel_stall_start_ = 0; + + api::VoiceAssistantAudio msg; + // Zero-copy: send_message() copies the data out before we consume it. + msg.data = this->audio_source_->data(); + msg.data_len = available; + if (this->audio_source2_ != nullptr) { + msg.data2 = this->audio_source2_->data(); + msg.data2_len = available2; + } + + this->api_client_->send_message(msg); + + this->audio_source_->consume(available); + if (this->audio_source2_ != nullptr) { + this->audio_source2_->consume(available2); + } + } +} + +void VoiceAssistant::handle_channel_stall_(size_t available, size_t available2) { + // Called when at least one configured channel has no audio exposed. When one channel has data and the + // other does not, watch how long the empty channel stays starved: Home Assistant has no stream timeout + // and would never tell us to stop, so a channel that fails outright would otherwise hang streaming + // forever with the live channel's chunk held. Stop the stream with an error after a prolonged imbalance. + if ((available == 0) && (available2 == 0)) { + // Both channels are idle (no audio buffered yet); normal, not a stalled channel. + this->audio_channel_stall_start_ = 0; + return; + } + + const uint32_t now = App.get_loop_component_start_time(); + if (this->audio_channel_stall_start_ == 0) { + this->audio_channel_stall_start_ = now; + } else if ((now - this->audio_channel_stall_start_) >= AUDIO_CHANNEL_STALL_TIMEOUT_MS) { + ESP_LOGW(TAG, "Mic channel %d stalled, stopping stream", (available == 0) ? 0 : 1); + this->audio_channel_stall_start_ = 0; + this->signal_stop_(); + this->set_state_(State::STOP_MICROPHONE, State::IDLE); + this->defer([this]() { + this->error_trigger_.trigger("mic-channel-stalled", "A microphone channel stopped producing audio"); + }); + } +} + void VoiceAssistant::loop() { if (this->api_client_ == nullptr && this->state_ != State::IDLE && this->state_ != State::STOP_MICROPHONE && this->state_ != State::STOPPING_MICROPHONE) { @@ -292,55 +374,25 @@ void VoiceAssistant::loop() { case State::STREAMING_MICROPHONE: { // pre_shift is ignored by RingBufferAudioSource (no intermediate transfer buffer to compact). if (this->audio_mode_ == AUDIO_MODE_API) { - // API audio - // Both microphone channels are sent, if configured - size_t available = this->audio_source_->fill(0, false); - size_t available2 = 0; - if (this->audio_source2_ != nullptr) { - available2 = this->audio_source2_->fill(0, false); - } - - while (available > 0 || available2 > 0) { - api::VoiceAssistantAudio msg; - - if (available > 0) { - // Zero-copy: send_message() copies the data out before we consume it - msg.data = this->audio_source_->data(); - msg.data_len = available; - } - - // Second microphone channel - if (available2 > 0) { - msg.data2 = this->audio_source2_->data(); - msg.data2_len = available2; - } - - this->api_client_->send_message(msg); - - if (available > 0) { - this->audio_source_->consume(available); - } - available = this->audio_source_->fill(0, false); - if (available2 > 0) { - this->audio_source2_->consume(available2); - } - if (this->audio_source2_ != nullptr) { - available2 = this->audio_source2_->fill(0, false); - } - } + this->stream_api_audio_(); } else { // UDP (will eventually be deprecated) // Only the primary microphone channel is used - while (this->audio_source_->fill(0, false) > 0) { + while (true) { + this->audio_source_->fill(0, false); + size_t available = this->audio_source_->available(); + if (available == 0) { + break; + } if (!this->udp_socket_running_) { if (!this->start_udp_socket_()) { this->set_state_(State::STOP_MICROPHONE, State::IDLE); break; } } - this->socket_->sendto(this->audio_source_->data(), this->audio_source_->available(), 0, - (struct sockaddr *) &this->dest_addr_, sizeof(this->dest_addr_)); - this->audio_source_->consume(this->audio_source_->available()); + this->socket_->sendto(this->audio_source_->data(), available, 0, (struct sockaddr *) &this->dest_addr_, + sizeof(this->dest_addr_)); + this->audio_source_->consume(available); } } // audio mode break; @@ -841,8 +893,8 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { }); State new_state = this->local_output_ ? State::STREAMING_RESPONSE : State::IDLE; if (new_state != this->state_) { - // Don't needlessly change the state. The intent progress stage may have already changed the state to streaming - // response. + // Don't needlessly change the state. The intent progress stage may have already changed the state to + // streaming response. this->set_state_(new_state, new_state); } break; diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index f3ea669e15..76b076a366 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -244,6 +244,12 @@ class VoiceAssistant : public Component { void signal_stop_(); void start_playback_timeout_(); + // Drains the exposed microphone audio and sends it to Home Assistant over the API in one loop() pass. + void stream_api_audio_(); + // Handles a pass where at least one configured channel has no audio exposed, timing out a channel that + // stalls. See audio_channel_stall_start_. + void handle_channel_stall_(size_t available, size_t available2); + std::unique_ptr socket_ = nullptr; struct sockaddr_storage dest_addr_; @@ -315,6 +321,13 @@ class VoiceAssistant : public Component { std::weak_ptr ring_buffer_; std::weak_ptr ring_buffer2_; + // When streaming multiple channels, the send loop holds an exposed chunk on one channel until the other + // channel also has audio so the channels are always sent together (an empty payload looks like + // end-of-stream to Home Assistant). Home Assistant has no stream timeout, so a channel that stops + // producing entirely would hang streaming forever. This records when such an imbalance began so a + // prolonged one can be detected and stopped; 0 means no imbalance is currently being timed. + uint32_t audio_channel_stall_start_{0}; + bool use_wake_word_; uint8_t noise_suppression_level_; uint8_t auto_gain_; From bac62cb7dec73d5e3311d16d607fb1485e4e9584 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 15:29:06 -0500 Subject: [PATCH 72/96] [core] Add cv.sensitive marker for schema-level sensitive fields (#16673) --- esphome/config_validation.py | 47 +++++++++++++++ script/build_language_schema.py | 39 +++++++++++- tests/script/test_build_language_schema.py | 69 +++++++++++++++++++++- tests/unit_tests/test_config_validation.py | 43 ++++++++++++++ 4 files changed, 193 insertions(+), 5 deletions(-) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index ca1fd8f5d4..1d5e27c9ae 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -487,6 +487,53 @@ def string_strict(value): ) +# Substring fallbacks for fields whose validator isn't explicitly wrapped in +# ``cv.sensitive``. Frontends and dump tooling should prefer the explicit +# marker; this list exists so we still mask obvious leaks in unmigrated or +# third-party schemas. Kept here as the single source of truth. +SENSITIVE_KEY_FRAGMENTS: frozenset[str] = frozenset( + { + "password", + "passcode", + "secret", + "token", + "api_key", + "apikey", + "psk", + } +) + + +class SensitiveValidator: + """Marker wrapper that flags a field as containing sensitive data (passwords, + encryption keys, PSKs, tokens). Frontends and dump tooling detect this marker + to mask the value; validation behavior is delegated to the inner validator. + """ + + def __init__(self, inner: Callable[[typing.Any], typing.Any]) -> None: + self.inner = inner + + def __call__(self, value: typing.Any) -> typing.Any: + return self.inner(value) + + def __repr__(self) -> str: + # Mirror the inner validator's repr so ``build_language_schema``'s + # ``known_schemas``/``extended_schemas`` dedup (keyed on ``repr(schema)``) + # treats two wrappers around the same inner as identical, and so + # voluptuous error messages stay readable. + return repr(self.inner) + + +def sensitive( + inner: Callable[[typing.Any], typing.Any] = string, +) -> SensitiveValidator: + """Mark a field as sensitive so that frontends mask it and dump tooling redacts it. + + Validation behavior is identical to ``inner`` (defaults to ``cv.string``). + """ + return SensitiveValidator(inner) + + def icon(value): """Validate that a given config value is a valid icon.""" from esphome.core.config import ICON_MAX_LENGTH diff --git a/script/build_language_schema.py b/script/build_language_schema.py index 9dff70af3c..6e4000e06e 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -39,7 +39,11 @@ parser.add_argument( ) parser.add_argument("--check", action="store_true", help="Check only for CI") -args = parser.parse_args() +# Module-level ``Namespace`` so helper functions can reference ``args`` +# without threading it through every call. ``main()`` fills it via +# ``parser.parse_args(namespace=args)``; tests import this module without +# invoking ``main()`` and rely on the defaults below. +args = argparse.Namespace(output_path=".", check=False) DUMP_RAW = False DUMP_UNKNOWN = False @@ -850,6 +854,12 @@ def convert(schema, config_var, path): convert(ext, config_var, f"{path}/ext{idx}") return + if isinstance(schema, cv.SensitiveValidator): + config_var["sensitive"] = True + config_var["sensitive_source"] = "explicit" + convert(schema.inner, config_var, f"{path}/sensitive") + return + if isinstance(schema, cv.All): i = 0 for inner in schema.validators: @@ -1125,6 +1135,25 @@ def convert_keys(converted, schema, path): # Do value convert(v, result, path + f"/{str(k)}") + + # Heuristic fallback when the field's validator wasn't explicitly + # wrapped in ``cv.sensitive``. Only applies to string-typed leaves so + # we don't mark unrelated nested schemas. ``sensitive_source`` lets + # consumers distinguish explicit markers from heuristic matches. Pull + # the field name from ``k.schema`` (voluptuous's stored key) rather + # than ``str(k)`` so we don't depend on the marker's ``__str__`` + # representation. + if ( + "sensitive" not in result + and result.get(S_TYPE) == "string" + and isinstance(k, (cv.Required, cv.Optional, cv.Inclusive, cv.Exclusive)) + and isinstance(k.schema, str) + ): + key_lower = k.schema.lower() + if any(frag in key_lower for frag in cv.SENSITIVE_KEY_FRAGMENTS): + result["sensitive"] = True + result["sensitive_source"] = "heuristic" + if "schema" not in converted: converted[S_TYPE] = "schema" converted["schema"] = {S_CONFIG_VARS: {}} @@ -1142,4 +1171,10 @@ def convert_keys(converted, schema, path): config_vars["string"] = config_vars.pop(key) -build_schema() +def main() -> None: + parser.parse_args(namespace=args) + build_schema() + + +if __name__ == "__main__": + main() diff --git a/tests/script/test_build_language_schema.py b/tests/script/test_build_language_schema.py index 59b8c7484b..dd1d88e74c 100644 --- a/tests/script/test_build_language_schema.py +++ b/tests/script/test_build_language_schema.py @@ -3,8 +3,11 @@ from __future__ import annotations import ast +import importlib.util from pathlib import Path +from esphome import config_validation as cv + SCRIPT_PATH = ( Path(__file__).resolve().parent.parent.parent / "script" @@ -12,10 +15,16 @@ SCRIPT_PATH = ( ) +def _load_script_module(): + spec = importlib.util.spec_from_file_location("build_language_schema", SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _extract_sort_obj(): - # build_language_schema.py runs argparse, loads every component, and - # calls build_schema() at import time, so a plain import isn't viable - # in a unit test. Pull just the pure helper out via AST instead. + # ``sort_obj`` is pure and self-contained; pulling it via AST avoids + # exercising the module-level component-loading state for these tests. tree = ast.parse(SCRIPT_PATH.read_text()) for node in tree.body: if isinstance(node, ast.FunctionDef) and node.name == "sort_obj": @@ -27,6 +36,7 @@ def _extract_sort_obj(): sort_obj = _extract_sort_obj() +_bls = _load_script_module() def test_sort_obj_sorts_dict_keys() -> None: @@ -96,3 +106,56 @@ def test_sort_obj_passes_through_scalars() -> None: assert sort_obj(42) == 42 assert sort_obj(None) is None assert sort_obj(True) is True + + +def test_convert_emits_explicit_sensitive_marker() -> None: + config_var: dict = {} + _bls.convert(cv.sensitive(cv.string), config_var, "/test") + + assert config_var["sensitive"] is True + assert config_var["sensitive_source"] == "explicit" + assert config_var["type"] == "string" + + +def test_convert_keys_emits_heuristic_sensitive_marker() -> None: + converted: dict = {} + _bls.convert_keys(converted, {cv.Optional("password"): cv.string}, "/root") + + entry = converted["schema"]["config_vars"]["password"] + assert entry["sensitive"] is True + assert entry["sensitive_source"] == "heuristic" + assert entry["type"] == "string" + + +def test_convert_keys_explicit_beats_heuristic() -> None: + # Key name matches a fragment but the validator is explicitly wrapped; + # the explicit branch should win and emit ``sensitive_source: explicit``. + converted: dict = {} + _bls.convert_keys( + converted, {cv.Optional("password"): cv.sensitive(cv.string)}, "/root" + ) + + entry = converted["schema"]["config_vars"]["password"] + assert entry["sensitive"] is True + assert entry["sensitive_source"] == "explicit" + + +def test_convert_keys_no_heuristic_for_non_string_leaves() -> None: + # Even though the key contains a fragment, a non-string leaf must not + # be flagged. Prevents false positives on unrelated fields whose name + # happens to embed a substring like "token". + converted: dict = {} + _bls.convert_keys(converted, {cv.Optional("password"): cv.boolean}, "/root") + + entry = converted["schema"]["config_vars"]["password"] + assert "sensitive" not in entry + assert "sensitive_source" not in entry + + +def test_convert_keys_no_marker_for_non_sensitive_field() -> None: + converted: dict = {} + _bls.convert_keys(converted, {cv.Optional("hostname"): cv.string}, "/root") + + entry = converted["schema"]["config_vars"]["hostname"] + assert "sensitive" not in entry + assert "sensitive_source" not in entry diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index fd6c0e95f2..2c34cbfb07 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -127,6 +127,49 @@ def test_string_string__invalid(value): config_validation.string_strict(value) +def test_sensitive__default_delegates_to_string() -> None: + validator = config_validation.sensitive() + + assert isinstance(validator, config_validation.SensitiveValidator) + assert validator.inner is config_validation.string + assert validator("hunter2") == "hunter2" + assert validator(42) == "42" + + +def test_sensitive__custom_inner_delegates_validation() -> None: + validator = config_validation.sensitive(config_validation.string_strict) + + assert validator.inner is config_validation.string_strict + assert validator("abc") == "abc" + with pytest.raises(Invalid, match="Must be string, got"): + validator(123) + + +def test_sensitive__is_detectable_via_isinstance() -> None: + validator = config_validation.sensitive() + + assert isinstance(validator, config_validation.SensitiveValidator) + + +def test_sensitive__repr_mirrors_inner() -> None: + # The schema dump dedups on ``repr(schema)``; mirroring the inner + # validator's repr keeps two ``cv.sensitive(cv.string)`` wrappers + # interchangeable for that purpose and avoids leaking the wrapper as + # noise in voluptuous error messages. + assert repr(config_validation.sensitive(config_validation.string)) == repr( + config_validation.string + ) + assert repr(config_validation.sensitive(config_validation.string)) == repr( + config_validation.sensitive(config_validation.string) + ) + + +def test_sensitive_key_fragments__covers_common_terms() -> None: + assert isinstance(config_validation.SENSITIVE_KEY_FRAGMENTS, frozenset) + for term in ("password", "passcode", "secret", "token", "api_key", "apikey", "psk"): + assert term in config_validation.SENSITIVE_KEY_FRAGMENTS + + @given( builds( lambda v: "mdi:" + v, From 96816e24916192a36daf7119b8fca81836067476 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 15:29:38 -0500 Subject: [PATCH 73/96] [core] Enable ruff DTZ (flake8-datetimez) lint family (#16660) --- esphome/__main__.py | 2 +- esphome/components/api/client.py | 2 +- esphome/components/zigbee/zigbee_zephyr.py | 5 ++++- esphome/config_validation.py | 4 +++- esphome/external_files.py | 9 ++++++--- esphome/git.py | 8 ++++---- esphome/mqtt.py | 6 +++--- esphome/storage_json.py | 5 ++++- pyproject.toml | 1 + tests/unit_tests/test_external_files.py | 4 ++-- tests/unit_tests/test_git.py | 18 +++++++++--------- tests/unit_tests/test_storage_json.py | 4 ++-- 12 files changed, 40 insertions(+), 28 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 5f281ce832..dd97c6eee9 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -639,7 +639,7 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: chunk = ser.read(ser.in_waiting or 1) if not chunk: continue - time_ = datetime.now() + time_ = datetime.now().astimezone() milliseconds = time_.microsecond // 1000 time_str = f"[{time_.hour:02}:{time_.minute:02}:{time_.second:02}.{milliseconds:03}]" diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index 327973a605..44edc035f9 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -119,7 +119,7 @@ async def async_run_logs( def on_log(msg: SubscribeLogsResponse) -> None: """Handle a new log message.""" - time_ = datetime.now() + time_ = datetime.now().astimezone() message: bytes = msg.message text = message.decode("utf8", "backslashreplace") nanoseconds = time_.microsecond // 1000 diff --git a/esphome/components/zigbee/zigbee_zephyr.py b/esphome/components/zigbee/zigbee_zephyr.py index aa16bbef53..39ecadfddf 100644 --- a/esphome/components/zigbee/zigbee_zephyr.py +++ b/esphome/components/zigbee/zigbee_zephyr.py @@ -161,7 +161,10 @@ async def _attr_to_code(config: ConfigType) -> None: zigbee_set_string(basic_attrs.mf_name, "esphome"), zigbee_set_string(basic_attrs.model_id, config[CONF_MODEL]), zigbee_set_string( - basic_attrs.date_code, datetime.datetime.now().strftime("%Y%m%d %H%M%S") + basic_attrs.date_code, + # Local build time, matching the esp32 implementation + # (App.get_build_time() in C++). + datetime.datetime.now().astimezone().strftime("%Y%m%d %H%M%S"), ), zigbee_assign( basic_attrs.power_source, diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 1d5e27c9ae..f826b254ac 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1183,7 +1183,9 @@ def date_time(date: bool, time: bool): format += "%p" try: - date_obj = datetime.strptime(value, format) + # The generated format never includes %z/%Z, so this parses a + # naive wall-clock date/time by design. + date_obj = datetime.strptime(value, format) # noqa: DTZ007 except ValueError as err: raise Invalid(f"Invalid {exc_message}: {err}") from err diff --git a/esphome/external_files.py b/esphome/external_files.py index dfabc54f47..4e73c8dc21 100644 --- a/esphome/external_files.py +++ b/esphome/external_files.py @@ -7,6 +7,7 @@ from datetime import UTC, datetime import logging import os from pathlib import Path +import time import requests @@ -141,9 +142,11 @@ def has_remote_file_changed( def is_file_recent(file_path: Path, refresh: TimePeriodSeconds) -> bool: if file_path.exists(): - creation_time = file_path.stat().st_ctime - current_time = datetime.now().timestamp() - return current_time - creation_time <= refresh.total_seconds + # st_mtime, not st_ctime: ctime is inode-change time on POSIX + # (bumped by chmod/chown/rename) so a metadata touch would make + # the file look fresh. + modification_time = file_path.stat().st_mtime + return time.time() - modification_time <= refresh.total_seconds return False diff --git a/esphome/git.py b/esphome/git.py index 094a6dae19..744ce35ef6 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -1,12 +1,12 @@ from collections.abc import Callable from dataclasses import dataclass -from datetime import datetime import hashlib import logging from pathlib import Path import re import subprocess import sys +import time import urllib.parse import esphome.config_validation as cv @@ -247,11 +247,11 @@ def clone_or_update( return repo_dir, None file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD") - # On first clone, FETCH_HEAD does not exists + # On first clone, FETCH_HEAD does not exist if not file_timestamp.exists(): file_timestamp = Path(repo_dir / ".git" / "HEAD") - age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime) - if refresh is None or age.total_seconds() > refresh.total_seconds: + age_seconds = time.time() - file_timestamp.stat().st_mtime + if refresh is None or age_seconds > refresh.total_seconds: # Try to update the repository, recovering from broken state if needed old_sha: str | None = None try: diff --git a/esphome/mqtt.py b/esphome/mqtt.py index d6bde0cbfd..c6a7a7558b 100644 --- a/esphome/mqtt.py +++ b/esphome/mqtt.py @@ -139,7 +139,7 @@ def show_discover(config, username=None, password=None, client_id=None): _LOGGER.info("Starting log output from %s", topic) def on_message(client, userdata, msg): - time_ = datetime.now().time().strftime("[%H:%M:%S]") + time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") if len(payload) > 0: message = time_ + " " + payload @@ -184,7 +184,7 @@ def get_esphome_device_ip( def on_message(client, userdata, msg): nonlocal dev_ip - time_ = datetime.now().time().strftime("[%H:%M:%S]") + time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") if len(payload) > 0: message = time_ + " " + payload @@ -253,7 +253,7 @@ def show_logs(config, topic=None, username=None, password=None, client_id=None): _LOGGER.info("Starting log output from %s", topic) def on_message(client, userdata, msg): - time_ = datetime.now().time().strftime("[%H:%M:%S]") + time_ = datetime.now().astimezone().time().strftime("[%H:%M:%S]") payload = msg.payload.decode(errors="backslashreplace") message = time_ + payload safe_print(message) diff --git a/esphome/storage_json.py b/esphome/storage_json.py index 7f8885ba5f..04f5881465 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -338,7 +338,10 @@ class EsphomeStorageJSON: @property def last_update_check(self) -> datetime | None: try: - return datetime.strptime(self.last_update_check_str, "%Y-%m-%dT%H:%M:%S") + # Stored format is naive ISO without %z; preserved for backward compat. + return datetime.strptime( # noqa: DTZ007 + self.last_update_check_str, "%Y-%m-%dT%H:%M:%S" + ) except Exception: # pylint: disable=broad-except return None diff --git a/pyproject.toml b/pyproject.toml index 6572078746..d92c7ba894 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,7 @@ exclude = ['generated'] select = [ "B", # flake8-bugbear "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez "E", # pycodestyle "EXE", # flake8-executable "F", # pyflakes/autoflake diff --git a/tests/unit_tests/test_external_files.py b/tests/unit_tests/test_external_files.py index 64ef149581..16cee9564f 100644 --- a/tests/unit_tests/test_external_files.py +++ b/tests/unit_tests/test_external_files.py @@ -120,7 +120,7 @@ def test_is_file_recent_with_old_file(setup_core: Path) -> None: old_time = time.time() - 7200 mock_stat = MagicMock() - mock_stat.st_ctime = old_time + mock_stat.st_mtime = old_time with patch.object(Path, "stat", return_value=mock_stat): refresh = TimePeriod(seconds=3600) @@ -147,7 +147,7 @@ def test_is_file_recent_with_zero_refresh(setup_core: Path) -> None: # Mock stat to return a time 10 seconds ago mock_stat = MagicMock() - mock_stat.st_ctime = time.time() - 10 + mock_stat.st_mtime = time.time() - 10 with patch.object(Path, "stat", return_value=mock_stat): refresh = TimePeriod(seconds=0) result = external_files.is_file_recent(test_file, refresh) diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index 690c47c183..62d2344069 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -1,8 +1,8 @@ """Tests for git.py module.""" -from datetime import datetime, timedelta import os from pathlib import Path +import time from typing import Any from unittest.mock import Mock, patch @@ -34,9 +34,9 @@ def _setup_old_repo(repo_dir: Path, days_old: int = 2) -> None: # Create FETCH_HEAD file with old timestamp fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - old_time = datetime.now() - timedelta(days=days_old) + old_time = time.time() - days_old * 86400 fetch_head.touch() - os.utime(fetch_head, (old_time.timestamp(), old_time.timestamp())) + os.utime(fetch_head, (old_time, old_time)) def _get_git_command_type(cmd: list[str]) -> str | None: @@ -285,10 +285,10 @@ def test_clone_or_update_with_refresh_updates_old_repo( # Create FETCH_HEAD file with old timestamp (2 days ago) fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - old_time = datetime.now() - timedelta(days=2) + old_time = time.time() - 2 * 86400 fetch_head.touch() # Create the file # Set modification time to 2 days ago - os.utime(fetch_head, (old_time.timestamp(), old_time.timestamp())) + os.utime(fetch_head, (old_time, old_time)) # Mock git command responses mock_run_git_command.return_value = "abc123" # SHA for rev-parse @@ -333,10 +333,10 @@ def test_clone_or_update_with_refresh_skips_fresh_repo( # Create FETCH_HEAD file with recent timestamp (1 hour ago) fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - recent_time = datetime.now() - timedelta(hours=1) + recent_time = time.time() - 3600 fetch_head.touch() # Create the file # Set modification time to 1 hour ago - os.utime(fetch_head, (recent_time.timestamp(), recent_time.timestamp())) + os.utime(fetch_head, (recent_time, recent_time)) # Call with refresh=1d (1 day) refresh = TimePeriodSeconds(days=1) @@ -409,10 +409,10 @@ def test_clone_or_update_with_none_refresh_always_updates( # Create FETCH_HEAD file with very recent timestamp (1 second ago) fetch_head = git_dir / "FETCH_HEAD" fetch_head.write_text("test") - recent_time = datetime.now() - timedelta(seconds=1) + recent_time = time.time() - 1 fetch_head.touch() # Create the file # Set modification time to 1 second ago - os.utime(fetch_head, (recent_time.timestamp(), recent_time.timestamp())) + os.utime(fetch_head, (recent_time, recent_time)) # Mock git command responses mock_run_git_command.return_value = "abc123" # SHA for rev-parse diff --git a/tests/unit_tests/test_storage_json.py b/tests/unit_tests/test_storage_json.py index ea37492cf4..b3f8a05605 100644 --- a/tests/unit_tests/test_storage_json.py +++ b/tests/unit_tests/test_storage_json.py @@ -576,8 +576,8 @@ def test_esphome_storage_json_last_update_check_property() -> None: assert result.hour == 10 assert result.minute == 30 - # Test setter - new_date = datetime(2024, 2, 20, 15, 45, 30) + # Test setter — naive datetime matches the storage round-trip format. + new_date = datetime(2024, 2, 20, 15, 45, 30) # noqa: DTZ001 storage.last_update_check = new_date assert storage.last_update_check_str == "2024-02-20T15:45:30" From 52ead52ef29fd01feabf6c6826b10b5f4fda200c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 15:29:54 -0500 Subject: [PATCH 74/96] [core] Enable ruff PGH (pygrep-hooks) lint family (#16651) --- esphome/components/esp32/__init__.py | 4 ++-- esphome/components/host/__init__.py | 2 +- esphome/components/libretiny/__init__.py | 2 +- .../libretiny/generate_components.py | 18 +++++++++++++----- esphome/components/light/__init__.py | 2 +- esphome/components/logger/__init__.py | 2 +- esphome/components/lvgl/helpers.py | 2 -- esphome/components/nrf52/__init__.py | 2 +- esphome/components/rp2040/__init__.py | 2 +- pyproject.toml | 1 + 10 files changed, 22 insertions(+), 15 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index e3bff8f934..7b94a26f54 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -56,7 +56,7 @@ from esphome.types import ConfigType from esphome.writer import clean_build, clean_cmake_cache from .boards import BOARDS, STANDARD_BOARDS -from .const import ( # noqa +from .const import ( KEY_ARDUINO_LIBRARIES, KEY_BOARD, KEY_COMPONENTS, @@ -86,7 +86,7 @@ from .const import ( # noqa ) # force import gpio to register pin schema -from .gpio import esp32_pin_to_code # noqa +from .gpio import esp32_pin_to_code # noqa: F401 _LOGGER = logging.getLogger(__name__) AUTO_LOAD = ["preferences"] diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index 8adbfb02ec..50deb1acf6 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -14,7 +14,7 @@ from esphome.core import CORE from .const import KEY_HOST # force import gpio to register pin schema -from .gpio import host_pin_to_code # noqa +from .gpio import host_pin_to_code # noqa: F401 CODEOWNERS = ["@esphome/core", "@clydebarrow"] AUTO_LOAD = ["network", "preferences"] diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index d1f1042501..afe0360c22 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -28,7 +28,7 @@ from esphome.core.config import BOARD_MAX_LENGTH from esphome.helpers import copy_file_if_changed from esphome.storage_json import StorageJSON -from . import gpio # noqa +from . import gpio # noqa: F401 from .const import ( COMPONENT_BK72XX, CONF_GPIO_RECOVER, diff --git a/esphome/components/libretiny/generate_components.py b/esphome/components/libretiny/generate_components.py index d5437895a6..6ca16f277f 100644 --- a/esphome/components/libretiny/generate_components.py +++ b/esphome/components/libretiny/generate_components.py @@ -1,7 +1,7 @@ # Copyright (c) Kuba Szczodrzyński 2023-06-01. # pylint: skip-file -# flake8: noqa +# ruff: noqa: C408, I001 import json import re @@ -313,8 +313,12 @@ def write_const( # build component constants comp_str = "\n".join(f'COMPONENT_{f} = "{f.lower()}"' for f in components) # replace the 2nd regex group only - repl = lambda m: m.group(1) + comp_str + m.group(3) - code = re.sub(comp_regex, repl, code, flags=re.DOTALL | re.MULTILINE) + code = re.sub( + comp_regex, + lambda m: m.group(1) + comp_str + m.group(3), + code, + flags=re.DOTALL | re.MULTILINE, + ) # regex for finding the family list block fam_regex = r"(# FAMILIES.+?\n)(.*?)(\n# FAMILIES)" @@ -337,8 +341,12 @@ def write_const( ] var_str = "\n".join(fam_lines) # replace the 2nd regex group only - repl = lambda m: m.group(1) + var_str + m.group(3) - code = re.sub(fam_regex, repl, code, flags=re.DOTALL | re.MULTILINE) + code = re.sub( + fam_regex, + lambda m: m.group(1) + var_str + m.group(3), + code, + flags=re.DOTALL | re.MULTILINE, + ) # format with black code = format_str(code, mode=FileMode()) diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index 68d9f85af2..7c4d7ed431 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -58,7 +58,7 @@ from .effects import ( RGB_EFFECTS, validate_effects, ) -from .types import ( # noqa +from .types import ( # noqa: F401 AddressableLight, AddressableLightState, ColorMode, diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index c6c440564a..5f160352cc 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -514,7 +514,7 @@ def validate_printf(value): (?:hh|h|ll|l|j|z|t|L|w|I|I32|I64)? # size [cCdiouxXeEfgGaAnpsSZ] # type ) - """ # noqa + """ matches = re.findall(cfmt, value[CONF_FORMAT], flags=re.VERBOSE) if len(matches) != len(value[CONF_ARGS]): raise cv.Invalid( diff --git a/esphome/components/lvgl/helpers.py b/esphome/components/lvgl/helpers.py index 6f70a1e3bd..3da8643308 100644 --- a/esphome/components/lvgl/helpers.py +++ b/esphome/components/lvgl/helpers.py @@ -6,7 +6,6 @@ from esphome.const import CONF_ARGS, CONF_FORMAT CONF_IF_NAN = "if_nan" -# noqa f_regex = re.compile( r""" ( # start of capture group 1 @@ -20,7 +19,6 @@ f_regex = re.compile( """, flags=re.VERBOSE, ) -# noqa c_regex = re.compile( r""" ( # start of capture group 1 diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 2aba208af7..4ba1ab5d4d 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -65,7 +65,7 @@ from .const import ( ) # force import gpio to register pin schema -from .gpio import nrf52_pin_to_code # noqa +from .gpio import nrf52_pin_to_code # noqa: F401 CODEOWNERS = ["@tomaszduda23"] AUTO_LOAD = ["zephyr", "preferences"] diff --git a/esphome/components/rp2040/__init__.py b/esphome/components/rp2040/__init__.py index 830c961476..6ec0ee08b8 100644 --- a/esphome/components/rp2040/__init__.py +++ b/esphome/components/rp2040/__init__.py @@ -50,7 +50,7 @@ from .const import ( ) # force import gpio to register pin schema -from .gpio import rp2040_pin_to_code # noqa +from .gpio import rp2040_pin_to_code # noqa: F401 _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@jesserockz"] diff --git a/pyproject.toml b/pyproject.toml index d92c7ba894..d2f30ea3d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,7 @@ select = [ "LOG", # flake8-logging "NPY", # numpy-specific rules "PERF", # performance + "PGH", # pygrep-hooks "PIE", # flake8-pie "PL", # pylint "PTH", # flake8-use-pathlib From 62b3b1cc7509e2b7a1ce5f26ebd08abe4fc8b495 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 27 May 2026 06:00:08 +0930 Subject: [PATCH 75/96] [lvgl] Support `rounded` property for meter arcs (#16669) --- esphome/components/lvgl/widgets/meter.py | 3 ++- tests/components/lvgl/lvgl-package.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/esphome/components/lvgl/widgets/meter.py b/esphome/components/lvgl/widgets/meter.py index 62ea14bdda..e2407fad5a 100644 --- a/esphome/components/lvgl/widgets/meter.py +++ b/esphome/components/lvgl/widgets/meter.py @@ -184,6 +184,7 @@ INDICATOR_ARC_SCHEMA = cv.Schema( cv.Optional(CONF_START_VALUE): lv_float, cv.Optional(CONF_END_VALUE): lv_float, cv.Optional(CONF_OPA, default=1.0): opacity, + cv.Optional(CONF_ROUNDED, default=False): cv.boolean, } ).add_extra(cv.has_at_most_one_key(CONF_VALUE, CONF_START_VALUE)) @@ -417,7 +418,7 @@ class MeterType(WidgetType): "arc_width": v[CONF_WIDTH], "arc_color": v[CONF_COLOR], "arc_opa": v[CONF_OPA], - "arc_rounded": v.get("arc_rounded", False), + "arc_rounded": v[CONF_ROUNDED], } if CONF_R_MOD in v: get_warnings().add( diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 0f4b961297..7af058e6b8 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -1313,6 +1313,7 @@ lvgl: width: 6 start_value: 0 end_value: 360 + rounded: true - id: page3 layout: Horizontal pad_all: 6px From 4d908798bcf41b8cbe151a2d5c1aa753d7aaaa40 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 18:45:50 -0400 Subject: [PATCH 76/96] [core] Remove deprecated custom_components folder loading (#16679) --- esphome/config.py | 1 - esphome/loader.py | 13 ------------- esphome/writer.py | 4 ++-- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/esphome/config.py b/esphome/config.py index 79d0d2b02b..9da39a387b 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -1005,7 +1005,6 @@ def validate_config( CORE.skip_external_update = skip_external_update loader.clear_component_meta_finders() - loader.install_custom_components_meta_finder() # 0. Load packages if CONF_PACKAGES in config: diff --git a/esphome/loader.py b/esphome/loader.py index c57c09274e..8823d82fc1 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -12,7 +12,6 @@ from types import ModuleType from typing import TYPE_CHECKING, Any from esphome.const import SOURCE_FILE_EXTENSIONS -from esphome.core import CORE from esphome.types import ConfigType if TYPE_CHECKING: @@ -206,18 +205,6 @@ def install_meta_finder( sys.meta_path.insert(0, ComponentMetaFinder(components_path, allowed_components)) -def install_custom_components_meta_finder(): - # Remove before 2026.6.0 - custom_components_dir = (Path(CORE.config_dir) / "custom_components").resolve() - if custom_components_dir.is_dir() and any(custom_components_dir.iterdir()): - _LOGGER.warning( - "The 'custom_components' folder is deprecated and will be removed in 2026.6.0. " - "Please use 'external_components' instead. " - "See https://esphome.io/components/external_components.html for more information." - ) - install_meta_finder(custom_components_dir) - - def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None: if domain in _COMPONENT_CACHE: return _COMPONENT_CACHE[domain] diff --git a/esphome/writer.py b/esphome/writer.py index ab014c5daa..ef7cbf5ac4 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -200,8 +200,8 @@ ESPHome automatically populates the build directory, and any changes to this directory will be removed the next time esphome is run. -For modifying esphome's core files, please use a development esphome install, -the custom_components folder or the external_components feature. +For modifying esphome's core files, please use a development esphome install +or the external_components feature. """ From b71d445e7963303f1ab76b6dd9ccdebf609115b1 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 18:46:45 -0400 Subject: [PATCH 77/96] [core] Remove deprecated const char* mark_failed/status_set_error (#16680) --- esphome/core/component.cpp | 33 +++++++-------------------------- esphome/core/component.h | 17 ----------------- 2 files changed, 7 insertions(+), 43 deletions(-) diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index e33652482e..2d80301897 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -32,10 +32,7 @@ static const char *const TAG = "component"; namespace { struct ComponentErrorMessage { const Component *component; - const char *message; - // Track if message is flash pointer (needs LOG_STR_ARG) or RAM pointer - // Remove before 2026.6.0 when deprecated const char* API is removed - bool is_flash_ptr; + const LogString *message; }; #ifdef USE_SETUP_PRIORITY_OVERRIDE @@ -56,9 +53,8 @@ std::vector *setup_priority_overrides = nullptr; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::vector *component_error_messages = nullptr; -// Helper to store error messages - reduces duplication between deprecated and new API -// Remove before 2026.6.0 when deprecated const char* API is removed -void store_component_error_message(const Component *component, const char *message, bool is_flash_ptr) { +// Helper to store error messages +void store_component_error_message(const Component *component, const LogString *message) { // Lazy allocate the error messages vector if needed if (!component_error_messages) { component_error_messages = new std::vector(); @@ -67,12 +63,11 @@ void store_component_error_message(const Component *component, const char *messa for (auto &entry : *component_error_messages) { if (entry.component == component) { entry.message = message; - entry.is_flash_ptr = is_flash_ptr; return; } } // Add new error message - component_error_messages->emplace_back(ComponentErrorMessage{component, message, is_flash_ptr}); + component_error_messages->emplace_back(ComponentErrorMessage{component, message}); } } // namespace @@ -209,21 +204,17 @@ void Component::call_dump_config_() { this->dump_config(); if (this->is_failed()) { // Look up error message from global vector - const char *error_msg = nullptr; - bool is_flash_ptr = false; + const LogString *error_msg = nullptr; if (component_error_messages) { for (const auto &entry : *component_error_messages) { if (entry.component == this) { error_msg = entry.message; - is_flash_ptr = entry.is_flash_ptr; break; } } } - // Log with appropriate format based on pointer type ESP_LOGE(TAG, " %s is marked FAILED: %s", LOG_STR_ARG(this->get_component_log_str()), - error_msg ? (is_flash_ptr ? LOG_STR_ARG((const LogString *) error_msg) : error_msg) - : LOG_STR_LITERAL("unspecified")); + error_msg ? LOG_STR_ARG(error_msg) : LOG_STR_LITERAL("unspecified")); } } @@ -390,23 +381,13 @@ void Component::status_set_warning(const LogString *message) { message ? LOG_STR_ARG(message) : LOG_STR_LITERAL("unspecified")); } void Component::status_set_error() { this->status_set_error((const LogString *) nullptr); } -void Component::status_set_error(const char *message) { - if (!this->set_status_flag_(STATUS_LED_ERROR)) - return; - ESP_LOGE(TAG, "%s set Error flag: %s", LOG_STR_ARG(this->get_component_log_str()), - message ? message : LOG_STR_LITERAL("unspecified")); - if (message != nullptr) { - store_component_error_message(this, message, false); - } -} void Component::status_set_error(const LogString *message) { if (!this->set_status_flag_(STATUS_LED_ERROR)) return; ESP_LOGE(TAG, "%s set Error flag: %s", LOG_STR_ARG(this->get_component_log_str()), message ? LOG_STR_ARG(message) : LOG_STR_LITERAL("unspecified")); if (message != nullptr) { - // Store the LogString pointer directly (safe because LogString is always in flash/static memory) - store_component_error_message(this, LOG_STR_ARG(message), true); + store_component_error_message(this, message); } } void Component::status_clear_warning_slow_path_() { diff --git a/esphome/core/component.h b/esphome/core/component.h index 5baf795ca6..ff10f1a8f1 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -220,18 +220,6 @@ class Component { */ void mark_failed(); - // Remove before 2026.6.0 - ESPDEPRECATED("Use mark_failed(LOG_STR(\"static string literal\")) instead. Do NOT use .c_str() from temporary " - "strings. Will stop working in 2026.6.0", - "2025.12.0") - void mark_failed(const char *message) { -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - this->status_set_error(message); -#pragma GCC diagnostic pop - this->mark_failed(); - } - void mark_failed(const LogString *message) { this->status_set_error(message); this->mark_failed(); @@ -296,11 +284,6 @@ class Component { void status_set_warning(const LogString *message); void status_set_error(); // Set error flag without message - // Remove before 2026.6.0 - ESPDEPRECATED("Use status_set_error(LOG_STR(\"static string literal\")) instead. Do NOT use .c_str() from temporary " - "strings. Will stop working in 2026.6.0", - "2025.12.0") - void status_set_error(const char *message); void status_set_error(const LogString *message); void status_clear_warning() { From 171ded35a58728ef48cb65198bcd457ea92f1fa9 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 18:47:16 -0400 Subject: [PATCH 78/96] [core] Remove cv.only_with_esp_idf and CORE.using_esp_idf (#16681) --- esphome/config_validation.py | 10 ---------- esphome/core/__init__.py | 9 --------- script/ci-custom.py | 13 ------------- 3 files changed, 32 deletions(-) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index f826b254ac..2f09fdc105 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -857,16 +857,6 @@ only_on_rp2040 = only_on(PLATFORM_RP2040) only_with_arduino = only_with_framework(Framework.ARDUINO) -def only_with_esp_idf(obj): - """Deprecated: use only_on_esp32 instead.""" - _LOGGER.warning( - "cv.only_with_esp_idf was deprecated in 2026.1, will change behavior in 2026.6. " - "ESP32 Arduino builds on top of ESP-IDF, so ESP-IDF features are available in both frameworks. " - "Use cv.only_on_esp32 and/or cv.only_with_arduino instead." - ) - return only_with_framework(Framework.ESP_IDF)(obj) - - # Adapted from: # https://github.com/alecthomas/voluptuous/issues/115#issuecomment-144464666 def has_at_least_one_key(*keys): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 182be38b18..df8fd0a756 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -859,15 +859,6 @@ class EsphomeCore: def using_arduino(self): return self.target_framework == "arduino" - @property - def using_esp_idf(self): - _LOGGER.warning( - "CORE.using_esp_idf was deprecated in 2026.1, will change behavior in 2026.6. " - "ESP32 Arduino builds on top of ESP-IDF, so ESP-IDF features are available in both frameworks. " - "Use CORE.is_esp32 and/or CORE.using_arduino instead." - ) - return self.target_framework == "esp-idf" - @property def using_toolchain_esp_idf(self): return self.toolchain == Toolchain.ESP_IDF diff --git a/script/ci-custom.py b/script/ci-custom.py index 1ac13e18f7..78ff6cf781 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -693,19 +693,6 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_find_check( - "CORE.using_esp_idf", - include=py_include, - exclude=["esphome/core/__init__.py", "script/ci-custom.py"], -) -def lint_using_esp_idf_deprecated(fname, line, col, content): - return ( - f"{highlight('CORE.using_esp_idf')} is deprecated and will change behavior in 2026.6. " - "ESP32 Arduino builds on top of ESP-IDF, so ESP-IDF features are available in both frameworks. " - f"Please use {highlight('CORE.is_esp32')} and/or {highlight('CORE.using_arduino')} instead." - ) - - @lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) def lint_pragma_once(fname, content): if "#pragma once" not in content: From fb0b73980bf188d5530e62c02dda451560164c2b Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 18:47:40 -0400 Subject: [PATCH 79/96] [wifi] Default ESP8266 min_auth_mode to WPA2 (#16682) --- esphome/components/wifi/__init__.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index f9cb391442..e5e57cc97d 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -326,23 +326,9 @@ def validate_variant(_): def _apply_min_auth_mode_default(config): - """Apply platform-specific default for min_auth_mode and warn ESP8266 users.""" - # Only apply defaults for platforms that support min_auth_mode + """Apply platform-specific default for min_auth_mode.""" if CONF_MIN_AUTH_MODE not in config and (CORE.is_esp8266 or CORE.is_esp32): - if CORE.is_esp8266: - _LOGGER.warning( - "The minimum WiFi authentication mode (wifi -> min_auth_mode) is not set. " - "This controls the weakest encryption your device will accept when connecting to WiFi. " - "Currently defaults to WPA (less secure), but will change to WPA2 (more secure) in 2026.6.0. " - "WPA uses TKIP encryption which has known security vulnerabilities and should be avoided. " - "WPA2 uses AES encryption which is significantly more secure. " - "To silence this warning, explicitly set min_auth_mode under 'wifi:'. " - "If your router supports WPA2 or WPA3, set 'min_auth_mode: WPA2'. " - "If your router only supports WPA, set 'min_auth_mode: WPA'." - ) - config[CONF_MIN_AUTH_MODE] = VALIDATE_WIFI_MIN_AUTH_MODE("WPA") - elif CORE.is_esp32: - config[CONF_MIN_AUTH_MODE] = VALIDATE_WIFI_MIN_AUTH_MODE("WPA2") + config[CONF_MIN_AUTH_MODE] = VALIDATE_WIFI_MIN_AUTH_MODE("WPA2") return config From eb1196c6b22ffe8da8c273d52bba546489ff0d04 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 18:48:17 -0400 Subject: [PATCH 80/96] [nfc] Remove deprecated heap-allocating format helpers (#16684) --- esphome/components/nfc/nfc.cpp | 11 ----------- esphome/components/nfc/nfc.h | 7 ------- 2 files changed, 18 deletions(-) diff --git a/esphome/components/nfc/nfc.cpp b/esphome/components/nfc/nfc.cpp index 99e476dbdf..76a391f1de 100644 --- a/esphome/components/nfc/nfc.cpp +++ b/esphome/components/nfc/nfc.cpp @@ -15,17 +15,6 @@ char *format_bytes_to(char *buffer, std::span bytes) { return format_hex_pretty_to(buffer, FORMAT_BYTES_BUFFER_SIZE, bytes.data(), bytes.size(), ' '); } -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -// Deprecated wrappers intentionally use heap-allocating version for backward compatibility -std::string format_uid(std::span uid) { - return format_hex_pretty(uid.data(), uid.size(), '-', false); // NOLINT -} -std::string format_bytes(std::span bytes) { - return format_hex_pretty(bytes.data(), bytes.size(), ' ', false); // NOLINT -} -#pragma GCC diagnostic pop - uint8_t guess_tag_type(uint8_t uid_length) { if (uid_length == 4) { return TAG_TYPE_MIFARE_CLASSIC; diff --git a/esphome/components/nfc/nfc.h b/esphome/components/nfc/nfc.h index 42ef993913..36b27ce5f6 100644 --- a/esphome/components/nfc/nfc.h +++ b/esphome/components/nfc/nfc.h @@ -63,13 +63,6 @@ static constexpr size_t FORMAT_BYTES_BUFFER_SIZE = 192; /// Format bytes to buffer with ' ' separator (e.g., "04 11 22 33"). Returns buffer for inline use. char *format_bytes_to(char *buffer, std::span bytes); -// Remove before 2026.6.0 -ESPDEPRECATED("Use format_uid_to() with stack buffer instead. Removed in 2026.6.0", "2025.12.0") -std::string format_uid(std::span uid); -// Remove before 2026.6.0 -ESPDEPRECATED("Use format_bytes_to() with stack buffer instead. Removed in 2026.6.0", "2025.12.0") -std::string format_bytes(std::span bytes); - uint8_t guess_tag_type(uint8_t uid_length); int8_t get_mifare_classic_ndef_start_index(std::vector &data); bool decode_mifare_classic_tlv(std::vector &data, uint32_t &message_length, uint8_t &message_start_index); From 6c4a8a324515f9d6dc3280ed4e82993022c5138b Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 18:49:44 -0400 Subject: [PATCH 81/96] [dsmr] Force BearSSL on ESP8266 to avoid mbedtls link failure (#16686) --- esphome/components/dsmr/dsmr.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/esphome/components/dsmr/dsmr.h b/esphome/components/dsmr/dsmr.h index 626a389c1f..e55db9f976 100644 --- a/esphome/components/dsmr/dsmr.h +++ b/esphome/components/dsmr/dsmr.h @@ -16,9 +16,14 @@ #include #include +// On ESP8266 Arduino, BearSSL is the native crypto. The mbedtls headers can +// still be in scope when a sibling component (e.g. wireguard) pulls in +// esp_mbedtls_esp8266, but that build leaves MBEDTLS_GCM_C disabled so the +// gcm.h symbols are unresolved at link time. Force BearSSL on ESP8266 to +// avoid that linker error. #if __has_include() #include -#elif __has_include() +#elif !defined(USE_ESP8266) && __has_include() #if __has_include() #include #endif @@ -33,7 +38,7 @@ namespace esphome::dsmr { #if __has_include() using Aes128GcmDecryptorImpl = dsmr_parser::Aes128GcmTfPsa; -#elif __has_include() +#elif !defined(USE_ESP8266) && __has_include() using Aes128GcmDecryptorImpl = dsmr_parser::Aes128GcmMbedTls; #else using Aes128GcmDecryptorImpl = dsmr_parser::Aes128GcmBearSsl; From f728cb437377ae32c9c2b0b99ca9c55d6c8d9925 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 18:50:20 -0400 Subject: [PATCH 82/96] [core] Remove deprecated seq/gens templates (#16685) --- esphome/core/automation.h | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/esphome/core/automation.h b/esphome/core/automation.h index 468ea3b382..ea522a4d2d 100644 --- a/esphome/core/automation.h +++ b/esphome/core/automation.h @@ -13,27 +13,6 @@ namespace esphome { -// C++20 std::index_sequence is now used for tuple unpacking -// Legacy seq<>/gens<> pattern deprecated but kept for backwards compatibility -// https://stackoverflow.com/questions/7858817/unpacking-a-tuple-to-call-a-matching-function-pointer/7858971#7858971 -// Remove before 2026.6.0 -// NOLINTBEGIN(readability-identifier-naming) -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - -template struct ESPDEPRECATED("Use std::index_sequence instead. Removed in 2026.6.0", "2025.12.0") seq {}; -template -struct ESPDEPRECATED("Use std::make_index_sequence instead. Removed in 2026.6.0", "2025.12.0") gens - : gens {}; -template struct gens<0, S...> { using type = seq; }; - -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic pop -#endif -// NOLINTEND(readability-identifier-naming) - /// Function-pointer-only templatable storage (4 bytes on 32-bit). /// Used by the TEMPLATABLE_VALUE macro for codegen-managed fields. /// Codegen wraps constants in stateless lambdas so only a function pointer is needed. From e174c44b283f0ed6f3748f0c796b9ee62922bdb4 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 19:15:25 -0400 Subject: [PATCH 83/96] [neopixelbus] Deprecate on ESP32 (#16676) --- esphome/components/neopixelbus/light.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/esphome/components/neopixelbus/light.py b/esphome/components/neopixelbus/light.py index 943fd141f6..2e18688af0 100644 --- a/esphome/components/neopixelbus/light.py +++ b/esphome/components/neopixelbus/light.py @@ -1,3 +1,5 @@ +import logging + from esphome import pins import esphome.codegen as cg from esphome.components import light @@ -22,6 +24,7 @@ from esphome.const import ( Framework, ) from esphome.core import CORE +from esphome.types import ConfigType from ._methods import ( METHOD_BIT_BANG, @@ -34,6 +37,8 @@ from ._methods import ( ) from .const import CHIP_TYPES, CONF_ASYNC, CONF_BUS, ONE_WIRE_CHIPS +_LOGGER = logging.getLogger(__name__) + neopixelbus_ns = cg.esphome_ns.namespace("neopixelbus") NeoPixelBusLightOutputBase = neopixelbus_ns.class_( "NeoPixelBusLightOutputBase", light.AddressableLight @@ -134,6 +139,17 @@ def _validate(config): return config +def _warn_esp32_deprecated(config: ConfigType) -> ConfigType: + if CORE.is_esp32: + _LOGGER.warning( + "'neopixelbus' on ESP32 is deprecated. The upstream library " + "(makuna/NeoPixelBus) is no longer actively maintained. Migrate " + "to 'esp32_rmt_led_strip'. Removal is targeted for 2027.1 but " + "may happen sooner once ESPHome moves to ESP-IDF 6." + ) + return config + + def _validate_method(value): if value is None: # default method is determined afterwards because it depends on the chip type chosen @@ -195,6 +211,7 @@ CONFIG_SCHEMA = cv.All( ).extend(cv.COMPONENT_SCHEMA), _choose_default_method, _validate, + _warn_esp32_deprecated, ) From a6ef67aa65892d7312ba7191bbe41742b2d0c80a Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 26 May 2026 19:34:52 -0400 Subject: [PATCH 84/96] [text_sensor] Remove deprecated public raw_state member (#16683) --- .../components/text_sensor/text_sensor.cpp | 19 ++++++------------- esphome/components/text_sensor/text_sensor.h | 10 ++-------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/esphome/components/text_sensor/text_sensor.cpp b/esphome/components/text_sensor/text_sensor.cpp index 31543117b8..d2483619a6 100644 --- a/esphome/components/text_sensor/text_sensor.cpp +++ b/esphome/components/text_sensor/text_sensor.cpp @@ -39,16 +39,13 @@ void TextSensor::publish_state(const char *state, size_t len) { #ifdef USE_TEXT_SENSOR_FILTER } else { // Has filters: need separate raw storage -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" // Only assign if changed to avoid heap allocation - if (len != this->raw_state.size() || memcmp(state, this->raw_state.data(), len) != 0) { - this->raw_state.assign(state, len); + if (len != this->raw_state_.size() || memcmp(state, this->raw_state_.data(), len) != 0) { + this->raw_state_.assign(state, len); } - this->raw_callback_.call(this->raw_state); - ESP_LOGV(TAG, "'%s': Received new state %s", this->name_.c_str(), this->raw_state.c_str()); - this->filter_list_->input(this->raw_state); -#pragma GCC diagnostic pop + this->raw_callback_.call(this->raw_state_); + ESP_LOGV(TAG, "'%s': Received new state %s", this->name_.c_str(), this->raw_state_.c_str()); + this->filter_list_->input(this->raw_state_); } #endif } @@ -89,11 +86,7 @@ const std::string &TextSensor::get_state() const { return this->state; } const std::string &TextSensor::get_raw_state() const { #ifdef USE_TEXT_SENSOR_FILTER if (this->filter_list_ != nullptr) { - // Suppress deprecation warning - get_raw_state() is the replacement API -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - return this->raw_state; -#pragma GCC diagnostic pop + return this->raw_state_; } #endif return this->state; // No filters, raw == filtered diff --git a/esphome/components/text_sensor/text_sensor.h b/esphome/components/text_sensor/text_sensor.h index 3f69e91c8d..aa48781f41 100644 --- a/esphome/components/text_sensor/text_sensor.h +++ b/esphome/components/text_sensor/text_sensor.h @@ -29,19 +29,12 @@ class TextSensor : public EntityBase { public: std::string state; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - /// @deprecated Use get_raw_state() instead. This member will be removed in ESPHome 2026.6.0. - ESPDEPRECATED("Use get_raw_state() instead of .raw_state. Will be removed in 2026.6.0", "2025.12.0") - std::string raw_state; - TextSensor() = default; ~TextSensor() = default; -#pragma GCC diagnostic pop /// Getter-syntax for .state. const std::string &get_state() const; - /// Getter-syntax for .raw_state + /// Returns the raw (pre-filter) state. const std::string &get_raw_state() const; void publish_state(const std::string &state); @@ -84,6 +77,7 @@ class TextSensor : public EntityBase { /// Notify frontend that state has changed (assumes this->state is already set) void notify_frontend_(); #ifdef USE_TEXT_SENSOR_FILTER + std::string raw_state_; ///< Backing storage for the raw (pre-filter) value. Only used when a filter is attached. LazyCallbackManager raw_callback_; ///< Storage for raw state callbacks. #endif LazyCallbackManager callback_; ///< Storage for filtered state callbacks. From 91ead4ff543e6d46f5e6aa987e7f4206e6de868a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 19:16:47 -0500 Subject: [PATCH 85/96] [core] Mark canonical sensitive fields with cv.sensitive (#16677) --- esphome/components/api/__init__.py | 2 +- esphome/components/esphome/ota/__init__.py | 2 +- esphome/components/http_request/ota/__init__.py | 2 +- esphome/components/mqtt/__init__.py | 2 +- esphome/components/web_server/__init__.py | 4 ++-- esphome/components/wifi/__init__.py | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index ca74483a2b..932702d47a 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -234,7 +234,7 @@ ACTIONS_SCHEMA = automation.validate_automation( ENCRYPTION_SCHEMA = cv.Schema( { - cv.Optional(CONF_KEY): validate_encryption_key, + cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), } ) diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index f7793b1493..66a33e1935 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -133,7 +133,7 @@ CONFIG_SCHEMA = cv.All( host=8082, ): cv.port, cv.Optional(CONF_ALLOW_PARTITION_ACCESS, default=False): cv.boolean, - cv.Optional(CONF_PASSWORD): cv.string, + cv.Optional(CONF_PASSWORD): cv.sensitive(), cv.Optional(CONF_NUM_ATTEMPTS): cv.invalid( f"'{CONF_SAFE_MODE}' (and its related configuration variables) has moved from 'ota' to its own component. See https://esphome.io/components/safe_mode" ), diff --git a/esphome/components/http_request/ota/__init__.py b/esphome/components/http_request/ota/__init__.py index fb59e51943..1bb54599dc 100644 --- a/esphome/components/http_request/ota/__init__.py +++ b/esphome/components/http_request/ota/__init__.py @@ -57,7 +57,7 @@ OTA_HTTP_REQUEST_FLASH_ACTION_SCHEMA = cv.All( cv.Optional(CONF_MD5): cv.templatable( cv.All(cv.string, cv.Length(min=32, max=32)) ), - cv.Optional(CONF_PASSWORD): cv.templatable(cv.string), + cv.Optional(CONF_PASSWORD): cv.sensitive(cv.templatable(cv.string)), cv.Optional(CONF_USERNAME): cv.templatable(cv.string), cv.Required(CONF_URL): cv.templatable(cv.url), } diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index cb6b9d144f..86bba11a60 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -232,7 +232,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_ON_BOOT, default=True): cv.boolean, cv.Optional(CONF_PORT, default=1883): cv.port, cv.Optional(CONF_USERNAME, default=""): cv.string, - cv.Optional(CONF_PASSWORD, default=""): cv.string, + cv.Optional(CONF_PASSWORD, default=""): cv.sensitive(), cv.Optional(CONF_CLEAN_SESSION, default=False): cv.boolean, cv.Optional(CONF_CLIENT_ID): cv.string, cv.SplitDefault(CONF_IDF_SEND_ASYNC, esp32=False): cv.All( diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 99a9b7518c..fd380a38dd 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -193,8 +193,8 @@ CONFIG_SCHEMA = cv.All( cv.Required(CONF_USERNAME): cv.All( cv.string_strict, cv.Length(min=1) ), - cv.Required(CONF_PASSWORD): cv.All( - cv.string_strict, cv.Length(min=1) + cv.Required(CONF_PASSWORD): cv.sensitive( + cv.All(cv.string_strict, cv.Length(min=1)) ), } ), diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index e5e57cc97d..4e7dcc82e5 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -251,7 +251,7 @@ EAP_AUTH_SCHEMA = cv.All( { cv.Optional(CONF_IDENTITY): cv.string_strict, cv.Optional(CONF_USERNAME): cv.string_strict, - cv.Optional(CONF_PASSWORD): cv.string_strict, + cv.Optional(CONF_PASSWORD): cv.sensitive(cv.string_strict), cv.Optional(CONF_CERTIFICATE_AUTHORITY): wpa2_eap.validate_certificate, cv.SplitDefault(CONF_TTLS_PHASE_2, esp32="mschapv2"): cv.All( cv.enum(TTLS_PHASE_2), cv.only_on_esp32 @@ -272,7 +272,7 @@ WIFI_NETWORK_BASE = cv.Schema( { cv.GenerateID(): cv.declare_id(WiFiAP), cv.Optional(CONF_SSID): cv.ssid, - cv.Optional(CONF_PASSWORD): validate_password, + cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), cv.Optional(CONF_CHANNEL): validate_channel, cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, } @@ -435,7 +435,7 @@ CONFIG_SCHEMA = cv.All( cv.ensure_list(WIFI_NETWORK_STA), cv.Length(max=MAX_WIFI_NETWORKS) ), cv.Optional(CONF_SSID): cv.ssid, - cv.Optional(CONF_PASSWORD): validate_password, + cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, cv.Optional(CONF_EAP): EAP_AUTH_SCHEMA, cv.Optional(CONF_AP): wifi_network_ap, @@ -851,7 +851,7 @@ async def final_step(): cv.Schema( { cv.Required(CONF_SSID): cv.templatable(cv.ssid), - cv.Required(CONF_PASSWORD): cv.templatable(validate_password), + cv.Required(CONF_PASSWORD): cv.sensitive(cv.templatable(validate_password)), cv.Optional(CONF_SAVE, default=True): cv.templatable(cv.boolean), cv.Optional(CONF_TIMEOUT, default="30000ms"): cv.templatable( cv.positive_time_period_milliseconds From 87d0e24d194164e1988ba742cd75b32176cf6cf0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 19:57:29 -0500 Subject: [PATCH 86/96] Bump aioesphomeapi from 45.2.2 to 45.3.1 (#16688) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 45401a7995..14dddbb1aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 -aioesphomeapi==45.2.2 +aioesphomeapi==45.3.1 zeroconf==0.149.16 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import From 8d19c55be2325d0a1c2dd06898ad4ec19c7c6602 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 19:58:13 -0500 Subject: [PATCH 87/96] Bump pytest-asyncio from 1.3.0 to 1.4.0 (#16687) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 102a9cae6e..aad1da0807 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -8,7 +8,7 @@ pre-commit pytest==9.0.3 pytest-cov==7.1.0 pytest-mock==3.15.1 -pytest-asyncio==1.3.0 +pytest-asyncio==1.4.0 pytest-xdist==3.8.0 asyncmock==0.4.2 hypothesis==6.92.1 From 49b9d9313edbf5ca45c5b2803bf42a64cf18f1a4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 21:49:31 -0500 Subject: [PATCH 88/96] [core] Sensitive redaction via yaml_util representer cv.sensitive(...) now returns a SensitiveStr (thin str subclass) so the tag travels with the validated value. yaml_util.dump constructs a per-call ESPHomeDumper subclass with a class-attribute redaction flag; the PyYAML representer for SensitiveStr renders values wrapped in literal \\033[8m...\\033[28m text when show_secrets is False and raw when True. The post-dump regex in command_config is deleted. Also tags wifi.ssid sites with cv.sensitive so SSID coverage isn't lost when the regex (which matched 'ssid:' via substring) goes away. No module-level mutable state; the per-call subclass keeps each dump invocation self-contained and thread-safe by construction. --- esphome/__main__.py | 5 -- esphome/components/wifi/__init__.py | 6 +-- esphome/config_validation.py | 10 +++- esphome/yaml_util.py | 43 ++++++++++++++++- tests/unit_tests/test_config_validation.py | 40 ++++++++++++++++ tests/unit_tests/test_yaml_util.py | 55 ++++++++++++++++++++++ 6 files changed, 148 insertions(+), 11 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index dd97c6eee9..7eef646ebb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1412,11 +1412,6 @@ def command_config(args: ArgsProtocol, config: ConfigType) -> int | None: if not CORE.verbose: config = strip_default_ids(config) output = yaml_util.dump(config, args.show_secrets) - # add the console decoration so the front-end can hide the secrets - if not args.show_secrets: - output = re.sub( - r"(password|key|psk|ssid)\: (.+)", r"\1: \\033[8m\2\\033[28m", output - ) if not CORE.quiet: safe_print(output) _LOGGER.info("Configuration is valid!") diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 4e7dcc82e5..b7719c80d1 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -271,7 +271,7 @@ EAP_AUTH_SCHEMA = cv.All( WIFI_NETWORK_BASE = cv.Schema( { cv.GenerateID(): cv.declare_id(WiFiAP), - cv.Optional(CONF_SSID): cv.ssid, + cv.Optional(CONF_SSID): cv.sensitive(cv.ssid), cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), cv.Optional(CONF_CHANNEL): validate_channel, cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, @@ -434,7 +434,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_NETWORKS): cv.All( cv.ensure_list(WIFI_NETWORK_STA), cv.Length(max=MAX_WIFI_NETWORKS) ), - cv.Optional(CONF_SSID): cv.ssid, + cv.Optional(CONF_SSID): cv.sensitive(cv.ssid), cv.Optional(CONF_PASSWORD): cv.sensitive(validate_password), cv.Optional(CONF_MANUAL_IP): STA_MANUAL_IP_SCHEMA, cv.Optional(CONF_EAP): EAP_AUTH_SCHEMA, @@ -850,7 +850,7 @@ async def final_step(): WiFiConfigureAction, cv.Schema( { - cv.Required(CONF_SSID): cv.templatable(cv.ssid), + cv.Required(CONF_SSID): cv.sensitive(cv.templatable(cv.ssid)), cv.Required(CONF_PASSWORD): cv.sensitive(cv.templatable(validate_password)), cv.Optional(CONF_SAVE, default=True): cv.templatable(cv.boolean), cv.Optional(CONF_TIMEOUT, default="30000ms"): cv.templatable( diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 2f09fdc105..0ef6d212fe 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -101,7 +101,7 @@ from esphome.schema_extractors import ( ) from esphome.util import parse_esphome_version from esphome.voluptuous_schema import _Schema -from esphome.yaml_util import make_data_base +from esphome.yaml_util import SensitiveStr, make_data_base _LOGGER = logging.getLogger(__name__) @@ -514,7 +514,13 @@ class SensitiveValidator: self.inner = inner def __call__(self, value: typing.Any) -> typing.Any: - return self.inner(value) + validated = self.inner(value) + # Tag string results so yaml_util.dump can mask them. Non-string + # results pass through unchanged; already-tagged values are not + # re-wrapped to keep nested cv.sensitive applications idempotent. + if isinstance(validated, str) and not isinstance(validated, SensitiveStr): + return SensitiveStr(validated) + return validated def __repr__(self) -> str: # Mirror the inner validator's repr so ``build_language_schema``'s diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 28f72ab831..19cb775a50 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -52,6 +52,16 @@ _load_listeners: list[Callable[[Path], None]] = [] DocumentPath = list[str | int] +class SensitiveStr(str): + """Marker subclass for validated strings that should be masked in + user-visible YAML output. ``cv.sensitive`` wraps validated values in this + type so ``dump()`` can render them with ANSI conceal codes without + needing a post-process regex. + """ + + __slots__ = () + + @contextmanager def track_yaml_loads() -> Generator[list[Path]]: """Context manager that records every file loaded by the YAML loader. @@ -808,11 +818,18 @@ def dump(dict_, show_secrets=False, sort_keys=False): if show_secrets: _SECRET_VALUES.clear() _SECRET_CACHE.clear() + + # Per-call subclass so the redaction flag lives on the dumper class itself, + # not in module-level mutable state. Cheap (one type object per call), + # encapsulated, and thread-safe by construction. + class _Dumper(ESPHomeDumper): + _redact_sensitive = not show_secrets + return yaml.dump( dict_, default_flow_style=False, allow_unicode=True, - Dumper=ESPHomeDumper, + Dumper=_Dumper, sort_keys=sort_keys, ) @@ -958,6 +975,10 @@ def format_path(path: DocumentPath, current_obj: Any) -> str: class ESPHomeDumper(yaml.SafeDumper): + # Default for the base class; per-call subclass in ``dump()`` overrides. + # When True, ``represent_sensitive`` wraps values in ANSI conceal codes. + _redact_sensitive: bool = False + def represent_mapping(self, tag, mapping, flow_style=None): value = [] node = yaml.MappingNode(tag, value, flow_style=flow_style) @@ -992,6 +1013,23 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_secret(value) return self.represent_scalar(tag="tag:yaml.org,2002:str", value=str(value)) + def represent_sensitive(self, value): + # ``!secret`` references win: keep the original representation so the + # dumped YAML round-trips back to ``!secret name`` instead of leaking + # the resolved value. + if is_secret(value): + return self.represent_secret(value) + if self._redact_sensitive: + # Emit the conceal sequence as literal ``\033`` text (not actual + # ESC bytes) so the dump matches the previous regex-based output + # and downstream consumers like device-builder, which match + # ``\033[8m...\033[28m`` against the rendered text, keep working. + return self.represent_scalar( + tag="tag:yaml.org,2002:str", + value=f"\\033[8m{value}\\033[28m", + ) + return self.represent_scalar(tag="tag:yaml.org,2002:str", value=str(value)) + # pylint: disable=arguments-renamed def represent_bool(self, value): return self.represent_scalar( @@ -1063,6 +1101,9 @@ ESPHomeDumper.add_multi_representer( ) ESPHomeDumper.add_multi_representer(bool, ESPHomeDumper.represent_bool) ESPHomeDumper.add_multi_representer(str, ESPHomeDumper.represent_stringify) +# Registered after ``str`` so ``add_multi_representer``'s MRO lookup prefers +# the more specific ``SensitiveStr`` representer over the bare-``str`` one. +ESPHomeDumper.add_multi_representer(SensitiveStr, ESPHomeDumper.represent_sensitive) ESPHomeDumper.add_multi_representer(int, ESPHomeDumper.represent_int) ESPHomeDumper.add_multi_representer(float, ESPHomeDumper.represent_float) ESPHomeDumper.add_multi_representer(_BaseAddress, ESPHomeDumper.represent_stringify) diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 2c34cbfb07..f45efeb20a 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -145,6 +145,46 @@ def test_sensitive__custom_inner_delegates_validation() -> None: validator(123) +def test_sensitive__wraps_string_result_in_sensitive_str() -> None: + from esphome.yaml_util import SensitiveStr + + validator = config_validation.sensitive() + result = validator("hunter2") + + assert isinstance(result, SensitiveStr) + assert isinstance(result, str) + assert result == "hunter2" + + +def test_sensitive__does_not_double_tag_already_sensitive() -> None: + from esphome.yaml_util import SensitiveStr + + # If the inner validator already returns a SensitiveStr (e.g., nested + # cv.sensitive wrappers), re-tagging is a no-op rather than a new + # SensitiveStr around the same value. + pre_tagged = SensitiveStr("hunter2") + + def inner(_value): + return pre_tagged + + validator = config_validation.sensitive(inner) + result = validator("anything") + + assert result is pre_tagged + + +def test_sensitive__non_string_result_passes_through() -> None: + # If an inner validator returns something other than a string (e.g., a + # Lambda template), the sensitive wrapper must not coerce it. + sentinel = object() + + def inner(_value): + return sentinel + + validator = config_validation.sensitive(inner) + assert validator("anything") is sentinel + + def test_sensitive__is_detectable_via_isinstance() -> None: validator = config_validation.sensitive() diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index d6fb5b81f2..6be090b869 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -15,6 +15,7 @@ from esphome.yaml_util import ( DiscoveredYamlFiles, ESPHomeDataBase, ESPLiteralValue, + SensitiveStr, discover_user_yaml_files, force_load_include_files, format_path, @@ -1340,3 +1341,57 @@ def test_frontmatter_included_file_stored(tmp_path: Path) -> None: assert main.resolve() not in core.CORE.frontmatter # Included file's frontmatter is captured assert core.CORE.frontmatter[inc.resolve()]["child_meta"] == "hello" + + +def test_sensitive_str__is_a_str_subclass() -> None: + value = SensitiveStr("hunter2") + assert isinstance(value, str) + assert value == "hunter2" + + +def test_dump__redacts_sensitive_str_by_default() -> None: + out = yaml_util.dump({"password": SensitiveStr("hunter2")}) + assert "\\033[8mhunter2\\033[28m" in out + assert "hunter2" not in out.replace( + "\\033[8mhunter2\\033[28m", "" + ) # the raw value is only present inside the wrap + + +def test_dump__show_secrets_emits_sensitive_str_raw() -> None: + out = yaml_util.dump({"password": SensitiveStr("hunter2")}, show_secrets=True) + assert "hunter2" in out + assert "\\033[8m" not in out + assert "\\033[28m" not in out + + +def test_dump__plain_str_is_not_redacted() -> None: + out = yaml_util.dump({"hostname": "myserver"}) + assert "myserver" in out + assert "\\033[8m" not in out + + +def test_dump__secret_reference_wins_over_redaction() -> None: + # If the value also has an entry in _SECRET_VALUES (i.e., it was loaded + # via !secret), the dump should render it as !secret , not as a + # redacted scalar. SensitiveStr layered on top must not change that. + value = SensitiveStr("hunter2") + yaml_util._SECRET_VALUES[str(value)] = "my_secret_name" + try: + out = yaml_util.dump({"password": value}) + assert "!secret" in out + assert "my_secret_name" in out + assert "\\033[8m" not in out + finally: + yaml_util._SECRET_VALUES.clear() + + +def test_dump__redaction_flag_does_not_leak_between_calls() -> None: + # Per-call _Dumper subclass means show_secrets in one call doesn't + # affect another. Run them in both orders to catch any leakage. + redacted = yaml_util.dump({"password": SensitiveStr("hunter2")}) + raw = yaml_util.dump({"password": SensitiveStr("hunter2")}, show_secrets=True) + redacted_again = yaml_util.dump({"password": SensitiveStr("hunter2")}) + + assert "\\033[8m" in redacted + assert "\\033[8m" not in raw + assert "\\033[8m" in redacted_again From 23df0229a27848796d93401319e9d8c7f8cd7ca0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 21:52:22 -0500 Subject: [PATCH 89/96] [core] Type represent_sensitive and hoist SensitiveStr import in test --- esphome/yaml_util.py | 2 +- tests/unit_tests/test_config_validation.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 19cb775a50..5355d03e9c 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -1013,7 +1013,7 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_secret(value) return self.represent_scalar(tag="tag:yaml.org,2002:str", value=str(value)) - def represent_sensitive(self, value): + def represent_sensitive(self, value: SensitiveStr) -> yaml.ScalarNode: # ``!secret`` references win: keep the original representation so the # dumped YAML round-trips back to ``!secret name`` instead of leaking # the resolved value. diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index f45efeb20a..74d9a5047a 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -27,6 +27,7 @@ from esphome.const import ( SCHEDULER_DONT_RUN, ) from esphome.core import CORE, HexInt, Lambda +from esphome.yaml_util import SensitiveStr def test_check_not_templatable__invalid(): @@ -146,8 +147,6 @@ def test_sensitive__custom_inner_delegates_validation() -> None: def test_sensitive__wraps_string_result_in_sensitive_str() -> None: - from esphome.yaml_util import SensitiveStr - validator = config_validation.sensitive() result = validator("hunter2") @@ -157,8 +156,6 @@ def test_sensitive__wraps_string_result_in_sensitive_str() -> None: def test_sensitive__does_not_double_tag_already_sensitive() -> None: - from esphome.yaml_util import SensitiveStr - # If the inner validator already returns a SensitiveStr (e.g., nested # cv.sensitive wrappers), re-tagging is a no-op rather than a new # SensitiveStr around the same value. From 2e53a226a3dbd0aeb44baf8b2431aecdb8593dac Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 21:55:35 -0500 Subject: [PATCH 90/96] [core] Keep legacy redaction regex as deprecation fallback Restore the substring regex as a second pass in command_config so sensitive-shaped fields that haven't been tagged with cv.sensitive(...) yet are still redacted. Each unique unmarked field name caught by the heuristic emits a one-time deprecation warning naming the field and the fix; the fallback itself is slated for removal in 2026.12.0. --- esphome/__main__.py | 35 ++++++++++++++++++++++++++ tests/unit_tests/test_main.py | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/esphome/__main__.py b/esphome/__main__.py index 7eef646ebb..d62fa1bafd 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1412,12 +1412,47 @@ def command_config(args: ArgsProtocol, config: ConfigType) -> int | None: if not CORE.verbose: config = strip_default_ids(config) output = yaml_util.dump(config, args.show_secrets) + if not args.show_secrets: + output = _redact_with_legacy_fallback(output) if not CORE.quiet: safe_print(output) _LOGGER.info("Configuration is valid!") return 0 +# Legacy substring redaction fallback. Catches sensitive-looking fields that +# aren't yet tagged with ``cv.sensitive(...)`` so config dumps don't regress +# for unmigrated/third-party schemas. Each match also logs a one-time +# deprecation warning naming the field; this fallback is slated for removal +# in 2026.12.0 once canonical sensitive fields are migrated. +# +# The negative lookahead ``(?!\\033\[8m)`` skips values already wrapped by the +# SensitiveStr representer, so explicit tagging silences the warning. +_LEGACY_REDACTION_RE = re.compile( + r"(?P\w*(?:password|key|psk|ssid)\w*)\: (?!\\033\[8m)(?P.+)" +) +_LEGACY_REDACTION_REMOVAL = "2026.12.0" + + +def _redact_with_legacy_fallback(output: str) -> str: + unmarked: set[str] = set() + + def _replace(m: re.Match[str]) -> str: + unmarked.add(m.group("key")) + return f"{m.group('key')}: \\033[8m{m.group('val')}\\033[28m" + + output = _LEGACY_REDACTION_RE.sub(_replace, output) + for key in sorted(unmarked): + _LOGGER.warning( + "Field '%s' is being redacted by a legacy substring heuristic. " + "Mark this field's schema validator with cv.sensitive(...) for " + "deterministic redaction; the heuristic will be removed in %s.", + key, + _LEGACY_REDACTION_REMOVAL, + ) + return output + + def command_config_hash(args: ArgsProtocol, config: ConfigType) -> int | None: # generating code might modify config, so it must be done in order to generate # a hash that will match what was generated when compiling and then running diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index f6b6d0b05f..a103f14da5 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -22,6 +22,7 @@ from esphome.__main__ import ( Purpose, _get_configured_xtal_freq, _make_crystal_freq_callback, + _redact_with_legacy_fallback, _resolve_network_devices, _validate_bootloader_binary, _validate_partition_table_binary, @@ -340,6 +341,52 @@ def mock_ram_strings_analyzer() -> Generator[Mock]: yield mock_class +def test_redact_with_legacy_fallback__wraps_unmarked_field( + caplog: pytest.LogCaptureFixture, +) -> None: + """Unmarked sensitive-shaped fields are redacted; a deprecation warning + is emitted naming the field.""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback("password: hunter2\n") + assert "password: \\033[8mhunter2\\033[28m" in out + assert any( + "password" in rec.message and "cv.sensitive" in rec.message + for rec in caplog.records + ) + + +def test_redact_with_legacy_fallback__skips_already_wrapped( + caplog: pytest.LogCaptureFixture, +) -> None: + """Values already wrapped by the SensitiveStr representer don't trigger + the heuristic or the warning.""" + wrapped = "password: \\033[8mhunter2\\033[28m\n" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback(wrapped) + assert out == wrapped + assert not any("legacy substring" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__captures_full_field_name( + caplog: pytest.LogCaptureFixture, +) -> None: + """The warning names the actual field, not just the matched fragment.""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + _redact_with_legacy_fallback("encryption_key: abc\n") + assert any("encryption_key" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__deduplicates_warnings( + caplog: pytest.LogCaptureFixture, +) -> None: + """One warning per unique field name even if it appears many times.""" + text = "password: a\npassword: b\npassword: c\n" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + _redact_with_legacy_fallback(text) + password_warnings = [rec for rec in caplog.records if "'password'" in rec.message] + assert len(password_warnings) == 1 + + def test_choose_upload_log_host_with_string_default() -> None: """Test with a single string default device.""" setup_core() From 27385cee325a9e758c9b8eed22ab3cf70a8b4d64 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 22:06:25 -0500 Subject: [PATCH 91/96] [core] Tighten legacy redaction regex to avoid mid-word matches The trailing \w* after the fragment over-matched fields like key_value_pair: which the prior regex did not catch. Drop it so the fragment must end the captured field name, preserving the previous matching scope while still capturing the full field name (via the leading \w*) for the warning message. --- esphome/__main__.py | 7 +++++-- tests/unit_tests/test_main.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index d62fa1bafd..7049f8025e 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1427,9 +1427,12 @@ def command_config(args: ArgsProtocol, config: ConfigType) -> int | None: # in 2026.12.0 once canonical sensitive fields are migrated. # # The negative lookahead ``(?!\\033\[8m)`` skips values already wrapped by the -# SensitiveStr representer, so explicit tagging silences the warning. +# SensitiveStr representer, so explicit tagging silences the warning. No +# trailing ``\w*`` after the fragment so the fragment must end the field +# name (preserves the prior regex's matching scope while letting the +# leading ``\w*`` capture the full field name for the warning). _LEGACY_REDACTION_RE = re.compile( - r"(?P\w*(?:password|key|psk|ssid)\w*)\: (?!\\033\[8m)(?P.+)" + r"(?P\w*(?:password|key|psk|ssid))\: (?!\\033\[8m)(?P.+)" ) _LEGACY_REDACTION_REMOVAL = "2026.12.0" diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index a103f14da5..ef8d922a8e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -387,6 +387,18 @@ def test_redact_with_legacy_fallback__deduplicates_warnings( assert len(password_warnings) == 1 +def test_redact_with_legacy_fallback__does_not_match_fragment_in_middle( + caplog: pytest.LogCaptureFixture, +) -> None: + """Fragment must end the field name; embedded matches like + ``key_value_pair`` are unrelated to a sensitive key and must not be + redacted (matching the prior regex's scope).""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback("key_value_pair: abc\n") + assert "\\033[8m" not in out + assert not any("legacy substring" in rec.message for rec in caplog.records) + + def test_choose_upload_log_host_with_string_default() -> None: """Test with a single string default device.""" setup_core() From e358b8328fc7237e26903ca12897f719ff9854f9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 22:13:49 -0500 Subject: [PATCH 92/96] [core] Cover command_config redaction wiring Add two tests that exercise command_config end-to-end: one confirms the legacy fallback wraps an unmarked sensitive field, the other confirms --show-secrets bypasses redaction. Closes the patch-coverage gap on the 'output = _redact_with_legacy_fallback(output)' line. --- tests/unit_tests/test_main.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index ef8d922a8e..1873988f83 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -30,6 +30,7 @@ from esphome.__main__ import ( command_analyze_memory, command_bundle, command_clean_all, + command_config, command_config_hash, command_rename, command_run, @@ -399,6 +400,41 @@ def test_redact_with_legacy_fallback__does_not_match_fragment_in_middle( assert not any("legacy substring" in rec.message for rec in caplog.records) +def test_command_config__invokes_legacy_fallback_when_redacting( + tmp_path: Path, capfd: CaptureFixture[str] +) -> None: + """``command_config`` runs the legacy fallback on the dumped output when + ``--show-secrets`` is off. Cover the wiring (not just the helper). + """ + setup_core(tmp_path=tmp_path, config={"esphome": {"name": "test"}}) + args = MockArgs() + args.show_secrets = False + + result = command_config(args, {"wifi": {"password": "hunter2"}}) + + assert result == 0 + output = capfd.readouterr().out + assert "\\033[8mhunter2\\033[28m" in output + + +def test_command_config__show_secrets_skips_redaction( + tmp_path: Path, capfd: CaptureFixture[str] +) -> None: + """With ``--show-secrets`` the helper isn't invoked and the value + renders raw. + """ + setup_core(tmp_path=tmp_path, config={"esphome": {"name": "test"}}) + args = MockArgs() + args.show_secrets = True + + result = command_config(args, {"wifi": {"password": "hunter2"}}) + + assert result == 0 + output = capfd.readouterr().out + assert "hunter2" in output + assert "\\033[8m" not in output + + def test_choose_upload_log_host_with_string_default() -> None: """Test with a single string default device.""" setup_core() From 6d689e8297c3528acbca1f6908776ec601c84675 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 22:18:09 -0500 Subject: [PATCH 93/96] [core] Suppress deprecation warning for lambda values in tagged fields Lambda values in cv.sensitive(cv.templatable(...)) fields still hit the legacy regex because Lambda isn't a str and therefore doesn't carry the SensitiveStr tag. The field IS tagged correctly; warning the author to add cv.sensitive would be misleading. Still wrap the first line so the user-visible output matches the prior regex. --- esphome/__main__.py | 11 +++++++++-- tests/unit_tests/test_main.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 7049f8025e..f4e9e89e57 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1441,8 +1441,15 @@ def _redact_with_legacy_fallback(output: str) -> str: unmarked: set[str] = set() def _replace(m: re.Match[str]) -> str: - unmarked.add(m.group("key")) - return f"{m.group('key')}: \\033[8m{m.group('val')}\\033[28m" + val = m.group("val") + # Lambda values in a ``cv.sensitive(cv.templatable(...))`` field still + # hit this branch because ``Lambda`` isn't a ``str`` and so doesn't + # carry the ``SensitiveStr`` tag. The field itself IS tagged; warning + # the author to add ``cv.sensitive`` would be misleading. Still wrap + # the first line so the user-visible output matches the prior regex. + if not val.startswith("!lambda"): + unmarked.add(m.group("key")) + return f"{m.group('key')}: \\033[8m{val}\\033[28m" output = _LEGACY_REDACTION_RE.sub(_replace, output) for key in sorted(unmarked): diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 1873988f83..77f852688d 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -388,6 +388,20 @@ def test_redact_with_legacy_fallback__deduplicates_warnings( assert len(password_warnings) == 1 +def test_redact_with_legacy_fallback__suppresses_warning_for_lambda( + caplog: pytest.LogCaptureFixture, +) -> None: + """Lambda values in cv.sensitive(cv.templatable(...)) fields hit the + legacy regex (Lambda isn't str so SensitiveStr tagging doesn't apply), + but the field IS tagged. The warning would mislead the author, so it's + suppressed; the first line still gets wrapped for visual parity.""" + text = ' ssid: !lambda |-\n return "x";\n' + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback(text) + assert "ssid: \\033[8m!lambda |-\\033[28m" in out + assert not any("legacy substring" in rec.message for rec in caplog.records) + + def test_redact_with_legacy_fallback__does_not_match_fragment_in_middle( caplog: pytest.LogCaptureFixture, ) -> None: From bc2a81156886d35b7afbc00b0b2c1252cc0beb76 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 22:23:40 -0500 Subject: [PATCH 94/96] [core] Skip !secret and !lambda in legacy fallback regex Address reviewer feedback: - legacy regex was wrapping password: !secret name and clobbering the dumper's user-friendly !secret round-trip; extend the negative lookahead to skip !secret (and !lambda) values entirely - drop the in-replacement !lambda check now that the lookahead handles it - reword the thread-safety claim in dump() since _SECRET_VALUES / _SECRET_CACHE remain module globals - reword the SensitiveStr representer registration comment to reflect PyYAML's MRO-walked dispatch rather than registration order - tighten the legacy regex comment Adds tests for the !secret and !lambda skip paths. --- esphome/__main__.py | 31 ++++++++++--------------------- esphome/yaml_util.py | 9 ++++----- tests/unit_tests/test_main.py | 22 ++++++++++++++++------ 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index f4e9e89e57..99370064ef 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1420,19 +1420,15 @@ def command_config(args: ArgsProtocol, config: ConfigType) -> int | None: return 0 -# Legacy substring redaction fallback. Catches sensitive-looking fields that -# aren't yet tagged with ``cv.sensitive(...)`` so config dumps don't regress -# for unmigrated/third-party schemas. Each match also logs a one-time -# deprecation warning naming the field; this fallback is slated for removal -# in 2026.12.0 once canonical sensitive fields are migrated. -# -# The negative lookahead ``(?!\\033\[8m)`` skips values already wrapped by the -# SensitiveStr representer, so explicit tagging silences the warning. No -# trailing ``\w*`` after the fragment so the fragment must end the field -# name (preserves the prior regex's matching scope while letting the -# leading ``\w*`` capture the full field name for the warning). +# Legacy substring redaction fallback for unmigrated schemas; removed in +# 2026.12.0 once canonical sensitive fields are tagged. The lookahead skips +# values that already render themselves: ``\033[8m`` (SensitiveStr wrap), +# ``!secret`` (preserves the user-friendly tag), ``!lambda`` (multi-line +# block; first line is structural). Un-anchored on purpose to match the +# prior regex; leading ``\w*`` captures the full field name for warnings. _LEGACY_REDACTION_RE = re.compile( - r"(?P\w*(?:password|key|psk|ssid))\: (?!\\033\[8m)(?P.+)" + r"(?P\w*(?:password|key|psk|ssid))\: " + r"(?!\\033\[8m|!secret\b|!lambda\b)(?P.+)" ) _LEGACY_REDACTION_REMOVAL = "2026.12.0" @@ -1441,15 +1437,8 @@ def _redact_with_legacy_fallback(output: str) -> str: unmarked: set[str] = set() def _replace(m: re.Match[str]) -> str: - val = m.group("val") - # Lambda values in a ``cv.sensitive(cv.templatable(...))`` field still - # hit this branch because ``Lambda`` isn't a ``str`` and so doesn't - # carry the ``SensitiveStr`` tag. The field itself IS tagged; warning - # the author to add ``cv.sensitive`` would be misleading. Still wrap - # the first line so the user-visible output matches the prior regex. - if not val.startswith("!lambda"): - unmarked.add(m.group("key")) - return f"{m.group('key')}: \\033[8m{val}\\033[28m" + unmarked.add(m.group("key")) + return f"{m.group('key')}: \\033[8m{m.group('val')}\\033[28m" output = _LEGACY_REDACTION_RE.sub(_replace, output) for key in sorted(unmarked): diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 5355d03e9c..2f3d25b4fc 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -819,9 +819,9 @@ def dump(dict_, show_secrets=False, sort_keys=False): _SECRET_VALUES.clear() _SECRET_CACHE.clear() - # Per-call subclass so the redaction flag lives on the dumper class itself, - # not in module-level mutable state. Cheap (one type object per call), - # encapsulated, and thread-safe by construction. + # Per-call subclass so the redaction flag doesn't leak across calls. + # (``_SECRET_VALUES`` / ``_SECRET_CACHE`` remain module globals; YAML + # processing is single-threaded today, so this isolates only the flag.) class _Dumper(ESPHomeDumper): _redact_sensitive = not show_secrets @@ -1101,8 +1101,7 @@ ESPHomeDumper.add_multi_representer( ) ESPHomeDumper.add_multi_representer(bool, ESPHomeDumper.represent_bool) ESPHomeDumper.add_multi_representer(str, ESPHomeDumper.represent_stringify) -# Registered after ``str`` so ``add_multi_representer``'s MRO lookup prefers -# the more specific ``SensitiveStr`` representer over the bare-``str`` one. +# MRO-walked dispatch; SensitiveStr's own entry wins over the str one. ESPHomeDumper.add_multi_representer(SensitiveStr, ESPHomeDumper.represent_sensitive) ESPHomeDumper.add_multi_representer(int, ESPHomeDumper.represent_int) ESPHomeDumper.add_multi_representer(float, ESPHomeDumper.represent_float) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 77f852688d..1c58e9b294 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -388,17 +388,27 @@ def test_redact_with_legacy_fallback__deduplicates_warnings( assert len(password_warnings) == 1 -def test_redact_with_legacy_fallback__suppresses_warning_for_lambda( +def test_redact_with_legacy_fallback__skips_lambda_values( caplog: pytest.LogCaptureFixture, ) -> None: - """Lambda values in cv.sensitive(cv.templatable(...)) fields hit the - legacy regex (Lambda isn't str so SensitiveStr tagging doesn't apply), - but the field IS tagged. The warning would mislead the author, so it's - suppressed; the first line still gets wrapped for visual parity.""" + """``!lambda`` first line is structural, body is unreachable by a + single-line regex anyway, and tagged fields shouldn't trigger a warning.""" text = ' ssid: !lambda |-\n return "x";\n' with caplog.at_level(logging.WARNING, logger="esphome.__main__"): out = _redact_with_legacy_fallback(text) - assert "ssid: \\033[8m!lambda |-\\033[28m" in out + assert out == text + assert not any("legacy substring" in rec.message for rec in caplog.records) + + +def test_redact_with_legacy_fallback__skips_secret_references( + caplog: pytest.LogCaptureFixture, +) -> None: + """``!secret name`` is the dumper's user-friendly representation; the + name isn't the secret, so wrapping it would clobber the round-trip.""" + text = " password: !secret wifi_password\n" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback(text) + assert out == text assert not any("legacy substring" in rec.message for rec in caplog.records) From 14a5ee87fe53e8b9aeaa2589d7e566d9043f36b0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 22:34:59 -0500 Subject: [PATCH 95/96] [core] Delegate non-redact path in represent_sensitive Hand off to represent_stringify for the !secret + plain-str branches instead of duplicating the is_secret check and scalar emission. --- esphome/yaml_util.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 2f3d25b4fc..bfe1fb0136 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -1014,21 +1014,18 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_scalar(tag="tag:yaml.org,2002:str", value=str(value)) def represent_sensitive(self, value: SensitiveStr) -> yaml.ScalarNode: - # ``!secret`` references win: keep the original representation so the - # dumped YAML round-trips back to ``!secret name`` instead of leaking - # the resolved value. - if is_secret(value): - return self.represent_secret(value) - if self._redact_sensitive: - # Emit the conceal sequence as literal ``\033`` text (not actual - # ESC bytes) so the dump matches the previous regex-based output - # and downstream consumers like device-builder, which match - # ``\033[8m...\033[28m`` against the rendered text, keep working. + # Only the redact-and-not-a-secret branch is unique to sensitive + # values; otherwise let ``represent_stringify`` handle ``!secret`` + # precedence and the plain-str fallthrough. Conceal sequence is + # emitted as literal ``\033`` text (not actual ESC bytes) so the + # output matches the prior regex format and device-builder's + # ``\033[8m...\033[28m`` parser keeps working. + if self._redact_sensitive and not is_secret(value): return self.represent_scalar( tag="tag:yaml.org,2002:str", value=f"\\033[8m{value}\\033[28m", ) - return self.represent_scalar(tag="tag:yaml.org,2002:str", value=str(value)) + return self.represent_stringify(value) # pylint: disable=arguments-renamed def represent_bool(self, value): From 4c6437b825508d8ffb8040717d87da550d2aa246 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 26 May 2026 22:46:41 -0500 Subject: [PATCH 96/96] [core] Anchor legacy redaction regex to avoid false-positive warnings The unanchored leading \w* could match fields like 'monkey:' (via the 'key' fragment), naming a non-sensitive field in the deprecation warning. Restrict the fragment to either start the name or follow '_', so warnings only fire when there's an actual sensitive-shaped field the author can migrate. --- esphome/__main__.py | 7 ++++--- tests/unit_tests/test_main.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 99370064ef..9bd19e517a 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1424,10 +1424,11 @@ def command_config(args: ArgsProtocol, config: ConfigType) -> int | None: # 2026.12.0 once canonical sensitive fields are tagged. The lookahead skips # values that already render themselves: ``\033[8m`` (SensitiveStr wrap), # ``!secret`` (preserves the user-friendly tag), ``!lambda`` (multi-line -# block; first line is structural). Un-anchored on purpose to match the -# prior regex; leading ``\w*`` captures the full field name for warnings. +# block; first line is structural). The fragment must either start the +# field name or follow ``_`` so the warning names a real field; this avoids +# false positives like ``monkey:`` matching the ``key`` fragment. _LEGACY_REDACTION_RE = re.compile( - r"(?P\w*(?:password|key|psk|ssid))\: " + r"(?P\b(?:\w+_)?(?:password|key|psk|ssid))\: " r"(?!\\033\[8m|!secret\b|!lambda\b)(?P.+)" ) _LEGACY_REDACTION_REMOVAL = "2026.12.0" diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 1c58e9b294..26b550669f 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -424,6 +424,18 @@ def test_redact_with_legacy_fallback__does_not_match_fragment_in_middle( assert not any("legacy substring" in rec.message for rec in caplog.records) +def test_redact_with_legacy_fallback__does_not_match_fragment_as_suffix( + caplog: pytest.LogCaptureFixture, +) -> None: + """Fragment must start the name or follow ``_``; ``monkey:`` shouldn't + fire a 'legacy heuristic' warning because there's no sensitive field + here — the user has nothing to migrate.""" + with caplog.at_level(logging.WARNING, logger="esphome.__main__"): + out = _redact_with_legacy_fallback("monkey: 1234\n") + assert "\\033[8m" not in out + assert not any("legacy substring" in rec.message for rec in caplog.records) + + def test_command_config__invokes_legacy_fallback_when_redacting( tmp_path: Path, capfd: CaptureFixture[str] ) -> None: