From 3926612281789284df820f21f159f8cf1bb24969 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 15:46:17 -0400 Subject: [PATCH 01/41] [core] Fix use-after-free when deleting a running StaticTask (#19048) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../micro_wake_word/micro_wake_word.cpp | 4 +-- .../mixer/speaker/mixer_speaker.cpp | 4 +-- .../resampler/speaker/resampler_speaker.cpp | 4 +-- .../speaker/media_player/audio_pipeline.cpp | 11 +++++-- esphome/core/static_task.cpp | 30 ++++++++++++++----- esphome/core/static_task.h | 17 +++++++---- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 3dadb78077..cebfe8e791 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -446,9 +446,9 @@ void MicroWakeWord::loop() { xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STOPPING); } - if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & EventGroupBits::TASK_STOPPED) && this->inference_task_.deallocate()) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 6128dc3767..0b79010773 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -382,8 +382,8 @@ void MixerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, MIXER_TASK_STATE_STOPPING); } - if (event_group_bits & MIXER_TASK_STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS); this->all_stopped_since_ms_ = 0; diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index f1ebd180cc..edda00ae06 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -153,8 +153,8 @@ void ResamplerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::STATE_STOPPING); } - if (event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::ALL_BITS); } diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index 010f0c50b3..c286a9d7d6 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -202,8 +202,15 @@ AudioPipelineState AudioPipeline::process_state() { if (!this->is_playing_) { // The tasks have been stopped for two ``process_state`` calls in a row, so delete the tasks if (this->read_task_.is_created() || this->decode_task_.is_created()) { - this->read_task_.deallocate(); - this->decode_task_.deallocate(); + // Both are attempted every time; a task that is still running on the other core is freed by a + // subsequent call, and freeing an already freed task succeeds without doing anything + bool read_task_freed = this->read_task_.deallocate(); + bool decode_task_freed = this->decode_task_.deallocate(); + if (!read_task_freed || !decode_task_freed) { + // A task is still running on the other core, so keep the pipeline in its current state and try + // again on the next call + return AudioPipelineState::PLAYING; + } if (this->hard_stop_) { // Stop command was sent, so immediately end the playback this->speaker_->stop(); diff --git a/esphome/core/static_task.cpp b/esphome/core/static_task.cpp index 4cfead44c2..4301108315 100644 --- a/esphome/core/static_task.cpp +++ b/esphome/core/static_task.cpp @@ -40,16 +40,31 @@ bool StaticTask::create(TaskFunction_t fn, const char *name, uint32_t stack_size return true; } -void StaticTask::destroy() { - if (this->handle_ != nullptr) { - TaskHandle_t handle = this->handle_; - this->handle_ = nullptr; - vTaskDelete(handle); +bool StaticTask::destroy() { + if (this->handle_ == nullptr) { + return true; } + + // Suspending takes the task off the ready and event lists, so nothing can schedule it again. It only asks + // the other core to yield though, so the task may still be running on it for a moment. + vTaskSuspend(this->handle_); + if (eTaskGetState(this->handle_) != eSuspended) { + // The task is still running on the other core and using its stack. Deleting it now would only put it on + // the termination list and return, so the caller has to try again once it has been swapped out. + return false; + } + + // The task cannot run again, so the delete completes right away instead of being left to the idle task. + TaskHandle_t handle = this->handle_; + this->handle_ = nullptr; + vTaskDelete(handle); + return true; } -void StaticTask::deallocate() { - this->destroy(); +bool StaticTask::deallocate() { + if (!this->destroy()) { + return false; + } if (this->stack_buffer_ != nullptr) { RAMAllocator allocator(this->use_psram_ ? RAMAllocator::ALLOC_EXTERNAL : RAMAllocator::ALLOC_INTERNAL); @@ -57,6 +72,7 @@ void StaticTask::deallocate() { this->stack_buffer_ = nullptr; this->stack_size_ = 0; } + return true; } } // namespace esphome diff --git a/esphome/core/static_task.h b/esphome/core/static_task.h index 5fd5b38f9e..e2996abeda 100644 --- a/esphome/core/static_task.h +++ b/esphome/core/static_task.h @@ -11,6 +11,7 @@ namespace esphome { /** Helper for FreeRTOS static task management. * Bundles TaskHandle_t, StaticTask_t, and the stack buffer into one object with create/destroy methods. + * Call destroy() and deallocate() from another task: a task cannot free the stack it is still running on. */ class StaticTask { public: @@ -23,7 +24,7 @@ class StaticTask { /// @brief Allocate stack and create task. /// @param fn Task function /// @param name Task name (for debug) - /// @param stack_size Stack size in StackType_t words + /// @param stack_size Stack size in bytes (StackType_t is a byte on ESP-IDF) /// @param param Parameter passed to task function /// @param priority FreeRTOS task priority /// @param use_psram If true, allocate stack in PSRAM; otherwise internal RAM @@ -31,11 +32,17 @@ class StaticTask { bool create(TaskFunction_t fn, const char *name, uint32_t stack_size, void *param, UBaseType_t priority, bool use_psram); - /// @brief Delete the task but keep the stack buffer allocated for reuse by a subsequent create() call. - void destroy(); + /// @brief Delete the task, keeping the stack buffer allocated for reuse by a subsequent create() call. + /// The task must have finished its work and parked itself, either suspended or blocked indefinitely: it is + /// suspended here so that it cannot be scheduled again, and it is given no chance to clean up. + /// @return true if the task was deleted; false if it is still running on another core, in which case the + /// caller should try again later. + bool destroy(); - /// @brief Delete the task (if running) and free the stack buffer. - void deallocate(); + /// @brief Delete the task (if created) and free the stack buffer. + /// @return true if the stack buffer was freed; false if the task is still running on another core, in + /// which case the caller should try again later. + bool deallocate(); protected: TaskHandle_t handle_{nullptr}; From 4ab9298ab3eedddbd45507785b4d2453ae867bba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:11:36 +0200 Subject: [PATCH 02/41] Bump esptool from 5.3.1 to 5.4.0 (#19023) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd3f7446f3..dfddbed00b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ tzlocal==5.4.4 # from time tzdata>=2026.3 # from time pyserial==3.5 platformio==6.1.19 -esptool==5.3.1 +esptool==5.4.0 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi From 5bb112f407e8edac9576a7eea1eafb9d94cb1f47 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:04:34 -0400 Subject: [PATCH 03/41] [audio][i2s_audio][micro_wake_word][microphone][mixer][resampler][speaker] Replace use_count() checks with lock and null test (#19046) --- esphome/components/audio/audio_reader.cpp | 3 +++ esphome/components/audio/audio_transfer_buffer.cpp | 12 ++++++------ .../i2s_audio/speaker/i2s_audio_speaker.cpp | 4 ++-- .../components/micro_wake_word/micro_wake_word.cpp | 2 +- esphome/components/microphone/microphone_source.h | 2 +- esphome/components/mixer/speaker/mixer_speaker.cpp | 12 ++++++------ .../resampler/speaker/resampler_speaker.cpp | 6 +++--- .../speaker/media_player/audio_pipeline.cpp | 12 +++++++----- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/esphome/components/audio/audio_reader.cpp b/esphome/components/audio/audio_reader.cpp index 4678ed548c..e69f33ac2d 100644 --- a/esphome/components/audio/audio_reader.cpp +++ b/esphome/components/audio/audio_reader.cpp @@ -58,6 +58,9 @@ esp_err_t AudioReader::add_sink(const std::weak_ptr &ou if (current_audio_file_ != nullptr) { // A transfer buffer isn't ncessary for a local file this->file_ring_buffer_ = output_ring_buffer.lock(); + if (this->file_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; + } return ESP_OK; } diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index a611549e58..01fd4bb68a 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -51,14 +51,14 @@ void AudioTransferBuffer::increase_buffer_length(size_t bytes) { this->buffer_le void AudioTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } } void AudioSinkTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } #ifdef USE_SPEAKER @@ -69,7 +69,7 @@ void AudioSinkTransferBuffer::clear_buffered_data() { } bool AudioTransferBuffer::has_buffered_data() const { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); @@ -144,7 +144,7 @@ size_t AudioSourceTransferBuffer::transfer_data_from_source(TickType_t ticks_to_ size_t bytes_to_read = AudioTransferBuffer::free(); size_t bytes_read = 0; if (bytes_to_read > 0) { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_read = this->ring_buffer_->read((void *) this->get_buffer_end(), bytes_to_read, ticks_to_wait); } @@ -161,7 +161,7 @@ size_t AudioSinkTransferBuffer::transfer_data_to_sink(TickType_t ticks_to_wait, bytes_written = this->speaker_->play(this->data_start_, this->available(), ticks_to_wait); } else #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_written = this->ring_buffer_->write_without_replacement((void *) this->data_start_, this->available(), ticks_to_wait); } else if (this->sink_callback_ != nullptr) { @@ -186,7 +186,7 @@ bool AudioSinkTransferBuffer::has_buffered_data() const { return (this->speaker_->has_buffered_data() || (this->available() > 0)); } #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1c2eb12904..b78a151ee4 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -218,8 +218,8 @@ size_t I2SAudioSpeakerBase::play(const uint8_t *data, size_t length, TickType_t } bool I2SAudioSpeakerBase::has_buffered_data() const { - if (this->audio_ring_buffer_.use_count() > 0) { - std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + if (temp_ring_buffer != nullptr) { return temp_ring_buffer->available() > 0; } return false; diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index cebfe8e791..cf239be696 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -129,7 +129,7 @@ void MicroWakeWord::setup() { return; } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (this->ring_buffer_.use_count() > 1) { + if (temp_ring_buffer != nullptr) { // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task // to drain it - reset() is a consumer operation and must run on the inference task's thread. // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. diff --git a/esphome/components/microphone/microphone_source.h b/esphome/components/microphone/microphone_source.h index 7be3b8cdb5..d7a3352432 100644 --- a/esphome/components/microphone/microphone_source.h +++ b/esphome/components/microphone/microphone_source.h @@ -48,7 +48,7 @@ class MicrophoneSource final { template void add_data_callback(F &&data_callback) { this->mic_->add_data_callback([this, data_callback](const std::vector &data) { if (this->enabled_ || this->passive_) { - if (this->processed_samples_.use_count() == 0) { + if (this->processed_samples_ == nullptr) { // Create vector if its unused this->processed_samples_ = std::make_shared>(); } diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 0b79010773..ef21da65c5 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -218,7 +218,7 @@ size_t SourceSpeaker::play(const uint8_t *data, size_t length, TickType_t ticks_ } size_t bytes_written = 0; std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer.use_count() > 0) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); if (bytes_written > 0) { @@ -250,14 +250,14 @@ esp_err_t SourceSpeaker::start_() { // avoids unnecessary single-frame splices. const size_t ring_buffer_size = (this->audio_stream_info_.ms_to_bytes(this->buffer_duration_ms_) / bytes_per_frame) * bytes_per_frame; - if (this->audio_source_.use_count() == 0) { + if (this->audio_source_ == nullptr) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); this->ring_buffer_ = temp_ring_buffer; } - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { return ESP_ERR_NO_MEM; } @@ -278,7 +278,7 @@ void SourceSpeaker::stop() { this->send_command_(SOURCE_SPEAKER_COMMAND_STOP); } void SourceSpeaker::finish() { this->send_command_(SOURCE_SPEAKER_COMMAND_FINISH); } bool SourceSpeaker::has_buffered_data() const { - return ((this->audio_source_.use_count() > 0) && this->audio_source_->has_buffered_data()); + return ((this->audio_source_ != nullptr) && this->audio_source_->has_buffered_data()); } void SourceSpeaker::set_mute_state(bool mute_state) { @@ -496,7 +496,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (speaker->is_running() && !speaker->get_pause_state()) { // Speaker is running and not paused, so it possibly can provide audio data std::shared_ptr audio_source = speaker->get_audio_source().lock(); - if (audio_source.use_count() == 0) { + if (audio_source == nullptr) { // No audio source allocated, so skip processing this speaker continue; } diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index edda00ae06..16d2d5dc9e 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -235,7 +235,7 @@ size_t ResamplerSpeaker::play(const uint8_t *data, size_t length, TickType_t tic bytes_written = this->output_speaker_->play(data, length, ticks_to_wait); } else { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); } else { @@ -299,7 +299,7 @@ bool ResamplerSpeaker::has_buffered_data() const { bool has_ring_buffer_data = false; if (this->requires_resampling_()) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { has_ring_buffer_data = (temp_ring_buffer->available() > 0); } } @@ -342,7 +342,7 @@ void ResamplerSpeaker::resample_task(void *params) { std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( this_resampler->audio_stream_info_.ms_to_bytes(this_resampler->buffer_duration_ms_)); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { this_resampler->ring_buffer_ = temp_ring_buffer; diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index c286a9d7d6..509984cfa2 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -322,17 +322,17 @@ void AudioPipeline::read_task(void *params) { if (err == ESP_OK) { size_t file_ring_buffer_size = this_pipeline->buffer_size_; - std::shared_ptr temp_ring_buffer; + std::shared_ptr temp_ring_buffer = this_pipeline->raw_file_ring_buffer_.lock(); - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(file_ring_buffer_size); this_pipeline->raw_file_ring_buffer_ = temp_ring_buffer; } - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { - reader->add_sink(this_pipeline->raw_file_ring_buffer_); + err = reader->add_sink(temp_ring_buffer); } } @@ -403,7 +403,9 @@ void AudioPipeline::decode_task(void *params) { make_unique(this_pipeline->transfer_buffer_size_, this_pipeline->transfer_buffer_size_); esp_err_t err = decoder->start(this_pipeline->current_audio_file_type_); - decoder->add_source(this_pipeline->raw_file_ring_buffer_); + if (err == ESP_OK) { + err = decoder->add_source(this_pipeline->raw_file_ring_buffer_); + } if (err != ESP_OK) { // Send specific error message From 006f31af9308fd85212cec8b5a9816273608dbba Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:05:02 -0400 Subject: [PATCH 04/41] [i2s_audio] Fix spurious driver failure (#19045) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index b78a151ee4..1382a87046 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -118,21 +118,24 @@ void I2SAudioSpeakerBase::loop() { break; } + // Still starting up or winding down from a previous run + if ((this->tx_handle_ != nullptr) || (this->speaker_task_handle_ != nullptr)) { + break; + } + if (this->start_i2s_driver(this->audio_stream_info_) != ESP_OK) { ESP_LOGE(TAG, "Driver failed to start; retrying in 1 second"); this->status_momentary_error("driver-failure", 1000); break; } - if (this->speaker_task_handle_ == nullptr) { - xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, - &this->speaker_task_handle_); + xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, + &this->speaker_task_handle_); - if (this->speaker_task_handle_ == nullptr) { - ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); - this->status_momentary_error("task-failure", 1000); - this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt - } + if (this->speaker_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); + this->status_momentary_error("task-failure", 1000); + this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt } break; case speaker::STATE_RUNNING: // Intentional fallthrough From 8f511a365a471d1614e7578a03ceb3c0dbc4470f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:07:06 +0200 Subject: [PATCH 05/41] [noise] Bump noise-c to 0.1.26 and libsodium to 1.10021.8 (#19030) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 4de706120e..d17ebf235e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.24") + cg.add_library("esphome/noise-c", "0.1.26") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.6") + cg.add_library("esphome/libsodium", "1.10021.8") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 779a05e7de..738773d1b5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.24 ; used by noise (api, ota) + esphome/noise-c@0.1.26 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 4f7f5a4a4c..00f22ca138 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.24"] + assert libs == ["esphome/noise-c @ 0.1.26"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.24", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 0.1.26", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.24"] + assert cls.calls == ["esphome/noise-c @ 0.1.26"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.24"] is None + assert compats["esphome/noise-c @ 0.1.26"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 14c52dda8d..b03bff19a2 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 6c5ab89d5f818ac501855479ea776984c5d3f16a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:08:16 +0200 Subject: [PATCH 06/41] [esphome][core] Give a lost OTA chunk ack time to be retransmitted (#19041) --- esphome/components/esphome/ota/ota_esphome.cpp | 5 ++++- esphome/espota2.py | 9 ++++++--- tests/unit_tests/test_espota2.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 1005ed214b..f853ed6a2d 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -41,7 +41,10 @@ const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake -static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer +// Milliseconds for data transfer. Covers the lwIP retransmit run seen in +// practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits +// longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries +static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/espota2.py b/esphome/espota2.py index ce403c398d..c683ffa323 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -96,6 +96,10 @@ UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # across the addresses on top of that. EXTRA_UPLOAD_ATTEMPTS = 2 UPLOAD_RETRY_DELAY = 5.0 +# Data phase timeout; must stay longer than the device's OTA_SOCKET_TIMEOUT_DATA +# (105 s) so a stalled session is gone before a retry, and long enough for lwIP +# to get a lost chunk ack through after the retransmit run seen in practice +DATA_PHASE_TIMEOUT = 160.0 _LOGGER = logging.getLogger(__name__) @@ -694,8 +698,7 @@ def perform_ota( _LOGGER.info("Handshake complete") - # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures - sock.settimeout(90.0) + sock.settimeout(DATA_PHASE_TIMEOUT) if extended_proto: send_check(sock, ota_type, "ota type") @@ -854,7 +857,7 @@ def run_ota_impl_( # clean up a half-open connection (its handshake watchdog runs at 20s); # moving on to the next address family stays immediate. Known limitation: # a silent mid-transfer drop with no reset can wedge the device until its - # 90s data timeout, which outlasts this budget; the retries target the + # 105s data timeout, which outlasts this budget; the retries target the # common failures where the device resets or closes the link promptly. total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 8867e2c215..2d65e8e079 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -416,6 +416,9 @@ def test_perform_ota_no_auth( "Update took 14.00 seconds (prepare 2.00, upload 5.00, commit 7.00)" in caplog.text ) + # The data phase timeout must outlast the device's 105 s data timeout + mock_socket.settimeout.assert_any_call(espota2.DATA_PHASE_TIMEOUT) + assert espota2.DATA_PHASE_TIMEOUT > 105.0 @pytest.mark.usefixtures("mock_time") From b947094f45f7bc8b193db6a75b732c9bdbcce41b Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:31:18 -0400 Subject: [PATCH 07/41] [sendspin] Add codec preference list to the media source (#19047) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/sendspin/__init__.py | 26 ++++-- .../sendspin/media_source/__init__.py | 31 +++++++ .../sendspin/test_media_source.py | 90 +++++++++++++++++++ .../sendspin/common-media_source.yaml | 1 + 4 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 tests/component_tests/sendspin/test_media_source.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 570fd3fadd..8ef11a7f90 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -30,6 +30,7 @@ CONF_SENDSPIN_ID = "sendspin_id" CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" +CONF_CODECS = "codecs" # Matches ARTWORK_MAX_SLOTS in sendspin-cpp. MAX_ARTWORK_SLOTS = 4 @@ -44,6 +45,20 @@ CODEC_FORMAT_OPUS = SendspinCodecFormat.enum("OPUS") CODEC_FORMAT_PCM = SendspinCodecFormat.enum("PCM") CODEC_FORMAT_UNSUPPORTED = SendspinCodecFormat.enum("UNSUPPORTED") +CODEC_FLAC = "flac" +CODEC_OPUS = "opus" +CODEC_PCM = "pcm" + +CODECS = { + CODEC_FLAC: CODEC_FORMAT_FLAC, + CODEC_OPUS: CODEC_FORMAT_OPUS, + CODEC_PCM: CODEC_FORMAT_PCM, +} + +# Opus only supports 48 kHz audio, so it is left out of the default list at other rates. +DEFAULT_CODECS = [CODEC_FLAC, CODEC_OPUS, CODEC_PCM] +OPUS_SAMPLE_RATE = 48000 + SendspinImageFormat = sendspin_library_ns.enum("SendspinImageFormat", is_class=True) IMAGE_FORMAT_JPEG = SendspinImageFormat.enum("JPEG") IMAGE_FORMAT_PNG = SendspinImageFormat.enum("PNG") @@ -286,16 +301,13 @@ async def to_code(config: ConfigType) -> None: if data.player_support: cg.add_define("USE_SENDSPIN_PLAYER", True) - # Configures the player role. We always assume support for 16 bits per sample mono and stereo FLAC, Opus, and PCM at the configured sample rate - # (with Opus only supported at 48 kHz since that's the only sample rate it supports). Users can configure the specific formats via the Sendspin server + # Configures the player role. Each configured codec is advertised for 16 bits per sample + # mono and stereo at the configured sample rate. The order is a preference order, both for + # the codecs themselves and for stereo over mono. player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - # OPUS only supports 48 kHz audio - codecs = [CODEC_FORMAT_FLAC] - if sample_rate == 48000: - codecs.append(CODEC_FORMAT_OPUS) - codecs.append(CODEC_FORMAT_PCM) + codecs = player_cfg[CONF_CODECS] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index 6af244d41f..6a9f1f18ba 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -13,11 +13,16 @@ from esphome.cpp_generator import MockObj, TemplateArgsType from esphome.types import ConfigType from .. import ( + CODEC_OPUS, + CODECS, + CONF_CODECS, CONF_DECODE_MEMORY, CONF_FIXED_DELAY, CONF_INITIAL_STATIC_DELAY, CONF_SENDSPIN_ID, + DEFAULT_CODECS, MEMORY_LOCATIONS, + OPUS_SAMPLE_RATE, SendspinHub, register_player_config, request_controller_support, @@ -49,10 +54,32 @@ DisableStaticDelayAdjustmentAction = sendspin_ns.class_( ) +def _resolve_codecs(config: ConfigType) -> ConfigType: + """Validate the codec preference list, filling in the default when it is not set.""" + sample_rate = config[CONF_SAMPLE_RATE] + if (codecs := config.get(CONF_CODECS)) is None: + config[CONF_CODECS] = [ + codec + for codec in DEFAULT_CODECS + if codec != CODEC_OPUS or sample_rate == OPUS_SAMPLE_RATE + ] + return config + + if len(set(codecs)) != len(codecs): + raise cv.Invalid("Each codec may only be listed once", path=[CONF_CODECS]) + if CODEC_OPUS in codecs and sample_rate != OPUS_SAMPLE_RATE: + raise cv.Invalid( + f"Codec '{CODEC_OPUS}' requires a {CONF_SAMPLE_RATE} of {OPUS_SAMPLE_RATE}", + path=[CONF_CODECS], + ) + return config + + def _register(config: ConfigType) -> ConfigType: request_controller_support() register_player_config( { + CONF_CODECS: config[CONF_CODECS], CONF_SAMPLE_RATE: config[CONF_SAMPLE_RATE], CONF_BUFFER_SIZE: config[CONF_BUFFER_SIZE], CONF_INITIAL_STATIC_DELAY: config[CONF_INITIAL_STATIC_DELAY], @@ -85,9 +112,13 @@ CONFIG_SCHEMA = cv.All( min=16000, max=96000 ), cv.Optional(CONF_DECODE_MEMORY): cv.one_of(*MEMORY_LOCATIONS, lower=True), + cv.Optional(CONF_CODECS): cv.All( + cv.ensure_list(cv.enum(CODECS, lower=True)), cv.Length(min=1) + ), } ), cv.only_on_esp32, + _resolve_codecs, _register, ) diff --git a/tests/component_tests/sendspin/test_media_source.py b/tests/component_tests/sendspin/test_media_source.py new file mode 100644 index 0000000000..6c2f79198d --- /dev/null +++ b/tests/component_tests/sendspin/test_media_source.py @@ -0,0 +1,90 @@ +"""Validation tests for the sendspin media_source platform. + +These cover the codec preference list, whose rejection branches a compile test +cannot reach: a `test*.yaml` can only assert that a configuration is accepted. +""" + +from typing import Any + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import CONF_CODECS, _get_data +from esphome.components.sendspin.media_source import CONFIG_SCHEMA +from esphome.const import PlatformFramework +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _media_source_config(**overrides: Any) -> ConfigType: + """Build a minimal valid media source config, allowing field overrides.""" + config: ConfigType = { + "id": "sendspin_media_source", + "sendspin_id": "sendspin_hub", + } + config.update(overrides) + return config + + +def test_default_codecs_at_48_khz(set_core_config: SetCoreConfigCallable) -> None: + """Every codec is advertised when the sample rate suits all of them.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config()) + + assert config[CONF_CODECS] == ["flac", "opus", "pcm"] + + +def test_default_codecs_drop_opus_at_other_rates( + set_core_config: SetCoreConfigCallable, +) -> None: + """Opus only supports 48 kHz, so it leaves the default list at other rates.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config(sample_rate=44100)) + + assert config[CONF_CODECS] == ["flac", "pcm"] + + +def test_configured_order_is_preserved(set_core_config: SetCoreConfigCallable) -> None: + """The list is a preference order, so it reaches the player role as written.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA(_media_source_config(codecs=["pcm", "flac"])) + + assert _get_data().player_config[CONF_CODECS] == ["pcm", "flac"] + + +def test_empty_codec_list_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A player with no codecs at all could never be given a stream.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="length of value must be at least 1"): + CONFIG_SCHEMA(_media_source_config(codecs=[])) + + +def test_duplicate_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A repeated codec has no meaning in a preference order.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="may only be listed once"): + CONFIG_SCHEMA(_media_source_config(codecs=["flac", "flac"])) + + +def test_unknown_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """Only codecs the player role can decode are accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="Unknown value"): + CONFIG_SCHEMA(_media_source_config(codecs=["mp3"])) + + +def test_opus_at_wrong_sample_rate_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + """Asking for Opus at a rate it cannot handle fails rather than silently + dropping the stated preference.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="requires a sample_rate of 48000"): + CONFIG_SCHEMA(_media_source_config(codecs=["opus"], sample_rate=44100)) diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 1977b79c04..0c136fbd43 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,3 +9,4 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal + codecs: [pcm, opus, flac] From 823d79c948eb4474423200d5a251210c31482b68 Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:10:55 +0200 Subject: [PATCH 08/41] [i2s_audio] Keep a start request that arrives while the speaker task stops (#19027) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../components/i2s_audio/speaker/i2s_audio_speaker.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 5e271e671e..1c2eb12904 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -91,7 +91,14 @@ void I2SAudioSpeakerBase::loop() { this->speaker_task_handle_ = nullptr; this->stop_i2s_driver_(); - xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + // ALL_BITS includes COMMAND_START. Take the bits from the clear itself, not from the snapshot at + // the top of loop(): the audio source's task can raise a start at any point above, including + // during stop_i2s_driver_(), and nothing would ever re-issue it. + const EventBits_t bits_before_clear = xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + if (bits_before_clear & SpeakerEventGroupBits::COMMAND_START) { + ESP_LOGD(TAG, "Start requested while stopping; keeping the request"); + xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); + } this->status_clear_error(); this->on_task_stopped(); From 628ebe23ec389d770e822f18de22753c167dff6f Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:13:03 +0200 Subject: [PATCH 09/41] [audio] Do not treat MP3_STREAM_INFO_CHANGED as a fatal decoder error (#19028) Co-authored-by: Claude --- esphome/components/audio/audio_decoder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index fe9ad9c9ad..051395606c 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -313,9 +313,10 @@ FileDecoderState AudioDecoder::decode_mp3_() { this->output_transfer_buffer_->increase_buffer_length( this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); } - } else if (result == micro_mp3::MP3_STREAM_INFO_READY) { - // First successful header parse: capture stream info and resize the output buffer to fit one full frame. - // microMP3 always outputs 16-bit PCM. + } else if (result == micro_mp3::MP3_STREAM_INFO_READY || result == micro_mp3::MP3_STREAM_INFO_CHANGED) { + // Header parsed: capture stream info and resize the output buffer to fit one full frame. + // microMP3 always outputs 16-bit PCM. MP3_STREAM_INFO_CHANGED is handled identically: despite its + // negative value it is documented as recoverable, so it must not reach the catch-all below. this->audio_stream_info_ = audio::AudioStreamInfo(16, this->mp3_decoder_->get_channels(), this->mp3_decoder_->get_sample_rate()); this->free_buffer_required_ = From e7f45a0d315442dcf789997d6e28de72f082e28a Mon Sep 17 00:00:00 2001 From: Ryan Ronnander <61520+ryan-ronnander@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:09:02 -0400 Subject: [PATCH 10/41] [mqtt] Restore brightness flag in light discovery (#18950) --- esphome/components/mqtt/mqtt_light.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/mqtt/mqtt_light.cpp b/esphome/components/mqtt/mqtt_light.cpp index aa47bdf996..a8b52a3839 100644 --- a/esphome/components/mqtt/mqtt_light.cpp +++ b/esphome/components/mqtt/mqtt_light.cpp @@ -67,6 +67,9 @@ void MQTTJSONLightComponent::send_discovery(JsonObject root, mqtt::SendDiscovery if (traits.supports_color_mode(ColorMode::RGB_COLD_WARM_WHITE)) color_modes.add(ESPHOME_F("rgbww")); + if (traits.supports_color_capability(ColorCapability::BRIGHTNESS)) + root[ESPHOME_F("brightness")] = true; + if (traits.supports_color_mode(ColorMode::COLOR_TEMPERATURE) || traits.supports_color_mode(ColorMode::COLD_WARM_WHITE)) { root[MQTT_MIN_MIREDS] = traits.get_min_mireds(); From d5cff6e9dfcdfce156483eecf205e64169a56dee Mon Sep 17 00:00:00 2001 From: AndreKR Date: Tue, 8 Sep 2026 03:13:51 +0200 Subject: [PATCH 11/41] [logger] Fix garbled stack traces (#17939) --- esphome/components/logger/logger_esp32.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index 05fc959ceb..c3d777299d 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -5,6 +5,7 @@ #include #include +#include #ifdef USE_LOGGER_UART_SELECTION_USB_SERIAL_JTAG #include @@ -76,7 +77,11 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { uart_config.parity = UART_PARITY_DISABLE; uart_config.stop_bits = UART_STOP_BITS_1; uart_config.flow_ctrl = UART_HW_FLOWCTRL_DISABLE; +#if SOC_UART_SUPPORT_XTAL_CLK + uart_config.source_clk = UART_SCLK_XTAL; +#else uart_config.source_clk = UART_SCLK_DEFAULT; +#endif uart_param_config(uart_num, &uart_config); // The logger only writes to UART, never reads, so use the minimum RX buffer. // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). From 934086365217965f95c21bda0593b3f2ec960615 Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Mon, 7 Sep 2026 18:27:49 -0700 Subject: [PATCH 12/41] [dallas_temp] filter 85 temp from sensor reset (#17877) Co-authored-by: Samuel Sieb --- esphome/components/dallas_temp/dallas_temp.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esphome/components/dallas_temp/dallas_temp.cpp b/esphome/components/dallas_temp/dallas_temp.cpp index ab4a8c458f..c418362ced 100644 --- a/esphome/components/dallas_temp/dallas_temp.cpp +++ b/esphome/components/dallas_temp/dallas_temp.cpp @@ -6,6 +6,7 @@ namespace esphome::dallas_temp { static const char *const TAG = "dallas.temp.sensor"; static const uint8_t DALLAS_MODEL_DS18S20 = 0x10; +static const uint8_t DALLAS_MODEL_DS18B20 = 0x28; static const uint8_t DALLAS_COMMAND_START_CONVERSION = 0x44; static const uint8_t DALLAS_COMMAND_READ_SCRATCH_PAD = 0xBE; static const uint8_t DALLAS_COMMAND_WRITE_SCRATCH_PAD = 0x4E; @@ -154,7 +155,14 @@ float DallasTemperatureSensor::get_temp_c_() { default: break; } - + // undocumented test for powerup measurement of 85 + // https://github.com/cpetrich/counterfeit_DS18B20#solution-to-the-85-c-problem + if ((this->address_ & 0xff) == DALLAS_MODEL_DS18B20) { + if ((temp == 85 * 16) && (this->scratch_pad_[6] == 0xc)) { + ESP_LOGD(TAG, "dropping reading caused by sensor reset"); + return NAN; + } + } return temp / 16.0f; } From 199acdf5222a923d5c7951af2e0ea632f45ba220 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 7 Sep 2026 18:57:50 -0700 Subject: [PATCH 13/41] [ble_client] Report Established from nodes that never read services (#17920) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/ble_client/automation.h | 34 +++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/esphome/components/ble_client/automation.h b/esphome/components/ble_client/automation.h index 94eeb83b3e..93aae23b6a 100644 --- a/esphome/components/ble_client/automation.h +++ b/esphome/components/ble_client/automation.h @@ -22,6 +22,23 @@ class Automation { static const char *const TAG; }; +// Base for nodes that never read the parent's services. +// The parent releases its services only once every node reports Established, so a node that never +// reports it keeps that memory allocated for the life of the connection. +class BLEClientServicelessNode : public BLEClientNode { + public: + // Final so that Established is always reported on SEARCH_CMPL, before the derived node sees the event. + void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) final { + if (event == ESP_GATTC_SEARCH_CMPL_EVT) + this->node_state = espbt::ClientState::ESTABLISHED; + this->on_gattc_event(event, gattc_if, param); + } + + protected: + // Derived nodes handle GATT events here rather than by overriding the handler above. + virtual void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) {} +}; + // implement on_connect automation. class BLEClientConnectTrigger final : public Trigger<>, public BLEClientNode { public: @@ -61,7 +78,7 @@ class BLEClientDisconnectTrigger final : public Trigger<>, public BLEClientNode } }; -class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientNode { +class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientServicelessNode { public: explicit BLEClientPasskeyRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -71,7 +88,7 @@ class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientN } }; -class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientNode { +class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientPasskeyNotificationTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -82,7 +99,7 @@ class BLEClientPasskeyNotificationTrigger final : public Trigger, publ } }; -class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientNode { +class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientNumericComparisonRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -315,19 +332,17 @@ template class BLEClientRemoveBondAction final : public Action class BLEClientConnectAction final : public Action, public BLEClientNode { +template class BLEClientConnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientConnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { case ESP_GATTC_SEARCH_CMPL_EVT: - this->node_state = espbt::ClientState::ESTABLISHED; this->parent()->run_later([this]() { this->play_next_tuple_(this->var_); }); break; // if the connection is closed, terminate the automation chain. @@ -364,14 +379,13 @@ template class BLEClientConnectAction final : public Action var_{}; }; -template class BLEClientDisconnectAction final : public Action, public BLEClientNode { +template class BLEClientDisconnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientDisconnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { From 442e4a1ec2c70bfc8507aa1cf5f471402806d9b0 Mon Sep 17 00:00:00 2001 From: Davide D M Date: Tue, 8 Sep 2026 03:59:05 +0200 Subject: [PATCH 14/41] [debug] Check reboot source pref on ESP_RST_WDT and guard against empty source (#17537) --- esphome/components/debug/debug_esp32.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esphome/components/debug/debug_esp32.cpp b/esphome/components/debug/debug_esp32.cpp index 969cd840cf..8e1a67224e 100644 --- a/esphome/components/debug/debug_esp32.cpp +++ b/esphome/components/debug/debug_esp32.cpp @@ -66,11 +66,15 @@ const char *DebugComponent::get_reset_reason_(std::spanmake_preference(REBOOT_MAX_LEN, fnv1_hash_extend(fnv1_hash(REBOOT_KEY), App.get_name().c_str())); char reboot_source[REBOOT_MAX_LEN]{}; - if (pref.load(&reboot_source)) { + if (pref.load(&reboot_source) && reboot_source[0] != '\0') { reboot_source[REBOOT_MAX_LEN - 1] = '\0'; snprintf(buf, size, "Reboot request from %s", reboot_source); } else { From c9729244af79e5b36a2b712c2fa5b91efa6a504f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:17:22 +1200 Subject: [PATCH 15/41] [udp] Use cv.invalid for relocated packet_transport options (#19032) --- esphome/components/udp/__init__.py | 16 +++------ tests/unit_tests/components/udp/__init__.py | 0 tests/unit_tests/components/udp/test_init.py | 37 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 tests/unit_tests/components/udp/__init__.py create mode 100644 tests/unit_tests/components/udp/test_init.py diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index a782d875b9..d96a731e9c 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -1,5 +1,4 @@ -from collections.abc import Callable -from typing import Any, NoReturn +from typing import Any from esphome import automation from esphome.automation import Trigger @@ -48,17 +47,10 @@ UDP_SCHEMA = cv.Schema( ) -def is_relocated(option: str) -> Callable[[Any], NoReturn]: - def validator(value: Any) -> NoReturn: - raise cv.Invalid( - f"The '{option}' option should now be configured in the 'packet_transport' component" - ) - - return validator - - RELOCATED = { - cv.Optional(x): is_relocated(x) + cv.Optional(x): cv.invalid( + f"The '{x}' option should now be configured in the 'packet_transport' component" + ) for x in ( CONF_PROVIDERS, CONF_ENCRYPTION, diff --git a/tests/unit_tests/components/udp/__init__.py b/tests/unit_tests/components/udp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/udp/test_init.py b/tests/unit_tests/components/udp/test_init.py new file mode 100644 index 0000000000..5afc92e9c6 --- /dev/null +++ b/tests/unit_tests/components/udp/test_init.py @@ -0,0 +1,37 @@ +"""Tests for the udp component configuration schema.""" + +from __future__ import annotations + +import pytest + +from esphome.components import udp +from esphome.components.packet_transport import ( + CONF_BINARY_SENSORS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_PROVIDERS, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, +) +import esphome.config_validation as cv + + +@pytest.mark.parametrize( + "option", + [ + CONF_PROVIDERS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, + CONF_BINARY_SENSORS, + ], +) +def test_relocated_option_rejected(option: str) -> None: + """Options that moved to packet_transport raise a pointing error.""" + with pytest.raises(cv.Invalid) as exc_info: + udp.CONFIG_SCHEMA({option: True}) + assert ( + f"The '{option}' option should now be configured in the 'packet_transport' component" + in str(exc_info.value) + ) From ca864c22b4c0810e9e4779bfa6d95b3597cc8abe Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:36:04 -0400 Subject: [PATCH 16/41] [tuya] Build without a network component (#18948) --- esphome/components/tuya/tuya.cpp | 17 +++++++++-- .../tuya/test-no-network.bk72xx-ard.yaml | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/components/tuya/test-no-network.bk72xx-ard.yaml diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index 82fb96d787..f9b4fe2453 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -1,10 +1,13 @@ #include "tuya.h" -#include "esphome/components/network/util.h" #include "esphome/core/gpio.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/util.h" +#ifdef USE_NETWORK +#include "esphome/components/network/util.h" +#endif + #ifdef USE_WIFI #include "esphome/components/wifi/wifi_component.h" #endif @@ -22,6 +25,14 @@ static const int MAX_RETRIES = 5; // Max bytes to log for datapoint values (larger values are truncated) static constexpr size_t MAX_DATAPOINT_LOG_BYTES = 16; +static bool network_is_connected() { +#ifdef USE_NETWORK + return network::is_connected(); +#else + return false; +#endif +} + void Tuya::setup() { this->set_interval("heartbeat", 15000, [this] { this->send_empty_command_(TuyaCommandType::HEARTBEAT); }); if (this->status_pin_ != nullptr) { @@ -554,14 +565,14 @@ void Tuya::send_empty_command_(TuyaCommandType command) { } void Tuya::set_status_pin_() { - bool is_network_ready = network::is_connected() && remote_is_connected(); + bool is_network_ready = network_is_connected() && remote_is_connected(); this->status_pin_->digital_write(is_network_ready); } uint8_t Tuya::get_wifi_status_code_() { uint8_t status = 0x02; - if (network::is_connected()) { + if (network_is_connected()) { status = 0x03; // Protocol version 3 also supports specifying when connected to "the cloud" diff --git a/tests/components/tuya/test-no-network.bk72xx-ard.yaml b/tests/components/tuya/test-no-network.bk72xx-ard.yaml new file mode 100644 index 0000000000..64207e94e3 --- /dev/null +++ b/tests/components/tuya/test-no-network.bk72xx-ard.yaml @@ -0,0 +1,29 @@ +# Tuya without any network component (no wifi/ethernet/api), as used on +# serial-only or BLE-only Tuya MCU boards. Regression test for +# https://github.com/esphome/esphome/issues/18942 +substitutions: + status_pin: P6 + +packages: + uart: !include ../../test_build_components/common/uart/bk72xx-ard.yaml + +tuya: + status_pin: ${status_pin} + +binary_sensor: + - platform: tuya + id: tuya_presence + sensor_datapoint: 101 + +sensor: + - platform: tuya + id: tuya_light_intensity + sensor_datapoint: 103 + +number: + - platform: tuya + id: tuya_far_detection + number_datapoint: 109 + min_value: 0 + max_value: 600 + step: 1 From 866ddb6e5729f11e2a3a107fa7145dc596b9a17a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 06:42:30 +0200 Subject: [PATCH 17/41] [core] Skip PlatformIO's private-package authorization probe (#18823) --- esphome/platformio/library.py | 6 ++- esphome/platformio/prefetch.py | 2 + esphome/platformio/runner.py | 14 ++++++- tests/unit_tests/test_platformio_library.py | 19 ++++++++++ tests/unit_tests/test_platformio_prefetch.py | 14 +++++++ tests/unit_tests/test_platformio_runner.py | 40 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 3ff60f8aaa..fb6779b807 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -616,11 +616,15 @@ def _make_registry_client() -> Any: elsewhere, not by the PlatformIO registry. """ from platformio.package.manager._registry import PackageManagerRegistryMixin + from platformio.registry.client import RegistryClient class _Registry(PackageManagerRegistryMixin): def __init__(self) -> None: - self._registry_client = None self.pkg_type = "library" + self._registry_client = RegistryClient() + # The probe sleeps ~500 ms per lookup (see runner.patch_registry_private_packages); + # instance-level so the ESPHome process never patches PlatformIO's class + self._registry_client.allowed_private_packages = lambda: False @staticmethod def is_system_compatible(value: Any, custom_system: Any = None) -> bool: diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 17a06cb9c1..e648192b73 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -951,8 +951,10 @@ def main(argv: list[str]) -> int: """Subprocess entry point: ``prefetch ``.""" from esphome.core import CORE from esphome.log import setup_log + from esphome.platformio.runner import patch_registry_private_packages signal.signal(signal.SIGTERM, _sigterm) + patch_registry_private_packages() raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") try: level = int(raw_level) if raw_level is not None else logging.INFO diff --git a/esphome/platformio/runner.py b/esphome/platformio/runner.py index 9bb2205a90..b9fbdec38d 100644 --- a/esphome/platformio/runner.py +++ b/esphome/platformio/runner.py @@ -2,7 +2,8 @@ Invoked via ``python -m esphome.platformio.runner`` instead of ``python -m platformio`` so that the patches (incremental rebuild -preservation, download retries) apply inside the subprocess. Running +preservation, download retries, skipping the private-package probe) apply +inside the subprocess. Running PlatformIO in a subprocess keeps its ``sys.path`` mutations and other global state from leaking into the ESPHome process. """ @@ -105,6 +106,16 @@ def patch_file_downloader() -> None: FileDownloader.__init__ = patched_init +def patch_registry_private_packages() -> None: + """Skip PlatformIO's private-package probe; it sleeps ~500 ms per lookup. + + ESPHome never uses private packages, so the answer is always False. + """ + from platformio.registry.client import RegistryClient + + RegistryClient.allowed_private_packages = staticmethod(lambda: False) # type: ignore[method-assign] + + _IGNORE_LIB_WARNINGS = "(?:Hash|Update)" # Regex patterns matched against each line of PlatformIO output. Lines that # match are dropped by RedirectText before they reach the parent process. @@ -152,6 +163,7 @@ FILTER_PLATFORMIO_LINES = [ def main() -> int: patch_structhash() patch_file_downloader() + patch_registry_private_packages() # Wrap stdout/stderr with RedirectText before PlatformIO runs: # diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 3bae39b3c1..512c883c37 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -7,6 +7,7 @@ exercised in their own test modules).""" import json import logging from pathlib import Path +from unittest.mock import Mock import pytest @@ -228,6 +229,24 @@ def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): _resolve_registry_version("owner", "pkg", set()) +def test_make_registry_client_skips_private_package_probe(monkeypatch): + """Our client answers the probe locally without patching PlatformIO's class.""" + from platformio.account.client import AccountClient + from platformio.registry.client import RegistryClient + + pio_probe = RegistryClient.__dict__["allowed_private_packages"] + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + client = lib._make_registry_client().get_registry_client_instance() + + assert client.allowed_private_packages() is False + assert RegistryClient.__dict__["allowed_private_packages"] is pio_probe + + def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: """Stub the registry lookup so tests never touch the network.""" monkeypatch.setattr( diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 77490fd861..14c52dda8d 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1225,6 +1225,20 @@ def test_main_runs_prefetch(tmp_path: Path) -> None: mock_prefetch.assert_called_once_with(tmp_path, "testenv") +def test_main_skips_private_package_probe_before_prefetch(tmp_path: Path) -> None: + """The registry probe patch is applied before any package manager runs.""" + order: list[str] = [] + with ( + patch.object(pf, "_prefetch", side_effect=lambda *_: order.append("prefetch")), + patch( + "esphome.platformio.runner.patch_registry_private_packages", + side_effect=lambda: order.append("patch"), + ), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + assert order == ["patch", "prefetch"] + + def test_main_bad_argv_is_a_distinct_exit( caplog: pytest.LogCaptureFixture, ) -> None: diff --git a/tests/unit_tests/test_platformio_runner.py b/tests/unit_tests/test_platformio_runner.py index f375aa457a..007455f45a 100644 --- a/tests/unit_tests/test_platformio_runner.py +++ b/tests/unit_tests/test_platformio_runner.py @@ -6,7 +6,9 @@ from collections.abc import Callable import io import sys from types import ModuleType +from unittest.mock import Mock +from platformio.registry.client import RegistryClient import pytest from esphome.platformio import runner @@ -30,6 +32,7 @@ def _prepare_main( monkeypatch.setattr(sys, "stderr", stream) monkeypatch.setattr(runner, "patch_structhash", lambda: None) monkeypatch.setattr(runner, "patch_file_downloader", lambda: None) + monkeypatch.setattr(runner, "patch_registry_private_packages", lambda: None) platformio = ModuleType("platformio") platformio_main = ModuleType("platformio.__main__") @@ -91,3 +94,40 @@ def test_main_still_filters_a_drained_partial_line( assert runner.main() == 0 assert buf.getvalue() == b"" + + +def test_main_applies_registry_private_packages_patch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The probe is patched before PlatformIO runs.""" + order: list[str] = [] + _prepare_main(monkeypatch, lambda: order.append("pio") or 0) + monkeypatch.setattr( + runner, "patch_registry_private_packages", lambda: order.append("patch") + ) + + assert runner.main() == 0 + assert order == ["patch", "pio"] + + +# Snapshot PlatformIO's own probe at import, before any test can patch it +_PIO_PROBE = RegistryClient.__dict__["allowed_private_packages"] + + +def test_patch_registry_private_packages_skips_account_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Answers False without touching the account client.""" + from platformio.account.client import AccountClient + + monkeypatch.setattr(RegistryClient, "allowed_private_packages", _PIO_PROBE) + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + runner.patch_registry_private_packages() + + assert RegistryClient.allowed_private_packages() is False + assert RegistryClient().allowed_private_packages() is False From f8a4cfa945ef765e469daa03edbe263346a03154 Mon Sep 17 00:00:00 2001 From: Gytis Date: Tue, 8 Sep 2026 08:31:30 +0200 Subject: [PATCH 18/41] [lvgl] Add missing label dependency to qrcode, keyboard and tabview (#18387) --- esphome/components/lvgl/widgets/keyboard.py | 3 +- esphome/components/lvgl/widgets/qrcode.py | 3 +- esphome/components/lvgl/widgets/tabview.py | 3 +- .../lvgl/config/keyboard_no_label.yaml | 32 +++++++++++++++++ .../lvgl/config/qrcode_no_label.yaml | 34 ++++++++++++++++++ .../lvgl/config/tabview_no_label.yaml | 35 +++++++++++++++++++ .../lvgl/test_widget_label_dependency.py | 32 +++++++++++++++++ 7 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/config/keyboard_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/qrcode_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/tabview_no_label.yaml create mode 100644 tests/component_tests/lvgl/test_widget_label_dependency.py diff --git a/esphome/components/lvgl/widgets/keyboard.py b/esphome/components/lvgl/widgets/keyboard.py index bcd2d2ae59..65516513a6 100644 --- a/esphome/components/lvgl/widgets/keyboard.py +++ b/esphome/components/lvgl/widgets/keyboard.py @@ -15,6 +15,7 @@ from ..defines import ( from ..types import LvCompound, LvType from . import Widget, WidgetType, get_widgets from .buttonmatrix import CONF_BUTTONMATRIX +from .label import CONF_LABEL from .textarea import CONF_TEXTAREA, lv_textarea_t CONF_KEYBOARD = "keyboard" @@ -49,7 +50,7 @@ class KeyboardType(WidgetType): ) def get_uses(self): - return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX + return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX, CONF_LABEL async def to_code(self, w: Widget, config: dict): add_lv_use("KEY_LISTENER") diff --git a/esphome/components/lvgl/widgets/qrcode.py b/esphome/components/lvgl/widgets/qrcode.py index df76ab6bb0..59af9168aa 100644 --- a/esphome/components/lvgl/widgets/qrcode.py +++ b/esphome/components/lvgl/widgets/qrcode.py @@ -10,6 +10,7 @@ from ..types import lv_obj_t from . import Widget, WidgetType from .canvas import CONF_CANVAS from .img import CONF_IMAGE +from .label import CONF_LABEL CONF_QRCODE = "qrcode" CONF_DARK_COLOR = "dark_color" @@ -41,7 +42,7 @@ class QrCodeType(WidgetType): ) def get_uses(self): - return CONF_CANVAS, CONF_IMAGE + return CONF_CANVAS, CONF_IMAGE, CONF_LABEL async def to_code(self, w: Widget, config): await w.set_property( diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index ee252ecf0b..77c88c48ff 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -28,6 +28,7 @@ from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t, lv_obj_t_ptr from . import Widget, WidgetType, add_widgets, get_widgets, set_obj_properties from .button import button_spec from .buttonmatrix import CONF_BUTTONMATRIX, buttonmatrix_spec +from .label import CONF_LABEL from .obj import obj_spec CONF_TABVIEW = "tabview" @@ -74,7 +75,7 @@ class TabviewType(WidgetType): ) def get_uses(self): - return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON + return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON, CONF_LABEL async def to_code(self, w: Widget, config: dict): await w.set_property( diff --git a/tests/component_tests/lvgl/config/keyboard_no_label.yaml b/tests/component_tests/lvgl/config/keyboard_no_label.yaml new file mode 100644 index 0000000000..7a45a537d3 --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_no_label.yaml @@ -0,0 +1,32 @@ +esphome: + name: test-keyboard-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - keyboard: + id: keyboard_widget diff --git a/tests/component_tests/lvgl/config/qrcode_no_label.yaml b/tests/component_tests/lvgl/config/qrcode_no_label.yaml new file mode 100644 index 0000000000..8bb1aafdd6 --- /dev/null +++ b/tests/component_tests/lvgl/config/qrcode_no_label.yaml @@ -0,0 +1,34 @@ +esphome: + name: test-qrcode-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - qrcode: + id: qr_widget + size: 100 + text: "esphome.io" diff --git a/tests/component_tests/lvgl/config/tabview_no_label.yaml b/tests/component_tests/lvgl/config/tabview_no_label.yaml new file mode 100644 index 0000000000..a3c16ab347 --- /dev/null +++ b/tests/component_tests/lvgl/config/tabview_no_label.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-tabview-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - tabview: + id: tabview_widget + tabs: + - name: "Tab 1" + id: tab_1 diff --git a/tests/component_tests/lvgl/test_widget_label_dependency.py b/tests/component_tests/lvgl/test_widget_label_dependency.py new file mode 100644 index 0000000000..9d3e24c8c5 --- /dev/null +++ b/tests/component_tests/lvgl/test_widget_label_dependency.py @@ -0,0 +1,32 @@ +"""Widgets whose LVGL C implementation creates or references labels +internally (tab titles, key legends, the QR canvas fallback) must declare +the label dependency in ``get_uses()``. Otherwise a config that contains +no ``label`` widget of its own compiles LVGL without ``LV_USE_LABEL`` and +fails at C compile time with undefined ``lv_label_*`` symbols. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.lvgl import defines as df + + +@pytest.mark.parametrize( + "yaml_file", + [ + "qrcode_no_label.yaml", + "keyboard_no_label.yaml", + "tabview_no_label.yaml", + ], +) +def test_label_less_config_enables_lv_use_label( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + yaml_file: str, +) -> None: + generate_main(component_config_path(yaml_file)) + assert "LV_USE_LABEL" in df.get_defines() From c3ce07755f32292af3da6466aa1fc4a2cfeca07d Mon Sep 17 00:00:00 2001 From: Bryan Li Date: Mon, 7 Sep 2026 23:36:42 -0700 Subject: [PATCH 19/41] [rf_bridge] Fix bucket sniffing with Portisch firmware (#17683) Co-authored-by: Bryan Li Co-authored-by: Claude Fable 5 --- esphome/components/rf_bridge/rf_bridge.cpp | 109 +++++++++++++++++---- esphome/components/rf_bridge/rf_bridge.h | 13 +++ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/esphome/components/rf_bridge/rf_bridge.cpp b/esphome/components/rf_bridge/rf_bridge.cpp index 549cce72df..a4a4da5d8c 100644 --- a/esphome/components/rf_bridge/rf_bridge.cpp +++ b/esphome/components/rf_bridge/rf_bridge.cpp @@ -18,6 +18,16 @@ void RFBridgeComponent::ack_() { } bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { + if (this->bucket_frame_candidate_ && byte == RF_CODE_START) { + // A queued next frame proves the trailing 0x55 really was the bucket + // frame's terminator: Portisch builds pulse entries from alternating + // signal edges, so the two level bits inside one pulse byte are always + // opposite — 0xAA (two high-level nibbles) cannot occur in pulse data. + // Finalize before this byte starts the new frame, so back-to-back + // deliveries are split even when loop() never observed a quiet gap + // between them. + this->finish_bucket_frame_(); + } size_t at = this->rx_buffer_.size(); this->rx_buffer_.push_back(byte); const uint8_t *raw = &this->rx_buffer_[0]; @@ -84,26 +94,21 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { break; } case RF_CODE_RFIN_BUCKET: { - if (byte != RF_CODE_STOP) { - return true; + if (at == 2) { + // The count byte: Portisch sends at most 7 buckets + sync, so 0 or + // >8 cannot be a genuine capture — reject before it can occupy the + // buffer for a full frame timeout. + return byte != 0 && byte <= B1_MAX_BUCKET_COUNT; } - - uint8_t buckets = raw[2] << 1; - std::string str; - char next_byte[3]; // 2 hex chars + null - - for (uint32_t i = 0; i <= at; i++) { - buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); - str += next_byte; - if ((i > 3) && buckets) { - buckets--; - } - if ((i < 3) || (buckets % 2) || (i == at - 1)) { - str += " "; - } - } - ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); - break; + // 0x55 is legal DATA inside a B1 frame: bucket durations are sent + // with only their HIGH byte masked to 7 bits, so a duration such as + // 0x0155 puts a raw 0x55 low byte inside the table — the first 0x55 + // must therefore not end the capture. The header declares the table + // length (raw[2] pairs), so a 0x55 there is always data; one at or + // past the first pulse index is a terminator CANDIDATE, confirmed + // once the UART goes quiet (finish_bucket_frame_ in loop()). + this->bucket_frame_candidate_ = byte == RF_CODE_STOP && at >= 3 + static_cast(raw[2]) * 2; + return true; } default: ESP_LOGW(TAG, "Unknown action: 0x%02X", action); @@ -119,6 +124,47 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { return false; } +void RFBridgeComponent::finish_bucket_frame_() { + if (this->rx_buffer_.size() < 4) { + // The candidate flag requires a header + non-empty bucket table, so + // this cannot happen while flag and buffer stay consistent; guard the + // raw[2] / size-1 reads against any future divergence anyway. + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + return; + } + const uint8_t *raw = this->rx_buffer_.data(); + const size_t at = this->rx_buffer_.size() - 1; + + uint8_t buckets = raw[2] << 1; + std::string str; + char next_byte[3]; // 2 hex chars + null + + for (uint32_t i = 0; i <= at; i++) { + buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); + str += next_byte; + if ((i > 3) && buckets) { + buckets--; + } + if ((i < 3) || (buckets % 2) || (i == at - 1)) { + str += " "; + } + } + ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); + + // Deliberately NOT ACKed: Portisch's B1 command handler leaves its + // last_sniffing_command at the previous mode (RF_CODE_RFIN), and its + // host-ACK handler re-arms sniffing from that stale value — so ACKing a + // bucket delivery silently reverts the radio to standard sniffing and + // ends bucket capture. Its delivery path is fire-and-forget and never + // waits for a host ACK. Stock Itead firmware never sends B1 frames, so + // suppressing this ACK cannot change stock-firmware behavior. + // https://github.com/esphome/esphome/issues/17682 + + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; +} + void RFBridgeComponent::write_byte_str_(const std::string &codes) { uint8_t code; int size = codes.length(); @@ -130,12 +176,31 @@ void RFBridgeComponent::write_byte_str_(const std::string &codes) { void RFBridgeComponent::loop() { const uint32_t now = App.get_loop_component_start_time(); - if (now - this->last_bridge_byte_ > 50) { + size_t avail = this->available(); + if (avail == 0 && this->bucket_frame_candidate_ && now - this->last_bridge_byte_ > BUCKET_CANDIDATE_QUIET_MS) { + // The trailing 0x55 was followed by UART quiet, so it really was the + // frame terminator and not an interior data byte. + this->finish_bucket_frame_(); + this->last_bridge_byte_ = now; + } + const bool receiving_bucket = this->rx_buffer_.size() >= 2 && this->rx_buffer_[1] == RF_CODE_RFIN_BUCKET; + if (receiving_bucket) { + // Never declare an in-progress bucket frame dead while its continuation + // bytes are already queued: a stalled loop() otherwise discards a live + // frame that the UART buffer proves is still arriving. + if (avail == 0 && now - this->last_bridge_byte_ > BUCKET_FRAME_TIMEOUT_MS) { + ESP_LOGD(TAG, "Discarding incomplete RFBridge Bucket frame (%u bytes)", + static_cast(this->rx_buffer_.size())); + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + this->last_bridge_byte_ = now; + } + } else if (now - this->last_bridge_byte_ > 50) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; this->last_bridge_byte_ = now; } - size_t avail = this->available(); while (avail > 0) { uint8_t buf[64]; size_t to_read = std::min(avail, sizeof(buf)); @@ -146,12 +211,14 @@ void RFBridgeComponent::loop() { for (size_t i = 0; i < to_read; i++) { if (this->rx_buffer_.size() > MAX_RX_BUFFER_SIZE) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } if (this->parse_bridge_byte_(buf[i])) { ESP_LOGVV(TAG, "Parsed: 0x%02X", buf[i]); this->last_bridge_byte_ = now; } else { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } } } diff --git a/esphome/components/rf_bridge/rf_bridge.h b/esphome/components/rf_bridge/rf_bridge.h index 5ad75650ab..cbb1880ec5 100644 --- a/esphome/components/rf_bridge/rf_bridge.h +++ b/esphome/components/rf_bridge/rf_bridge.h @@ -30,6 +30,17 @@ static const uint8_t RF_CODE_BEEP = 0xC0; static const uint8_t RF_CODE_STOP = 0x55; static const uint8_t RF_DEBOUNCE = 200; static const size_t MAX_RX_BUFFER_SIZE = 512; +// ~10 byte times at 19200 baud: long enough to prove the UART went quiet +// after a possible bucket-frame terminator, short enough to finish well +// before the next radio capture can be delivered. +static const uint32_t BUCKET_CANDIDATE_QUIET_MS = 5; +// Portisch drains a B1 frame's header, bucket table, and pulse data as +// separate UART writes, so an in-progress bucket frame tolerates a longer +// inter-region gap than the generic 50 ms inter-byte timeout. +static const uint32_t BUCKET_FRAME_TIMEOUT_MS = 250; +// Portisch's uart_put_RF_buckets sends at most 7 buckets plus the sync +// bucket, so a B1 count byte above 8 (or 0) is malformed for any protocol. +static const uint8_t B1_MAX_BUCKET_COUNT = 8; struct RFBridgeData { uint16_t sync; @@ -67,10 +78,12 @@ class RFBridgeComponent final : public uart::UARTDevice, public Component { void ack_(); void decode_(); bool parse_bridge_byte_(uint8_t byte); + void finish_bucket_frame_(); void write_byte_str_(const std::string &codes); std::vector rx_buffer_; uint32_t last_bridge_byte_{0}; + bool bucket_frame_candidate_{false}; CallbackManager data_callback_; CallbackManager advanced_data_callback_; From 7660dd7fa7059a6154e65797c26e45d27aa8bf78 Mon Sep 17 00:00:00 2001 From: raykholo Date: Tue, 8 Sep 2026 02:57:33 -0400 Subject: [PATCH 20/41] [anova] Re-assert temperature unit on every poll cycle (#17141) --- esphome/components/anova/anova.cpp | 107 ++++++++++++++--------------- esphome/components/anova/anova.h | 13 +++- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/esphome/components/anova/anova.cpp b/esphome/components/anova/anova.cpp index 6e382872e2..b0769bb622 100644 --- a/esphome/components/anova/anova.cpp +++ b/esphome/components/anova/anova.cpp @@ -13,7 +13,7 @@ void Anova::dump_config() { LOG_CLIMATE("", "Anova BLE Cooker", this); } void Anova::setup() { this->codec_ = make_unique(); - this->current_request_ = 0; + this->poll_step_ = PollStep::IDLE; } void Anova::loop() { @@ -22,6 +22,15 @@ void Anova::loop() { this->disable_loop(); } +void Anova::write_request_(AnovaPacket *pkt) { + auto status = + esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, + pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); + if (status) { + ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } +} + void Anova::control(const ClimateCall &call) { auto mode_val = call.get_mode(); if (mode_val.has_value()) { @@ -38,22 +47,11 @@ void Anova::control(const ClimateCall &call) { ESP_LOGW(TAG, "Unsupported mode: %d", mode); return; } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(pkt); } auto target_temp = call.get_target_temperature(); if (target_temp.has_value()) { - auto *pkt = this->codec_->get_set_target_temp_request(*target_temp); - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(this->codec_->get_set_target_temp_request(*target_temp)); } } @@ -62,6 +60,7 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ case ESP_GATTC_DISCONNECT_EVT: { this->current_temperature = NAN; this->target_temperature = NAN; + this->poll_step_ = PollStep::IDLE; this->publish_state(); break; } @@ -83,8 +82,8 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } case ESP_GATTC_REG_FOR_NOTIFY_EVT: { this->node_state = espbt::ClientState::ESTABLISHED; - this->current_request_ = 0; - this->update(); + this->poll_step_ = PollStep::IDLE; + this->update(); // begin the first poll cycle immediately break; } case ESP_GATTC_NOTIFY_EVT: { @@ -101,33 +100,30 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ this->mode = this->codec_->running_ ? climate::CLIMATE_MODE_HEAT : climate::CLIMATE_MODE_OFF; } if (this->codec_->has_unit()) { - this->fahrenheit_ = (this->codec_->unit_ == 'f'); - ESP_LOGD(TAG, "Anova units is %s", this->fahrenheit_ ? "fahrenheit" : "celsius"); - this->current_request_++; + ESP_LOGD(TAG, "Anova units is %s", (this->codec_->unit_ == 'f') ? "fahrenheit" : "celsius"); } this->publish_state(); - if (this->current_request_ > 1) { - AnovaPacket *pkt = nullptr; - switch (this->current_request_++) { - case 2: - pkt = this->codec_->get_read_target_temp_request(); - break; - case 3: - pkt = this->codec_->get_read_current_temp_request(); - break; - default: - this->current_request_ = 1; - break; - } - if (pkt != nullptr) { - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - } + // Advance the poll cycle to its next request based on the reply we got. + switch (this->poll_step_) { + case PollStep::SET_UNIT: + this->poll_step_ = PollStep::STATUS; + this->write_request_(this->codec_->get_read_device_status_request()); + break; + case PollStep::STATUS: + this->poll_step_ = PollStep::TARGET; + this->write_request_(this->codec_->get_read_target_temp_request()); + break; + case PollStep::TARGET: + this->poll_step_ = PollStep::CURRENT; + this->write_request_(this->codec_->get_read_current_temp_request()); + break; + case PollStep::CURRENT: + this->poll_step_ = PollStep::IDLE; // full cycle complete + break; + default: + // A reply to an ad-hoc control() write, outside a managed cycle. + break; } break; } @@ -136,27 +132,26 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } } -void Anova::set_unit_of_measurement(const char *unit) { this->fahrenheit_ = !strncmp(unit, "f", 1); } +void Anova::set_unit_of_measurement(const char *unit) { this->want_fahrenheit_ = !strncmp(unit, "f", 1); } void Anova::update() { if (this->node_state != espbt::ClientState::ESTABLISHED) return; - - if (this->current_request_ < 2) { - AnovaPacket *pkt; - if (this->current_request_ == 0) { - pkt = this->codec_->get_set_unit_request(this->fahrenheit_ ? 'f' : 'c'); - } else { - pkt = this->codec_->get_read_device_status_request(); - } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - this->current_request_++; + if (this->poll_step_ != PollStep::IDLE) { + // The previous cycle never finished within a full polling interval -- a + // reply was missed or a write failed. Restart the cycle rather than stall; + // the polling interval itself acts as the timeout. A late reply from the + // abandoned cycle is harmless: state decoding happens on every notify + // regardless of step, and each notify sends at most one follow-up request. + ESP_LOGW(TAG, "[%s] Poll cycle incomplete (step %u); restarting cycle", this->parent_->address_str(), + static_cast(this->poll_step_)); } + // Re-assert the configured unit at the start of every poll cycle, then fall + // through the status/temperature reads via the notification handler. Always + // command the configured unit (want_fahrenheit_) -- never the last value the + // device reported, or a drift to 'c' would lock itself in. + this->poll_step_ = PollStep::SET_UNIT; + this->write_request_(this->codec_->get_set_unit_request(this->want_fahrenheit_ ? 'f' : 'c')); } } // namespace esphome::anova diff --git a/esphome/components/anova/anova.h b/esphome/components/anova/anova.h index 49b1100c37..a0fa03df01 100644 --- a/esphome/components/anova/anova.h +++ b/esphome/components/anova/anova.h @@ -37,11 +37,20 @@ class Anova final : public climate::Climate, public esphome::ble_client::BLEClie void set_unit_of_measurement(const char *unit); protected: + // A poll cycle re-asserts the configured unit, then reads device state. + // Re-asserting every cycle prevents the cooker from silently reverting to + // its default (Celsius); previously the unit was only set once on + // connection, so a drift persisted (and corrupted the F/C interpretation of + // subsequent readings) until the BLE link was re-established. + enum class PollStep : uint8_t { SET_UNIT, STATUS, TARGET, CURRENT, IDLE }; + + void write_request_(AnovaPacket *pkt); + std::unique_ptr codec_; void control(const climate::ClimateCall &call) override; uint16_t char_handle_; - uint8_t current_request_; - bool fahrenheit_; + bool want_fahrenheit_{true}; // configured target unit; never overwritten by device replies + PollStep poll_step_{PollStep::IDLE}; }; } // namespace esphome::anova From f8b2e53609051bf6ac9a626305a7e6304c67393f Mon Sep 17 00:00:00 2001 From: John <34163498+CircuitSetup@users.noreply.github.com> Date: Tue, 8 Sep 2026 04:02:49 -0400 Subject: [PATCH 21/41] [atm90e32] Verify offset calibration writes (#18701) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/atm90e32/atm90e32.cpp | 360 ++++++++++-------- esphome/components/atm90e32/atm90e32.h | 71 ++-- tests/components/atm90e32/__init__.py | 5 + .../offset_register_verification_test.cpp | 62 +++ 4 files changed, 322 insertions(+), 176 deletions(-) create mode 100644 tests/components/atm90e32/__init__.py create mode 100644 tests/components/atm90e32/offset_register_verification_test.cpp diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp index d948b3741d..23701e7834 100644 --- a/esphome/components/atm90e32/atm90e32.cpp +++ b/esphome/components/atm90e32/atm90e32.cpp @@ -9,6 +9,10 @@ namespace esphome::atm90e32 { static const char *const TAG = "atm90e32"; +static const LogString *offset_calibration_name(bool power_offsets) { + return power_offsets ? LOG_STR("Power offset") : LOG_STR("Offset"); +} + static uint32_t pref_hash(const char *prefix, const char *name_space) { auto hash = fnv1_hash(prefix); return fnv1_hash_extend(hash, name_space); @@ -203,13 +207,12 @@ void ATM90E32Component::setup() { // Initialize flash storage for power offset calibrations uint32_t po_hash = pref_hash("_power_offset_calibration_", cs); - this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); + this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); bool migrated_power_offset = false; if (has_distinct_legacy_namespace) { uint32_t legacy_po_hash = pref_hash("_power_offset_calibration_", legacy_cs); - auto legacy_power_offset_pref = - global_preferences->make_preference(legacy_po_hash, true); - PowerOffsetCalibration power_offset_data[3]{}; + auto legacy_power_offset_pref = global_preferences->make_preference(legacy_po_hash, true); + OffsetCalibration power_offset_data[3]{}; int migration_status = migrate_legacy_pref_if_needed(this->power_offset_pref_, legacy_power_offset_pref, &power_offset_data); migrated_power_offset = migration_status > 0; @@ -224,20 +227,20 @@ void ATM90E32Component::setup() { global_preferences->sync(); } - this->restore_offset_calibrations_(); - this->restore_power_offset_calibrations_(); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } else { ESP_LOGI(TAG, "[CALIBRATION][%s] Power & Voltage/Current offset calibration is disabled. Using config file values.", cs); for (uint8_t phase = 0; phase < 3; ++phase) { this->write16_(this->voltage_offset_registers[phase], - static_cast(this->offset_phase_[phase].voltage_offset_)); + static_cast(this->offset_phase_[phase].first_offset)); this->write16_(this->current_offset_registers[phase], - static_cast(this->offset_phase_[phase].current_offset_)); + static_cast(this->offset_phase_[phase].second_offset)); this->write16_(this->power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].active_power_offset)); + static_cast(this->power_offset_phase_[phase].first_offset)); this->write16_(this->reactive_power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].reactive_power_offset)); + static_cast(this->power_offset_phase_[phase].second_offset)); } } @@ -317,8 +320,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].voltage_offset_, - this->config_offset_phase_[phase].current_offset_, this->offset_phase_[phase].current_offset_); + this->config_offset_phase_[phase].first_offset, this->offset_phase_[phase].first_offset, + this->config_offset_phase_[phase].second_offset, this->offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -335,10 +338,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].active_power_offset, - this->config_power_offset_phase_[phase].reactive_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->config_power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].first_offset, + this->config_power_offset_phase_[phase].second_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -372,7 +373,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\\n", cs); } @@ -385,8 +386,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); } @@ -756,36 +756,68 @@ void ATM90E32Component::save_gain_calibration_to_memory_() { } } -void ATM90E32Component::save_offset_calibration_to_memory_() { +void ATM90E32Component::finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); - bool success = this->offset_pref_.save(&this->offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_offset_calibration_ = true; - for (bool &phase : this->offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save offset calibration to memory!", cs); - } -} + const LogString *name = offset_calibration_name(power_offsets); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; -void ATM90E32Component::save_power_offset_calibration_to_memory_() { - const char *cs = this->get_calibration_id_(); - bool success = this->power_offset_pref_.save(&this->power_offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_power_offset_calibration_ = true; - for (bool &phase : this->power_offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Power offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save power offset calibration to memory!", cs); + const bool writes_verified = this->verify_offset_writes_(type); + bool saved = false; + bool synced = false; + if (writes_verified) { + saved = preference->save(offsets); + synced = global_preferences->sync(); } + + if (writes_verified && saved && synced) { + this->using_saved_calibrations_ = true; + *has_stored = true; + *restored = true; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration saved to memory. %s calibration completed and verified.", cs, + LOG_STR_ARG(name), LOG_STR_ARG(name)); + return; + } + + if (writes_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save %s calibration to memory!", cs, LOG_STR_ARG(name)); + } + + for (uint8_t phase = 0; phase < 3; phase++) { + this->write_offsets_to_registers_(phase, previous[phase].first_offset, previous[phase].second_offset, type); + } + const bool rollback_verified = this->verify_offset_writes_(type); + + bool rollback_persisted = false; + if (writes_verified) { + OffsetCalibration rollback[3]{}; + prepare_offset_rollback(previous, previous_restored, rollback); + const bool rollback_saved = preference->save(&rollback); + const bool rollback_synced = global_preferences->sync(); + rollback_persisted = rollback_saved && rollback_synced; + if (!rollback_saved || !rollback_synced) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to persist restored %s calibration values!", cs, LOG_STR_ARG(name)); + } + } + + *restored = previous_restored; + if (rollback_persisted) + *has_stored = previous_restored; + this->using_saved_calibrations_ = previous_using_saved; + if (!rollback_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; rollback readback verification failed.", cs, + LOG_STR_ARG(name)); + return; + } + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; previous values restored.", cs, LOG_STR_ARG(name)); } void ATM90E32Component::run_offset_calibrations() { @@ -803,11 +835,16 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->offset_phase_[0], this->offset_phase_[1], this->offset_phase_[2]}; + const bool previous_restored = this->restored_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = calibrate_offset(phase, true); int16_t current_offset = calibrate_offset(phase, false); - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); @@ -815,7 +852,8 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] ==================================================================\n", cs); - this->save_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); } void ATM90E32Component::run_power_offset_calibrations() { @@ -834,18 +872,25 @@ void ATM90E32Component::run_power_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->power_offset_phase_[0], this->power_offset_phase_[1], + this->power_offset_phase_[2]}; + const bool previous_restored = this->restored_power_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; ++phase) { int16_t active_offset = calibrate_power_offset(phase, false); int16_t reactive_offset = calibrate_power_offset(phase, true); - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - this->save_power_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } void ATM90E32Component::write_gains_to_registers_() { @@ -859,35 +904,26 @@ void ATM90E32Component::write_gains_to_registers_() { this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } -void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset) { - // Save to runtime - this->offset_phase_[phase].voltage_offset_ = voltage_offset; - this->phase_[phase].voltage_offset_ = voltage_offset; +void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + OffsetCalibration &offsets = power_offsets ? this->power_offset_phase_[phase] : this->offset_phase_[phase]; + offsets.first_offset = first_offset; + offsets.second_offset = second_offset; + if (power_offsets) { + this->phase_[phase].active_power_offset_ = first_offset; + this->phase_[phase].reactive_power_offset_ = second_offset; + } else { + this->phase_[phase].voltage_offset_ = first_offset; + this->phase_[phase].current_offset_ = second_offset; + } - // Save to flash-storable struct - this->offset_phase_[phase].current_offset_ = current_offset; - this->phase_[phase].current_offset_ = current_offset; - - // Write to registers + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(voltage_offset_registers[phase], static_cast(voltage_offset)); - this->write16_(current_offset_registers[phase], static_cast(current_offset)); - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); -} - -void ATM90E32Component::write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset) { - // Save to runtime - this->phase_[phase].active_power_offset_ = p_offset; - this->phase_[phase].reactive_power_offset_ = q_offset; - - // Save to flash-storable struct - this->power_offset_phase_[phase].active_power_offset = p_offset; - this->power_offset_phase_[phase].reactive_power_offset = q_offset; - - // Write to registers - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(this->power_offset_registers[phase], static_cast(p_offset)); - this->write16_(this->reactive_power_offset_registers[phase], static_cast(q_offset)); + this->write16_(first_registers[phase], static_cast(first_offset)); + this->write16_(second_registers[phase], static_cast(second_offset)); this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } @@ -947,89 +983,78 @@ void ATM90E32Component::restore_gain_calibrations_() { ESP_LOGW(TAG, "[CALIBRATION][%s] No stored gain calibrations found. Using config file values.", cs); } -void ATM90E32Component::restore_offset_calibrations_() { +void ATM90E32Component::restore_offset_calibrations_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); + const LogString *name = power_offsets ? LOG_STR("power offset") : LOG_STR("offset"); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + OffsetCalibration(*config_offsets)[3] = + power_offsets ? &this->config_power_offset_phase_ : &this->config_offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; + const bool *has_first = power_offsets ? this->has_config_active_power_offset_ : this->has_config_voltage_offset_; + const bool *has_second = power_offsets ? this->has_config_reactive_power_offset_ : this->has_config_current_offset_; + for (uint8_t i = 0; i < 3; ++i) - this->config_offset_phase_[i] = this->offset_phase_[i]; - - bool have_data = this->offset_pref_.load(&this->offset_phase_); + (*config_offsets)[i] = (*offsets)[i]; + const bool have_data = preference->load(offsets); bool all_zero = true; if (have_data) { - for (auto &phase : this->offset_phase_) { - if (phase.voltage_offset_ != 0 || phase.current_offset_ != 0) { + for (const auto &phase : *offsets) { + if (phase.first_offset != 0 || phase.second_offset != 0) { all_zero = false; break; } } } - if (have_data && !all_zero) { - this->restored_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; phase++) { - auto &offset = this->offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_voltage_offset_[phase] && - offset.voltage_offset_ != this->config_offset_phase_[phase].voltage_offset_) - mismatch = true; - if (this->has_config_current_offset_[phase] && - offset.current_offset_ != this->config_offset_phase_[phase].current_offset_) - mismatch = true; - if (mismatch) - this->offset_calibration_mismatch_[phase] = true; + *has_stored = have_data && !all_zero; + *restored = false; + for (uint8_t phase = 0; phase < 3; phase++) { + mismatches[phase] = false; + if (*has_stored) { + mismatches[phase] = + (has_first[phase] && (*offsets)[phase].first_offset != (*config_offsets)[phase].first_offset) || + (has_second[phase] && (*offsets)[phase].second_offset != (*config_offsets)[phase].second_offset); } - } else { + } + + if (!*has_stored) { for (uint8_t phase = 0; phase < 3; phase++) - this->offset_phase_[phase] = this->config_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored offset calibrations found. Using default values.", cs); + (*offsets)[phase] = (*config_offsets)[phase]; + ESP_LOGW(TAG, "[CALIBRATION][%s] No stored %s calibrations found. Using default values.", cs, LOG_STR_ARG(name)); } for (uint8_t phase = 0; phase < 3; phase++) { - write_offsets_to_registers_(phase, this->offset_phase_[phase].voltage_offset_, - this->offset_phase_[phase].current_offset_); + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); } -} - -void ATM90E32Component::restore_power_offset_calibrations_() { - const char *cs = this->get_calibration_id_(); - for (uint8_t i = 0; i < 3; ++i) - this->config_power_offset_phase_[i] = this->power_offset_phase_[i]; - - bool have_data = this->power_offset_pref_.load(&this->power_offset_phase_); - - bool all_zero = true; - if (have_data) { - for (auto &phase : this->power_offset_phase_) { - if (phase.active_power_offset != 0 || phase.reactive_power_offset != 0) { - all_zero = false; - break; - } - } + const bool initial_values_verified = this->verify_offset_writes_(type); + if (initial_values_verified) { + const auto state = resolve_offset_restore_state(*has_stored, true, false); + *restored = state.restored; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration values verified.", cs, LOG_STR_ARG(name)); + return; } - if (have_data && !all_zero) { - this->restored_power_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; ++phase) { - auto &offset = this->power_offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_active_power_offset_[phase] && - offset.active_power_offset != this->config_power_offset_phase_[phase].active_power_offset) - mismatch = true; - if (this->has_config_reactive_power_offset_[phase] && - offset.reactive_power_offset != this->config_power_offset_phase_[phase].reactive_power_offset) - mismatch = true; - if (mismatch) - this->power_offset_calibration_mismatch_[phase] = true; - } + this->using_saved_calibrations_ = false; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + for (uint8_t phase = 0; phase < 3; phase++) { + (*offsets)[phase] = (*config_offsets)[phase]; + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); + } + const auto state = resolve_offset_restore_state(*has_stored, false, this->verify_offset_writes_(type)); + *restored = state.restored; + if (state.values_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore failed verification; config values verified.", cs, + LOG_STR_ARG(name)); } else { - for (uint8_t phase = 0; phase < 3; ++phase) - this->power_offset_phase_[phase] = this->config_power_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored power offsets found. Using default values.", cs); - } - - for (uint8_t phase = 0; phase < 3; ++phase) { - write_power_offsets_to_registers_(phase, this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore and config fallback both failed verification.", cs, + LOG_STR_ARG(name)); } } @@ -1084,14 +1109,14 @@ void ATM90E32Component::clear_gain_calibrations() { void ATM90E32Component::clear_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_offset_calibration_) { + if (!this->has_stored_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored offset calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\n", cs); return; @@ -1104,10 +1129,11 @@ void ATM90E32Component::clear_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = - this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].voltage_offset_ : 0; + this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].first_offset : 0; int16_t current_offset = - this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].current_offset_ : 0; - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); } @@ -1117,6 +1143,7 @@ void ATM90E32Component::clear_offset_calibrations() { this->offset_pref_.save(&zero_offsets); // Clear stored values in flash global_preferences->sync(); + this->has_stored_offset_calibration_ = false; this->restored_offset_calibration_ = false; for (bool &phase : this->offset_calibration_mismatch_) phase = false; @@ -1126,15 +1153,14 @@ void ATM90E32Component::clear_offset_calibrations() { void ATM90E32Component::clear_power_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_power_offset_calibration_) { + if (!this->has_stored_power_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored power offsets to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); return; @@ -1147,20 +1173,21 @@ void ATM90E32Component::clear_power_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t active_offset = - this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].active_power_offset : 0; - int16_t reactive_offset = this->has_config_reactive_power_offset_[phase] - ? this->config_power_offset_phase_[phase].reactive_power_offset - : 0; - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].first_offset : 0; + int16_t reactive_offset = + this->has_config_reactive_power_offset_[phase] ? this->config_power_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - PowerOffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; + OffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; this->power_offset_pref_.save(&zero_power_offsets); global_preferences->sync(); + this->has_stored_power_offset_calibration_ = false; this->restored_power_offset_calibration_ = false; for (bool &phase : this->power_offset_calibration_mismatch_) phase = false; @@ -1215,6 +1242,31 @@ bool ATM90E32Component::verify_gain_writes_() { return success; // Return true if all writes were successful, false otherwise } +bool ATM90E32Component::verify_offset_writes_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + const char *cs = this->get_calibration_id_(); + const LogString *name = offset_calibration_name(power_offsets); + const LogString *first_name = power_offsets ? LOG_STR("active") : LOG_STR("voltage"); + const LogString *second_name = power_offsets ? LOG_STR("reactive") : LOG_STR("current"); + const OffsetCalibration *offsets = power_offsets ? this->power_offset_phase_ : this->offset_phase_; + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; + bool success = true; + for (uint8_t phase = 0; phase < 3; phase++) { + const uint16_t first = this->read16_(first_registers[phase]); + const uint16_t second = this->read16_(second_registers[phase]); + if (!offset_register_value_matches(first, offsets[phase].first_offset) || + !offset_register_value_matches(second, offsets[phase].second_offset)) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s readback failed for Phase %s: %s %d/%d, %s %d/%d.", cs, LOG_STR_ARG(name), + phase_labels[phase], LOG_STR_ARG(first_name), static_cast(first), offsets[phase].first_offset, + LOG_STR_ARG(second_name), static_cast(second), offsets[phase].second_offset); + success = false; + } + } + return success; +} + #ifdef USE_TEXT_SENSOR void ATM90E32Component::check_phase_status() { uint16_t state0 = this->read16_(ATM90E32_REGISTER_EMMSTATE0); diff --git a/esphome/components/atm90e32/atm90e32.h b/esphome/components/atm90e32/atm90e32.h index c636e5065a..fe7d903962 100644 --- a/esphome/components/atm90e32/atm90e32.h +++ b/esphome/components/atm90e32/atm90e32.h @@ -13,6 +13,40 @@ namespace esphome::atm90e32 { +inline bool offset_register_value_matches(uint16_t actual, int16_t expected) { + return actual == static_cast(expected); +} + +struct OffsetCalibration { + int16_t first_offset{0}; + int16_t second_offset{0}; +}; + +static_assert(sizeof(OffsetCalibration[3]) == 12, "Offset calibration preference layout must remain compatible"); + +enum class OffsetCalibrationType : uint8_t { + OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT, + OFFSET_CALIBRATION_TYPE_POWER, +}; + +struct OffsetRestoreState { + bool restored; + bool values_verified; +}; + +inline OffsetRestoreState resolve_offset_restore_state(bool has_stored_values, bool initial_values_verified, + bool fallback_values_verified) { + if (initial_values_verified) + return {has_stored_values, true}; + return {false, fallback_values_verified}; +} + +inline void prepare_offset_rollback(const OffsetCalibration (&previous)[3], bool had_stored_values, + OffsetCalibration (&rollback)[3]) { + for (uint8_t phase = 0; phase < 3; phase++) + rollback[phase] = had_stored_values ? previous[phase] : OffsetCalibration{}; +} + class ATM90E32Component final : public PollingComponent, public spi::SPIDevice { @@ -71,19 +105,19 @@ class ATM90E32Component final : public PollingComponent, this->has_config_current_gain_[phase] = true; } void set_voltage_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].voltage_offset_ = offset; + this->offset_phase_[phase].first_offset = offset; this->has_config_voltage_offset_[phase] = true; } void set_current_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].current_offset_ = offset; + this->offset_phase_[phase].second_offset = offset; this->has_config_current_offset_[phase] = true; } void set_active_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].active_power_offset = offset; + this->power_offset_phase_[phase].first_offset = offset; this->has_config_active_power_offset_[phase] = true; } void set_reactive_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].reactive_power_offset = offset; + this->power_offset_phase_[phase].second_offset = offset; this->has_config_reactive_power_offset_[phase] = true; } void set_freq_sensor(sensor::Sensor *freq_sensor) { freq_sensor_ = freq_sensor; } @@ -171,16 +205,16 @@ class ATM90E32Component final : public PollingComponent, float get_chip_temperature_(); bool get_publish_interval_flag_() { return publish_interval_flag_; }; void set_publish_interval_flag_(bool flag) { publish_interval_flag_ = flag; }; - void restore_offset_calibrations_(); - void restore_power_offset_calibrations_(); + void restore_offset_calibrations_(OffsetCalibrationType type); void restore_gain_calibrations_(); - void save_offset_calibration_to_memory_(); void save_gain_calibration_to_memory_(); - void save_power_offset_calibration_to_memory_(); - void write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset); - void write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset); + void finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type); + void write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type); void write_gains_to_registers_(); bool verify_gain_writes_(); + bool verify_offset_writes_(OffsetCalibrationType type); bool validate_spi_read_(uint16_t expected, const char *context = nullptr); void log_calibration_status_(); const char *get_calibration_id_(); @@ -219,19 +253,10 @@ class ATM90E32Component final : public PollingComponent, uint32_t cumulative_reverse_active_energy_{0}; } phase_[3]; - struct OffsetCalibration { - int16_t voltage_offset_{0}; - int16_t current_offset_{0}; - } offset_phase_[3]; - + OffsetCalibration offset_phase_[3]; OffsetCalibration config_offset_phase_[3]; - - struct PowerOffsetCalibration { - int16_t active_power_offset{0}; - int16_t reactive_power_offset{0}; - } power_offset_phase_[3]; - - PowerOffsetCalibration config_power_offset_phase_[3]; + OffsetCalibration power_offset_phase_[3]; + OffsetCalibration config_power_offset_phase_[3]; struct GainCalibration { uint16_t voltage_gain{1}; @@ -265,6 +290,8 @@ class ATM90E32Component final : public PollingComponent, bool enable_offset_calibration_{false}; bool enable_gain_calibration_{false}; const char *instance_id_{nullptr}; + bool has_stored_offset_calibration_{false}; + bool has_stored_power_offset_calibration_{false}; bool restored_offset_calibration_{false}; bool restored_power_offset_calibration_{false}; bool restored_gain_calibration_{false}; diff --git a/tests/components/atm90e32/__init__.py b/tests/components/atm90e32/__init__.py new file mode 100644 index 0000000000..37d6797e2d --- /dev/null +++ b/tests/components/atm90e32/__init__.py @@ -0,0 +1,5 @@ +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + manifest.dependencies = manifest.dependencies + ["sensor", "spi"] diff --git a/tests/components/atm90e32/offset_register_verification_test.cpp b/tests/components/atm90e32/offset_register_verification_test.cpp new file mode 100644 index 0000000000..3bb3eb76ea --- /dev/null +++ b/tests/components/atm90e32/offset_register_verification_test.cpp @@ -0,0 +1,62 @@ +#include + +#include "esphome/components/atm90e32/atm90e32.h" + +namespace esphome::atm90e32::testing { + +TEST(ATM90E32OffsetRegisterVerification, AcceptsExactSignedReadback) { + EXPECT_TRUE(offset_register_value_matches(0x007B, 123)); + EXPECT_TRUE(offset_register_value_matches(0xFF85, -123)); +} + +TEST(ATM90E32OffsetRegisterVerification, RejectsMismatchedReadback) { + EXPECT_FALSE(offset_register_value_matches(0x007C, 123)); + EXPECT_FALSE(offset_register_value_matches(0xFF84, -123)); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedStoredValuesAsRestored) { + const auto state = resolve_offset_restore_state(true, true, false); + + EXPECT_TRUE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedConfigFallbackAsNotRestored) { + const auto state = resolve_offset_restore_state(true, false, true); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsFailedConfigFallbackAsUnverified) { + const auto state = resolve_offset_restore_state(true, false, false); + + EXPECT_FALSE(state.restored); + EXPECT_FALSE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsConfigWithoutStoredValuesAsNotRestored) { + const auto state = resolve_offset_restore_state(false, true, false); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetPersistence, RollsBackStoredValuesOrZeroSentinel) { + const OffsetCalibration previous[3]{{1, -1}, {2, -2}, {3, -3}}; + OffsetCalibration rollback[3]{}; + + prepare_offset_rollback(previous, true, rollback); + for (uint8_t phase = 0; phase < 3; phase++) { + EXPECT_EQ(rollback[phase].first_offset, previous[phase].first_offset); + EXPECT_EQ(rollback[phase].second_offset, previous[phase].second_offset); + } + + prepare_offset_rollback(previous, false, rollback); + for (const auto &phase : rollback) { + EXPECT_EQ(phase.first_offset, 0); + EXPECT_EQ(phase.second_offset, 0); + } +} + +} // namespace esphome::atm90e32::testing From f89b9e704c7dfabce1e8ce670dd2c6e7aa9ea086 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:21:48 +0200 Subject: [PATCH 22/41] Bump bundled esphome-device-builder to 1.14.5 (#19040) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index da76ab7b6a..ac84ee4689 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 RUN \ platformio settings set enable_telemetry No \ From 9b6facb20d5461dfaf47fd3993a7b4601f082c34 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 15:46:17 -0400 Subject: [PATCH 23/41] [core] Fix use-after-free when deleting a running StaticTask (#19048) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../micro_wake_word/micro_wake_word.cpp | 4 +-- .../mixer/speaker/mixer_speaker.cpp | 4 +-- .../resampler/speaker/resampler_speaker.cpp | 4 +-- .../speaker/media_player/audio_pipeline.cpp | 11 +++++-- esphome/core/static_task.cpp | 30 ++++++++++++++----- esphome/core/static_task.h | 17 +++++++---- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 3dadb78077..cebfe8e791 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -446,9 +446,9 @@ void MicroWakeWord::loop() { xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STOPPING); } - if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & EventGroupBits::TASK_STOPPED) && this->inference_task_.deallocate()) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 6128dc3767..0b79010773 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -382,8 +382,8 @@ void MixerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, MIXER_TASK_STATE_STOPPING); } - if (event_group_bits & MIXER_TASK_STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS); this->all_stopped_since_ms_ = 0; diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index f1ebd180cc..edda00ae06 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -153,8 +153,8 @@ void ResamplerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::STATE_STOPPING); } - if (event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::ALL_BITS); } diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index 010f0c50b3..c286a9d7d6 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -202,8 +202,15 @@ AudioPipelineState AudioPipeline::process_state() { if (!this->is_playing_) { // The tasks have been stopped for two ``process_state`` calls in a row, so delete the tasks if (this->read_task_.is_created() || this->decode_task_.is_created()) { - this->read_task_.deallocate(); - this->decode_task_.deallocate(); + // Both are attempted every time; a task that is still running on the other core is freed by a + // subsequent call, and freeing an already freed task succeeds without doing anything + bool read_task_freed = this->read_task_.deallocate(); + bool decode_task_freed = this->decode_task_.deallocate(); + if (!read_task_freed || !decode_task_freed) { + // A task is still running on the other core, so keep the pipeline in its current state and try + // again on the next call + return AudioPipelineState::PLAYING; + } if (this->hard_stop_) { // Stop command was sent, so immediately end the playback this->speaker_->stop(); diff --git a/esphome/core/static_task.cpp b/esphome/core/static_task.cpp index 4cfead44c2..4301108315 100644 --- a/esphome/core/static_task.cpp +++ b/esphome/core/static_task.cpp @@ -40,16 +40,31 @@ bool StaticTask::create(TaskFunction_t fn, const char *name, uint32_t stack_size return true; } -void StaticTask::destroy() { - if (this->handle_ != nullptr) { - TaskHandle_t handle = this->handle_; - this->handle_ = nullptr; - vTaskDelete(handle); +bool StaticTask::destroy() { + if (this->handle_ == nullptr) { + return true; } + + // Suspending takes the task off the ready and event lists, so nothing can schedule it again. It only asks + // the other core to yield though, so the task may still be running on it for a moment. + vTaskSuspend(this->handle_); + if (eTaskGetState(this->handle_) != eSuspended) { + // The task is still running on the other core and using its stack. Deleting it now would only put it on + // the termination list and return, so the caller has to try again once it has been swapped out. + return false; + } + + // The task cannot run again, so the delete completes right away instead of being left to the idle task. + TaskHandle_t handle = this->handle_; + this->handle_ = nullptr; + vTaskDelete(handle); + return true; } -void StaticTask::deallocate() { - this->destroy(); +bool StaticTask::deallocate() { + if (!this->destroy()) { + return false; + } if (this->stack_buffer_ != nullptr) { RAMAllocator allocator(this->use_psram_ ? RAMAllocator::ALLOC_EXTERNAL : RAMAllocator::ALLOC_INTERNAL); @@ -57,6 +72,7 @@ void StaticTask::deallocate() { this->stack_buffer_ = nullptr; this->stack_size_ = 0; } + return true; } } // namespace esphome diff --git a/esphome/core/static_task.h b/esphome/core/static_task.h index 5fd5b38f9e..e2996abeda 100644 --- a/esphome/core/static_task.h +++ b/esphome/core/static_task.h @@ -11,6 +11,7 @@ namespace esphome { /** Helper for FreeRTOS static task management. * Bundles TaskHandle_t, StaticTask_t, and the stack buffer into one object with create/destroy methods. + * Call destroy() and deallocate() from another task: a task cannot free the stack it is still running on. */ class StaticTask { public: @@ -23,7 +24,7 @@ class StaticTask { /// @brief Allocate stack and create task. /// @param fn Task function /// @param name Task name (for debug) - /// @param stack_size Stack size in StackType_t words + /// @param stack_size Stack size in bytes (StackType_t is a byte on ESP-IDF) /// @param param Parameter passed to task function /// @param priority FreeRTOS task priority /// @param use_psram If true, allocate stack in PSRAM; otherwise internal RAM @@ -31,11 +32,17 @@ class StaticTask { bool create(TaskFunction_t fn, const char *name, uint32_t stack_size, void *param, UBaseType_t priority, bool use_psram); - /// @brief Delete the task but keep the stack buffer allocated for reuse by a subsequent create() call. - void destroy(); + /// @brief Delete the task, keeping the stack buffer allocated for reuse by a subsequent create() call. + /// The task must have finished its work and parked itself, either suspended or blocked indefinitely: it is + /// suspended here so that it cannot be scheduled again, and it is given no chance to clean up. + /// @return true if the task was deleted; false if it is still running on another core, in which case the + /// caller should try again later. + bool destroy(); - /// @brief Delete the task (if running) and free the stack buffer. - void deallocate(); + /// @brief Delete the task (if created) and free the stack buffer. + /// @return true if the stack buffer was freed; false if the task is still running on another core, in + /// which case the caller should try again later. + bool deallocate(); protected: TaskHandle_t handle_{nullptr}; From 0a1e2acbcba4521391742824c617c8cf206beb63 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:04:34 -0400 Subject: [PATCH 24/41] [audio][i2s_audio][micro_wake_word][microphone][mixer][resampler][speaker] Replace use_count() checks with lock and null test (#19046) --- esphome/components/audio/audio_reader.cpp | 3 +++ esphome/components/audio/audio_transfer_buffer.cpp | 12 ++++++------ .../i2s_audio/speaker/i2s_audio_speaker.cpp | 4 ++-- .../components/micro_wake_word/micro_wake_word.cpp | 2 +- esphome/components/microphone/microphone_source.h | 2 +- esphome/components/mixer/speaker/mixer_speaker.cpp | 12 ++++++------ .../resampler/speaker/resampler_speaker.cpp | 6 +++--- .../speaker/media_player/audio_pipeline.cpp | 12 +++++++----- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/esphome/components/audio/audio_reader.cpp b/esphome/components/audio/audio_reader.cpp index 4678ed548c..e69f33ac2d 100644 --- a/esphome/components/audio/audio_reader.cpp +++ b/esphome/components/audio/audio_reader.cpp @@ -58,6 +58,9 @@ esp_err_t AudioReader::add_sink(const std::weak_ptr &ou if (current_audio_file_ != nullptr) { // A transfer buffer isn't ncessary for a local file this->file_ring_buffer_ = output_ring_buffer.lock(); + if (this->file_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; + } return ESP_OK; } diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index a611549e58..01fd4bb68a 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -51,14 +51,14 @@ void AudioTransferBuffer::increase_buffer_length(size_t bytes) { this->buffer_le void AudioTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } } void AudioSinkTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } #ifdef USE_SPEAKER @@ -69,7 +69,7 @@ void AudioSinkTransferBuffer::clear_buffered_data() { } bool AudioTransferBuffer::has_buffered_data() const { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); @@ -144,7 +144,7 @@ size_t AudioSourceTransferBuffer::transfer_data_from_source(TickType_t ticks_to_ size_t bytes_to_read = AudioTransferBuffer::free(); size_t bytes_read = 0; if (bytes_to_read > 0) { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_read = this->ring_buffer_->read((void *) this->get_buffer_end(), bytes_to_read, ticks_to_wait); } @@ -161,7 +161,7 @@ size_t AudioSinkTransferBuffer::transfer_data_to_sink(TickType_t ticks_to_wait, bytes_written = this->speaker_->play(this->data_start_, this->available(), ticks_to_wait); } else #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_written = this->ring_buffer_->write_without_replacement((void *) this->data_start_, this->available(), ticks_to_wait); } else if (this->sink_callback_ != nullptr) { @@ -186,7 +186,7 @@ bool AudioSinkTransferBuffer::has_buffered_data() const { return (this->speaker_->has_buffered_data() || (this->available() > 0)); } #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1c2eb12904..b78a151ee4 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -218,8 +218,8 @@ size_t I2SAudioSpeakerBase::play(const uint8_t *data, size_t length, TickType_t } bool I2SAudioSpeakerBase::has_buffered_data() const { - if (this->audio_ring_buffer_.use_count() > 0) { - std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + if (temp_ring_buffer != nullptr) { return temp_ring_buffer->available() > 0; } return false; diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index cebfe8e791..cf239be696 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -129,7 +129,7 @@ void MicroWakeWord::setup() { return; } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (this->ring_buffer_.use_count() > 1) { + if (temp_ring_buffer != nullptr) { // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task // to drain it - reset() is a consumer operation and must run on the inference task's thread. // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. diff --git a/esphome/components/microphone/microphone_source.h b/esphome/components/microphone/microphone_source.h index 7be3b8cdb5..d7a3352432 100644 --- a/esphome/components/microphone/microphone_source.h +++ b/esphome/components/microphone/microphone_source.h @@ -48,7 +48,7 @@ class MicrophoneSource final { template void add_data_callback(F &&data_callback) { this->mic_->add_data_callback([this, data_callback](const std::vector &data) { if (this->enabled_ || this->passive_) { - if (this->processed_samples_.use_count() == 0) { + if (this->processed_samples_ == nullptr) { // Create vector if its unused this->processed_samples_ = std::make_shared>(); } diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 0b79010773..ef21da65c5 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -218,7 +218,7 @@ size_t SourceSpeaker::play(const uint8_t *data, size_t length, TickType_t ticks_ } size_t bytes_written = 0; std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer.use_count() > 0) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); if (bytes_written > 0) { @@ -250,14 +250,14 @@ esp_err_t SourceSpeaker::start_() { // avoids unnecessary single-frame splices. const size_t ring_buffer_size = (this->audio_stream_info_.ms_to_bytes(this->buffer_duration_ms_) / bytes_per_frame) * bytes_per_frame; - if (this->audio_source_.use_count() == 0) { + if (this->audio_source_ == nullptr) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); this->ring_buffer_ = temp_ring_buffer; } - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { return ESP_ERR_NO_MEM; } @@ -278,7 +278,7 @@ void SourceSpeaker::stop() { this->send_command_(SOURCE_SPEAKER_COMMAND_STOP); } void SourceSpeaker::finish() { this->send_command_(SOURCE_SPEAKER_COMMAND_FINISH); } bool SourceSpeaker::has_buffered_data() const { - return ((this->audio_source_.use_count() > 0) && this->audio_source_->has_buffered_data()); + return ((this->audio_source_ != nullptr) && this->audio_source_->has_buffered_data()); } void SourceSpeaker::set_mute_state(bool mute_state) { @@ -496,7 +496,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (speaker->is_running() && !speaker->get_pause_state()) { // Speaker is running and not paused, so it possibly can provide audio data std::shared_ptr audio_source = speaker->get_audio_source().lock(); - if (audio_source.use_count() == 0) { + if (audio_source == nullptr) { // No audio source allocated, so skip processing this speaker continue; } diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index edda00ae06..16d2d5dc9e 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -235,7 +235,7 @@ size_t ResamplerSpeaker::play(const uint8_t *data, size_t length, TickType_t tic bytes_written = this->output_speaker_->play(data, length, ticks_to_wait); } else { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); } else { @@ -299,7 +299,7 @@ bool ResamplerSpeaker::has_buffered_data() const { bool has_ring_buffer_data = false; if (this->requires_resampling_()) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { has_ring_buffer_data = (temp_ring_buffer->available() > 0); } } @@ -342,7 +342,7 @@ void ResamplerSpeaker::resample_task(void *params) { std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( this_resampler->audio_stream_info_.ms_to_bytes(this_resampler->buffer_duration_ms_)); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { this_resampler->ring_buffer_ = temp_ring_buffer; diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index c286a9d7d6..509984cfa2 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -322,17 +322,17 @@ void AudioPipeline::read_task(void *params) { if (err == ESP_OK) { size_t file_ring_buffer_size = this_pipeline->buffer_size_; - std::shared_ptr temp_ring_buffer; + std::shared_ptr temp_ring_buffer = this_pipeline->raw_file_ring_buffer_.lock(); - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(file_ring_buffer_size); this_pipeline->raw_file_ring_buffer_ = temp_ring_buffer; } - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { - reader->add_sink(this_pipeline->raw_file_ring_buffer_); + err = reader->add_sink(temp_ring_buffer); } } @@ -403,7 +403,9 @@ void AudioPipeline::decode_task(void *params) { make_unique(this_pipeline->transfer_buffer_size_, this_pipeline->transfer_buffer_size_); esp_err_t err = decoder->start(this_pipeline->current_audio_file_type_); - decoder->add_source(this_pipeline->raw_file_ring_buffer_); + if (err == ESP_OK) { + err = decoder->add_source(this_pipeline->raw_file_ring_buffer_); + } if (err != ESP_OK) { // Send specific error message From ac79173f4ae83ff10c6a571140091be6ec7878b1 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:05:02 -0400 Subject: [PATCH 25/41] [i2s_audio] Fix spurious driver failure (#19045) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index b78a151ee4..1382a87046 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -118,21 +118,24 @@ void I2SAudioSpeakerBase::loop() { break; } + // Still starting up or winding down from a previous run + if ((this->tx_handle_ != nullptr) || (this->speaker_task_handle_ != nullptr)) { + break; + } + if (this->start_i2s_driver(this->audio_stream_info_) != ESP_OK) { ESP_LOGE(TAG, "Driver failed to start; retrying in 1 second"); this->status_momentary_error("driver-failure", 1000); break; } - if (this->speaker_task_handle_ == nullptr) { - xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, - &this->speaker_task_handle_); + xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, + &this->speaker_task_handle_); - if (this->speaker_task_handle_ == nullptr) { - ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); - this->status_momentary_error("task-failure", 1000); - this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt - } + if (this->speaker_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); + this->status_momentary_error("task-failure", 1000); + this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt } break; case speaker::STATE_RUNNING: // Intentional fallthrough From 9c16aba6f78af2657cd9e5876d0e771c72692fcd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:07:06 +0200 Subject: [PATCH 26/41] [noise] Bump noise-c to 0.1.26 and libsodium to 1.10021.8 (#19030) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 4de706120e..d17ebf235e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.24") + cg.add_library("esphome/noise-c", "0.1.26") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.6") + cg.add_library("esphome/libsodium", "1.10021.8") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 779a05e7de..738773d1b5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.24 ; used by noise (api, ota) + esphome/noise-c@0.1.26 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 4f7f5a4a4c..00f22ca138 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.24"] + assert libs == ["esphome/noise-c @ 0.1.26"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.24", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 0.1.26", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.24"] + assert cls.calls == ["esphome/noise-c @ 0.1.26"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.24"] is None + assert compats["esphome/noise-c @ 0.1.26"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 14c52dda8d..b03bff19a2 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 42fffd16fef3d08f58b5ace59b0545e551c07e64 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:08:16 +0200 Subject: [PATCH 27/41] [esphome][core] Give a lost OTA chunk ack time to be retransmitted (#19041) --- esphome/components/esphome/ota/ota_esphome.cpp | 5 ++++- esphome/espota2.py | 9 ++++++--- tests/unit_tests/test_espota2.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 1005ed214b..f853ed6a2d 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -41,7 +41,10 @@ const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake -static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer +// Milliseconds for data transfer. Covers the lwIP retransmit run seen in +// practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits +// longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries +static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/espota2.py b/esphome/espota2.py index ce403c398d..c683ffa323 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -96,6 +96,10 @@ UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # across the addresses on top of that. EXTRA_UPLOAD_ATTEMPTS = 2 UPLOAD_RETRY_DELAY = 5.0 +# Data phase timeout; must stay longer than the device's OTA_SOCKET_TIMEOUT_DATA +# (105 s) so a stalled session is gone before a retry, and long enough for lwIP +# to get a lost chunk ack through after the retransmit run seen in practice +DATA_PHASE_TIMEOUT = 160.0 _LOGGER = logging.getLogger(__name__) @@ -694,8 +698,7 @@ def perform_ota( _LOGGER.info("Handshake complete") - # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures - sock.settimeout(90.0) + sock.settimeout(DATA_PHASE_TIMEOUT) if extended_proto: send_check(sock, ota_type, "ota type") @@ -854,7 +857,7 @@ def run_ota_impl_( # clean up a half-open connection (its handshake watchdog runs at 20s); # moving on to the next address family stays immediate. Known limitation: # a silent mid-transfer drop with no reset can wedge the device until its - # 90s data timeout, which outlasts this budget; the retries target the + # 105s data timeout, which outlasts this budget; the retries target the # common failures where the device resets or closes the link promptly. total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 8867e2c215..2d65e8e079 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -416,6 +416,9 @@ def test_perform_ota_no_auth( "Update took 14.00 seconds (prepare 2.00, upload 5.00, commit 7.00)" in caplog.text ) + # The data phase timeout must outlast the device's 105 s data timeout + mock_socket.settimeout.assert_any_call(espota2.DATA_PHASE_TIMEOUT) + assert espota2.DATA_PHASE_TIMEOUT > 105.0 @pytest.mark.usefixtures("mock_time") From d2bc056f0ab3ea93e62c7ca468f7a3da17a3f421 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:31:18 -0400 Subject: [PATCH 28/41] [sendspin] Add codec preference list to the media source (#19047) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/sendspin/__init__.py | 26 ++++-- .../sendspin/media_source/__init__.py | 31 +++++++ .../sendspin/test_media_source.py | 90 +++++++++++++++++++ .../sendspin/common-media_source.yaml | 1 + 4 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 tests/component_tests/sendspin/test_media_source.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 570fd3fadd..8ef11a7f90 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -30,6 +30,7 @@ CONF_SENDSPIN_ID = "sendspin_id" CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" +CONF_CODECS = "codecs" # Matches ARTWORK_MAX_SLOTS in sendspin-cpp. MAX_ARTWORK_SLOTS = 4 @@ -44,6 +45,20 @@ CODEC_FORMAT_OPUS = SendspinCodecFormat.enum("OPUS") CODEC_FORMAT_PCM = SendspinCodecFormat.enum("PCM") CODEC_FORMAT_UNSUPPORTED = SendspinCodecFormat.enum("UNSUPPORTED") +CODEC_FLAC = "flac" +CODEC_OPUS = "opus" +CODEC_PCM = "pcm" + +CODECS = { + CODEC_FLAC: CODEC_FORMAT_FLAC, + CODEC_OPUS: CODEC_FORMAT_OPUS, + CODEC_PCM: CODEC_FORMAT_PCM, +} + +# Opus only supports 48 kHz audio, so it is left out of the default list at other rates. +DEFAULT_CODECS = [CODEC_FLAC, CODEC_OPUS, CODEC_PCM] +OPUS_SAMPLE_RATE = 48000 + SendspinImageFormat = sendspin_library_ns.enum("SendspinImageFormat", is_class=True) IMAGE_FORMAT_JPEG = SendspinImageFormat.enum("JPEG") IMAGE_FORMAT_PNG = SendspinImageFormat.enum("PNG") @@ -286,16 +301,13 @@ async def to_code(config: ConfigType) -> None: if data.player_support: cg.add_define("USE_SENDSPIN_PLAYER", True) - # Configures the player role. We always assume support for 16 bits per sample mono and stereo FLAC, Opus, and PCM at the configured sample rate - # (with Opus only supported at 48 kHz since that's the only sample rate it supports). Users can configure the specific formats via the Sendspin server + # Configures the player role. Each configured codec is advertised for 16 bits per sample + # mono and stereo at the configured sample rate. The order is a preference order, both for + # the codecs themselves and for stereo over mono. player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - # OPUS only supports 48 kHz audio - codecs = [CODEC_FORMAT_FLAC] - if sample_rate == 48000: - codecs.append(CODEC_FORMAT_OPUS) - codecs.append(CODEC_FORMAT_PCM) + codecs = player_cfg[CONF_CODECS] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index 6af244d41f..6a9f1f18ba 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -13,11 +13,16 @@ from esphome.cpp_generator import MockObj, TemplateArgsType from esphome.types import ConfigType from .. import ( + CODEC_OPUS, + CODECS, + CONF_CODECS, CONF_DECODE_MEMORY, CONF_FIXED_DELAY, CONF_INITIAL_STATIC_DELAY, CONF_SENDSPIN_ID, + DEFAULT_CODECS, MEMORY_LOCATIONS, + OPUS_SAMPLE_RATE, SendspinHub, register_player_config, request_controller_support, @@ -49,10 +54,32 @@ DisableStaticDelayAdjustmentAction = sendspin_ns.class_( ) +def _resolve_codecs(config: ConfigType) -> ConfigType: + """Validate the codec preference list, filling in the default when it is not set.""" + sample_rate = config[CONF_SAMPLE_RATE] + if (codecs := config.get(CONF_CODECS)) is None: + config[CONF_CODECS] = [ + codec + for codec in DEFAULT_CODECS + if codec != CODEC_OPUS or sample_rate == OPUS_SAMPLE_RATE + ] + return config + + if len(set(codecs)) != len(codecs): + raise cv.Invalid("Each codec may only be listed once", path=[CONF_CODECS]) + if CODEC_OPUS in codecs and sample_rate != OPUS_SAMPLE_RATE: + raise cv.Invalid( + f"Codec '{CODEC_OPUS}' requires a {CONF_SAMPLE_RATE} of {OPUS_SAMPLE_RATE}", + path=[CONF_CODECS], + ) + return config + + def _register(config: ConfigType) -> ConfigType: request_controller_support() register_player_config( { + CONF_CODECS: config[CONF_CODECS], CONF_SAMPLE_RATE: config[CONF_SAMPLE_RATE], CONF_BUFFER_SIZE: config[CONF_BUFFER_SIZE], CONF_INITIAL_STATIC_DELAY: config[CONF_INITIAL_STATIC_DELAY], @@ -85,9 +112,13 @@ CONFIG_SCHEMA = cv.All( min=16000, max=96000 ), cv.Optional(CONF_DECODE_MEMORY): cv.one_of(*MEMORY_LOCATIONS, lower=True), + cv.Optional(CONF_CODECS): cv.All( + cv.ensure_list(cv.enum(CODECS, lower=True)), cv.Length(min=1) + ), } ), cv.only_on_esp32, + _resolve_codecs, _register, ) diff --git a/tests/component_tests/sendspin/test_media_source.py b/tests/component_tests/sendspin/test_media_source.py new file mode 100644 index 0000000000..6c2f79198d --- /dev/null +++ b/tests/component_tests/sendspin/test_media_source.py @@ -0,0 +1,90 @@ +"""Validation tests for the sendspin media_source platform. + +These cover the codec preference list, whose rejection branches a compile test +cannot reach: a `test*.yaml` can only assert that a configuration is accepted. +""" + +from typing import Any + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import CONF_CODECS, _get_data +from esphome.components.sendspin.media_source import CONFIG_SCHEMA +from esphome.const import PlatformFramework +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _media_source_config(**overrides: Any) -> ConfigType: + """Build a minimal valid media source config, allowing field overrides.""" + config: ConfigType = { + "id": "sendspin_media_source", + "sendspin_id": "sendspin_hub", + } + config.update(overrides) + return config + + +def test_default_codecs_at_48_khz(set_core_config: SetCoreConfigCallable) -> None: + """Every codec is advertised when the sample rate suits all of them.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config()) + + assert config[CONF_CODECS] == ["flac", "opus", "pcm"] + + +def test_default_codecs_drop_opus_at_other_rates( + set_core_config: SetCoreConfigCallable, +) -> None: + """Opus only supports 48 kHz, so it leaves the default list at other rates.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config(sample_rate=44100)) + + assert config[CONF_CODECS] == ["flac", "pcm"] + + +def test_configured_order_is_preserved(set_core_config: SetCoreConfigCallable) -> None: + """The list is a preference order, so it reaches the player role as written.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA(_media_source_config(codecs=["pcm", "flac"])) + + assert _get_data().player_config[CONF_CODECS] == ["pcm", "flac"] + + +def test_empty_codec_list_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A player with no codecs at all could never be given a stream.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="length of value must be at least 1"): + CONFIG_SCHEMA(_media_source_config(codecs=[])) + + +def test_duplicate_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A repeated codec has no meaning in a preference order.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="may only be listed once"): + CONFIG_SCHEMA(_media_source_config(codecs=["flac", "flac"])) + + +def test_unknown_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """Only codecs the player role can decode are accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="Unknown value"): + CONFIG_SCHEMA(_media_source_config(codecs=["mp3"])) + + +def test_opus_at_wrong_sample_rate_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + """Asking for Opus at a rate it cannot handle fails rather than silently + dropping the stated preference.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="requires a sample_rate of 48000"): + CONFIG_SCHEMA(_media_source_config(codecs=["opus"], sample_rate=44100)) diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 1977b79c04..0c136fbd43 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,3 +9,4 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal + codecs: [pcm, opus, flac] From 008677298ada0a95adeef14c5ac88d1895d5fb2f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:54:44 +1200 Subject: [PATCH 29/41] Bump version to 2026.9.0b3 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 060de51d3a..97ce92240c 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0b2 +PROJECT_NUMBER = 2026.9.0b3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 287804ace3..b013098f33 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0b2" +__version__ = "2026.9.0b3" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From fd598057efdfa689a10b53329753a936a15016a1 Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Wed, 9 Sep 2026 14:22:23 +0200 Subject: [PATCH 30/41] [sendspin] Fix codec enum codegen when codecs is not set (#19055) --- esphome/components/sendspin/__init__.py | 2 +- tests/components/sendspin/common-media_source.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 8ef11a7f90..c1970ab132 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -307,7 +307,7 @@ async def to_code(config: ConfigType) -> None: player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - codecs = player_cfg[CONF_CODECS] + codecs = [CODECS[codec] for codec in player_cfg[CONF_CODECS]] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 0c136fbd43..1977b79c04 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,4 +9,3 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal - codecs: [pcm, opus, flac] From 58ca3456845ca130ac106796225e8ea4cb9c5107 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:42:09 -0400 Subject: [PATCH 31/41] [ci] Refresh integration test durations (#19049) --- .../integration_test_durations.json | 293 +++++++++--------- 1 file changed, 152 insertions(+), 141 deletions(-) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json index 5a5aac3b22..b4a7f4e1ae 100644 --- a/tests/integration/integration_test_durations.json +++ b/tests/integration/integration_test_durations.json @@ -1,143 +1,154 @@ { - "tests/integration/test_action_concurrent_reentry.py": 57.91, - "tests/integration/test_addressable_light_transition.py": 21.25, - "tests/integration/test_alarm_control_panel_state_transitions.py": 70.71, - "tests/integration/test_api_action_metadata.py": 66.6, - "tests/integration/test_api_action_responses.py": 36.1, - "tests/integration/test_api_action_timeout.py": 68.86, - "tests/integration/test_api_conditional_memory.py": 15.48, - "tests/integration/test_api_custom_services.py": 18.77, - "tests/integration/test_api_get_time_response_timezone.py": 21.08, - "tests/integration/test_api_homeassistant.py": 65.59, - "tests/integration/test_api_homeassistant_action_no_subscriber.py": 18.44, - "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 15.05, - "tests/integration/test_api_list_entities_backpressure.py": 13.88, - "tests/integration/test_api_message_size_batching.py": 29.98, - "tests/integration/test_api_reboot_timeout.py": 16.05, - "tests/integration/test_api_string_lambda.py": 15.31, - "tests/integration/test_api_vv_logging.py": 19.28, - "tests/integration/test_api_zero_psk_provisioning.py": 31.5, - "tests/integration/test_areas_and_devices.py": 24.95, - "tests/integration/test_automation_wait_actions.py": 20.92, - "tests/integration/test_automations.py": 35.19, - "tests/integration/test_batch_delay_zero_rapid_transitions.py": 17.99, - "tests/integration/test_binary_sensor_autorepeat_filter.py": 20.39, - "tests/integration/test_binary_sensor_invalidate_state.py": 18.41, - "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 24.69, - "tests/integration/test_build_info.py": 18.7, - "tests/integration/test_camera_mock.py": 16.23, - "tests/integration/test_climate_control_action.py": 21.14, - "tests/integration/test_climate_custom_modes.py": 20.74, - "tests/integration/test_continuation_actions.py": 16.81, - "tests/integration/test_cover_control_action.py": 20.34, - "tests/integration/test_crc8_helper.py": 9.36, - "tests/integration/test_device_id_in_state.py": 44.67, - "tests/integration/test_duplicate_entities.py": 23.58, - "tests/integration/test_entity_icon.py": 34.35, - "tests/integration/test_fan_turn_on_action.py": 24.23, - "tests/integration/test_fnv1_hash_object_id.py": 16.21, - "tests/integration/test_fnv1a_hash.py": 13.38, - "tests/integration/test_gpio_expander_cache.py": 13.06, - "tests/integration/test_host_logger_thread_safety.py": 23.66, - "tests/integration/test_host_mode_basic.py": 8.01, - "tests/integration/test_host_mode_batch_delay.py": 21.0, - "tests/integration/test_host_mode_climate_basic_state.py": 22.14, - "tests/integration/test_host_mode_climate_control.py": 19.39, - "tests/integration/test_host_mode_empty_string_options.py": 21.76, - "tests/integration/test_host_mode_entity_fields.py": 29.61, - "tests/integration/test_host_mode_fan_preset.py": 20.01, - "tests/integration/test_host_mode_many_entities.py": 39.08, - "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.92, - "tests/integration/test_host_mode_noise_encryption.py": 42.42, - "tests/integration/test_host_mode_reconnect.py": 3.41, - "tests/integration/test_host_mode_sensor.py": 22.96, - "tests/integration/test_host_ota.py": 29.5, - "tests/integration/test_host_preferences.py": 16.06, - "tests/integration/test_host_preferences_suspend_resume.py": 18.71, - "tests/integration/test_improv_serial_uart.py": 20.22, - "tests/integration/test_large_message_batching.py": 26.56, - "tests/integration/test_legacy_area.py": 22.72, - "tests/integration/test_legacy_climate_compat.py": 14.13, - "tests/integration/test_legacy_fan_compat.py": 14.33, - "tests/integration/test_light_automations.py": 18.81, - "tests/integration/test_light_binary_effect_off_phase.py": 8.38, - "tests/integration/test_light_calls.py": 21.88, - "tests/integration/test_light_constant_brightness.py": 59.45, - "tests/integration/test_light_control_action.py": 31.91, - "tests/integration/test_light_dim_relative_action.py": 14.43, - "tests/integration/test_light_effect_zero_brightness.py": 25.05, - "tests/integration/test_light_initial_state.py": 18.97, - "tests/integration/test_light_toggle_action.py": 17.44, - "tests/integration/test_lock_automations.py": 18.9, - "tests/integration/test_logger_buffered_recursion_guard.py": 18.2, - "tests/integration/test_loop_disable_enable.py": 63.35, - "tests/integration/test_loop_interval_decoupling.py": 17.7, - "tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.56, - "tests/integration/test_micros_to_millis.py": 15.89, - "tests/integration/test_multi_click_trigger.py": 17.23, - "tests/integration/test_multi_device_preferences.py": 19.4, - "tests/integration/test_noise_encryption_key_protection.py": 72.59, - "tests/integration/test_object_id_api_verification.py": 19.22, - "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 16.77, - "tests/integration/test_object_id_no_friendly_name.py": 45.8, - "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 86.73, - "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 40.4, - "tests/integration/test_online_image_bmp.py": 37.24, - "tests/integration/test_oversized_payloads.py": 55.75, - "tests/integration/test_preference_key_stability.py": 25.49, - "tests/integration/test_runtime_stats.py": 29.81, - "tests/integration/test_safe_mode_loop_runs.py": 6.26, - "tests/integration/test_scheduler_blocking_warning.py": 37.98, - "tests/integration/test_scheduler_bulk_cleanup.py": 18.67, - "tests/integration/test_scheduler_defer_cancel.py": 18.46, - "tests/integration/test_scheduler_defer_cancel_regular.py": 16.34, - "tests/integration/test_scheduler_defer_fifo_simple.py": 18.26, - "tests/integration/test_scheduler_defer_stress.py": 17.74, - "tests/integration/test_scheduler_heap_stress.py": 3.89, - "tests/integration/test_scheduler_internal_id_no_collision.py": 20.01, - "tests/integration/test_scheduler_interval_reschedule.py": 16.29, - "tests/integration/test_scheduler_interval_zero_coerced.py": 16.09, - "tests/integration/test_scheduler_null_name.py": 14.69, - "tests/integration/test_scheduler_numeric_id_test.py": 17.08, - "tests/integration/test_scheduler_pool.py": 19.88, - "tests/integration/test_scheduler_rapid_cancellation.py": 4.42, - "tests/integration/test_scheduler_recursive_timeout.py": 4.3, - "tests/integration/test_scheduler_removed_item_race.py": 15.49, - "tests/integration/test_scheduler_self_keyed.py": 25.77, - "tests/integration/test_scheduler_simultaneous_callbacks.py": 14.84, - "tests/integration/test_scheduler_string_test.py": 15.42, - "tests/integration/test_script_array_params.py": 12.73, - "tests/integration/test_script_delay_params.py": 12.69, - "tests/integration/test_script_queued.py": 20.38, - "tests/integration/test_script_queued_idle_loop.py": 25.06, - "tests/integration/test_script_wait_on_boot.py": 15.67, - "tests/integration/test_select_stringref_trigger.py": 19.48, - "tests/integration/test_sensor_filters_delta.py": 27.62, - "tests/integration/test_sensor_filters_ring_buffer.py": 20.27, - "tests/integration/test_sensor_filters_sliding_window.py": 56.28, - "tests/integration/test_sensor_filters_value_list.py": 20.6, - "tests/integration/test_sensor_timeout_filter.py": 22.21, - "tests/integration/test_socket_wake_gate_tcp.py": 16.37, - "tests/integration/test_status_flags.py": 29.68, - "tests/integration/test_strftime_to.py": 17.42, - "tests/integration/test_syslog.py": 18.39, - "tests/integration/test_template_alarm_control_panel_many_sensors.py": 25.61, - "tests/integration/test_template_text_save.py": 19.16, - "tests/integration/test_text_command.py": 16.43, - "tests/integration/test_text_sensor_raw_state.py": 17.19, - "tests/integration/test_uart_mock_ld2410.py": 37.0, - "tests/integration/test_uart_mock_ld2412.py": 40.82, - "tests/integration/test_uart_mock_ld2420.py": 32.7, - "tests/integration/test_uart_mock_ld2450.py": 32.84, - "tests/integration/test_uart_mock_modbus.py": 548.87, - "tests/integration/test_udp.py": 16.67, - "tests/integration/test_use_address_runtime.py": 27.26, - "tests/integration/test_valve_control_action.py": 24.58, - "tests/integration/test_varint_five_byte_device_id.py": 22.5, - "tests/integration/test_wait_until_mid_loop_timing.py": 22.05, - "tests/integration/test_wait_until_on_boot.py": 10.37, - "tests/integration/test_wait_until_ordering.py": 18.23, - "tests/integration/test_wait_until_reentrant_restart.py": 19.35, - "tests/integration/test_wake_loop_forces_phase_b.py": 17.83, - "tests/integration/test_water_heater_template.py": 25.7 + "tests/integration/test_action_concurrent_reentry.py": 30.48, + "tests/integration/test_addressable_light_transition.py": 42.1, + "tests/integration/test_alarm_control_panel_state_transitions.py": 35.76, + "tests/integration/test_api_action_metadata.py": 22.35, + "tests/integration/test_api_action_responses.py": 30.31, + "tests/integration/test_api_action_timeout.py": 34.73, + "tests/integration/test_api_conditional_memory.py": 18.35, + "tests/integration/test_api_custom_services.py": 15.99, + "tests/integration/test_api_get_time_response_timezone.py": 24.21, + "tests/integration/test_api_homeassistant.py": 33.77, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 20.8, + "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 23.55, + "tests/integration/test_api_list_entities_backpressure.py": 23.04, + "tests/integration/test_api_message_size_batching.py": 27.31, + "tests/integration/test_api_reboot_timeout.py": 29.32, + "tests/integration/test_api_string_lambda.py": 14.9, + "tests/integration/test_api_vv_logging.py": 26.25, + "tests/integration/test_api_zero_psk_provisioning.py": 38.19, + "tests/integration/test_areas_and_devices.py": 27.52, + "tests/integration/test_automation_wait_actions.py": 24.25, + "tests/integration/test_automations.py": 36.02, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 18.46, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 17.47, + "tests/integration/test_binary_sensor_invalidate_state.py": 14.79, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 21.52, + "tests/integration/test_build_info.py": 21.42, + "tests/integration/test_camera_mock.py": 17.02, + "tests/integration/test_climate_control_action.py": 26.56, + "tests/integration/test_climate_custom_modes.py": 18.82, + "tests/integration/test_continuation_actions.py": 20.39, + "tests/integration/test_cover_control_action.py": 19.91, + "tests/integration/test_crc8_helper.py": 16.73, + "tests/integration/test_device_id_in_state.py": 58.41, + "tests/integration/test_duplicate_entities.py": 30.76, + "tests/integration/test_entity_icon.py": 25.34, + "tests/integration/test_fan_turn_on_action.py": 23.64, + "tests/integration/test_fnv1_hash_object_id.py": 25.44, + "tests/integration/test_fnv1a_hash.py": 20.85, + "tests/integration/test_gpio_expander_cache.py": 14.42, + "tests/integration/test_host_logger_thread_safety.py": 21.31, + "tests/integration/test_host_mode_basic.py": 2.65, + "tests/integration/test_host_mode_batch_delay.py": 22.21, + "tests/integration/test_host_mode_climate_basic_state.py": 27.12, + "tests/integration/test_host_mode_climate_control.py": 21.57, + "tests/integration/test_host_mode_empty_string_options.py": 27.17, + "tests/integration/test_host_mode_entity_fields.py": 30.1, + "tests/integration/test_host_mode_fan_preset.py": 17.55, + "tests/integration/test_host_mode_many_entities.py": 38.98, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.82, + "tests/integration/test_host_mode_noise_encryption.py": 39.84, + "tests/integration/test_host_mode_reconnect.py": 13.1, + "tests/integration/test_host_mode_sensor.py": 22.17, + "tests/integration/test_host_ota.py": 92.05, + "tests/integration/test_host_preferences.py": 20.29, + "tests/integration/test_host_preferences_suspend_resume.py": 15.02, + "tests/integration/test_improv_serial_uart.py": 30.15, + "tests/integration/test_large_message_batching.py": 25.84, + "tests/integration/test_legacy_area.py": 21.24, + "tests/integration/test_legacy_climate_compat.py": 17.34, + "tests/integration/test_legacy_fan_compat.py": 22.6, + "tests/integration/test_light_automations.py": 29.13, + "tests/integration/test_light_binary_effect_off_phase.py": 33.99, + "tests/integration/test_light_calls.py": 26.81, + "tests/integration/test_light_constant_brightness.py": 25.0, + "tests/integration/test_light_control_action.py": 25.57, + "tests/integration/test_light_dim_relative_action.py": 21.4, + "tests/integration/test_light_effect_zero_brightness.py": 19.65, + "tests/integration/test_light_initial_state.py": 17.58, + "tests/integration/test_light_toggle_action.py": 28.28, + "tests/integration/test_lock_automations.py": 23.3, + "tests/integration/test_logger_buffered_recursion_guard.py": 22.96, + "tests/integration/test_loop_disable_enable.py": 16.18, + "tests/integration/test_loop_interval_decoupling.py": 25.19, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 20.59, + "tests/integration/test_lvgl_headless_render.py": 87.78, + "tests/integration/test_micros_to_millis.py": 18.73, + "tests/integration/test_multi_click_trigger.py": 24.2, + "tests/integration/test_multi_device_preferences.py": 20.52, + "tests/integration/test_noise_encryption_key_protection.py": 19.1, + "tests/integration/test_object_id_api_verification.py": 26.24, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 14.88, + "tests/integration/test_object_id_no_friendly_name.py": 61.27, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 82.32, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 46.03, + "tests/integration/test_online_image_bmp.py": 34.21, + "tests/integration/test_oversized_payloads.py": 62.75, + "tests/integration/test_preference_key_stability.py": 26.8, + "tests/integration/test_runtime_stats.py": 28.26, + "tests/integration/test_safe_mode_loop_runs.py": 18.14, + "tests/integration/test_scheduler_blocking_warning.py": 28.7, + "tests/integration/test_scheduler_bulk_cleanup.py": 20.73, + "tests/integration/test_scheduler_defer_cancel.py": 22.99, + "tests/integration/test_scheduler_defer_cancel_regular.py": 21.97, + "tests/integration/test_scheduler_defer_fifo_simple.py": 24.15, + "tests/integration/test_scheduler_defer_stress.py": 23.11, + "tests/integration/test_scheduler_heap_stress.py": 20.2, + "tests/integration/test_scheduler_internal_id_no_collision.py": 23.75, + "tests/integration/test_scheduler_interval_reschedule.py": 15.32, + "tests/integration/test_scheduler_interval_zero_coerced.py": 20.1, + "tests/integration/test_scheduler_null_name.py": 17.43, + "tests/integration/test_scheduler_numeric_id_test.py": 25.51, + "tests/integration/test_scheduler_pool.py": 24.22, + "tests/integration/test_scheduler_rapid_cancellation.py": 24.01, + "tests/integration/test_scheduler_recursive_timeout.py": 22.94, + "tests/integration/test_scheduler_removed_item_race.py": 23.07, + "tests/integration/test_scheduler_self_keyed.py": 18.43, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 21.99, + "tests/integration/test_scheduler_string_test.py": 17.27, + "tests/integration/test_script_array_params.py": 4.59, + "tests/integration/test_script_delay_params.py": 22.46, + "tests/integration/test_script_queued.py": 25.24, + "tests/integration/test_script_queued_idle_loop.py": 5.04, + "tests/integration/test_script_wait_on_boot.py": 21.77, + "tests/integration/test_sdl_headless_screenshot.py": 19.23, + "tests/integration/test_select_stringref_trigger.py": 19.31, + "tests/integration/test_sensor_filters_delta.py": 25.92, + "tests/integration/test_sensor_filters_ring_buffer.py": 22.39, + "tests/integration/test_sensor_filters_sliding_window.py": 57.93, + "tests/integration/test_sensor_filters_value_list.py": 20.32, + "tests/integration/test_sensor_timeout_filter.py": 25.35, + "tests/integration/test_snapshot_display.py": 19.7, + "tests/integration/test_socket_wake_gate_tcp.py": 14.5, + "tests/integration/test_status_flags.py": 33.83, + "tests/integration/test_strftime_to.py": 17.64, + "tests/integration/test_syslog.py": 24.49, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 24.81, + "tests/integration/test_template_climate_basic.py": 15.28, + "tests/integration/test_template_climate_custom_modes.py": 25.07, + "tests/integration/test_template_climate_nonoptimistic.py": 24.25, + "tests/integration/test_template_climate_on_control_ordering.py": 24.09, + "tests/integration/test_template_climate_publish_all_fields.py": 17.78, + "tests/integration/test_template_climate_sensor_push.py": 17.42, + "tests/integration/test_template_climate_set_actions.py": 23.63, + "tests/integration/test_template_climate_two_point_temperature.py": 25.19, + "tests/integration/test_template_text_save.py": 17.88, + "tests/integration/test_text_command.py": 22.71, + "tests/integration/test_text_sensor_raw_state.py": 25.17, + "tests/integration/test_uart_mock_ld2410.py": 58.15, + "tests/integration/test_uart_mock_ld2412.py": 61.14, + "tests/integration/test_uart_mock_ld2420.py": 33.87, + "tests/integration/test_uart_mock_ld2450.py": 26.06, + "tests/integration/test_uart_mock_modbus.py": 391.79, + "tests/integration/test_udp.py": 7.38, + "tests/integration/test_use_address_runtime.py": 24.09, + "tests/integration/test_valve_control_action.py": 23.22, + "tests/integration/test_varint_five_byte_device_id.py": 17.93, + "tests/integration/test_wait_until_mid_loop_timing.py": 22.26, + "tests/integration/test_wait_until_on_boot.py": 17.46, + "tests/integration/test_wait_until_ordering.py": 11.89, + "tests/integration/test_wait_until_reentrant_restart.py": 22.88, + "tests/integration/test_wake_loop_forces_phase_b.py": 16.6, + "tests/integration/test_water_heater_template.py": 19.66 } From c66fa812086f20aead9e56bf42b15f541b2e5bc8 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:09:49 +1200 Subject: [PATCH 32/41] [core] Consolidate setup scripts into a cross-platform setup.py (#18856) --- script/git-hooks/post-checkout | 46 ++- script/setup | 74 +---- script/setup.bat | 29 +- script/setup.py | 222 +++++++++++++ tests/script/test_setup.py | 562 +++++++++++++++++++++++++++++++++ 5 files changed, 827 insertions(+), 106 deletions(-) create mode 100755 script/setup.py create mode 100644 tests/script/test_setup.py diff --git a/script/git-hooks/post-checkout b/script/git-hooks/post-checkout index 853c2b0352..73c1cb0f13 100755 --- a/script/git-hooks/post-checkout +++ b/script/git-hooks/post-checkout @@ -1,27 +1,49 @@ #!/bin/sh # Prepare the dev environment for a new checkout or worktree. # -# Installed into the git hooks directory by script/setup. Deliberately tiny and -# self-contained: it stays valid on branches where script/setup does not exist, -# and simply does nothing there. +# Installed into the git hooks directory by script/setup.py. Deliberately tiny +# and self-contained: it stays valid on branches where the setup script does not +# exist, and simply does nothing there. # $3 is 1 for a branch checkout, 0 for a file checkout. [ "$3" = "1" ] || exit 0 top=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0 -# This also runs on ordinary branch switches, where there is nothing to do. +# This also runs on ordinary branch switches, where there is nothing to do. Both +# layouts are checked because git for Windows runs hooks under its own bundled +# shell, where the environment lives in venv/Scripts rather than venv/bin. [ -x "$top/venv/bin/python" ] && exit 0 -[ -x "$top/script/setup" ] || exit 0 +[ -f "$top/venv/Scripts/python.exe" ] && exit 0 + +# Branches from before the setup script moved to Python carry only the shell +# entry point, so whichever one the checked out branch has is used. +py= +if [ -f "$top/script/setup.py" ]; then + # The interpreter goes by different names across platforms, and on Windows + # "python3" is often a stub that opens the app store instead of running + # anything, so each candidate is tried before it is used. Doing nothing is the + # right outcome when none of them work. + for candidate in "python3" "python" "py -3"; do + # Unquoted on purpose: the launcher candidate is a command plus a flag. + if $candidate -c "" >/dev/null 2>&1; then + py=$candidate + break + fi + done + [ -n "$py" ] || exit 0 +elif ! [ -x "$top/script/setup" ]; then + exit 0 +fi # Every worktree shares the hooks directory of the checkout it was created -# from, and the script/setup run below is the one from whichever branch was just +# from, and the setup script run below is the one from whichever branch was just # checked out. Older branches install their own pre-commit hook without checking # for a worktree: that moves the shared hook aside as pre-commit.legacy and # replaces it with one tied to this worktree's virtual environment, so commits # break in every checkout. To rule that out, the hooks directory is copied -# before script/setup runs and put back exactly as it was afterwards, including -# removing any file script/setup added. +# before the setup script runs and put back exactly as it was afterwards, +# including removing any file the setup script added. hooks=$(git rev-parse --path-format=absolute --git-path hooks 2>/dev/null) || exit 0 snap=$(mktemp -d "$hooks/.post-checkout.XXXXXX") || exit 0 cp -p "$hooks"/* "$snap"/ 2>/dev/null @@ -29,7 +51,13 @@ cp -p "$hooks"/* "$snap"/ 2>/dev/null # Clear VIRTUAL_ENV so a checkout made from a shell with an environment already # activated still gets its own, rather than having the active one repointed at # this working tree. -env -u VIRTUAL_ENV "$top/script/setup" +unset VIRTUAL_ENV +if [ -n "$py" ]; then + # Unquoted on purpose, as above. + $py "$top/script/setup.py" +else + "$top/script/setup" +fi status=$? for f in "$hooks"/*; do diff --git a/script/setup b/script/setup index b96af6e8f3..91bcb88154 100755 --- a/script/setup +++ b/script/setup @@ -1,71 +1,7 @@ #!/usr/bin/env bash -# Set up ESPHome dev environment +# Set up ESPHome dev environment. +# +# The work is done by setup.py, which script/setup.bat also runs, so the Unix +# and Windows entry points share one implementation. -set -e - -cd "$(dirname "$0")/.." -if [ -n "$VIRTUAL_ENV" ]; then - # A virtual environment is already active (e.g. the devcontainer's pre-provisioned - # esphome-venv). Install into it rather than creating a ./venv in the workspace. - venv_state=active -elif [ -x venv/bin/python ]; then - # Reuse the environment from an earlier run, so this script can be run again - # at any time to pick up dependency changes. - venv_state=reused - source venv/bin/activate -else - venv_state=created - # --clear replaces a partial environment left behind by an interrupted run. - if [ -x "$(command -v uv)" ]; then - uv venv --clear --seed venv - else - python3 -m venv --clear venv - fi - source venv/bin/activate -fi - -if ! [ -x "$(command -v uv)" ]; then - python3 -m pip install uv -fi - -uv pip install setuptools wheel -uv pip install -e ".[dev,test]" --config-settings editable_mode=compat - -# A worktree shares one git hooks directory with the main checkout it was -# created from, so hooks are installed from the main checkout only. Installing -# from a worktree would point the shared hook at that worktree's virtual -# environment, breaking it for everyone once the worktree is removed. -git_dir="$(git rev-parse --absolute-git-dir 2>/dev/null || true)" -common_dir="$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" -if [ -n "$common_dir" ] && [ "$git_dir" = "$common_dir" ]; then - # --overwrite replaces any hook already in place. Without it, prek finds a - # previously installed pre-commit hook, moves it aside to - # .git/hooks/pre-commit.legacy and keeps calling it, so every commit would - # run both tools. - prek install --overwrite - - # Prepares the virtual environment for new checkouts and worktrees. Installed - # once here, it covers every worktree created from this checkout. - if [ -d "$common_dir/hooks" ]; then - cp script/git-hooks/post-checkout "$common_dir/hooks/post-checkout" - chmod +x "$common_dir/hooks/post-checkout" - fi -fi - -mkdir -p .temp - -echo -echo -case "$venv_state" in - created) - echo "Virtual environment created at ./venv. Run 'source venv/bin/activate' to use it." - ;; - reused) - echo "Dependencies updated in the existing ./venv. Run 'source venv/bin/activate' to use it." - ;; - active) - echo "Dependencies installed into the active virtual environment:" - echo " $VIRTUAL_ENV" - echo "It is already active in this shell, so no 'source venv/bin/activate' is needed." - ;; -esac +exec python3 "$(dirname "$0")/setup.py" "$@" diff --git a/script/setup.bat b/script/setup.bat index 809d05ae93..405121b139 100644 --- a/script/setup.bat +++ b/script/setup.bat @@ -1,28 +1 @@ -@echo off - -if defined VIRTUAL_ENV goto :install - -echo Starting the Virtual Environment -python -m venv venv -call venv/Scripts/activate -echo Running the Virtual Environment - -:install - -echo Installing required packages... - -python.exe -m pip install --upgrade pip - -pip3 install -r requirements.txt -r requirements_test.txt -r requirements_dev.txt -pip3 install setuptools wheel -pip3 install -e ".[dev,test]" --config-settings editable_mode=compat - -rem --overwrite replaces any hook already in place. Without it, prek finds a -rem previously installed pre-commit hook, moves it aside to -rem .git/hooks/pre-commit.legacy and keeps calling it, so every commit would -rem run both tools. -prek install --overwrite - -echo . -echo . -echo Virtual environment created. Run 'venv/Scripts/activate' to use it. +@python "%~dp0setup.py" %* diff --git a/script/setup.py b/script/setup.py new file mode 100755 index 0000000000..62129b8c05 --- /dev/null +++ b/script/setup.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Set up the ESPHome development environment. + +Shared implementation behind script/setup and script/setup.bat, so the Unix and +Windows entry points cannot drift apart. Uses only the standard library: it runs +before any dependency has been installed. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import sysconfig + +MIN_PYTHON = (3, 12) + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_VENV = ROOT / "venv" +POST_CHECKOUT_HOOK = ROOT / "script" / "git-hooks" / "post-checkout" + +# State of the environment the dependencies end up in, used for the closing +# message. +VENV_ACTIVE = "active" +VENV_REUSED = "reused" +VENV_CREATED = "created" + + +def bin_dir(venv: Path) -> Path: + """Return the directory holding a virtual environment's executables. + + The "venv" scheme resolves to bin on Unix and Scripts on Windows, so the + layout does not have to be hardcoded here. + """ + base = str(venv) + return Path( + sysconfig.get_path("scripts", "venv", vars={"base": base, "platbase": base}) + ) + + +def venv_python(venv: Path) -> Path: + """Return the path to a virtual environment's interpreter.""" + name = "python.exe" if os.name == "nt" else "python" + return bin_dir(venv) / name + + +def run(command: list[str], env: dict[str, str] | None = None) -> None: + """Run a command, aborting the whole script if it fails.""" + print(f"+ {' '.join(command)}", flush=True) + result = subprocess.run(command, cwd=ROOT, env=env, check=False) + if result.returncode != 0: + # Some tools fail without printing anything, so name the step that broke. + print( + f"Failed with exit code {result.returncode}: {command[0]}", file=sys.stderr + ) + raise SystemExit(result.returncode) + + +def git_output(*args: str) -> str: + """Return the trimmed output of a git command, or "" if it cannot be run.""" + try: + result = subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=False + ) + except OSError: + # Git is not required to install the dependencies, only to install hooks. + return "" + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def create_venv(venv: Path) -> None: + """Create a virtual environment, replacing anything already at the path.""" + # --clear replaces a partial environment left behind by an interrupted run. + if (uv := shutil.which("uv")) is not None: + run([uv, "venv", "--clear", "--seed", str(venv)]) + else: + run([sys.executable, "-m", "venv", "--clear", str(venv)]) + + +def venv_environment(venv: Path) -> dict[str, str]: + """Return the environment child processes need to target a virtual env. + + Equivalent to sourcing the environment's activate script: tools such as uv + and prek pick the environment up from VIRTUAL_ENV and PATH. + """ + env = dict(os.environ) + env["VIRTUAL_ENV"] = str(venv) + env.pop("PYTHONHOME", None) + path = str(bin_dir(venv)) + # An empty entry would be appended if PATH is unset, and on Unix that means + # the working directory is searched for executables. + if existing := env.get("PATH"): + path = os.pathsep.join([path, existing]) + env["PATH"] = path + return env + + +def find_uv(venv: Path, env: dict[str, str]) -> str: + """Return the path to uv, installing it into the environment if needed.""" + if (uv := shutil.which("uv", path=env["PATH"])) is not None: + return uv + run([str(venv_python(venv)), "-m", "pip", "install", "uv"], env=env) + if (uv := shutil.which("uv", path=env["PATH"])) is not None: + return uv + raise SystemExit("uv could not be installed, aborting.") + + +def install_dependencies(venv: Path, env: dict[str, str]) -> None: + """Install ESPHome and its development dependencies into the environment.""" + uv = find_uv(venv, env) + run([uv, "pip", "install", "setuptools", "wheel"], env=env) + # The dev and test extras pull in requirements_dev.txt and + # requirements_test.txt, and the package itself pulls in requirements.txt, + # so this single install covers every requirements file. + run( + [ + uv, + "pip", + "install", + "-e", + ".[dev,test]", + "--config-settings", + "editable_mode=compat", + ], + env=env, + ) + + +def install_git_hooks(env: dict[str, str]) -> None: + """Install the git hooks, but only when run from the main checkout. + + A worktree shares one git hooks directory with the main checkout it was + created from. Installing from a worktree would point the shared hook at that + worktree's virtual environment, breaking it for everyone once the worktree is + removed. + """ + git_dir = git_output("rev-parse", "--absolute-git-dir") + common_dir = git_output("rev-parse", "--path-format=absolute", "--git-common-dir") + if not git_dir or not common_dir or Path(git_dir) != Path(common_dir): + return + + prek = shutil.which("prek", path=env["PATH"]) + if prek is None: + raise SystemExit("prek was not installed, aborting.") + # --overwrite replaces any hook already in place. Without it, prek finds a + # previously installed pre-commit hook, moves it aside to + # .git/hooks/pre-commit.legacy and keeps calling it, so every commit would + # run both tools. + run([prek, "install", "--overwrite"], env=env) + + # Prepares the virtual environment for new checkouts and worktrees. Installed + # once here, it covers every worktree created from this checkout. + hooks_dir = Path(common_dir) / "hooks" + if hooks_dir.is_dir(): + installed = hooks_dir / "post-checkout" + shutil.copyfile(POST_CHECKOUT_HOOK, installed) + installed.chmod(0o755) + + +def activate_hint() -> str: + """Return the command that activates the environment this script creates.""" + activate = bin_dir(DEFAULT_VENV).relative_to(ROOT) / "activate" + if os.name == "nt": + return str(activate) + return f"source {activate.as_posix()}" + + +def report(state: str, venv: Path) -> None: + """Print the closing message for the environment that was set up.""" + location = f"./{DEFAULT_VENV.name}" + print() + print() + if state == VENV_ACTIVE: + print("Dependencies installed into the active virtual environment:") + print(f" {venv}") + print( + f"It is already active in this shell, so no '{activate_hint()}' is needed." + ) + elif state == VENV_REUSED: + print( + f"Dependencies updated in the existing {location}. " + f"Run '{activate_hint()}' to use it." + ) + else: + print( + f"Virtual environment created at {location}. " + f"Run '{activate_hint()}' to use it." + ) + + +def main() -> None: + """Set up the development environment.""" + if sys.version_info < MIN_PYTHON: + raise SystemExit( + f"ESPHome needs Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]} or newer, " + f"but this is Python {sys.version.split()[0]}." + ) + + # A virtual environment that is already active (for example the + # devcontainer's pre-provisioned esphome-venv) is installed into rather than + # creating a ./venv in the workspace. + if active := os.environ.get("VIRTUAL_ENV"): + state, venv = VENV_ACTIVE, Path(active) + elif venv_python(DEFAULT_VENV).is_file(): + # Reuse the environment from an earlier run, so this script can be run + # again at any time to pick up dependency changes. + state, venv = VENV_REUSED, DEFAULT_VENV + else: + state, venv = VENV_CREATED, DEFAULT_VENV + create_venv(venv) + + env = venv_environment(venv) + install_dependencies(venv, env) + install_git_hooks(env) + (ROOT / ".temp").mkdir(exist_ok=True) + report(state, venv) + + +if __name__ == "__main__": + main() diff --git a/tests/script/test_setup.py b/tests/script/test_setup.py new file mode 100644 index 0000000000..3e816c4b05 --- /dev/null +++ b/tests/script/test_setup.py @@ -0,0 +1,562 @@ +"""Tests for script/setup.py.""" + +import importlib.util +import os +from pathlib import Path, PurePosixPath, PureWindowsPath +import runpy +import sys +from types import ModuleType +from unittest.mock import Mock, call, patch + +import pytest + +_SCRIPT = Path(__file__).parents[2] / "script" / "setup.py" + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location("script_setup", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def script_setup() -> ModuleType: + """Fresh import of script/setup.py, isolated from other tests.""" + return _load_module() + + +# --- bin_dir / venv_python / activate_hint ----------------------------------- + + +def test_bin_dir_matches_host_layout(script_setup: ModuleType, tmp_path: Path) -> None: + """The venv scheme resolves to Scripts on Windows and bin everywhere else.""" + expected = "Scripts" if os.name == "nt" else "bin" + assert script_setup.bin_dir(tmp_path) == tmp_path / expected + + +# Both flavours are exercised on every host. Pure paths are used because a real +# Path refuses to change flavour: PosixPath cannot be built on Windows, and +# WindowsPath cannot be built on Unix. + + +def test_venv_python_posix(script_setup: ModuleType, tmp_path: Path) -> None: + with ( + patch.object( + script_setup, "bin_dir", return_value=PurePosixPath("/x/venv/bin") + ), + patch.object(script_setup.os, "name", "posix"), + ): + result = script_setup.venv_python(tmp_path) + assert result == PurePosixPath("/x/venv/bin/python") + + +def test_venv_python_nt(script_setup: ModuleType, tmp_path: Path) -> None: + with ( + patch.object( + script_setup, "bin_dir", return_value=PureWindowsPath(r"C:\x\venv\Scripts") + ), + patch.object(script_setup.os, "name", "nt"), + ): + result = script_setup.venv_python(tmp_path) + assert result == PureWindowsPath(r"C:\x\venv\Scripts\python.exe") + + +def test_activate_hint_posix(script_setup: ModuleType) -> None: + with ( + patch.object(script_setup, "ROOT", PurePosixPath("/x")), + patch.object( + script_setup, "bin_dir", return_value=PurePosixPath("/x/venv/bin") + ), + patch.object(script_setup.os, "name", "posix"), + ): + hint = script_setup.activate_hint() + assert hint == "source venv/bin/activate" + + +def test_activate_hint_nt(script_setup: ModuleType) -> None: + with ( + patch.object(script_setup, "ROOT", PureWindowsPath(r"C:\x")), + patch.object( + script_setup, "bin_dir", return_value=PureWindowsPath(r"C:\x\venv\Scripts") + ), + patch.object(script_setup.os, "name", "nt"), + ): + hint = script_setup.activate_hint() + # The nt branch returns str(activate) as-is, skipping the "source " prefix. + assert hint == r"venv\Scripts\activate" + + +# --- run ----------------------------------------------------------------- + + +def test_run_success(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run: + script_setup.run(["echo", "hi"]) + mock_run.assert_called_once_with( + ["echo", "hi"], cwd=script_setup.ROOT, env=None, check=False + ) + + +def test_run_failure_raises_system_exit_with_code( + script_setup: ModuleType, capsys: pytest.CaptureFixture[str] +) -> None: + with ( + patch.object(script_setup.subprocess, "run", return_value=Mock(returncode=7)), + pytest.raises(SystemExit) as excinfo, + ): + script_setup.run(["false"]) + assert excinfo.value.code == 7 + assert "Failed with exit code 7: false" in capsys.readouterr().err + + +# --- git_output ------------------------------------------------------------ + + +def test_git_output_success_strips_stdout(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, + "run", + return_value=Mock(returncode=0, stdout=" /repo/.git \n"), + ) as mock_run: + result = script_setup.git_output("rev-parse", "--absolute-git-dir") + assert result == "/repo/.git" + mock_run.assert_called_once_with( + ["git", "rev-parse", "--absolute-git-dir"], + cwd=script_setup.ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def test_git_output_nonzero_returncode_is_empty(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, + "run", + return_value=Mock(returncode=1, stdout="whatever"), + ): + assert script_setup.git_output("status") == "" + + +def test_git_output_oserror_is_empty(script_setup: ModuleType) -> None: + with patch.object(script_setup.subprocess, "run", side_effect=OSError("no git")): + assert script_setup.git_output("status") == "" + + +# --- create_venv ----------------------------------------------------------- + + +def test_create_venv_uses_uv_when_present( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.create_venv(venv) + mock_run.assert_called_once_with( + ["/usr/bin/uv", "venv", "--clear", "--seed", str(venv)], + cwd=script_setup.ROOT, + env=None, + check=False, + ) + + +def test_create_venv_falls_back_to_venv_module( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + with ( + patch.object(script_setup.shutil, "which", return_value=None), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.create_venv(venv) + mock_run.assert_called_once_with( + [sys.executable, "-m", "venv", "--clear", str(venv)], + cwd=script_setup.ROOT, + env=None, + check=False, + ) + + +# --- venv_environment -------------------------------------------------------- + + +def test_venv_environment_sets_virtual_env_and_prepends_path( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + venv = tmp_path / "venv" + monkeypatch.setenv("PYTHONHOME", "/somewhere") + monkeypatch.setenv("PATH", "/usr/bin:/bin") + env = script_setup.venv_environment(venv) + assert env["VIRTUAL_ENV"] == str(venv) + assert "PYTHONHOME" not in env + expected_prefix = str(script_setup.bin_dir(venv)) + os.pathsep + assert env["PATH"] == expected_prefix + "/usr/bin:/bin" + + +def test_venv_environment_path_fallback_when_unset( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + venv = tmp_path / "venv" + monkeypatch.delenv("PATH", raising=False) + env = script_setup.venv_environment(venv) + # No trailing separator: an empty PATH entry means "search the cwd". + assert env["PATH"] == str(script_setup.bin_dir(venv)) + + +# --- find_uv ----------------------------------------------------------------- + + +def test_find_uv_found_immediately(script_setup: ModuleType, tmp_path: Path) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + result = script_setup.find_uv(venv, env) + assert result == "/usr/bin/uv" + mock_run.assert_not_called() + + +def test_find_uv_installed_then_found(script_setup: ModuleType, tmp_path: Path) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", side_effect=[None, "/usr/bin/uv"]), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + result = script_setup.find_uv(venv, env) + assert result == "/usr/bin/uv" + mock_run.assert_called_once_with( + [str(script_setup.venv_python(venv)), "-m", "pip", "install", "uv"], + cwd=script_setup.ROOT, + env=env, + check=False, + ) + + +def test_find_uv_still_missing_raises_system_exit( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", side_effect=[None, None]), + patch.object(script_setup.subprocess, "run", return_value=Mock(returncode=0)), + pytest.raises(SystemExit, match="uv could not be installed"), + ): + script_setup.find_uv(venv, env) + + +# --- install_dependencies ----------------------------------------------------- + + +def test_install_dependencies_installs_setuptools_then_project( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_dependencies(venv, env) + assert mock_run.call_args_list == [ + call( + ["/usr/bin/uv", "pip", "install", "setuptools", "wheel"], + cwd=script_setup.ROOT, + env=env, + check=False, + ), + call( + [ + "/usr/bin/uv", + "pip", + "install", + "-e", + ".[dev,test]", + "--config-settings", + "editable_mode=compat", + ], + cwd=script_setup.ROOT, + env=env, + check=False, + ), + ] + + +# --- install_git_hooks --------------------------------------------------------- + + +def _fake_git_output(git_dir: str, common_dir: str): + def _run(*args: str) -> str: + if "--absolute-git-dir" in args: + return git_dir + return common_dir + + return _run + + +def test_install_git_hooks_returns_early_when_git_dir_empty( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, "git_output", side_effect=_fake_git_output("", "/repo/.git") + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_returns_early_when_common_dir_empty( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, "git_output", side_effect=_fake_git_output("/repo/.git", "") + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_returns_early_for_worktree( + script_setup: ModuleType, +) -> None: + """A worktree's git-dir differs from the shared common-dir.""" + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output("/repo/.git/worktrees/wt", "/repo/.git"), + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_missing_prek_raises_system_exit( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output("/repo/.git", "/repo/.git"), + ), + patch.object(script_setup.shutil, "which", return_value=None), + patch.object(script_setup.subprocess, "run") as mock_run, + pytest.raises(SystemExit, match="prek was not installed"), + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_happy_path_installs_hook( + script_setup: ModuleType, tmp_path: Path +) -> None: + env = {"PATH": "/usr/bin"} + common_dir = tmp_path / "repo" / ".git" + hooks_dir = common_dir / "hooks" + hooks_dir.mkdir(parents=True) + source_hook = tmp_path / "post-checkout" + source_hook.write_text("#!/bin/sh\necho post-checkout\n") + + with ( + patch.object(script_setup, "POST_CHECKOUT_HOOK", source_hook), + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output(str(common_dir), str(common_dir)), + ), + patch.object(script_setup.shutil, "which", return_value="/usr/bin/prek"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_git_hooks(env) + + mock_run.assert_called_once_with( + ["/usr/bin/prek", "install", "--overwrite"], + cwd=script_setup.ROOT, + env=env, + check=False, + ) + installed = hooks_dir / "post-checkout" + assert installed.read_text() == source_hook.read_text() + if os.name != "nt": + # Windows has no POSIX permission bits for chmod to set. + assert (installed.stat().st_mode & 0o777) == 0o755 + + +def test_install_git_hooks_skips_copy_when_hooks_dir_missing( + script_setup: ModuleType, tmp_path: Path +) -> None: + """The prek install still runs when the hooks directory does not exist.""" + env = {"PATH": "/usr/bin"} + common_dir = tmp_path / "repo" / ".git" + common_dir.mkdir(parents=True) # no "hooks" subdirectory created + + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output(str(common_dir), str(common_dir)), + ), + patch.object(script_setup.shutil, "which", return_value="/usr/bin/prek"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_git_hooks(env) + + mock_run.assert_called_once() + assert not (common_dir / "hooks").exists() + + +# --- report ------------------------------------------------------------------ + + +def test_report_active_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + venv = Path("/opt/esphome-venv") + script_setup.report(script_setup.VENV_ACTIVE, venv) + out = capsys.readouterr().out + assert "Dependencies installed into the active virtual environment:" in out + assert str(venv) in out + assert "is already active in this shell" in out + + +def test_report_reused_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + script_setup.report(script_setup.VENV_REUSED, script_setup.DEFAULT_VENV) + out = capsys.readouterr().out + assert "Dependencies updated in the existing ./venv" in out + + +def test_report_created_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + script_setup.report(script_setup.VENV_CREATED, script_setup.DEFAULT_VENV) + out = capsys.readouterr().out + assert "Virtual environment created at ./venv" in out + + +# --- main -------------------------------------------------------------------- + + +def test_main_raises_system_exit_when_python_too_old( + script_setup: ModuleType, +) -> None: + with ( + patch.object(script_setup.sys, "version_info", (3, 11, 5)), + pytest.raises(SystemExit, match="ESPHome needs Python 3.12"), + ): + script_setup.main() + + +def test_main_uses_active_virtual_env( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + active_venv = tmp_path / "active-venv" + monkeypatch.setenv("VIRTUAL_ENV", str(active_venv)) + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_not_called() + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_ACTIVE, active_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_main_reuses_existing_venv( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + default_venv = tmp_path / "venv" + python_path = script_setup.venv_python(default_venv) + python_path.parent.mkdir(parents=True) + python_path.touch() + + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "DEFAULT_VENV", default_venv), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_not_called() + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_REUSED, default_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_main_creates_new_venv( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + default_venv = tmp_path / "venv" # does not exist yet + + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "DEFAULT_VENV", default_venv), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_called_once_with(default_venv) + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_CREATED, default_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_run_as_script_calls_main(tmp_path: Path) -> None: + """The __main__ guard runs the whole flow, with every side effect stubbed.""" + completed = Mock(returncode=0, stdout="") + with ( + patch("subprocess.run", return_value=completed) as mock_run, + patch("shutil.which", return_value="/usr/bin/uv"), + patch("pathlib.Path.mkdir") as mock_mkdir, + patch.dict(os.environ, {"VIRTUAL_ENV": str(tmp_path / "env")}), + ): + runpy.run_path(str(_SCRIPT), run_name="__main__") + + # The dependency install ran, and git reported no hooks directory to touch. + assert mock_run.called + mock_mkdir.assert_called_once_with(exist_ok=True) From 4868b498cf80cf6fb6c59544a84797f74736bbfe Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:09:58 +1200 Subject: [PATCH 33/41] [ci] Ask stale PR authors to merge dev instead of rebasing (#19064) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index aa31094f81..38d2418ac6 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -33,7 +33,7 @@ jobs: and will be closed if no further activity occurs within 7 days. If you are the author of this PR, please leave a comment if you want - to keep it open. Also, please rebase your PR onto the latest dev + to keep it open. Also, please merge the latest dev branch into your branch to ensure that it's up to date with the latest changes. Thank you for your contribution! From 5ed59af9204af5e4a4638d10379375b598dd8ace Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:11:13 +1200 Subject: [PATCH 34/41] [template] Surface value metadata on template entity forms (#17545) --- .../template/binary_sensor/__init__.py | 14 +++- .../components/template/button/__init__.py | 6 +- esphome/components/template/cover/__init__.py | 7 +- esphome/components/template/event/__init__.py | 6 +- .../components/template/number/__init__.py | 9 ++- .../components/template/sensor/__init__.py | 22 +++++- .../components/template/switch/__init__.py | 7 +- .../template/text_sensor/__init__.py | 8 +- esphome/components/template/valve/__init__.py | 7 +- esphome/config_validation.py | 32 ++++++++ .../template/test_template_visibility.py | 76 +++++++++++++++++++ tests/unit_tests/test_config_validation.py | 29 +++++++ 12 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 tests/component_tests/template/test_template_visibility.py diff --git a/esphome/components/template/binary_sensor/__init__.py b/esphome/components/template/binary_sensor/__init__.py index 8f57df91c5..07028f7dff 100644 --- a/esphome/components/template/binary_sensor/__init__.py +++ b/esphome/components/template/binary_sensor/__init__.py @@ -2,7 +2,13 @@ from esphome import automation import esphome.codegen as cg from esphome.components import binary_sensor import esphome.config_validation as cv -from esphome.const import CONF_CONDITION, CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import ( + CONF_CONDITION, + CONF_DEVICE_CLASS, + CONF_ID, + CONF_LAMBDA, + CONF_STATE, +) from esphome.cpp_generator import LambdaExpression from .. import template_ns @@ -12,7 +18,11 @@ TemplateBinarySensor = template_ns.class_( ) CONFIG_SCHEMA = ( - binary_sensor.binary_sensor_schema(TemplateBinarySensor) + cv.with_visibility( + binary_sensor.binary_sensor_schema(TemplateBinarySensor), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Exclusive(CONF_LAMBDA, CONF_CONDITION): cv.returning_lambda, diff --git a/esphome/components/template/button/__init__.py b/esphome/components/template/button/__init__.py index e0101dfc8f..9c6fa13c19 100644 --- a/esphome/components/template/button/__init__.py +++ b/esphome/components/template/button/__init__.py @@ -1,10 +1,14 @@ from esphome.components import button +import esphome.config_validation as cv +from esphome.const import CONF_DEVICE_CLASS from .. import template_ns TemplateButton = template_ns.class_("TemplateButton", button.Button) -CONFIG_SCHEMA = button.button_schema(TemplateButton) +CONFIG_SCHEMA = cv.with_visibility( + button.button_schema(TemplateButton), cv.Visibility.UI, CONF_DEVICE_CLASS +) async def to_code(config): diff --git a/esphome/components/template/cover/__init__.py b/esphome/components/template/cover/__init__.py index 7cb50df84c..0e6f96e9f5 100644 --- a/esphome/components/template/cover/__init__.py +++ b/esphome/components/template/cover/__init__.py @@ -6,6 +6,7 @@ from esphome.const import ( CONF_ASSUMED_STATE, CONF_CLOSE_ACTION, CONF_CURRENT_OPERATION, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPEN_ACTION, @@ -38,7 +39,11 @@ CONF_HAS_POSITION = "has_position" CONF_TOGGLE_ACTION = "toggle_action" CONFIG_SCHEMA = ( - cover.cover_schema(TemplateCover) + cv.with_visibility( + cover.cover_schema(TemplateCover), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/components/template/event/__init__.py b/esphome/components/template/event/__init__.py index cf9c7f4c3d..bdcbd456d5 100644 --- a/esphome/components/template/event/__init__.py +++ b/esphome/components/template/event/__init__.py @@ -1,7 +1,7 @@ import esphome.codegen as cg from esphome.components import event import esphome.config_validation as cv -from esphome.const import CONF_EVENT_TYPES +from esphome.const import CONF_DEVICE_CLASS, CONF_EVENT_TYPES from .. import template_ns @@ -9,7 +9,9 @@ CODEOWNERS = ["@nohat"] TemplateEvent = template_ns.class_("TemplateEvent", event.Event, cg.Component) -CONFIG_SCHEMA = event.event_schema(TemplateEvent).extend( +CONFIG_SCHEMA = cv.with_visibility( + event.event_schema(TemplateEvent), cv.Visibility.UI, CONF_DEVICE_CLASS +).extend( { cv.Required(CONF_EVENT_TYPES): cv.ensure_list(cv.string_strict), } diff --git a/esphome/components/template/number/__init__.py b/esphome/components/template/number/__init__.py index 2f4c9cbffe..3b6485fec3 100644 --- a/esphome/components/template/number/__init__.py +++ b/esphome/components/template/number/__init__.py @@ -3,6 +3,7 @@ import esphome.codegen as cg from esphome.components import number import esphome.config_validation as cv from esphome.const import ( + CONF_DEVICE_CLASS, CONF_ID, CONF_INITIAL_VALUE, CONF_LAMBDA, @@ -12,6 +13,7 @@ from esphome.const import ( CONF_RESTORE_VALUE, CONF_SET_ACTION, CONF_STEP, + CONF_UNIT_OF_MEASUREMENT, ) from .. import template_ns @@ -46,7 +48,12 @@ def validate(config): CONFIG_SCHEMA = cv.All( - number.number_schema(TemplateNumber) + cv.with_visibility( + number.number_schema(TemplateNumber), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + CONF_UNIT_OF_MEASUREMENT, + ) .extend( { cv.Required(CONF_MAX_VALUE): cv.float_, diff --git a/esphome/components/template/sensor/__init__.py b/esphome/components/template/sensor/__init__.py index 0c875bba0f..55537a5636 100644 --- a/esphome/components/template/sensor/__init__.py +++ b/esphome/components/template/sensor/__init__.py @@ -2,7 +2,16 @@ from esphome import automation import esphome.codegen as cg from esphome.components import sensor import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import ( + CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, + CONF_FORCE_UPDATE, + CONF_ID, + CONF_LAMBDA, + CONF_STATE, + CONF_STATE_CLASS, + CONF_UNIT_OF_MEASUREMENT, +) from .. import template_ns @@ -11,9 +20,14 @@ TemplateSensor = template_ns.class_( ) CONFIG_SCHEMA = ( - sensor.sensor_schema( - TemplateSensor, - accuracy_decimals=1, + cv.with_visibility( + sensor.sensor_schema(TemplateSensor, accuracy_decimals=1), + cv.Visibility.UI, + CONF_UNIT_OF_MEASUREMENT, + CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, + CONF_STATE_CLASS, + CONF_FORCE_UPDATE, ) .extend( { diff --git a/esphome/components/template/switch/__init__.py b/esphome/components/template/switch/__init__.py index ca986365ed..37303abb0d 100644 --- a/esphome/components/template/switch/__init__.py +++ b/esphome/components/template/switch/__init__.py @@ -4,6 +4,7 @@ from esphome.components import switch import esphome.config_validation as cv from esphome.const import ( CONF_ASSUMED_STATE, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPTIMISTIC, @@ -31,7 +32,11 @@ def validate(config): CONFIG_SCHEMA = cv.All( - switch.switch_schema(TemplateSwitch) + cv.with_visibility( + switch.switch_schema(TemplateSwitch), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/components/template/text_sensor/__init__.py b/esphome/components/template/text_sensor/__init__.py index ddbdd6dadb..77f5c2ff7c 100644 --- a/esphome/components/template/text_sensor/__init__.py +++ b/esphome/components/template/text_sensor/__init__.py @@ -3,7 +3,7 @@ import esphome.codegen as cg from esphome.components import text_sensor from esphome.components.text_sensor import TextSensorPublishAction import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_STATE from .. import template_ns @@ -12,7 +12,11 @@ TemplateTextSensor = template_ns.class_( ) CONFIG_SCHEMA = ( - text_sensor.text_sensor_schema() + cv.with_visibility( + text_sensor.text_sensor_schema(), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.GenerateID(): cv.declare_id(TemplateTextSensor), diff --git a/esphome/components/template/valve/__init__.py b/esphome/components/template/valve/__init__.py index a2d0c19880..11b35dad23 100644 --- a/esphome/components/template/valve/__init__.py +++ b/esphome/components/template/valve/__init__.py @@ -6,6 +6,7 @@ from esphome.const import ( CONF_ASSUMED_STATE, CONF_CLOSE_ACTION, CONF_CURRENT_OPERATION, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPEN_ACTION, @@ -36,7 +37,11 @@ CONF_HAS_POSITION = "has_position" CONF_TOGGLE_ACTION = "toggle_action" CONFIG_SCHEMA = ( - valve.valve_schema(TemplateValve) + cv.with_visibility( + valve.valve_schema(TemplateValve), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 685a9d04b3..a38fb2ed82 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -4,6 +4,7 @@ from __future__ import annotations from collections.abc import Callable from contextlib import contextmanager, suppress +import copy from datetime import datetime from ipaddress import ( AddressValueError, @@ -419,6 +420,37 @@ class Required(vol.Required): self.visibility: Visibility | None = visibility +def with_visibility(schema: Schema, visibility: Visibility, *keys: str) -> Schema: + """Return a copy of ``schema`` with the given ``keys`` re-marked at ``visibility``. + + Lets a platform override the editor :class:`Visibility` of fields it + inherits from a shared schema builder — without that builder needing a + visibility parameter of its own. The canonical use is a ``template`` + platform promoting the value metadata its user is expected to define + (``device_class``, ``unit_of_measurement``, …) onto the main form: + + CONFIG_SCHEMA = cv.with_visibility( + sensor.sensor_schema(TemplateSensor), + cv.Visibility.UI, + CONF_DEVICE_CLASS, CONF_UNIT_OF_MEASUREMENT, + ) + + The original marker's key, default and validator are preserved; only the + visibility changes, and the input ``schema`` is left untouched. Raises if + a requested key is not present so typos fail at schema-build time. + """ + wanted = {str(k) for k in keys} + overrides = {} + for marker, validator in schema.schema.items(): + if str(marker) in wanted: + marker = copy.copy(marker) + marker.visibility = visibility + overrides[marker] = validator + if missing := wanted - {str(m) for m in overrides}: + raise ValueError(f"with_visibility: keys not in schema: {sorted(missing)}") + return schema.extend(overrides) + + class FinalExternalInvalid(Invalid): """Represents an invalid value in the final validation phase where the path should not be prepended.""" diff --git a/tests/component_tests/template/test_template_visibility.py b/tests/component_tests/template/test_template_visibility.py new file mode 100644 index 0000000000..a50a27e1f7 --- /dev/null +++ b/tests/component_tests/template/test_template_visibility.py @@ -0,0 +1,76 @@ +"""The template platforms surface value-describing metadata on the main form. + +Hardware platforms get sensible defaults for unit/device_class/etc., so those +fields fall through to the editor's advanced disclosure. A ``template`` entity +has no such defaults -- the user is expected to define them -- so the template +platforms pass ``visibility=cv.Visibility.UI`` to promote them onto the form. +""" + +from __future__ import annotations + +import importlib + +import pytest + +import esphome.config_validation as cv + + +def _markers(schema: cv.Schema) -> dict[str, object]: + s = schema + if hasattr(s, "validators"): + # cv.All -> the schema is the first validator. + s = s.validators[0] + return {str(k): k for k in s.schema} + + +@pytest.mark.parametrize( + ("platform", "fields"), + [ + ( + "sensor", + [ + "unit_of_measurement", + "accuracy_decimals", + "device_class", + "state_class", + "force_update", + ], + ), + ("binary_sensor", ["device_class"]), + ("switch", ["device_class"]), + ("cover", ["device_class"]), + ("button", ["device_class"]), + ("valve", ["device_class"]), + ("event", ["device_class"]), + ("text_sensor", ["device_class"]), + ("number", ["device_class", "unit_of_measurement"]), + ], +) +def test_template_metadata_is_ui(platform: str, fields: list[str]) -> None: + mod = importlib.import_module(f"esphome.components.template.{platform}") + markers = _markers(mod.CONFIG_SCHEMA) + for field in fields: + assert markers[field].visibility is cv.Visibility.UI, f"{platform}.{field}" + + +def test_template_sensor_promotion_preserves_defaults() -> None: + """Promoting to UI must not drop the fields' defaults.""" + from esphome.components.template.sensor import CONFIG_SCHEMA + + markers = _markers(CONFIG_SCHEMA) + assert markers["accuracy_decimals"].default() == 1 + assert markers["force_update"].default() is False + + +def test_hardware_platform_metadata_not_promoted() -> None: + """Without ``visibility=`` the builders leave metadata unset. + + Unset markers fall through to the consumer's ``Optional`` default of + advanced, so hardware platforms are unaffected by the template promotion. + """ + from esphome.components import binary_sensor, sensor + + hw_sensor = _markers(sensor.sensor_schema(device_class="temperature")) + assert hw_sensor["device_class"].visibility is None + hw_bs = _markers(binary_sensor.binary_sensor_schema(device_class="motion")) + assert hw_bs["device_class"].visibility is None diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 457b9d017b..4092b4c0d5 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -1394,6 +1394,35 @@ def test_entity_metadata_visibility_hints() -> None: assert web["web_server"].visibility is advanced +def test_with_visibility_remarks_keys() -> None: + """``with_visibility`` re-marks the named keys, preserving each field's + default and validator, without touching the other keys or the input schema. + """ + base = cv.Schema( + { + cv.Optional("a", default=7): cv.int_, + cv.Optional("b", visibility=cv.Visibility.ADVANCED): cv.string, + } + ) + promoted = cv.with_visibility(base, cv.Visibility.UI, "a") + + pm = {str(k): k for k in promoted.schema} + assert pm["a"].visibility is cv.Visibility.UI # re-marked + assert pm["a"].default() == 7 # default preserved + assert pm["b"].visibility is cv.Visibility.ADVANCED # sibling untouched + assert promoted({}) == {"a": 7} # validator/default still applied + + # The input schema is left untouched (no shared-marker mutation). + assert {str(k): k for k in base.schema}["a"].visibility is None + + +def test_with_visibility_unknown_key_raises() -> None: + """A key not present in the schema is a typo — fail at build time.""" + base = cv.Schema({cv.Optional("a"): cv.int_}) + with pytest.raises(ValueError, match="not in schema"): + cv.with_visibility(base, cv.Visibility.UI, "nope") + + def _wrap_str(value: str) -> ESPHomeDataBase: """Wrap a raw string as an ESPHomeDataBase, mimicking a YAML-loaded value.""" return make_data_base(value) From ddbd89dd2a5bb35b24217dae93a4a45ce5da476f Mon Sep 17 00:00:00 2001 From: Robin Thoni Date: Thu, 10 Sep 2026 06:06:44 +0200 Subject: [PATCH 35/41] [network] Improve `network::is_connected()` to better handle multiple interfaces (#18999) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/network/util.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/esphome/components/network/util.h b/esphome/components/network/util.h index 65a578c22f..57c5a66833 100644 --- a/esphome/components/network/util.h +++ b/esphome/components/network/util.h @@ -26,30 +26,34 @@ namespace esphome::network { /// Return whether the node is connected to the network (through wifi, eth, ...) ESPHOME_ALWAYS_INLINE inline bool is_connected() { + // With a single interface enabled the checks below collapse to `if (x) return true; return false;`, which + // clang-tidy wants folded into one return. Keep the per-interface form so every enabled interface is checked. + // NOLINTBEGIN(readability-simplify-boolean-expr) #ifdef USE_ETHERNET if (ethernet::global_eth_component != nullptr && ethernet::global_eth_component->is_connected()) return true; #endif #ifdef USE_MODEM - if (modem::global_modem_component != nullptr) - return modem::global_modem_component->is_connected(); + if (modem::global_modem_component != nullptr && modem::global_modem_component->is_connected()) + return true; #endif #ifdef USE_WIFI - if (wifi::global_wifi_component != nullptr) - return wifi::global_wifi_component->is_connected(); + if (wifi::global_wifi_component != nullptr && wifi::global_wifi_component->is_connected()) + return true; #endif #ifdef USE_OPENTHREAD - if (openthread::global_openthread_component != nullptr) - return openthread::global_openthread_component->is_connected(); + if (openthread::global_openthread_component != nullptr && openthread::global_openthread_component->is_connected()) + return true; #endif #ifdef USE_HOST return true; // Assume it's connected #endif return false; + // NOLINTEND(readability-simplify-boolean-expr) } /// Return whether the network is disabled: every configured interface with a From 05f7d5e4f1b1d5ff14f0d4c30ce984ed319ca112 Mon Sep 17 00:00:00 2001 From: Anton Sergunov Date: Thu, 10 Sep 2026 10:14:43 +0600 Subject: [PATCH 36/41] [mlx90614] pec validation (#6689) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/mlx90614/mlx90614.cpp | 185 +++++++++++++++++------ esphome/components/mlx90614/mlx90614.h | 7 +- 2 files changed, 145 insertions(+), 47 deletions(-) diff --git a/esphome/components/mlx90614/mlx90614.cpp b/esphome/components/mlx90614/mlx90614.cpp index 2d3b6631bc..508b3743d1 100644 --- a/esphome/components/mlx90614/mlx90614.cpp +++ b/esphome/components/mlx90614/mlx90614.cpp @@ -26,44 +26,129 @@ static const uint8_t MLX90614_ID4 = 0x3F; static const char *const TAG = "mlx90614"; +// The EEPROM cell has a limited number of write cycles, so stop retrying after a few failures +static constexpr uint8_t EMISSIVITY_WRITE_ATTEMPTS = 3; + +// SMBus packet error code: CRC-8 with polynomial 0x07, MSB first +static uint8_t crc8_pec(const uint8_t *data, uint8_t len) { return crc8(data, len, 0x00, 0x07, true); } + void MLX90614Component::setup() { - if (!this->write_emissivity_()) { - ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); - this->mark_failed(); + if (std::isnan(this->emissivity_)) { return; } + this->emissivity_write_attempts_ = EMISSIVITY_WRITE_ATTEMPTS; + this->try_write_emissivity_(); + if (this->emissivity_write_attempts_ != 0) { + this->status_set_warning(LOG_STR("Failed to write emissivity, will retry")); + } +} + +void MLX90614Component::try_write_emissivity_() { + if (this->emissivity_write_attempts_ == 0) { + return; + } + if (this->write_emissivity_()) { + this->emissivity_write_attempts_ = 0; + return; + } + if (--this->emissivity_write_attempts_ == 0) { + ESP_LOGE(TAG, "Giving up on writing emissivity after %u attempts", EMISSIVITY_WRITE_ATTEMPTS); + this->emissivity_write_failed_ = true; + } } bool MLX90614Component::write_emissivity_() { - if (std::isnan(this->emissivity_)) + // Skip the write when the EEPROM already holds the desired value to save write cycles + uint16_t current_emissivity; + if (this->read_register_(MLX90614_EMISSIVITY, current_emissivity) != i2c::ERROR_OK) { + return false; + } + + const auto desired_emissivity = static_cast(this->emissivity_ * 0xFFFF); + if (current_emissivity == desired_emissivity) { return true; - uint16_t value = (uint16_t) (this->emissivity_ * 65535); - if (!this->write_bytes_(MLX90614_EMISSIVITY, 0)) { - return false; } - delay(10); - if (!this->write_bytes_(MLX90614_EMISSIVITY, value)) { - return false; - } - delay(10); - return true; + + return this->write_register_(MLX90614_EMISSIVITY, desired_emissivity); } -bool MLX90614Component::write_bytes_(uint8_t reg, uint16_t data) { +bool MLX90614Component::write_register_(uint8_t reg, uint16_t data) { + // The PEC covers the whole write transaction: SLA+W, command, data low, data high uint8_t buf[5]; buf[0] = this->address_ << 1; buf[1] = reg; - buf[2] = data & 0xFF; - buf[3] = data >> 8; - buf[4] = crc8(buf, 4, 0x00, 0x07, true); - return this->write_bytes(reg, buf + 2, 3); + + // See datasheet 8.3.3.1 EEPROM write sequence + // 1. Write 0x0000 into the cell of interest (erases the cell) + buf[2] = buf[3] = 0; + buf[4] = crc8_pec(buf, 4); + auto ec = this->write_register(reg, buf + 2, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't erase register 0x%02X, error %d", reg, ec); + return false; + } + + // 2. Wait at least 5ms + delay(10); + + // 3. Write the new value + if (data != 0) { + buf[2] = data & 0xFF; + buf[3] = data >> 8; + buf[4] = crc8_pec(buf, 4); + ec = this->write_register(reg, buf + 2, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't write register 0x%02X, error %d", reg, ec); + return false; + } + // 4. Wait at least 5ms + delay(10); + } + + // 5. Read back to confirm the value was stored + uint16_t read_back; + ec = this->read_register_(reg, read_back); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't check register 0x%02X value, error %d", reg, ec); + return false; + } + + if (read_back != data) { + ESP_LOGW(TAG, "Read back mismatch on register 0x%02X. Expected 0x%04X, got 0x%04X", reg, data, read_back); + return false; + } + + return true; +} + +i2c::ErrorCode MLX90614Component::read_register_(uint8_t reg, uint16_t &data) { + // The PEC covers the whole read transaction: SLA+W, command, SLA+R, data low, data high + uint8_t buf[6]; + buf[0] = this->address_ << 1; + buf[1] = reg; + buf[2] = (this->address_ << 1) | 0x01; + + const auto ec = this->read_register(reg, buf + 3, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "i2c read error %d", ec); + return ec; + } + + const auto expected_pec = crc8_pec(buf, 5); + if (buf[5] != expected_pec) { + ESP_LOGW(TAG, "i2c CRC error. Expected 0x%02X, got 0x%02X", expected_pec, buf[5]); + return i2c::ERROR_CRC; + } + + data = encode_uint16(buf[4], buf[3]); + return i2c::ERROR_OK; } void MLX90614Component::dump_config() { ESP_LOGCONFIG(TAG, "MLX90614:"); LOG_I2C_DEVICE(this); - if (this->is_failed()) { - ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); + if (this->emissivity_write_attempts_ != 0) { + ESP_LOGW(TAG, " Emissivity not written yet, will retry"); } LOG_UPDATE_INTERVAL(this); LOG_SENSOR(" ", "Ambient", this->ambient_sensor_); @@ -71,33 +156,41 @@ void MLX90614Component::dump_config() { } void MLX90614Component::update() { - uint8_t emissivity[3]; - if (this->read_register(MLX90614_EMISSIVITY, emissivity, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; + // Temperature reads run regardless of the emissivity state so a failure still shows up as NAN + this->try_write_emissivity_(); + + // Publishes NAN on a bus or CRC failure so a stuck reading is visible instead of silently stale + auto publish_sensor = [this](sensor::Sensor *sensor, uint8_t reg) { + if (sensor == nullptr) { + return i2c::ERROR_OK; + } + + uint16_t raw; + const auto ec = this->read_register_(reg, raw); + if (ec != i2c::ERROR_OK) { + sensor->publish_state(NAN); + return ec; + } + + // Bit 15 set means the device flagged the reading as invalid + const float temperature = (raw & 0x8000) ? NAN : raw * 0.02f - 273.15f; + ESP_LOGD(TAG, "'%s': Got temperature=%.1f°C", sensor->get_name().c_str(), temperature); + sensor->publish_state(temperature); + return ec; + }; + + const auto object_ec = publish_sensor(this->object_sensor_, MLX90614_TEMPERATURE_OBJECT_1); + const auto ambient_ec = publish_sensor(this->ambient_sensor_, MLX90614_TEMPERATURE_AMBIENT); + + if (object_ec != i2c::ERROR_OK || ambient_ec != i2c::ERROR_OK) { + this->status_set_warning(LOG_STR("Failed to read some sensors")); + } else if (this->emissivity_write_failed_) { + this->status_set_warning(LOG_STR("Failed to write emissivity")); + } else if (this->emissivity_write_attempts_ != 0) { + this->status_set_warning(LOG_STR("Failed to write emissivity, will retry")); + } else { + this->status_clear_warning(); } - uint8_t raw_object[3]; - if (this->read_register(MLX90614_TEMPERATURE_OBJECT_1, raw_object, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; - } - - uint8_t raw_ambient[3]; - if (this->read_register(MLX90614_TEMPERATURE_AMBIENT, raw_ambient, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; - } - - float ambient = raw_ambient[1] & 0x80 ? NAN : encode_uint16(raw_ambient[1], raw_ambient[0]) * 0.02f - 273.15f; - float object = raw_object[1] & 0x80 ? NAN : encode_uint16(raw_object[1], raw_object[0]) * 0.02f - 273.15f; - - ESP_LOGD(TAG, "Got Temperature=%.1f°C Ambient=%.1f°C", object, ambient); - - if (this->ambient_sensor_ != nullptr && !std::isnan(ambient)) - this->ambient_sensor_->publish_state(ambient); - if (this->object_sensor_ != nullptr && !std::isnan(object)) - this->object_sensor_->publish_state(object); - this->status_clear_warning(); } } // namespace esphome::mlx90614 diff --git a/esphome/components/mlx90614/mlx90614.h b/esphome/components/mlx90614/mlx90614.h index 882ee45186..758792aced 100644 --- a/esphome/components/mlx90614/mlx90614.h +++ b/esphome/components/mlx90614/mlx90614.h @@ -18,13 +18,18 @@ class MLX90614Component final : public PollingComponent, public i2c::I2CDevice { void set_emissivity(float emissivity) { emissivity_ = emissivity; } protected: + void try_write_emissivity_(); bool write_emissivity_(); - bool write_bytes_(uint8_t reg, uint16_t data); + bool write_register_(uint8_t reg, uint16_t data); + i2c::ErrorCode read_register_(uint8_t reg, uint16_t &data); sensor::Sensor *ambient_sensor_{nullptr}; sensor::Sensor *object_sensor_{nullptr}; float emissivity_{NAN}; + // Remaining attempts to program the emissivity EEPROM cell, bounded to limit cell wear + uint8_t emissivity_write_attempts_{0}; + bool emissivity_write_failed_{false}; }; } // namespace esphome::mlx90614 From 54706e869c13abc0f688a91c0f326c779799b858 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:18:15 +0200 Subject: [PATCH 37/41] [deep_sleep] disable loop (#18962) --- esphome/components/deep_sleep/deep_sleep_bk72xx.cpp | 2 +- esphome/components/deep_sleep/deep_sleep_component.cpp | 3 ++- esphome/components/deep_sleep/deep_sleep_component.h | 5 +++++ esphome/components/deep_sleep/deep_sleep_esp32.cpp | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp index 2c97dc3211..a955095875 100644 --- a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp +++ b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp @@ -44,7 +44,7 @@ bool DeepSleepComponent::prepare_to_sleep_() { this->status_set_warning(); ESP_LOGV(TAG, "Waiting for pin to switch state to enter deep sleep..."); } - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return false; } } diff --git a/esphome/components/deep_sleep/deep_sleep_component.cpp b/esphome/components/deep_sleep/deep_sleep_component.cpp index 9a3e537e05..d33102bf4f 100644 --- a/esphome/components/deep_sleep/deep_sleep_component.cpp +++ b/esphome/components/deep_sleep/deep_sleep_component.cpp @@ -17,6 +17,7 @@ void DeepSleepComponent::setup() { void DeepSleepComponent::schedule_sleep_() { this->next_enter_deep_sleep_ = false; + this->disable_loop(); const optional run_duration = get_run_duration_(); if (run_duration.has_value()) { ESP_LOGI(TAG, "Scheduling in %" PRIu32 " ms", *run_duration); @@ -45,7 +46,7 @@ void DeepSleepComponent::loop() { void DeepSleepComponent::begin_sleep(bool manual) { if (this->prevent_ && !manual) { - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return; } diff --git a/esphome/components/deep_sleep/deep_sleep_component.h b/esphome/components/deep_sleep/deep_sleep_component.h index 208f88d707..0bbca4c5c4 100644 --- a/esphome/components/deep_sleep/deep_sleep_component.h +++ b/esphome/components/deep_sleep/deep_sleep_component.h @@ -190,6 +190,11 @@ class DeepSleepComponent final : public Component { void schedule_sleep_(); bool should_teardown_(); + void defer_sleep_() { + this->next_enter_deep_sleep_ = true; + this->enable_loop(); + } + #ifdef USE_BK72XX bool pin_prevents_sleep_(WakeUpPinItem &pin_item) const; bool get_real_pin_state_(InternalGPIOPin &pin) const { return (pin.digital_read() ^ pin.is_inverted()); } diff --git a/esphome/components/deep_sleep/deep_sleep_esp32.cpp b/esphome/components/deep_sleep/deep_sleep_esp32.cpp index 3fa1a1f1ed..20297028b2 100644 --- a/esphome/components/deep_sleep/deep_sleep_esp32.cpp +++ b/esphome/components/deep_sleep/deep_sleep_esp32.cpp @@ -100,7 +100,7 @@ bool DeepSleepComponent::prepare_to_sleep_() { this->status_set_warning(); ESP_LOGW(TAG, "Waiting for wakeup pin state change"); } - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return false; } return true; From a88ec7d90b6b19813f2e06bb012c7192b04b3c73 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:07:09 +0200 Subject: [PATCH 38/41] [logger] Flush uart before sleep in idf 6 (#18975) --- esphome/components/logger/logger_esp32.cpp | 13 +++++++++++-- sdkconfig.defaults | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index c3d777299d..8579708559 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -3,6 +3,7 @@ #include "esphome/components/esp32/crash_handler.h" #include +#include #include #include @@ -16,8 +17,10 @@ #include #endif #endif - -#include "esp_idf_version.h" +#if defined(CONFIG_PM_ENABLE) && defined(CONFIG_FREERTOS_USE_TICKLESS_IDLE) && \ + (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0)) +#include "esp_sleep.h" +#endif #include "freertos/FreeRTOS.h" #include @@ -87,6 +90,12 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). const int min_rx_buffer_size = UART_HW_FIFO_LEN(uart_num) + 1; uart_driver_install(uart_num, min_rx_buffer_size, tx_buffer_size, 0, nullptr, 0); +#if defined(CONFIG_PM_ENABLE) && defined(CONFIG_FREERTOS_USE_TICKLESS_IDLE) && \ + (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0)) + // Always flush before going to light sleep. Could be disabled for devices + // without TOP_PD or if source_clk = UART_SCLK_RTC + esp_sleep_set_console_uart_handling_mode(ESP_SLEEP_ALWAYS_FLUSH_UART); +#endif } void Logger::pre_setup() { diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 2bd702f48e..f4fe331df4 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -17,6 +17,8 @@ CONFIG_ESP_TASK_WDT_INIT=y CONFIG_ESP_TASK_WDT_PANIC=y CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=n CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1=n +CONFIG_FREERTOS_USE_TICKLESS_IDLE=y +CONFIG_PM_ENABLE=y # esp32_ble CONFIG_BT_ENABLED=y From 66f829c760358a291a9a97d90f9b981d8ac6a6ec Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:08:07 +0200 Subject: [PATCH 39/41] [zigbee] wake loop on defer/set_timeout (#19050) --- esphome/components/zigbee/time/zigbee_time_zephyr.cpp | 2 ++ esphome/components/zigbee/zigbee_esp32.cpp | 5 ++++- esphome/components/zigbee/zigbee_zephyr.cpp | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/esphome/components/zigbee/time/zigbee_time_zephyr.cpp b/esphome/components/zigbee/time/zigbee_time_zephyr.cpp index 92d238629a..3f14d0a62d 100644 --- a/esphome/components/zigbee/time/zigbee_time_zephyr.cpp +++ b/esphome/components/zigbee/time/zigbee_time_zephyr.cpp @@ -1,6 +1,7 @@ #include "zigbee_time_zephyr.h" #if defined(USE_ZIGBEE) && defined(USE_NRF52) && defined(USE_TIME) #include "esphome/core/log.h" +#include "esphome/core/application.h" namespace esphome::zigbee { @@ -47,6 +48,7 @@ void ZigbeeTime::set_epoch_time(uint32_t epoch) { this->synchronize_epoch_(epoch); this->has_time_ = true; }); + App.wake_loop_threadsafe(); } void ZigbeeTime::zcl_device_cb_(zb_bufid_t bufid) { diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index cd094306f4..4f9c70da75 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -49,7 +49,8 @@ void ZigbeeComponent::factory_reset() { void ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(ezb_bdb_comm_mode_mask_t mode) { if (!esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { - global_zigbee->set_timeout("zb_init", 10, [mode]() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(mode); }); + global_zigbee->set_timeout("zb_init", 100, [mode]() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(mode); }); + App.wake_loop_threadsafe(); return; } if (ezb_bdb_start_top_level_commissioning(mode) != EZB_ERR_NONE) { @@ -88,6 +89,7 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { global_zigbee->set_timeout("zb_init", 1000, []() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(EZB_BDB_MODE_INITIALIZATION); }); + App.wake_loop_threadsafe(); } } break; case EZB_BDB_SIGNAL_STEERING: { @@ -113,6 +115,7 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(EZB_BDB_MODE_NETWORK_STEERING); }); } + App.wake_loop_threadsafe(); } } break; case EZB_ZDO_SIGNAL_LEAVE: { diff --git a/esphome/components/zigbee/zigbee_zephyr.cpp b/esphome/components/zigbee/zigbee_zephyr.cpp index b8bb0a2036..286c83b8f5 100644 --- a/esphome/components/zigbee/zigbee_zephyr.cpp +++ b/esphome/components/zigbee/zigbee_zephyr.cpp @@ -1,10 +1,10 @@ #include "zigbee_zephyr.h" #if defined(USE_ZIGBEE) && defined(USE_NRF52) #include "esphome/core/log.h" +#include "esphome/core/application.h" #include #include #include "esphome/core/hal.h" -#include "esphome/core/wake.h" extern "C" { #include @@ -120,7 +120,7 @@ void ZigbeeComponent::zcl_device_cb(zb_bufid_t bufid) { /* Set default response value. */ p_device_cb_param->status = RET_OK; - esphome::wake_loop_threadsafe(); + App.wake_loop_threadsafe(); // endpoints are enumerated from 1 if (global_zigbee->callbacks_.size() >= endpoint) { @@ -138,6 +138,7 @@ void ZigbeeComponent::on_join_(bool factory_new) { ESP_LOGD(TAG, "Joined the network"); this->join_cb_.call(factory_new); }); + App.wake_loop_threadsafe(); } void ZigbeeComponent::on_start_() { @@ -145,6 +146,7 @@ void ZigbeeComponent::on_start_() { ESP_LOGD(TAG, "Started zigbee stack"); this->start_cb_.call(); }); + App.wake_loop_threadsafe(); } #ifdef USE_ZIGBEE_WIPE_ON_BOOT From 99241483026d54d47f8f06bdd2415e02c0cd3ebb Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:59:23 +0000 Subject: [PATCH 40/41] Bump aioesphomeapi from 46.3.0 to 46.4.0 (#19071) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dfddbed00b..72c42dad32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.4.0 click==8.3.3 -aioesphomeapi==46.3.0 +aioesphomeapi==46.4.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.151.3 puremagic==2.2.0 From 280fac11e6a8b571f6859dc4f9203e470cbbf1d4 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 10 Sep 2026 10:03:33 -0500 Subject: [PATCH 41/41] [serial_proxy] Add tap interface and port mode (#18955) Co-authored-by: puddly <32534428+puddly@users.noreply.github.com> --- esphome/components/api/api.proto | 37 +++- esphome/components/api/api_connection.cpp | 16 +- esphome/components/api/api_connection.h | 1 + esphome/components/api/api_pb2.cpp | 13 ++ esphome/components/api/api_pb2.h | 21 ++ esphome/components/api/api_pb2_dump.cpp | 18 ++ esphome/components/api/api_pb2_service.cpp | 11 ++ esphome/components/api/api_pb2_service.h | 3 + esphome/components/serial_proxy/__init__.py | 1 + .../components/serial_proxy/serial_proxy.cpp | 182 +++++++++++++++--- .../components/serial_proxy/serial_proxy.h | 103 +++++++++- esphome/core/defines.h | 1 + .../components/serial_proxy/serial_proxy.h | 3 + .../serial_proxy/test-tap.esp32-idf.yaml | 14 ++ 14 files changed, 394 insertions(+), 30 deletions(-) create mode 100644 tests/components/serial_proxy/test-tap.esp32-idf.yaml diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 3a0e0abea9..21972decad 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -77,6 +77,7 @@ service APIConnection { rpc serial_proxy_set_modem_pins(SerialProxySetModemPinsRequest) returns (void) {} rpc serial_proxy_get_modem_pins(SerialProxyGetModemPinsRequest) returns (void) {} rpc serial_proxy_request(SerialProxyRequest) returns (void) {} + rpc serial_proxy_set_mode(SerialProxySetModeRequest) returns (void) {} } @@ -2726,7 +2727,8 @@ enum SerialProxyParity { SERIAL_PROXY_PARITY_ODD = 2; } -// Configure UART parameters for a serial proxy instance +// Configure UART parameters for a serial proxy instance. Only the subscribed client may +// configure the port; others are refused with PORT_IN_USE (since API 1.17). message SerialProxyConfigureRequest { option (id) = 138; option (source) = SOURCE_CLIENT; @@ -2752,7 +2754,8 @@ message SerialProxyDataReceived { bytes data = 2; // Raw data received from the serial device } -// Write data to a serial device +// Write data to a serial device. Only the subscribed client may write; writes from +// others are ignored (since API 1.17). message SerialProxyWriteRequest { option (id) = 140; option (source) = SOURCE_CLIENT; @@ -2763,7 +2766,8 @@ message SerialProxyWriteRequest { bytes data = 2; // Raw data to write to the serial device } -// Set modem control pin states (RTS and DTR) +// Set modem control pin states (RTS and DTR). Only the subscribed client may set them; +// others are refused with PORT_IN_USE (since API 1.17). message SerialProxySetModemPinsRequest { option (id) = 141; option (source) = SOURCE_CLIENT; @@ -2802,6 +2806,7 @@ enum SerialProxyRequestType { // error the device answers with INVALID_ARGUMENT. SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3; // Acknowledges a SerialProxyConfigureRequest SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4; // Acknowledges a SerialProxySetModemPinsRequest + SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5; // Acknowledges a SerialProxySetModeRequest (since API 1.17) } enum SerialProxyStatus { @@ -2814,7 +2819,8 @@ enum SerialProxyStatus { SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6; // Invalid instance index or parameter value } -// Generic request message for simple serial proxy operations +// Generic request message for simple serial proxy operations. FLUSH requires an active +// subscription; it is refused with PORT_IN_USE otherwise (since API 1.17). message SerialProxyRequest { option (id) = 144; option (source) = SOURCE_CLIENT; @@ -2838,6 +2844,29 @@ message SerialProxyRequestResponse { string error_message = 4; // Additional detail on failure (optional) } +// How a port treats the bytes passing through it. RAW is a plain byte pipe; PROTOCOL +// activates the port's protocol-aware tap (if one is configured), letting it observe +// traffic and inject protocol bytes such as acknowledgements. Which protocol the tap +// speaks is a property of the device configuration, discoverable from the tap +// component's own API surface. A client that is about to flash firmware selects RAW +// first, which definitively disables that injection. +enum SerialProxyMode { + SERIAL_PROXY_MODE_RAW = 0; + SERIAL_PROXY_MODE_PROTOCOL = 1; +} + +// Only the subscribed client may change the mode; any other caller -- including one that +// never subscribed -- is refused with PORT_IN_USE. PROTOCOL is refused with NOT_SUPPORTED +// when the port has no protocol-aware tap configured. +message SerialProxySetModeRequest { + option (id) = 152; + option (source) = SOURCE_CLIENT; + option (ifdef) = "USE_SERIAL_PROXY"; + + uint32 instance = 1; + SerialProxyMode mode = 2; +} + // ==================== BLUETOOTH CONNECTION PARAMS ==================== message BluetoothSetConnectionParamsRequest { option (id) = 145; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index da4b7d7702..d910f6fc67 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1661,6 +1661,7 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { break; case enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE: case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE: // Response-only discriminators; never valid in a request ESP_LOGW(TAG, "Response-only serial proxy request type: %" PRIu32, static_cast(msg.type)); status = enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT; @@ -1673,6 +1674,19 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { send_serial_proxy_ack(this, msg.instance, msg.type, status); } +void APIConnection::on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE, + enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT); + return; + } + serial_proxy::SerialProxyResult result = proxies[msg.instance]->set_mode_from_client(this, msg.mode); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE, + serial_proxy_result_to_status(result)); +} + void APIConnection::send_serial_proxy_data(const SerialProxyDataReceived &msg) { if (!this->send_message(msg)) { ESP_LOGV(TAG, "Serial proxy data dropped, TCP buffer full"); @@ -1799,7 +1813,7 @@ bool APIConnection::send_hello_response_(const HelloRequest &msg) { HelloResponse resp; resp.api_version_major = 1; - resp.api_version_minor = 16; + resp.api_version_minor = 17; // Send only the version string - the client only logs this for debugging and doesn't use it otherwise resp.server_info = ESPHOME_VERSION_REF; resp.name = StringRef(App.get_name()); diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index a4c49dccf4..c19a33ca9b 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -244,6 +244,7 @@ class APIConnection final : public APIServerConnectionBase { void on_serial_proxy_set_modem_pins_request(const SerialProxySetModemPinsRequest &msg); void on_serial_proxy_get_modem_pins_request(const SerialProxyGetModemPinsRequest &msg); void on_serial_proxy_request(const SerialProxyRequest &msg); + void on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &msg); void send_serial_proxy_data(const SerialProxyDataReceived &msg); #endif diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 2de1f0a15c..7f162d9c15 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -4253,6 +4253,19 @@ uint32_t SerialProxyRequestResponse::calculate_size() const { size += ProtoSize::calc_length(1, this->error_message.size()); return size; } +bool SerialProxySetModeRequest::decode_varint(uint32_t field_id, proto_varint_value_t value) { + switch (field_id) { + case 1: + this->instance = value; + break; + case 2: + this->mode = static_cast(value); + break; + default: + return false; + } + return true; +} #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS bool BluetoothSetConnectionParamsRequest::decode_varint(uint32_t field_id, proto_varint_value_t value) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 5c3429a63a..799aaa27b5 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -356,6 +356,7 @@ enum SerialProxyRequestType : uint32_t { SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2, SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3, SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4, + SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5, }; enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_OK = 0, @@ -366,6 +367,10 @@ enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_PORT_IN_USE = 5, SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6, }; +enum SerialProxyMode : uint32_t { + SERIAL_PROXY_MODE_RAW = 0, + SERIAL_PROXY_MODE_PROTOCOL = 1, +}; #endif } // namespace enums @@ -3403,6 +3408,22 @@ class SerialProxyRequestResponse final : public ProtoMessage { protected: }; +class SerialProxySetModeRequest final : public ProtoDecodableMessage { + public: + static constexpr uint16_t MESSAGE_TYPE = 152; + static constexpr uint8_t ESTIMATED_SIZE = 6; +#ifdef HAS_PROTO_MESSAGE_DUMP + const LogString *message_name() const override { return LOG_STR("serial_proxy_set_mode_request"); } +#endif + uint32_t instance{0}; + enums::SerialProxyMode mode{}; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: + bool decode_varint(uint32_t field_id, proto_varint_value_t value) override; +}; #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS class BluetoothSetConnectionParamsRequest final : public ProtoDecodableMessage { diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index dced81ee30..bb244973a1 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -854,6 +854,8 @@ template<> const char *proto_enum_to_string(enums return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_CONFIGURE"); case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS"); + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE: + return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODE"); default: return ESPHOME_PSTR("UNKNOWN"); } @@ -878,6 +880,16 @@ template<> const char *proto_enum_to_string(enums::Ser return ESPHOME_PSTR("UNKNOWN"); } } +template<> const char *proto_enum_to_string(enums::SerialProxyMode value) { + switch (value) { + case enums::SERIAL_PROXY_MODE_RAW: + return ESPHOME_PSTR("SERIAL_PROXY_MODE_RAW"); + case enums::SERIAL_PROXY_MODE_PROTOCOL: + return ESPHOME_PSTR("SERIAL_PROXY_MODE_PROTOCOL"); + default: + return ESPHOME_PSTR("UNKNOWN"); + } +} #endif const char *HelloRequest::dump_to(DumpBuffer &out) const { @@ -2805,6 +2817,12 @@ const char *SerialProxyRequestResponse::dump_to(DumpBuffer &out) const { dump_field(out, ESPHOME_PSTR("error_message"), this->error_message); return out.c_str(); } +const char *SerialProxySetModeRequest::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("SerialProxySetModeRequest")); + dump_field(out, ESPHOME_PSTR("instance"), this->instance); + dump_field(out, ESPHOME_PSTR("mode"), static_cast(this->mode)); + return out.c_str(); +} #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS const char *BluetoothSetConnectionParamsRequest::dump_to(DumpBuffer &out) const { diff --git a/esphome/components/api/api_pb2_service.cpp b/esphome/components/api/api_pb2_service.cpp index 65c7b8858c..172062be63 100644 --- a/esphome/components/api/api_pb2_service.cpp +++ b/esphome/components/api/api_pb2_service.cpp @@ -712,6 +712,17 @@ void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const ui this->on_device_capabilities_request(); break; } +#ifdef USE_SERIAL_PROXY + case SerialProxySetModeRequest::MESSAGE_TYPE: { + SerialProxySetModeRequest msg; + msg.decode(msg_data, msg_size); +#ifdef HAS_PROTO_MESSAGE_DUMP + this->log_receive_message_(LOG_STR("on_serial_proxy_set_mode_request"), msg); +#endif + this->on_serial_proxy_set_mode_request(msg); + break; + } +#endif default: break; } diff --git a/esphome/components/api/api_pb2_service.h b/esphome/components/api/api_pb2_service.h index 6abdf7093e..a4dfd6a366 100644 --- a/esphome/components/api/api_pb2_service.h +++ b/esphome/components/api/api_pb2_service.h @@ -235,6 +235,9 @@ class APIServerConnectionBase { void on_serial_proxy_request(const SerialProxyRequest &value){}; #endif +#ifdef USE_SERIAL_PROXY + void on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &value){}; +#endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS void on_bluetooth_set_connection_params_request(const BluetoothSetConnectionParamsRequest &value){}; #endif diff --git a/esphome/components/serial_proxy/__init__.py b/esphome/components/serial_proxy/__init__.py index 4186fcf8b1..b6e780fabd 100644 --- a/esphome/components/serial_proxy/__init__.py +++ b/esphome/components/serial_proxy/__init__.py @@ -30,6 +30,7 @@ MULTI_CONF = True serial_proxy_ns = cg.esphome_ns.namespace("serial_proxy") SerialProxy = serial_proxy_ns.class_("SerialProxy", cg.Component, uart.UARTDevice) +SerialProxyTap = serial_proxy_ns.class_("SerialProxyTap") api_enums_ns = cg.esphome_ns.namespace("api").namespace("enums") SerialProxyPortType = api_enums_ns.enum("SerialProxyPortType") diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index c1c1510643..129745c1c9 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -29,26 +29,57 @@ void SerialProxy::setup() { #ifdef USE_API // instance_index_ is fixed at registration time; pre-set it so loop() only needs to update data this->outgoing_msg_.instance = this->instance_index_; +#endif +#ifdef USE_SERIAL_PROXY_TAP + // A tap sets itself up before this runs (its setup priority is higher), so it may + // already be waiting on the port -- a boot-time handshake with the device, say. Leaving + // the loop enabled is what lets that finish; without it the tap would stall until a + // client happened to subscribe. + if (this->tap_ != nullptr && this->tap_->tap_needs_port()) { + return; + } #endif // No subscriber at startup; disable loop until a client subscribes this->disable_loop(); } -void SerialProxy::loop() { -#ifdef USE_API - // Safety check — loop should only run when subscribed, but guard against races - if (this->api_connection_ == nullptr) [[unlikely]] { - this->disable_loop(); +#ifdef USE_SERIAL_PROXY_TAP +void SerialProxy::reset_mode_() { + // The mode belongs to a session, not to the port. Carrying a departed client's choice + // over to the next one would inject protocol bytes into a stream that never asked for + // them -- a firmware upload, or any client built before this request existed and so + // unable to turn it off. Guessing RAW is the safe direction: a client that wanted + // protocol handling and did not ask for it merely sends its own acknowledgements. + if (this->mode_ == api::enums::SERIAL_PROXY_MODE_RAW) { return; } + ESP_LOGD(TAG, "Session ended, returning serial proxy [%" PRIu32 "] to RAW mode", this->instance_index_); + this->mode_ = api::enums::SERIAL_PROXY_MODE_RAW; +} +#endif +void SerialProxy::loop() { +#ifdef USE_API // Detect subscriber disconnect - if (this->api_connection_->is_marked_for_removal() || !this->api_connection_->is_connection_setup() || - !api_is_connected()) { + if (this->api_connection_ != nullptr && (this->api_connection_->is_marked_for_removal() || + !this->api_connection_->is_connection_setup() || !api_is_connected())) { ESP_LOGW(TAG, "Subscriber disconnected"); this->api_connection_ = nullptr; + this->reset_mode_(); + } + + // With no subscriber there is normally nothing to do, but a tap may still need the port + // read -- it does its protocol work precisely while nobody else is listening. + if (this->api_connection_ == nullptr) [[unlikely]] { +#ifdef USE_SERIAL_PROXY_TAP + if (this->tap_ == nullptr || !this->tap_->tap_needs_port()) { + this->disable_loop(); + return; + } +#else this->disable_loop(); return; +#endif } // Read available data from UART and forward to subscribed client @@ -69,11 +100,54 @@ void __attribute__((noinline)) SerialProxy::read_and_send_(size_t available) { if (!this->read_array(buffer, to_read)) return; +#ifdef USE_SERIAL_PROXY_TAP + // Before forwarding, so a tap that answers the device (an acknowledgement, say) is not + // waiting on the network round trip to a subscriber that may not even exist. + if (this->tap_observing_()) { + this->tap_->on_device_rx(buffer, to_read); + } +#endif + + if (this->api_connection_ == nullptr) { + return; + } this->outgoing_msg_.set_data(buffer, to_read); this->api_connection_->send_serial_proxy_data(this->outgoing_msg_); } #endif +#ifdef USE_SERIAL_PROXY_TAP + +bool SerialProxy::tap_observing_() const { + if (this->tap_ == nullptr) { + return false; + } + // With no subscriber, a tap doing its own protocol work (the boot-time handshake with + // the device, say) is served regardless of mode -- nobody has chosen one yet. Once a + // subscriber holds the port, the mode alone decides, so RAW stays inert. + if (this->api_connection_ == nullptr && this->tap_->tap_needs_port()) { + return true; + } + // Otherwise the mode decides. RAW must be inert: a client that flips to RAW before + // flashing firmware is entitled to a byte pipe with nothing injecting protocol bytes + // into it, and "the tap turned out not to recognise the stream" is not good enough. + return this->mode_ == api::enums::SERIAL_PROXY_MODE_PROTOCOL; +} + +void SerialProxy::tap_pump() { +#ifdef USE_API + // Nothing would consume the bytes; leave them in the FIFO + if (!this->tap_observing_() && this->api_connection_ == nullptr) { + return; + } + const size_t available = this->available(); + if (available > 0) { + this->read_and_send_(available); + } +#endif +} +#endif + void SerialProxy::dump_config() { ESP_LOGCONFIG(TAG, "Serial Proxy [%" PRIu32 "]:\n" @@ -92,8 +166,9 @@ void SerialProxy::dump_config() { SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, uint8_t stop_bits, uint8_t data_size) { #ifdef USE_API - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring configure request from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring configure request from client without port subscription [%" PRIu32 "]", + this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -159,24 +234,80 @@ SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uin return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } +SerialProxyResult SerialProxy::set_mode_from_client(api::APIConnection *api_connection, + api::enums::SerialProxyMode mode) { +#ifdef USE_API + // Only the live subscriber may change the mode, so the mode cannot outlive a session + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring mode request from client without port subscription [%" PRIu32 "]", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; + } +#endif + // Values come from a remote client + if (mode != api::enums::SERIAL_PROXY_MODE_RAW && mode != api::enums::SERIAL_PROXY_MODE_PROTOCOL) { + ESP_LOGW(TAG, "Invalid mode: %" PRIu32, static_cast(mode)); + return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT; + } + // PROTOCOL on a port with no tap would be a silent no-op; refuse so the client knows +#ifdef USE_SERIAL_PROXY_TAP + const bool has_tap = this->tap_ != nullptr; +#else + const bool has_tap = false; +#endif + if (mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL && !has_tap) { + ESP_LOGW(TAG, "No tap on serial proxy [%" PRIu32 "]; PROTOCOL mode unavailable", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; + } + ESP_LOGD(TAG, "Serial proxy [%" PRIu32 "] mode set to %s", this->instance_index_, + mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL ? LOG_STR_LITERAL("PROTOCOL") : LOG_STR_LITERAL("RAW")); +#ifdef USE_SERIAL_PROXY_TAP + const bool leaving_protocol_mode = + this->mode_ != api::enums::SERIAL_PROXY_MODE_RAW && mode == api::enums::SERIAL_PROXY_MODE_RAW; + this->mode_ = mode; + + // Only for an explicit client request, not for reset_mode_() at the end of a session: + // an ordinary disconnect says nothing about the device, whereas a client deliberately + // asking for raw bytes usually precedes changing what the device is. + if (leaving_protocol_mode && this->tap_ != nullptr) { + this->tap_->on_protocol_disabled(); + } +#endif + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; +} + void SerialProxy::write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) { #ifdef USE_API - // Bytes from a client other than the live subscriber would interleave with the - // subscriber's traffic on the wire - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring write from client without port access [%" PRIu32 "]", this->instance_index_); + // Bytes from anyone but the live subscriber would interleave with the subscriber's + // traffic -- or with an active tap's -- on the wire + if (!this->is_subscriber_(api_connection)) { + if (this->api_connection_ != nullptr) { + ESP_LOGW(TAG, "Ignoring write from client that does not hold serial proxy [%" PRIu32 "]", this->instance_index_); + } else { + // A legacy client streaming writes without subscribing would flood WARN, one per + // request; writes are the only high-rate, unacknowledged operation, so keep this + // visible without drowning the log + ESP_LOGV(TAG, "Ignoring write from client without port subscription [%" PRIu32 "]", this->instance_index_); + } return; } #endif if (data == nullptr || len == 0) return; this->write_array(data, len); + +#ifdef USE_SERIAL_PROXY_TAP + // After the write, so the tap observes the same ordering the device does + if (this->tap_observing_()) { + this->tap_->on_client_tx(data, len); + } +#endif } SerialProxyResult SerialProxy::set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { #ifdef USE_API - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring modem pin request from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring modem pin request from client without port subscription [%" PRIu32 "]", + this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -210,8 +341,8 @@ uint32_t SerialProxy::get_modem_pins() const { SerialProxyResult SerialProxy::flush_port(api::APIConnection *api_connection) { #ifdef USE_API // Flushing stalls the port, so it gets the same ownership check as writes - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring flush from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring flush from client without port subscription [%" PRIu32 "]", this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -230,11 +361,6 @@ SerialProxyResult SerialProxy::flush_port(api::APIConnection *api_connection) { } #ifdef USE_API -bool SerialProxy::port_claimed_by_other_(api::APIConnection *api_connection) const { - return this->api_connection_ != nullptr && this->api_connection_ != api_connection && - this->api_connection_->is_connection_setup(); -} - SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type) { switch (type) { @@ -252,6 +378,10 @@ SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_conn return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } ESP_LOGW(TAG, "Previous subscriber disconnected; taking over subscription"); + // End the dead client's session before starting the new one, so its mode + // cannot leak into a session that never asked for it + this->api_connection_ = nullptr; + this->reset_mode_(); } this->api_connection_ = api_connection; this->enable_loop(); @@ -264,7 +394,15 @@ SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_conn return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } this->api_connection_ = nullptr; + this->reset_mode_(); +#ifdef USE_SERIAL_PROXY_TAP + // Keep the loop alive for a tap that still needs the port (mirrors loop()) + if (this->tap_ == nullptr || !this->tap_->tap_needs_port()) { + this->disable_loop(); + } +#else this->disable_loop(); +#endif ESP_LOGV(TAG, "API connection unsubscribed from serial proxy [%" PRIu32 "]", this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_OK; default: diff --git a/esphome/components/serial_proxy/serial_proxy.h b/esphome/components/serial_proxy/serial_proxy.h index a0e47ee686..e3f4264cfa 100644 --- a/esphome/components/serial_proxy/serial_proxy.h +++ b/esphome/components/serial_proxy/serial_proxy.h @@ -26,6 +26,7 @@ class APIConnection; namespace enums { enum SerialProxyPortType : uint32_t; enum SerialProxyRequestType : uint32_t; +enum SerialProxyMode : uint32_t; } // namespace enums } // namespace esphome::api @@ -52,6 +53,36 @@ enum class SerialProxyResult : uint8_t { /// Maximum bytes to read from UART in a single loop iteration inline constexpr size_t SERIAL_PROXY_MAX_READ_SIZE = 256; +#ifdef USE_SERIAL_PROXY_TAP +/// Observes a port's traffic without owning it, and may inject bytes of its own. +/// +/// This exists so protocol-aware behaviour can be layered onto a plain byte pipe without +/// the pipe knowing anything about the protocol: the tap is compiled in only when some +/// component asks for one, so a proxy carrying an RS485 meter pays nothing for it. +/// +/// A tap is an observer, never a gatekeeper -- it cannot suppress or alter the bytes +/// flowing in either direction, so a misbehaving tap cannot corrupt the stream. +class SerialProxyTap { + public: + /// Bytes read from the device, before they are forwarded to any subscriber. + virtual void on_device_rx(const uint8_t *data, size_t len) = 0; + + /// Bytes a subscriber sent towards the device, after they have been written. + virtual void on_client_tx(const uint8_t *data, size_t len) = 0; + + /// True when the port must keep reading even with no subscriber attached, so a tap can + /// do its own protocol work while nobody is listening. Honoured only while no + /// subscriber holds the port; with one attached, the port mode alone decides. + virtual bool tap_needs_port() const = 0; + + /// A client explicitly turned protocol handling off for this port. Distinct from the + /// automatic reset when a session ends: this one means a client intends to do something + /// else with the device -- reflash it, most likely -- so anything the tap believes about + /// it should be treated as suspect. + virtual void on_protocol_disabled() = 0; +}; +#endif + class SerialProxy final : public uart::UARTDevice, public Component { public: void setup() override; @@ -77,6 +108,9 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Get the port type api::enums::SerialProxyPortType get_port_type() const { return this->port_type_; } + /// Handle a mode change requested by an API client + SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode); + /// Configure UART parameters and apply them /// @param api_connection The API connection requesting the change /// @param baudrate Baud rate in bits per second @@ -121,13 +155,67 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Set the DTR GPIO pin (from YAML configuration) void set_dtr_pin(GPIOPin *pin) { this->dtr_pin_ = pin; } +#ifdef USE_SERIAL_PROXY_TAP + /// Attach a traffic observer. At most one, set once at setup time. + void set_tap(SerialProxyTap *tap) { this->tap_ = tap; } + + /// Write bytes originating from the tap rather than from a client. Bypasses the + /// subscriber ownership check, but only while the tap is being served bytes -- so a + /// port in RAW mode with a subscriber attached stays inert. Returns false when the + /// bytes were dropped for that reason. + bool write_from_tap(const uint8_t *data, size_t len) { + if (!this->tap_observing_()) { + return false; + } + this->write_array(data, len); + return true; + } + + /// Whether the tap is currently being served bytes. Can flip false with no callback + /// (a subscriber attaching in RAW mode, say), so a tap should check before starting + /// protocol work and when a reply seems overdue. + bool tap_is_observed() const { return this->tap_observing_(); } + + /// Resume reading after a tap's needs change. loop() disables itself when there is + /// neither a subscriber nor a tap that wants the port, so a tap starting fresh work + /// must ask for it back. Must be called from the main loop. + void tap_request_port() { this->enable_loop(); } + + /// Whether the underlying device is present. On a USB UART this tracks enumeration, so + /// a tap can notice the device being unplugged and plugged back in. + bool is_device_connected() const { return this->parent_->is_connected(); } + + /// Run one read-and-dispatch cycle immediately. Lets a tap make progress before the + /// main loop is running -- during setup, for instance, while a component is still + /// blocking on can_proceed(). Must not be called from on_device_rx() or + /// on_client_tx(): each nested cycle costs a 256-byte stack frame. + void tap_pump(); +#endif + protected: #ifdef USE_API - /// Read from UART and send to API client (slow path with 256-byte stack buffer) + /// Read from UART, hand the bytes to any tap, and forward them to a subscriber + /// (slow path with a 256-byte stack buffer) void read_and_send_(size_t available); - /// True when a live subscriber other than the given connection holds the port - bool port_claimed_by_other_(api::APIConnection *api_connection) const; + /// True when the given connection is the live subscriber. Every port operation + /// (write, configure, modem pins, flush, mode) requires this, so an unsubscribed + /// client can never share the wire with the subscriber or an active tap. + bool is_subscriber_(api::APIConnection *api_connection) const { return this->api_connection_ == api_connection; } +#endif + +#ifdef USE_SERIAL_PROXY_TAP + /// Return the port to RAW when a subscriber goes away, so the mode never outlives it + void reset_mode_(); +#else + /// Without a tap, PROTOCOL is refused, so the mode is fixed at RAW and there is + /// nothing to reset + void reset_mode_() {} +#endif + +#ifdef USE_SERIAL_PROXY_TAP + /// True when the tap should be shown the traffic passing through this port + bool tap_observing_() const; #endif /// Instance index for identifying this proxy in API messages @@ -147,6 +235,11 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Port type api::enums::SerialProxyPortType port_type_{}; +#ifdef USE_SERIAL_PROXY_TAP + /// How the bytes passing through are treated; zero is SERIAL_PROXY_MODE_RAW + api::enums::SerialProxyMode mode_{}; +#endif + /// Optional GPIO pins for modem control GPIOPin *rts_pin_{nullptr}; GPIOPin *dtr_pin_{nullptr}; @@ -154,6 +247,10 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Current modem pin states bool rts_state_{false}; bool dtr_state_{false}; + +#ifdef USE_SERIAL_PROXY_TAP + SerialProxyTap *tap_{nullptr}; +#endif }; } // namespace esphome::serial_proxy diff --git a/esphome/core/defines.h b/esphome/core/defines.h index eaece6d5ff..c3b16d833a 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -181,6 +181,7 @@ #define USE_SENSOR #define USE_SENSOR_FILTER #define USE_SERIAL_PROXY +#define USE_SERIAL_PROXY_TAP #define USE_SETUP_PRIORITY_OVERRIDE #define USE_STATUS_LED #define USE_STATUS_SENSOR diff --git a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h index 6fc20f3350..7da6fff017 100644 --- a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h +++ b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h @@ -40,6 +40,9 @@ class SerialProxy { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {} + SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) { + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } diff --git a/tests/components/serial_proxy/test-tap.esp32-idf.yaml b/tests/components/serial_proxy/test-tap.esp32-idf.yaml new file mode 100644 index 0000000000..5522e53c47 --- /dev/null +++ b/tests/components/serial_proxy/test-tap.esp32-idf.yaml @@ -0,0 +1,14 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +# Compile the tap code paths; no tap is attached, so this exercises the +# null-tap branches that a normal build never defines. +esphome: + platformio_options: + build_flags: + - "-DUSE_SERIAL_PROXY_TAP" + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + serial_proxy: !include common.yaml