From 722cbfe843cad4aa729ef848016261fdb4bb884d Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 26 May 2026 14:05:57 -0400 Subject: [PATCH] [voice_assistant] Never send zero-length audio to Home Assistant (#16634) --- .../voice_assistant/voice_assistant.cpp | 136 ++++++++++++------ .../voice_assistant/voice_assistant.h | 13 ++ 2 files changed, 107 insertions(+), 42 deletions(-) diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index af1b98da02..f13ea39fa2 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -4,6 +4,7 @@ #ifdef USE_VOICE_ASSISTANT #include "esphome/components/socket/socket.h" +#include "esphome/core/application.h" #include "esphome/core/log.h" #include @@ -26,6 +27,11 @@ static const size_t SEND_BUFFER_SIZE = SEND_BUFFER_SAMPLES * sizeof(int16_t); static const size_t RECEIVE_SIZE = 1024; static const size_t SPEAKER_BUFFER_SIZE = 16 * RECEIVE_SIZE; +// If one microphone channel keeps producing audio while another configured channel produces none for this +// long, treat the silent channel as failed and stop the stream. A working microphone exposes a chunk every +// SEND_BUFFER_SAMPLES (32 ms), so this is far longer than any legitimate gap between chunks. +static const uint32_t AUDIO_CHANNEL_STALL_TIMEOUT_MS = 2000; + VoiceAssistant::VoiceAssistant() { global_voice_assistant = this; } void VoiceAssistant::setup() { @@ -168,6 +174,9 @@ void VoiceAssistant::clear_buffers_() { this->audio_source2_->clear_buffered_data(); } + // Reset the multi-channel stall watchdog (see audio_channel_stall_start_). + this->audio_channel_stall_start_ = 0; + #ifdef USE_SPEAKER if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) { memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE); @@ -200,6 +209,79 @@ void VoiceAssistant::reset_conversation_id() { ESP_LOGD(TAG, "reset conversation ID"); } +void VoiceAssistant::stream_api_audio_() { + // Both microphone channels are sent together, if configured. Home Assistant feeds one of the + // channels to its speech-to-text stream and treats an empty payload on that channel as + // end-of-stream, and the device cannot know which channel it picked, so only send once every + // configured channel has audio exposed, and always send them together. We don't target any + // particular message size: Home Assistant re-chunks the audio, and each fill() exposes at most + // SEND_BUFFER_SIZE bytes. + while (true) { + // fill() exposes a new chunk, or returns 0 if a previous chunk is still exposed; available() + // reports the currently exposed bytes either way. + this->audio_source_->fill(0, false); + size_t available = this->audio_source_->available(); + size_t available2 = 0; + if (this->audio_source2_ != nullptr) { + this->audio_source2_->fill(0, false); + available2 = this->audio_source2_->available(); + } + + const bool channel_empty = (available == 0); + const bool channel2_empty = (this->audio_source2_ != nullptr) && (available2 == 0); + if (channel_empty || channel2_empty) { + // A configured channel has no audio yet, so keep any chunk exposed on the other channel for the + // next pass rather than sending an empty payload. + this->handle_channel_stall_(available, available2); + break; + } + + // Both channels have audio exposed; clear any in-progress stall timer. + this->audio_channel_stall_start_ = 0; + + api::VoiceAssistantAudio msg; + // Zero-copy: send_message() copies the data out before we consume it. + msg.data = this->audio_source_->data(); + msg.data_len = available; + if (this->audio_source2_ != nullptr) { + msg.data2 = this->audio_source2_->data(); + msg.data2_len = available2; + } + + this->api_client_->send_message(msg); + + this->audio_source_->consume(available); + if (this->audio_source2_ != nullptr) { + this->audio_source2_->consume(available2); + } + } +} + +void VoiceAssistant::handle_channel_stall_(size_t available, size_t available2) { + // Called when at least one configured channel has no audio exposed. When one channel has data and the + // other does not, watch how long the empty channel stays starved: Home Assistant has no stream timeout + // and would never tell us to stop, so a channel that fails outright would otherwise hang streaming + // forever with the live channel's chunk held. Stop the stream with an error after a prolonged imbalance. + if ((available == 0) && (available2 == 0)) { + // Both channels are idle (no audio buffered yet); normal, not a stalled channel. + this->audio_channel_stall_start_ = 0; + return; + } + + const uint32_t now = App.get_loop_component_start_time(); + if (this->audio_channel_stall_start_ == 0) { + this->audio_channel_stall_start_ = now; + } else if ((now - this->audio_channel_stall_start_) >= AUDIO_CHANNEL_STALL_TIMEOUT_MS) { + ESP_LOGW(TAG, "Mic channel %d stalled, stopping stream", (available == 0) ? 0 : 1); + this->audio_channel_stall_start_ = 0; + this->signal_stop_(); + this->set_state_(State::STOP_MICROPHONE, State::IDLE); + this->defer([this]() { + this->error_trigger_.trigger("mic-channel-stalled", "A microphone channel stopped producing audio"); + }); + } +} + void VoiceAssistant::loop() { if (this->api_client_ == nullptr && this->state_ != State::IDLE && this->state_ != State::STOP_MICROPHONE && this->state_ != State::STOPPING_MICROPHONE) { @@ -292,55 +374,25 @@ void VoiceAssistant::loop() { case State::STREAMING_MICROPHONE: { // pre_shift is ignored by RingBufferAudioSource (no intermediate transfer buffer to compact). if (this->audio_mode_ == AUDIO_MODE_API) { - // API audio - // Both microphone channels are sent, if configured - size_t available = this->audio_source_->fill(0, false); - size_t available2 = 0; - if (this->audio_source2_ != nullptr) { - available2 = this->audio_source2_->fill(0, false); - } - - while (available > 0 || available2 > 0) { - api::VoiceAssistantAudio msg; - - if (available > 0) { - // Zero-copy: send_message() copies the data out before we consume it - msg.data = this->audio_source_->data(); - msg.data_len = available; - } - - // Second microphone channel - if (available2 > 0) { - msg.data2 = this->audio_source2_->data(); - msg.data2_len = available2; - } - - this->api_client_->send_message(msg); - - if (available > 0) { - this->audio_source_->consume(available); - } - available = this->audio_source_->fill(0, false); - if (available2 > 0) { - this->audio_source2_->consume(available2); - } - if (this->audio_source2_ != nullptr) { - available2 = this->audio_source2_->fill(0, false); - } - } + this->stream_api_audio_(); } else { // UDP (will eventually be deprecated) // Only the primary microphone channel is used - while (this->audio_source_->fill(0, false) > 0) { + while (true) { + this->audio_source_->fill(0, false); + size_t available = this->audio_source_->available(); + if (available == 0) { + break; + } if (!this->udp_socket_running_) { if (!this->start_udp_socket_()) { this->set_state_(State::STOP_MICROPHONE, State::IDLE); break; } } - this->socket_->sendto(this->audio_source_->data(), this->audio_source_->available(), 0, - (struct sockaddr *) &this->dest_addr_, sizeof(this->dest_addr_)); - this->audio_source_->consume(this->audio_source_->available()); + this->socket_->sendto(this->audio_source_->data(), available, 0, (struct sockaddr *) &this->dest_addr_, + sizeof(this->dest_addr_)); + this->audio_source_->consume(available); } } // audio mode break; @@ -841,8 +893,8 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) { }); State new_state = this->local_output_ ? State::STREAMING_RESPONSE : State::IDLE; if (new_state != this->state_) { - // Don't needlessly change the state. The intent progress stage may have already changed the state to streaming - // response. + // Don't needlessly change the state. The intent progress stage may have already changed the state to + // streaming response. this->set_state_(new_state, new_state); } break; diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index f3ea669e15..76b076a366 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -244,6 +244,12 @@ class VoiceAssistant : public Component { void signal_stop_(); void start_playback_timeout_(); + // Drains the exposed microphone audio and sends it to Home Assistant over the API in one loop() pass. + void stream_api_audio_(); + // Handles a pass where at least one configured channel has no audio exposed, timing out a channel that + // stalls. See audio_channel_stall_start_. + void handle_channel_stall_(size_t available, size_t available2); + std::unique_ptr socket_ = nullptr; struct sockaddr_storage dest_addr_; @@ -315,6 +321,13 @@ class VoiceAssistant : public Component { std::weak_ptr ring_buffer_; std::weak_ptr ring_buffer2_; + // When streaming multiple channels, the send loop holds an exposed chunk on one channel until the other + // channel also has audio so the channels are always sent together (an empty payload looks like + // end-of-stream to Home Assistant). Home Assistant has no stream timeout, so a channel that stops + // producing entirely would hang streaming forever. This records when such an imbalance began so a + // prolonged one can be detected and stopped; 0 means no imbalance is currently being timed. + uint32_t audio_channel_stall_start_{0}; + bool use_wake_word_; uint8_t noise_suppression_level_; uint8_t auto_gain_;