From b11ad26c4f70050d1656200234b0642760bdea68 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Thu, 19 Feb 2026 10:20:19 -0600 Subject: [PATCH] [audio] Support decoding audio directly from flash (#14098) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/components/audio/audio_decoder.cpp | 102 ++++++++++-------- esphome/components/audio/audio_decoder.h | 26 +++-- .../audio/audio_transfer_buffer.cpp | 17 ++- .../components/audio/audio_transfer_buffer.h | 67 +++++++++++- 4 files changed, 159 insertions(+), 53 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index ee6d7d0a15..7794b5b0d3 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -15,8 +15,8 @@ static const uint32_t READ_WRITE_TIMEOUT_MS = 20; // Timeout for transferring a static const uint32_t MAX_POTENTIALLY_FAILED_COUNT = 10; -AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) { - this->input_transfer_buffer_ = AudioSourceTransferBuffer::create(input_buffer_size); +AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) + : input_buffer_size_(input_buffer_size) { this->output_transfer_buffer_ = AudioSinkTransferBuffer::create(output_buffer_size); } @@ -29,11 +29,20 @@ AudioDecoder::~AudioDecoder() { } esp_err_t AudioDecoder::add_source(std::weak_ptr &input_ring_buffer) { - if (this->input_transfer_buffer_ != nullptr) { - this->input_transfer_buffer_->set_source(input_ring_buffer); - return ESP_OK; + auto source = AudioSourceTransferBuffer::create(this->input_buffer_size_); + if (source == nullptr) { + return ESP_ERR_NO_MEM; } - return ESP_ERR_NO_MEM; + source->set_source(input_ring_buffer); + this->input_buffer_ = std::move(source); + return ESP_OK; +} + +esp_err_t AudioDecoder::add_source(const uint8_t *data_pointer, size_t length) { + auto source = make_unique(); + source->set_data(data_pointer, length); + this->input_buffer_ = std::move(source); + return ESP_OK; } esp_err_t AudioDecoder::add_sink(std::weak_ptr &output_ring_buffer) { @@ -54,8 +63,16 @@ esp_err_t AudioDecoder::add_sink(speaker::Speaker *speaker) { } #endif +esp_err_t AudioDecoder::add_sink(AudioSinkCallback *callback) { + if (this->output_transfer_buffer_ != nullptr) { + this->output_transfer_buffer_->set_sink(callback); + return ESP_OK; + } + return ESP_ERR_NO_MEM; +} + esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { - if ((this->input_transfer_buffer_ == nullptr) || (this->output_transfer_buffer_ == nullptr)) { + if (this->output_transfer_buffer_ == nullptr) { return ESP_ERR_NO_MEM; } @@ -112,6 +129,10 @@ esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { } AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { + if (this->input_buffer_ == nullptr) { + return AudioDecoderState::FAILED; + } + if (stop_gracefully) { if (this->output_transfer_buffer_->available() == 0) { if (this->end_of_file_) { @@ -119,7 +140,7 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { return AudioDecoderState::FINISHED; } - if (!this->input_transfer_buffer_->has_buffered_data()) { + if (!this->input_buffer_->has_buffered_data()) { // If all the internal buffers are empty, the decoding is done return AudioDecoderState::FINISHED; } @@ -170,10 +191,10 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { // Only shift data on the first loop iteration to avoid unnecessary, slow moves // If the decoder buffers internally, then never shift - size_t bytes_read = this->input_transfer_buffer_->transfer_data_from_source( - pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), first_loop_iteration && !this->decoder_buffers_internally_); + size_t bytes_read = this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), + first_loop_iteration && !this->decoder_buffers_internally_); - if (!first_loop_iteration && (this->input_transfer_buffer_->available() < bytes_processed)) { + if (!first_loop_iteration && (this->input_buffer_->available() < bytes_processed)) { // Less data is available than what was processed in last iteration, so don't attempt to decode. // This attempts to avoid the decoder from consistently trying to decode an incomplete frame. The transfer buffer // will shift the remaining data to the start and copy more from the source the next time the decode function is @@ -181,19 +202,21 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { break; } - bytes_available_before_processing = this->input_transfer_buffer_->available(); + bytes_available_before_processing = this->input_buffer_->available(); if ((this->potentially_failed_count_ > 0) && (bytes_read == 0)) { // Failed to decode in last attempt and there is no new data - if ((this->input_transfer_buffer_->free() == 0) && first_loop_iteration) { - // The input buffer is full. Since it previously failed on the exact same data, we can never recover + if ((this->input_buffer_->free() == 0) && first_loop_iteration) { + // The input buffer is full (or read-only, e.g. const flash source). Since it previously failed on the exact + // same data, we can never recover. For const sources this is correct: the entire file is already available, so + // a decode failure is genuine, not a transient out-of-data condition. state = FileDecoderState::FAILED; } else { // Attempt to get more data next time state = FileDecoderState::IDLE; } - } else if (this->input_transfer_buffer_->available() == 0) { + } else if (this->input_buffer_->available() == 0) { // No data to decode, attempt to get more data next time state = FileDecoderState::IDLE; } else { @@ -224,7 +247,7 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } first_loop_iteration = false; - bytes_processed = bytes_available_before_processing - this->input_transfer_buffer_->available(); + bytes_processed = bytes_available_before_processing - this->input_buffer_->available(); if (state == FileDecoderState::POTENTIALLY_FAILED) { ++this->potentially_failed_count_; @@ -243,8 +266,7 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { FileDecoderState AudioDecoder::decode_flac_() { if (!this->audio_stream_info_.has_value()) { // Header hasn't been read - auto result = this->flac_decoder_->read_header(this->input_transfer_buffer_->get_buffer_start(), - this->input_transfer_buffer_->available()); + auto result = this->flac_decoder_->read_header(this->input_buffer_->data(), this->input_buffer_->available()); if (result > esp_audio_libs::flac::FLAC_DECODER_HEADER_OUT_OF_DATA) { // Serrious error reading FLAC header, there is no recovery @@ -252,7 +274,7 @@ FileDecoderState AudioDecoder::decode_flac_() { } size_t bytes_consumed = this->flac_decoder_->get_bytes_index(); - this->input_transfer_buffer_->decrease_buffer_length(bytes_consumed); + this->input_buffer_->consume(bytes_consumed); if (result == esp_audio_libs::flac::FLAC_DECODER_HEADER_OUT_OF_DATA) { return FileDecoderState::MORE_TO_PROCESS; @@ -273,8 +295,7 @@ FileDecoderState AudioDecoder::decode_flac_() { } uint32_t output_samples = 0; - auto result = this->flac_decoder_->decode_frame(this->input_transfer_buffer_->get_buffer_start(), - this->input_transfer_buffer_->available(), + auto result = this->flac_decoder_->decode_frame(this->input_buffer_->data(), this->input_buffer_->available(), this->output_transfer_buffer_->get_buffer_end(), &output_samples); if (result == esp_audio_libs::flac::FLAC_DECODER_ERROR_OUT_OF_DATA) { @@ -283,7 +304,7 @@ FileDecoderState AudioDecoder::decode_flac_() { } size_t bytes_consumed = this->flac_decoder_->get_bytes_index(); - this->input_transfer_buffer_->decrease_buffer_length(bytes_consumed); + this->input_buffer_->consume(bytes_consumed); if (result > esp_audio_libs::flac::FLAC_DECODER_ERROR_OUT_OF_DATA) { // Corrupted frame, don't retry with current buffer content, wait for new sync @@ -305,26 +326,25 @@ FileDecoderState AudioDecoder::decode_flac_() { #ifdef USE_AUDIO_MP3_SUPPORT FileDecoderState AudioDecoder::decode_mp3_() { // Look for the next sync word - int buffer_length = (int) this->input_transfer_buffer_->available(); - int32_t offset = - esp_audio_libs::helix_decoder::MP3FindSyncWord(this->input_transfer_buffer_->get_buffer_start(), buffer_length); + int buffer_length = (int) this->input_buffer_->available(); + int32_t offset = esp_audio_libs::helix_decoder::MP3FindSyncWord(this->input_buffer_->data(), buffer_length); if (offset < 0) { // New data may have the sync word - this->input_transfer_buffer_->decrease_buffer_length(buffer_length); + this->input_buffer_->consume(buffer_length); return FileDecoderState::POTENTIALLY_FAILED; } // Advance read pointer to match the offset for the syncword - this->input_transfer_buffer_->decrease_buffer_length(offset); - const uint8_t *buffer_start = this->input_transfer_buffer_->get_buffer_start(); + this->input_buffer_->consume(offset); + const uint8_t *buffer_start = this->input_buffer_->data(); - buffer_length = (int) this->input_transfer_buffer_->available(); + buffer_length = (int) this->input_buffer_->available(); int err = esp_audio_libs::helix_decoder::MP3Decode(this->mp3_decoder_, &buffer_start, &buffer_length, (int16_t *) this->output_transfer_buffer_->get_buffer_end(), 0); - size_t consumed = this->input_transfer_buffer_->available() - buffer_length; - this->input_transfer_buffer_->decrease_buffer_length(consumed); + size_t consumed = this->input_buffer_->available() - buffer_length; + this->input_buffer_->consume(consumed); if (err) { switch (err) { @@ -363,9 +383,8 @@ FileDecoderState AudioDecoder::decode_opus_() { size_t bytes_consumed, samples_decoded; micro_opus::OggOpusResult result = this->opus_decoder_->decode( - this->input_transfer_buffer_->get_buffer_start(), this->input_transfer_buffer_->available(), - this->output_transfer_buffer_->get_buffer_end(), this->output_transfer_buffer_->free(), bytes_consumed, - samples_decoded); + this->input_buffer_->data(), this->input_buffer_->available(), this->output_transfer_buffer_->get_buffer_end(), + this->output_transfer_buffer_->free(), bytes_consumed, samples_decoded); if (result == micro_opus::OGG_OPUS_OK) { if (!processed_header && this->opus_decoder_->is_initialized()) { @@ -379,7 +398,7 @@ FileDecoderState AudioDecoder::decode_opus_() { this->output_transfer_buffer_->increase_buffer_length( this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); } - this->input_transfer_buffer_->decrease_buffer_length(bytes_consumed); + this->input_buffer_->consume(bytes_consumed); } else if (result == micro_opus::OGG_OPUS_OUTPUT_BUFFER_TOO_SMALL) { // Reallocate to decode the packet on the next call this->free_buffer_required_ = this->opus_decoder_->get_required_output_buffer_size(); @@ -399,11 +418,11 @@ FileDecoderState AudioDecoder::decode_wav_() { if (!this->audio_stream_info_.has_value()) { // Header hasn't been processed - esp_audio_libs::wav_decoder::WAVDecoderResult result = this->wav_decoder_->decode_header( - this->input_transfer_buffer_->get_buffer_start(), this->input_transfer_buffer_->available()); + esp_audio_libs::wav_decoder::WAVDecoderResult result = + this->wav_decoder_->decode_header(this->input_buffer_->data(), this->input_buffer_->available()); if (result == esp_audio_libs::wav_decoder::WAV_DECODER_SUCCESS_IN_DATA) { - this->input_transfer_buffer_->decrease_buffer_length(this->wav_decoder_->bytes_processed()); + this->input_buffer_->consume(this->wav_decoder_->bytes_processed()); this->audio_stream_info_ = audio::AudioStreamInfo( this->wav_decoder_->bits_per_sample(), this->wav_decoder_->num_channels(), this->wav_decoder_->sample_rate()); @@ -419,7 +438,7 @@ FileDecoderState AudioDecoder::decode_wav_() { } } else { if (!this->wav_has_known_end_ || (this->wav_bytes_left_ > 0)) { - size_t bytes_to_copy = this->input_transfer_buffer_->available(); + size_t bytes_to_copy = this->input_buffer_->available(); if (this->wav_has_known_end_) { bytes_to_copy = std::min(bytes_to_copy, this->wav_bytes_left_); @@ -428,9 +447,8 @@ FileDecoderState AudioDecoder::decode_wav_() { bytes_to_copy = std::min(bytes_to_copy, this->output_transfer_buffer_->free()); if (bytes_to_copy > 0) { - std::memcpy(this->output_transfer_buffer_->get_buffer_end(), this->input_transfer_buffer_->get_buffer_start(), - bytes_to_copy); - this->input_transfer_buffer_->decrease_buffer_length(bytes_to_copy); + std::memcpy(this->output_transfer_buffer_->get_buffer_end(), this->input_buffer_->data(), bytes_to_copy); + this->input_buffer_->consume(bytes_to_copy); this->output_transfer_buffer_->increase_buffer_length(bytes_to_copy); if (this->wav_has_known_end_) { this->wav_bytes_left_ -= bytes_to_copy; diff --git a/esphome/components/audio/audio_decoder.h b/esphome/components/audio/audio_decoder.h index cad16110ae..726baa289e 100644 --- a/esphome/components/audio/audio_decoder.h +++ b/esphome/components/audio/audio_decoder.h @@ -50,12 +50,12 @@ enum class FileDecoderState : uint8_t { class AudioDecoder { /* * @brief Class that facilitates decoding an audio file. - * The audio file is read from a ring buffer source, decoded, and sent to an audio sink (ring buffer or speaker - * component). + * The audio file is read from a source (ring buffer or const data pointer), decoded, and sent to an audio sink + * (ring buffer, speaker component, or callback). * Supports wav, flac, mp3, and ogg opus formats. */ public: - /// @brief Allocates the input and output transfer buffers + /// @brief Allocates the output transfer buffer and stores the input buffer size for later use by add_source() /// @param input_buffer_size Size of the input transfer buffer in bytes. /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioDecoder(size_t input_buffer_size, size_t output_buffer_size); @@ -80,6 +80,17 @@ class AudioDecoder { esp_err_t add_sink(speaker::Speaker *speaker); #endif + /// @brief Adds a const data pointer as the source for raw file data. Does not allocate a transfer buffer. + /// @param data_pointer Pointer to the const audio data (e.g., stored in flash memory) + /// @param length Size of the data in bytes + /// @return ESP_OK + esp_err_t add_source(const uint8_t *data_pointer, size_t length); + + /// @brief Adds a callback as the sink for decoded audio. + /// @param callback Pointer to the AudioSinkCallback implementation + /// @return ESP_OK if successful, ESP_ERR_NO_MEM if the transfer buffer wasn't allocated + esp_err_t add_sink(AudioSinkCallback *callback); + /// @brief Sets up decoding the file /// @param audio_file_type AudioFileType of the file /// @return ESP_OK if successful, ESP_ERR_NO_MEM if the transfer buffers fail to allocate, or ESP_ERR_NOT_SUPPORTED if @@ -120,25 +131,26 @@ class AudioDecoder { #endif FileDecoderState decode_wav_(); - std::unique_ptr input_transfer_buffer_; + std::unique_ptr input_buffer_; std::unique_ptr output_transfer_buffer_; AudioFileType audio_file_type_{AudioFileType::NONE}; optional audio_stream_info_{}; + size_t input_buffer_size_{0}; size_t free_buffer_required_{0}; size_t wav_bytes_left_{0}; uint32_t potentially_failed_count_{0}; + uint32_t accumulated_frames_written_{0}; + uint32_t playback_ms_{0}; + bool end_of_file_{false}; bool wav_has_known_end_{false}; bool decoder_buffers_internally_{false}; bool pause_output_{false}; - - uint32_t accumulated_frames_written_{0}; - uint32_t playback_ms_{0}; }; } // namespace audio } // namespace esphome diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index a8be55d62f..5cd7cf9e63 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -142,7 +142,7 @@ size_t AudioSourceTransferBuffer::transfer_data_from_source(TickType_t ticks_to_ this->data_start_ = this->buffer_; } - size_t bytes_to_read = this->free(); + size_t bytes_to_read = AudioTransferBuffer::free(); size_t bytes_read = 0; if (bytes_to_read > 0) { if (this->ring_buffer_.use_count() > 0) { @@ -193,6 +193,21 @@ bool AudioSinkTransferBuffer::has_buffered_data() const { return (this->available() > 0); } +size_t AudioSourceTransferBuffer::free() const { return AudioTransferBuffer::free(); } + +bool AudioSourceTransferBuffer::has_buffered_data() const { return AudioTransferBuffer::has_buffered_data(); } + +void ConstAudioSourceBuffer::set_data(const uint8_t *data, size_t length) { + this->data_start_ = data; + this->length_ = length; +} + +void ConstAudioSourceBuffer::consume(size_t bytes) { + bytes = std::min(bytes, this->length_); + this->length_ -= bytes; + this->data_start_ += bytes; +} + } // namespace audio } // namespace esphome diff --git a/esphome/components/audio/audio_transfer_buffer.h b/esphome/components/audio/audio_transfer_buffer.h index 22c22cc9ae..c32d4d0e41 100644 --- a/esphome/components/audio/audio_transfer_buffer.h +++ b/esphome/components/audio/audio_transfer_buffer.h @@ -32,7 +32,7 @@ class AudioTransferBuffer { /// @brief Destructor that deallocates the transfer buffer ~AudioTransferBuffer(); - /// @brief Returns a pointer to the start of the transfer buffer where available() bytes of exisiting data can be read + /// @brief Returns a pointer to the start of the transfer buffer where available() bytes of existing data can be read uint8_t *get_buffer_start() const { return this->data_start_; } /// @brief Returns a pointer to the end of the transfer buffer where free() bytes of new data can be written @@ -129,10 +129,41 @@ class AudioSinkTransferBuffer : public AudioTransferBuffer { AudioSinkCallback *sink_callback_{nullptr}; }; -class AudioSourceTransferBuffer : public AudioTransferBuffer { +/// @brief Abstract interface for reading audio data from a buffer. +/// Provides a common read interface for both mutable transfer buffers and read-only const buffers. +class AudioReadableBuffer { + public: + virtual ~AudioReadableBuffer() = default; + + /// @brief Returns a pointer to the start of readable data + virtual const uint8_t *data() const = 0; + + /// @brief Returns the number of bytes available to read + virtual size_t available() const = 0; + + /// @brief Returns the number of free bytes available to write. Defaults to 0 for read-only buffers. + virtual size_t free() const { return 0; } + + /// @brief Advances past consumed data + /// @param bytes Number of bytes consumed + virtual void consume(size_t bytes) = 0; + + /// @brief Tests if there is any buffered data + virtual bool has_buffered_data() const = 0; + + /// @brief Refills the buffer from its source. No-op by default for read-only buffers. + /// @param ticks_to_wait FreeRTOS ticks to block while waiting for data + /// @param pre_shift If true, shifts existing data to the start of the buffer before reading + /// @return Number of bytes read + virtual size_t fill(TickType_t ticks_to_wait, bool pre_shift) { return 0; } + size_t fill(TickType_t ticks_to_wait) { return this->fill(ticks_to_wait, true); } +}; + +class AudioSourceTransferBuffer : public AudioTransferBuffer, public AudioReadableBuffer { /* * @brief A class that implements a transfer buffer for audio sources. * Supports reading audio data from a ring buffer into the transfer buffer for processing. + * Implements AudioReadableBuffer for use by consumers that only need read access. */ public: /// @brief Creates a new source transfer buffer. @@ -140,7 +171,7 @@ class AudioSourceTransferBuffer : public AudioTransferBuffer { /// @return unique_ptr if successfully allocated, nullptr otherwise static std::unique_ptr create(size_t buffer_size); - /// @brief Reads any available data from the sink into the transfer buffer. + /// @brief Reads any available data from the source into the transfer buffer. /// @param ticks_to_wait FreeRTOS ticks to block while waiting for the source to have enough data /// @param pre_shift If true, any unwritten data is moved to the start of the buffer before transferring from the /// source. Defaults to true. @@ -150,6 +181,36 @@ class AudioSourceTransferBuffer : public AudioTransferBuffer { /// @brief Adds a ring buffer as the transfer buffer's source. /// @param ring_buffer weak_ptr to the allocated ring buffer void set_source(const std::weak_ptr &ring_buffer) { this->ring_buffer_ = ring_buffer.lock(); }; + + // AudioReadableBuffer interface + const uint8_t *data() const override { return this->data_start_; } + size_t available() const override { return this->buffer_length_; } + size_t free() const override; + void consume(size_t bytes) override { this->decrease_buffer_length(bytes); } + bool has_buffered_data() const override; + size_t fill(TickType_t ticks_to_wait, bool pre_shift) override { + return this->transfer_data_from_source(ticks_to_wait, pre_shift); + } +}; + +/// @brief A lightweight read-only audio buffer for const data sources (e.g., flash memory). +/// Does not allocate memory or transfer data from external sources. +class ConstAudioSourceBuffer : public AudioReadableBuffer { + public: + /// @brief Sets the data pointer and length for the buffer + /// @param data Pointer to the const audio data + /// @param length Size of the data in bytes + void set_data(const uint8_t *data, size_t length); + + // AudioReadableBuffer interface + const uint8_t *data() const override { return this->data_start_; } + size_t available() const override { return this->length_; } + void consume(size_t bytes) override; + bool has_buffered_data() const override { return this->length_ > 0; } + + protected: + const uint8_t *data_start_{nullptr}; + size_t length_{0}; }; } // namespace audio