diff --git a/CODEOWNERS b/CODEOWNERS index 61f1f4722cc..3bb01931381 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -684,6 +684,7 @@ esphome/components/ultrasonic/* @ssieb @swoboda1337 esphome/components/unicode/* @esphome/core esphome/components/update/* @jesserockz esphome/components/uponor_smatrix/* @kroimon +esphome/components/usb_audio/* @ab-tools esphome/components/usb_cdc_acm/* @kbx81 esphome/components/usb_host/* @clydebarrow esphome/components/usb_uart/* @clydebarrow diff --git a/esphome/components/usb_audio/__init__.py b/esphome/components/usb_audio/__init__.py new file mode 100644 index 00000000000..fd1333d46f6 --- /dev/null +++ b/esphome/components/usb_audio/__init__.py @@ -0,0 +1,152 @@ +from collections.abc import Callable + +import esphome.codegen as cg +from esphome.components import audio, esp32, usb_host +import esphome.config_validation as cv +from esphome.const import ( + CONF_BITS_PER_SAMPLE, + CONF_BUFFER_SIZE, + CONF_ID, + CONF_NUM_CHANNELS, + CONF_PLATFORM, + CONF_SAMPLE_RATE, +) +from esphome.cpp_generator import MockObj +import esphome.final_validate as fv +from esphome.types import ConfigType + +CODEOWNERS = ["@ab-tools"] +DEPENDENCIES = ["esp32"] +# usb_host installs the USB host library, services its events and limits the variants +AUTO_LOAD = ["audio", "usb_host"] +DOMAIN = "usb_audio" + +usb_audio_ns = cg.esphome_ns.namespace("usb_audio") +USBAudioComponent = usb_audio_ns.class_("USBAudioComponent", cg.Component) +AudioStreamInfo = audio.audio_ns.class_("AudioStreamInfo") + +CONF_USB_AUDIO_ID = "usb_audio_id" +BITS_PER_SAMPLE_OPTIONS = [8, 16, 24, 32] +DEFAULT_BUFFER_SIZE = 6400 + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(USBAudioComponent), + } +).extend(cv.COMPONENT_SCHEMA) + + +def stream_schema(default_sample_rate: int, default_num_channels: int) -> cv.Schema: + """Stream format and buffer options shared by the speaker and microphone platforms.""" + return cv.Schema( + { + cv.GenerateID(CONF_USB_AUDIO_ID): cv.use_id(USBAudioComponent), + cv.Optional(CONF_SAMPLE_RATE, default=default_sample_rate): cv.int_range( + min=8000, max=96000 + ), + cv.Optional(CONF_BITS_PER_SAMPLE, default="16bit"): cv.All( + cv.float_with_unit("bits", "bit"), cv.one_of(*BITS_PER_SAMPLE_OPTIONS) + ), + cv.Optional(CONF_NUM_CHANNELS, default=default_num_channels): cv.int_range( + min=1, max=2 + ), + # The default depends on the format; see apply_buffer_size() + cv.Optional(CONF_BUFFER_SIZE): cv.positive_not_null_int, + } + ).extend(cv.COMPONENT_SCHEMA) + + +def set_stream_limits(config: ConfigType) -> ConfigType: + """Pin the audio stream limits to the one configured format.""" + bits_per_sample = int(config[CONF_BITS_PER_SAMPLE]) + channels = config[CONF_NUM_CHANNELS] + sample_rate = config[CONF_SAMPLE_RATE] + # The limits validator edits the config in place and returns None + audio.set_stream_limits( + min_bits_per_sample=bits_per_sample, + max_bits_per_sample=bits_per_sample, + min_channels=channels, + max_channels=channels, + min_sample_rate=sample_rate, + max_sample_rate=sample_rate, + )(config) + return config + + +def bytes_per_frame(config: ConfigType) -> int: + """Bytes in one frame (one sample for every channel) of a platform's configured format.""" + return config[CONF_NUM_CHANNELS] * int(config[CONF_BITS_PER_SAMPLE]) // 8 + + +def apply_buffer_size(config: ConfigType, min_size: int, minimum: str) -> ConfigType: + """Default buffer_size to DEFAULT_BUFFER_SIZE, or the format's minimum when that is larger; reject smaller ones. + + `minimum` describes what min_size holds, for the error message. + """ + if CONF_BUFFER_SIZE not in config: + config[CONF_BUFFER_SIZE] = max(DEFAULT_BUFFER_SIZE, min_size) + elif config[CONF_BUFFER_SIZE] < min_size: + raise cv.Invalid( + f"buffer_size must be at least {min_size} bytes ({minimum}) for this format", + path=[CONF_BUFFER_SIZE], + ) + return config + + +def final_validate_one_per_hub(domain: str) -> Callable[[ConfigType], ConfigType]: + """The hub streams one speaker and one microphone, so reject a second platform of the same kind on a hub.""" + + def validator(config: ConfigType) -> ConfigType: + hub_id = str(config[CONF_USB_AUDIO_ID]) + count = sum( + 1 + for platform_config in fv.full_config.get().get(domain, []) + if platform_config.get(CONF_PLATFORM) == DOMAIN + and str(platform_config.get(CONF_USB_AUDIO_ID)) == hub_id + ) + if count > 1: + raise cv.Invalid( + f"Only one usb_audio {domain} per usb_audio component is supported", + path=[CONF_USB_AUDIO_ID], + ) + return config + + return validator + + +def max_packet_bytes(config: ConfigType) -> int: + """Largest isochronous packet of a platform's format: a 1 ms packet, plus one frame for rate adaptation.""" + return (-(-config[CONF_SAMPLE_RATE] // 1000) + 1) * bytes_per_frame(config) + + +def request_speaker_fifo(config: ConfigType) -> ConfigType: + """Ask usb_host for USB FIFO space for the speaker's isochronous OUT packets.""" + usb_host.request_periodic_fifo(out_bytes=max_packet_bytes(config)) + return config + + +def request_microphone_fifo(config: ConfigType) -> ConfigType: + """Ask usb_host for USB FIFO space for the microphone's isochronous IN packets.""" + usb_host.request_periodic_fifo(in_bytes=max_packet_bytes(config)) + return config + + +def stream_info(config: ConfigType) -> MockObj: + """The C++ AudioStreamInfo for a platform's configured format.""" + return AudioStreamInfo( + int(config[CONF_BITS_PER_SAMPLE]), + config[CONF_NUM_CHANNELS], + config[CONF_SAMPLE_RATE], + ) + + +async def to_code(config: ConfigType) -> None: + # Keep in sync with esphome/idf_component.yml. usb_host adds espressif/usb on IDF 6.0 and later. + esp32.add_idf_component(name="espressif/usb_host_uac", ref="1.5.0") + # 18 ms of transfers in flight instead of the driver's 9 ms, so a late transfer completion does not leave the bus + # idle while the ring buffer still holds audio. + esp32.set_idf_sdkconfig_default("CONFIG_UAC_NUM_ISOC_URBS", 6) + + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + cg.add_define("USE_USB_AUDIO") diff --git a/esphome/components/usb_audio/microphone/__init__.py b/esphome/components/usb_audio/microphone/__init__.py new file mode 100644 index 00000000000..51ab3ac47ea --- /dev/null +++ b/esphome/components/usb_audio/microphone/__init__.py @@ -0,0 +1,56 @@ +import esphome.codegen as cg +from esphome.components import microphone +import esphome.config_validation as cv +from esphome.const import CONF_BUFFER_SIZE, CONF_ID, CONF_SAMPLE_RATE +from esphome.types import ConfigType + +from .. import ( + CONF_USB_AUDIO_ID, + apply_buffer_size, + bytes_per_frame, + final_validate_one_per_hub, + request_microphone_fifo, + set_stream_limits, + stream_info, + stream_schema, + usb_audio_ns, +) + +USBAudioMicrophone = usb_audio_ns.class_( + "USBAudioMicrophone", microphone.Microphone, cg.Component +) + +# Keep in sync with USBAudioMicrophone::READ_TIMEOUT_MS in usb_audio_microphone.h +READ_PERIOD_MS = 20 + + +def validate_buffer_size(config: ConfigType) -> ConfigType: + """The driver buffer must hold the audio that arrives between two reads, with room to spare.""" + bytes_per_second = config[CONF_SAMPLE_RATE] * bytes_per_frame(config) + min_size = bytes_per_second * 2 * READ_PERIOD_MS // 1000 + return apply_buffer_size(config, min_size, f"{2 * READ_PERIOD_MS} ms of audio") + + +CONFIG_SCHEMA = cv.All( + microphone.MICROPHONE_SCHEMA.extend(stream_schema(16000, 1)).extend( + { + cv.GenerateID(): cv.declare_id(USBAudioMicrophone), + } + ), + set_stream_limits, + validate_buffer_size, + request_microphone_fifo, +) + +FINAL_VALIDATE_SCHEMA = final_validate_one_per_hub("microphone") + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable( + config[CONF_ID], stream_info(config), config[CONF_BUFFER_SIZE] + ) + await cg.register_component(var, config) + await microphone.register_microphone(var, config) + hub = await cg.get_variable(config[CONF_USB_AUDIO_ID]) + await cg.register_parented(var, hub) + cg.add(hub.register_microphone(var)) diff --git a/esphome/components/usb_audio/microphone/usb_audio_microphone.cpp b/esphome/components/usb_audio/microphone/usb_audio_microphone.cpp new file mode 100644 index 00000000000..9d699baccf2 --- /dev/null +++ b/esphome/components/usb_audio/microphone/usb_audio_microphone.cpp @@ -0,0 +1,182 @@ +#include "usb_audio_microphone.h" + +#ifdef USE_USB_AUDIO + +#include "esphome/core/log.h" + +#include +#include + +extern "C" { +#include "esp_err.h" +} + +namespace esphome::usb_audio { + +ESPHOME_LOG_TAG(TAG, "usb_audio.mic"); + +void USBAudioMicrophone::setup() { + this->frame_.reserve(this->audio_stream_info_.ms_to_bytes(2 * READ_TIMEOUT_MS)); + if (!this->mic_task_.create(&USBAudioMicrophone::mic_task, "usb_mic", MIC_TASK_STACK_SIZE, this, MIC_TASK_PRIORITY, + false)) { + this->mark_failed(LOG_STR("Failed to create microphone task")); + return; + } + // start() and stop() wake the loop. + this->disable_loop(); +} + +void USBAudioMicrophone::dump_config() { + ESP_LOGCONFIG(TAG, + "USB Microphone:\n" + " Sample rate: %" PRIu32 " Hz\n" + " Bits per sample: %u\n" + " Channels: %u\n" + " Buffer size: %" PRIu32, + this->audio_stream_info_.get_sample_rate(), this->audio_stream_info_.get_bits_per_sample(), + this->audio_stream_info_.get_channels(), this->buffer_size_); +} + +void USBAudioMicrophone::loop() { + const uint8_t commands = this->commands_.exchange(0); + if (commands & COMMAND_STOP) { + // Also when a failed start left the microphone STOPPED, so the next session's retries start from the minimum. + this->cancel_timeout(RETRY_START_ID); + this->retry_delay_ms_ = RETRY_MIN_MS; + } + if ((commands & COMMAND_STOP) && + (this->state_ == microphone::STATE_STARTING || this->state_ == microphone::STATE_RUNNING)) { + this->parent_->suspend(Endpoint::MICROPHONE); + // The task notices within one read timeout; the stop finishes below once it is idle. + this->running_ = false; + this->state_ = microphone::STATE_STOPPING; + } + if (this->state_ == microphone::STATE_STOPPING && this->task_idle_) { + this->state_ = microphone::STATE_STOPPED; + } + // A retry can race the last stop(), so only start while someone is still listening. + if ((commands & COMMAND_START) && this->listeners_ > 0) { + if (this->state_ == microphone::STATE_STOPPED) { + this->state_ = microphone::STATE_STARTING; + // Answers through on_stream_state() now or once the device connects. + this->parent_->resume(Endpoint::MICROPHONE); + } else if (this->state_ == microphone::STATE_STOPPING) { + // Retry once the task is idle. + this->commands_.fetch_or(COMMAND_START); + } + } + if (this->state_ != microphone::STATE_STOPPING) { + this->disable_loop(); + } +} + +void USBAudioMicrophone::request_(uint8_t set, uint8_t clear) { + this->commands_.fetch_and(static_cast(~clear)); + this->commands_.fetch_or(set); + this->enable_loop_soon_any_context(); +} + +// Every MicrophoneSource forwards its own start() and stop(), so capture runs while any of them is listening, like +// i2s_audio's listener count. +void USBAudioMicrophone::start() { + if (this->is_failed()) { + return; + } + uint8_t listeners = this->listeners_.load(); + do { + if (listeners >= MAX_LISTENERS) { + return; + } + } while (!this->listeners_.compare_exchange_weak(listeners, listeners + 1)); + if (listeners == 0) { + this->request_(COMMAND_START, COMMAND_STOP); + } +} + +void USBAudioMicrophone::stop() { + uint8_t listeners = this->listeners_.load(); + do { + if (listeners == 0) { + return; + } + } while (!this->listeners_.compare_exchange_weak(listeners, listeners - 1)); + if (listeners == 1) { + this->request_(COMMAND_STOP, COMMAND_START); + } +} + +void USBAudioMicrophone::on_stream_state(StreamEvent event) { + switch (event) { + case StreamEvent::RUNNING: + if (this->state_ == microphone::STATE_STARTING) { + this->task_idle_ = false; + this->running_ = true; + xTaskNotifyGive(this->mic_task_.get_handle()); + this->state_ = microphone::STATE_RUNNING; + this->retry_delay_ms_ = RETRY_MIN_MS; + } + break; + case StreamEvent::LOST: + // The hub restarts the stream on reconnect, since the microphone still wants it, so keep waiting in STARTING; + // this also covers a disconnect while the stream was still starting. + if (this->state_ == microphone::STATE_RUNNING) { + ESP_LOGW(TAG, "USB device disconnected, waiting for it to return"); + this->running_ = false; + this->state_ = microphone::STATE_STARTING; + } + break; + case StreamEvent::FAILED: + // Only a start or resume fails, so the task is idle. + if (this->state_ == microphone::STATE_STARTING) { + this->state_ = microphone::STATE_STOPPED; + // A MicrophoneSource calls start() once and then waits for data, and start() only asks for the stream when + // the first listener arrives, so retry here for the listeners still held. + if (this->listeners_ > 0) { + this->set_timeout(RETRY_START_ID, this->retry_delay_ms_, [this]() { + if (this->listeners_ > 0) { + this->request_(COMMAND_START, 0); + } + }); + this->retry_delay_ms_ = std::min(this->retry_delay_ms_ * 2, RETRY_MAX_MS); + } + } + break; + } +} + +void USBAudioMicrophone::mic_task(void *param) { static_cast(param)->mic_task_loop_(); } + +void USBAudioMicrophone::mic_task_loop_() { + while (true) { + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + + while (this->running_) { + size_t bytes_read = 0; + this->frame_.resize(this->frame_.capacity()); + esp_err_t err = + this->parent_->read_microphone(this->frame_.data(), this->frame_.size(), &bytes_read, READ_TIMEOUT_MS); + if (!this->running_) { + break; + } + + if (err == ESP_OK && bytes_read > 0) { + this->frame_.resize(bytes_read); + this->data_callbacks_.call(this->frame_); + } else if (err == ESP_ERR_TIMEOUT) { + // The driver already waited READ_TIMEOUT_MS for data; read again. + } else if (err == ESP_ERR_INVALID_STATE) { + // The stream is starting or suspended. + vTaskDelay(pdMS_TO_TICKS(READ_TIMEOUT_MS)); + } else if (err != ESP_OK) { + ESP_LOGW(TAG, "Read error: %s", esp_err_to_name(err)); + vTaskDelay(pdMS_TO_TICKS(READ_TIMEOUT_MS)); + } + } + + this->task_idle_ = true; + } +} + +} // namespace esphome::usb_audio + +#endif diff --git a/esphome/components/usb_audio/microphone/usb_audio_microphone.h b/esphome/components/usb_audio/microphone/usb_audio_microphone.h new file mode 100644 index 00000000000..1b2ea744e0e --- /dev/null +++ b/esphome/components/usb_audio/microphone/usb_audio_microphone.h @@ -0,0 +1,76 @@ +#pragma once + +#include "esphome/core/defines.h" + +#ifdef USE_USB_AUDIO + +#include "esphome/components/usb_audio/usb_audio.h" + +#include "esphome/components/audio/audio.h" +#include "esphome/components/microphone/microphone.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" +#include "esphome/core/static_task.h" + +#include +#include + +namespace esphome::usb_audio { + +class USBAudioMicrophone : public microphone::Microphone, + public Component, + public Parented, + public USBAudioStream { + public: + USBAudioMicrophone(const audio::AudioStreamInfo &stream_info, uint32_t buffer_size) : USBAudioStream(buffer_size) { + this->audio_stream_info_ = stream_info; + } + + void setup() override; + void dump_config() override; + void loop() override; + + // Safe from any task; loop() acts on the request. + void start() override; + void stop() override; + + audio::AudioStreamInfo get_usb_stream_info() override { return this->audio_stream_info_; } + void on_stream_state(StreamEvent event) override; + bool stream_idle() const override { return this->task_idle_; } + + protected: + static constexpr uint8_t COMMAND_START = 1 << 0; + static constexpr uint8_t COMMAND_STOP = 1 << 1; + // Same limit as i2s_audio's microphone. + static constexpr uint8_t MAX_LISTENERS = 16; + static constexpr uint32_t READ_TIMEOUT_MS = 20; + // A failed start is retried while listeners are held, backing off so a device that keeps failing does not flood the + // log. + static constexpr uint32_t RETRY_MIN_MS = 1000; + static constexpr uint32_t RETRY_MAX_MS = 30000; + // Scheduler ids. + static constexpr uint32_t RETRY_START_ID = 0; + static constexpr uint32_t MIC_TASK_STACK_SIZE = 4096; + static constexpr UBaseType_t MIC_TASK_PRIORITY = 18; + + static void mic_task(void *param); + void mic_task_loop_(); + void request_(uint8_t set, uint8_t clear); + + // The data callback takes a std::vector, so this is reserved once in setup() and only ever resized within its + // capacity by the mic task. + std::vector frame_; + // Created once in setup() and parked between streams. + StaticTask mic_task_; + std::atomic commands_{0}; + // start() calls not yet matched by a stop(); capture stops when the last listener stops. + std::atomic listeners_{0}; + std::atomic running_{false}; + std::atomic task_idle_{true}; + // Main loop only. + uint32_t retry_delay_ms_{RETRY_MIN_MS}; +}; + +} // namespace esphome::usb_audio + +#endif diff --git a/esphome/components/usb_audio/speaker/__init__.py b/esphome/components/usb_audio/speaker/__init__.py new file mode 100644 index 00000000000..62555f89d3c --- /dev/null +++ b/esphome/components/usb_audio/speaker/__init__.py @@ -0,0 +1,62 @@ +import esphome.codegen as cg +from esphome.components import speaker +import esphome.config_validation as cv +from esphome.const import CONF_BUFFER_SIZE, CONF_ID, CONF_SAMPLE_RATE +from esphome.types import ConfigType + +from .. import ( + CONF_USB_AUDIO_ID, + apply_buffer_size, + bytes_per_frame, + final_validate_one_per_hub, + request_speaker_fifo, + set_stream_limits, + stream_info, + stream_schema, + usb_audio_ns, +) + +USBAudioSpeaker = usb_audio_ns.class_("USBAudioSpeaker", speaker.Speaker, cg.Component) + +CONF_WRITE_TIMEOUT = "write_timeout" + +# Keep in sync with the usb_host_uac default of CONFIG_UAC_NUM_PACKETS_PER_URB (1 ms packets per transfer) +PACKETS_PER_TRANSFER = 3 + + +def validate_buffer_size(config: ConfigType) -> ConfigType: + """The driver only sends complete transfers, so the buffer must hold two of them to keep audio flowing.""" + frames_per_packet = -(-config[CONF_SAMPLE_RATE] // 1000) + transfer_size = PACKETS_PER_TRANSFER * frames_per_packet * bytes_per_frame(config) + return apply_buffer_size(config, 2 * transfer_size, "two USB transfers") + + +CONFIG_SCHEMA = cv.All( + speaker.SPEAKER_SCHEMA.extend(stream_schema(48000, 2)).extend( + { + cv.GenerateID(): cv.declare_id(USBAudioSpeaker), + cv.Optional( + CONF_WRITE_TIMEOUT, default="20ms" + ): cv.positive_time_period_milliseconds, + } + ), + set_stream_limits, + validate_buffer_size, + request_speaker_fifo, +) + +FINAL_VALIDATE_SCHEMA = final_validate_one_per_hub("speaker") + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable( + config[CONF_ID], + stream_info(config), + config[CONF_BUFFER_SIZE], + config[CONF_WRITE_TIMEOUT].total_milliseconds, + ) + await cg.register_component(var, config) + await speaker.register_speaker(var, config) + hub = await cg.get_variable(config[CONF_USB_AUDIO_ID]) + await cg.register_parented(var, hub) + cg.add(hub.register_speaker(var)) diff --git a/esphome/components/usb_audio/speaker/usb_audio_speaker.cpp b/esphome/components/usb_audio/speaker/usb_audio_speaker.cpp new file mode 100644 index 00000000000..2e1613b8ccb --- /dev/null +++ b/esphome/components/usb_audio/speaker/usb_audio_speaker.cpp @@ -0,0 +1,359 @@ +#include "usb_audio_speaker.h" + +#ifdef USE_USB_AUDIO + +#include +#include + +#include "esphome/core/hal.h" +#include "esphome/core/log.h" +#include "esp_err.h" +#include "esp_timer.h" +#include "sdkconfig.h" + +namespace esphome::usb_audio { + +ESPHOME_LOG_TAG(TAG, "usb_audio.spk"); + +static constexpr size_t MAX_TIMEOUT_RETRIES = 4; +// Zeros are silence for the signed PCM the driver streams; 240 is a whole number of frames for every format. +static constexpr uint8_t SILENCE[240] = {}; + +void USBAudioSpeaker::setup() { + const auto &info = this->usb_stream_info_; + this->write_chunk_ = info.frames_to_bytes(std::max(info.bytes_to_frames(this->buffer_size_ / 4), 1)); + // The driver sends 1 ms packets, grouped into transfers it keeps in flight. + this->frame_bytes_ = info.frames_to_bytes(1); + this->packet_bytes_ = info.frames_to_bytes(info.get_sample_rate() / 1000); + this->packet_frac_ = info.get_sample_rate() % 1000; + this->transfer_bytes_ = info.frames_to_bytes((info.get_sample_rate() + 999) / 1000 * CONFIG_UAC_NUM_PACKETS_PER_URB); + this->transfer_us_ = CONFIG_UAC_NUM_PACKETS_PER_URB * 1000; + this->in_flight_us_ = CONFIG_UAC_NUM_ISOC_URBS * this->transfer_us_; + // About 10 ms per report, like the I2S speaker's DMA buffers. + this->report_frames_ = std::max(info.get_sample_rate() / 100, 1); + // start(), stop() and finish() wake the loop. + this->disable_loop(); +} + +void USBAudioSpeaker::dump_config() { + ESP_LOGCONFIG(TAG, + "USB Speaker:\n" + " Sample rate: %" PRIu32 " Hz\n" + " Bits per sample: %u\n" + " Channels: %u\n" + " Buffer size: %" PRIu32 "\n" + " Write timeout: %" PRIu32 " ms", + this->usb_stream_info_.get_sample_rate(), this->usb_stream_info_.get_bits_per_sample(), + this->usb_stream_info_.get_channels(), this->buffer_size_, this->write_timeout_ms_); +} + +void USBAudioSpeaker::loop() { + this->deliver_reports_(); + const uint8_t commands = this->commands_.exchange(0); + if (commands & COMMAND_STOP) { + this->stop_(); + } + if ((commands & COMMAND_START) && this->state_ == speaker::STATE_STOPPED) { + // Suspending flushed the driver's ring buffer, and resuming restarts its packet sizing. + portENTER_CRITICAL(&this->taken_lock_); + this->ring_bytes_ = 0; + this->ring_padding_ = 0; + // Resuming resets the driver's accumulator, then prepares every transfer, which advances it by their packets. + this->packet_frac_accum_ = CONFIG_UAC_NUM_ISOC_URBS * CONFIG_UAC_NUM_PACKETS_PER_URB * this->packet_frac_ % 1000; + this->unreported_frames_ = 0; + this->report_frames_pending_ = 0; + this->last_finish_us_ = 0; + portEXIT_CRITICAL(&this->taken_lock_); + this->state_ = speaker::STATE_STARTING; + // Answers through on_stream_state() now or once the device connects. + this->parent_->resume(Endpoint::SPEAKER); + } + + if (commands & COMMAND_FINISH) { + if (this->state_ == speaker::STATE_RUNNING) { + this->begin_finishing_(); + } else if (this->state_ == speaker::STATE_STARTING) { + // The stream is still starting; on_stream_state() finishes it once it runs. + this->finish_after_start_ = true; + } + } + if (this->finishing_ && this->pad_remaining_ > 0) { + this->pad_with_silence_(); + } + // has_buffered_data_() turns false once the driver has taken the last whole transfer and it has had time to play. + if (this->finishing_ && this->pad_remaining_ == 0 && !this->has_buffered_data_(esp_timer_get_time())) { + this->stop_(); + } + if (!this->finishing_) { + this->disable_loop(); + } +} + +void USBAudioSpeaker::request_(uint8_t set, uint8_t clear) { + this->commands_.fetch_and(static_cast(~clear)); + this->commands_.fetch_or(set); + this->enable_loop_soon_any_context(); +} + +void USBAudioSpeaker::start() { + if (this->state_ == speaker::STATE_STARTING || this->state_ == speaker::STATE_RUNNING) { + return; + } + this->request_(COMMAND_START, COMMAND_STOP); +} + +void USBAudioSpeaker::stop() { this->request_(COMMAND_STOP, COMMAND_START | COMMAND_FINISH); } + +void USBAudioSpeaker::finish() { this->request_(COMMAND_FINISH, 0); } + +void USBAudioSpeaker::begin_finishing_() { + if (this->finishing_) { + return; + } + this->finishing_ = true; + this->pad_remaining_ = this->transfer_bytes_; + // loop() pads the last transfer and stops once the audio has played. + this->enable_loop(); +} + +void USBAudioSpeaker::stop_() { + this->finishing_ = false; + this->finish_after_start_ = false; + this->pad_remaining_ = 0; + if (this->state_ == speaker::STATE_STOPPED) { + return; + } + this->parent_->suspend(Endpoint::SPEAKER); + this->state_ = speaker::STATE_STOPPED; +} + +void USBAudioSpeaker::on_stream_state(StreamEvent event) { + if (event == StreamEvent::RUNNING) { + if (this->state_ == speaker::STATE_STARTING) { + this->state_ = speaker::STATE_RUNNING; + if (this->finish_after_start_) { + this->finish_after_start_ = false; + this->begin_finishing_(); + } + } + return; + } + // Playback stops on a lost device as well as a failed start; the next start() opens it again. + if (event == StreamEvent::LOST && this->state_ == speaker::STATE_RUNNING) { + ESP_LOGW(TAG, "USB device disconnected, stopping playback"); + } + this->stop_(); +} + +size_t USBAudioSpeaker::play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) { + // 0 ticks is one non-blocking pass; portMAX_DELAY is capped by the configured write timeout. + const uint32_t budget_ms = ticks_to_wait == portMAX_DELAY ? this->write_timeout_ms_ : pdTICKS_TO_MS(ticks_to_wait); + return this->play_internal_(data, length, budget_ms); +} + +// Callers such as voice_assistant and rtttl call this from loop(), so it never blocks. +size_t USBAudioSpeaker::play(const uint8_t *data, size_t length) { return this->play(data, length, 0); } + +size_t USBAudioSpeaker::play_internal_(const uint8_t *data, size_t length, uint32_t budget_ms) { + if (this->audio_stream_info_ != this->usb_stream_info_) { + // Playing it through the configured stream would change the speed and channel layout. + if (!this->format_mismatch_.exchange(true)) { + const auto &info = this->audio_stream_info_; + ESP_LOGE(TAG, + "Audio format (%" PRIu32 " Hz, %u bits, %u channels) does not match the USB stream; use a resampler " + "speaker", + info.get_sample_rate(), info.get_bits_per_sample(), info.get_channels()); + } + if (budget_ms > 0) { + vTaskDelay(pdMS_TO_TICKS(budget_ms)); + } + return 0; + } + this->format_mismatch_ = false; + + if (this->state_ != speaker::STATE_RUNNING) { + this->start(); + // Wait out the budget so a pipeline task retrying play() does not spin. + if (budget_ms > 0) { + vTaskDelay(pdMS_TO_TICKS(budget_ms)); + } + return 0; + } + + // Counted before the hub's handle is read, so the hub never closes the device under this call. + this->writes_in_flight_.fetch_add(1); + + // Runs on the caller's task, so the cached loop time does not apply. + const uint32_t start_ms = millis(); + uint32_t now = start_ms; + size_t written = 0; + size_t timeout_retries = 0; + while (written < length) { + const uint32_t elapsed = now - start_ms; + if (budget_ms > 0 && elapsed >= budget_ms) { + break; + } + const size_t chunk = std::min({length - written, this->write_chunk_, this->ring_room_()}); + if (chunk == 0) { + // Back-pressure: wait for the driver to take a transfer. + if (budget_ms == 0) { + break; + } + vTaskDelay(1); + now = millis(); + continue; + } + const uint32_t timeout_ms = budget_ms > 0 ? budget_ms - elapsed : 0; + const esp_err_t err = this->write_(data + written, chunk, timeout_ms, false); + now = millis(); + if (err == ESP_OK) { + written += chunk; + timeout_retries = 0; + } else if (err == ESP_ERR_INVALID_STATE) { + // The device went away. + break; + } else if (err != ESP_ERR_TIMEOUT) { + ESP_LOGW(TAG, "Write error: %s", esp_err_to_name(err)); + break; + } else if (budget_ms == 0 || ++timeout_retries >= MAX_TIMEOUT_RETRIES) { + // A full driver buffer is back-pressure; the caller retries. + break; + } + } + this->writes_in_flight_.fetch_sub(1); + return written; +} + +size_t USBAudioSpeaker::ring_room_() const { + // Whole frames up to buffer_size_. ring_bytes_ is never below the real fill, so a write of this much does not wait. + const uint32_t ring = this->ring_bytes_; + const uint32_t room = ring < this->buffer_size_ ? this->buffer_size_ - ring : 0; + return room - room % this->frame_bytes_; +} + +esp_err_t USBAudioSpeaker::write_(const uint8_t *data, size_t length, uint32_t timeout_ms, bool padding) { + // Counted first: the driver submits transfers inside the write, and on_transfer_taken() must see these bytes then. + // A failed write stores nothing, since the driver only writes whole chunks. + this->ring_bytes_.fetch_add(length); + if (padding) { + this->ring_padding_.fetch_add(length); + } + const esp_err_t err = this->parent_->write_speaker(data, length, timeout_ms); + if (err != ESP_OK) { + this->ring_bytes_.fetch_sub(length); + if (padding) { + this->ring_padding_.fetch_sub(length); + } + } + return err; +} + +void USBAudioSpeaker::pad_with_silence_() { + if (this->state_ != speaker::STATE_RUNNING) { + this->pad_remaining_ = 0; + return; + } + // Counted before the hub's handle is read, so the hub never closes the device under this call. + this->writes_in_flight_.fetch_add(1); + const size_t piece = sizeof(SILENCE) - sizeof(SILENCE) % this->usb_stream_info_.frames_to_bytes(1); + while (this->pad_remaining_ > 0) { + const size_t chunk = std::min({this->pad_remaining_, piece, this->ring_room_()}); + if (chunk == 0 || this->write_(SILENCE, chunk, 0, true) != ESP_OK) { + // A full buffer; the rest is written on a later loop(). + break; + } + this->pad_remaining_ -= chunk; + } + this->writes_in_flight_.fetch_sub(1); +} + +void USBAudioSpeaker::on_transfer_taken() { + bool report = false; + portENTER_CRITICAL(&this->taken_lock_); + // Mirror the driver: the next transfer is CONFIG_UAC_NUM_PACKETS_PER_URB packets, and the driver takes it only when + // the ring buffer holds all of it; otherwise it parks the transfer until the next write. + uint32_t accum = this->packet_frac_accum_; + uint32_t size = 0; + for (int i = 0; i < CONFIG_UAC_NUM_PACKETS_PER_URB; i++) { + accum += this->packet_frac_; + size += this->packet_bytes_; + if (accum >= 1000) { + size += this->frame_bytes_; + accum -= 1000; + } + } + const uint32_t ring = this->ring_bytes_; + if (ring >= size) { + this->packet_frac_accum_ = accum; + this->ring_bytes_.fetch_sub(size); + // Padding sits at the end of the ring buffer and is not audio from the caller, so it is not reported. + const uint32_t padding = std::min(this->ring_padding_, ring); + const uint32_t audio = std::min(size, ring - padding); + if (audio < size) { + this->ring_padding_.fetch_sub(std::min(size - audio, padding)); + } + const int64_t now_us = esp_timer_get_time(); + this->last_taken_us_ = static_cast(now_us); + // The bus plays one packet per millisecond, so while it streams a transfer finishes one transfer after the one + // taken before it, however late this callback runs. It cannot finish later than now plus the transfers in flight, + // so a callback that runs on time pulls back any lateness the chain picked up. When the bus is idle, after a start + // (the driver submits nothing until the first write) or once it ran dry, the transfer plays right away. + const int64_t finish_us = this->last_finish_us_ < now_us + ? now_us + this->transfer_us_ + : std::min(this->last_finish_us_ + this->transfer_us_, now_us + this->in_flight_us_); + this->last_finish_us_ = finish_us; + this->unreported_frames_ += this->usb_stream_info_.bytes_to_frames(audio); + // Batched like the I2S speaker's DMA buffers: about every 10 ms, and once the caller's audio runs out. + if (this->unreported_frames_ > 0 && (this->unreported_frames_ >= this->report_frames_ || audio == ring - padding)) { + this->report_frames_pending_ += this->unreported_frames_; + this->report_finish_us_ = finish_us; + this->unreported_frames_ = 0; + report = true; + } + } + portEXIT_CRITICAL(&this->taken_lock_); + if (report) { + // The listeners can block, so loop() calls them; the frames and their finish time travel together, so the delay + // does not change what they compute. + this->enable_loop_soon_any_context(); + } +} + +void USBAudioSpeaker::deliver_reports_() { + portENTER_CRITICAL(&this->taken_lock_); + const uint32_t frames = this->report_frames_pending_; + const int64_t finish_us = this->report_finish_us_; + this->report_frames_pending_ = 0; + portEXIT_CRITICAL(&this->taken_lock_); + if (frames > 0) { + this->audio_output_callback_(frames, finish_us); + } +} + +bool USBAudioSpeaker::has_buffered_data_(int64_t now_us) const { + if (this->state_ != speaker::STATE_RUNNING) { + return false; + } + // The driver still holds a whole transfer to take, or one it took has not finished playing. Less than a transfer + // stays in the ring buffer until the next write. + return this->ring_bytes_ >= this->transfer_bytes_ || + static_cast(static_cast(now_us) - this->last_taken_us_) < + static_cast(this->in_flight_us_); +} + +bool USBAudioSpeaker::has_buffered_data() const { return this->has_buffered_data_(esp_timer_get_time()); } + +// The hub's worker sends volume and mute to the device. +void USBAudioSpeaker::set_volume(float volume) { + speaker::Speaker::set_volume(volume); + this->parent_->set_speaker_volume_level(volume); +} + +void USBAudioSpeaker::set_mute_state(bool mute_state) { + speaker::Speaker::set_mute_state(mute_state); + this->parent_->set_speaker_mute_state(mute_state); +} + +} // namespace esphome::usb_audio + +#endif diff --git a/esphome/components/usb_audio/speaker/usb_audio_speaker.h b/esphome/components/usb_audio/speaker/usb_audio_speaker.h new file mode 100644 index 00000000000..a4edda7e0ea --- /dev/null +++ b/esphome/components/usb_audio/speaker/usb_audio_speaker.h @@ -0,0 +1,112 @@ +#pragma once + +#include "esphome/core/defines.h" + +#ifdef USE_USB_AUDIO + +#include "esphome/components/usb_audio/usb_audio.h" + +#include "esphome/components/audio/audio.h" +#include "esphome/components/speaker/speaker.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +#include + +namespace esphome::usb_audio { + +class USBAudioSpeaker : public speaker::Speaker, + public Component, + public Parented, + public USBAudioStream { + public: + USBAudioSpeaker(const audio::AudioStreamInfo &stream_info, uint32_t buffer_size, uint32_t write_timeout_ms) + : USBAudioStream(buffer_size), usb_stream_info_(stream_info), write_timeout_ms_(write_timeout_ms) { + this->audio_stream_info_ = stream_info; + } + + void setup() override; + void dump_config() override; + void loop() override; + + // Safe from any task; loop() acts on the request. + void start() override; + void stop() override; + void finish() override; + + size_t play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) override; + size_t play(const uint8_t *data, size_t length) override; + bool has_buffered_data() const override; + + void set_volume(float volume) override; + void set_mute_state(bool mute_state) override; + + audio::AudioStreamInfo get_usb_stream_info() override { return this->usb_stream_info_; } + void on_stream_state(StreamEvent event) override; + bool stream_idle() const override { return this->writes_in_flight_ == 0; } + void on_transfer_taken() override; + + protected: + static constexpr uint8_t COMMAND_START = 1 << 0; + static constexpr uint8_t COMMAND_STOP = 1 << 1; + static constexpr uint8_t COMMAND_FINISH = 1 << 2; + void request_(uint8_t set, uint8_t clear); + void deliver_reports_(); + void begin_finishing_(); + void stop_(); + size_t play_internal_(const uint8_t *data, size_t length, uint32_t budget_ms); + /// Write silence until a whole USB transfer is padded, so the driver sends a final partial transfer. + void pad_with_silence_(); + /// Write to the driver, counting the bytes in ring_bytes_ (and in ring_padding_ for silence padding). + esp_err_t write_(const uint8_t *data, size_t length, uint32_t timeout_ms, bool padding); + /// Bytes that can be written without filling the driver's ring buffer past buffer_size_. + size_t ring_room_() const; + bool has_buffered_data_(int64_t now_us) const; + + // The configured format the USB stream runs with. Callers such as the media pipeline set audio_stream_info_ for + // each item, so play() checks it against this instead of reopening the stream with another format. + const audio::AudioStreamInfo usb_stream_info_; + uint32_t write_timeout_ms_; + // Frame-aligned size of each driver write, set in setup(). + uint32_t write_chunk_{0}; + // The driver's packet sizing, mirrored so on_transfer_taken() knows how much each transfer takes: whole frames per + // 1 ms packet, plus a frame whenever the remainder of sample_rate / 1000 adds up. Set in setup(). + uint32_t frame_bytes_{0}; + uint32_t packet_bytes_{0}; + uint32_t packet_frac_{0}; + // Bytes in the largest USB transfer, how long one transfer plays and how long all the driver keeps in flight play, + // and the batch of frames reported at a time; set in setup(). + uint32_t transfer_bytes_{0}; + uint32_t transfer_us_{0}; + uint32_t in_flight_us_{0}; + uint32_t report_frames_{0}; + // Silence still to write while finishing. + uint32_t pad_remaining_{0}; + // Bytes in the driver's ring buffer: added before each write, since the driver can take a transfer inside it, and + // taken off for each transfer it takes. ring_padding_ is the silence padding at its end. + std::atomic ring_bytes_{0}; + std::atomic ring_padding_{0}; + // Low 32 bits of esp_timer_get_time() when the driver last took a transfer. + std::atomic last_taken_us_{0}; + // on_transfer_taken() runs on the USB driver's task, which must not block, and inside writes, so its state is + // guarded by a spinlock and the reports it collects are delivered from loop(). Reset while the stream is stopped. + portMUX_TYPE taken_lock_ = portMUX_INITIALIZER_UNLOCKED; + uint32_t packet_frac_accum_{0}; + uint32_t unreported_frames_{0}; + uint32_t report_frames_pending_{0}; + int64_t report_finish_us_{0}; + // When the last transfer taken finishes playing; reset on a start, so the next one starts the chain from now. + int64_t last_finish_us_{0}; + std::atomic commands_{0}; + // play() calls and silence padding that may be using the device handle; the hub closes a lost device only at zero. + std::atomic writes_in_flight_{0}; + // Set while play() rejects audio in a format other than usb_stream_info_, so the error is logged once. + std::atomic format_mismatch_{false}; + bool finishing_{false}; + // finish() arrived while the stream was still starting. + bool finish_after_start_{false}; +}; + +} // namespace esphome::usb_audio + +#endif diff --git a/esphome/components/usb_audio/usb_audio.cpp b/esphome/components/usb_audio/usb_audio.cpp new file mode 100644 index 00000000000..ec9537e1dce --- /dev/null +++ b/esphome/components/usb_audio/usb_audio.cpp @@ -0,0 +1,876 @@ +#include "usb_audio.h" + +#ifdef USE_USB_AUDIO + +#include +#include +#include + +#include "esphome/core/application.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +extern "C" { +#include "esp_err.h" +} + +namespace esphome::usb_audio { + +ESPHOME_LOG_TAG(TAG, "usb_audio"); + +// The driver task completes and resubmits the speaker's transfers, with only CONFIG_UAC_NUM_ISOC_URBS of them +// queued on the bus, so it runs above the network stack like the I2S speaker task; each wakeup is short. +static constexpr UBaseType_t UAC_TASK_PRIORITY = 19; +static constexpr uint32_t UAC_TASK_STACK_SIZE = 4096; +// Below the driver's task, so its transfers are not held up by control requests. +static constexpr UBaseType_t WORKER_TASK_PRIORITY = 5; +static constexpr uint32_t WORKER_TASK_STACK_SIZE = 4096; +// The worker retries volume and mute up to this many times, this far apart. +static constexpr uint8_t MAX_CONTROL_ATTEMPTS = 5; +static constexpr uint32_t CONTROL_RETRY_MS = 100; +// A failed close is retried up to this many times, this far apart, before the handle is given up. +static constexpr uint8_t MAX_CLOSE_ATTEMPTS = 5; +static constexpr uint32_t CLOSE_RETRY_MS = 500; + +static const LogString *endpoint_name(Endpoint endpoint) { + return endpoint == Endpoint::SPEAKER ? LOG_STR("Speaker") : LOG_STR("Microphone"); +} + +static uint8_t discrete_rate_count(const uac_host_dev_alt_param_t &alt) { + return std::min(alt.sample_freq_type, UAC_FREQ_NUM_MAX); +} + +static bool alt_supports_rate(const uac_host_dev_alt_param_t &alt, uint32_t sample_rate) { + if (alt.sample_freq_type == 0) { + return sample_rate >= alt.sample_freq_lower && sample_rate <= alt.sample_freq_upper; + } + const uint8_t count = discrete_rate_count(alt); + return std::find(alt.sample_freq, alt.sample_freq + count, sample_rate) != alt.sample_freq + count; +} + +void USBAudioComponent::setup() { + this->event_queue_ = xQueueCreateStatic(EVENT_QUEUE_LENGTH, sizeof(USBAudioEvent), this->event_queue_storage_.data(), + &this->event_queue_struct_); + this->op_queue_ = + xQueueCreateStatic(OP_QUEUE_LENGTH, sizeof(Op), this->op_queue_storage_.data(), &this->op_queue_struct_); + + uac_host_driver_config_t uac_cfg = { + .create_background_task = true, + .task_priority = UAC_TASK_PRIORITY, + .stack_size = UAC_TASK_STACK_SIZE, + .core_id = 0, + .callback = &USBAudioComponent::driver_event_stub, + .callback_arg = this, + }; + + esp_err_t err = uac_host_install(&uac_cfg); + if (err != ESP_OK) { + ESP_LOGE(TAG, "uac_host_install failed: %s", esp_err_to_name(err)); + this->mark_failed(); + return; + } + this->uac_installed_ = true; + + // Opening, starting, suspending and closing streams and setting volume are control transfers that can block for + // seconds, so they run here instead of on the main loop. + if (!this->worker_.create(&USBAudioComponent::worker_task, "usb_audio", WORKER_TASK_STACK_SIZE, this, + WORKER_TASK_PRIORITY, false)) { + this->mark_failed(LOG_STR("Failed to create worker task")); + return; + } + // The USB callbacks and the worker wake the loop when there is work. + this->disable_loop(); +} + +void USBAudioComponent::loop() { + this->process_events_(App.get_loop_component_start_time()); + + if (const uint32_t errors = this->transfer_errors_.exchange(0); errors != 0) { + ESP_LOGW(TAG, "%" PRIu32 " USB transfer errors", errors); + } + + // Stay awake while a lost device still waits to be closed. + if (std::any_of(this->endpoints_.begin(), this->endpoints_.end(), + [](const EndpointState &ep) { return ep.closing != nullptr; })) { + return; + } + this->disable_loop(); +} + +void USBAudioComponent::process_events_(uint32_t now) { + this->finish_closes_(now); + + USBAudioEvent event; + // Only the events already queued: one posted again while this runs waits for the next loop(). + for (UBaseType_t queued = uxQueueMessagesWaiting(this->event_queue_); + queued > 0 && xQueueReceive(this->event_queue_, &event, 0) == pdTRUE; queued--) { + this->process_event_(event, now); + } +} + +void USBAudioComponent::dump_config() { + const auto &speaker = this->endpoint_(Endpoint::SPEAKER); + const auto µphone = this->endpoint_(Endpoint::MICROPHONE); + ESP_LOGCONFIG(TAG, + "USB Audio:\n" + " Device connected: %s\n" + " Speaker registered: %s\n" + " Microphone registered: %s", + YESNO(speaker.handle.load() != nullptr || microphone.handle.load() != nullptr), + YESNO(speaker.platform != nullptr), YESNO(microphone.platform != nullptr)); +} + +bool USBAudioComponent::teardown() { + if (this->event_queue_ == nullptr) { + return true; + } + // Stop the platforms first, then let the worker close each device once its platform is idle. + this->begin_close_(Endpoint::SPEAKER); + this->begin_close_(Endpoint::MICROPHONE); + // The loop time does not advance while components tear down. + const uint32_t now = millis(); + this->process_events_(now); + if (!this->finish_closes_(now)) { + return false; + } + // Wait for the worker to finish whatever it was still running. + if (uxQueueMessagesWaiting(this->op_queue_) != 0 || this->driver_busy_) { + return false; + } + xQueueReset(this->event_queue_); + + if (this->uac_installed_) { + esp_err_t err = uac_host_uninstall(); + if (err == ESP_ERR_INVALID_STATE) { + return false; + } + if (err != ESP_OK) { + ESP_LOGW(TAG, "uac_host_uninstall returned %s", esp_err_to_name(err)); + } + this->uac_installed_ = false; + } + return true; +} + +void USBAudioComponent::resume(Endpoint endpoint) { + auto &ep = this->endpoint_(endpoint); + ep.active = true; + ep.request_seq++; + const uac_host_device_handle_t handle = ep.handle; + if (handle == nullptr) { + if (ep.open_failed && !ep.open_pending) { + // The device is still plugged in but its last open failed; try again for this start. + ep.open_failed = false; + this->open_stream_(endpoint, ep.failed_addr, ep.failed_iface); + return; + } + ESP_LOGD(TAG, "%s interface not connected yet", LOG_STR_ARG(endpoint_name(endpoint))); + return; + } + if (ep.start_pending) { + // The pending start reports back to the platform, and checks `active` first. + return; + } + if (!ep.stream_started) { + if (ep.start_rejected) { + // Already probed and logged for this device; a retrying caller must not flood the log. + this->stream_failed_(endpoint); + return; + } + this->start_stream_(endpoint); + return; + } + // Queued even while an earlier resume or suspend is pending: the worker runs them in order, so the driver ends in + // the latest requested state, and only this request's RESUMED reports back. + Op op; + op.type = Op::Type::RESUME; + op.endpoint = endpoint; + op.handle = handle; + op.seq = ep.request_seq; + if (!this->queue_op_(op)) { + this->stream_failed_(endpoint); + } +} + +void USBAudioComponent::suspend(Endpoint endpoint) { + auto &ep = this->endpoint_(endpoint); + ep.active = false; + ep.request_seq++; + const uac_host_device_handle_t handle = ep.handle; + // A start still running on the worker is suspended once its STARTED arrives. + if (!ep.stream_started || handle == nullptr) { + return; + } + Op op; + op.type = Op::Type::SUSPEND; + op.endpoint = endpoint; + op.handle = handle; + this->queue_op_(op); +} + +bool USBAudioComponent::enter_driver_(uint32_t &timeout_ms) { + const uint32_t start = millis(); + while (true) { + // Counted before the busy check, so the worker waits for this call to leave the driver before it takes it. + this->driver_users_.fetch_add(1); + if (!this->driver_busy_) { + const uint32_t waited = millis() - start; + timeout_ms = waited < timeout_ms ? timeout_ms - waited : 0; + return true; + } + this->driver_users_.fetch_sub(1); + // A control request takes a few milliseconds, so go on as soon as the worker is done with it. + if (millis() - start >= timeout_ms) { + return false; + } + vTaskDelay(1); + } +} + +esp_err_t USBAudioComponent::read_microphone(uint8_t *buffer, size_t size, size_t *bytes_read, uint32_t timeout_ms) { + *bytes_read = 0; + if (!this->enter_driver_(timeout_ms)) { + return ESP_ERR_TIMEOUT; + } + uac_host_device_handle_t handle = this->endpoint_(Endpoint::MICROPHONE).handle; + if (handle == nullptr) { + this->driver_users_.fetch_sub(1); + return ESP_ERR_INVALID_STATE; + } + // The driver collects up to `size` bytes for the whole timeout and returns what it got, or ESP_ERR_TIMEOUT when + // nothing arrived. + uint32_t actual = 0; + esp_err_t err = uac_host_device_read(handle, buffer, static_cast(size), &actual, pdMS_TO_TICKS(timeout_ms)); + this->driver_users_.fetch_sub(1); + *bytes_read = actual; + return err; +} + +esp_err_t USBAudioComponent::write_speaker(const uint8_t *data, size_t length, uint32_t timeout_ms) { + if (!this->enter_driver_(timeout_ms)) { + // A non-blocking caller retries later. + return ESP_ERR_TIMEOUT; + } + uac_host_device_handle_t handle = this->endpoint_(Endpoint::SPEAKER).handle; + if (handle == nullptr) { + this->driver_users_.fetch_sub(1); + return ESP_ERR_INVALID_STATE; + } + esp_err_t err = uac_host_device_write(handle, const_cast(data), static_cast(length), + pdMS_TO_TICKS(timeout_ms)); + this->driver_users_.fetch_sub(1); + if (err == ESP_FAIL || err == ESP_ERR_INVALID_STATE) { + // Ring buffer full or stream inactive; report as timeout so callers back off and retry. + err = ESP_ERR_TIMEOUT; + } + return err; +} + +void USBAudioComponent::process_event_(const USBAudioEvent &event, uint32_t now) { + switch (event.type) { + case USBAudioEvent::Type::CONNECTED: { + auto &ep = this->endpoint_(event.endpoint); + if (ep.closing != nullptr) { + // A device that came back waits until its old handle is closed; the queue keeps draining meanwhile, so the + // worker's CLOSED can arrive. + ep.connect_deferred = true; + ep.deferred_addr = event.addr; + ep.deferred_iface = event.iface_num; + break; + } + this->open_stream_(event.endpoint, event.addr, event.iface_num); + break; + } + case USBAudioEvent::Type::DISCONNECTED: + this->handle_disconnect_(event.handle); + break; + case USBAudioEvent::Type::OPENED: + this->on_opened_(event); + break; + case USBAudioEvent::Type::STARTED: + this->on_started_(event); + break; + case USBAudioEvent::Type::RESUMED: + this->on_resumed_(event); + break; + case USBAudioEvent::Type::CLOSED: + this->on_closed_(event, now); + break; + } +} + +void USBAudioComponent::open_stream_(Endpoint endpoint, uint8_t addr, uint8_t iface_num) { + auto &ep = this->endpoint_(endpoint); + const LogString *name = endpoint_name(endpoint); + if (ep.platform == nullptr) { + ESP_LOGD(TAG, "%s interface connected but not configured", LOG_STR_ARG(name)); + return; + } + if (ep.open_pending) { + // Either a duplicate, or a replacement for a device that went away during the pending open; on_opened_() tells + // them apart and replays a replacement. + ep.connect_deferred = true; + ep.deferred_addr = addr; + ep.deferred_iface = iface_num; + return; + } + if (ep.handle.load() != nullptr) { + ESP_LOGW(TAG, "%s interface already open, ignoring duplicate event", LOG_STR_ARG(name)); + return; + } + Op op; + op.type = Op::Type::OPEN; + op.endpoint = endpoint; + op.addr = addr; + op.iface_num = iface_num; + if (!this->queue_op_(op)) { + // The event is already off the queue, so post it again for a later loop() instead of losing the device. + USBAudioEvent event; + event.type = USBAudioEvent::Type::CONNECTED; + event.endpoint = endpoint; + event.addr = addr; + event.iface_num = iface_num; + this->post_event_(event); + return; + } + ep.open_pending = true; +} + +void USBAudioComponent::on_opened_(const USBAudioEvent &event) { + auto &ep = this->endpoint_(event.endpoint); + ep.open_pending = false; + bool lost_meanwhile = false; + for (auto &lost : this->lost_while_opening_) { + if (event.ok && lost == event.handle) { + lost = nullptr; + lost_meanwhile = true; + } + } + if (std::none_of(this->endpoints_.begin(), this->endpoints_.end(), + [](const EndpointState &other) { return other.open_pending; })) { + // Only a pending open can still claim an entry; a failed one reports no handle, so its entry would stay behind + // and could match a later device that reuses the freed handle. + this->lost_while_opening_.fill(nullptr); + } + if (!event.ok) { + if (ep.connect_deferred) { + // A newer connect arrived during the failed open; try that one. + ep.connect_deferred = false; + this->open_stream_(event.endpoint, ep.deferred_addr, ep.deferred_iface); + return; + } + // Keep where the device is, so the next start() retries the open without a replug. + ep.open_failed = true; + ep.failed_addr = event.addr; + ep.failed_iface = event.iface_num; + if (ep.active) { + this->stream_failed_(event.endpoint); + } + return; + } + ep.open_failed = false; + ep.handle = event.handle; + ep.iface = event.iface_num; + ESP_LOGI(TAG, "%s interface %u connected (addr=%u)", LOG_STR_ARG(endpoint_name(event.endpoint)), ep.iface, + event.addr); + if (lost_meanwhile) { + // It went away while the worker was opening it; a connect deferred meanwhile is opened once it is closed. + this->handle_disconnect_(event.handle); + return; + } + // The device that was being opened is still here, so a connect deferred meanwhile was a duplicate. + ep.connect_deferred = false; + if (ep.active) { + this->start_stream_(event.endpoint); + } +} + +void USBAudioComponent::start_stream_(Endpoint endpoint) { + auto &ep = this->endpoint_(endpoint); + Op op; + op.type = Op::Type::START; + op.endpoint = endpoint; + op.handle = ep.handle; + if (!this->queue_op_(op)) { + this->stream_failed_(endpoint); + return; + } + ep.start_pending = true; +} + +void USBAudioComponent::on_started_(const USBAudioEvent &event) { + auto &ep = this->endpoint_(event.endpoint); + ep.start_pending = false; + if (event.handle != ep.handle.load()) { + // The device went away meanwhile; the platform was already told. + return; + } + if (!event.ok) { + ep.start_rejected = event.rejected; + if (ep.active) { + this->stream_failed_(event.endpoint); + } + return; + } + ep.stream_started = true; + if (event.endpoint == Endpoint::SPEAKER) { + // Send the current volume and mute to the newly started stream. + this->volume_.pending = true; + this->mute_.pending = true; + this->queue_controls_(); + } + if (!ep.active) { + // The platform stopped while the stream was starting. + this->suspend(event.endpoint); + return; + } + ep.platform->on_stream_state(StreamEvent::RUNNING); +} + +void USBAudioComponent::on_resumed_(const USBAudioEvent &event) { + auto &ep = this->endpoint_(event.endpoint); + if (event.handle != ep.handle.load() || event.seq != ep.request_seq || !ep.active) { + // The device went away, or a later suspend or resume is queued behind this one and decides the state. + return; + } + if (!event.ok) { + this->stream_failed_(event.endpoint); + return; + } + ep.platform->on_stream_state(StreamEvent::RUNNING); +} + +void USBAudioComponent::stream_failed_(Endpoint endpoint) { + auto &ep = this->endpoint_(endpoint); + // The platform goes back to stopped, so a reconnect must not restart the stream; its next start() retries. + ep.active = false; + ep.platform->on_stream_state(StreamEvent::FAILED); +} + +void USBAudioComponent::set_speaker_volume_level(float volume) { + this->speaker_volume_ = static_cast(std::lround(std::clamp(volume, 0.0f, 1.0f) * 100.0f)); + this->volume_.pending = true; + this->queue_controls_(); +} + +void USBAudioComponent::set_speaker_mute_state(bool mute_state) { + this->speaker_muted_ = mute_state; + this->mute_.pending = true; + this->queue_controls_(); +} + +void USBAudioComponent::queue_controls_() { + if (this->op_queue_ == nullptr) { + return; + } + if (this->controls_queued_.exchange(true)) { + // The queued op sends the latest values, so lifecycle ops keep their queue slots. + return; + } + Op op; + op.type = Op::Type::CONTROLS; + if (xQueueSend(this->op_queue_, &op, 0) != pdTRUE) { + // The controls stay pending and the next start or change queues them again. + this->controls_queued_ = false; + } +} + +void USBAudioComponent::handle_disconnect_(uac_host_device_handle_t handle) { + for (size_t i = 0; i < this->endpoints_.size(); i++) { + auto &ep = this->endpoints_[i]; + if (handle != nullptr && ep.handle.load() == handle) { + const auto endpoint = static_cast(i); + ESP_LOGI(TAG, "%s interface %u disconnected", LOG_STR_ARG(endpoint_name(endpoint)), ep.iface); + this->begin_close_(endpoint); + return; + } + } + if (std::any_of(this->endpoints_.begin(), this->endpoints_.end(), + [](const EndpointState &ep) { return ep.open_pending; })) { + // Its open is still running on the worker; close it once it reports back. + for (auto &lost : this->lost_while_opening_) { + if (lost == nullptr || lost == handle) { + lost = handle; + return; + } + } + // Both endpoints are already waiting on a lost open; the oldest one is the least likely to still report back. + this->lost_while_opening_[0] = handle; + return; + } + ESP_LOGD(TAG, "Disconnect event for an interface with no active stream"); +} + +void USBAudioComponent::begin_close_(Endpoint endpoint) { + auto &ep = this->endpoint_(endpoint); + // Platform reads and writes see nullptr from here on and return at once. + const uac_host_device_handle_t handle = ep.handle.exchange(nullptr); + if (handle == nullptr) { + return; + } + ep.closing = handle; + ep.close_queued = false; + // `active` is kept, so the stream restarts when the device returns. + ep.platform->on_stream_state(StreamEvent::LOST); +} + +bool USBAudioComponent::finish_closes_(uint32_t now) { + bool done = true; + for (size_t i = 0; i < this->endpoints_.size(); i++) { + auto &ep = this->endpoints_[i]; + if (ep.closing == nullptr) { + continue; + } + done = false; + // Closing frees the driver's interface, so wait until no platform task can still be using the handle. + if (ep.close_queued || !ep.platform->stream_idle()) { + continue; + } + if (ep.close_attempts > 0 && static_cast(now - ep.close_retry_ms) < 0) { + // A failed close waits a moment before it is retried. + continue; + } + Op op; + op.type = Op::Type::CLOSE; + op.endpoint = static_cast(i); + op.handle = ep.closing; + op.stop = ep.stream_started; + ep.close_queued = this->queue_op_(op); + } + return done; +} + +void USBAudioComponent::on_closed_(const USBAudioEvent &event, uint32_t now) { + auto &ep = this->endpoint_(event.endpoint); + ep.close_queued = false; + if (!event.ok && ++ep.close_attempts < MAX_CLOSE_ATTEMPTS) { + // Keep the handle, so finish_closes_() retries the close. + ep.close_retry_ms = now + CLOSE_RETRY_MS; + return; + } + if (!event.ok) { + ESP_LOGE(TAG, "Giving up closing the %s interface after %u attempts", LOG_STR_ARG(endpoint_name(event.endpoint)), + ep.close_attempts); + } + ep.close_attempts = 0; + ep.closing = nullptr; + ep.stream_started = false; + ep.start_rejected = false; + ep.start_pending = false; + if (ep.connect_deferred) { + // The device came back while its old handle was closing. + ep.connect_deferred = false; + this->open_stream_(event.endpoint, ep.deferred_addr, ep.deferred_iface); + } +} + +bool USBAudioComponent::queue_op_(const Op &op) { + if (xQueueSend(this->op_queue_, &op, 0) != pdTRUE) { + ESP_LOGW(TAG, "USB audio work queue full, dropping %s request", LOG_STR_ARG(endpoint_name(op.endpoint))); + return false; + } + return true; +} + +void USBAudioComponent::post_event_(const USBAudioEvent &event) { + if (xQueueSend(this->event_queue_, &event, 0) != pdTRUE) { + ESP_LOGW(TAG, "USB audio event queue full, dropping event"); + } + this->enable_loop_soon_any_context(); +} + +void USBAudioComponent::worker_task(void *param) { + auto *self = static_cast(param); + Op op; + while (true) { + if (xQueueReceive(self->op_queue_, &op, portMAX_DELAY) != pdTRUE) { + continue; + } + self->lock_driver_(); + self->run_op_(op); + self->unlock_driver_(); + } +} + +void USBAudioComponent::lock_driver_() { + this->driver_busy_ = true; + // Reads and writes give up within their timeout; wait for those already inside the driver. + while (this->driver_users_ != 0) { + vTaskDelay(1); + } +} + +void USBAudioComponent::unlock_driver_() { this->driver_busy_ = false; } + +void USBAudioComponent::run_op_(const Op &op) { + const LogString *name = endpoint_name(op.endpoint); + USBAudioEvent result; + result.endpoint = op.endpoint; + result.handle = op.handle; + + switch (op.type) { + case Op::Type::OPEN: { + uint32_t buffer_size = this->endpoint_(op.endpoint).platform->get_buffer_size(); + uint32_t threshold = std::max(buffer_size / 4, 1U); + if (op.endpoint == Endpoint::SPEAKER) { + // The driver raises TX_DONE after taking a transfer only while the ring buffer holds at most the threshold, + // and it needs the threshold below the size. The speaker never fills past its buffer_size, so it hears about + // every transfer even when a waiting write lands right after one. + threshold = buffer_size; + buffer_size += SPEAKER_RING_SLACK; + } + uac_host_device_config_t dev_cfg = { + .addr = op.addr, + .iface_num = op.iface_num, + .buffer_size = buffer_size, + .buffer_threshold = threshold, + .callback = &USBAudioComponent::device_event_stub, + .callback_arg = this, + }; + uac_host_device_handle_t handle{nullptr}; + esp_err_t err = uac_host_device_open(&dev_cfg, &handle); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Failed to open %s interface: %s", LOG_STR_ARG(name), esp_err_to_name(err)); + } + result.type = USBAudioEvent::Type::OPENED; + result.ok = err == ESP_OK; + result.handle = handle; + result.addr = op.addr; + result.iface_num = op.iface_num; + this->post_event_(result); + return; + } + case Op::Type::START: { + result.type = USBAudioEvent::Type::STARTED; + uac_host_stream_config_t stm_cfg{}; + if (this->populate_stream_config_(op.endpoint, op.handle, stm_cfg, result.rejected)) { + esp_err_t err = uac_host_device_start(op.handle, &stm_cfg); + // Already started or active is ESP_OK; ESP_ERR_INVALID_STATE means the interface is not ready. + result.ok = err == ESP_OK; + // The host cannot allocate the endpoint's pipe, for example when its max packet size exceeds the USB FIFO. + result.rejected = err == ESP_ERR_NOT_SUPPORTED; + if (!result.ok) { + ESP_LOGW(TAG, "Failed to start %s stream: %s", LOG_STR_ARG(name), esp_err_to_name(err)); + } + } + this->post_event_(result); + return; + } + case Op::Type::RESUME: { + esp_err_t err = uac_host_device_resume(op.handle); + result.type = USBAudioEvent::Type::RESUMED; + result.seq = op.seq; + // Already started or active is ESP_OK; ESP_ERR_INVALID_STATE means the interface is not ready. + result.ok = err == ESP_OK; + if (!result.ok) { + ESP_LOGW(TAG, "Failed to resume %s stream: %s", LOG_STR_ARG(name), esp_err_to_name(err)); + } + this->post_event_(result); + return; + } + case Op::Type::SUSPEND: { + esp_err_t err = uac_host_device_suspend(op.handle); + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, "Failed to suspend %s stream: %s", LOG_STR_ARG(name), esp_err_to_name(err)); + } + return; + } + case Op::Type::CLOSE: { + if (op.stop) { + esp_err_t err = uac_host_device_stop(op.handle); + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, "Failed to stop %s stream: %s", LOG_STR_ARG(name), esp_err_to_name(err)); + } + } + esp_err_t err = uac_host_device_close(op.handle); + result.type = USBAudioEvent::Type::CLOSED; + result.ok = err == ESP_OK; + if (!result.ok) { + // The interface may still be held, so the main loop keeps the handle and retries. + ESP_LOGW(TAG, "uac_host_device_close failed: %s", esp_err_to_name(err)); + this->post_event_(result); + return; + } + if (op.endpoint == Endpoint::SPEAKER) { + // A reconnected device may support different controls. + for (ControlState *control : {&this->volume_, &this->mute_}) { + control->supported = true; + control->attempts = 0; + } + } + this->post_event_(result); + return; + } + case Op::Type::CONTROLS: { + // Cleared first, so a change made during the transfer queues another op. + this->controls_queued_ = false; + if (this->apply_pending_speaker_controls_()) { + // Give the device a moment before retrying; the queue keeps any other work in order. + this->unlock_driver_(); + vTaskDelay(pdMS_TO_TICKS(CONTROL_RETRY_MS)); + this->lock_driver_(); + this->queue_controls_(); + } + return; + } + } +} + +bool USBAudioComponent::populate_stream_config_(Endpoint endpoint, uac_host_device_handle_t handle, + uac_host_stream_config_t &stream_config, bool &rejected) { + const LogString *name = endpoint_name(endpoint); + const audio::AudioStreamInfo info = this->endpoint_(endpoint).platform->get_usb_stream_info(); + const uint8_t channels = info.get_channels(); + const uint8_t bits = info.get_bits_per_sample(); + const uint32_t sample_rate = info.get_sample_rate(); + + uac_host_dev_info_t dev_info{}; + esp_err_t err = uac_host_get_device_info(handle, &dev_info); + if (err != ESP_OK) { + ESP_LOGW(TAG, "Failed to query %s interface info: %s", LOG_STR_ARG(name), esp_err_to_name(err)); + return false; + } + + // uac_host_device_start() picks the alternate setting itself from channels, bit depth and rate: the last match with + // discrete rates, or the first match with a rate range. Find the one it will pick, so its subframe size is checked. + uint8_t selected_alt = 0; + uint8_t selected_subframe = 0; + for (uint8_t alt = 1; alt <= dev_info.iface_alt_num; alt++) { + uac_host_dev_alt_param_t alt_param{}; + if (uac_host_get_device_alt_param(handle, alt, &alt_param) != ESP_OK || alt_param.channels != channels || + alt_param.bit_resolution != bits || !alt_supports_rate(alt_param, sample_rate)) { + continue; + } + selected_alt = alt; + selected_subframe = alt_param.subframe_size; + if (alt_param.sample_freq_type == 0) { + break; + } + } + + // ESPHome's samples are packed, so a device that pads them (e.g. 24-bit samples in 4-byte subframes) cannot be + // streamed unchanged. + if (selected_alt != 0 && selected_subframe * 8 == bits) { + stream_config.channels = channels; + stream_config.bit_resolution = bits; + stream_config.sample_freq = sample_rate; + stream_config.flags = 0; + return true; + } + + rejected = true; + if (selected_alt != 0) { + ESP_LOGW(TAG, "%s interface %u would stream alt %u with %u-bit samples padded to %u bytes, which is not supported", + LOG_STR_ARG(name), dev_info.iface_num, selected_alt, bits, selected_subframe); + } + ESP_LOGW(TAG, "%s interface %u does not support requested format (channels=%u bits=%u sample_rate=%" PRIu32 " Hz)", + LOG_STR_ARG(name), dev_info.iface_num, channels, bits, sample_rate); + for (uint8_t alt = 1; alt <= dev_info.iface_alt_num; alt++) { + uac_host_dev_alt_param_t alt_param{}; + if (uac_host_get_device_alt_param(handle, alt, &alt_param) != ESP_OK) { + continue; + } + if (alt_param.sample_freq_type == 0) { + ESP_LOGI(TAG, " alt %u: channels=%u bits=%u subframe=%u bytes sample rate range=%" PRIu32 "-%" PRIu32 " Hz", alt, + alt_param.channels, alt_param.bit_resolution, alt_param.subframe_size, alt_param.sample_freq_lower, + alt_param.sample_freq_upper); + continue; + } + ESP_LOGI(TAG, " alt %u: channels=%u bits=%u subframe=%u bytes sample rates:", alt, alt_param.channels, + alt_param.bit_resolution, alt_param.subframe_size); + const uint8_t count = discrete_rate_count(alt_param); + for (uint8_t i = 0; i < count; i++) { + // Some devices list a rate more than once. + if (std::find(alt_param.sample_freq, alt_param.sample_freq + i, alt_param.sample_freq[i]) == + alt_param.sample_freq + i) { + ESP_LOGI(TAG, " %" PRIu32 " Hz", alt_param.sample_freq[i]); + } + } + } + return false; +} + +bool USBAudioComponent::apply_pending_speaker_controls_() { + const uac_host_device_handle_t handle = this->endpoint_(Endpoint::SPEAKER).handle; + if (handle == nullptr) { + // Sent once the stream starts. + return false; + } + bool retry = false; + if (this->volume_.pending.exchange(false) && this->volume_.supported) { + retry |= this->apply_control_(this->volume_, uac_host_device_set_volume(handle, this->speaker_volume_), + LOG_STR("volume")); + } + if (this->mute_.pending.exchange(false) && this->mute_.supported) { + retry |= this->apply_control_(this->mute_, uac_host_device_set_mute(handle, this->speaker_muted_), LOG_STR("mute")); + } + return retry; +} + +bool USBAudioComponent::apply_control_(ControlState &control, esp_err_t err, const LogString *name) { + if (err == ESP_OK) { + control.attempts = 0; + return false; + } + if (err == ESP_ERR_NOT_SUPPORTED) { + ESP_LOGW(TAG, "Speaker %s control not supported by device", LOG_STR_ARG(name)); + control.supported = false; + return false; + } + if (err == ESP_ERR_INVALID_STATE) { + // The stream is not started yet; starting it queues the controls again. + control.pending = true; + return false; + } + if (++control.attempts >= MAX_CONTROL_ATTEMPTS) { + ESP_LOGW(TAG, "Failed to set speaker %s: %s", LOG_STR_ARG(name), esp_err_to_name(err)); + control.attempts = 0; + return false; + } + control.pending = true; + return true; +} + +void USBAudioComponent::driver_event_stub(uint8_t addr, uint8_t iface_num, const uac_host_driver_event_t event, + void *user_data) { + auto *self = static_cast(user_data); + USBAudioEvent evt; + switch (event) { + case UAC_HOST_DRIVER_EVENT_TX_CONNECTED: + evt.endpoint = Endpoint::SPEAKER; + break; + case UAC_HOST_DRIVER_EVENT_RX_CONNECTED: + evt.endpoint = Endpoint::MICROPHONE; + break; + default: + return; + } + evt.type = USBAudioEvent::Type::CONNECTED; + evt.addr = addr; + evt.iface_num = iface_num; + self->post_event_(evt); +} + +void USBAudioComponent::device_event_stub(uac_host_device_handle_t handle, const uac_host_device_event_t event, + void *user_data) { + auto *self = static_cast(user_data); + if (event == UAC_HOST_DEVICE_EVENT_TX_DONE) { + auto &speaker = self->endpoint_(Endpoint::SPEAKER); + // Only the speaker's own interface; the handle is cleared first when the device goes away. + if (speaker.platform != nullptr && handle == speaker.handle.load()) { + speaker.platform->on_transfer_taken(); + } + } else if (event == UAC_HOST_DEVICE_EVENT_TRANSFER_ERROR) { + self->transfer_errors_.fetch_add(1, std::memory_order_relaxed); + self->enable_loop_soon_any_context(); + } else if (event == UAC_HOST_DRIVER_EVENT_DISCONNECTED) { + USBAudioEvent evt; + evt.type = USBAudioEvent::Type::DISCONNECTED; + evt.handle = handle; + self->post_event_(evt); + } +} + +} // namespace esphome::usb_audio + +#endif diff --git a/esphome/components/usb_audio/usb_audio.h b/esphome/components/usb_audio/usb_audio.h new file mode 100644 index 00000000000..a283fb18f2e --- /dev/null +++ b/esphome/components/usb_audio/usb_audio.h @@ -0,0 +1,239 @@ +#pragma once + +#include "esphome/core/defines.h" + +#ifdef USE_USB_AUDIO + +#include +#include + +#include "esphome/components/audio/audio.h" +#include "esphome/core/component.h" +#include "esphome/core/static_task.h" + +extern "C" { +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "usb/uac_host.h" +} + +namespace esphome::usb_audio { + +enum class Endpoint : uint8_t { + SPEAKER, + MICROPHONE, +}; + +/// What the hub reports to a platform about its stream. +enum class StreamEvent : uint8_t { + // The stream runs. + RUNNING, + // The device went away; the hub restarts the stream when it returns while the platform still wants it. + LOST, + // The stream failed to start; the platform goes back to stopped and its next start() retries. + FAILED, +}; + +/// A speaker or microphone platform that streams through the hub. The hub calls it on the main loop only. +class USBAudioStream { + public: + explicit USBAudioStream(uint32_t buffer_size) : buffer_size_(buffer_size) {} + + uint32_t get_buffer_size() const { return this->buffer_size_; } + /// The fixed format the hub opens the stream with. + virtual audio::AudioStreamInfo get_usb_stream_info() = 0; + virtual void on_stream_state(StreamEvent event) = 0; + /// No platform task is inside a read or write call on the device; the hub closes a lost device only then. + virtual bool stream_idle() const = 0; + /// Speaker only: the driver tried to take its next transfer from the ring buffer, which it does each time a transfer + /// completes and on each write, and takes it only when the ring buffer holds all of it. Called on the USB driver's + /// task or inside a write. + virtual void on_transfer_taken() {} + + protected: + uint32_t buffer_size_; +}; + +/// Posted to the main loop, by the USB driver task (CONNECTED, DISCONNECTED) or the hub's worker (the results). +struct USBAudioEvent { + enum class Type : uint8_t { + CONNECTED, + // The driver reports disconnects through the device callback, not the driver callback. + DISCONNECTED, + OPENED, + STARTED, + RESUMED, + CLOSED, + }; + + uac_host_device_handle_t handle{nullptr}; + uint8_t addr{0}; + uint8_t iface_num{0}; + // RESUMED: the request it answers; only the latest request's result is acted on. + uint8_t seq{0}; + Type type{Type::CONNECTED}; + Endpoint endpoint{Endpoint::SPEAKER}; + bool ok{false}; + // STARTED: the start will fail the same way until the device reconnects: the device does not support the configured + // format, or the host cannot allocate its pipe. + bool rejected{false}; +}; + +class USBAudioComponent : public Component { + public: + void setup() override; + void dump_config() override; + void loop() override; + // Just after usb_host, which installs the USB host library that uac_host_install needs. + float get_setup_priority() const override { return setup_priority::BUS - 1.0f; } + bool teardown() override; + + void register_speaker(USBAudioStream *speaker) { this->endpoint_(Endpoint::SPEAKER).platform = speaker; } + void register_microphone(USBAudioStream *microphone) { this->endpoint_(Endpoint::MICROPHONE).platform = microphone; } + + /// Start streaming as soon as the device is connected; the platform hears back through on_stream_state(). + void resume(Endpoint endpoint); + void suspend(Endpoint endpoint); + + /// Safe from any task; the worker sends the values to the device. + void set_speaker_volume_level(float volume); + void set_speaker_mute_state(bool mute_state); + + // Safe from any task. While the worker holds the driver for a control transfer, these wait for it within timeout_ms + // and return ESP_ERR_TIMEOUT if it is still busy, instead of blocking on the driver's interface lock for seconds. + esp_err_t read_microphone(uint8_t *buffer, size_t size, size_t *bytes_read, uint32_t timeout_ms); + esp_err_t write_speaker(const uint8_t *data, size_t length, uint32_t timeout_ms); + + protected: + static constexpr size_t EVENT_QUEUE_LENGTH = 16; + static constexpr size_t OP_QUEUE_LENGTH = 8; + // Extra driver ring buffer space above the speaker's buffer_size, see Op::Type::OPEN. + static constexpr uint32_t SPEAKER_RING_SLACK = 4; + + struct EndpointState { + // Read by the platform tasks and the worker; cleared first when the device goes away. + std::atomic handle{nullptr}; + // A lost device, closed by the worker once the platform reports stream_idle(). + uac_host_device_handle_t closing{nullptr}; + USBAudioStream *platform{nullptr}; + uint8_t iface{0}; + bool stream_started{false}; + // The platform asked for the stream and has not stopped it. + bool active{false}; + // Starting the connected device failed for good (see USBAudioEvent::rejected); cleared when it is closed. + bool start_rejected{false}; + // An open or start is queued on the worker; its result event clears it. + bool open_pending{false}; + bool start_pending{false}; + bool close_queued{false}; + // Counts resume() and suspend() calls, so a RESUMED that a later suspend overtook is ignored. + uint8_t request_seq{0}; + // A connect that arrived while the old handle was still opening or closing; opened once that finishes. + bool connect_deferred{false}; + uint8_t deferred_addr{0}; + uint8_t deferred_iface{0}; + // The last open failed; the next resume() retries it at this address and interface. + bool open_failed{false}; + uint8_t failed_addr{0}; + uint8_t failed_iface{0}; + // A failed close keeps `closing` and is retried after this time, a limited number of times. + uint8_t close_attempts{0}; + uint32_t close_retry_ms{0}; + }; + + struct ControlState { + std::atomic pending{false}; + // Only touched by the worker, and reset while the speaker is closed. + bool supported{true}; + uint8_t attempts{0}; + }; + + /// A blocking driver call, run on the worker. + struct Op { + enum class Type : uint8_t { + OPEN, + START, + RESUME, + SUSPEND, + CLOSE, + CONTROLS, + }; + + uac_host_device_handle_t handle{nullptr}; + uint8_t addr{0}; + uint8_t iface_num{0}; + // RESUME: copied to the result, see EndpointState::request_seq. + uint8_t seq{0}; + Type type{Type::OPEN}; + Endpoint endpoint{Endpoint::SPEAKER}; + // CLOSE: stop the stream before closing. + bool stop{false}; + }; + + EndpointState &endpoint_(Endpoint endpoint) { return this->endpoints_[static_cast(endpoint)]; } + const EndpointState &endpoint_(Endpoint endpoint) const { return this->endpoints_[static_cast(endpoint)]; } + void process_events_(uint32_t now); + void process_event_(const USBAudioEvent &event, uint32_t now); + void open_stream_(Endpoint endpoint, uint8_t addr, uint8_t iface_num); + void on_opened_(const USBAudioEvent &event); + void on_started_(const USBAudioEvent &event); + void on_resumed_(const USBAudioEvent &event); + void on_closed_(const USBAudioEvent &event, uint32_t now); + void start_stream_(Endpoint endpoint); + void stream_failed_(Endpoint endpoint); + void handle_disconnect_(uac_host_device_handle_t handle); + void begin_close_(Endpoint endpoint); + bool finish_closes_(uint32_t now); + bool queue_op_(const Op &op); + void queue_controls_(); + void post_event_(const USBAudioEvent &event); + + // Worker side: the only place that makes blocking driver calls after setup(). + static void worker_task(void *param); + void run_op_(const Op &op); + /// Keep platform reads and writes out of the driver while the worker uses it. + void lock_driver_(); + /// Platform side: wait, within timeout_ms, until the worker is not using the driver, and count the caller as a + /// driver user; timeout_ms is reduced by the wait. Returns false when the worker still holds it. + bool enter_driver_(uint32_t &timeout_ms); + void unlock_driver_(); + bool populate_stream_config_(Endpoint endpoint, uac_host_device_handle_t handle, + uac_host_stream_config_t &stream_config, bool &rejected); + /// Send pending volume and mute; returns true when one should be retried later. + bool apply_pending_speaker_controls_(); + bool apply_control_(ControlState &control, esp_err_t err, const LogString *name); + + static void driver_event_stub(uint8_t addr, uint8_t iface_num, uac_host_driver_event_t event, void *user_data); + static void device_event_stub(uac_host_device_handle_t handle, uac_host_device_event_t event, void *user_data); + + StaticTask worker_; + // Filled from the USB driver task and the worker, drained on the main loop. + QueueHandle_t event_queue_{nullptr}; + // Filled on the main loop (and by volume/mute changes from any task), drained by the worker. + QueueHandle_t op_queue_{nullptr}; + std::array endpoints_{}; + // Disconnects for devices whose open is still running on the worker, one per endpoint since a speakerphone opens + // two interfaces; each is closed once its OPENED arrives. + std::array lost_while_opening_{}; + // Counted on the USB driver task, logged from loop(). + std::atomic transfer_errors_{0}; + StaticQueue_t event_queue_struct_{}; + StaticQueue_t op_queue_struct_{}; + std::array event_queue_storage_{}; + std::array op_queue_storage_{}; + ControlState volume_{}; + ControlState mute_{}; + std::atomic speaker_volume_{100}; + std::atomic speaker_muted_{false}; + // A CONTROLS op is queued; the pending flags carry the latest values, so one is enough. + std::atomic controls_queued_{false}; + // Set while the worker runs a driver call; platform reads and writes counted in driver_users_ back off. + std::atomic driver_busy_{false}; + std::atomic driver_users_{0}; + bool uac_installed_{false}; +}; + +} // namespace esphome::usb_audio + +#endif diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 931ccf3c789..70838d897c8 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -557,6 +557,11 @@ defined(USE_ESP32_VARIANT_ESP32S31) || defined(USE_ESP32_VARIANT_ESP32H4) #define USE_SERIAL_PROXY_USB_IDENTITY #endif +// usb_host_uac supports only these variants +#if defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || \ + defined(USE_ESP32_VARIANT_ESP32S31) || defined(USE_ESP32_VARIANT_ESP32H4) +#define USE_USB_AUDIO +#endif #ifdef USE_ARDUINO #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 3, 12) diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index dd837d61632..2c12eec6d47 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -108,6 +108,11 @@ dependencies: version: "1.4.1" rules: - if: "idf_version >=6.0.0 && target in [esp32s2, esp32s3, esp32s31, esp32p4, esp32h4]" + # Keep in sync with esphome/components/usb_audio/__init__.py + espressif/usb_host_uac: + version: "1.5.0" + rules: + - if: "target in [esp32s2, esp32s3, esp32s31, esp32p4, esp32h4]" esp32async/asynctcp: version: 3.4.91 # sendspin is ESP-IDF only (it needs noise-c as an ESP-IDF component, see below) diff --git a/tests/component_tests/usb_audio/__init__.py b/tests/component_tests/usb_audio/__init__.py new file mode 100644 index 00000000000..284b0c6d298 --- /dev/null +++ b/tests/component_tests/usb_audio/__init__.py @@ -0,0 +1 @@ +"""Tests for the usb_audio platforms' config validation.""" diff --git a/tests/component_tests/usb_audio/test_validation.py b/tests/component_tests/usb_audio/test_validation.py new file mode 100644 index 00000000000..c7b0300b3a1 --- /dev/null +++ b/tests/component_tests/usb_audio/test_validation.py @@ -0,0 +1,132 @@ +"""Tests for the usb_audio buffer floors, the one-platform-per-hub check and the USB FIFO requests.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components import usb_audio, usb_host +from esphome.components.usb_audio import ( + CONF_USB_AUDIO_ID, + microphone as usb_microphone, + speaker as usb_speaker, +) +from esphome.config import Config +from esphome.const import ( + CONF_BITS_PER_SAMPLE, + CONF_BUFFER_SIZE, + CONF_NUM_CHANNELS, + CONF_PLATFORM, + CONF_SAMPLE_RATE, + PlatformFramework, +) +from esphome.core import CORE, ID +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _stream(sample_rate: int, channels: int, bits: int, buffer_size: int) -> ConfigType: + return { + CONF_SAMPLE_RATE: sample_rate, + CONF_NUM_CHANNELS: channels, + CONF_BITS_PER_SAMPLE: float(bits), + CONF_BUFFER_SIZE: buffer_size, + } + + +def _entry(hub: str, platform: str = "usb_audio") -> ConfigType: + return {CONF_PLATFORM: platform, CONF_USB_AUDIO_ID: ID(hub)} + + +def _set_full_config( + set_core_config: SetCoreConfigCallable, **domains: list[ConfigType] +) -> None: + full = Config() + full.update(domains) + set_core_config(PlatformFramework.ESP32_IDF, full_config=full) + + +def test_speaker_buffer_floor() -> None: + # Two transfers of 3 packets x 48 frames x 4 bytes at 48 kHz 16-bit stereo. + usb_speaker.validate_buffer_size(_stream(48000, 2, 16, 1152)) + with pytest.raises(cv.Invalid, match="at least 1152 bytes"): + usb_speaker.validate_buffer_size(_stream(48000, 2, 16, 1151)) + + +def test_speaker_floor_rounds_up_fractional_rates() -> None: + # 44.1 kHz packets carry up to 45 frames. + with pytest.raises(cv.Invalid, match="at least 1080 bytes"): + usb_speaker.validate_buffer_size(_stream(44100, 2, 16, 1079)) + + +def test_speaker_default_buffer_meets_floor() -> None: + usb_speaker.validate_buffer_size(_stream(96000, 2, 32, 6400)) + + +def test_microphone_buffer_floor() -> None: + # 16 kHz 16-bit mono is 32000 bytes/s, so two read periods (40 ms) are 1280 bytes. + usb_microphone.validate_buffer_size(_stream(16000, 1, 16, 1280)) + with pytest.raises(cv.Invalid, match="at least 1280 bytes"): + usb_microphone.validate_buffer_size(_stream(16000, 1, 16, 1279)) + + +def _unset_buffer(sample_rate: int, channels: int, bits: int) -> ConfigType: + config = _stream(sample_rate, channels, bits, 0) + del config[CONF_BUFFER_SIZE] + return config + + +def test_default_buffer_size() -> None: + assert ( + usb_speaker.validate_buffer_size(_unset_buffer(48000, 2, 16))[CONF_BUFFER_SIZE] + == 6400 + ) + assert ( + usb_microphone.validate_buffer_size(_unset_buffer(16000, 1, 16))[ + CONF_BUFFER_SIZE + ] + == 6400 + ) + + +def test_default_buffer_size_grows_to_the_floor() -> None: + # 48 kHz 16-bit stereo is 192000 bytes/s, so 40 ms need 7680 bytes, more than the 6400 default. + assert ( + usb_microphone.validate_buffer_size(_unset_buffer(48000, 2, 16))[ + CONF_BUFFER_SIZE + ] + == 7680 + ) + + +def test_one_speaker_per_hub_accepted(set_core_config: SetCoreConfigCallable) -> None: + entries = [_entry("hub_a"), _entry("hub_b"), _entry("hub_a", "i2s_audio")] + _set_full_config(set_core_config, speaker=entries) + usb_speaker.FINAL_VALIDATE_SCHEMA(entries[0]) + usb_speaker.FINAL_VALIDATE_SCHEMA(entries[1]) + + +def test_two_speakers_on_one_hub_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + entries = [_entry("hub_a"), _entry("hub_a")] + _set_full_config(set_core_config, speaker=entries) + with pytest.raises(cv.Invalid, match="Only one usb_audio speaker"): + usb_speaker.FINAL_VALIDATE_SCHEMA(entries[0]) + + +def test_two_microphones_on_one_hub_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + entries = [_entry("hub_a"), _entry("hub_a")] + _set_full_config(set_core_config, microphone=entries, speaker=[_entry("hub_a")]) + with pytest.raises(cv.Invalid, match="Only one usb_audio microphone"): + usb_microphone.FINAL_VALIDATE_SCHEMA(entries[0]) + + +def test_platforms_request_usb_fifo_space( + set_core_config: SetCoreConfigCallable, +) -> None: + # 48 kHz 16-bit stereo packets carry up to 49 frames of 4 bytes; 16 kHz 16-bit mono up to 17 frames of 2 bytes. + set_core_config(PlatformFramework.ESP32_IDF) + usb_audio.request_speaker_fifo(_stream(48000, 2, 16, 6400)) + usb_audio.request_microphone_fifo(_stream(16000, 1, 16, 6400)) + assert CORE.data[usb_host.DOMAIN][usb_host.KEY_PERIODIC_FIFO] == [34, 196] diff --git a/tests/components/usb_audio/common.yaml b/tests/components/usb_audio/common.yaml new file mode 100644 index 00000000000..7e60ca0da02 --- /dev/null +++ b/tests/components/usb_audio/common.yaml @@ -0,0 +1,21 @@ +usb_audio: + id: usb_audio_id + +microphone: + - platform: usb_audio + id: usb_mic + usb_audio_id: usb_audio_id + sample_rate: 48000 + bits_per_sample: 16bit + num_channels: 2 + buffer_size: 9600 + +speaker: + - platform: usb_audio + id: usb_speaker + usb_audio_id: usb_audio_id + sample_rate: 48000 + bits_per_sample: 16bit + num_channels: 2 + buffer_size: 6400 + write_timeout: 20ms diff --git a/tests/components/usb_audio/test.esp32-p4-idf.yaml b/tests/components/usb_audio/test.esp32-p4-idf.yaml new file mode 100644 index 00000000000..53588849d1a --- /dev/null +++ b/tests/components/usb_audio/test.esp32-p4-idf.yaml @@ -0,0 +1,2 @@ +packages: + usb_audio: !include common.yaml diff --git a/tests/components/usb_audio/test.esp32-s3-idf.yaml b/tests/components/usb_audio/test.esp32-s3-idf.yaml new file mode 100644 index 00000000000..53588849d1a --- /dev/null +++ b/tests/components/usb_audio/test.esp32-s3-idf.yaml @@ -0,0 +1,2 @@ +packages: + usb_audio: !include common.yaml