diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index cb57db9ce8..5a8f139e1e 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1329,8 +1329,7 @@ void APIConnection::on_voice_assistant_announce_request(const VoiceAssistantAnno } } -bool APIConnection::send_voice_assistant_get_configuration_response_( - const VoiceAssistantConfigurationRequest & /*msg*/) { +bool APIConnection::send_voice_assistant_get_configuration_response_(const VoiceAssistantConfigurationRequest &msg) { VoiceAssistantConfigurationResponse resp; if (!this->check_voice_assistant_api_connection_()) { // send_message encodes synchronously, so this stack local outlives the encode @@ -1339,7 +1338,9 @@ bool APIConnection::send_voice_assistant_get_configuration_response_( return this->send_message(resp); } - auto &config = voice_assistant::global_voice_assistant->get_configuration(); + // VoiceAssistant::get_configuration merges compiled models with cached external wake words (deduped by id), + // so the response is built entirely from config.available_wake_words. + auto &config = voice_assistant::global_voice_assistant->get_configuration(msg.external_wake_words); for (auto &wake_word : config.available_wake_words) { resp.available_wake_words.emplace_back(); auto &resp_wake_word = resp.available_wake_words.back(); diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py index f41adfd8de..a5335cc2fb 100644 --- a/esphome/components/voice_assistant/__init__.py +++ b/esphome/components/voice_assistant/__init__.py @@ -2,6 +2,7 @@ from esphome import automation from esphome.automation import register_action, register_condition import esphome.codegen as cg from esphome.components import media_player, micro_wake_word, microphone, speaker +from esphome.components.http_request import CONF_HTTP_REQUEST_ID, HttpRequestComponent import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -14,8 +15,18 @@ from esphome.const import ( CONF_ON_START, CONF_SPEAKER, ) +from esphome.types import ConfigType + + +def AUTO_LOAD(config: ConfigType) -> list[str]: + """Auto-load the components needed to download runtime wake word models over HTTP.""" + base = ["audio", "ring_buffer", "socket"] + # Runtime model loading verifies downloads (sha256) and parses manifests (json). + if config and CONF_HTTP_REQUEST_ID in config: + return base + ["sha256", "json"] + return base + -AUTO_LOAD = ["audio", "ring_buffer", "socket"] DEPENDENCIES = ["api", "microphone"] CODEOWNERS = ["@jesserockz", "@kahrendt"] @@ -88,6 +99,15 @@ def tts_stream_validate(config): return config +def _runtime_model_validate(config): + # Downloading wake word models is only useful alongside micro_wake_word, which runs them. + if CONF_HTTP_REQUEST_ID in config and CONF_MICRO_WAKE_WORD not in config: + raise cv.Invalid( + f"'{CONF_HTTP_REQUEST_ID}' requires '{CONF_MICRO_WAKE_WORD}' to be configured on the voice assistant" + ) + return config + + CONFIG_SCHEMA = cv.All( cv.Schema( { @@ -113,6 +133,7 @@ CONFIG_SCHEMA = cv.All( cv.Exclusive(CONF_SPEAKER, "output"): cv.use_id(speaker.Speaker), cv.Optional(CONF_USE_WAKE_WORD, default=False): cv.boolean, cv.Optional(CONF_MICRO_WAKE_WORD): cv.use_id(micro_wake_word.MicroWakeWord), + cv.Optional(CONF_HTTP_REQUEST_ID): cv.use_id(HttpRequestComponent), cv.Optional(CONF_VAD_THRESHOLD): cv.invalid( "VAD threshold is no longer supported, as it requires the deprecated esp_adf external component. Use an i2s_audio microphone/speaker instead. Additionally, you may need to configure the audio_adc and audio_dac components depending on your hardware." ), @@ -183,6 +204,7 @@ CONFIG_SCHEMA = cv.All( } ).extend(cv.COMPONENT_SCHEMA), tts_stream_validate, + _runtime_model_validate, ) FINAL_VALIDATE_SCHEMA = cv.All( @@ -215,6 +237,13 @@ async def to_code(config): mww = await cg.get_variable(config[CONF_MICRO_WAKE_WORD]) cg.add(var.set_micro_wake_word(mww)) + if (http_request_id := config.get(CONF_HTTP_REQUEST_ID)) is not None: + http_req = await cg.get_variable(http_request_id) + cg.add(var.set_http_request(http_req)) + # sha256's and json's own to_code emit USE_SHA256/USE_JSON and their build flags; they are + # pulled in via AUTO_LOAD above. + cg.add_define("USE_VOICE_ASSISTANT_RUNTIME_MODEL") + if CONF_MEDIA_PLAYER in config: mp = await cg.get_variable(config[CONF_MEDIA_PLAYER]) cg.add(var.set_media_player(mp)) diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 76ae145b16..56e8dc38ab 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -6,10 +6,20 @@ #include "esphome/components/socket/socket.h" #include "esphome/core/application.h" #include "esphome/core/log.h" +#include "esphome/core/preferences.h" #include #include +#if defined(USE_MICRO_WAKE_WORD) && defined(USE_VOICE_ASSISTANT_RUNTIME_MODEL) +#include "esphome/components/micro_wake_word/model_data.h" +#include "esphome/components/json/json_util.h" +#include "esphome/components/sha256/sha256.h" + +#include +#include +#endif + namespace esphome::voice_assistant { static const char *const TAG = "voice_assistant"; @@ -1086,25 +1096,63 @@ void VoiceAssistant::on_announce(const api::VoiceAssistantAnnounceRequest &msg) void VoiceAssistant::on_set_configuration(const std::vector &active_wake_words) { #ifdef USE_MICRO_WAKE_WORD if (this->micro_wake_word_) { - // Disable all wake words first + // Disable every wake word first. disable() persists the state for runtime models via the unified pref path. for (auto &model : this->micro_wake_word_->get_wake_words()) { model->disable(); } - // Enable only active wake words - for (const auto &ww_id : active_wake_words) { - for (auto &model : this->micro_wake_word_->get_wake_words()) { - if (model->get_id() == ww_id) { - model->enable(); - ESP_LOGD(TAG, "Enabled wake word: %s (id=%s)", model->get_wake_word().c_str(), model->get_id().c_str()); - } +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL + // Reset the optimistic pending list -- it tracks the most recent request only. + this->pending_active_wake_words_.clear(); + + // Evict runtime (downloaded) models that are no longer active, freeing their PSRAM buffer immediately + // rather than leaving it resident behind a merely-disabled model. Without this, trialing many advertised + // wake words accumulates one PSRAM buffer per model tried (HA keeps advertising them, so the stale-model + // cleanup never fires), which can exhaust PSRAM and make unrelated allocations (audio playback, TLS, etc.) + // fail. Compiled-in models are only disabled (above); switching back to an evicted model re-downloads it. + // get_runtime_model_ids() returns a copy, so removing while iterating is safe. + for (const auto &id : this->micro_wake_word_->get_runtime_model_ids()) { + if (std::find(active_wake_words.begin(), active_wake_words.end(), id) == active_wake_words.end()) { + this->micro_wake_word_->remove_runtime_model(id); } } +#endif + + // Enable the requested wake words. + for (const auto &ww_id : active_wake_words) { + // Already loaded (compiled or previously downloaded)? enable() persists the state. + if (auto *model = this->micro_wake_word_->get_model_by_id(ww_id)) { + model->enable(); + ESP_LOGD(TAG, "Enabled wake word: %s (id=%s)", model->get_wake_word().c_str(), model->get_id().c_str()); + continue; + } + +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL + // Not loaded -- must be an external model we can download. + CachedExternalWakeWord *cached = this->find_cached_wake_word_(ww_id); + if (cached == nullptr) { + ESP_LOGE(TAG, "Unknown wake word ID: %s", ww_id.c_str()); + continue; + } + if (!this->is_wake_word_pending_(ww_id)) { + ESP_LOGD(TAG, "Queuing download for wake word %s", ww_id.c_str()); + this->model_download_queue_.push_back(*cached); // copy -- the task holds its own entries + this->pending_active_wake_words_.push_back(ww_id); // report active until the load resolves + } +#else + ESP_LOGE(TAG, "Unknown wake word ID: %s (runtime model loading not enabled)", ww_id.c_str()); +#endif + } + +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL + this->try_start_model_load_task_(); +#endif } #endif }; -const Configuration &VoiceAssistant::get_configuration() { +const Configuration &VoiceAssistant::get_configuration( + const std::vector &external_wake_words) { this->config_.available_wake_words.clear(); this->config_.active_wake_words.clear(); @@ -1112,6 +1160,18 @@ const Configuration &VoiceAssistant::get_configuration() { if (this->micro_wake_word_) { this->config_.max_active_wake_words = 1; +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL + // Rebuild the external wake word cache from this request (drops entries HA no longer advertises). + this->cache_external_wake_words_(external_wake_words); + + // Unload runtime models whose wake word is no longer advertised, before the loops below would list them. + this->remove_stale_runtime_models_(); + + // Re-download any models the user had enabled before a reboot; they become active asynchronously. + this->restore_runtime_models_(); +#endif + + // Add built-in wake words (already loaded models) for (auto &model : this->micro_wake_word_->get_wake_words()) { if (model->is_enabled()) { this->config_.active_wake_words.push_back(model->get_id()); @@ -1125,6 +1185,31 @@ const Configuration &VoiceAssistant::get_configuration() { } this->config_.available_wake_words.push_back(std::move(wake_word)); } + +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL + // Advertise cached external wake words that aren't already loaded (loaded ones are listed above). + for (const auto &cached_ww : this->external_wake_words_cache_) { + if (this->micro_wake_word_->get_model_by_id(cached_ww.id) != nullptr) { + continue; + } + + WakeWord wake_word; + wake_word.id = cached_ww.id; + wake_word.wake_word = cached_ww.wake_word; + for (const auto &lang : cached_ww.trained_languages) { + wake_word.trained_languages.push_back(lang); + } + this->config_.available_wake_words.push_back(std::move(wake_word)); + } + + // Optimistically include pending (queued but not yet loaded) wake words as active. + for (const auto &pending_id : this->pending_active_wake_words_) { + if (std::find(this->config_.active_wake_words.begin(), this->config_.active_wake_words.end(), pending_id) == + this->config_.active_wake_words.end()) { + this->config_.active_wake_words.push_back(pending_id); + } + } +#endif } else { #endif // No microWakeWord @@ -1136,6 +1221,383 @@ const Configuration &VoiceAssistant::get_configuration() { return this->config_; }; +#if defined(USE_MICRO_WAKE_WORD) && defined(USE_VOICE_ASSISTANT_RUNTIME_MODEL) +namespace { + +// Background download task stack. TLS handshakes are stack-hungry; measure with uxTaskGetStackHighWaterMark +// against an HTTPS manifest during hardware testing and raise to 12288 if it runs close. +constexpr uint32_t MODEL_LOAD_TASK_STACK_SIZE = 8192; +// Manifests are small JSON documents; reject anything implausibly large before allocating for it. +constexpr size_t MAX_MANIFEST_SIZE = 8192; +// Chunk size for streaming an HTTP body into its destination buffer. +constexpr size_t MODEL_DOWNLOAD_CHUNK_SIZE = 1024; +// Sanity bounds for the model parameters declared in the manifest. +constexpr size_t MAX_SLIDING_WINDOW_SIZE = 50; +constexpr size_t MAX_TENSOR_ARENA_SIZE = 1024 * 1024; + +// Verifies a buffer against an expected hex-encoded SHA256. A free function (not a method) so it can never +// read VoiceAssistant state, and so the hasher stays within a single stack frame as the hardware-accelerated +// SHA path requires. +bool verify_model_sha256(const uint8_t *data, size_t size, const std::string &expected_hex) { + sha256::SHA256 hasher; + hasher.init(); + hasher.add(data, size); + hasher.calculate(); + if (hasher.equals_hex(expected_hex.c_str())) { + return true; + } + char actual_hex[65]; + hasher.get_hex(actual_hex); + ESP_LOGE(TAG, "Model hash mismatch: expected %s, got %s", expected_hex.c_str(), actual_hex); + return false; +} + +} // namespace + +void VoiceAssistant::cache_external_wake_words_(const std::vector &wake_words) { + // Rebuild from scratch so entries HA no longer advertises drop out. In-flight downloads are unaffected: the + // load task holds its own copies of the entries it is working on. + this->external_wake_words_cache_.clear(); + for (const auto &ww : wake_words) { + if (ww.model_type != "micro") { + continue; // microWakeWord only + } + // Copy every StringRef into an owning string; the proto StringRefs point into the receive buffer and + // dangle once this handler returns. + CachedExternalWakeWord entry; + entry.id = ww.id.str(); + entry.wake_word = ww.wake_word.str(); + entry.trained_languages = ww.trained_languages; + entry.model_type = ww.model_type.str(); + entry.model_size = ww.model_size; + entry.model_hash = ww.model_hash.str(); + entry.url = ww.url.str(); + ESP_LOGD(TAG, "Cached external wake word: %s (manifest: %s)", entry.id.c_str(), entry.url.c_str()); + this->external_wake_words_cache_.push_back(std::move(entry)); + } +} + +void VoiceAssistant::remove_stale_runtime_models_() { + // Runtime models whose wake word HA no longer advertises are unloaded entirely, freeing the interpreter, + // arenas, and the PSRAM model buffer. The enabled preference is deliberately left alone: if HA ever + // advertises the wake word again, restore_runtime_models_ re-downloads it in the state the user left it. + for (const auto &id : this->micro_wake_word_->get_runtime_model_ids()) { + if (this->find_cached_wake_word_(id) == nullptr) { + this->micro_wake_word_->remove_runtime_model(id); + } + } +} + +void VoiceAssistant::restore_runtime_models_() { + for (const auto &cached_ww : this->external_wake_words_cache_) { + // Skip anything already loaded or already queued/downloading. + if (this->micro_wake_word_->get_model_by_id(cached_ww.id) != nullptr || this->is_wake_word_pending_(cached_ww.id)) { + continue; + } + + // Only re-download models the user had enabled before the reboot. + auto pref = global_preferences->make_preference(fnv1_hash(cached_ww.id)); + bool enabled = false; + if (pref.load(&enabled) && enabled) { + ESP_LOGD(TAG, "Restoring runtime model %s", cached_ww.id.c_str()); + this->model_download_queue_.push_back(cached_ww); + this->pending_active_wake_words_.push_back(cached_ww.id); + } + } + + this->try_start_model_load_task_(); +} + +CachedExternalWakeWord *VoiceAssistant::find_cached_wake_word_(const std::string &id) { + for (auto &entry : this->external_wake_words_cache_) { + if (entry.id == id) { + return &entry; + } + } + return nullptr; +} + +bool VoiceAssistant::is_wake_word_pending_(const std::string &id) const { + return std::find(this->pending_active_wake_words_.begin(), this->pending_active_wake_words_.end(), id) != + this->pending_active_wake_words_.end(); +} + +void VoiceAssistant::erase_pending_wake_word_(const std::string &id) { + auto it = std::find(this->pending_active_wake_words_.begin(), this->pending_active_wake_words_.end(), id); + if (it != this->pending_active_wake_words_.end()) { + this->pending_active_wake_words_.erase(it); + } +} + +void VoiceAssistant::mark_model_load_failed_(const std::string &id) { + // Persist disabled so a broken model isn't retried on every boot, and drop the optimistic active entry so + // HA sees the real (inactive) state. + auto pref = global_preferences->make_preference(fnv1_hash(id)); + bool enabled = false; + pref.save(&enabled); + this->erase_pending_wake_word_(id); +} + +void VoiceAssistant::try_start_model_load_task_() { + if (this->http_request_ == nullptr || this->micro_wake_word_ == nullptr) { + return; // Required components not configured + } + // One task at a time; nothing to do if it is already running or there is no queued work. + if (this->model_load_task_handle_ != nullptr || this->model_download_queue_.empty()) { + return; + } + + // Drop queued entries that are already loaded (a repeated activation raced with an in-flight download). + auto &queue = this->model_download_queue_; + queue.erase(std::remove_if(queue.begin(), queue.end(), + [this](const CachedExternalWakeWord &ww) { + return this->micro_wake_word_->get_model_by_id(ww.id) != nullptr; + }), + queue.end()); + if (queue.empty()) { + return; + } + + auto *params = new ModelLoadTaskParams{this, std::move(this->model_download_queue_), + this->micro_wake_word_->get_features_step_size()}; + this->model_download_queue_.clear(); // moved-from vector: make it definitively empty + + BaseType_t result = xTaskCreate(VoiceAssistant::model_load_task, "model_load", MODEL_LOAD_TASK_STACK_SIZE, params, 1, + &this->model_load_task_handle_); + + if (result != pdPASS || this->model_load_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Failed to create model load task"); + this->model_load_task_handle_ = nullptr; + // We're on the main loop: do the failure bookkeeping inline for every queued model. + for (const auto &cached_ww : params->models_to_load) { + this->mark_model_load_failed_(cached_ww.id); + } + delete params; + return; + } +} + +void VoiceAssistant::model_load_task(void *params) { + ModelLoadTaskParams *task_params = static_cast(params); + VoiceAssistant *this_va = task_params->voice_assistant; + const uint8_t features_step_size = task_params->features_step_size; + + ESP_LOGD(TAG, "Model load task started for %zu model(s)", task_params->models_to_load.size()); + + for (const auto &cached_ww : task_params->models_to_load) { + // Copy everything the handoffs need out of cached_ww up front. cached_ww is owned by task_params (freed + // when this task exits), so no deferred lambda may capture it by reference. + const std::string id = cached_ww.id; + ESP_LOGD(TAG, "Processing model: %s", id.c_str()); + + // Shared failure step: persist disabled and drop the optimistic active entry, both on the main loop. + auto fail = [this_va, id]() { this_va->defer([this_va, id]() { this_va->mark_model_load_failed_(id); }); }; + + // 1. Download the manifest. + auto manifest_container = this_va->http_request_->get(cached_ww.url); + if (!manifest_container || manifest_container->status_code != 200) { + ESP_LOGW(TAG, "Failed to download manifest for %s from %s", id.c_str(), cached_ww.url.c_str()); + if (manifest_container) { + manifest_container->end(); + } + fail(); + continue; + } + size_t manifest_size = manifest_container->content_length; + if (manifest_size == 0 || manifest_size > MAX_MANIFEST_SIZE) { + ESP_LOGW(TAG, "Manifest for %s has an invalid content length %zu", id.c_str(), manifest_size); + manifest_container->end(); + fail(); + continue; + } + std::string manifest_str; + manifest_str.resize(manifest_size); + auto manifest_read = + http_request::http_read_fully(manifest_container.get(), reinterpret_cast(manifest_str.data()), + manifest_size, MODEL_DOWNLOAD_CHUNK_SIZE, this_va->http_request_->get_timeout()); + size_t manifest_bytes = manifest_container->get_bytes_read(); + manifest_container->end(); + if (manifest_read.status != http_request::HttpReadStatus::OK) { + ESP_LOGW(TAG, "Failed to read manifest for %s", id.c_str()); + fail(); + continue; + } + manifest_str.resize(manifest_bytes); // trim to what actually arrived + + // 2. Parse the manifest. + std::string model_url; + std::string wake_word; + float probability_cutoff = 0.0f; + uint32_t sliding_window_size = 0; + uint32_t tensor_arena_size = 0; + int manifest_feature_step_size = -1; + bool parse_success = json::parse_json(manifest_str, [&](JsonObject root) -> bool { + if (!root["model"].is() || !root["wake_word"].is() || + !root["micro"].is()) { + ESP_LOGE(TAG, "Manifest does not contain required fields"); + return false; + } + model_url = root["model"].as(); + wake_word = root["wake_word"].as(); + + JsonObject micro = root["micro"]; + if (!micro["probability_cutoff"].is() || !micro["sliding_window_size"].is() || + !micro["tensor_arena_size"].is() || !micro["feature_step_size"].is()) { + ESP_LOGE(TAG, "Manifest micro section does not contain required fields"); + return false; + } + probability_cutoff = micro["probability_cutoff"]; + sliding_window_size = micro["sliding_window_size"]; + tensor_arena_size = micro["tensor_arena_size"]; + manifest_feature_step_size = micro["feature_step_size"]; + return true; + }); + if (!parse_success) { + ESP_LOGW(TAG, "Failed to parse manifest JSON for %s", id.c_str()); + fail(); + continue; + } + + // A model trained with a different feature step size would silently produce garbage inferences. + if (manifest_feature_step_size != static_cast(features_step_size)) { + ESP_LOGE(TAG, "Model %s feature step size %d does not match device's %u; rejecting", id.c_str(), + manifest_feature_step_size, features_step_size); + fail(); + continue; + } + + // Validate model hyper-parameters for sanity: a probability cutoff outside [0, 1] overflows the + // uint8_t quantization, a zero sliding window divides by zero when averaging, and an + // out-of-range arena could fail to allocate memory. + if (probability_cutoff < 0.0f || probability_cutoff > 1.0f || sliding_window_size == 0 || + sliding_window_size > MAX_SLIDING_WINDOW_SIZE || tensor_arena_size == 0 || + tensor_arena_size > MAX_TENSOR_ARENA_SIZE) { + ESP_LOGE(TAG, "Model %s has out-of-range parameters (cutoff %.3f, window %" PRIu32 ", arena %" PRIu32 ")", + id.c_str(), probability_cutoff, sliding_window_size, tensor_arena_size); + fail(); + continue; + } + + // 3. Resolve a relative "model" filename against the manifest URL; absolute URLs are used as-is. + if (model_url.compare(0, 7, "http://") != 0 && model_url.compare(0, 8, "https://") != 0) { + size_t slash_pos = cached_ww.url.find_last_of('/'); + if (slash_pos != std::string::npos) { + model_url = cached_ww.url.substr(0, slash_pos + 1) + model_url; + } + } + ESP_LOGD(TAG, "Resolved model URL for %s: %s", id.c_str(), model_url.c_str()); + + // 4. Download the model. + auto container = this_va->http_request_->get(model_url); + if (!container || container->status_code != 200) { + ESP_LOGW(TAG, "Failed to connect to model URL for %s", id.c_str()); + if (container) { + container->end(); + } + fail(); + continue; + } + size_t model_size = container->content_length; + if (model_size == 0) { + ESP_LOGW(TAG, "Model %s reported a zero-length body", id.c_str()); + container->end(); + fail(); + continue; + } + if (cached_ww.model_size != 0 && cached_ww.model_size != model_size) { + ESP_LOGW(TAG, "Model %s size %zu disagrees with the advertised %" PRIu32 " (SHA256 is authoritative)", id.c_str(), + model_size, cached_ww.model_size); + } + auto model_data = std::make_shared(); + if (!model_data->allocate(model_size)) { + ESP_LOGW(TAG, "Failed to allocate %zu bytes for model %s", model_size, id.c_str()); + container->end(); + fail(); + continue; + } + auto model_read = http_request::http_read_fully(container.get(), model_data->get_write_pointer(), model_size, + MODEL_DOWNLOAD_CHUNK_SIZE, this_va->http_request_->get_timeout()); + size_t model_bytes = container->get_bytes_read(); + container->end(); + if (model_read.status != http_request::HttpReadStatus::OK || model_bytes != model_size) { + ESP_LOGW(TAG, "Failed to read model %s (%zu of %zu bytes)", id.c_str(), model_bytes, model_size); + fail(); + continue; + } + + // 5. Verify the SHA256 (static helper -- never touches VA state, keeps the hasher in one stack frame). + if (!verify_model_sha256(model_data->get_write_pointer(), model_size, cached_ww.model_hash)) { + ESP_LOGW(TAG, "SHA256 validation failed for model %s", id.c_str()); + fail(); + continue; + } + + // 6. Validate the TFLite header. + if (!model_data->validate_and_mark_ready()) { + ESP_LOGW(TAG, "TFLite validation failed for model %s", id.c_str()); + fail(); + continue; + } + + // 7. Hand off to the main loop. Every capture is by value (strings, POD, the shared_ptr). + ESP_LOGI(TAG, "Loaded model %s (%zu bytes); handing off to micro_wake_word", id.c_str(), model_size); + const std::string wake_word_copy = wake_word; + const std::vector trained_languages = cached_ww.trained_languages; + const uint8_t quantized_cutoff = static_cast(probability_cutoff * 255); + const size_t window = sliding_window_size; + const size_t arena = tensor_arena_size; + this_va->defer([this_va, id, wake_word_copy, trained_languages, model_data, quantized_cutoff, window, arena]() { + // The world may have changed while the download was in flight; re-check against current main-loop state. + if (this_va->find_cached_wake_word_(id) == nullptr) { + // HA stopped advertising this wake word: discard the download. The model buffer is freed when the + // last shared_ptr reference (this lambda's capture) drops. + ESP_LOGW(TAG, "Discarding downloaded model %s: no longer advertised", id.c_str()); + this_va->erase_pending_wake_word_(id); + return; + } + // A set_configuration while the download was in flight may have withdrawn the activation request. + const bool still_wanted = this_va->is_wake_word_pending_(id); + + auto model = make_unique(id, model_data, quantized_cutoff, window, wake_word_copy, + trained_languages, arena); + auto *raw = model.get(); + if (!this_va->micro_wake_word_->add_runtime_model(std::move(model))) { + if (this_va->micro_wake_word_->get_model_by_id(id) != nullptr) { + // A duplicate download slipped through (an earlier pass already added the model, e.g. a config + // change re-queued it while it was in flight). Benign: leave the existing model enabled. + this_va->erase_pending_wake_word_(id); + } else { + // Genuine failure (e.g. the pause handshake timed out). Don't retry it on every boot. + this_va->mark_model_load_failed_(id); + } + return; + } + if (still_wanted) { + raw->enable(); // persists pref = true via the unified path + ESP_LOGI(TAG, "Enabled runtime model %s", id.c_str()); + } else { + // Deactivated while downloading: keep the model loaded for instant re-enable, but leave it off. + raw->disable(); // persists pref = false + ESP_LOGI(TAG, "Loaded runtime model %s (left disabled: activation was withdrawn)", id.c_str()); + } + this_va->erase_pending_wake_word_(id); + }); + } + + ESP_LOGD(TAG, "Model load task completed"); + + // Drain anything queued while the task was busy, then let the next request start a fresh task. Deleting + // task_params before these deferred lambdas run is safe because every capture above is by value. + this_va->defer([this_va]() { + this_va->model_load_task_handle_ = nullptr; + this_va->try_start_model_load_task_(); + }); + + delete task_params; + vTaskDelete(nullptr); +} +#endif // USE_MICRO_WAKE_WORD && USE_VOICE_ASSISTANT_RUNTIME_MODEL + VoiceAssistant *global_voice_assistant = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) } // namespace esphome::voice_assistant diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index d46b089c2e..a861283cf9 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -18,6 +18,12 @@ #endif #ifdef USE_MICRO_WAKE_WORD #include "esphome/components/micro_wake_word/micro_wake_word.h" +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL +// micro_wake_word is ESP32-only, so USE_VOICE_ASSISTANT_RUNTIME_MODEL always implies ESP32/FreeRTOS. +#include "esphome/components/http_request/http_request.h" +#include +#include +#endif #endif #ifdef USE_SPEAKER #include "esphome/components/speaker/speaker.h" @@ -25,6 +31,7 @@ #include "esphome/components/socket/socket.h" #include +#include #include namespace esphome::voice_assistant { @@ -42,6 +49,7 @@ enum VoiceAssistantFeature : uint32_t { FEATURE_ANNOUNCE = 1 << 4, FEATURE_START_CONVERSATION = 1 << 5, FEATURE_MULTI_CHANNEL_AUDIO = 1 << 6, + FEATURE_EXTERNAL_WAKE_WORDS = 1 << 7, }; enum class State { @@ -104,6 +112,30 @@ enum class MediaPlayerResponseState { }; #endif +#if defined(USE_MICRO_WAKE_WORD) && defined(USE_VOICE_ASSISTANT_RUNTIME_MODEL) +class VoiceAssistant; + +// Owning copy of VoiceAssistantExternalWakeWord. The protobuf message uses StringRef +// which points into the receive buffer and becomes dangling once the API handler returns, +// so any data that needs to outlive the handler (cache, async task) is copied into this struct. +struct CachedExternalWakeWord { + std::string id; + std::string wake_word; + std::vector trained_languages; + std::string model_type; + uint32_t model_size{0}; + std::string model_hash; + std::string url; +}; + +struct ModelLoadTaskParams { + VoiceAssistant *voice_assistant; + std::vector models_to_load; + // Captured at launch so the task never has to call back into micro_wake_word to validate manifests. + uint8_t features_step_size; +}; +#endif + class VoiceAssistant final : public Component { public: VoiceAssistant(); @@ -119,6 +151,9 @@ class VoiceAssistant final : public Component { void set_microphone_source2(microphone::MicrophoneSource *mic_source2) { this->mic_source2_ = mic_source2; } #ifdef USE_MICRO_WAKE_WORD void set_micro_wake_word(micro_wake_word::MicroWakeWord *mww) { this->micro_wake_word_ = mww; } +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL + void set_http_request(http_request::HttpRequestComponent *http_request) { this->http_request_ = http_request; } +#endif #endif #ifdef USE_SPEAKER void set_speaker(speaker::Speaker *speaker) { @@ -166,6 +201,11 @@ class VoiceAssistant final : public Component { } #endif +#if defined(USE_MICRO_WAKE_WORD) && defined(USE_VOICE_ASSISTANT_RUNTIME_MODEL) + // Indicate support for external wake word models that can be downloaded at runtime + flags |= VoiceAssistantFeature::FEATURE_EXTERNAL_WAKE_WORDS; +#endif + return flags; } @@ -177,7 +217,7 @@ class VoiceAssistant final : public Component { void on_timer_event(const api::VoiceAssistantTimerEventResponse &msg); void on_announce(const api::VoiceAssistantAnnounceRequest &msg); void on_set_configuration(const std::vector &active_wake_words); - const Configuration &get_configuration(); + const Configuration &get_configuration(const std::vector &external_wake_words); bool is_running() const { return this->state_ != State::IDLE; } void set_continuous(bool continuous) { this->continuous_ = continuous; } @@ -344,6 +384,37 @@ class VoiceAssistant final : public Component { #ifdef USE_MICRO_WAKE_WORD micro_wake_word::MicroWakeWord *micro_wake_word_{nullptr}; +#ifdef USE_VOICE_ASSISTANT_RUNTIME_MODEL + /* Runtime model management. Every member and method below is touched only on the main loop; the + background download task communicates results back via defer() with everything captured by value. */ + + // External wake words advertised by Home Assistant, rebuilt from each configuration request so stale + // entries drop out. Ownership of a loaded model itself lives in the WakeWordModel (micro_wake_word owns it). + std::vector external_wake_words_cache_; + // Wake word IDs HA asked us to activate but whose model isn't loaded yet. Reported as active in + // get_configuration so HA's UI reflects the request immediately; entries clear on load success or failure. + std::vector pending_active_wake_words_; + // Models the download task should fetch next. Filled on the main loop, handed to the task by std::move. + std::vector model_download_queue_; + + void cache_external_wake_words_(const std::vector &wake_words); + // Unloads runtime models whose wake word is no longer in the cache, freeing their memory. + void remove_stale_runtime_models_(); + void restore_runtime_models_(); + CachedExternalWakeWord *find_cached_wake_word_(const std::string &id); + bool is_wake_word_pending_(const std::string &id) const; + void erase_pending_wake_word_(const std::string &id); + // Persists a model as disabled (so it isn't retried every boot) and drops it from the optimistic list. + void mark_model_load_failed_(const std::string &id); + + // Starts the background download task if work is queued and no task is already running. + void try_start_model_load_task_(); + // FreeRTOS task body: downloads and validates queued models, handing each off to the main loop via defer(). + static void model_load_task(void *params); + TaskHandle_t model_load_task_handle_{nullptr}; + + http_request::HttpRequestComponent *http_request_{nullptr}; +#endif #endif }; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index ad24d27369..3cff670e96 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -337,6 +337,7 @@ #define USE_SPEAKER_MEDIA_PLAYER_ON_OFF #define USE_SPI #define USE_VOICE_ASSISTANT +#define USE_VOICE_ASSISTANT_RUNTIME_MODEL #define USE_WEBSERVER #define USE_WEBSERVER_AUTH #define USE_WEBSERVER_AUTH_DIGEST diff --git a/tests/components/voice_assistant/test-runtime-models.esp32-idf.yaml b/tests/components/voice_assistant/test-runtime-models.esp32-idf.yaml new file mode 100644 index 0000000000..5f6064b021 --- /dev/null +++ b/tests/components/voice_assistant/test-runtime-models.esp32-idf.yaml @@ -0,0 +1,49 @@ +substitutions: + i2s_din_pin: GPIO34 + i2s_dout_pin: GPIO32 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml + +wifi: + ssid: MySSID + password: password1 + +api: + +# Runtime wake word model downloading pulls in sha256 + json via AUTO_LOAD and defines +# USE_VOICE_ASSISTANT_RUNTIME_MODEL, compiling the whole download/validation code path. +http_request: + id: model_http_request + verify_ssl: false + +micro_wake_word: + id: mww_id + microphone: va_mic_id_external + # Empty models list: exercises the optional-models path. Wake words arrive at runtime from Home Assistant. + models: [] + +microphone: + - platform: i2s_audio + id: va_mic_id_external + i2s_audio_id: i2s_audio_bus + i2s_din_pin: ${i2s_din_pin} + adc_type: external + pdm: false + +speaker: + - platform: i2s_audio + id: va_speaker_id + i2s_audio_id: i2s_audio_bus + dac_type: external + i2s_dout_pin: ${i2s_dout_pin} + +voice_assistant: + microphone: + microphone: va_mic_id_external + gain_factor: 4 + channels: 0 + speaker: va_speaker_id + micro_wake_word: mww_id + http_request_id: model_http_request + conversation_timeout: 60s