diff --git a/esphome/components/micro_wake_word/__init__.py b/esphome/components/micro_wake_word/__init__.py index c427f28028..255923f878 100644 --- a/esphome/components/micro_wake_word/__init__.py +++ b/esphome/components/micro_wake_word/__init__.py @@ -432,7 +432,7 @@ CONFIG_SCHEMA = cv.All( min_channels=1, max_channels=1, ), - cv.Required(CONF_MODELS): cv.ensure_list( + cv.Optional(CONF_MODELS, default=[]): cv.ensure_list( cv.maybe_simple_value(MODEL_SCHEMA, key=CONF_MODEL) ), cv.Optional(CONF_ON_WAKE_WORD_DETECTED): automation.validate_automation( @@ -555,6 +555,9 @@ async def to_code(config): # Use the general model loading code for the VAD codegen config[CONF_MODELS].append(vad_model) + # Default feature step size for runtime models + feature_step_size = 10 + for i, model_parameters in enumerate(config[CONF_MODELS]): model_config = model_parameters.get(CONF_MODEL) data = [] @@ -573,6 +576,9 @@ async def to_code(config): manifest[KEY_MICRO][CONF_SLIDING_WINDOW_SIZE], ) + # Update feature step size from manifest + feature_step_size = manifest[KEY_MICRO][CONF_FEATURE_STEP_SIZE] + if manifest[KEY_WAKE_WORD] == "vad": cg.add( var.add_vad_model( @@ -602,7 +608,7 @@ async def to_code(config): cg.add(var.add_wake_word_model(wake_word_model)) - cg.add(var.set_features_step_size(manifest[KEY_MICRO][CONF_FEATURE_STEP_SIZE])) + cg.add(var.set_features_step_size(feature_step_size)) cg.add(var.set_stop_after_detection(config[CONF_STOP_AFTER_DETECTION])) if on_wake_word_detection_config := config.get(CONF_ON_WAKE_WORD_DETECTED): diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 237d72229d..3dadb78077 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -9,6 +9,8 @@ #include "esphome/components/audio/audio_transfer_buffer.h" +#include + #ifdef USE_OTA #include "esphome/components/ota/ota_backend.h" #endif @@ -35,21 +37,34 @@ static const UBaseType_t INFERENCE_TASK_PRIORITY = 3; enum EventGroupBits : uint32_t { COMMAND_STOP = (1 << 0), // Signals the inference task should stop COMMAND_RESET_RING_BUFFER = (1 << 1), // Signals the inference task to discard buffered audio + COMMAND_PAUSE_MODELS = (1 << 2), // Asks the inference task to pause at a safe point so the model lists can be + // mutated from the main loop TASK_STARTING = (1 << 3), TASK_RUNNING = (1 << 4), TASK_STOPPING = (1 << 5), TASK_STOPPED = (1 << 6), + MODELS_PAUSED = (1 << 7), // Inference task acknowledges it is paused and holds no iterators + COMMAND_RESUME_MODELS = (1 << 8), // Main loop signals the inference task it may resume iterating + ERROR_MEMORY = (1 << 9), ERROR_INFERENCE = (1 << 10), WARNING_FULL_RING_BUFFER = (1 << 13), + WARNING_MODELS_RESUME_TIMEOUT = (1 << 14), // The paused inference task gave up waiting to be released ERROR_BITS = ERROR_MEMORY | ERROR_INFERENCE, ALL_BITS = 0xfffff, // 24 total bits available in an event group }; +// How long the main loop waits for the inference task to acknowledge a pause request before giving up. +// The task checks for the command at the top of its loop, which runs at least every DATA_TIMEOUT_MS. +static const uint32_t MODELS_PAUSE_TIMEOUT_MS = 500; +// How long the paused inference task waits to be resumed before rechecking on its own. Only reached if +// the main loop abandoned the handshake (e.g. it timed out first), so recovery just needs to be bounded. +static const uint32_t MODELS_RESUME_TIMEOUT_MS = 1000; + float MicroWakeWord::get_setup_priority() const { return setup_priority::AFTER_CONNECTION; } static const LogString *micro_wake_word_state_to_string(State state) { @@ -176,6 +191,20 @@ void MicroWakeWord::inference_task(void *params) { xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_RUNNING); while (!(xEventGroupGetBits(this_mww->event_group_) & (COMMAND_STOP | ERROR_BITS))) { + if (xEventGroupGetBits(this_mww->event_group_) & EventGroupBits::COMMAND_PAUSE_MODELS) { + // Safe point: no iterators into wake_word_models_ are held here. Acknowledge the pause and wait for the + // main loop to finish mutating the model lists before resuming. + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::MODELS_PAUSED); + EventBits_t resume_bits = xEventGroupWaitBits(this_mww->event_group_, EventGroupBits::COMMAND_RESUME_MODELS, + pdTRUE, pdTRUE, pdMS_TO_TICKS(MODELS_RESUME_TIMEOUT_MS)); + if (!(resume_bits & EventGroupBits::COMMAND_RESUME_MODELS)) { + // Nobody released us, so the main loop abandoned the handshake and did not mutate the lists. + // Rechecking the pause command below is safe, but the wait cost a second of detection, so report it. + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::WARNING_MODELS_RESUME_TIMEOUT); + } + continue; + } + if (xEventGroupGetBits(this_mww->event_group_) & EventGroupBits::COMMAND_RESET_RING_BUFFER) { // Producer asked us to drain; run the consumer-side reset from this thread. audio_source->clear_buffered_data(); @@ -232,6 +261,130 @@ std::vector MicroWakeWord::get_wake_words() { void MicroWakeWord::add_wake_word_model(WakeWordModel *model) { this->wake_word_models_.push_back(model); } +bool MicroWakeWord::try_lock_models_() { + // When the inference task isn't running it holds no iterators into wake_word_models_, so the lists can be + // mutated without a handshake. The main loop is the only caller, so this state cannot change between here + // and the matching unlock_models_() call. + if (!this->inference_task_.is_created() || this->state_ == State::STOPPED) { + return true; + } + + // The task is running and iterates wake_word_models_. Ask it to pause at a safe point before we mutate. + // Clear any stale acknowledgement from an abandoned handshake first. + xEventGroupClearBits(this->event_group_, EventGroupBits::MODELS_PAUSED); + xEventGroupSetBits(this->event_group_, EventGroupBits::COMMAND_PAUSE_MODELS); + + EventBits_t bits = xEventGroupWaitBits(this->event_group_, EventGroupBits::MODELS_PAUSED, pdFALSE, pdTRUE, + pdMS_TO_TICKS(MODELS_PAUSE_TIMEOUT_MS)); + + if (!(bits & EventGroupBits::MODELS_PAUSED)) { + // The task never acknowledged (e.g. it is busy stopping). Withdraw the request and refuse to mutate a + // list it might be iterating. + xEventGroupClearBits(this->event_group_, EventGroupBits::COMMAND_PAUSE_MODELS); + return false; + } + return true; +} + +void MicroWakeWord::unlock_models_() { + if (!this->inference_task_.is_created() || this->state_ == State::STOPPED) { + return; // Nothing was paused + } + xEventGroupClearBits(this->event_group_, EventGroupBits::MODELS_PAUSED | EventGroupBits::COMMAND_PAUSE_MODELS); + xEventGroupSetBits(this->event_group_, EventGroupBits::COMMAND_RESUME_MODELS); +} + +bool MicroWakeWord::add_runtime_model(std::unique_ptr model) { + if (!model) { + ESP_LOGE(TAG, "Cannot add null runtime model"); + return false; + } + + const std::string model_id = model->get_id(); + + // A model without usable data can never load, so keep it out of the lists entirely. Otherwise it would be + // advertised to Home Assistant as selectable and the inference task would silently disable it again every + // time it was enabled. + if (!model->has_model_data()) { + ESP_LOGE(TAG, "Runtime model '%s' has no valid data", model_id.c_str()); + return false; + } + + // Reject a duplicate id against every model (compiled or runtime). The inference task only ever reads + // wake_word_models_, so scanning it here (on the main loop) needs no synchronization. + for (auto *existing : this->wake_word_models_) { + if (existing->get_id() == model_id) { + ESP_LOGW(TAG, "Wake word model '%s' already exists", model_id.c_str()); + return false; + } + } + + if (!this->try_lock_models_()) { + ESP_LOGE(TAG, "Timed out pausing inference task; not adding runtime model '%s'", model_id.c_str()); + return false; + } + + this->wake_word_models_.push_back(model.get()); + this->runtime_models_.push_back(std::move(model)); + + this->unlock_models_(); + ESP_LOGD(TAG, "Added runtime model '%s'", model_id.c_str()); + return true; +} + +bool MicroWakeWord::remove_runtime_model(const std::string &model_id) { + // Only runtime-downloaded models can be removed; compiled-in models never appear in runtime_models_. + auto runtime_it = + std::find_if(this->runtime_models_.begin(), this->runtime_models_.end(), + [&model_id](const std::unique_ptr &m) { return m->get_id() == model_id; }); + if (runtime_it == this->runtime_models_.end()) { + return false; + } + + if (!this->try_lock_models_()) { + ESP_LOGE(TAG, "Timed out pausing inference task; not removing runtime model '%s'", model_id.c_str()); + return false; + } + + WakeWordModel *raw = runtime_it->get(); + auto models_it = std::find(this->wake_word_models_.begin(), this->wake_word_models_.end(), raw); + if (models_it != this->wake_word_models_.end()) { + this->wake_word_models_.erase(models_it); + } + + // Queued detection events hold a pointer into the model being destroyed, so drop them. The inference task + // is parked, so no new events can be queued concurrently. Losing an undelivered detection from another + // model is acceptable for this rare operation. + xQueueReset(this->detection_queue_); + + // Free the interpreter and arenas (safe: the task is parked, not mid-inference), then destroy the model. + // Its ModelData releases the PSRAM model buffer once the last shared_ptr reference drops. + raw->unload_model(); + this->runtime_models_.erase(runtime_it); + + this->unlock_models_(); + ESP_LOGI(TAG, "Removed runtime model '%s'", model_id.c_str()); + return true; +} + +std::vector MicroWakeWord::get_runtime_model_ids() { + std::vector ids; + ids.reserve(this->runtime_models_.size()); + for (const auto &model : this->runtime_models_) { + ids.push_back(model->get_id()); + } + return ids; +} + +WakeWordModel *MicroWakeWord::get_model_by_id(const std::string &model_id) { + for (auto *model : this->wake_word_models_) { + if (model->get_id() == model_id) { + return model; + } + } + return nullptr; +} + #ifdef USE_MICRO_WAKE_WORD_VAD void MicroWakeWord::add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size, size_t tensor_arena_size) { @@ -270,6 +423,12 @@ void MicroWakeWord::loop() { "word detection accuracy will temporarily be reduced."); } + if (event_group_bits & EventGroupBits::WARNING_MODELS_RESUME_TIMEOUT) { + xEventGroupClearBits(this->event_group_, EventGroupBits::WARNING_MODELS_RESUME_TIMEOUT); + ESP_LOGW(TAG, "Inference task paused for %" PRIu32 " ms without being released, so it resumed on its own", + MODELS_RESUME_TIMEOUT_MS); + } + if (event_group_bits & EventGroupBits::TASK_STARTING) { ESP_LOGD(TAG, "Inference task has started, attempting to allocate memory for buffers"); xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STARTING); diff --git a/esphome/components/micro_wake_word/micro_wake_word.h b/esphome/components/micro_wake_word/micro_wake_word.h index aebb5b2595..03f4a86fd4 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.h +++ b/esphome/components/micro_wake_word/micro_wake_word.h @@ -66,6 +66,32 @@ class MicroWakeWord final : public Component void add_wake_word_model(WakeWordModel *model); + /// @brief Adds a runtime-downloaded wake word model. Must be called from the main loop. + /// If the inference task is running it is paused at a safe point before the model lists are mutated, + /// so the task never observes a half-updated vector. + /// Callers should check get_model_by_id() before constructing the model: a WakeWordModel permanently + /// claims a preference backend that is not released when the model is destroyed, so building one only to + /// have it rejected here costs internal RAM that never comes back. + /// @return True if the model was added, false if it has no valid data, on a duplicate id, or if the task + /// could not be paused + bool add_runtime_model(std::unique_ptr model); + + /// @brief Removes a runtime-downloaded wake word model and frees its interpreter, arenas, and model buffer. + /// Must be called from the main loop. If the inference task is running it is paused at a safe point first, + /// and any queued detection events are dropped (they hold pointers into the model being destroyed). + /// @return True if the model was removed, false if the id is not a runtime model or the task could not be paused + bool remove_runtime_model(const std::string &model_id); + + /// @brief Returns the wake word model with the given id, or nullptr if none matches (compiled or runtime). + /// Must be called from the main loop, as the returned pointer is invalidated by remove_runtime_model(). + WakeWordModel *get_model_by_id(const std::string &model_id); + + /// @brief Returns the ids of all runtime-downloaded models. Must be called from the main loop. + std::vector get_runtime_model_ids(); + + /// @brief Returns the feature step size (ms) the frontend is configured for. Runtime models must match it. + uint8_t get_features_step_size() const { return this->features_step_size_; } + #ifdef USE_MICRO_WAKE_WORD_VAD void add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size, size_t tensor_arena_size); @@ -85,6 +111,7 @@ class MicroWakeWord final : public Component std::weak_ptr ring_buffer_; std::vector wake_word_models_; + std::vector> runtime_models_; #ifdef USE_MICRO_WAKE_WORD_VAD std::unique_ptr vad_model_; @@ -119,6 +146,13 @@ class MicroWakeWord final : public Component /// @brief Resumes the inference task void resume_task_(); + /// @brief Parks the inference task at a safe point (or verifies it isn't running) so the model lists may be + /// mutated from the main loop. Every successful call must be paired with unlock_models_(). + /// @return True if the lists may be mutated, false if the running task never acknowledged the pause request + bool try_lock_models_(); + /// @brief Releases the inference task parked by a successful try_lock_models_() call + void unlock_models_(); + void set_state_(State state); /// @brief Generates a spectrogram feature from an input buffer of audio samples. The frontend buffers samples diff --git a/esphome/components/micro_wake_word/model_data.cpp b/esphome/components/micro_wake_word/model_data.cpp new file mode 100644 index 0000000000..a7326ab77a --- /dev/null +++ b/esphome/components/micro_wake_word/model_data.cpp @@ -0,0 +1,100 @@ +#include "model_data.h" + +#ifdef USE_ESP32 + +#include +#include "esphome/core/log.h" + +#include +#include + +namespace esphome::micro_wake_word { + +static const char *const TAG = "micro_wake_word"; + +ModelData::~ModelData() { this->deallocate_(); } + +bool ModelData::allocate(size_t size) { + // Reject up front: reallocating to zero frees the buffer and returns null, which would leave data_ pointing at + // freed memory. A zero-length model is never usable anyway. + if (size == 0) { + ESP_LOGE(TAG, "Refusing to allocate a zero-length model"); + return false; + } + + // Already allocated, so reallocate to the new size + if (this->data_) { + uint8_t *new_allocation = this->allocator_.reallocate(this->data_, size); + if (new_allocation == nullptr) { + ESP_LOGE(TAG, "Failed to reallocate %zu bytes", size); + return false; + } + this->data_ = new_allocation; + this->size_ = size; + this->valid_ = false; // Need to revalidate with new data + return true; + } + + // Try to allocate in PSRAM first + this->data_ = this->allocator_.allocate(size); + if (this->data_ == nullptr) { + ESP_LOGE(TAG, "Failed to allocate %zu bytes", size); + return false; + } + + this->size_ = size; + this->valid_ = false; + return true; +} + +void ModelData::deallocate_() { + if (this->data_ != nullptr) { + this->allocator_.deallocate(this->data_, this->size_); + this->data_ = nullptr; + this->size_ = 0; + this->valid_ = false; + } +} + +const uint8_t *ModelData::get_model_pointer() const { return this->valid_ ? this->data_ : nullptr; } + +uint8_t *ModelData::get_write_pointer() { + this->valid_ = false; // Mark invalid while writing + return this->data_; +} + +bool ModelData::validate_and_mark_ready() { + // The magic number lives in bytes 4-7, so we need at least 8 bytes to read it. + if (!this->data_ || this->size_ < 8) { + ESP_LOGE(TAG, "Model data is null or too small"); + return false; + } + + // Check TFLite magic number "TFL3" in bytes 4-7 + if (memcmp(this->data_ + 4, "TFL3", 4) != 0) { + ESP_LOGE(TAG, "Invalid TFLite model magic number"); + return false; + } + + // Bytes 0-3 hold the offset of the root table. tflite::GetModel only adds that offset to the start of the + // buffer, so check it lands inside the buffer before reading through it. + uint32_t root_offset; + memcpy(&root_offset, this->data_, sizeof(root_offset)); + if (root_offset >= this->size_) { + ESP_LOGE(TAG, "TFLite model root offset is out of bounds"); + return false; + } + + const tflite::Model *model = tflite::GetModel(this->data_); + if (model->version() != TFLITE_SCHEMA_VERSION) { + ESP_LOGE(TAG, "TFLite model version mismatch (expected %d, got %d)", TFLITE_SCHEMA_VERSION, model->version()); + return false; + } + + this->valid_ = true; + return true; +} + +} // namespace esphome::micro_wake_word + +#endif // USE_ESP32 diff --git a/esphome/components/micro_wake_word/model_data.h b/esphome/components/micro_wake_word/model_data.h new file mode 100644 index 0000000000..0f0e08c718 --- /dev/null +++ b/esphome/components/micro_wake_word/model_data.h @@ -0,0 +1,60 @@ +#pragma once + +#ifdef USE_ESP32 + +#include +#include +#include "esphome/core/helpers.h" + +namespace esphome::micro_wake_word { + +// Owns the buffer holding a runtime-downloaded TFLite model. The buffer prefers PSRAM but falls back to +// internal RAM, so a device without PSRAM can still hold a single model. It is filled over HTTP, checked +// for integrity by the caller (SHA256) and for a usable TFLite header here, then kept alive for the +// lifetime of the WakeWordModel that uses it. Only ever held behind a std::shared_ptr, so copies and +// moves are disabled. +class ModelData { + public: + ModelData() = default; + ~ModelData(); + + // Non-copyable, non-movable + ModelData(const ModelData &) = delete; + ModelData &operator=(const ModelData &) = delete; + ModelData(ModelData &&) = delete; + ModelData &operator=(ModelData &&) = delete; + + // Allocate memory for model + bool allocate(size_t size); + + // Get stable pointer for TFLite (only valid after validate_and_mark_ready()) + const uint8_t *get_model_pointer() const; + + // Get writable pointer for downloading (invalidates the model) + uint8_t *get_write_pointer(); + + // Validate TFLite model and mark as ready for use + bool validate_and_mark_ready(); + + // Check if model is valid and ready for use + bool is_valid() const { return this->valid_; } + + // Get size of model data + size_t size() const { return this->size_; } + + // Check if memory is allocated + bool is_allocated() const { return this->data_ != nullptr; } + + protected: + // Deallocate memory + void deallocate_(); + + uint8_t *data_{nullptr}; + size_t size_{0}; + bool valid_{false}; + RAMAllocator allocator_{RAMAllocator::NONE}; +}; + +} // namespace esphome::micro_wake_word + +#endif // USE_ESP32 diff --git a/esphome/components/micro_wake_word/streaming_model.cpp b/esphome/components/micro_wake_word/streaming_model.cpp index 1cdc06b352..72984f04fb 100644 --- a/esphome/components/micro_wake_word/streaming_model.cpp +++ b/esphome/components/micro_wake_word/streaming_model.cpp @@ -26,6 +26,11 @@ void VADModel::log_model_config() { } bool StreamingModel::load_model_() { + if (this->model_start_ == nullptr) { + ESP_LOGE(TAG, "Streaming model has no data to load"); + return false; + } + RAMAllocator arena_allocator; if (this->var_arena_ == nullptr) { @@ -188,6 +193,13 @@ void StreamingModel::unload_model() { } bool StreamingModel::perform_streaming_inference(const int8_t features[PREPROCESSOR_FEATURE_SIZE]) { + if (this->model_start_ == nullptr) { + // No usable model data, and that cannot change for this object. Skip the model instead of reporting a + // failure, because a false return here stops the inference task for every other model too. + this->enabled_ = false; + return true; + } + if (this->enabled_ && !this->loaded_) { // Model is enabled but isn't loaded if (!this->load_model_()) { @@ -269,6 +281,41 @@ WakeWordModel::WakeWordModel(const std::string &id, const uint8_t *model_start, } }; +WakeWordModel::WakeWordModel(const std::string &id, std::shared_ptr model_data, + uint8_t default_probability_cutoff, size_t sliding_window_average_size, + const std::string &wake_word, std::vector trained_languages, + size_t tensor_arena_size) { + this->id_ = id; + this->model_data_ = std::move(model_data); + // Callers are expected to pass a validated buffer, so this is normally the stable model pointer. Tolerate a + // null or unvalidated handle rather than dereferencing it blindly: model_start_ stays null and the model is + // never loaded. + this->model_start_ = this->model_data_ ? this->model_data_->get_model_pointer() : nullptr; + if (this->model_start_ == nullptr) { + ESP_LOGE(TAG, "Model '%s' has no valid data and will not be loaded", id.c_str()); + } + this->default_probability_cutoff_ = default_probability_cutoff; + this->probability_cutoff_ = default_probability_cutoff; + this->sliding_window_size_ = sliding_window_average_size; + this->recent_streaming_probabilities_.resize(sliding_window_average_size, 0); + this->wake_word_ = wake_word; + this->trained_languages_ = std::move(trained_languages); + this->tensor_arena_size_ = tensor_arena_size; + this->register_streaming_ops_(this->streaming_op_resolver_); + this->current_stride_step_ = 0; + this->internal_only_ = false; // Runtime models are always exposed to Home Assistant + + this->pref_ = global_preferences->make_preference(fnv1_hash(id)); + bool enabled; + if (this->pref_.load(&enabled)) { + // Use the enabled state loaded from flash + this->enabled_ = enabled; + } else { + // No saved state: stay disabled. The activation flow calls enable() explicitly after adding. + this->enabled_ = false; + } +}; + void WakeWordModel::enable() { this->enabled_ = true; if (!this->internal_only_) { diff --git a/esphome/components/micro_wake_word/streaming_model.h b/esphome/components/micro_wake_word/streaming_model.h index 07ba78d1f4..1cb9d6eba5 100644 --- a/esphome/components/micro_wake_word/streaming_model.h +++ b/esphome/components/micro_wake_word/streaming_model.h @@ -3,9 +3,11 @@ #ifdef USE_ESP32 #include "preprocessor_settings.h" +#include "model_data.h" #include "esphome/core/preferences.h" +#include #include #include #include @@ -27,6 +29,10 @@ struct DetectionEvent { class StreamingModel { public: + // Runtime models are heap owned and destroyed while the device is running, so freeing the arenas cannot + // depend on the owner calling unload_model() first. unload_model() is not virtual and is safe to repeat. + virtual ~StreamingModel() { this->unload_model(); } + virtual void log_model_config() = 0; virtual DetectionEvent determine_detected() = 0; @@ -51,6 +57,9 @@ class StreamingModel { /// @brief Return true if the model is enabled. bool is_enabled() const { return this->enabled_; } + /// @brief Return true if the model has usable data. A model without it can never be loaded or run. + bool has_model_data() const { return this->model_start_ != nullptr; } + bool get_unprocessed_probability_status() const { return this->unprocessed_probability_status_; } // Quantized probability cutoffs mapping 0.0 - 1.0 to 0 - 255 @@ -86,7 +95,7 @@ class StreamingModel { size_t tensor_arena_size_; std::vector recent_streaming_probabilities_; - const uint8_t *model_start_; + const uint8_t *model_start_{nullptr}; uint8_t *tensor_arena_{nullptr}; uint8_t *var_arena_{nullptr}; std::unique_ptr interpreter_; @@ -96,7 +105,7 @@ class StreamingModel { class WakeWordModel final : public StreamingModel { public: - /// @brief Constructs a wake word model object + /// @brief Constructs a wake word model object with compile-time model data /// @param id (std::string) identifier for this model /// @param model_start (const uint8_t *) pointer to the start of the model's TFLite FlatBuffer /// @param default_probability_cutoff (uint8_t) probability cutoff for acceping the wake word has been said @@ -110,6 +119,23 @@ class WakeWordModel final : public StreamingModel { size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size, bool default_enabled, bool internal_only); + /// @brief Constructs a wake word model object with a runtime-downloaded model + /// @param id (std::string) identifier for this model + /// @param model_data (std::shared_ptr) owning handle to the downloaded model buffer; must be valid + /// @param default_probability_cutoff (uint8_t) probability cutoff for acceping the wake word has been said + /// @param sliding_window_average_size (size_t) the length of the sliding window computing the mean rolling + /// probability + /// @param wake_word (std::string) Friendly name of the wake word + /// @param trained_languages (std::vector) Languages the model was trained on + /// @param tensor_arena_size (size_t) Size in bytes for allocating the tensor arena + WakeWordModel(const std::string &id, std::shared_ptr model_data, uint8_t default_probability_cutoff, + size_t sliding_window_average_size, const std::string &wake_word, + std::vector trained_languages, size_t tensor_arena_size); + + // model_data_ is a member of this class, so it is destroyed before ~StreamingModel() runs. Unload here, while + // the buffer is still alive, so the interpreter is never torn down over freed model data. + ~WakeWordModel() override { this->unload_model(); } + void log_model_config() override; /// @brief Checks for the wake word by comparing the mean probability in the sliding window with the probability @@ -132,6 +158,10 @@ class WakeWordModel final : public StreamingModel { bool get_internal_only() { return this->internal_only_; } protected: + // Kept for runtime-downloaded models so the model buffer stays alive for the model's lifetime. + // Null for compiled-in models (their data lives in flash). + std::shared_ptr model_data_; + std::string id_; std::string wake_word_; std::vector trained_languages_; diff --git a/tests/components/micro_wake_word/validate.esp32-idf.yaml b/tests/components/micro_wake_word/validate.esp32-idf.yaml new file mode 100644 index 0000000000..d87b19bdcf --- /dev/null +++ b/tests/components/micro_wake_word/validate.esp32-idf.yaml @@ -0,0 +1,21 @@ +# Config-only test: micro_wake_word without any compiled-in models. Covers the optional models +# schema, which validates without a model list. Wake word models are added at runtime instead, +# which voice_assistant wires up. +substitutions: + mic_din_pin: GPIO36 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml + +microphone: + - platform: i2s_audio + id: echo_microphone + i2s_audio_id: i2s_audio_bus + i2s_din_pin: ${mic_din_pin} + adc_type: external + pdm: true + bits_per_sample: 16bit + +micro_wake_word: + microphone: echo_microphone + # models is omitted entirely, so the default empty list applies