diff --git a/esphome/components/micro_wake_word/__init__.py b/esphome/components/micro_wake_word/__init__.py index f04f784ebcf..9477ebc850e 100644 --- a/esphome/components/micro_wake_word/__init__.py +++ b/esphome/components/micro_wake_word/__init__.py @@ -44,6 +44,7 @@ DOMAIN = "micro_wake_word" CONF_FEATURE_STEP_SIZE = "feature_step_size" CONF_MODELS = "models" CONF_ON_WAKE_WORD_DETECTED = "on_wake_word_detected" +CONF_PREFER_INTERNAL_RAM = "prefer_internal_ram" CONF_PROBABILITY_CUTOFF = "probability_cutoff" CONF_SLIDING_WINDOW_AVERAGE_SIZE = "sliding_window_average_size" CONF_SLIDING_WINDOW_SIZE = "sliding_window_size" @@ -426,6 +427,7 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_VAD): _maybe_empty_vad_schema, cv.Optional(CONF_STOP_AFTER_DETECTION, default=True): cv.boolean, + cv.Optional(CONF_PREFER_INTERNAL_RAM, default=False): cv.boolean, cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_MODEL): cv.invalid( f"The {CONF_MODEL} parameter has moved to be a list element under the {CONF_MODELS} parameter." @@ -519,6 +521,8 @@ async def to_code(config): cg.add(var.set_microphone_source(mic_source)) cg.add_define("USE_MICRO_WAKE_WORD") + if config[CONF_PREFER_INTERNAL_RAM]: + cg.add_define("USE_MICRO_WAKE_WORD_INTERNAL_RAM") ota.request_ota_state_listeners() if config.get(CONF_TASK_STACK_IN_PSRAM): @@ -569,6 +573,7 @@ async def to_code(config): cg.add( var.add_vad_model( prog_arr, + len(data), quantized_probability_cutoff, sliding_window_size, manifest[KEY_MICRO][CONF_TENSOR_ARENA_SIZE], @@ -581,6 +586,7 @@ async def to_code(config): model_parameters[CONF_ID], str(model_parameters[CONF_ID]), prog_arr, + len(data), quantized_probability_cutoff, sliding_window_size, manifest[KEY_WAKE_WORD], diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 6d76e399868..4ae66d69bf0 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -386,9 +386,10 @@ WakeWordModel *MicroWakeWord::get_model_by_id(const std::string &model_id) { } #ifdef USE_MICRO_WAKE_WORD_VAD -void MicroWakeWord::add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size, - size_t tensor_arena_size) { - this->vad_model_ = make_unique(model_start, probability_cutoff, sliding_window_size, tensor_arena_size); +void MicroWakeWord::add_vad_model(const uint8_t *model_start, size_t model_size, uint8_t probability_cutoff, + size_t sliding_window_size, size_t tensor_arena_size) { + this->vad_model_ = + make_unique(model_start, model_size, probability_cutoff, sliding_window_size, tensor_arena_size); } #endif diff --git a/esphome/components/micro_wake_word/micro_wake_word.h b/esphome/components/micro_wake_word/micro_wake_word.h index 03f4a86fd47..862d40733bf 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.h +++ b/esphome/components/micro_wake_word/micro_wake_word.h @@ -93,8 +93,8 @@ class MicroWakeWord final : public Component uint8_t get_features_step_size() const { return this->features_step_size_; } #ifdef USE_MICRO_WAKE_WORD_VAD - void add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size, - size_t tensor_arena_size); + void add_vad_model(const uint8_t *model_start, size_t model_size, uint8_t probability_cutoff, + size_t sliding_window_size, size_t tensor_arena_size); // Intended for the voice assistant component to fetch VAD status bool get_vad_state() { return this->vad_state_; } diff --git a/esphome/components/micro_wake_word/streaming_model.cpp b/esphome/components/micro_wake_word/streaming_model.cpp index 7cbd11711a2..14e6046ba04 100644 --- a/esphome/components/micro_wake_word/streaming_model.cpp +++ b/esphome/components/micro_wake_word/streaming_model.cpp @@ -4,6 +4,10 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM +#include +#endif ESPHOME_LOG_TAG(TAG, "micro_wake_word"); @@ -62,16 +66,43 @@ bool StreamingModel::load_model_() { } if (this->tensor_arena_ == nullptr) { +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + RAMAllocator preferred_allocator(RAMAllocator::PREFER_INTERNAL); + this->tensor_arena_ = preferred_allocator.allocate(this->tensor_arena_size_); + if (this->tensor_arena_ != nullptr && !esp_ptr_internal(this->tensor_arena_)) { + ESP_LOGW(TAG, "Could not allocate %zu bytes of internal RAM for tensor arena; using default memory", + this->tensor_arena_size_); + } +#else this->tensor_arena_ = arena_allocator.allocate(this->tensor_arena_size_); +#endif if (this->tensor_arena_ == nullptr) { ESP_LOGE(TAG, "Could not allocate the streaming model's tensor arena."); return false; } } +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + if (!esp_ptr_internal(this->model_start_) && this->model_copy_ == nullptr) { + RAMAllocator internal_allocator(RAMAllocator::ALLOC_INTERNAL); + this->model_copy_ = internal_allocator.make_unique_array_for_overwrite(this->model_size_); + if (this->model_copy_ == nullptr) { + ESP_LOGW(TAG, "Could not allocate %zu bytes of internal RAM for model; using original model data", + this->model_size_); + } else { + std::memcpy(this->model_copy_.get(), this->model_start_, this->model_size_); + } + } +#endif + if (this->interpreter_ == nullptr) { + const uint8_t *model_data = this->model_start_; +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + if (this->model_copy_ != nullptr) + model_data = this->model_copy_.get(); +#endif this->interpreter_ = - make_unique(tflite::GetModel(this->model_start_), this->streaming_op_resolver_, + make_unique(tflite::GetModel(model_data), this->streaming_op_resolver_, this->tensor_arena_, this->tensor_arena_size_, this->mrv_); if (this->interpreter_->AllocateTensors() != kTfLiteOk) { ESP_LOGE(TAG, "Failed to allocate tensors for the streaming model"); @@ -176,6 +207,9 @@ size_t StreamingModel::probe_arena_size_() { void StreamingModel::unload_model() { this->interpreter_.reset(); +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + this->model_copy_.reset(); +#endif RAMAllocator arena_allocator; @@ -255,11 +289,15 @@ void StreamingModel::reset_probabilities() { this->ignore_windows_ = -MIN_SLICES_BEFORE_DETECTION; } -WakeWordModel::WakeWordModel(const std::string &id, const uint8_t *model_start, uint8_t default_probability_cutoff, - size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size, - bool default_enabled, bool internal_only) { +WakeWordModel::WakeWordModel(const std::string &id, const uint8_t *model_start, size_t model_size, + uint8_t default_probability_cutoff, size_t sliding_window_average_size, + const std::string &wake_word, size_t tensor_arena_size, bool default_enabled, + bool internal_only) { this->id_ = id; this->model_start_ = model_start; +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + this->model_size_ = model_size; +#endif this->default_probability_cutoff_ = default_probability_cutoff; this->probability_cutoff_ = default_probability_cutoff; this->sliding_window_size_ = sliding_window_average_size; @@ -291,6 +329,9 @@ WakeWordModel::WakeWordModel(const std::string &id, std::shared_ptr m // null or unvalidated handle rather than dereferencing it blindly: model_start_ stays null and the model is // never loaded. this->model_start_ = this->model_data_ ? this->model_data_->get_model_pointer() : nullptr; +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + this->model_size_ = this->model_data_ ? this->model_data_->size() : 0; +#endif if (this->model_start_ == nullptr) { ESP_LOGE(TAG, "Model '%s' has no valid data and will not be loaded", id.c_str()); } @@ -354,9 +395,12 @@ DetectionEvent WakeWordModel::determine_detected() { return detection_event; } -VADModel::VADModel(const uint8_t *model_start, uint8_t default_probability_cutoff, size_t sliding_window_size, - size_t tensor_arena_size) { +VADModel::VADModel(const uint8_t *model_start, size_t model_size, uint8_t default_probability_cutoff, + size_t sliding_window_size, size_t tensor_arena_size) { this->model_start_ = model_start; +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + this->model_size_ = model_size; +#endif this->default_probability_cutoff_ = default_probability_cutoff; this->probability_cutoff_ = default_probability_cutoff; this->sliding_window_size_ = sliding_window_size; diff --git a/esphome/components/micro_wake_word/streaming_model.h b/esphome/components/micro_wake_word/streaming_model.h index 1cb9d6eba54..730afe32bf7 100644 --- a/esphome/components/micro_wake_word/streaming_model.h +++ b/esphome/components/micro_wake_word/streaming_model.h @@ -5,6 +5,7 @@ #include "preprocessor_settings.h" #include "model_data.h" +#include "esphome/core/helpers.h" #include "esphome/core/preferences.h" #include @@ -96,6 +97,10 @@ class StreamingModel { std::vector recent_streaming_probabilities_; const uint8_t *model_start_{nullptr}; +#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM + size_t model_size_{0}; + RAMUniquePtr model_copy_; +#endif uint8_t *tensor_arena_{nullptr}; uint8_t *var_arena_{nullptr}; std::unique_ptr interpreter_; @@ -108,6 +113,7 @@ class WakeWordModel final : public StreamingModel { /// @brief Constructs a wake word model object with compile-time model data /// @param id (std::string) identifier for this model /// @param model_start (const uint8_t *) pointer to the start of the model's TFLite FlatBuffer + /// @param model_size (size_t) size of the compiled-in model data /// @param default_probability_cutoff (uint8_t) probability cutoff for acceping the wake word has been said /// @param sliding_window_average_size (size_t) the length of the sliding window computing the mean rolling /// probability @@ -115,9 +121,9 @@ class WakeWordModel final : public StreamingModel { /// @param tensor_arena_size (size_t) Size in bytes for allocating the tensor arena /// @param default_enabled (bool) If true, it will be enabled by default on first boot /// @param internal_only (bool) If true, the model will not be exposed to HomeAssistant as an available model - WakeWordModel(const std::string &id, const uint8_t *model_start, uint8_t default_probability_cutoff, - size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size, - bool default_enabled, bool internal_only); + WakeWordModel(const std::string &id, const uint8_t *model_start, size_t model_size, + uint8_t default_probability_cutoff, size_t sliding_window_average_size, const std::string &wake_word, + size_t tensor_arena_size, bool default_enabled, bool internal_only); /// @brief Constructs a wake word model object with a runtime-downloaded model /// @param id (std::string) identifier for this model @@ -173,8 +179,8 @@ class WakeWordModel final : public StreamingModel { class VADModel final : public StreamingModel { public: - VADModel(const uint8_t *model_start, uint8_t default_probability_cutoff, size_t sliding_window_size, - size_t tensor_arena_size); + VADModel(const uint8_t *model_start, size_t model_size, uint8_t default_probability_cutoff, + size_t sliding_window_size, size_t tensor_arena_size); void log_model_config() override; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 730e17fb53a..c4c20e80a6d 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -523,6 +523,7 @@ #ifdef USE_ESP32 #define USE_MICRO_WAKE_WORD +#define USE_MICRO_WAKE_WORD_INTERNAL_RAM #define USE_MICRO_WAKE_WORD_VAD #if defined(USE_ESP32_VARIANT_ESP32C6) || defined(USE_ESP32_VARIANT_ESP32H2) #define USE_OPENTHREAD diff --git a/tests/components/micro_wake_word/common.yaml b/tests/components/micro_wake_word/common.yaml index 9ac1056ba49..2b9df607c87 100644 --- a/tests/components/micro_wake_word/common.yaml +++ b/tests/components/micro_wake_word/common.yaml @@ -13,6 +13,7 @@ microphone: micro_wake_word: microphone: echo_microphone task_stack_in_psram: true + vad: on_wake_word_detected: - logger.log: "Wake word detected" - micro_wake_word.stop: diff --git a/tests/components/micro_wake_word/test-internal.esp32-idf.yaml b/tests/components/micro_wake_word/test-internal.esp32-idf.yaml new file mode 100644 index 00000000000..bddab918c2a --- /dev/null +++ b/tests/components/micro_wake_word/test-internal.esp32-idf.yaml @@ -0,0 +1,9 @@ +substitutions: + mic_din_pin: GPIO36 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml + micro_wake_word: !include common.yaml + +micro_wake_word: + prefer_internal_ram: true diff --git a/tests/components/micro_wake_word/test.esp32-s3-idf.yaml b/tests/components/micro_wake_word/test.esp32-s3-idf.yaml index a1b7b754236..8806d8046d9 100644 --- a/tests/components/micro_wake_word/test.esp32-s3-idf.yaml +++ b/tests/components/micro_wake_word/test.esp32-s3-idf.yaml @@ -3,5 +3,7 @@ substitutions: packages: i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-s3-idf.yaml + micro_wake_word: !include common.yaml -<<: !include common.yaml +micro_wake_word: + prefer_internal_ram: true