github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-10-14 10:12:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

[micro_wake_word] Add opt-in internal RAM placement (#20529)

Browse Source 
Co-authored-by: J. Nick Koston <nick@koston.org>
This commit is contained in:
embe
2026-10-11 11:49:49 -10:00
committed by GitHub
co-authored by J. Nick Koston
parent 7d7db6bf90
commit 8060dadd6c
9 changed files with 87 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/components/micro_wake_word/__init__.py
+6
View File
@@ -44,6 +44,7 @@ DOMAIN = "micro_wake_word"
CONF_FEATURE_STEP_SIZE = "feature_step_size"
CONF_MODELS = "models"
CONF_ON_WAKE_WORD_DETECTED = "on_wake_word_detected"
CONF_PREFER_INTERNAL_RAM = "prefer_internal_ram"
CONF_PROBABILITY_CUTOFF = "probability_cutoff"
CONF_SLIDING_WINDOW_AVERAGE_SIZE = "sliding_window_average_size"
CONF_SLIDING_WINDOW_SIZE = "sliding_window_size"
@@ -426,6 +427,7 @@ CONFIG_SCHEMA = cv.All(
),
cv.Optional(CONF_VAD): _maybe_empty_vad_schema,
cv.Optional(CONF_STOP_AFTER_DETECTION, default=True): cv.boolean,
cv.Optional(CONF_PREFER_INTERNAL_RAM, default=False): cv.boolean,
cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram,
cv.Optional(CONF_MODEL): cv.invalid(
f"The {CONF_MODEL} parameter has moved to be a list element under the {CONF_MODELS} parameter."
@@ -519,6 +521,8 @@ async def to_code(config):
cg.add(var.set_microphone_source(mic_source))
cg.add_define("USE_MICRO_WAKE_WORD")
if config[CONF_PREFER_INTERNAL_RAM]:
cg.add_define("USE_MICRO_WAKE_WORD_INTERNAL_RAM")
ota.request_ota_state_listeners()
if config.get(CONF_TASK_STACK_IN_PSRAM):
@@ -569,6 +573,7 @@ async def to_code(config):
cg.add(
var.add_vad_model(
prog_arr,
len(data),
quantized_probability_cutoff,
sliding_window_size,
manifest[KEY_MICRO][CONF_TENSOR_ARENA_SIZE],
@@ -581,6 +586,7 @@ async def to_code(config):
model_parameters[CONF_ID],
str(model_parameters[CONF_ID]),
prog_arr,
len(data),
quantized_probability_cutoff,
sliding_window_size,
manifest[KEY_WAKE_WORD],
esphome/components/micro_wake_word/micro_wake_word.cpp
+4 -3
View File
@@ -386,9 +386,10 @@ WakeWordModel *MicroWakeWord::get_model_by_id(const std::string &model_id) {
}
#ifdef USE_MICRO_WAKE_WORD_VAD
void MicroWakeWord::add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size,
size_t tensor_arena_size) {
this->vad_model_ = make_unique<VADModel>(model_start, probability_cutoff, sliding_window_size, tensor_arena_size);
void MicroWakeWord::add_vad_model(const uint8_t *model_start, size_t model_size, uint8_t probability_cutoff,
size_t sliding_window_size, size_t tensor_arena_size) {
this->vad_model_ =
make_unique<VADModel>(model_start, model_size, probability_cutoff, sliding_window_size, tensor_arena_size);
}
#endif
esphome/components/micro_wake_word/micro_wake_word.h
+2 -2
View File
@@ -93,8 +93,8 @@ class MicroWakeWord final : public Component
uint8_t get_features_step_size() const { return this->features_step_size_; }
#ifdef USE_MICRO_WAKE_WORD_VAD
void add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size,
size_t tensor_arena_size);
void add_vad_model(const uint8_t *model_start, size_t model_size, uint8_t probability_cutoff,
size_t sliding_window_size, size_t tensor_arena_size);
// Intended for the voice assistant component to fetch VAD status
bool get_vad_state() { return this->vad_state_; }
esphome/components/micro_wake_word/streaming_model.cpp
+50 -6
View File
@@ -4,6 +4,10 @@
#include "esphome/core/helpers.h"
#include "esphome/core/log.h"
#include <cstring>
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
#include <esp_memory_utils.h>
#endif
ESPHOME_LOG_TAG(TAG, "micro_wake_word");
@@ -62,16 +66,43 @@ bool StreamingModel::load_model_() {
}
if (this->tensor_arena_ == nullptr) {
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
RAMAllocator<uint8_t> preferred_allocator(RAMAllocator<uint8_t>::PREFER_INTERNAL);
this->tensor_arena_ = preferred_allocator.allocate(this->tensor_arena_size_);
if (this->tensor_arena_ != nullptr && !esp_ptr_internal(this->tensor_arena_)) {
ESP_LOGW(TAG, "Could not allocate %zu bytes of internal RAM for tensor arena; using default memory",
this->tensor_arena_size_);
}
#else
this->tensor_arena_ = arena_allocator.allocate(this->tensor_arena_size_);
#endif
if (this->tensor_arena_ == nullptr) {
ESP_LOGE(TAG, "Could not allocate the streaming model's tensor arena.");
return false;
}
}
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
if (!esp_ptr_internal(this->model_start_) && this->model_copy_ == nullptr) {
RAMAllocator<uint8_t> internal_allocator(RAMAllocator<uint8_t>::ALLOC_INTERNAL);
this->model_copy_ = internal_allocator.make_unique_array_for_overwrite(this->model_size_);
if (this->model_copy_ == nullptr) {
ESP_LOGW(TAG, "Could not allocate %zu bytes of internal RAM for model; using original model data",
this->model_size_);
} else {
std::memcpy(this->model_copy_.get(), this->model_start_, this->model_size_);
}
}
#endif
if (this->interpreter_ == nullptr) {
const uint8_t *model_data = this->model_start_;
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
if (this->model_copy_ != nullptr)
model_data = this->model_copy_.get();
#endif
this->interpreter_ =
make_unique<tflite::MicroInterpreter>(tflite::GetModel(this->model_start_), this->streaming_op_resolver_,
make_unique<tflite::MicroInterpreter>(tflite::GetModel(model_data), this->streaming_op_resolver_,
this->tensor_arena_, this->tensor_arena_size_, this->mrv_);
if (this->interpreter_->AllocateTensors() != kTfLiteOk) {
ESP_LOGE(TAG, "Failed to allocate tensors for the streaming model");
@@ -176,6 +207,9 @@ size_t StreamingModel::probe_arena_size_() {
void StreamingModel::unload_model() {
this->interpreter_.reset();
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
this->model_copy_.reset();
#endif
RAMAllocator<uint8_t> arena_allocator;
@@ -255,11 +289,15 @@ void StreamingModel::reset_probabilities() {
this->ignore_windows_ = -MIN_SLICES_BEFORE_DETECTION;
}
WakeWordModel::WakeWordModel(const std::string &id, const uint8_t *model_start, uint8_t default_probability_cutoff,
size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size,
bool default_enabled, bool internal_only) {
WakeWordModel::WakeWordModel(const std::string &id, const uint8_t *model_start, size_t model_size,
uint8_t default_probability_cutoff, size_t sliding_window_average_size,
const std::string &wake_word, size_t tensor_arena_size, bool default_enabled,
bool internal_only) {
this->id_ = id;
this->model_start_ = model_start;
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
this->model_size_ = model_size;
#endif
this->default_probability_cutoff_ = default_probability_cutoff;
this->probability_cutoff_ = default_probability_cutoff;
this->sliding_window_size_ = sliding_window_average_size;
@@ -291,6 +329,9 @@ WakeWordModel::WakeWordModel(const std::string &id, std::shared_ptr<ModelData> m
// null or unvalidated handle rather than dereferencing it blindly: model_start_ stays null and the model is
// never loaded.
this->model_start_ = this->model_data_ ? this->model_data_->get_model_pointer() : nullptr;
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
this->model_size_ = this->model_data_ ? this->model_data_->size() : 0;
#endif
if (this->model_start_ == nullptr) {
ESP_LOGE(TAG, "Model '%s' has no valid data and will not be loaded", id.c_str());
}
@@ -354,9 +395,12 @@ DetectionEvent WakeWordModel::determine_detected() {
return detection_event;
}
VADModel::VADModel(const uint8_t *model_start, uint8_t default_probability_cutoff, size_t sliding_window_size,
size_t tensor_arena_size) {
VADModel::VADModel(const uint8_t *model_start, size_t model_size, uint8_t default_probability_cutoff,
size_t sliding_window_size, size_t tensor_arena_size) {
this->model_start_ = model_start;
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
this->model_size_ = model_size;
#endif
this->default_probability_cutoff_ = default_probability_cutoff;
this->probability_cutoff_ = default_probability_cutoff;
this->sliding_window_size_ = sliding_window_size;
esphome/components/micro_wake_word/streaming_model.h
+11 -5
View File
@@ -5,6 +5,7 @@
#include "preprocessor_settings.h"
#include "model_data.h"
#include "esphome/core/helpers.h"
#include "esphome/core/preferences.h"
#include <memory>
@@ -96,6 +97,10 @@ class StreamingModel {
std::vector<uint8_t> recent_streaming_probabilities_;
const uint8_t *model_start_{nullptr};
#ifdef USE_MICRO_WAKE_WORD_INTERNAL_RAM
size_t model_size_{0};
RAMUniquePtr<uint8_t[]> model_copy_;
#endif
uint8_t *tensor_arena_{nullptr};
uint8_t *var_arena_{nullptr};
std::unique_ptr<tflite::MicroInterpreter> interpreter_;
@@ -108,6 +113,7 @@ class WakeWordModel final : public StreamingModel {
/// @brief Constructs a wake word model object with compile-time model data
/// @param id (std::string) identifier for this model
/// @param model_start (const uint8_t *) pointer to the start of the model's TFLite FlatBuffer
/// @param model_size (size_t) size of the compiled-in model data
/// @param default_probability_cutoff (uint8_t) probability cutoff for acceping the wake word has been said
/// @param sliding_window_average_size (size_t) the length of the sliding window computing the mean rolling
/// probability
@@ -115,9 +121,9 @@ class WakeWordModel final : public StreamingModel {
/// @param tensor_arena_size (size_t) Size in bytes for allocating the tensor arena
/// @param default_enabled (bool) If true, it will be enabled by default on first boot
/// @param internal_only (bool) If true, the model will not be exposed to HomeAssistant as an available model
WakeWordModel(const std::string &id, const uint8_t *model_start, uint8_t default_probability_cutoff,
size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size,
bool default_enabled, bool internal_only);
WakeWordModel(const std::string &id, const uint8_t *model_start, size_t model_size,
uint8_t default_probability_cutoff, size_t sliding_window_average_size, const std::string &wake_word,
size_t tensor_arena_size, bool default_enabled, bool internal_only);
/// @brief Constructs a wake word model object with a runtime-downloaded model
/// @param id (std::string) identifier for this model
@@ -173,8 +179,8 @@ class WakeWordModel final : public StreamingModel {
class VADModel final : public StreamingModel {
public:
VADModel(const uint8_t *model_start, uint8_t default_probability_cutoff, size_t sliding_window_size,
size_t tensor_arena_size);
VADModel(const uint8_t *model_start, size_t model_size, uint8_t default_probability_cutoff,
size_t sliding_window_size, size_t tensor_arena_size);
void log_model_config() override;
esphome/core/defines.h
+1
View File
@@ -523,6 +523,7 @@
#ifdef USE_ESP32
#define USE_MICRO_WAKE_WORD
#define USE_MICRO_WAKE_WORD_INTERNAL_RAM
#define USE_MICRO_WAKE_WORD_VAD
#if defined(USE_ESP32_VARIANT_ESP32C6) || defined(USE_ESP32_VARIANT_ESP32H2)
#define USE_OPENTHREAD
tests/components/micro_wake_word/common.yaml
+1
View File
@@ -13,6 +13,7 @@ microphone:
micro_wake_word:
microphone: echo_microphone
task_stack_in_psram: true
vad:
on_wake_word_detected:
- logger.log: "Wake word detected"
- micro_wake_word.stop:
tests/components/micro_wake_word/test-internal.esp32-idf.yaml
+9
View File
@@ -0,0 +1,9 @@
substitutions:
mic_din_pin: GPIO36
packages:
i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml
micro_wake_word: !include common.yaml
micro_wake_word:
prefer_internal_ram: true
tests/components/micro_wake_word/test.esp32-s3-idf.yaml
+3 -1
View File
@@ -3,5 +3,7 @@ substitutions:
packages:
i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-s3-idf.yaml
micro_wake_word: !include common.yaml
<<: !include common.yaml
micro_wake_word:
prefer_internal_ram: true