diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp
index 7d33b6c49f8..41b7123269a 100644
--- a/esphome/components/mixer/speaker/mixer_speaker.cpp
+++ b/esphome/components/mixer/speaker/mixer_speaker.cpp
@@ -385,7 +385,8 @@ void MixerSpeaker::loop() {
   // Retries on a subsequent loop if the task is still running on the other core
   if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) {
     ESP_LOGD(TAG, "Stopped");
-    xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS);
+    // Keep a start request that arrived while the task was stopping, otherwise it is lost for good
+    xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS & ~MIXER_TASK_COMMAND_START);
     this->all_stopped_since_ms_ = 0;
   }