diff --git a/esphome/components/speaker_source/speaker_source_media_player.cpp b/esphome/components/speaker_source/speaker_source_media_player.cpp
index 215f3942d5..6a000b8f21 100644
--- a/esphome/components/speaker_source/speaker_source_media_player.cpp
+++ b/esphome/components/speaker_source/speaker_source_media_player.cpp
@@ -196,10 +196,19 @@ size_t SpeakerSourceMediaPlayer::handle_media_output_(uint8_t pipeline, media_so
       vTaskDelay(pdMS_TO_TICKS(timeout_ms));
       return 0;
     }
+    // Reserve frames before the speaker can play them and report their progress,
+    // then release the portion it did not accept.
+    const uint32_t reserved_frames = stream_info.bytes_to_frames(length);
+    ps.pending_frames.fetch_add(reserved_frames, std::memory_order_relaxed);
     size_t bytes_written = ps.speaker->play(data, length, pdMS_TO_TICKS(timeout_ms));
-    if (bytes_written > 0) {
-      // Track frames sent to speaker for this source
-      ps.pending_frames.fetch_add(stream_info.bytes_to_frames(bytes_written), std::memory_order_relaxed);
+    const uint32_t unused_frames = reserved_frames - stream_info.bytes_to_frames(bytes_written);
+    if (unused_frames > 0) {
+      // The main loop may reset pending_frames while play() is blocked.
+      // Clamp the release so that a concurrent reset cannot cause underflow.
+      uint32_t pending = ps.pending_frames.load(std::memory_order_relaxed);
+      while (!ps.pending_frames.compare_exchange_weak(pending, pending - std::min(pending, unused_frames),
+                                                      std::memory_order_relaxed)) {
+      }
     }
     return bytes_written;
   }
diff --git a/tests/components/speaker_source/test.esp32-idf.yaml b/tests/components/speaker_source/test.esp32-idf.yaml
index 5a2fd16938..a8d47dd253 100644
--- a/tests/components/speaker_source/test.esp32-idf.yaml
+++ b/tests/components/speaker_source/test.esp32-idf.yaml
@@ -4,4 +4,4 @@ substitutions:
 packages:
   i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml
 
-<<: !include common.yaml
+  speaker_source: !include common.yaml