diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py
index 2a5304be77e..14a08188949 100644
--- a/esphome/components/audio/__init__.py
+++ b/esphome/components/audio/__init__.py
@@ -339,10 +339,7 @@ async def to_code(config: ConfigType) -> None:
     # HTTPS streams verify the server against the root certificate bundle
     require_certificate_bundle()
 
-    add_idf_component(
-        name="esphome/esp-audio-libs",
-        ref="3.2.1",
-    )
+    add_idf_component(name="esphome/esp-audio-libs", ref="4.0.0")
 
     data = _get_data()
 
diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp
index ef21da65c5a..7d33b6c49f8 100644
--- a/esphome/components/mixer/speaker/mixer_speaker.cpp
+++ b/esphome/components/mixer/speaker/mixer_speaker.cpp
@@ -306,9 +306,9 @@ size_t SourceSpeaker::process_data_from_source(std::shared_ptr<audio::RingBuffer
 
   uint32_t samples_to_duck = this->audio_stream_info_.bytes_to_samples(bytes_read);
   if (samples_to_duck > 0) {
-    esp_audio_libs::ducking::apply(audio_source->mutable_data(),
-                                   static_cast<uint8_t>(this->audio_stream_info_.get_bits_per_sample() / 8),
-                                   samples_to_duck, this->ducking_state_);
+    this->ducking_ramp_.process(audio_source->mutable_data(),
+                                static_cast<uint8_t>(this->audio_stream_info_.get_bits_per_sample() / 8),
+                                samples_to_duck);
   }
 
   return bytes_read;
@@ -316,7 +316,7 @@ size_t SourceSpeaker::process_data_from_source(std::shared_ptr<audio::RingBuffer
 
 void SourceSpeaker::apply_ducking(uint8_t decibel_reduction, uint32_t duration) {
   const uint32_t transition_samples = duration > 0 ? this->audio_stream_info_.ms_to_samples(duration) : 0;
-  esp_audio_libs::ducking::set_target(this->ducking_state_, decibel_reduction, transition_samples);
+  this->ducking_ramp_.set_target_db_reduction_over(decibel_reduction, transition_samples);
 }
 
 void SourceSpeaker::enter_stopping_state_() {
diff --git a/esphome/components/mixer/speaker/mixer_speaker.h b/esphome/components/mixer/speaker/mixer_speaker.h
index 00e89d17826..494443d6951 100644
--- a/esphome/components/mixer/speaker/mixer_speaker.h
+++ b/esphome/components/mixer/speaker/mixer_speaker.h
@@ -11,7 +11,7 @@
 #include "esphome/core/helpers.h"
 #include "esphome/core/static_task.h"
 
-#include <ducking.h>  // esp-audio-libs
+#include <gain.h>  // esp-audio-libs
 
 #include <freertos/event_groups.h>
 
@@ -108,7 +108,7 @@ class SourceSpeaker final : public speaker::Speaker, public Component {
 
   bool pause_state_{false};
 
-  esp_audio_libs::ducking::DuckingState ducking_state_{};
+  esp_audio_libs::gain::GainRamp ducking_ramp_;
 
   std::atomic<uint32_t> pending_playback_frames_{0};
   std::atomic<uint32_t> playback_delay_frames_{0};  // Frames in output pipeline when this source started contributing
diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml
index e817a253d9c..b3cd5ee09b5 100644
--- a/esphome/idf_component.yml
+++ b/esphome/idf_component.yml
@@ -4,7 +4,7 @@ dependencies:
   esphome/dlms_parser:
     version: 1.1.0
   esphome/esp-audio-libs:
-    version: 3.2.1
+    version: 4.0.0
   esphome/esp-micro-speech-features:
     version: 1.2.3
   esphome/micro-decoder: