diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h
index 55f3bdc1f35..04938a00fad 100644
--- a/esphome/components/api/api_connection.h
+++ b/esphome/components/api/api_connection.h
@@ -316,10 +316,15 @@ class APIConnection final : public APIServerConnectionBase {
   void on_noise_encryption_set_key_request(const NoiseEncryptionSetKeyRequest &msg);
 #endif
 
+  static constexpr uint32_t CONNECT_GRACE_MS = 1000;
   bool is_authenticated() {
     return static_cast<ConnectionState>(this->flags_.connection_state) == ConnectionState::AUTHENTICATED;
   }
-  bool is_handshake_complete() const { return this->helper_->is_handshake_complete(); }
+  // A connection that is still setting up within its grace period; an older
+  // unauthenticated one is a stale half open client and no longer counts
+  bool is_still_connecting(uint32_t now) {
+    return !this->is_authenticated() && now - this->last_traffic_ < CONNECT_GRACE_MS;
+  }
   bool is_connection_setup() {
     return static_cast<ConnectionState>(this->flags_.connection_state) == ConnectionState::CONNECTED ||
            this->is_authenticated();
diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h
index a09d93b7df1..ff8aa7834c0 100644
--- a/esphome/components/api/api_frame_helper.h
+++ b/esphome/components/api/api_frame_helper.h
@@ -118,8 +118,6 @@ class APIFrameHelper {
   virtual APIError loop() = 0;
   virtual APIError read_packet(ReadPacketBuffer *buffer) = 0;
   bool can_write_without_blocking() { return this->state_ == State::DATA && this->overflow_buf_.empty(); }
-  /// Transport handshake done (immediately for plaintext)
-  bool is_handshake_complete() const { return this->state_ == State::DATA; }
   int getpeername(struct sockaddr *addr, socklen_t *addrlen) { return socket_->getpeername(addr, addrlen); }
   APIError close() {
     if (state_ == State::CLOSED)
diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp
index dc19824f0f6..b01a3b689c0 100644
--- a/esphome/components/api/api_server.cpp
+++ b/esphome/components/api/api_server.cpp
@@ -196,14 +196,15 @@ void APIServer::loop() {
 }
 
 #ifdef USE_API_NOISE
-// Refill only while no api client waits on a noise handshake; an OTA
-// handshake is not visible here and just pays the refill it triggered.
+// Refill only while no api client is still connecting; an OTA handshake is
+// not visible here and just pays the refill it triggered.
 void APIServer::prepare_spare_ephemeral_() {
   if (noise::has_spare_ephemeral() || !network::is_connected()) {
     return;
   }
+  const uint32_t now = App.get_loop_component_start_time();
   for (auto &client : this->active_clients()) {
-    if (!client->is_handshake_complete()) {
+    if (client->is_still_connecting(now)) {
       return;
     }
   }