[api] Provision encryption keys over an encrypted zero-PSK noise connection

esphome/components/api/__init__.py

@@ -488,8 +488,11 @@ async def to_code(config: ConfigType) -> None:
             cg.add_define("USE_API_NOISE_PSK_FROM_YAML")
         else:
             # No key provided, but encryption desired
-            # This will allow a plaintext client to provide a noise key,
-            # send it to the device, and then switch to noise.
+            # Until a key is set, the device accepts both Noise connections
+            # using the well-known all-zeros PSK (preferred: the key travels
+            # encrypted, protecting against passive sniffing) and plaintext
+            # connections (deprecated, remove after 2027.2.0) so a client can
+            # provide a noise key and the device then switches to noise only.
             # The key will be saved in flash and used for future connections
             # and plaintext disabled. Only a factory reset can remove it.
             cg.add_define("USE_API_PLAINTEXT")

esphome/components/api/api.proto

@@ -310,6 +310,11 @@ message DeviceInfoResponse {
 
   // Serial proxy instance metadata
   repeated SerialProxyInfo serial_proxies = 25 [(field_ifdef) = "USE_SERIAL_PROXY", (fixed_array_size_define) = "SERIAL_PROXY_COUNT"];
+
+  // Device is unprovisioned and accepts Noise handshakes with the well-known
+  // all-zeros PSK, so the api encryption key can be provisioned without being
+  // sent in plaintext (protects against passive sniffing, not active MITM)
+  bool api_encryption_provisionable = 26 [(field_ifdef) = "USE_API_NOISE"];
 }
 
 message ListEntitiesRequest {

esphome/components/api/api_connection.cpp

@@ -198,6 +198,29 @@ APIConnection::~APIConnection() {
 #endif
 }
 
+#if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT)
+void APIConnection::upgrade_helper_to_noise_() {
+  // The client opened with a Noise hello while this device has no encryption
+  // key set. Replace the plaintext helper with a Noise helper so the key can
+  // be provisioned over an encrypted channel: the noise context PSK is all
+  // zeros when unprovisioned, and NNpsk0 still runs a fresh ephemeral X25519
+  // exchange, so a passive listener cannot read the session. A publicly known
+  // PSK authenticates nobody; this protects against sniffing only.
+  auto *plaintext = static_cast<APIPlaintextFrameHelper *>(this->helper_.get());
+  uint8_t header[3];
+  uint8_t header_len = plaintext->get_consumed_header(header);
+  auto *noise = new APINoiseFrameHelper(plaintext->release_socket_for_switch(), this->parent_->get_noise_ctx());
+  this->helper_.reset(noise);  // destroys the plaintext helper
+  // Restore the peername-based client name (Hello has not arrived yet)
+  char peername[socket::SOCKADDR_STR_LEN];
+  noise->set_client_name(noise->get_peername_to(peername), strlen(peername));
+  APIError err = noise->init_from_handoff(header, header_len);
+  if (err != APIError::OK) {
+    this->fatal_error_with_log_(LOG_STR("Noise handoff failed"), err);
+  }
+}
+#endif  // USE_API_NOISE && USE_API_PLAINTEXT
+
 void APIConnection::destroy_active_iterator_() {
   switch (this->active_iterator_) {
     case ActiveIterator::LIST_ENTITIES:
@@ -255,6 +278,11 @@ void APIConnection::loop() {
       if (err == APIError::WOULD_BLOCK) {
         // No more data available
         break;
+#if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT)
+      } else if (err == APIError::PROTOCOL_SWITCH_TO_NOISE) {
+        this->upgrade_helper_to_noise_();
+        return;
+#endif
       } else if (err != APIError::OK) {
         this->fatal_error_with_log_(LOG_STR("Reading failed"), err);
         return;
@@ -1860,6 +1888,12 @@ bool APIConnection::send_device_info_response_() {
 #endif
 #ifdef USE_API_NOISE
   resp.api_encryption_supported = true;
+#ifdef USE_API_PLAINTEXT
+  // Dual build (encryption compiled in, no key from YAML): while no key is
+  // set, the key can be provisioned over a zero-PSK Noise connection instead
+  // of plaintext
+  resp.api_encryption_provisionable = !this->parent_->get_noise_ctx().has_psk();
+#endif
 #endif
 #ifdef USE_DEVICES
   size_t device_index = 0;
@@ -2037,10 +2071,23 @@ bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptio
     }
   } else if (base64_decode(msg.key, msg.key_len, psk.data(), psk.size()) != psk.size()) {
     ESP_LOGW(TAG, "Invalid encryption key length");
+  } else if (psk == psk_t{}) {
+    // The all-zeros key is reserved: it marks the device as unprovisioned and
+    // serves as the well-known provisioning PSK. Accepting it would report
+    // success without enabling encryption (or silently clear an existing key).
+    ESP_LOGW(TAG, "Rejecting all-zero encryption key");
   } else if (!this->parent_->save_noise_psk(psk, true)) {
     ESP_LOGW(TAG, "Failed to save encryption key");
   } else {
     resp.success = true;
+#ifdef USE_API_PLAINTEXT
+    if (this->helper_->frame_footer_size() == 0) {
+      // Plaintext transport has no frame footer; Noise always has the MAC footer.
+      // Remove after 2027.2.0 together with plaintext support on keyless devices.
+      ESP_LOGW(TAG, "Key was received on an unencrypted connection; this is deprecated and will stop "
+                    "working in 2027.2.0. Update Home Assistant to provision the key encrypted");
+    }
+#endif
   }
 
   return this->send_message(resp);

esphome/components/api/api_connection.h

@@ -626,6 +626,11 @@ class APIConnection final : public APIServerConnectionBase {
   void destroy_active_iterator_();
   void begin_iterator_(ActiveIterator type);
   void finalize_iterator_sync_();
+#if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT)
+  // Swap the plaintext helper for a Noise helper after the client opened
+  // with a Noise hello on an unprovisioned device (zero-PSK provisioning).
+  void upgrade_helper_to_noise_();
+#endif
 #ifdef USE_CAMERA
   std::unique_ptr<camera::CameraImageReader> image_reader_;
 #endif

esphome/components/api/api_frame_helper.cpp

@@ -96,6 +96,11 @@ const LogString *api_error_to_logstr(APIError err) {
   } else if (err == APIError::BAD_HANDSHAKE_ERROR_BYTE) {
     return LOG_STR("BAD_HANDSHAKE_ERROR_BYTE");
   }
+#endif
+#if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT)
+  else if (err == APIError::PROTOCOL_SWITCH_TO_NOISE) {
+    return LOG_STR("PROTOCOL_SWITCH_TO_NOISE");
+  }
 #endif
   return LOG_STR("UNKNOWN");
 }

esphome/components/api/api_frame_helper.h

@@ -88,6 +88,11 @@ enum class APIError : uint16_t {
   HANDSHAKESTATE_SPLIT_FAILED = 1020,
   BAD_HANDSHAKE_ERROR_BYTE = 1021,
 #endif
+#if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT)
+  // Not an error: an unprovisioned device received a Noise client hello on a
+  // plaintext connection; the caller must hand the socket off to a Noise helper.
+  PROTOCOL_SWITCH_TO_NOISE = 1023,
+#endif
 };
 
 const LogString *api_error_to_logstr(APIError err);
@@ -200,6 +205,12 @@ class APIFrameHelper {
   // or track that they stopped early and retry without this check.
   // See Socket::ready() for details.
   bool is_socket_ready() const { return socket_ != nullptr && socket_->ready(); }
+#if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT)
+  // Move the socket out of this helper so a replacement helper can take it
+  // over (plaintext to Noise handoff on unprovisioned devices). The drained
+  // helper must be destroyed right after.
+  std::unique_ptr<socket::Socket> release_socket_for_switch() { return std::move(this->socket_); }
+#endif
   // Release excess memory from internal buffers after initial sync
   void release_buffers() {
     // rx_buf_: Safe to clear only if no partial read in progress.

esphome/components/api/api_frame_helper_noise.cpp

@@ -109,6 +109,33 @@ APIError APINoiseFrameHelper::init() {
   state_ = State::CLIENT_HELLO;
   return APIError::OK;
 }
+#ifdef USE_API_PLAINTEXT
+APIError APINoiseFrameHelper::init_from_handoff(const uint8_t *header, uint8_t header_len) {
+  APIError err = this->init();
+  if (err != APIError::OK) {
+    return err;
+  }
+  // Seed the header bytes the plaintext helper consumed before detecting the
+  // Noise indicator; try_read_frame_ resumes from rx_header_buf_len_.
+  std::memcpy(this->rx_header_buf_, header, header_len);
+  this->rx_header_buf_len_ = header_len;
+  // Pump the handshake without gating on socket_->ready(): on LWIP the
+  // plaintext helper's partial read can drain rcvevent while the rest of the
+  // client hello sits in the lastdata cache, so ready() may report false even
+  // though data is available. Reads stop naturally on EWOULDBLOCK.
+  while (this->state_ != State::DATA) {
+    err = this->state_action_();
+    if (err == APIError::WOULD_BLOCK) {
+      return APIError::OK;
+    }
+    if (err != APIError::OK) {
+      return err;
+    }
+  }
+  return APIError::OK;
+}
+#endif  // USE_API_PLAINTEXT
+
 // Helper for handling handshake frame errors
 APIError APINoiseFrameHelper::handle_handshake_frame_error_(APIError aerr) {
   if (aerr == APIError::BAD_INDICATOR) {

esphome/components/api/api_frame_helper_noise.h

@@ -22,6 +22,13 @@ class APINoiseFrameHelper final : public APIFrameHelper {
   }
   ~APINoiseFrameHelper() override;
   APIError init() override;
+#ifdef USE_API_PLAINTEXT
+  // Take over a connection whose first bytes were consumed by a plaintext
+  // helper on an unprovisioned device (see APIError::PROTOCOL_SWITCH_TO_NOISE).
+  // Seeds the already-read header bytes and pumps the handshake state machine
+  // until it would block.
+  APIError init_from_handoff(const uint8_t *header, uint8_t header_len);
+#endif
   APIError loop() override;
   APIError read_packet(ReadPacketBuffer *buffer) override;
   APIError write_protobuf_packet(uint8_t type, ProtoWriteBuffer buffer) override;

esphome/components/api/api_frame_helper_plaintext.cpp

@@ -89,6 +89,17 @@ APIError APIPlaintextFrameHelper::try_read_frame_() {
     // If this was the first read, validate the indicator byte
     if (rx_header_buf_pos_ == 0 && received > 0) {
       if (rx_header_buf_[0] != 0x00) {
+#ifdef USE_API_NOISE
+        // Dual build (encryption supported but no key set): a 0x01 first byte
+        // is a Noise client hello. Hand the connection off to a Noise helper
+        // running the all-zeros provisioning PSK so the encryption key can be
+        // set without crossing the wire in plaintext. Preserve the bytes we
+        // already consumed; they are the start of the Noise 3-byte header.
+        if (rx_header_buf_[0] == 0x01) {
+          rx_header_buf_pos_ = static_cast<uint8_t>(received);
+          return APIError::PROTOCOL_SWITCH_TO_NOISE;
+        }
+#endif
         state_ = State::FAILED;
         HELPER_LOG("Bad indicator byte %u", rx_header_buf_[0]);
         return APIError::BAD_INDICATOR;

esphome/components/api/api_frame_helper_plaintext.h

@@ -23,6 +23,15 @@ class APIPlaintextFrameHelper final : public APIFrameHelper {
   APIError read_packet(ReadPacketBuffer *buffer) override;
   APIError write_protobuf_packet(uint8_t type, ProtoWriteBuffer buffer) override;
   APIError write_protobuf_messages(ProtoWriteBuffer buffer, std::span<const MessageInfo> messages) override;
+#ifdef USE_API_NOISE
+  // After try_read_frame_ returned PROTOCOL_SWITCH_TO_NOISE: copy out the
+  // header bytes already consumed from the socket (at most 3, the size of the
+  // Noise fixed header) so the replacement Noise helper can be seeded with them.
+  uint8_t get_consumed_header(uint8_t out[3]) const {
+    memcpy(out, this->rx_header_buf_, this->rx_header_buf_pos_);
+    return this->rx_header_buf_pos_;
+  }
+#endif
 
  protected:
   APIError try_read_frame_();

esphome/components/api/api_pb2.cpp

@@ -170,6 +170,9 @@ uint8_t *DeviceInfoResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_
   for (const auto &it : this->serial_proxies) {
     ProtoEncode::encode_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 25, it);
   }
+#endif
+#ifdef USE_API_NOISE
+  ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 26, this->api_encryption_provisionable);
 #endif
   return pos;
 }
@@ -232,6 +235,9 @@ uint32_t DeviceInfoResponse::calculate_size() const {
   for (const auto &it : this->serial_proxies) {
     size += ProtoSize::calc_message_force(2, it.calculate_size());
   }
+#endif
+#ifdef USE_API_NOISE
+  size += ProtoSize::calc_bool(2, this->api_encryption_provisionable);
 #endif
   return size;
 }

esphome/components/api/api_pb2.h

@@ -533,7 +533,7 @@ class SerialProxyInfo final : public ProtoMessage {
 class DeviceInfoResponse final : public ProtoMessage {
  public:
   static constexpr uint8_t MESSAGE_TYPE = 10;
-  static constexpr uint16_t ESTIMATED_SIZE = 309;
+  static constexpr uint16_t ESTIMATED_SIZE = 312;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const LogString *message_name() const override { return LOG_STR("device_info_response"); }
 #endif
@@ -588,6 +588,9 @@ class DeviceInfoResponse final : public ProtoMessage {
 #endif
 #ifdef USE_SERIAL_PROXY
   std::array<SerialProxyInfo, SERIAL_PROXY_COUNT> serial_proxies{};
+#endif
+#ifdef USE_API_NOISE
+  bool api_encryption_provisionable{false};
 #endif
   uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const;
   uint32_t calculate_size() const;

esphome/components/api/api_pb2_dump.cpp

@@ -982,6 +982,9 @@ const char *DeviceInfoResponse::dump_to(DumpBuffer &out) const {
     it.dump_to(out);
     out.append("\n");
   }
+#endif
+#ifdef USE_API_NOISE
+  dump_field(out, ESPHOME_PSTR("api_encryption_provisionable"), this->api_encryption_provisionable);
 #endif
   return out.c_str();
 }

esphome/components/mdns/mdns_component.cpp

@@ -108,6 +108,11 @@ void MDNSComponent::compile_records_(StaticVector<MDNSService, MDNS_SERVICE_COUN
 #endif
 #ifdef USE_API_NOISE
     txt_count++;  // api_encryption or api_encryption_supported
+#ifdef USE_API_PLAINTEXT
+    if (!api::global_api_server->get_noise_ctx().has_psk()) {
+      txt_count++;  // api_provisioning
+    }
+#endif
 #endif
 #ifdef ESPHOME_PROJECT_NAME
     txt_count += 2;  // project_name and project_version
@@ -166,6 +171,15 @@ void MDNSComponent::compile_records_(StaticVector<MDNSService, MDNS_SERVICE_COUN
     bool has_psk = api::global_api_server->get_noise_ctx().has_psk();
     const char *encryption_key = has_psk ? TXT_API_ENCRYPTION : TXT_API_ENCRYPTION_SUPPORTED;
     txt_records.push_back({MDNS_STR(encryption_key), MDNS_STR(NOISE_ENCRYPTION)});
+#ifdef USE_API_PLAINTEXT
+    if (!has_psk) {
+      // Unprovisioned dual-build device: advertise that the encryption key can
+      // be provisioned over a zero-PSK Noise connection (no plaintext needed)
+      MDNS_STATIC_CONST_CHAR(TXT_API_PROVISIONING, "api_provisioning");
+      MDNS_STATIC_CONST_CHAR(VALUE_ZERO_PSK, "zero-psk");
+      txt_records.push_back({MDNS_STR(TXT_API_PROVISIONING), MDNS_STR(VALUE_ZERO_PSK)});
+    }
+#endif
 #endif
 
 #ifdef ESPHOME_PROJECT_NAME

requirements.txt

@@ -9,7 +9,7 @@ pyserial==3.5
 platformio==6.1.19
 esptool==5.3.1
 click==8.3.3
-aioesphomeapi==45.5.2
+aioesphomeapi==45.6.0
 zeroconf==0.150.0
 puremagic==2.2.0
 ruamel.yaml==0.19.1 # dashboard_import

tests/components/api/test-dynamic-encryption.esp32-idf.yaml

@@ -1,5 +1,11 @@
-<<: !include common-base.yaml
+packages:
+  common: !include common-base.yaml
 
 wifi:
   ssid: MySSID
   password: password1
+
+# Encryption enabled without a key: compiles both frame helpers so the key
+# can be provisioned at runtime (zero-PSK noise or deprecated plaintext)
+api:
+  encryption:

tests/integration/fixtures/api_zero_psk_provisioning.yaml

@@ -0,0 +1,6 @@
+esphome:
+  name: zero-psk-provision-test
+host:
+api:
+  encryption:
+logger:

tests/integration/fixtures/api_zero_psk_provisioning_plaintext.yaml

@@ -0,0 +1,6 @@
+esphome:
+  name: zero-psk-plaintext-test
+host:
+api:
+  encryption:
+logger:

tests/integration/fixtures/api_zero_psk_provisioning_rejects.yaml

@@ -0,0 +1,6 @@
+esphome:
+  name: zero-psk-rejects-test
+host:
+api:
+  encryption:
+logger:

tests/integration/test_api_zero_psk_provisioning.py

@@ -0,0 +1,136 @@
+"""Integration tests for provisioning the encryption key over a zero-PSK connection.
+
+A device with `api: encryption:` but no key accepts Noise handshakes using the
+well-known all-zeros PSK. The ephemeral X25519 exchange protects the key from
+passive sniffing while it is provisioned; plaintext provisioning still works
+but is deprecated.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+
+from aioesphomeapi import InvalidEncryptionKeyAPIError, RequiresEncryptionAPIError
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+# The well-known provisioning PSK: base64 of 32 zero bytes
+ZERO_PSK = base64.b64encode(bytes(32)).decode()
+# A real key to provision
+NEW_KEY = base64.b64encode(b"n" * 32)
+# Time for the device to activate a newly saved key (100ms timer plus margin)
+KEY_ACTIVATION_DELAY = 0.5
+
+
+@pytest.fixture(autouse=True)
+def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
+    """Keep host preferences per-test so every run starts unprovisioned."""
+    monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs"))
+
+
+@pytest.mark.asyncio
+async def test_api_zero_psk_provisioning(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Provision a key over the zero-PSK channel and verify the switch to it."""
+    async with run_compiled(yaml_config):
+        # The unprovisioned device accepts the all-zeros PSK; the handshake's
+        # ephemeral-ephemeral DH encrypts everything that follows
+        async with api_client_connected(noise_psk=ZERO_PSK) as client:
+            device_info = await client.device_info()
+            assert device_info.name == "zero-psk-provision-test"
+            assert device_info.api_encryption_supported is True
+            assert device_info.api_encryption_provisionable is True
+
+            assert await client.noise_encryption_set_key(NEW_KEY) is True
+
+        # The device activates the new key shortly after responding
+        await asyncio.sleep(KEY_ACTIVATION_DELAY)
+
+        # The new key now works, and the device is no longer provisionable
+        async with api_client_connected(noise_psk=NEW_KEY.decode()) as client:
+            device_info = await client.device_info()
+            assert device_info.name == "zero-psk-provision-test"
+            assert device_info.api_encryption_provisionable is False
+
+        # The zero PSK no longer works
+        with pytest.raises(InvalidEncryptionKeyAPIError):
+            async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client:
+                await client.device_info()
+
+        # Plaintext no longer works
+        with pytest.raises(RequiresEncryptionAPIError):
+            async with api_client_connected(timeout=5) as client:
+                await client.device_info()
+
+
+@pytest.mark.asyncio
+async def test_api_zero_psk_provisioning_rejects(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Exercise the paths that must not provision a key."""
+    async with run_compiled(yaml_config):
+        # A wrong (non-zero) PSK fails against the zero provisioning PSK
+        with pytest.raises(InvalidEncryptionKeyAPIError):
+            async with api_client_connected(
+                noise_psk=base64.b64encode(b"w" * 32).decode(), timeout=5
+            ) as client:
+                await client.device_info()
+
+        # A plaintext client and a zero-PSK client can be connected at the
+        # same time while the device is unprovisioned
+        async with (
+            api_client_connected() as plaintext_client,
+            api_client_connected(noise_psk=ZERO_PSK) as noise_client,
+        ):
+            plaintext_info = await plaintext_client.device_info()
+            noise_info = await noise_client.device_info()
+            # Both transports advertise provisioning support so old and new
+            # clients can decide how to provision
+            assert plaintext_info.api_encryption_provisionable is True
+            assert noise_info.api_encryption_provisionable is True
+
+            # The all-zeros key is reserved as the provisioning PSK and is
+            # rejected on both transports
+            zero_key = base64.b64encode(bytes(32))
+            assert await noise_client.noise_encryption_set_key(zero_key) is False
+            assert await plaintext_client.noise_encryption_set_key(zero_key) is False
+
+        # Nothing was provisioned: the zero PSK still works
+        async with api_client_connected(noise_psk=ZERO_PSK) as client:
+            assert (await client.device_info()).api_encryption_provisionable is True
+
+
+@pytest.mark.asyncio
+async def test_api_zero_psk_provisioning_plaintext(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """The legacy plaintext provisioning path still works and warns."""
+    log_lines: list[str] = []
+    async with run_compiled(yaml_config, line_callback=log_lines.append):
+        async with api_client_connected() as client:
+            device_info = await client.device_info()
+            assert device_info.name == "zero-psk-plaintext-test"
+
+            assert await client.noise_encryption_set_key(NEW_KEY) is True
+
+        await asyncio.sleep(KEY_ACTIVATION_DELAY)
+
+        # The deprecation warning was logged
+        assert any("unencrypted connection" in line for line in log_lines)
+
+        # The new key works; the zero PSK does not
+        async with api_client_connected(noise_psk=NEW_KEY.decode()) as client:
+            assert (await client.device_info()).name == "zero-psk-plaintext-test"
+
+        with pytest.raises(InvalidEncryptionKeyAPIError):
+            async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client:
+                await client.device_info()