Share the noise wire constants and reject formatter

esphome/components/api/api_frame_helper_noise.cpp

@@ -20,6 +20,12 @@ namespace esphome::api {
 
 using noise::noise_err_to_logstr;
 
+// api_frame_helper.h keeps its own MAX_HANDSHAKE_SIZE because that header is
+// also compiled in plaintext-only builds without the noise component; keep
+// the two definitions from drifting apart.
+static_assert(MAX_HANDSHAKE_SIZE == noise::MAX_HANDSHAKE_SIZE,
+              "api and noise component handshake size limits must match");
+
 static const char *const TAG = "api.noise";
 #ifdef USE_ESP8266
 static constexpr char PROLOGUE_INIT[] PROGMEM = "NoiseAPIInit";
@@ -158,9 +164,9 @@ APIError APINoiseFrameHelper::loop() {
  */
 APIError APINoiseFrameHelper::try_read_frame_() {
   // read header
-  if (rx_header_buf_len_ < 3) {
+  if (rx_header_buf_len_ < noise::FRAME_HEADER_SIZE) {
     // no header information yet
-    uint8_t to_read = 3 - rx_header_buf_len_;
+    uint8_t to_read = static_cast<uint8_t>(noise::FRAME_HEADER_SIZE) - rx_header_buf_len_;
     ssize_t received = this->socket_->read(&rx_header_buf_[rx_header_buf_len_], to_read);
     APIError err = handle_socket_read_result_(received);
     if (err != APIError::OK) {
@@ -172,7 +178,7 @@ APIError APINoiseFrameHelper::try_read_frame_() {
       return APIError::WOULD_BLOCK;
     }
 
-    if (rx_header_buf_[0] != 0x01) {
+    if (rx_header_buf_[0] != noise::FRAME_INDICATOR) {
       state_ = State::FAILED;
       HELPER_LOG("Bad indicator byte %u", rx_header_buf_[0]);
       return APIError::BAD_INDICATOR;
@@ -332,7 +338,7 @@ APIError APINoiseFrameHelper::state_action_handshake_read_() {
   if (this->rx_buf_.empty()) {
     this->send_explicit_handshake_reject_(LOG_STR("Empty handshake message"));
     return APIError::BAD_HANDSHAKE_ERROR_BYTE;
-  } else if (this->rx_buf_[0] != 0x00) {
+  } else if (this->rx_buf_[0] != noise::HANDSHAKE_STATUS_OK) {
     HELPER_LOG("Bad handshake error byte: %u", this->rx_buf_[0]);
     this->send_explicit_handshake_reject_(LOG_STR("Bad handshake error byte"));
     return APIError::BAD_HANDSHAKE_ERROR_BYTE;
@@ -373,27 +379,7 @@ APIError APINoiseFrameHelper::state_action_handshake_write_() {
 void APINoiseFrameHelper::send_explicit_handshake_reject_(const LogString *reason) {
   // Max reject message: "Bad handshake packet len" (24) + 1 (failure byte) = 25 bytes
   uint8_t data[32];
-  data[0] = 0x01;  // failure
-
-#ifdef USE_STORE_LOG_STR_IN_FLASH
-  // On ESP8266 with flash strings, we need to use PROGMEM-aware functions
-  size_t reason_len = strlen_P(reinterpret_cast<PGM_P>(reason));
-  reason_len = std::min(reason_len, sizeof(data) - 1);
-  if (reason_len > 0) {
-    memcpy_P(data + 1, reinterpret_cast<PGM_P>(reason), reason_len);
-  }
-#else
-  // Normal memory access
-  const char *reason_str = LOG_STR_ARG(reason);
-  size_t reason_len = strlen(reason_str);
-  reason_len = std::min(reason_len, sizeof(data) - 1);
-  if (reason_len > 0) {
-    // NOLINTNEXTLINE(bugprone-not-null-terminated-result) - binary protocol, not a C string
-    std::memcpy(data + 1, reason_str, reason_len);
-  }
-#endif
-
-  size_t data_size = reason_len + 1;
+  size_t data_size = noise::format_reject_payload(data, sizeof(data), reason);
 
   // temporarily remove failed state
   auto orig_state = state_;
@@ -456,12 +442,10 @@ APIError APINoiseFrameHelper::read_packet(ReadPacketBuffer *buffer) {
 // Returns APIError::OK on success.
 APIError APINoiseFrameHelper::encrypt_noise_message_(uint8_t *buf_start, uint16_t payload_size, uint8_t message_type,
                                                      uint16_t &encrypted_len_out) {
-  // Write noise header
-  buf_start[0] = 0x01;  // indicator
-  // buf_start[1], buf_start[2] to be set after encryption
+  // The noise frame header is written after encryption, when the size is known
 
   // Write message header (to be encrypted)
-  constexpr uint8_t msg_offset = 3;
+  constexpr uint8_t msg_offset = noise::FRAME_HEADER_SIZE;
   buf_start[msg_offset] = static_cast<uint8_t>(message_type >> 8);      // type high byte
   buf_start[msg_offset + 1] = static_cast<uint8_t>(message_type);       // type low byte
   buf_start[msg_offset + 2] = static_cast<uint8_t>(payload_size >> 8);  // data_len high byte
@@ -479,11 +463,10 @@ APIError APINoiseFrameHelper::encrypt_noise_message_(uint8_t *buf_start, uint16_
   if (aerr != APIError::OK)
     return aerr;
 
-  // Fill in the encrypted size
-  buf_start[1] = static_cast<uint8_t>(mbuf.size >> 8);
-  buf_start[2] = static_cast<uint8_t>(mbuf.size);
+  // Fill in the frame header now that the encrypted size is known
+  noise::write_frame_header(buf_start, static_cast<uint16_t>(mbuf.size));
 
-  encrypted_len_out = static_cast<uint16_t>(3 + mbuf.size);  // indicator + size + encrypted data
+  encrypted_len_out = static_cast<uint16_t>(noise::FRAME_HEADER_SIZE + mbuf.size);
   return APIError::OK;
 }
 
@@ -533,9 +516,7 @@ APIError APINoiseFrameHelper::write_protobuf_messages(ProtoWriteBuffer buffer, s
 
 APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) {
   uint8_t header[3];
-  header[0] = 0x01;  // indicator
-  header[1] = (uint8_t) (len >> 8);
-  header[2] = (uint8_t) len;
+  noise::write_frame_header(header, len);
 
   if (len == 0) {
     return this->write_raw_buf_(header, 3);

esphome/components/api/api_frame_helper_noise.h

@@ -14,7 +14,7 @@ class APINoiseFrameHelper final : public APIFrameHelper {
   // Pos 1-2: encrypted payload size (16-bit big-endian)
   // Pos 3-6: encrypted type (16-bit) + data_len (16-bit)
   // Pos 7+: actual payload data
-  static constexpr uint8_t HEADER_PADDING = 1 + 2 + 2 + 2;  // indicator + size + type + data_len
+  static constexpr uint8_t HEADER_PADDING = noise::FRAME_HEADER_SIZE + 2 + 2;  // frame header + type + data_len
 
   APINoiseFrameHelper(std::unique_ptr<socket::Socket> socket, APINoiseContext &ctx)
       : APIFrameHelper(std::move(socket)), ctx_(ctx) {
@@ -67,7 +67,7 @@ class APINoiseFrameHelper final : public APIFrameHelper {
   // Fixed-size header buffer for noise protocol:
   // 1 byte for indicator + 2 bytes for message size (16-bit value, not varint)
   // Note: Maximum message size is UINT16_MAX (65535), with a limit of 128 bytes during handshake phase
-  uint8_t rx_header_buf_[3];
+  uint8_t rx_header_buf_[noise::FRAME_HEADER_SIZE];
   uint8_t rx_header_buf_len_ = 0;
   // 4 bytes total, no padding
 };

esphome/components/noise/noise.cpp

@@ -4,8 +4,15 @@
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
+#include <algorithm>
+#include <cstring>
+
 #include <noise/protocol.h>
 
+#ifdef USE_ESP8266
+#include <pgmspace.h>
+#endif
+
 namespace esphome::noise {
 
 static const char *const TAG = "noise";
@@ -48,6 +55,34 @@ const LogString *noise_err_to_logstr(int err) {
   return LOG_STR("UNKNOWN");
 }
 
+const LogString *reject_reason_for(int err) {
+  return err == NOISE_ERROR_MAC_FAILURE ? LOG_STR("Handshake MAC failure") : LOG_STR("Handshake error");
+}
+
+size_t format_reject_payload(uint8_t *buf, size_t capacity, const LogString *reason) {
+  if (capacity == 0) {
+    return 0;
+  }
+  buf[0] = HANDSHAKE_STATUS_REJECT;
+#ifdef USE_STORE_LOG_STR_IN_FLASH
+  // On ESP8266 with flash strings, we need to use PROGMEM-aware functions
+  size_t reason_len = strlen_P(reinterpret_cast<PGM_P>(reason));
+  reason_len = std::min(reason_len, capacity - 1);
+  if (reason_len > 0) {
+    memcpy_P(buf + 1, reinterpret_cast<PGM_P>(reason), reason_len);
+  }
+#else
+  const char *reason_str = LOG_STR_ARG(reason);
+  size_t reason_len = strlen(reason_str);
+  reason_len = std::min(reason_len, capacity - 1);
+  if (reason_len > 0) {
+    // NOLINTNEXTLINE(bugprone-not-null-terminated-result) - binary protocol, not a C string
+    std::memcpy(buf + 1, reason_str, reason_len);
+  }
+#endif
+  return reason_len + 1;
+}
+
 extern "C" {
 // declare how noise generates random bytes (here with a good HWRNG based on the RF system)
 void noise_rand_bytes(void *output, size_t len) {

esphome/components/noise/noise.h

@@ -38,5 +38,31 @@ class NoiseContext {
 /// Convert a noise error code to a readable error
 const LogString *noise_err_to_logstr(int err);
 
+// Shared wire format for the noise transports (api and ota): every frame is
+// FRAME_INDICATOR, a 16-bit big-endian payload length, then the payload.
+// Handshake payloads start with a status byte; transport payloads end with
+// the ChaCha20-Poly1305 MAC.
+static constexpr uint8_t FRAME_INDICATOR = 0x01;
+static constexpr size_t FRAME_HEADER_SIZE = 3;
+static constexpr size_t MAC_SIZE = 16;
+static constexpr size_t MAX_HANDSHAKE_SIZE = 128;
+static constexpr uint8_t HANDSHAKE_STATUS_OK = 0x00;
+static constexpr uint8_t HANDSHAKE_STATUS_REJECT = 0x01;
+
+inline void write_frame_header(uint8_t *buf, uint16_t payload_len) {
+  buf[0] = FRAME_INDICATOR;
+  buf[1] = (uint8_t) (payload_len >> 8);
+  buf[2] = (uint8_t) payload_len;
+}
+
+/// Fill buf with a handshake reject payload (status byte plus the reason
+/// text, PROGMEM aware); returns the payload length. buf needs capacity for
+/// the status byte plus the truncated reason.
+size_t format_reject_payload(uint8_t *buf, size_t capacity, const LogString *reason);
+
+/// Reject reason for a failed handshake read. The MAC failure string is a
+/// wire contract: clients match it to report a wrong key.
+const LogString *reject_reason_for(int err);
+
 }  // namespace esphome::noise
 #endif  // USE_NOISE