Fix EZSP and ASH protocol parsing/forwarding

esphome/components/zigbee_proxy/ash_protocol.cpp

@@ -33,6 +33,14 @@ static const uint16_t CRC_TABLE[256] = {
     0x1CE0, 0x0CC1, 0xEF1F, 0xFF3E, 0xCF5D, 0xDF7C, 0xAF9B, 0xBFBA, 0x8FD9, 0x9FF8, 0x6E17, 0x7E36, 0x4E55, 0x5E74,
     0x2E93, 0x3EB2, 0x0ED1, 0x1EF0};
 
+void ash_randomize(uint8_t *data, size_t length) {
+  uint8_t rand = 0x42;
+  for (size_t i = 0; i < length; i++) {
+    data[i] ^= rand;
+    rand = (rand & 0x01) ? static_cast<uint8_t>((rand >> 1) ^ 0xB8) : static_cast<uint8_t>(rand >> 1);
+  }
+}
+
 uint16_t ZigbeeProxy::calculate_crc_(const uint8_t *data, size_t length, uint16_t init) {
   uint16_t crc = init;
   for (size_t i = 0; i < length; i++) {
@@ -65,6 +73,22 @@ bool ZigbeeProxy::validate_frame_crc_() {
   return true;
 }
 
+bool ZigbeeProxy::handle_ack_num_(uint8_t ack_num) {
+  // ackNum means "I expect frame N next", i.e. everything up to N-1 arrived, so a
+  // pending frame numbered ack_num-1 has been acknowledged. Carried by DATA, ACK
+  // and NAK alike.
+  if (!this->tx_buffer_pending_ || ack_num != ((this->tx_pending_frame_num_ + 1) & ASH_MAX_SEQUENCE)) {
+    return false;
+  }
+
+  uint32_t rtt = millis() - this->ack_timer_start_;
+  this->update_adaptive_timeout_(rtt);
+  ESP_LOGV(TAG, "Frame %d acknowledged, RTT: %u ms", this->tx_pending_frame_num_, rtt);
+  this->clear_tx_buffer_();
+  this->drain_ncp_tx_queue_();
+  return true;
+}
+
 void ZigbeeProxy::parse_control_byte_(uint8_t control) {
   // Decode frame type based on bit patterns:
   // DATA:  0xxxxxxx (bit 7 = 0)
@@ -114,14 +138,10 @@ void ZigbeeProxy::parse_control_byte_(uint8_t control) {
   // Handle frame based on type
   switch (frame_type) {
     case AshFrameType::DATA: {
-      // Process the piggybacked ACK first: ackNum means "I expect frame N next" = "I received
-      // up to N-1", and it is valid regardless of the DATA frame's own sequence ordering
-      if (this->tx_buffer_pending_ && ack_num == ((this->tx_pending_frame_num_ + 1) & ASH_MAX_SEQUENCE)) {
-        uint32_t rtt = millis() - this->ack_timer_start_;
-        this->update_adaptive_timeout_(rtt);
-        this->clear_tx_buffer_();
-        ESP_LOGV(TAG, "ACK received (piggybacked in DATA), RTT: %u ms", rtt);
-        this->drain_ncp_tx_queue_();
+      // Process the piggybacked ACK first: ackNum is valid regardless of the DATA
+      // frame's own sequence ordering
+      if (this->handle_ack_num_(ack_num)) {
+        ESP_LOGV(TAG, "ACK received (piggybacked in DATA)");
       }
 
       // Check sequence number
@@ -152,8 +172,11 @@ void ZigbeeProxy::parse_control_byte_(uint8_t control) {
       size_t payload_length = this->rx_buffer_index_ > 3 ? this->rx_buffer_index_ - 3 : 0;
       const uint8_t *payload = this->rx_buffer_.data() + 1;
 
-      // During boot sequence, route to boot handler
+      // During boot sequence, route to boot handler. Frames consumed locally must
+      // be derandomized first; proxied frames below are passed through untouched
+      // so the client's own randomization survives end to end.
       if (this->boot_sequence_active_ && payload_length > 0) {
+        ash_randomize(this->rx_buffer_.data() + 1, payload_length);
         this->handle_boot_data_frame_(payload, payload_length);
       } else if (this->api_connection_ != nullptr && payload_length > 0) {
         // Forward EZSP payload to client via client-side ASH DATA frame
@@ -163,19 +186,19 @@ void ZigbeeProxy::parse_control_byte_(uint8_t control) {
     }
 
     case AshFrameType::ACK:
-      // Check if this ACKs our pending frame
-      // ackNum means "I expect frame N next" = "I received all frames up to N-1"
-      // So if ackNum == pending+1, our pending frame was acknowledged
-      if (this->tx_buffer_pending_ && ack_num == ((this->tx_pending_frame_num_ + 1) & ASH_MAX_SEQUENCE)) {
-        uint32_t rtt = millis() - this->ack_timer_start_;
-        this->update_adaptive_timeout_(rtt);
-        this->clear_tx_buffer_();
-        ESP_LOGV(TAG, "ACK received for frame %d, RTT: %u ms", this->tx_pending_frame_num_, rtt);
-        this->drain_ncp_tx_queue_();
-      }
+      this->handle_ack_num_(ack_num);
       break;
 
     case AshFrameType::NAK:
+      // A NAK carries valid ACK information like any other frame: ackNum is the
+      // next frame the NCP expects, so everything before it did arrive. Honour
+      // that first -- retransmitting an already-acknowledged frame otherwise
+      // burns all ASH_MAX_RETRIES and drops the link. bellows applies the same
+      // ACK handling to DATA, ACK and NAK alike.
+      if (this->handle_ack_num_(ack_num)) {
+        ESP_LOGW(TAG, "NAK received for frame %d (already acknowledged, not retransmitting)", ack_num);
+        break;
+      }
       ESP_LOGW(TAG, "NAK received for frame %d, retransmitting", ack_num);
       if (this->tx_buffer_pending_) {
         this->handle_retransmission_();
@@ -202,13 +225,28 @@ void ZigbeeProxy::parse_control_byte_(uint8_t control) {
 }
 
 bool ZigbeeProxy::parse_byte_(uint8_t byte) {
-  // ASH_CAN (0x1A) resets the parser state - discard any partial frame
   static constexpr uint8_t ASH_CAN_BYTE = 0x1A;
-  if (byte == ASH_CAN_BYTE) {
-    this->rx_buffer_index_ = 0;
-    this->escape_next_byte_ = false;
-    this->parsing_state_ = ParsingState::WAIT_FLAG_START;
-    return false;
+  static constexpr uint8_t ASH_XON_BYTE = 0x11;
+  static constexpr uint8_t ASH_XOFF_BYTE = 0x13;
+
+  // Reserved bytes are only meaningful when they appear *bare* in the stream, so
+  // they must be filtered here, before unescaping, and never afterwards. A frame
+  // whose control or data byte happens to equal one of them arrives stuffed (0x11
+  // is sent as 7D 31), and unescaping yields the real value -- so filtering after
+  // unescaping silently eats a valid control byte, shifting the whole frame by one
+  // and failing CRC on every retransmission. This mirrors bellows, which strips
+  // flow control from the raw buffer and only then unstuffs.
+  if (!this->escape_next_byte_) {
+    if (byte == ASH_CAN_BYTE) {
+      // Cancel: discard any partial frame
+      this->rx_buffer_index_ = 0;
+      this->parsing_state_ = ParsingState::WAIT_FLAG_START;
+      return false;
+    }
+    if (byte == ASH_XON_BYTE || byte == ASH_XOFF_BYTE) {
+      // Flow control: not part of any frame, may appear anywhere
+      return false;
+    }
   }
 
   switch (this->parsing_state_) {
@@ -222,11 +260,6 @@ bool ZigbeeProxy::parse_byte_(uint8_t byte) {
       if (this->escape_next_byte_) {
         byte ^= ASH_XOR_BYTE;
         this->escape_next_byte_ = false;
-        // After unescaping, check if it's a CAN byte (0x1A)
-        if (byte == ASH_CAN_BYTE) {
-          this->rx_buffer_index_ = 0;
-          return false;
-        }
       }
 
       if (byte == ASH_FLAG_BYTE) {
@@ -242,13 +275,12 @@ bool ZigbeeProxy::parse_byte_(uint8_t byte) {
         //   - ACK frames:  0x80-0x9F (bits 7-6 = 10, bit 5 = 0)
         //   - NAK frames:  0xA0-0xBF (bits 7-6 = 10, bit 5 = 1)
         //   - RST/RSTACK/ERROR: 0xC0-0xC2 (bits 7-6 = 11)
-        // Skip reserved bytes that cannot be valid control bytes
-        if (byte != 0x11 && byte != 0x13) {
-          this->rx_buffer_index_ = 0;
-          this->rx_buffer_[this->rx_buffer_index_++] = byte;
-          this->parsing_state_ = ParsingState::WAIT_DATA;
-          ESP_LOGV(TAG, "Frame start detected (control byte 0x%02X)", byte);
-        }
+        // Bare flow-control bytes were already filtered above, so anything
+        // reaching here is genuine frame content.
+        this->rx_buffer_index_ = 0;
+        this->rx_buffer_[this->rx_buffer_index_++] = byte;
+        this->parsing_state_ = ParsingState::WAIT_DATA;
+        ESP_LOGV(TAG, "Frame start detected (control byte 0x%02X)", byte);
       } else if ((byte & 0x80) != 0) {
         // Before connected, only accept control/management frames (bit 7 set)
         // This handles RSTACK (0xC1), ACK (0x8X), NAK (0xAX), ERROR (0xC2)

esphome/components/zigbee_proxy/ash_protocol.h

@@ -39,6 +39,18 @@ static constexpr uint8_t NCP_TX_QUEUE_SIZE = 2;
 // IEEE address size
 static constexpr size_t ZIGBEE_IEEE_ADDR_SIZE = 8;  // 64-bit IEEE address
 
+// ASH data randomization. The Data Field of every DATA frame is XORed with a
+// pseudo-random sequence (LFSR seeded at 0x42, polynomial 0xB8) before
+// transmission and again after reception; the operation is its own inverse.
+//
+// Proxied client traffic must NOT be passed through this: the client randomizes
+// and the NCP derandomizes, so payloads travel end to end untouched and the
+// proxy stays transparent. Apply it only to frames this component originates or
+// consumes itself, i.e. the boot-harvest EZSP commands and their responses.
+// Sending an unrandomized command makes the NCP derandomize it into garbage and
+// answer with an error frame that decodes as a plausible-looking wrong value.
+void ash_randomize(uint8_t *data, size_t length);
+
 // ASH Frame Types (encoded in control byte)
 // DATA format:    0ffrPPPP - bit 7=0, bits 6-4=frmNum, bit 3=reTx, bits 2-0=ackNum
 // ACK/NAK format: 10XnrPPP - bit 5 distinguishes ACK(0) from NAK(1)

esphome/components/zigbee_proxy/ezsp_commands.h

@@ -6,7 +6,7 @@
 namespace esphome::zigbee_proxy {
 
 // EZSP Protocol Versions
-static constexpr uint8_t EZSP_MIN_VERSION = 8;   // Minimum supported version
+static constexpr uint8_t EZSP_MIN_VERSION = 13;  // Minimum supported version
 static constexpr uint8_t EZSP_MAX_VERSION = 13;  // Maximum version we request
 
 // EZSP Frame Control bits
@@ -14,8 +14,18 @@ static constexpr uint8_t EZSP_FRAME_CONTROL_COMMAND = 0x00;   // Host to NCP
 static constexpr uint8_t EZSP_FRAME_CONTROL_RESPONSE = 0x80;  // NCP to Host
 static constexpr uint8_t EZSP_FRAME_CONTROL_CALLBACK = 0x90;  // Async callback from NCP
 
+// High byte of the 16-bit frame control, carrying frameFormatVersion = 1. Every
+// command after version negotiation must set this: omitting it leaves the NCP
+// reading the frame ID's low byte as frame_control_high, so the command is
+// discarded and the reply is an error frame rather than the expected response.
+static constexpr uint8_t EZSP_FRAME_CONTROL_EXTENDED = 0x01;
+
 // Legacy EZSP frame format (v4-v7): [sequence] [frame_control] [frame_id]
 // Extended EZSP frame format (v8+): [sequence] [frame_control_low] [frame_control_high] [frame_id_low] [frame_id_high]
+//
+// Only the `version` command and its response use the legacy format, because the
+// NCP starts in legacy mode and has not yet learned the negotiated version.
+// Everything after that is extended, with no per-NCP exceptions.
 
 // EZSP Frame IDs - Commands (host to NCP)
 static constexpr uint16_t EZSP_VERSION = 0x0000;                 // Version negotiation
@@ -23,6 +33,7 @@ static constexpr uint16_t EZSP_NETWORK_INIT = 0x0017;            // Initialize n
 static constexpr uint16_t EZSP_NETWORK_STATE = 0x0018;           // Get network state
 static constexpr uint16_t EZSP_GET_EUI64 = 0x0026;               // Get IEEE address
 static constexpr uint16_t EZSP_GET_NETWORK_PARAMETERS = 0x0028;  // Get network parameters
+static constexpr uint16_t EZSP_LAUNCH_STANDALONE_BOOTLOADER = 0x008F;  // Reboot into the bootloader
 
 // EZSP Frame IDs - Callbacks (NCP to host, async)
 static constexpr uint16_t EZSP_STACK_STATUS_HANDLER = 0x0019;  // Stack status callback
@@ -36,21 +47,29 @@ enum class EzspNetworkStatus : uint8_t {
   LEAVING_NETWORK = 0x04,
 };
 
-// Ember Status codes (subset)
-enum class EmberStatus : uint8_t {
-  SUCCESS = 0x00,
-  NETWORK_UP = 0x90,
-  NETWORK_DOWN = 0x91,
-  NOT_JOINED = 0x93,
+// Status codes (subset). EZSP v13+ / EmberZNet 8.x report sl_status_t, not the
+// legacy 8-bit EmberStatus, and the two disagree on every value that matters
+// here: legacy NETWORK_UP/NOT_JOINED were 0x90/0x93.
+enum class SlStatus : uint8_t {
+  OK = 0x00,
+  NETWORK_UP = 0x15,
+  NETWORK_DOWN = 0x16,
+  NOT_JOINED = 0x17,
 };
 
+// sl_status_t is 32-bit little-endian on the wire, so a status field occupies
+// four bytes even though every code used here fits in the first one.
+static constexpr size_t SL_STATUS_SIZE = 4;
+
 // Network parameters structure offsets (in getNetworkParameters response)
-// Response format: [status] [nodeType] [parameters...]
-// Parameters: [extendedPanId (8)] [panId (2)] [radioTxPower] [radioChannel] [joinMethod] ...
+// Response format: [status (4)] [nodeType (1)] [parameters (20)] = 25 bytes
+// Parameters: [extendedPanId (8)] [panId (2)] [radioTxPower (1)] [radioChannel (1)]
+//             [joinMethod (1)] [nwkManagerId (2)] [nwkUpdateId (1)] [channels (4)]
 static constexpr size_t NETWORK_PARAMS_STATUS_OFFSET = 0;
-static constexpr size_t NETWORK_PARAMS_NODE_TYPE_OFFSET = 1;
-static constexpr size_t NETWORK_PARAMS_EXT_PAN_ID_OFFSET = 2;
-static constexpr size_t NETWORK_PARAMS_PAN_ID_OFFSET = 10;
-static constexpr size_t NETWORK_PARAMS_CHANNEL_OFFSET = 13;
+static constexpr size_t NETWORK_PARAMS_NODE_TYPE_OFFSET = 4;
+static constexpr size_t NETWORK_PARAMS_EXT_PAN_ID_OFFSET = 5;
+static constexpr size_t NETWORK_PARAMS_PAN_ID_OFFSET = 13;
+static constexpr size_t NETWORK_PARAMS_CHANNEL_OFFSET = 16;
+static constexpr size_t NETWORK_PARAMS_RESPONSE_SIZE = 25;
 
 }  // namespace esphome::zigbee_proxy

esphome/components/zigbee_proxy/zigbee_proxy.cpp

@@ -208,6 +208,13 @@ void ZigbeeProxy::zigbee_proxy_frame(api::APIConnection *api_connection, const a
     return;
   }
 
+  if (this->in_raw_relay_()) {
+    // The NCP is in the bootloader, so the client is speaking its menu/XMODEM
+    // protocol rather than ASH. Pass it straight through.
+    this->write_array(msg.data, msg.data_len);
+    return;
+  }
+
   // Feed raw bytes into the client-side ASH parser
   for (size_t i = 0; i < msg.data_len; i++) {
     this->client_parse_byte_(msg.data[i]);
@@ -274,6 +281,7 @@ void ZigbeeProxy::set_usb_uart_channel(usb_uart::USBUartChannel *channel) {
 // ASH Protocol State Machine
 void ZigbeeProxy::reset_ash_protocol_() {
   ESP_LOGV(TAG, "Resetting ASH protocol");
+  this->raw_buffer_index_ = 0;
   this->ash_state_ = AshState::CONNECTING;
   this->tx_sequence_ = 0;
   this->rx_sequence_ = 0;
@@ -284,6 +292,11 @@ void ZigbeeProxy::reset_ash_protocol_() {
   this->setup_time_ = millis();
   this->boot_start_time_ = this->setup_time_;
 
+  // An NCP reset drops it back to legacy framing, so the extended-format version
+  // handshake has to be redone before any other command is accepted.
+  this->ezsp_version_ = 0;
+  this->ezsp_version_confirmed_ = false;
+
   // Start boot sequence
   this->boot_state_ = BootState::WAIT_RSTACK;
   this->boot_sequence_active_ = true;
@@ -521,8 +534,12 @@ void ZigbeeProxy::handle_retransmission_() {
 }
 
 // Boot-time NCP initialization sequence
-// Sequence: RST -> RSTACK -> version() -> getEui64() -> networkInit() -> stackStatus ->
-//           getNetworkParameters() -> RST -> RSTACK
+// Sequence: RST -> RSTACK -> version() -> networkInit() -> stackStatus ->
+//           getNetworkParameters() -> getEui64() -> RST -> RSTACK
+//
+// getEui64 comes last, after networkInit has brought the stack up: asking earlier
+// makes the NCP answer with error frame 0x0058 instead of the address. bellows
+// orders it the same way.
 
 void ZigbeeProxy::advance_boot_state_() {
   switch (this->boot_state_) {
@@ -558,16 +575,15 @@ void ZigbeeProxy::advance_boot_state_() {
 }
 
 void ZigbeeProxy::handle_boot_data_frame_(const uint8_t *data, size_t length) {
-  // EZSP frame format depends on negotiated version:
-  // Legacy (v4-v7): [sequence] [frame_control] [frame_id] [data...]  (3-byte header)
-  // Extended (v8+): [sequence] [frame_control_low] [frame_control_high] [frame_id_low] [frame_id_high] [data...]
+  // EZSP frame format is decided solely by whether version negotiation has
+  // completed, never by the frame's length:
+  // Legacy: [sequence] [frame_control] [frame_id] [data...]  (3-byte header)
+  // Extended (v13+): [sequence] [frame_control_low] [frame_control_high] [frame_id_low] [frame_id_high] [data...]
   //
-  // Note: Some NCPs may respond in legacy format for a few frames after version negotiation
-  // before fully switching to extended format. We handle this by falling back to legacy
-  // parsing if the frame is too short for extended format. This heuristic (extended iff
-  // negotiated v8+ AND frame >= 5 bytes) is validated against EFR32 EmberZNet 7.x NCPs;
-  // a legacy-format response of 5+ bytes would be misparsed, but such NCPs have not been
-  // observed in practice.
+  // Only the `version` response arrives in legacy format, before ezsp_version_ is
+  // set. Inferring the format from the length instead misparses a short extended
+  // frame -- an error reply, say -- as a legacy one, which then surfaces as a
+  // plausible but wrong payload rather than as a protocol error.
 
   if (length < 3) {
     ESP_LOGW(TAG, "Boot frame too short: %u bytes", length);
@@ -579,8 +595,12 @@ void ZigbeeProxy::handle_boot_data_frame_(const uint8_t *data, size_t length) {
   const uint8_t *payload;
   size_t payload_length;
 
-  // Determine format: prefer extended for v8+, but fall back to legacy if frame is too short
-  bool use_extended = (this->ezsp_version_ >= 8) && (length >= 5);
+  bool use_extended = this->ezsp_version_ >= EZSP_MIN_VERSION;
+
+  if (use_extended && length < 5) {
+    ESP_LOGW(TAG, "Extended EZSP frame too short: %u bytes", length);
+    return;
+  }
 
   if (use_extended) {
     frame_control = data[1];
@@ -619,7 +639,7 @@ void ZigbeeProxy::handle_boot_data_frame_(const uint8_t *data, size_t length) {
       if (frame_id == EZSP_STACK_STATUS_HANDLER && is_callback) {
         this->handle_stack_status_(payload, payload_length);
       } else if (frame_id == EZSP_NETWORK_INIT && is_response) {
-        // networkInit response contains EmberStatus
+        // networkInit response contains a 32-bit sl_status_t
         // Some NCPs proceed directly without stackStatusHandler callback
         if (payload_length >= 1) {
           uint8_t status = payload[0];
@@ -656,44 +676,52 @@ void ZigbeeProxy::send_ezsp_version_() {
       EZSP_MAX_VERSION             // Desired protocol version
   };
 
+  ash_randomize(cmd, sizeof(cmd));
   ESP_LOGV(TAG, "Sending EZSP version command (legacy format, requesting v%d)", EZSP_MAX_VERSION);
   this->send_data_frame_(cmd, sizeof(cmd), false);
 }
 
 void ZigbeeProxy::send_get_eui64_() {
-  // getEui64 command - use legacy format for boot sequence compatibility
+  // Extended format: version negotiation has completed, so the NCP now requires
+  // the 16-bit frame control and 16-bit frame ID.
   uint8_t cmd[] = {
-      this->ezsp_sequence_++,      // Sequence
-      EZSP_FRAME_CONTROL_COMMAND,  // Frame control
-      EZSP_GET_EUI64 & 0xFF        // Frame ID
+      this->ezsp_sequence_++,        // Sequence
+      EZSP_FRAME_CONTROL_COMMAND,    // Frame control (low)
+      EZSP_FRAME_CONTROL_EXTENDED,   // Frame control (high)
+      EZSP_GET_EUI64 & 0xFF,         // Frame ID (low)
+      (EZSP_GET_EUI64 >> 8) & 0xFF,  // Frame ID (high)
   };
-  ESP_LOGV(TAG, "Sending EZSP getEui64 command (legacy format)");
+  ash_randomize(cmd, sizeof(cmd));
+  ESP_LOGV(TAG, "Sending EZSP getEui64 command");
   this->send_data_frame_(cmd, sizeof(cmd), false);
 }
 
 void ZigbeeProxy::send_network_init_() {
-  // networkInit command - use legacy format for compatibility
-  // Some NCPs need a command or two in legacy format after version negotiation
-  // before fully switching to extended format.
   // networkInitStruct: [bitmask (2 bytes)] - use 0x0000 for default
   uint8_t cmd[] = {
-      this->ezsp_sequence_++,      // Sequence
-      EZSP_FRAME_CONTROL_COMMAND,  // Frame control
-      EZSP_NETWORK_INIT & 0xFF,    // Frame ID
-      0x00, 0x00                   // networkInitStruct bitmask (default)
+      this->ezsp_sequence_++,           // Sequence
+      EZSP_FRAME_CONTROL_COMMAND,       // Frame control (low)
+      EZSP_FRAME_CONTROL_EXTENDED,      // Frame control (high)
+      EZSP_NETWORK_INIT & 0xFF,         // Frame ID (low)
+      (EZSP_NETWORK_INIT >> 8) & 0xFF,  // Frame ID (high)
+      0x00,                             // networkInitStruct bitmask (default)
+      0x00,
   };
-  ESP_LOGV(TAG, "Sending EZSP networkInit command (legacy format)");
+  ash_randomize(cmd, sizeof(cmd));
+  ESP_LOGV(TAG, "Sending EZSP networkInit command");
   this->send_data_frame_(cmd, sizeof(cmd), false);
 }
 
 void ZigbeeProxy::send_get_network_params_() {
-  // getNetworkParameters command - use legacy format for boot sequence compatibility
   uint8_t cmd[] = {
-      this->ezsp_sequence_++,             // Sequence
-      EZSP_FRAME_CONTROL_COMMAND,         // Frame control
-      EZSP_GET_NETWORK_PARAMETERS & 0xFF  // Frame ID
+      this->ezsp_sequence_++,                      // Sequence
+      EZSP_FRAME_CONTROL_COMMAND,                  // Frame control (low)
+      EZSP_FRAME_CONTROL_EXTENDED,                 // Frame control (high)
+      EZSP_GET_NETWORK_PARAMETERS & 0xFF,          // Frame ID (low)
+      (EZSP_GET_NETWORK_PARAMETERS >> 8) & 0xFF,   // Frame ID (high)
   };
-  ESP_LOGV(TAG, "Sending EZSP getNetworkParameters command (legacy format)");
+  ash_randomize(cmd, sizeof(cmd));
+  ESP_LOGV(TAG, "Sending EZSP getNetworkParameters command");
   this->send_data_frame_(cmd, sizeof(cmd), false);
 }
 
@@ -723,7 +751,7 @@ void ZigbeeProxy::handle_version_response_(const uint8_t *data, size_t length) {
       // NCP accepted our requested version - treat as success
       ESP_LOGV(TAG, "NCP accepted EZSP v%d", ncp_version);
       this->ezsp_version_ = ncp_version;
-      this->boot_state_ = BootState::SEND_GET_EUI64;
+      this->boot_state_ = BootState::SEND_NETWORK_INIT;
       this->advance_boot_state_();
       return;
     }
@@ -740,6 +768,7 @@ void ZigbeeProxy::handle_version_response_(const uint8_t *data, size_t length) {
         0x00,                        // Frame ID (version)
         ncp_version                  // Use NCP's version
     };
+    ash_randomize(cmd, sizeof(cmd));
     ESP_LOGV(TAG, "Re-sending EZSP version command (requesting v%d)", ncp_version);
     this->send_data_frame_(cmd, sizeof(cmd), false);
     // Stay in WAIT_VERSION state
@@ -766,7 +795,29 @@ void ZigbeeProxy::handle_version_response_(const uint8_t *data, size_t length) {
     return;
   }
 
-  this->boot_state_ = BootState::SEND_GET_EUI64;
+  if (!this->ezsp_version_confirmed_) {
+    // Repeat `version` in the negotiated extended format. The NCP answers the
+    // initial legacy command with its own version, but keeps rejecting extended
+    // frames (error frame 0x0058) until the handshake is completed in that format.
+    this->ezsp_version_confirmed_ = true;
+    this->ezsp_requested_version_ = this->ezsp_version_;
+
+    uint8_t cmd[] = {
+        this->ezsp_sequence_++,       // Sequence
+        EZSP_FRAME_CONTROL_COMMAND,   // Frame control (low)
+        EZSP_FRAME_CONTROL_EXTENDED,  // Frame control (high)
+        EZSP_VERSION & 0xFF,          // Frame ID (low)
+        (EZSP_VERSION >> 8) & 0xFF,   // Frame ID (high)
+        this->ezsp_version_,          // desiredProtocolVersion
+    };
+    ash_randomize(cmd, sizeof(cmd));
+    ESP_LOGV(TAG, "Confirming EZSP v%d in extended format", this->ezsp_version_);
+    this->send_data_frame_(cmd, sizeof(cmd), false);
+    // Stay in WAIT_VERSION for the confirmation response
+    return;
+  }
+
+  this->boot_state_ = BootState::SEND_NETWORK_INIT;
   this->advance_boot_state_();
 }
 
@@ -777,8 +828,8 @@ void ZigbeeProxy::handle_eui64_response_(const uint8_t *data, size_t length) {
   } else {
     ESP_LOGW(TAG, "getEui64 response too short: %u bytes", length);
   }
-  // Proceed to networkInit either way; the proxy works without an IEEE address
-  this->boot_state_ = BootState::SEND_NETWORK_INIT;
+  // Harvest is done either way; the proxy works without an IEEE address
+  this->boot_state_ = BootState::SEND_FINAL_RST;
   this->advance_boot_state_();
 }
 
@@ -793,14 +844,16 @@ void ZigbeeProxy::handle_stack_status_(const uint8_t *data, size_t length) {
   ESP_LOGV(TAG, "Stack status: 0x%02X", status);
 
   // Check for network up status
-  if (status == static_cast<uint8_t>(EmberStatus::NETWORK_UP) || status == static_cast<uint8_t>(EmberStatus::SUCCESS)) {
+  if (status == static_cast<uint8_t>(SlStatus::NETWORK_UP) || status == static_cast<uint8_t>(SlStatus::OK)) {
     ESP_LOGV(TAG, "Network is up, querying parameters");
     this->boot_state_ = BootState::SEND_GET_NETWORK_PARAMS;
     this->advance_boot_state_();
-  } else if (status == static_cast<uint8_t>(EmberStatus::NOT_JOINED)) {
-    // No network configured - that's fine, we just won't have network info
+  } else if (status == static_cast<uint8_t>(SlStatus::NOT_JOINED)) {
+    // No network configured, so there are no parameters to read -- but still read
+    // the EUI64. It is a hardware address that exists regardless of membership, and
+    // it is what lets a client tell an unformed radio apart from an unreachable one.
     ESP_LOGD(TAG, "No network configured on NCP");
-    this->boot_state_ = BootState::SEND_FINAL_RST;
+    this->boot_state_ = BootState::SEND_GET_EUI64;
     this->advance_boot_state_();
   } else {
     ESP_LOGW(TAG, "Unexpected stack status: 0x%02X, continuing anyway", status);
@@ -814,21 +867,21 @@ void ZigbeeProxy::handle_network_params_response_(const uint8_t *data, size_t le
   // getNetworkParameters response:
   // [status] [nodeType] [extendedPanId (8)] [panId (2)] [radioTxPower] [radioChannel] ...
   // A 1-byte response (status only) is normal when the NCP has no network configured.
-  if (length < 14) {
-    if (length == 1) {
+  if (length < NETWORK_PARAMS_RESPONSE_SIZE) {
+    if (length <= SL_STATUS_SIZE) {
       ESP_LOGD(TAG, "NCP has no network configured (status=0x%02X)", data[0]);
     } else {
       ESP_LOGW(TAG, "getNetworkParameters response unexpected length: %u bytes", length);
     }
-    this->boot_state_ = BootState::SEND_FINAL_RST;
+    this->boot_state_ = BootState::SEND_GET_EUI64;
     this->advance_boot_state_();
     return;
   }
 
   uint8_t status = data[NETWORK_PARAMS_STATUS_OFFSET];
-  if (status != static_cast<uint8_t>(EmberStatus::SUCCESS)) {
+  if (status != static_cast<uint8_t>(SlStatus::OK)) {
     ESP_LOGW(TAG, "getNetworkParameters failed with status: 0x%02X", status);
-    this->boot_state_ = BootState::SEND_FINAL_RST;
+    this->boot_state_ = BootState::SEND_GET_EUI64;
     this->advance_boot_state_();
     return;
   }
@@ -857,8 +910,8 @@ void ZigbeeProxy::handle_network_params_response_(const uint8_t *data, size_t le
            this->network_info_.extended_pan_id[1], this->network_info_.extended_pan_id[0], this->network_info_.pan_id,
            this->network_info_.channel);
 
-  // Now reset NCP to clean state
-  this->boot_state_ = BootState::SEND_FINAL_RST;
+  // The stack is up, so the EUI64 can finally be read
+  this->boot_state_ = BootState::SEND_GET_EUI64;
   this->advance_boot_state_();
 }
 
@@ -959,29 +1012,99 @@ void ZigbeeProxy::check_wifi_zigbee_conflict_() {
 #endif
 }
 
-// Bootloader detection - fed consecutive raw byte pairs while the ASH link is not CONNECTED
-// (bootloader output only ever appears in place of the RSTACK after a reset)
+// Bootloader detection, fed consecutive raw byte pairs. The Gecko bootloader speaks
+// neither ASH nor EZSP, so once detected the NCP link is relayed verbatim in both
+// directions (see process_uart_slow_ and zigbee_proxy_frame) and normal ASH parsing
+// is suspended -- a flasher needs the raw menu and XMODEM streams to reach the client.
 void ZigbeeProxy::check_bootloader_mode_(uint8_t prev_byte, uint8_t byte) {
-  // Check for Silicon Labs bootloader menu prompt (0xC1 0x0D)
+  // An ASH RSTACK means the application is running again, so ASH resumes. This is the
+  // only way out of the relay: while it is active nothing else parses the NCP stream.
+  if (prev_byte == ASH_FLAG_BYTE && byte == static_cast<uint8_t>(AshFrameType::RSTACK)) {
+    if (this->in_raw_relay_()) {
+      ESP_LOGI(TAG, "NCP returned to application mode, resuming ASH");
+      this->bootloader_state_ = BootloaderState::NORMAL;
+      // Clear the failure that put us in the relay, so the RSTACK below is parsed
+      // as ASH and the link comes back up on its own.
+      this->ash_state_ = AshState::CONNECTING;
+      this->parsing_state_ = ParsingState::WAIT_FLAG_START;
+      this->raw_buffer_index_ = 0;
+    }
+    return;
+  }
+
+  // Silicon Labs bootloader menu prompt (0xC1 0x0D)
   if (prev_byte == 0xC1 && byte == 0x0D) {
     if (this->bootloader_state_ != BootloaderState::MENU) {
-      ESP_LOGW(TAG, "NCP in bootloader menu mode detected\n"
-                    "  Please flash NCP firmware or power cycle the device");
+      ESP_LOGW(TAG, "NCP in bootloader menu mode, relaying raw bytes to client");
       this->bootloader_state_ = BootloaderState::MENU;
     }
     return;
   }
 
-  // Check for upload begin (0x43)
-  if (byte == 0x43) {
-    if (this->bootloader_state_ != BootloaderState::DETECTED) {
-      ESP_LOGW(TAG, "NCP bootloader upload mode detected");
-      this->bootloader_state_ = BootloaderState::DETECTED;
-    }
+  // XMODEM-CRC poll ('C'), only meaningful once the bootloader is already talking:
+  // treating it as an entry condition on its own would false-positive on ASH payload.
+  if (byte == 0x43 && this->bootloader_state_ == BootloaderState::MENU) {
+    ESP_LOGW(TAG, "NCP bootloader upload mode detected");
+    this->bootloader_state_ = BootloaderState::DETECTED;
     return;
   }
 }
 
+// The NCP is not speaking ASH: either it is in its bootloader, or the ASH link gave
+// up entirely. Relaying raw in both cases is what lets a flasher reach -- and recover
+// -- a device that is already sitting in the bootloader when the proxy starts.
+bool ZigbeeProxy::in_raw_relay_() const {
+  return this->bootloader_state_ != BootloaderState::NORMAL || this->ash_state_ == AshState::FAILED;
+}
+
+// True if a client-bound EZSP payload is `launchStandaloneBootloader`. The payload is
+// still randomized here (it is forwarded to the NCP untouched), so only the frame ID
+// bytes are unmasked, using the fixed prefix of the ASH pseudo-random sequence.
+bool ZigbeeProxy::is_launch_bootloader_command_(const uint8_t *payload, size_t length) {
+  // Extended EZSP header: [seq] [fc_lo] [fc_hi] [id_lo] [id_hi]
+  if (length < 5) {
+    return false;
+  }
+
+  uint8_t header[5];
+  memcpy(header, payload, sizeof(header));
+  ash_randomize(header, sizeof(header));
+
+  // Commands only; a response or callback carrying the same ID is the NCP's reply
+  if ((header[1] & (EZSP_FRAME_CONTROL_RESPONSE | EZSP_FRAME_CONTROL_CALLBACK)) != 0) {
+    return false;
+  }
+
+  uint16_t frame_id = header[3] | (static_cast<uint16_t>(header[4]) << 8);
+  return frame_id == EZSP_LAUNCH_STANDALONE_BOOTLOADER;
+}
+
+// Relay raw NCP bytes to the client, batching them so an XMODEM transfer does not
+// become one API message per byte. Flushed when the UART drains or the buffer fills.
+void ZigbeeProxy::queue_raw_to_client_(uint8_t byte) {
+  if (this->api_connection_ == nullptr) {
+    return;
+  }
+
+  this->raw_buffer_[this->raw_buffer_index_++] = byte;
+  if (this->raw_buffer_index_ >= this->raw_buffer_.size()) {
+    this->flush_raw_to_client_();
+  }
+}
+
+void ZigbeeProxy::flush_raw_to_client_() {
+  if (this->raw_buffer_index_ == 0) {
+    return;
+  }
+
+  // Best effort: the bootloader protocols carry their own retries, and there is no
+  // ASH session to resynchronize here.
+  if (!this->send_to_client_(this->raw_buffer_.data(), this->raw_buffer_index_)) {
+    ESP_LOGW(TAG, "Dropped %u raw bytes to client (API TX buffer full)", this->raw_buffer_index_);
+  }
+  this->raw_buffer_index_ = 0;
+}
+
 // UART processing (precondition: available() > 0, see inline process_uart_ in the header)
 void ZigbeeProxy::process_uart_slow_() {
   do {
@@ -993,17 +1116,23 @@ void ZigbeeProxy::process_uart_slow_() {
     // Verbose logging for debugging (ESP_LOGV already checks log level)
     ESP_LOGV(TAG, "RX: 0x%02X", byte);
 
-    if (this->ash_state_ != AshState::CONNECTED) {
-      this->check_bootloader_mode_(this->last_rx_byte_, byte);
-      this->last_rx_byte_ = byte;
-    } else if (this->bootloader_state_ != BootloaderState::NORMAL) {
-      // Normal traffic while connected clears any stale bootloader detection
-      ESP_LOGV(TAG, "NCP returned to normal operation");
-      this->bootloader_state_ = BootloaderState::NORMAL;
+    // Runs unconditionally: a client can launch the bootloader over EZSP while the
+    // ASH link is up, so detection cannot be gated on the link being down.
+    this->check_bootloader_mode_(this->last_rx_byte_, byte);
+    this->last_rx_byte_ = byte;
+
+    if (this->in_raw_relay_()) {
+      // Bootloader traffic is not ASH. Relay it untouched and keep the parser out of
+      // it: feeding menu text or XMODEM to parse_byte_ would fail CRC and NAK the NCP
+      // mid-transfer.
+      this->queue_raw_to_client_(byte);
+      continue;
     }
 
     this->parse_byte_(byte);
   } while (this->available());
+
+  this->flush_raw_to_client_();
 }
 
 // ==================== Client-side ASH session ====================
@@ -1135,13 +1264,22 @@ void ZigbeeProxy::forward_ncp_error_to_client_(const uint8_t *data, size_t lengt
 
 void ZigbeeProxy::client_parse_byte_(uint8_t byte) {
   static constexpr uint8_t ASH_CAN_BYTE = 0x1A;
+  static constexpr uint8_t ASH_XON_BYTE = 0x11;
+  static constexpr uint8_t ASH_XOFF_BYTE = 0x13;
 
-  // CAN byte resets parser
-  if (byte == ASH_CAN_BYTE) {
-    this->client_rx_buffer_index_ = 0;
-    this->client_escape_next_byte_ = false;
-    this->client_parsing_state_ = ParsingState::WAIT_FLAG_START;
-    return;
+  // Reserved bytes count only when bare, never as the second half of an escape
+  // sequence (see the matching comment in parse_byte_).
+  if (!this->client_escape_next_byte_) {
+    if (byte == ASH_CAN_BYTE) {
+      // Cancel: discard any partial frame
+      this->client_rx_buffer_index_ = 0;
+      this->client_parsing_state_ = ParsingState::WAIT_FLAG_START;
+      return;
+    }
+    if (byte == ASH_XON_BYTE || byte == ASH_XOFF_BYTE) {
+      // Flow control: not part of any frame
+      return;
+    }
   }
 
   switch (this->client_parsing_state_) {
@@ -1275,10 +1413,21 @@ void ZigbeeProxy::client_parse_control_byte_(uint8_t control) {
         // Forward EZSP payload to NCP via right-side ASH; NAK without consuming if the
         // NCP link is down or the TX queue is full so the client retransmits
         ESP_LOGV(TAG, "Client DATA → NCP, EZSP payload %u bytes", payload_length);
+        bool launching_bootloader = is_launch_bootloader_command_(payload, payload_length);
         if (!this->send_frame(payload, payload_length)) {
           this->client_send_nak_frame_(this->client_rx_sequence_);
           return;
         }
+
+        if (launching_bootloader) {
+          // The NCP is about to reboot into its bootloader, which speaks neither ASH
+          // nor EZSP. Switch to the raw relay now: waiting to recognize bootloader
+          // output would deadlock, since that output only appears once the client's
+          // (non-ASH) bytes reach the NCP, and those are exactly what the relay
+          // carries. Cleared again when an ASH RSTACK shows the app is back.
+          ESP_LOGI(TAG, "Client launched NCP bootloader, relaying raw bytes");
+          this->bootloader_state_ = BootloaderState::MENU;
+        }
       }
 
       // Accepted: advance the sequence and ACK immediately rather than relying on the

esphome/components/zigbee_proxy/zigbee_proxy.h

@@ -113,6 +113,9 @@ class ZigbeeProxy : public uart::UARTDevice, public Component {
   void send_rst_frame_();
   void handle_rstack_frame_(const uint8_t *data, size_t length);
   void handle_error_frame_(const uint8_t *data, size_t length);
+  // Applies a frame's ackNum to the pending TX frame. Returns true if it
+  // acknowledged one. Valid on DATA, ACK and NAK frames alike.
+  bool handle_ack_num_(uint8_t ack_num);
   bool send_ack_frame_(uint8_t ack_num);
   bool send_nak_frame_(uint8_t ack_num);
   bool send_data_frame_(const uint8_t *data, size_t length, bool retransmit = false);
@@ -166,8 +169,13 @@ class ZigbeeProxy : public uart::UARTDevice, public Component {
   // WiFi/Zigbee channel conflict detection
   void check_wifi_zigbee_conflict_();
 
-  // Bootloader detection (fed consecutive raw byte pairs while not CONNECTED)
+  // Bootloader detection (fed consecutive raw byte pairs)
   void check_bootloader_mode_(uint8_t prev_byte, uint8_t byte);
+  // Raw NCP <-> client relay used while the NCP is in its bootloader
+  bool in_raw_relay_() const;
+  static bool is_launch_bootloader_command_(const uint8_t *payload, size_t length);
+  void queue_raw_to_client_(uint8_t byte);
+  void flush_raw_to_client_();
 
   // UART processing
   // Inline fast-path: UART::available() is cheap (ring-buffer head/tail compare on most
@@ -282,11 +290,18 @@ class ZigbeeProxy : public uart::UARTDevice, public Component {
   uint8_t ezsp_version_{0};            // NCP's EZSP protocol version
   uint8_t ezsp_sequence_{0};           // EZSP frame sequence number
   uint8_t ezsp_requested_version_{0};  // Version we last requested (for re-negotiation)
+  // The NCP keeps using legacy framing until `version` is repeated in the
+  // negotiated (extended) format; until then it rejects every extended command
+  // with frame ID 0x0058. Tracks whether that second handshake has happened.
+  bool ezsp_version_confirmed_{false};
 
   bool tx_buffer_pending_{false};        // True if waiting for ACK from NCP
   bool escape_next_byte_{false};         // True if next NCP byte should be unescaped
   bool client_escape_next_byte_{false};  // True if next client byte should be unescaped
   bool network_info_ready_{false};       // True when network info retrieved
+
+  std::array<uint8_t, 128> raw_buffer_{};  // Bootloader relay batching buffer
+  size_t raw_buffer_index_{0};
   bool boot_sequence_active_{false};     // True during boot-time init
 };