[serial_proxy] Acknowledge set_mode requests

Follow the acknowledgement pattern from #18312: set_mode now returns a
SerialProxyResult and the handler answers with SerialProxyRequestResponse
(type SET_MODE). This matters most for a client switching to RAW before
flashing firmware through the port: without an ack, a refused request
(another client holds the port) is silently dropped and the client cannot
tell that protocol bytes may still be injected.

esphome/components/api/api.proto

@@ -2803,6 +2803,7 @@ enum SerialProxyRequestType {
   // error the device answers with INVALID_ARGUMENT.
   SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3;      // Acknowledges a SerialProxyConfigureRequest
   SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4; // Acknowledges a SerialProxySetModemPinsRequest
+  SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5;       // Acknowledges a SerialProxySetModeRequest (since API 1.17)
 }
 
 enum SerialProxyStatus {

esphome/components/api/api_connection.cpp

@@ -1661,6 +1661,7 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) {
       break;
     case enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE:
     case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS:
+    case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE:
       // Response-only discriminators; never valid in a request
       ESP_LOGW(TAG, "Response-only serial proxy request type: %" PRIu32, static_cast<uint32_t>(msg.type));
       status = enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT;
@@ -1677,9 +1678,13 @@ void APIConnection::on_serial_proxy_set_mode_request(const SerialProxySetModeReq
   auto &proxies = App.get_serial_proxies();
   if (msg.instance >= proxies.size()) {
     ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance);
+    send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE,
+                          enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT);
     return;
   }
-  proxies[msg.instance]->set_mode(this, msg.mode);
+  serial_proxy::SerialProxyResult result = proxies[msg.instance]->set_mode(this, msg.mode);
+  send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE,
+                        serial_proxy_result_to_status(result));
 }
 
 void APIConnection::send_serial_proxy_data(const SerialProxyDataReceived &msg) {

esphome/components/api/api_pb2.h

@@ -356,6 +356,7 @@ enum SerialProxyRequestType : uint32_t {
   SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2,
   SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3,
   SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4,
+  SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5,
 };
 enum SerialProxyStatus : uint32_t {
   SERIAL_PROXY_STATUS_OK = 0,

esphome/components/api/api_pb2_dump.cpp

@@ -854,6 +854,8 @@ template<> const char *proto_enum_to_string<enums::SerialProxyRequestType>(enums
       return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_CONFIGURE");
     case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS:
       return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS");
+    case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE:
+      return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODE");
     default:
       return ESPHOME_PSTR("UNKNOWN");
   }

esphome/components/serial_proxy/serial_proxy.cpp

@@ -231,11 +231,11 @@ SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uin
   return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
 }
 
-void SerialProxy::set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) {
+SerialProxyResult SerialProxy::set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) {
 #ifdef USE_API
   if (this->port_claimed_by_other_(api_connection)) {
     ESP_LOGW(TAG, "Ignoring mode request from client without port access [%" PRIu32 "]", this->instance_index_);
-    return;
+    return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE;
   }
 #endif
   ESP_LOGD(TAG, "Serial proxy [%" PRIu32 "] mode set to %s", this->instance_index_,
@@ -252,6 +252,7 @@ void SerialProxy::set_mode(api::APIConnection *api_connection, api::enums::Seria
     this->tap_->on_protocol_disabled();
   }
 #endif
+  return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
 }
 
 void SerialProxy::write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {

esphome/components/serial_proxy/serial_proxy.h

@@ -114,7 +114,7 @@ class SerialProxy final : public uart::UARTDevice, public Component {
   api::enums::SerialProxyMode get_mode() const { return this->mode_; }
 
   /// Handle a mode change requested by an API client
-  void set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode);
+  SerialProxyResult set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode);
 
   /// Configure UART parameters and apply them
   /// @param api_connection The API connection requesting the change

tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h

@@ -40,7 +40,9 @@ class SerialProxy {
     return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
   }
   void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {}
-  void set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) {}
+  SerialProxyResult set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) {
+    return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
+  }
   SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) {
     return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
   }