[serial_proxy] Make set_mode acknowledgements report the real outcome

- Refuse PROTOCOL with NOT_SUPPORTED when the port has no tap, so a
  client cannot mistake a plain pipe for an active tap
- Skip tap_pump() when neither the tap nor a subscriber would receive
  the bytes, instead of draining the FIFO into the void
- Rename the client-facing overload to set_mode_from_client, matching
  write_from_client
- Document that PORT_IN_USE also covers callers that never subscribed,
  and that the YAML mode applies only until the first session ends

esphome/components/api/api.proto

@@ -2851,7 +2851,9 @@ enum SerialProxyMode {
   SERIAL_PROXY_MODE_PROTOCOL = 1;
 }
 
-// Only the subscribed client may change the mode; others are refused with PORT_IN_USE.
+// Only the subscribed client may change the mode; any other caller -- including one that
+// never subscribed -- is refused with PORT_IN_USE. PROTOCOL is refused with NOT_SUPPORTED
+// when the port has no protocol-aware tap configured.
 message SerialProxySetModeRequest {
   option (id) = 152;
   option (source) = SOURCE_CLIENT;

esphome/components/api/api_connection.cpp

@@ -1682,7 +1682,7 @@ void APIConnection::on_serial_proxy_set_mode_request(const SerialProxySetModeReq
                           enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT);
     return;
   }
-  serial_proxy::SerialProxyResult result = proxies[msg.instance]->set_mode(this, msg.mode);
+  serial_proxy::SerialProxyResult result = proxies[msg.instance]->set_mode_from_client(this, msg.mode);
   send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE,
                         serial_proxy_result_to_status(result));
 }

esphome/components/serial_proxy/__init__.py

@@ -41,10 +41,11 @@ SERIAL_PROXY_PORT_TYPES = {
 }
 
 SerialProxyMode = api_enums_ns.enum("SerialProxyMode")
-# The mode a port starts in. `raw` is a plain byte pipe; `protocol` activates the
+# The mode a port boots into. `raw` is a plain byte pipe; `protocol` activates the
 # port's tap (if one is configured), letting it observe traffic and inject protocol
-# bytes such as acknowledgements. Clients may change it at runtime, so this only
-# decides what the device boots into.
+# bytes such as acknowledgements. The mode returns to `raw` whenever a client session
+# ends, so this value applies only until the first session ends; after that, clients
+# select the mode at runtime.
 SERIAL_PROXY_MODES = {
     "RAW": SerialProxyMode.SERIAL_PROXY_MODE_RAW,
     "PROTOCOL": SerialProxyMode.SERIAL_PROXY_MODE_PROTOCOL,

esphome/components/serial_proxy/serial_proxy.cpp

@@ -134,6 +134,10 @@ bool SerialProxy::tap_observing_() const {
 
 void SerialProxy::tap_pump() {
 #ifdef USE_API
+  // Nothing would consume the bytes; leave them in the FIFO
+  if (!this->tap_observing_() && this->api_connection_ == nullptr) {
+    return;
+  }
   const size_t available = this->available();
   if (available > 0) {
     this->read_and_send_(available);
@@ -230,7 +234,8 @@ SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uin
   return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
 }
 
-SerialProxyResult SerialProxy::set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) {
+SerialProxyResult SerialProxy::set_mode_from_client(api::APIConnection *api_connection,
+                                                    api::enums::SerialProxyMode mode) {
 #ifdef USE_API
   // Only the live subscriber may change the mode, so the mode cannot outlive a session
   if (this->api_connection_ != api_connection) {
@@ -243,6 +248,16 @@ SerialProxyResult SerialProxy::set_mode(api::APIConnection *api_connection, api:
     ESP_LOGW(TAG, "Invalid mode: %" PRIu32, static_cast<uint32_t>(mode));
     return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT;
   }
+  // PROTOCOL on a port with no tap would be a silent no-op; refuse so the client knows
+  if (mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL) {
+#ifdef USE_SERIAL_PROXY_TAP
+    if (this->tap_ == nullptr)
+#endif
+    {
+      ESP_LOGW(TAG, "No tap on serial proxy [%" PRIu32 "]; PROTOCOL mode unavailable", this->instance_index_);
+      return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED;
+    }
+  }
   ESP_LOGD(TAG, "Serial proxy [%" PRIu32 "] mode set to %s", this->instance_index_,
            mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL ? LOG_STR_LITERAL("PROTOCOL") : LOG_STR_LITERAL("RAW"));
   const bool leaving_protocol_mode =

esphome/components/serial_proxy/serial_proxy.h

@@ -115,7 +115,7 @@ class SerialProxy final : public uart::UARTDevice, public Component {
   api::enums::SerialProxyMode get_mode() const { return this->mode_; }
 
   /// Handle a mode change requested by an API client
-  SerialProxyResult set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode);
+  SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode);
 
   /// Configure UART parameters and apply them
   /// @param api_connection The API connection requesting the change
@@ -176,7 +176,7 @@ class SerialProxy final : public uart::UARTDevice, public Component {
 
   /// Resume reading after a tap's needs change. loop() disables itself when there is
   /// neither a subscriber nor a tap that wants the port, so a tap starting fresh work
-  /// must ask for it back.
+  /// must ask for it back. Must be called from the main loop.
   void tap_request_port() { this->enable_loop(); }
 
   /// Whether the underlying device is present. On a USB UART this tracks enumeration, so

tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h

@@ -40,7 +40,7 @@ class SerialProxy {
     return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
   }
   void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {}
-  SerialProxyResult set_mode(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) {
+  SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) {
     return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
   }
   SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) {