[serial_proxy] Drop the YAML mode option

The boot mode had no coherent job left: before any subscriber the tap
is served via tap_needs_port() regardless of mode, 1.17+ clients select
the mode explicitly after subscribing, and the only remaining effect
was arming the tap for a first-session client that never asked for it
and could not turn it off. The mode is now purely a session property
of the API: ports always boot RAW.

Also polish the tap contract per review: expose tap_is_observed(),
return false from write_from_tap() when the bytes are dropped, and
document that tap_pump() must not be called from tap callbacks.

esphome/components/serial_proxy/__init__.py

@@ -18,7 +18,7 @@ from esphome import pins
 import esphome.codegen as cg
 from esphome.components import uart
 import esphome.config_validation as cv
-from esphome.const import CONF_ID, CONF_MODE, CONF_NAME
+from esphome.const import CONF_ID, CONF_NAME
 from esphome.core import CORE, coroutine_with_priority
 from esphome.coroutine import CoroPriority
 from esphome.types import ConfigType
@@ -40,17 +40,6 @@ SERIAL_PROXY_PORT_TYPES = {
     "RS485": SerialProxyPortType.SERIAL_PROXY_PORT_TYPE_RS485,
 }
 
-SerialProxyMode = api_enums_ns.enum("SerialProxyMode")
-# The mode a port boots into. `raw` is a plain byte pipe; `protocol` activates the
-# port's tap (if one is configured), letting it observe traffic and inject protocol
-# bytes such as acknowledgements. The mode returns to `raw` whenever a client session
-# ends, so this value applies only until the first session ends; after that, clients
-# select the mode at runtime.
-SERIAL_PROXY_MODES = {
-    "RAW": SerialProxyMode.SERIAL_PROXY_MODE_RAW,
-    "PROTOCOL": SerialProxyMode.SERIAL_PROXY_MODE_PROTOCOL,
-}
-
 CONF_DTR_PIN = "dtr_pin"
 CONF_PORT_TYPE = "port_type"
 CONF_RTS_PIN = "rts_pin"
@@ -75,9 +64,6 @@ CONFIG_SCHEMA = (
             cv.GenerateID(): cv.declare_id(SerialProxy),
             cv.Required(CONF_NAME): cv.string_strict,
             cv.Required(CONF_PORT_TYPE): cv.enum(SERIAL_PROXY_PORT_TYPES, upper=True),
-            cv.Optional(CONF_MODE, default="RAW"): cv.enum(
-                SERIAL_PROXY_MODES, upper=True
-            ),
             cv.Optional(CONF_RTS_PIN): pins.gpio_output_pin_schema,
             cv.Optional(CONF_DTR_PIN): pins.gpio_output_pin_schema,
         }
@@ -102,7 +88,6 @@ async def to_code(config: ConfigType) -> None:
     cg.add(cg.App.register_serial_proxy(var))
     cg.add(var.set_name(config[CONF_NAME]))
     cg.add(var.set_port_type(config[CONF_PORT_TYPE]))
-    cg.add(var.set_mode(config[CONF_MODE]))
     cg.add_define("USE_SERIAL_PROXY")
 
     # Track instance count for the FINAL priority define

esphome/components/serial_proxy/serial_proxy.cpp

@@ -147,21 +147,18 @@ void SerialProxy::tap_pump() {
 #endif
 
 void SerialProxy::dump_config() {
-  ESP_LOGCONFIG(
-      TAG,
-      "Serial Proxy [%" PRIu32 "]:\n"
-      "  Name: %s\n"
-      "  Port Type: %s\n"
-      "  Mode: %s\n"
-      "  RTS Pin: %s\n"
-      "  DTR Pin: %s",
-      this->instance_index_, this->name_ != nullptr ? this->name_ : "",
-      this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS485   ? LOG_STR_LITERAL("RS485")
-      : this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS232 ? LOG_STR_LITERAL("RS232")
-                                                                     : LOG_STR_LITERAL("TTL"),
-      this->mode_ == api::enums::SERIAL_PROXY_MODE_PROTOCOL ? LOG_STR_LITERAL("PROTOCOL") : LOG_STR_LITERAL("RAW"),
-      this->rts_pin_ != nullptr ? LOG_STR_LITERAL("configured") : LOG_STR_LITERAL("not configured"),
-      this->dtr_pin_ != nullptr ? LOG_STR_LITERAL("configured") : LOG_STR_LITERAL("not configured"));
+  ESP_LOGCONFIG(TAG,
+                "Serial Proxy [%" PRIu32 "]:\n"
+                "  Name: %s\n"
+                "  Port Type: %s\n"
+                "  RTS Pin: %s\n"
+                "  DTR Pin: %s",
+                this->instance_index_, this->name_ != nullptr ? this->name_ : "",
+                this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS485   ? LOG_STR_LITERAL("RS485")
+                : this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS232 ? LOG_STR_LITERAL("RS232")
+                                                                               : LOG_STR_LITERAL("TTL"),
+                this->rts_pin_ != nullptr ? LOG_STR_LITERAL("configured") : LOG_STR_LITERAL("not configured"),
+                this->dtr_pin_ != nullptr ? LOG_STR_LITERAL("configured") : LOG_STR_LITERAL("not configured"));
 }
 
 SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control,

esphome/components/serial_proxy/serial_proxy.h

@@ -108,12 +108,6 @@ class SerialProxy final : public uart::UARTDevice, public Component {
   /// Get the port type
   api::enums::SerialProxyPortType get_port_type() const { return this->port_type_; }
 
-  /// Set the initial mode (from YAML configuration)
-  void set_mode(api::enums::SerialProxyMode mode) { this->mode_ = mode; }
-
-  /// Get the current mode
-  api::enums::SerialProxyMode get_mode() const { return this->mode_; }
-
   /// Handle a mode change requested by an API client
   SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode);
 
@@ -167,13 +161,21 @@ class SerialProxy final : public uart::UARTDevice, public Component {
 
   /// Write bytes originating from the tap rather than from a client. Bypasses the
   /// subscriber ownership check, but only while the tap is being served bytes -- so a
-  /// port in RAW mode with a subscriber attached stays inert.
-  void write_from_tap(const uint8_t *data, size_t len) {
-    if (this->tap_observing_()) {
-      this->write_array(data, len);
+  /// port in RAW mode with a subscriber attached stays inert. Returns false when the
+  /// bytes were dropped for that reason.
+  bool write_from_tap(const uint8_t *data, size_t len) {
+    if (!this->tap_observing_()) {
+      return false;
     }
+    this->write_array(data, len);
+    return true;
   }
 
+  /// Whether the tap is currently being served bytes. Can flip false with no callback
+  /// (a subscriber attaching in RAW mode, say), so a tap should check before starting
+  /// protocol work and when a reply seems overdue.
+  bool tap_is_observed() const { return this->tap_observing_(); }
+
   /// Resume reading after a tap's needs change. loop() disables itself when there is
   /// neither a subscriber nor a tap that wants the port, so a tap starting fresh work
   /// must ask for it back. Must be called from the main loop.
@@ -185,7 +187,8 @@ class SerialProxy final : public uart::UARTDevice, public Component {
 
   /// Run one read-and-dispatch cycle immediately. Lets a tap make progress before the
   /// main loop is running -- during setup, for instance, while a component is still
-  /// blocking on can_proceed().
+  /// blocking on can_proceed(). Must not be called from on_device_rx() or
+  /// on_client_tx(): each nested cycle costs a 256-byte stack frame.
   void tap_pump();
 #endif
 

tests/components/serial_proxy/common.yaml

@@ -8,4 +8,3 @@ serial_proxy:
   - id: serial_proxy_1
     name: Test Serial Port
     port_type: RS232
-    mode: protocol