diff --git a/CODEOWNERS b/CODEOWNERS index d60dbc729d9..cb415bb625a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -435,6 +435,7 @@ esphome/components/sen5x/* @martgras esphome/components/sen6x/* @martgras @mebner86 @mikelawrence @tuct esphome/components/sensirion_common/* @martgras esphome/components/sensor/* @esphome/core +esphome/components/serial_proxy/* @kbx81 esphome/components/sfa30/* @ghsensdev esphome/components/sgp40/* @SenexCrenshaw esphome/components/sgp4x/* @martgras @SenexCrenshaw diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 0e28e19db24..81efe5ece58 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -2618,6 +2618,14 @@ enum SerialProxyRequestType { SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2; // Flush the serial port (block until all TX data is sent) } +enum SerialProxyStatus { + SERIAL_PROXY_STATUS_OK = 0; // Completed successfully; TX drain confirmed + SERIAL_PROXY_STATUS_ASSUMED_SUCCESS = 1; // Platform cannot confirm TX drain; success assumed + SERIAL_PROXY_STATUS_ERROR = 2; // Driver or hardware error + SERIAL_PROXY_STATUS_TIMEOUT = 3; // Timed out before TX completed + SERIAL_PROXY_STATUS_NOT_SUPPORTED = 4; // Request type not supported by this instance +} + // Generic request message for simple serial proxy operations message SerialProxyRequest { option (id) = 144; @@ -2628,6 +2636,18 @@ message SerialProxyRequest { SerialProxyRequestType type = 2; // Request type } +// Response to a SerialProxyRequest (e.g. flush completion or failure) +message SerialProxyRequestResponse { + option (id) = 147; + option (source) = SOURCE_SERVER; + option (ifdef) = "USE_SERIAL_PROXY"; + + uint32 instance = 1; // Instance index (0-based) + SerialProxyRequestType type = 2; // Which request type this responds to + SerialProxyStatus status = 3; // Result status + string error_message = 4; // Additional detail on failure (optional) +} + // ==================== BLUETOOTH CONNECTION PARAMS ==================== message BluetoothSetConnectionParamsRequest { option (id) = 145; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 5409565c6eb..0029c2e34fc 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1445,6 +1445,89 @@ void APIConnection::on_infrared_rf_transmit_raw_timings_request(const InfraredRF void APIConnection::send_infrared_rf_receive_event(const InfraredRFReceiveEvent &msg) { this->send_message(msg); } #endif +#ifdef USE_SERIAL_PROXY +void APIConnection::on_serial_proxy_configure_request(const SerialProxyConfigureRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %u out of range (max %u)", msg.instance, + static_cast(proxies.size())); + return; + } + proxies[msg.instance]->configure(msg.baudrate, msg.flow_control, static_cast(msg.parity), msg.stop_bits, + msg.data_size); +} + +void APIConnection::on_serial_proxy_write_request(const SerialProxyWriteRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %u out of range", msg.instance); + return; + } + proxies[msg.instance]->write_from_client(msg.data, msg.data_len); +} + +void APIConnection::on_serial_proxy_set_modem_pins_request(const SerialProxySetModemPinsRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %u out of range", msg.instance); + return; + } + proxies[msg.instance]->set_modem_pins(msg.line_states); +} + +void APIConnection::on_serial_proxy_get_modem_pins_request(const SerialProxyGetModemPinsRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %u out of range", msg.instance); + return; + } + SerialProxyGetModemPinsResponse resp{}; + resp.instance = msg.instance; + resp.line_states = proxies[msg.instance]->get_modem_pins(); + this->send_message(resp); +} + +void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %u out of range", msg.instance); + return; + } + switch (msg.type) { + case enums::SERIAL_PROXY_REQUEST_TYPE_SUBSCRIBE: + case enums::SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE: + proxies[msg.instance]->serial_proxy_request(this, msg.type); + break; + case enums::SERIAL_PROXY_REQUEST_TYPE_FLUSH: { + SerialProxyRequestResponse resp{}; + resp.instance = msg.instance; + resp.type = enums::SERIAL_PROXY_REQUEST_TYPE_FLUSH; + switch (proxies[msg.instance]->flush_port()) { + case uart::FlushResult::SUCCESS: + resp.status = enums::SERIAL_PROXY_STATUS_OK; + break; + case uart::FlushResult::ASSUMED_SUCCESS: + resp.status = enums::SERIAL_PROXY_STATUS_ASSUMED_SUCCESS; + break; + case uart::FlushResult::TIMEOUT: + resp.status = enums::SERIAL_PROXY_STATUS_TIMEOUT; + break; + case uart::FlushResult::FAILED: + resp.status = enums::SERIAL_PROXY_STATUS_ERROR; + break; + } + this->send_message(resp); + break; + } + default: + ESP_LOGW(TAG, "Unknown serial proxy request type: %u", static_cast(msg.type)); + break; + } +} + +void APIConnection::send_serial_proxy_data(const SerialProxyDataReceived &msg) { this->send_message(msg); } +#endif + #ifdef USE_INFRARED uint16_t APIConnection::try_send_infrared_info(EntityBase *entity, APIConnection *conn, uint32_t remaining_size) { auto *infrared = static_cast(entity); @@ -1666,6 +1749,16 @@ bool APIConnection::send_device_info_response_() { resp.zwave_proxy_feature_flags = zwave_proxy::global_zwave_proxy->get_feature_flags(); resp.zwave_home_id = zwave_proxy::global_zwave_proxy->get_home_id(); #endif +#ifdef USE_SERIAL_PROXY + size_t serial_proxy_index = 0; + for (auto const &proxy : App.get_serial_proxies()) { + if (serial_proxy_index >= SERIAL_PROXY_COUNT) + break; + auto &info = resp.serial_proxies[serial_proxy_index++]; + info.name = StringRef(proxy->get_name()); + info.port_type = proxy->get_port_type(); + } +#endif #ifdef USE_API_NOISE resp.api_encryption_supported = true; #endif diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 3d864a7cf23..20cc4d4f864 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -189,6 +189,15 @@ class APIConnection final : public APIServerConnectionBase { void send_infrared_rf_receive_event(const InfraredRFReceiveEvent &msg); #endif +#ifdef USE_SERIAL_PROXY + void on_serial_proxy_configure_request(const SerialProxyConfigureRequest &msg) override; + void on_serial_proxy_write_request(const SerialProxyWriteRequest &msg) override; + void on_serial_proxy_set_modem_pins_request(const SerialProxySetModemPinsRequest &msg) override; + void on_serial_proxy_get_modem_pins_request(const SerialProxyGetModemPinsRequest &msg) override; + void on_serial_proxy_request(const SerialProxyRequest &msg) override; + void send_serial_proxy_data(const SerialProxyDataReceived &msg); +#endif + #ifdef USE_EVENT void send_event(event::Event *event); #endif @@ -254,6 +263,7 @@ class APIConnection final : public APIServerConnectionBase { return static_cast(this->flags_.connection_state) == ConnectionState::CONNECTED || this->is_authenticated(); } + bool is_marked_for_removal() const { return this->flags_.remove; } uint8_t get_log_subscription_level() const { return this->flags_.log_subscription; } // Get client API version for feature detection diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index c438043da40..07b4901bfeb 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -3836,6 +3836,20 @@ bool SerialProxyRequest::decode_varint(uint32_t field_id, ProtoVarInt value) { } return true; } +void SerialProxyRequestResponse::encode(ProtoWriteBuffer &buffer) const { + buffer.encode_uint32(1, this->instance); + buffer.encode_uint32(2, static_cast(this->type)); + buffer.encode_uint32(3, static_cast(this->status)); + buffer.encode_string(4, this->error_message); +} +uint32_t SerialProxyRequestResponse::calculate_size() const { + uint32_t size = 0; + size += ProtoSize::calc_uint32(1, this->instance); + size += ProtoSize::calc_uint32(1, static_cast(this->type)); + size += ProtoSize::calc_uint32(1, static_cast(this->status)); + size += ProtoSize::calc_length(1, this->error_message.size()); + return size; +} #endif #ifdef USE_BLUETOOTH_PROXY bool BluetoothSetConnectionParamsRequest::decode_varint(uint32_t field_id, ProtoVarInt value) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 58cd75028dc..09587f943dc 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -333,6 +333,13 @@ enum SerialProxyRequestType : uint32_t { SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE = 1, SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2, }; +enum SerialProxyStatus : uint32_t { + SERIAL_PROXY_STATUS_OK = 0, + SERIAL_PROXY_STATUS_ASSUMED_SUCCESS = 1, + SERIAL_PROXY_STATUS_ERROR = 2, + SERIAL_PROXY_STATUS_TIMEOUT = 3, + SERIAL_PROXY_STATUS_NOT_SUPPORTED = 4, +}; #endif } // namespace enums @@ -3219,6 +3226,25 @@ class SerialProxyRequest final : public ProtoDecodableMessage { protected: bool decode_varint(uint32_t field_id, ProtoVarInt value) override; }; +class SerialProxyRequestResponse final : public ProtoMessage { + public: + static constexpr uint8_t MESSAGE_TYPE = 147; + static constexpr uint8_t ESTIMATED_SIZE = 17; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *message_name() const override { return "serial_proxy_request_response"; } +#endif + uint32_t instance{0}; + enums::SerialProxyRequestType type{}; + enums::SerialProxyStatus status{}; + StringRef error_message{}; + void encode(ProtoWriteBuffer &buffer) const; + uint32_t calculate_size() const; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: +}; #endif #ifdef USE_BLUETOOTH_PROXY class BluetoothSetConnectionParamsRequest final : public ProtoDecodableMessage { diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index c4e850ffba8..86346563e49 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -789,6 +789,22 @@ template<> const char *proto_enum_to_string(enums return "UNKNOWN"; } } +template<> const char *proto_enum_to_string(enums::SerialProxyStatus value) { + switch (value) { + case enums::SERIAL_PROXY_STATUS_OK: + return "SERIAL_PROXY_STATUS_OK"; + case enums::SERIAL_PROXY_STATUS_ASSUMED_SUCCESS: + return "SERIAL_PROXY_STATUS_ASSUMED_SUCCESS"; + case enums::SERIAL_PROXY_STATUS_ERROR: + return "SERIAL_PROXY_STATUS_ERROR"; + case enums::SERIAL_PROXY_STATUS_TIMEOUT: + return "SERIAL_PROXY_STATUS_TIMEOUT"; + case enums::SERIAL_PROXY_STATUS_NOT_SUPPORTED: + return "SERIAL_PROXY_STATUS_NOT_SUPPORTED"; + default: + return "UNKNOWN"; + } +} #endif const char *HelloRequest::dump_to(DumpBuffer &out) const { @@ -2608,6 +2624,14 @@ const char *SerialProxyRequest::dump_to(DumpBuffer &out) const { dump_field(out, "type", static_cast(this->type)); return out.c_str(); } +const char *SerialProxyRequestResponse::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, "SerialProxyRequestResponse"); + dump_field(out, "instance", this->instance); + dump_field(out, "type", static_cast(this->type)); + dump_field(out, "status", static_cast(this->status)); + dump_field(out, "error_message", this->error_message); + return out.c_str(); +} #endif #ifdef USE_BLUETOOTH_PROXY const char *BluetoothSetConnectionParamsRequest::dump_to(DumpBuffer &out) const { diff --git a/esphome/components/api/api_pb2_service.h b/esphome/components/api/api_pb2_service.h index a031d2d969e..10fd88d8e13 100644 --- a/esphome/components/api/api_pb2_service.h +++ b/esphome/components/api/api_pb2_service.h @@ -233,6 +233,7 @@ class APIServerConnectionBase : public ProtoService { #ifdef USE_SERIAL_PROXY virtual void on_serial_proxy_request(const SerialProxyRequest &value){}; #endif + #ifdef USE_BLUETOOTH_PROXY virtual void on_bluetooth_set_connection_params_request(const BluetoothSetConnectionParamsRequest &value){}; #endif diff --git a/esphome/components/serial_proxy/__init__.py b/esphome/components/serial_proxy/__init__.py new file mode 100644 index 00000000000..f9b8c375d21 --- /dev/null +++ b/esphome/components/serial_proxy/__init__.py @@ -0,0 +1,104 @@ +""" +Serial Proxy component for ESPHome. + +WARNING: This component is EXPERIMENTAL. The API (both Python configuration +and C++ interfaces) may change at any time without following the normal +breaking changes policy. Use at your own risk. + +Once the API is considered stable, this warning will be removed. + +Provides a proxy to/from a serial interface on the ESPHome device, allowing +Home Assistant to connect to the serial port and send/receive data to/from +an arbitrary serial device. +""" + +from dataclasses import dataclass + +from esphome import pins +import esphome.codegen as cg +from esphome.components import uart +import esphome.config_validation as cv +from esphome.const import CONF_ID, CONF_NAME +from esphome.core import CORE, coroutine_with_priority +from esphome.coroutine import CoroPriority + +CODEOWNERS = ["@kbx81"] +DEPENDENCIES = ["api", "uart"] + +MULTI_CONF = True + +serial_proxy_ns = cg.esphome_ns.namespace("serial_proxy") +SerialProxy = serial_proxy_ns.class_("SerialProxy", cg.Component, uart.UARTDevice) + +api_enums_ns = cg.esphome_ns.namespace("api").namespace("enums") +SerialProxyPortType = api_enums_ns.enum("SerialProxyPortType") +SERIAL_PROXY_PORT_TYPES = { + "TTL": SerialProxyPortType.SERIAL_PROXY_PORT_TYPE_TTL, + "RS232": SerialProxyPortType.SERIAL_PROXY_PORT_TYPE_RS232, + "RS485": SerialProxyPortType.SERIAL_PROXY_PORT_TYPE_RS485, +} + +CONF_DTR_PIN = "dtr_pin" +CONF_PORT_TYPE = "port_type" +CONF_RTS_PIN = "rts_pin" + +DOMAIN = "serial_proxy" + + +@dataclass +class SerialProxyData: + count: int = 0 + + +def _get_data() -> SerialProxyData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = SerialProxyData() + return CORE.data[DOMAIN] + + +CONFIG_SCHEMA = ( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(SerialProxy), + cv.Required(CONF_NAME): cv.string_strict, + cv.Required(CONF_PORT_TYPE): cv.enum(SERIAL_PROXY_PORT_TYPES, upper=True), + cv.Optional(CONF_RTS_PIN): pins.gpio_output_pin_schema, + cv.Optional(CONF_DTR_PIN): pins.gpio_output_pin_schema, + } + ) + .extend(cv.COMPONENT_SCHEMA) + .extend(uart.UART_DEVICE_SCHEMA) +) + + +@coroutine_with_priority(CoroPriority.FINAL) +async def _add_serial_proxy_count_define(): + """Emit the SERIAL_PROXY_COUNT define once with the final instance count.""" + count = _get_data().count + if count > 0: + cg.add_define("SERIAL_PROXY_COUNT", count) + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) + cg.add(cg.App.register_serial_proxy(var)) + cg.add(var.set_name(config[CONF_NAME])) + cg.add(var.set_port_type(config[CONF_PORT_TYPE])) + cg.add_define("USE_SERIAL_PROXY") + + # Track instance count for the FINAL priority define + data = _get_data() + if data.count == 0: + # Schedule the count define job only once (on the first instance) + CORE.add_job(_add_serial_proxy_count_define) + data.count += 1 + + if CONF_RTS_PIN in config: + rts_pin = await cg.gpio_pin_expression(config[CONF_RTS_PIN]) + cg.add(var.set_rts_pin(rts_pin)) + + if CONF_DTR_PIN in config: + dtr_pin = await cg.gpio_pin_expression(config[CONF_DTR_PIN]) + cg.add(var.set_dtr_pin(dtr_pin)) diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp new file mode 100644 index 00000000000..340f9b0cb85 --- /dev/null +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -0,0 +1,188 @@ +#include "serial_proxy.h" + +#ifdef USE_SERIAL_PROXY + +#include "esphome/core/log.h" +#include "esphome/core/util.h" + +#ifdef USE_API +#include "esphome/components/api/api_connection.h" +#include "esphome/components/api/api_server.h" +#endif + +namespace esphome::serial_proxy { + +static const char *const TAG = "serial_proxy"; + +void SerialProxy::setup() { + // Set up modem control pins if configured + if (this->rts_pin_ != nullptr) { + this->rts_pin_->setup(); + this->rts_pin_->digital_write(this->rts_state_); + } + if (this->dtr_pin_ != nullptr) { + this->dtr_pin_->setup(); + this->dtr_pin_->digital_write(this->dtr_state_); + } +#ifdef USE_API + // instance_index_ is fixed at registration time; pre-set it so loop() only needs to update data + this->outgoing_msg_.instance = this->instance_index_; +#endif +} + +void SerialProxy::loop() { +#ifdef USE_API + // Detect subscriber disconnect + if (this->api_connection_ != nullptr && (this->api_connection_->is_marked_for_removal() || + !this->api_connection_->is_connection_setup() || !api_is_connected())) { + ESP_LOGW(TAG, "Subscriber disconnected"); + this->api_connection_ = nullptr; + } + + if (this->api_connection_ == nullptr) + return; + + // Read available data from UART and forward to subscribed client + size_t available = this->available(); + if (available == 0) + return; + + // Read in chunks up to SERIAL_PROXY_MAX_READ_SIZE + uint8_t buffer[SERIAL_PROXY_MAX_READ_SIZE]; + size_t to_read = std::min(available, sizeof(buffer)); + + if (!this->read_array(buffer, to_read)) + return; + + this->outgoing_msg_.set_data(buffer, to_read); + this->api_connection_->send_serial_proxy_data(this->outgoing_msg_); +#endif +} + +void SerialProxy::dump_config() { + ESP_LOGCONFIG(TAG, + "Serial Proxy [%u]:\n" + " Name: %s\n" + " Port Type: %s\n" + " RTS Pin: %s\n" + " DTR Pin: %s", + this->instance_index_, this->name_ != nullptr ? this->name_ : "", + this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS485 ? "RS485" + : this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS232 ? "RS232" + : "TTL", + this->rts_pin_ != nullptr ? "configured" : "not configured", + this->dtr_pin_ != nullptr ? "configured" : "not configured"); +} + +void SerialProxy::configure(uint32_t baudrate, bool flow_control, uint8_t parity, uint8_t stop_bits, + uint8_t data_size) { + ESP_LOGD(TAG, "Configuring serial proxy [%u]: baud=%u, flow_ctrl=%s, parity=%u, stop=%u, data=%u", + this->instance_index_, baudrate, YESNO(flow_control), parity, stop_bits, data_size); + + auto *uart_comp = this->parent_; + if (uart_comp == nullptr) { + ESP_LOGE(TAG, "UART component not available"); + return; + } + + // Validate all parameters before applying any (values come from a remote client) + if (baudrate == 0) { + ESP_LOGW(TAG, "Invalid baud rate: 0"); + return; + } + if (stop_bits < 1 || stop_bits > 2) { + ESP_LOGW(TAG, "Invalid stop bits: %u (must be 1 or 2)", stop_bits); + return; + } + if (data_size < 5 || data_size > 8) { + ESP_LOGW(TAG, "Invalid data bits: %u (must be 5-8)", data_size); + return; + } + if (parity > 2) { + ESP_LOGW(TAG, "Invalid parity: %u (must be 0-2)", parity); + return; + } + + // Apply validated parameters + uart_comp->set_baud_rate(baudrate); + uart_comp->set_stop_bits(stop_bits); + uart_comp->set_data_bits(data_size); + + // Map parity value to UARTParityOptions + static const uart::UARTParityOptions PARITY_MAP[] = { + uart::UART_CONFIG_PARITY_NONE, + uart::UART_CONFIG_PARITY_EVEN, + uart::UART_CONFIG_PARITY_ODD, + }; + uart_comp->set_parity(PARITY_MAP[parity]); + + // load_settings() is available on ESP8266 and ESP32 platforms +#if defined(USE_ESP8266) || defined(USE_ESP32) + uart_comp->load_settings(true); +#endif + + if (flow_control) { + ESP_LOGW(TAG, "Hardware flow control requested but is not yet supported"); + } +} + +void SerialProxy::write_from_client(const uint8_t *data, size_t len) { + if (data == nullptr || len == 0) + return; + this->write_array(data, len); +} + +void SerialProxy::set_modem_pins(uint32_t line_states) { + const bool rts = (line_states & SERIAL_PROXY_LINE_STATE_FLAG_RTS) != 0; + const bool dtr = (line_states & SERIAL_PROXY_LINE_STATE_FLAG_DTR) != 0; + ESP_LOGV(TAG, "Setting modem pins [%u]: RTS=%s, DTR=%s", this->instance_index_, ONOFF(rts), ONOFF(dtr)); + + if (this->rts_pin_ != nullptr) { + this->rts_state_ = rts; + this->rts_pin_->digital_write(rts); + } + if (this->dtr_pin_ != nullptr) { + this->dtr_state_ = dtr; + this->dtr_pin_->digital_write(dtr); + } +} + +uint32_t SerialProxy::get_modem_pins() const { + return (this->rts_state_ ? SERIAL_PROXY_LINE_STATE_FLAG_RTS : 0u) | + (this->dtr_state_ ? SERIAL_PROXY_LINE_STATE_FLAG_DTR : 0u); +} + +uart::FlushResult SerialProxy::flush_port() { + ESP_LOGV(TAG, "Flushing serial proxy [%u]", this->instance_index_); + return this->flush(); +} + +#ifdef USE_API +void SerialProxy::serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type) { + switch (type) { + case api::enums::SERIAL_PROXY_REQUEST_TYPE_SUBSCRIBE: + if (this->api_connection_ != nullptr) { + ESP_LOGE(TAG, "Only one API subscription is allowed at a time"); + return; + } + this->api_connection_ = api_connection; + ESP_LOGV(TAG, "API connection subscribed to serial proxy [%u]", this->instance_index_); + break; + case api::enums::SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE: + if (this->api_connection_ != api_connection) { + ESP_LOGV(TAG, "API connection is not subscribed to serial proxy [%u]", this->instance_index_); + return; + } + this->api_connection_ = nullptr; + ESP_LOGV(TAG, "API connection unsubscribed from serial proxy [%u]", this->instance_index_); + break; + default: + ESP_LOGW(TAG, "Unknown serial proxy request type: %u", static_cast(type)); + break; + } +} +#endif + +} // namespace esphome::serial_proxy + +#endif // USE_SERIAL_PROXY diff --git a/esphome/components/serial_proxy/serial_proxy.h b/esphome/components/serial_proxy/serial_proxy.h new file mode 100644 index 00000000000..52f0654ff0c --- /dev/null +++ b/esphome/components/serial_proxy/serial_proxy.h @@ -0,0 +1,129 @@ +#pragma once + +// WARNING: This component is EXPERIMENTAL. The API may change at any time +// without following the normal breaking changes policy. Use at your own risk. +// Once the API is considered stable, this warning will be removed. + +#include "esphome/core/defines.h" + +#ifdef USE_SERIAL_PROXY + +#include "esphome/core/component.h" +#include "esphome/core/hal.h" +#include "esphome/components/uart/uart.h" + +// Include api_pb2.h only when the API is enabled. The full include is needed +// to hold SerialProxyDataReceived by value as a pre-allocated member. +// Guarding prevents pulling conflicting Zephyr logging macro names into +// translation units that include this header without USE_API defined. +#ifdef USE_API +#include "esphome/components/api/api_pb2.h" +#endif + +// Forward-declare types needed outside the USE_API guard. +namespace esphome::api { +class APIConnection; +namespace enums { +enum SerialProxyPortType : uint32_t; +enum SerialProxyRequestType : uint32_t; +} // namespace enums +} // namespace esphome::api + +namespace esphome::serial_proxy { + +/// Bit flags for the line_states field exchanged with API clients. +/// Bit positions are stable API — new signals must use the next available bit. +enum SerialProxyLineStateFlag : uint32_t { + SERIAL_PROXY_LINE_STATE_FLAG_RTS = 1 << 0, ///< RTS (Request To Send) + SERIAL_PROXY_LINE_STATE_FLAG_DTR = 1 << 1, ///< DTR (Data Terminal Ready) +}; + +/// Maximum bytes to read from UART in a single loop iteration +inline constexpr size_t SERIAL_PROXY_MAX_READ_SIZE = 256; + +class SerialProxy : public uart::UARTDevice, public Component { + public: + void setup() override; + void loop() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::AFTER_CONNECTION; } + + /// Get the instance index (position in Application's serial_proxies_ vector) + uint32_t get_instance_index() const { return this->instance_index_; } + + /// Set the instance index (called by Application::register_serial_proxy) + void set_instance_index(uint32_t index) { this->instance_index_ = index; } + + /// Set the human-readable port name (from YAML configuration) + void set_name(const char *name) { this->name_ = name; } + + /// Get the human-readable port name + const char *get_name() const { return this->name_; } + + /// Set the port type (from YAML configuration) + void set_port_type(api::enums::SerialProxyPortType port_type) { this->port_type_ = port_type; } + + /// Get the port type + api::enums::SerialProxyPortType get_port_type() const { return this->port_type_; } + + /// Configure UART parameters and apply them + /// @param baudrate Baud rate in bits per second + /// @param flow_control True to enable hardware flow control + /// @param parity Parity setting (0=none, 1=even, 2=odd) + /// @param stop_bits Number of stop bits (1 or 2) + /// @param data_size Number of data bits (5-8) + void configure(uint32_t baudrate, bool flow_control, uint8_t parity, uint8_t stop_bits, uint8_t data_size); + + /// Handle a subscribe/unsubscribe request from an API client + void serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type); + + /// Write data received from an API client to the serial device + /// @param data Pointer to data buffer + /// @param len Number of bytes to write + void write_from_client(const uint8_t *data, size_t len); + + /// Set modem pin states from a bitmask of SerialProxyLineStateFlag values + void set_modem_pins(uint32_t line_states); + + /// Get current modem pin states as a bitmask of SerialProxyLineStateFlag values + uint32_t get_modem_pins() const; + + /// Flush the serial port (block until all TX data is sent) + uart::FlushResult flush_port(); + + /// Set the RTS GPIO pin (from YAML configuration) + void set_rts_pin(GPIOPin *pin) { this->rts_pin_ = pin; } + + /// Set the DTR GPIO pin (from YAML configuration) + void set_dtr_pin(GPIOPin *pin) { this->dtr_pin_ = pin; } + + protected: + /// Instance index for identifying this proxy in API messages + uint32_t instance_index_{0}; + + /// Subscribed API client (only one allowed at a time) + api::APIConnection *api_connection_{nullptr}; + +#ifdef USE_API + /// Pre-allocated outgoing message; instance field is set once in setup() + api::SerialProxyDataReceived outgoing_msg_; +#endif + + /// Human-readable port name (points to a string literal in flash) + const char *name_{nullptr}; + + /// Port type + api::enums::SerialProxyPortType port_type_{}; + + /// Optional GPIO pins for modem control + GPIOPin *rts_pin_{nullptr}; + GPIOPin *dtr_pin_{nullptr}; + + /// Current modem pin states + bool rts_state_{false}; + bool dtr_state_{false}; +}; + +} // namespace esphome::serial_proxy + +#endif // USE_SERIAL_PROXY diff --git a/esphome/core/application.h b/esphome/core/application.h index ef18566182b..5baf0469b64 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -105,6 +105,9 @@ void socket_wake(); // NOLINT(readability-redundant-declaration) #ifdef USE_INFRARED #include "esphome/components/infrared/infrared.h" #endif +#ifdef USE_SERIAL_PROXY +#include "esphome/components/serial_proxy/serial_proxy.h" +#endif #ifdef USE_EVENT #include "esphome/components/event/event.h" #endif @@ -382,6 +385,13 @@ class Application { } #endif +#ifdef USE_SERIAL_PROXY + void register_serial_proxy(serial_proxy::SerialProxy *proxy) { + proxy->set_instance_index(this->serial_proxies_.size()); + this->serial_proxies_.push_back(proxy); + } +#endif + #ifdef USE_EVENT void register_event(event::Event *event) { this->events_.push_back(event); @@ -623,6 +633,10 @@ class Application { GET_ENTITY_METHOD(infrared::Infrared, infrared, infrareds) #endif +#ifdef USE_SERIAL_PROXY + auto &get_serial_proxies() const { return this->serial_proxies_; } +#endif + #ifdef USE_EVENT auto &get_events() const { return this->events_; } GET_ENTITY_METHOD(event::Event, event, events) @@ -888,6 +902,9 @@ class Application { #ifdef USE_INFRARED StaticVector infrareds_{}; #endif +#ifdef USE_SERIAL_PROXY + StaticVector serial_proxies_{}; +#endif #ifdef USE_UPDATE StaticVector updates_{}; #endif diff --git a/esphome/core/defines.h b/esphome/core/defines.h index dc0ee138564..73791889544 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -107,6 +107,7 @@ #define MDNS_SERVICE_COUNT 3 #define USE_MDNS_DYNAMIC_TXT #define MDNS_DYNAMIC_TXT_COUNT 2 +#define SERIAL_PROXY_COUNT 2 #define SNTP_SERVER_COUNT 3 #define USE_MEDIA_PLAYER #define USE_MEDIA_SOURCE @@ -120,6 +121,7 @@ #define USE_SENSOR #define USE_SETUP_HEAP_STATS #define USE_SENSOR_FILTER +#define USE_SERIAL_PROXY #define USE_SETUP_PRIORITY_OVERRIDE #define USE_STATUS_LED #define USE_STATUS_SENSOR diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index ca560e8250a..63e1006b03c 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -30,11 +30,6 @@ static constexpr uint32_t MAX_LOGICALLY_DELETED_ITEMS = 5; // max delay to start an interval sequence static constexpr uint32_t MAX_INTERVAL_DELAY = 5000; -// Prevent inlining of SchedulerItem deletion. On BK7231N (Thumb-1), GCC inlines -// ~unique_ptr (~30 bytes each) at every destruction site. Defining -// the deleter in the .cpp file ensures a single copy of the destructor + operator delete. -void Scheduler::SchedulerItemDeleter::operator()(SchedulerItem *ptr) const noexcept { delete ptr; } - #if defined(ESPHOME_LOG_HAS_VERBOSE) || defined(ESPHOME_DEBUG_SCHEDULER) // Helper struct for formatting scheduler item names consistently in logs // Uses a stack buffer to avoid heap allocation @@ -122,8 +117,8 @@ uint32_t Scheduler::calculate_interval_offset_(uint32_t delay) { bool Scheduler::is_retry_cancelled_locked_(Component *component, NameType name_type, const char *static_name, uint32_t hash_or_id) { for (auto *container : {&this->items_, &this->to_add_}) { - for (auto &item : *container) { - if (item && this->is_item_removed_locked_(item.get()) && + for (auto *item : *container) { + if (item != nullptr && this->is_item_removed_locked_(item) && this->matches_item_locked_(item, component, name_type, static_name, hash_or_id, SchedulerItem::TIMEOUT, /* match_retry= */ true, /* skip_removed= */ false)) { return true; @@ -147,17 +142,31 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type return; } - // Take lock early to protect scheduler_item_pool_ access + // Take lock early to protect scheduler_item_pool_ access and retry-cancelled check LockGuard guard{this->lock_}; + // For retries, check if there's a cancelled timeout first - before allocating an item. + // Skip check for anonymous retries (STATIC_STRING with nullptr) - they can't be cancelled by name + // Skip check for defer (delay=0) - deferred retries bypass the cancellation check + if (is_retry && delay != 0 && (name_type != NameType::STATIC_STRING || static_name != nullptr) && + type == SchedulerItem::TIMEOUT && + this->is_retry_cancelled_locked_(component, name_type, static_name, hash_or_id)) { +#ifdef ESPHOME_DEBUG_SCHEDULER + SchedulerNameLog skip_name_log; + ESP_LOGD(TAG, "Skipping retry '%s' - found cancelled item", + skip_name_log.format(name_type, static_name, hash_or_id)); +#endif + return; + } + // Create and populate the scheduler item - auto item = this->get_item_from_pool_locked_(); + SchedulerItem *item = this->get_item_from_pool_locked_(); item->component = component; item->set_name(name_type, static_name, hash_or_id); item->type = type; item->callback = std::move(func); // Reset remove flag - recycled items may have been cancelled (remove=true) in previous use - this->set_item_removed_(item.get(), false); + this->set_item_removed_(item, false); item->is_retry = is_retry; // Determine target container: defer_queue_ for deferred items, to_add_ for everything else. @@ -193,29 +202,15 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type } #ifdef ESPHOME_DEBUG_SCHEDULER - this->debug_log_timer_(item.get(), name_type, static_name, hash_or_id, type, delay, now_64); + this->debug_log_timer_(item, name_type, static_name, hash_or_id, type, delay, now_64); #endif /* ESPHOME_DEBUG_SCHEDULER */ - - // For retries, check if there's a cancelled timeout first - // Skip check for anonymous retries (STATIC_STRING with nullptr) - they can't be cancelled by name - if (is_retry && (name_type != NameType::STATIC_STRING || static_name != nullptr) && - type == SchedulerItem::TIMEOUT && - this->is_retry_cancelled_locked_(component, name_type, static_name, hash_or_id)) { - // Skip scheduling - the retry was cancelled -#ifdef ESPHOME_DEBUG_SCHEDULER - SchedulerNameLog skip_name_log; - ESP_LOGD(TAG, "Skipping retry '%s' - found cancelled item", - skip_name_log.format(name_type, static_name, hash_or_id)); -#endif - return; - } } // Common epilogue: atomic cancel-and-add (unless skip_cancel is true) if (!skip_cancel) { this->cancel_item_locked_(component, name_type, static_name, hash_or_id, type); } - target->push_back(std::move(item)); + target->push_back(item); } void HOT Scheduler::set_timeout(Component *component, const char *name, uint32_t timeout, @@ -395,7 +390,7 @@ optional HOT Scheduler::next_schedule_in(uint32_t now) { if (this->cleanup_() == 0) return {}; - auto &item = this->items_[0]; + SchedulerItem *item = this->items_[0]; const auto now_64 = this->millis_64_from_(now); const uint64_t next_exec = item->get_next_execution(); if (next_exec < now_64) @@ -414,13 +409,13 @@ void Scheduler::full_cleanup_removed_items_() { // Compact in-place: move valid items forward, recycle removed ones size_t write = 0; for (size_t read = 0; read < this->items_.size(); ++read) { - if (!is_item_removed_locked_(this->items_[read].get())) { + if (!is_item_removed_locked_(this->items_[read])) { if (write != read) { - this->items_[write] = std::move(this->items_[read]); + this->items_[write] = this->items_[read]; } ++write; } else { - this->recycle_item_main_loop_(std::move(this->items_[read])); + this->recycle_item_main_loop_(this->items_[read]); } } this->items_.erase(this->items_.begin() + write, this->items_.end()); @@ -444,7 +439,7 @@ void Scheduler::compact_defer_queue_locked_() { // and recycled on the next loop iteration. size_t remaining = this->defer_queue_.size() - this->defer_queue_front_; for (size_t i = 0; i < remaining; i++) { - this->defer_queue_[i] = std::move(this->defer_queue_[this->defer_queue_front_ + i]); + this->defer_queue_[i] = this->defer_queue_[this->defer_queue_front_ + i]; } // Use erase() instead of resize() to avoid instantiating _M_default_append // (saves ~156 bytes flash). Erasing from the end is O(1) - no shifting needed. @@ -469,26 +464,26 @@ void HOT Scheduler::call(uint32_t now) { if (now_64 - last_print > 2000) { last_print = now_64; - std::vector old_items; + std::vector old_items; ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64, this->items_.size(), this->scheduler_item_pool_.size(), now_64); // Cleanup before debug output this->cleanup_(); while (!this->items_.empty()) { - SchedulerItemPtr item; + SchedulerItem *item; { LockGuard guard{this->lock_}; item = this->pop_raw_locked_(); } SchedulerNameLog name_log; - bool is_cancelled = is_item_removed_(item.get()); + bool is_cancelled = is_item_removed_(item); ESP_LOGD(TAG, " %s '%s/%s' interval=%" PRIu32 " next_execution in %" PRIu64 "ms at %" PRIu64 "%s", item->get_type_str(), LOG_STR_ARG(item->get_source()), name_log.format(item->get_name_type(), item->get_name(), item->get_name_hash_or_id()), item->interval, item->get_next_execution() - now_64, item->get_next_execution(), is_cancelled ? " [CANCELLED]" : ""); - old_items.push_back(std::move(item)); + old_items.push_back(item); } ESP_LOGD(TAG, "\n"); @@ -512,7 +507,7 @@ void HOT Scheduler::call(uint32_t now) { } while (!this->items_.empty()) { // Don't copy-by value yet - auto &item = this->items_[0]; + SchedulerItem *item = this->items_[0]; if (item->get_next_execution() > now_64) { // Not reached timeout yet, done for this call break; @@ -532,7 +527,7 @@ void HOT Scheduler::call(uint32_t now) { // Multi-threaded platforms without atomics: must take lock to safely read remove flag { LockGuard guard{this->lock_}; - if (is_item_removed_locked_(item.get())) { + if (is_item_removed_locked_(item)) { this->recycle_item_main_loop_(this->pop_raw_locked_()); this->to_remove_--; continue; @@ -540,7 +535,7 @@ void HOT Scheduler::call(uint32_t now) { } #else // Single-threaded or multi-threaded with atomics: can check without lock - if (is_item_removed_(item.get())) { + if (is_item_removed_(item)) { LockGuard guard{this->lock_}; this->recycle_item_main_loop_(this->pop_raw_locked_()); this->to_remove_--; @@ -561,18 +556,18 @@ void HOT Scheduler::call(uint32_t now) { // Warning: During callback(), a lot of stuff can happen, including: // - timeouts/intervals get added, potentially invalidating vector pointers // - timeouts/intervals get cancelled - now = this->execute_item_(item.get(), now); + now = this->execute_item_(item, now); LockGuard guard{this->lock_}; // Only pop after function call, this ensures we were reachable // during the function call and know if we were cancelled. - auto executed_item = this->pop_raw_locked_(); + SchedulerItem *executed_item = this->pop_raw_locked_(); - if (this->is_item_removed_locked_(executed_item.get())) { + if (this->is_item_removed_locked_(executed_item)) { // We were removed/cancelled in the function call, recycle and continue this->to_remove_--; - this->recycle_item_main_loop_(std::move(executed_item)); + this->recycle_item_main_loop_(executed_item); continue; } @@ -580,10 +575,10 @@ void HOT Scheduler::call(uint32_t now) { executed_item->set_next_execution(now_64 + executed_item->interval); // Add new item directly to to_add_ // since we have the lock held - this->to_add_.push_back(std::move(executed_item)); + this->to_add_.push_back(executed_item); } else { // Timeout completed - recycle it - this->recycle_item_main_loop_(std::move(executed_item)); + this->recycle_item_main_loop_(executed_item); } has_added_items |= !this->to_add_.empty(); @@ -592,17 +587,33 @@ void HOT Scheduler::call(uint32_t now) { if (has_added_items) { this->process_to_add(); } + +#ifdef ESPHOME_DEBUG_SCHEDULER + // Verify no items were leaked during this call() cycle. + // All items must be in items_, to_add_, defer_queue_, or the pool. + // Safe to check here because: + // - process_defer_queue_ has already run its cleanup_defer_queue_locked_(), + // so defer_queue_ contains no nullptr slots inflating the count. + // - The while loop above has finished, so no items are held in local variables; + // every item has been returned to a container (items_, to_add_, or pool). + // Lock needed to get a consistent snapshot of all containers. + { + LockGuard guard{this->lock_}; + this->debug_verify_no_leak_(); + } +#endif } void HOT Scheduler::process_to_add() { LockGuard guard{this->lock_}; - for (auto &it : this->to_add_) { - if (is_item_removed_locked_(it.get())) { + for (auto *&it : this->to_add_) { + if (is_item_removed_locked_(it)) { // Recycle cancelled items - this->recycle_item_main_loop_(std::move(it)); + this->recycle_item_main_loop_(it); + it = nullptr; continue; } - this->items_.push_back(std::move(it)); + this->items_.push_back(it); std::push_heap(this->items_.begin(), this->items_.end(), SchedulerItem::cmp); } this->to_add_.clear(); @@ -628,20 +639,18 @@ size_t HOT Scheduler::cleanup_() { // leading to race conditions LockGuard guard{this->lock_}; while (!this->items_.empty()) { - auto &item = this->items_[0]; - if (!this->is_item_removed_locked_(item.get())) + SchedulerItem *item = this->items_[0]; + if (!this->is_item_removed_locked_(item)) break; this->to_remove_--; this->recycle_item_main_loop_(this->pop_raw_locked_()); } return this->items_.size(); } -Scheduler::SchedulerItemPtr HOT Scheduler::pop_raw_locked_() { +Scheduler::SchedulerItem *HOT Scheduler::pop_raw_locked_() { std::pop_heap(this->items_.begin(), this->items_.end(), SchedulerItem::cmp); - // Move the item out before popping - this is the item that was at the front of the heap - auto item = std::move(this->items_.back()); - + SchedulerItem *item = this->items_.back(); this->items_.pop_back(); return item; } @@ -699,7 +708,7 @@ bool HOT Scheduler::cancel_item_locked_(Component *component, NameType name_type return total_cancelled > 0; } -bool HOT Scheduler::SchedulerItem::cmp(const SchedulerItemPtr &a, const SchedulerItemPtr &b) { +bool HOT Scheduler::SchedulerItem::cmp(SchedulerItem *a, SchedulerItem *b) { // High bits are almost always equal (change only on 32-bit rollover ~49 days) // Optimize for common case: check low bits first when high bits are equal return (a->next_execution_high_ == b->next_execution_high_) ? (a->next_execution_low_ > b->next_execution_low_) @@ -710,23 +719,26 @@ bool HOT Scheduler::SchedulerItem::cmp(const SchedulerItemPtr &a, const Schedule // IMPORTANT: Caller must hold the scheduler lock before calling this function. // This protects scheduler_item_pool_ from concurrent access by other threads // that may be acquiring items from the pool in set_timer_common_(). -void Scheduler::recycle_item_main_loop_(SchedulerItemPtr item) { - if (!item) +void Scheduler::recycle_item_main_loop_(SchedulerItem *item) { + if (item == nullptr) return; if (this->scheduler_item_pool_.size() < MAX_POOL_SIZE) { // Clear callback to release captured resources item->callback = nullptr; - this->scheduler_item_pool_.push_back(std::move(item)); + this->scheduler_item_pool_.push_back(item); #ifdef ESPHOME_DEBUG_SCHEDULER ESP_LOGD(TAG, "Recycled item to pool (pool size now: %zu)", this->scheduler_item_pool_.size()); #endif } else { #ifdef ESPHOME_DEBUG_SCHEDULER ESP_LOGD(TAG, "Pool full (size: %zu), deleting item", this->scheduler_item_pool_.size()); +#endif + delete item; +#ifdef ESPHOME_DEBUG_SCHEDULER + this->debug_live_items_--; #endif } - // else: unique_ptr will delete the item when it goes out of scope } #ifdef ESPHOME_DEBUG_SCHEDULER @@ -753,21 +765,54 @@ void Scheduler::debug_log_timer_(const SchedulerItem *item, NameType name_type, // Helper to get or create a scheduler item from the pool // IMPORTANT: Caller must hold the scheduler lock before calling this function. -Scheduler::SchedulerItemPtr Scheduler::get_item_from_pool_locked_() { - SchedulerItemPtr item; +Scheduler::SchedulerItem *Scheduler::get_item_from_pool_locked_() { if (!this->scheduler_item_pool_.empty()) { - item = std::move(this->scheduler_item_pool_.back()); + SchedulerItem *item = this->scheduler_item_pool_.back(); this->scheduler_item_pool_.pop_back(); #ifdef ESPHOME_DEBUG_SCHEDULER ESP_LOGD(TAG, "Reused item from pool (pool size now: %zu)", this->scheduler_item_pool_.size()); #endif - } else { - item = SchedulerItemPtr(new SchedulerItem()); -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Allocated new item (pool empty)"); -#endif + return item; } +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Allocated new item (pool empty)"); +#endif + auto *item = new SchedulerItem(); +#ifdef ESPHOME_DEBUG_SCHEDULER + this->debug_live_items_++; +#endif return item; } +#ifdef ESPHOME_DEBUG_SCHEDULER +bool Scheduler::debug_verify_no_leak_() const { + // Invariant: every live SchedulerItem must be in exactly one container. + // debug_live_items_ tracks allocations minus deletions. + size_t accounted = this->items_.size() + this->to_add_.size() + this->scheduler_item_pool_.size(); +#ifndef ESPHOME_THREAD_SINGLE + accounted += this->defer_queue_.size(); +#endif + if (accounted != this->debug_live_items_) { + ESP_LOGE(TAG, + "SCHEDULER LEAK DETECTED: live=%" PRIu32 " but accounted=%" PRIu32 " (items=%" PRIu32 " to_add=%" PRIu32 + " pool=%" PRIu32 +#ifndef ESPHOME_THREAD_SINGLE + " defer=%" PRIu32 +#endif + ")", + static_cast(this->debug_live_items_), static_cast(accounted), + static_cast(this->items_.size()), static_cast(this->to_add_.size()), + static_cast(this->scheduler_item_pool_.size()) +#ifndef ESPHOME_THREAD_SINGLE + , + static_cast(this->defer_queue_.size()) +#endif + ); + assert(false); + return false; + } + return true; +} +#endif + } // namespace esphome diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 9f48ed7e927..6d67492fca6 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -2,7 +2,6 @@ #include "esphome/core/defines.h" #include -#include #include #include #ifdef ESPHOME_THREAD_MULTI_ATOMICS @@ -144,19 +143,6 @@ class Scheduler { }; protected: - struct SchedulerItem; - - // Custom deleter for SchedulerItem unique_ptr that prevents the compiler from - // inlining the destructor at every destruction site. On BK7231N (Thumb-1), GCC - // inlines ~unique_ptr (~30 bytes: null check + ~std::function + - // operator delete) at every destruction site, while ESP32/ESP8266/RTL8720CF outline - // it into a single helper. This noinline deleter ensures only one copy exists. - // operator() is defined in scheduler.cpp to prevent inlining. - struct SchedulerItemDeleter { - void operator()(SchedulerItem *ptr) const noexcept; - }; - using SchedulerItemPtr = std::unique_ptr; - struct SchedulerItem { // Ordered by size to minimize padding Component *component; @@ -216,14 +202,14 @@ class Scheduler { name_.static_name = nullptr; } - // Destructor - no dynamic memory to clean up + // Destructor - no dynamic memory to clean up (callback's std::function handles its own) ~SchedulerItem() = default; // Delete copy operations to prevent accidental copies SchedulerItem(const SchedulerItem &) = delete; SchedulerItem &operator=(const SchedulerItem &) = delete; - // Delete move operations: SchedulerItem objects are only managed via unique_ptr, never moved directly + // Delete move operations: SchedulerItem objects are managed via raw pointers, never moved directly SchedulerItem(SchedulerItem &&) = delete; SchedulerItem &operator=(SchedulerItem &&) = delete; @@ -247,7 +233,7 @@ class Scheduler { name_type_ = type; } - static bool cmp(const SchedulerItemPtr &a, const SchedulerItemPtr &b); + static bool cmp(SchedulerItem *a, SchedulerItem *b); // Note: We use 48 bits total (32 + 16), stored in a 64-bit value for API compatibility. // The upper 16 bits of the 64-bit value are always zero, which is fine since @@ -298,12 +284,13 @@ class Scheduler { // Returns the number of items remaining after cleanup // IMPORTANT: This method should only be called from the main thread (loop task). size_t cleanup_(); - // Remove and return the front item from the heap + // Remove and return the front item from the heap as a raw pointer. + // Caller takes ownership and must either recycle or delete the item. // IMPORTANT: Caller must hold the scheduler lock before calling this function. - SchedulerItemPtr pop_raw_locked_(); + SchedulerItem *pop_raw_locked_(); // Get or create a scheduler item from the pool // IMPORTANT: Caller must hold the scheduler lock before calling this function. - SchedulerItemPtr get_item_from_pool_locked_(); + SchedulerItem *get_item_from_pool_locked_(); private: // Helper to cancel items - must be called with lock held @@ -327,19 +314,16 @@ class Scheduler { // Helper function to check if item matches criteria for cancellation // name_type determines matching: STATIC_STRING uses static_name, others use hash_or_id // IMPORTANT: Must be called with scheduler lock held - inline bool HOT matches_item_locked_(const SchedulerItemPtr &item, Component *component, NameType name_type, + inline bool HOT matches_item_locked_(SchedulerItem *item, Component *component, NameType name_type, const char *static_name, uint32_t hash_or_id, SchedulerItem::Type type, bool match_retry, bool skip_removed = true) const { // THREAD SAFETY: Check for nullptr first to prevent LoadProhibited crashes. On multi-threaded - // platforms, items can be moved out of defer_queue_ during processing, leaving nullptr entries. - // PR #11305 added nullptr checks in callers (mark_matching_items_removed_locked_()), but this check - // provides defense-in-depth: helper - // functions should be safe regardless of caller behavior. + // platforms, items can be nulled in defer_queue_ during processing. // Fixes: https://github.com/esphome/esphome/issues/11940 - if (!item) + if (item == nullptr) return false; - if (item->component != component || item->type != type || - (skip_removed && this->is_item_removed_locked_(item.get())) || (match_retry && !item->is_retry)) { + if (item->component != component || item->type != type || (skip_removed && this->is_item_removed_locked_(item)) || + (match_retry && !item->is_retry)) { return false; } // Name type must match @@ -361,10 +345,12 @@ class Scheduler { } // Helper to recycle a SchedulerItem back to the pool. + // Takes a raw pointer — caller transfers ownership. The item is either added to the + // pool or deleted if the pool is full. // IMPORTANT: Only call from main loop context! Recycling clears the callback, // so calling from another thread while the callback is executing causes use-after-free. // IMPORTANT: Caller must hold the scheduler lock before calling this function. - void recycle_item_main_loop_(SchedulerItemPtr item); + void recycle_item_main_loop_(SchedulerItem *item); // Helper to perform full cleanup when too many items are cancelled void full_cleanup_removed_items_(); @@ -420,27 +406,28 @@ class Scheduler { // Merge lock acquisitions: instead of separate locks for move-out and recycle (2N+1 total), // recycle each item after re-acquiring the lock for the next iteration (N+1 total). // The lock is held across: recycle → loop condition → move-out, then released for execution. - SchedulerItemPtr item; + SchedulerItem *item; this->lock_.lock(); while (this->defer_queue_front_ < defer_queue_end) { - // SAFETY: Moving out the unique_ptr leaves a nullptr in the vector at defer_queue_front_. - // This is intentional and safe because: + // Take ownership of the item, leaving nullptr in the vector slot. + // This is safe because: // 1. The vector is only cleaned up by cleanup_defer_queue_locked_() at the end of this function // 2. Any code iterating defer_queue_ MUST check for nullptr items (see mark_matching_items_removed_locked_) // 3. The lock protects concurrent access, but the nullptr remains until cleanup - item = std::move(this->defer_queue_[this->defer_queue_front_]); + item = this->defer_queue_[this->defer_queue_front_]; + this->defer_queue_[this->defer_queue_front_] = nullptr; this->defer_queue_front_++; this->lock_.unlock(); // Execute callback without holding lock to prevent deadlocks // if the callback tries to call defer() again - if (!this->should_skip_item_(item.get())) { - now = this->execute_item_(item.get(), now); + if (!this->should_skip_item_(item)) { + now = this->execute_item_(item, now); } this->lock_.lock(); - this->recycle_item_main_loop_(std::move(item)); + this->recycle_item_main_loop_(item); } // Clean up the queue (lock already held from last recycle or initial acquisition) this->cleanup_defer_queue_locked_(); @@ -520,18 +507,14 @@ class Scheduler { // name_type determines matching: STATIC_STRING uses static_name, others use hash_or_id // Returns the number of items marked for removal // IMPORTANT: Must be called with scheduler lock held - __attribute__((noinline)) size_t mark_matching_items_removed_locked_(std::vector &container, + __attribute__((noinline)) size_t mark_matching_items_removed_locked_(std::vector &container, Component *component, NameType name_type, const char *static_name, uint32_t hash_or_id, SchedulerItem::Type type, bool match_retry) { size_t count = 0; - for (auto &item : container) { - // Skip nullptr items (can happen in defer_queue_ when items are being processed) - // The defer_queue_ uses index-based processing: items are std::moved out but left in the - // vector as nullptr until cleanup. Even though this function is called with lock held, - // the vector can still contain nullptr items from the processing loop. This check prevents crashes. - if (item && this->matches_item_locked_(item, component, name_type, static_name, hash_or_id, type, match_retry)) { - this->set_item_removed_(item.get(), true); + for (auto *item : container) { + if (this->matches_item_locked_(item, component, name_type, static_name, hash_or_id, type, match_retry)) { + this->set_item_removed_(item, true); count++; } } @@ -539,15 +522,15 @@ class Scheduler { } Mutex lock_; - std::vector items_; - std::vector to_add_; + std::vector items_; + std::vector to_add_; #ifndef ESPHOME_THREAD_SINGLE // Single-core platforms don't need the defer queue and save ~32 bytes of RAM // Using std::vector instead of std::deque avoids 512-byte chunked allocations // Index tracking avoids O(n) erase() calls when draining the queue each loop - std::vector defer_queue_; // FIFO queue for defer() calls - size_t defer_queue_front_{0}; // Index of first valid item in defer_queue_ (tracks consumed items) -#endif /* ESPHOME_THREAD_SINGLE */ + std::vector defer_queue_; // FIFO queue for defer() calls + size_t defer_queue_front_{0}; // Index of first valid item in defer_queue_ (tracks consumed items) +#endif /* ESPHOME_THREAD_SINGLE */ uint32_t to_remove_{0}; // Memory pool for recycling SchedulerItem objects to reduce heap churn. @@ -558,7 +541,18 @@ class Scheduler { // - The pool significantly reduces heap fragmentation which is critical because heap allocation/deallocation // can stall the entire system, causing timing issues and dropped events for any components that need // to synchronize between tasks (see https://github.com/esphome/backlog/issues/52) - std::vector scheduler_item_pool_; + std::vector scheduler_item_pool_; + +#ifdef ESPHOME_DEBUG_SCHEDULER + // Leak detection: tracks total live SchedulerItem allocations. + // Invariant: debug_live_items_ == items_.size() + to_add_.size() + defer_queue_.size() + scheduler_item_pool_.size() + // Verified periodically in call() to catch leaks early. + size_t debug_live_items_{0}; + + // Verify the scheduler memory invariant: all allocated items are accounted for. + // Returns true if no leak detected. Logs an error and asserts on failure. + bool debug_verify_no_leak_() const; +#endif }; } // namespace esphome diff --git a/tests/components/serial_proxy/common.yaml b/tests/components/serial_proxy/common.yaml new file mode 100644 index 00000000000..6f03cf95dff --- /dev/null +++ b/tests/components/serial_proxy/common.yaml @@ -0,0 +1,10 @@ +wifi: + ssid: MySSID + password: password1 + +api: + +serial_proxy: + - id: serial_proxy_1 + name: Test Serial Port + port_type: RS232 diff --git a/tests/components/serial_proxy/test.esp32-idf.yaml b/tests/components/serial_proxy/test.esp32-idf.yaml new file mode 100644 index 00000000000..b415125e84b --- /dev/null +++ b/tests/components/serial_proxy/test.esp32-idf.yaml @@ -0,0 +1,8 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + +<<: !include common.yaml diff --git a/tests/components/serial_proxy/test.esp8266-ard.yaml b/tests/components/serial_proxy/test.esp8266-ard.yaml new file mode 100644 index 00000000000..96ab4ef6aca --- /dev/null +++ b/tests/components/serial_proxy/test.esp8266-ard.yaml @@ -0,0 +1,8 @@ +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO2 + +packages: + uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + +<<: !include common.yaml diff --git a/tests/components/serial_proxy/test.rp2040-ard.yaml b/tests/components/serial_proxy/test.rp2040-ard.yaml new file mode 100644 index 00000000000..b28f2b5e05e --- /dev/null +++ b/tests/components/serial_proxy/test.rp2040-ard.yaml @@ -0,0 +1,8 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +packages: + uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + +<<: !include common.yaml diff --git a/tests/integration/fixtures/scheduler_bulk_cleanup.yaml b/tests/integration/fixtures/scheduler_bulk_cleanup.yaml index de876da8c47..3d2c47a0de5 100644 --- a/tests/integration/fixtures/scheduler_bulk_cleanup.yaml +++ b/tests/integration/fixtures/scheduler_bulk_cleanup.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-bulk-cleanup external_components: diff --git a/tests/integration/fixtures/scheduler_defer_cancel.yaml b/tests/integration/fixtures/scheduler_defer_cancel.yaml index 9e3f927c33c..92ae0062aca 100644 --- a/tests/integration/fixtures/scheduler_defer_cancel.yaml +++ b/tests/integration/fixtures/scheduler_defer_cancel.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-defer-cancel host: diff --git a/tests/integration/fixtures/scheduler_defer_cancels_regular.yaml b/tests/integration/fixtures/scheduler_defer_cancels_regular.yaml index fb6b1791dc4..cf7f6ec7338 100644 --- a/tests/integration/fixtures/scheduler_defer_cancels_regular.yaml +++ b/tests/integration/fixtures/scheduler_defer_cancels_regular.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-defer-cancel-regular host: diff --git a/tests/integration/fixtures/scheduler_defer_fifo_simple.yaml b/tests/integration/fixtures/scheduler_defer_fifo_simple.yaml index 7384082ac2d..f69e5c6c67b 100644 --- a/tests/integration/fixtures/scheduler_defer_fifo_simple.yaml +++ b/tests/integration/fixtures/scheduler_defer_fifo_simple.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-defer-fifo-simple host: diff --git a/tests/integration/fixtures/scheduler_defer_stress.yaml b/tests/integration/fixtures/scheduler_defer_stress.yaml index 0d9c1d14051..70eac01daf6 100644 --- a/tests/integration/fixtures/scheduler_defer_stress.yaml +++ b/tests/integration/fixtures/scheduler_defer_stress.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-defer-stress-test external_components: diff --git a/tests/integration/fixtures/scheduler_heap_stress.yaml b/tests/integration/fixtures/scheduler_heap_stress.yaml index d4d340b68ba..486a5d12764 100644 --- a/tests/integration/fixtures/scheduler_heap_stress.yaml +++ b/tests/integration/fixtures/scheduler_heap_stress.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-heap-stress-test external_components: diff --git a/tests/integration/fixtures/scheduler_internal_id_no_collision.yaml b/tests/integration/fixtures/scheduler_internal_id_no_collision.yaml index 46dbb8e728d..e696e99efa6 100644 --- a/tests/integration/fixtures/scheduler_internal_id_no_collision.yaml +++ b/tests/integration/fixtures/scheduler_internal_id_no_collision.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-internal-id-test on_boot: priority: -100 diff --git a/tests/integration/fixtures/scheduler_null_name.yaml b/tests/integration/fixtures/scheduler_null_name.yaml index 42eaacdd439..d5488761d68 100644 --- a/tests/integration/fixtures/scheduler_null_name.yaml +++ b/tests/integration/fixtures/scheduler_null_name.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-null-name host: diff --git a/tests/integration/fixtures/scheduler_numeric_id_test.yaml b/tests/integration/fixtures/scheduler_numeric_id_test.yaml index 1669f026f5c..25decf20f51 100644 --- a/tests/integration/fixtures/scheduler_numeric_id_test.yaml +++ b/tests/integration/fixtures/scheduler_numeric_id_test.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-numeric-id-test on_boot: priority: -100 diff --git a/tests/integration/fixtures/scheduler_rapid_cancellation.yaml b/tests/integration/fixtures/scheduler_rapid_cancellation.yaml index 4824654c5c6..530b8241f58 100644 --- a/tests/integration/fixtures/scheduler_rapid_cancellation.yaml +++ b/tests/integration/fixtures/scheduler_rapid_cancellation.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: sched-rapid-cancel-test external_components: diff --git a/tests/integration/fixtures/scheduler_recursive_timeout.yaml b/tests/integration/fixtures/scheduler_recursive_timeout.yaml index f1168802f6e..66b6f4b19bb 100644 --- a/tests/integration/fixtures/scheduler_recursive_timeout.yaml +++ b/tests/integration/fixtures/scheduler_recursive_timeout.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: sched-recursive-timeout external_components: diff --git a/tests/integration/fixtures/scheduler_removed_item_race.yaml b/tests/integration/fixtures/scheduler_removed_item_race.yaml index 2f8a7fb987b..55d2197d7ce 100644 --- a/tests/integration/fixtures/scheduler_removed_item_race.yaml +++ b/tests/integration/fixtures/scheduler_removed_item_race.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-removed-item-race host: diff --git a/tests/integration/fixtures/scheduler_retry_test.yaml b/tests/integration/fixtures/scheduler_retry_test.yaml index ffe9082a69f..cdf71152bdc 100644 --- a/tests/integration/fixtures/scheduler_retry_test.yaml +++ b/tests/integration/fixtures/scheduler_retry_test.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-retry-test on_boot: priority: -100 diff --git a/tests/integration/fixtures/scheduler_simultaneous_callbacks.yaml b/tests/integration/fixtures/scheduler_simultaneous_callbacks.yaml index 446ee7fdc0e..c15edc3ffd5 100644 --- a/tests/integration/fixtures/scheduler_simultaneous_callbacks.yaml +++ b/tests/integration/fixtures/scheduler_simultaneous_callbacks.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: sched-simul-callbacks-test external_components: diff --git a/tests/integration/fixtures/scheduler_string_lifetime.yaml b/tests/integration/fixtures/scheduler_string_lifetime.yaml index ebd5052b8bf..5ae5a1914e7 100644 --- a/tests/integration/fixtures/scheduler_string_lifetime.yaml +++ b/tests/integration/fixtures/scheduler_string_lifetime.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: scheduler-string-lifetime-test external_components: diff --git a/tests/integration/fixtures/scheduler_string_name_stress.yaml b/tests/integration/fixtures/scheduler_string_name_stress.yaml index d1ef55c8d5f..8f68d1d1023 100644 --- a/tests/integration/fixtures/scheduler_string_name_stress.yaml +++ b/tests/integration/fixtures/scheduler_string_name_stress.yaml @@ -1,4 +1,5 @@ esphome: + debug_scheduler: true # Enable scheduler leak detection name: sched-string-name-stress external_components: