From a1760a198047d92a36fe9c8af01ca5c86f736d80 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 1 Mar 2026 10:23:10 -1000 Subject: [PATCH 01/20] [improv_serial] Add missing USE_IMPROV_SERIAL define to fix WiFi scan filtering (#14359) --- esphome/components/improv_serial/__init__.py | 1 + esphome/components/wifi/wifi_component.cpp | 2 +- esphome/components/wifi/wifi_component.h | 4 ++-- esphome/components/wifi/wifi_component_esp8266.cpp | 2 +- esphome/components/wifi/wifi_component_esp_idf.cpp | 2 +- esphome/components/wifi/wifi_component_libretiny.cpp | 2 +- esphome/components/wifi/wifi_component_pico_w.cpp | 2 +- esphome/core/defines.h | 1 + 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/esphome/components/improv_serial/__init__.py b/esphome/components/improv_serial/__init__.py index 9a2ac2f40f..4266f5b78b 100644 --- a/esphome/components/improv_serial/__init__.py +++ b/esphome/components/improv_serial/__init__.py @@ -43,3 +43,4 @@ async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) await improv_base.setup_improv_core(var, config, "improv_serial") + cg.add_define("USE_IMPROV_SERIAL") diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 1e6961b8bd..7d5d0133c1 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -2121,7 +2121,7 @@ bool WiFiComponent::can_proceed() { #endif void WiFiComponent::set_reboot_timeout(uint32_t reboot_timeout) { this->reboot_timeout_ = reboot_timeout; } -bool WiFiComponent::is_connected() { +bool WiFiComponent::is_connected() const { return this->state_ == WIFI_COMPONENT_STATE_STA_CONNECTED && this->wifi_sta_connect_status_() == WiFiSTAConnectStatus::CONNECTED && !this->error_from_callback_; } diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 63c7039f21..a6f03a08d9 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -443,7 +443,7 @@ class WiFiComponent : public Component { void set_reboot_timeout(uint32_t reboot_timeout); - bool is_connected(); + bool is_connected() const; void set_power_save_mode(WiFiPowerSaveMode power_save); void set_min_auth_mode(WifiMinAuthMode min_auth_mode) { min_auth_mode_ = min_auth_mode; } @@ -677,7 +677,7 @@ class WiFiComponent : public Component { bool wifi_apply_hostname_(); bool wifi_sta_connect_(const WiFiAP &ap); void wifi_pre_setup_(); - WiFiSTAConnectStatus wifi_sta_connect_status_(); + WiFiSTAConnectStatus wifi_sta_connect_status_() const; bool wifi_scan_start_(bool passive); #ifdef USE_WIFI_AP diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 8911bf15e0..bd6a18a99b 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -622,7 +622,7 @@ void WiFiComponent::wifi_pre_setup_() { this->wifi_mode_(false, false); } -WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() { +WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() const { station_status_t status = wifi_station_get_connect_status(); if (status == STATION_GOT_IP) return WiFiSTAConnectStatus::CONNECTED; diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 57bbceb1b8..734d186205 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -921,7 +921,7 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { } } -WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() { +WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() const { if (s_sta_connected && this->got_ipv4_address_) { #if USE_NETWORK_IPV6 && (USE_NETWORK_MIN_IPV6_ADDR_COUNT > 0) if (this->num_ipv6_addresses_ >= USE_NETWORK_MIN_IPV6_ADDR_COUNT) { diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index 1c5490a3ac..4c4150e44d 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -634,7 +634,7 @@ void WiFiComponent::wifi_pre_setup_() { // Make sure WiFi is in clean state before anything starts this->wifi_mode_(false, false); } -WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() { +WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() const { // Use state machine instead of querying WiFi.status() directly // State is updated in main loop from queued events, ensuring thread safety switch (s_sta_state) { diff --git a/esphome/components/wifi/wifi_component_pico_w.cpp b/esphome/components/wifi/wifi_component_pico_w.cpp index 9b2c077dc5..270425d8c2 100644 --- a/esphome/components/wifi/wifi_component_pico_w.cpp +++ b/esphome/components/wifi/wifi_component_pico_w.cpp @@ -120,7 +120,7 @@ const char *get_disconnect_reason_str(uint8_t reason) { return "UNKNOWN"; } -WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() { +WiFiSTAConnectStatus WiFiComponent::wifi_sta_connect_status_() const { // Use cyw43_wifi_link_status instead of cyw43_tcpip_link_status because the Arduino // framework's __wrap_cyw43_cb_tcpip_init is a no-op — the SDK's internal netif // (cyw43_state.netif[]) is never initialized. cyw43_tcpip_link_status checks that netif's diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 5c982c94b1..181425c162 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -55,6 +55,7 @@ #define USE_HOMEASSISTANT_TIME #define USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT 8000 // NOLINT #define USE_IMAGE +#define USE_IMPROV_SERIAL #define USE_IMPROV_SERIAL_NEXT_URL #define USE_INFRARED #define USE_IR_RF From 48b5cae6c4692c6f563b1c250aa266e31b0b1b98 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 1 Mar 2026 11:32:44 -1000 Subject: [PATCH 02/20] [api] Use StringRef for user service string arguments (#13974) --- esphome/automation.py | 57 +++++- esphome/components/api/__init__.py | 15 +- esphome/components/api/api_connection.cpp | 43 +++-- esphome/components/api/api_frame_helper.h | 4 + .../components/api/api_frame_helper_noise.cpp | 30 ++- .../api/api_frame_helper_plaintext.cpp | 7 +- esphome/components/api/user_services.cpp | 5 + esphome/components/button/__init__.py | 4 +- esphome/components/cover/__init__.py | 20 +- esphome/components/fan/__init__.py | 9 +- esphome/components/globals/__init__.py | 1 + esphome/components/light/automation.py | 12 +- esphome/components/logger/__init__.py | 4 +- esphome/components/mqtt/__init__.py | 7 +- esphome/components/number/__init__.py | 6 + esphome/components/output/__init__.py | 7 +- esphome/components/script/__init__.py | 2 + esphome/components/select/__init__.py | 7 + esphome/components/switch/__init__.py | 14 +- esphome/components/text/__init__.py | 1 + esphome/core/string_ref.h | 14 ++ esphome/util.py | 16 +- tests/unit_tests/test_automation.py | 177 ++++++++++++++++++ 23 files changed, 406 insertions(+), 56 deletions(-) create mode 100644 tests/unit_tests/test_automation.py diff --git a/esphome/automation.py b/esphome/automation.py index 2439b1ddc4..d9b8b2ec57 100644 --- a/esphome/automation.py +++ b/esphome/automation.py @@ -57,8 +57,23 @@ def maybe_conf(conf, *validators): return validate -def register_action(name: str, action_type: MockObjClass, schema: cv.Schema): - return ACTION_REGISTRY.register(name, action_type, schema) +def register_action( + name: str, + action_type: MockObjClass, + schema: cv.Schema, + *, + synchronous: bool = False, +): + """Register an action type. + + Actions default to ``synchronous=False`` (safe default), meaning string + arguments use owning std::string to prevent dangling references. + + Set ``synchronous=True`` only for actions that complete synchronously + and never store trigger arguments for later execution. This allows + the code generator to use non-owning StringRef for zero-copy access. + """ + return ACTION_REGISTRY.register(name, action_type, schema, synchronous=synchronous) def register_condition(name: str, condition_type: MockObjClass, schema: cv.Schema): @@ -335,7 +350,9 @@ async def component_is_idle_condition_to_code( @register_action( - "delay", DelayAction, cv.templatable(cv.positive_time_period_milliseconds) + "delay", + DelayAction, + cv.templatable(cv.positive_time_period_milliseconds), ) async def delay_action_to_code( config: ConfigType, @@ -366,6 +383,7 @@ async def delay_action_to_code( cv.has_at_least_one_key(CONF_THEN, CONF_ELSE), cv.has_at_least_one_key(CONF_CONDITION, CONF_ANY, CONF_ALL), ), + synchronous=True, ) async def if_action_to_code( config: ConfigType, @@ -394,6 +412,7 @@ async def if_action_to_code( cv.Required(CONF_THEN): validate_action_list, } ), + synchronous=True, ) async def while_action_to_code( config: ConfigType, @@ -417,6 +436,7 @@ async def while_action_to_code( cv.Required(CONF_THEN): validate_action_list, } ), + synchronous=True, ) async def repeat_action_to_code( config: ConfigType, @@ -461,7 +481,12 @@ async def wait_until_action_to_code( return var -@register_action("lambda", LambdaAction, cv.lambda_) +# Lambda executes user C++ inline and returns — synchronous by execution model. +# User code could theoretically store the StringRef for deferred use, but StringRef +# is a view type and storing views beyond their scope is always unsafe regardless +# of this optimization. Marking non-synchronous would disable StringRef for nearly +# all user services since most use lambda. +@register_action("lambda", LambdaAction, cv.lambda_, synchronous=True) async def lambda_action_to_code( config: ConfigType, action_id: ID, @@ -480,6 +505,7 @@ async def lambda_action_to_code( cv.Required(CONF_ID): cv.use_id(cg.PollingComponent), } ), + synchronous=True, ) async def component_update_action_to_code( config: ConfigType, @@ -499,6 +525,7 @@ async def component_update_action_to_code( cv.Required(CONF_ID): cv.use_id(cg.PollingComponent), } ), + synchronous=True, ) async def component_suspend_action_to_code( config: ConfigType, @@ -521,6 +548,7 @@ async def component_suspend_action_to_code( ), } ), + synchronous=True, ) async def component_resume_action_to_code( config: ConfigType, @@ -578,6 +606,27 @@ async def build_condition_list( return conditions +def has_non_synchronous_actions(actions: ConfigType) -> bool: + """Check if a validated action list contains any non-synchronous actions. + + Non-synchronous actions (delay, wait_until, script.wait, etc.) store + trigger args for later execution, making non-owning types like StringRef + unsafe. Actions that haven't been audited default to non-synchronous. + """ + if isinstance(actions, list): + return any(has_non_synchronous_actions(item) for item in actions) + if isinstance(actions, dict): + for key in actions: + if key in ACTION_REGISTRY and not ACTION_REGISTRY[key].synchronous: + return True + return any( + has_non_synchronous_actions(v) + for v in actions.values() + if isinstance(v, (list, dict)) + ) + return False + + async def build_automation( trigger: MockObj, args: TemplateArgsType, config: ConfigType ) -> MockObj: diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3f7cafb485..d7b6bec357 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -76,7 +76,7 @@ SERVICE_ARG_NATIVE_TYPES: dict[str, MockObj] = { "bool": cg.bool_, "int": cg.int32, "float": cg.float_, - "string": cg.std_string, + "string": cg.StringRef, "bool[]": cg.FixedVector.template(cg.bool_).operator("const").operator("ref"), "int[]": cg.FixedVector.template(cg.int32).operator("const").operator("ref"), "float[]": cg.FixedVector.template(cg.float_).operator("const").operator("ref"), @@ -380,9 +380,18 @@ async def to_code(config: ConfigType) -> None: if is_optional: func_args.append((cg.bool_, "return_response")) + # Check if action chain has non-synchronous actions that would make + # non-owning StringRef dangle (rx_buf_ reused after delay) + has_non_synchronous = automation.has_non_synchronous_actions( + conf.get(CONF_THEN, []) + ) + service_arg_names: list[str] = [] for name, var_ in conf[CONF_VARIABLES].items(): native = SERVICE_ARG_NATIVE_TYPES[var_] + # Fall back to std::string for string args if non-synchronous actions exist + if has_non_synchronous and native is cg.StringRef: + native = cg.std_string service_template_args.append(native) func_args.append((native, name)) service_arg_names.append(name) @@ -509,11 +518,13 @@ HOMEASSISTANT_ACTION_ACTION_SCHEMA = cv.All( "homeassistant.action", HomeAssistantServiceCallAction, HOMEASSISTANT_ACTION_ACTION_SCHEMA, + synchronous=True, ) @automation.register_action( "homeassistant.service", HomeAssistantServiceCallAction, HOMEASSISTANT_ACTION_ACTION_SCHEMA, + synchronous=True, ) async def homeassistant_service_to_code( config: ConfigType, @@ -604,6 +615,7 @@ HOMEASSISTANT_EVENT_ACTION_SCHEMA = cv.Schema( "homeassistant.event", HomeAssistantServiceCallAction, HOMEASSISTANT_EVENT_ACTION_SCHEMA, + synchronous=True, ) async def homeassistant_event_to_code(config, action_id, template_arg, args): cg.add_define("USE_API_HOMEASSISTANT_SERVICES") @@ -644,6 +656,7 @@ HOMEASSISTANT_TAG_SCANNED_ACTION_SCHEMA = cv.maybe_simple_value( "homeassistant.tag_scanned", HomeAssistantServiceCallAction, HOMEASSISTANT_TAG_SCANNED_ACTION_SCHEMA, + synchronous=True, ) async def homeassistant_tag_scanned_to_code(config, action_id, template_arg, args): cg.add_define("USE_API_HOMEASSISTANT_SERVICES") diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 8b2efdde51..215af611db 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1711,37 +1711,42 @@ void APIConnection::on_home_assistant_state_response(const HomeAssistantStateRes return; } + // Null-terminate state in-place for safe c_str() usage (e.g., parse_number in callbacks). + // Safe: decode is complete, byte after string data was already consumed during parse, + // and frame helpers reserve RX_BUF_NULL_TERMINATOR extra byte in rx_buf_. + // const_cast is safe: msg references mutable rx_buf_ data; the const& handler + // signature is a generated protobuf pattern, not a true immutability contract. + if (!msg.state.empty()) { + const_cast(msg.state.c_str())[msg.state.size()] = '\0'; + } + for (auto &it : this->parent_->get_state_subs()) { - // Compare entity_id: check length matches and content matches - size_t entity_id_len = strlen(it.entity_id); - if (entity_id_len != msg.entity_id.size() || - memcmp(it.entity_id, msg.entity_id.c_str(), msg.entity_id.size()) != 0) { + if (msg.entity_id != it.entity_id) { continue; } - // Compare attribute: either both have matching attribute, or both have none - size_t sub_attr_len = it.attribute != nullptr ? strlen(it.attribute) : 0; - if (sub_attr_len != msg.attribute.size() || - (sub_attr_len > 0 && memcmp(it.attribute, msg.attribute.c_str(), sub_attr_len) != 0)) { + // If subscriber has attribute filter (non-null), message attribute must match it; + // if subscriber has no filter (nullptr), message must have no attribute. + if (it.attribute != nullptr ? msg.attribute != it.attribute : !msg.attribute.empty()) { continue; } - // Create null-terminated state for callback (parse_number needs null-termination) - // HA state max length is 255 characters, but attributes can be much longer - // Use stack buffer for common case (states), heap fallback for large attributes - size_t state_len = msg.state.size(); - SmallBufferWithHeapFallback state_buf_alloc(state_len + 1); - char *state_buf = reinterpret_cast(state_buf_alloc.get()); - if (state_len > 0) { - memcpy(state_buf, msg.state.c_str(), state_len); - } - state_buf[state_len] = '\0'; - it.callback(StringRef(state_buf, state_len)); + it.callback(msg.state); } } #endif #ifdef USE_API_USER_DEFINED_ACTIONS void APIConnection::on_execute_service_request(const ExecuteServiceRequest &msg) { + // Null-terminate string args in-place for safe c_str() usage in YAML service triggers. + // Safe: full ExecuteServiceRequest decode is complete, all bytes in rx_buf_ consumed, + // and frame helpers reserve RX_BUF_NULL_TERMINATOR extra byte for the last field. + // const_cast is safe: msg references mutable rx_buf_ data; the const& handler + // signature is a generated protobuf pattern, not a true immutability contract. + for (auto &arg : const_cast(msg).args) { + if (!arg.string_.empty()) { + const_cast(arg.string_.c_str())[arg.string_.size()] = '\0'; + } + } bool found = false; #ifdef USE_API_USER_DEFINED_ACTION_RESPONSES // Register the call and get a unique server-generated action_call_id diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h index 03f3814bb9..2b4e9ea3cd 100644 --- a/esphome/components/api/api_frame_helper.h +++ b/esphome/components/api/api_frame_helper.h @@ -29,6 +29,10 @@ static constexpr uint16_t MAX_MESSAGE_SIZE = 8192; // 8 KiB for ESP8266 static constexpr uint16_t MAX_MESSAGE_SIZE = 32768; // 32 KiB for ESP32 and other platforms #endif +// Extra byte reserved in rx_buf_ beyond the message size so protobuf +// StringRef fields can be null-terminated in-place after decode. +static constexpr uint16_t RX_BUF_NULL_TERMINATOR = 1; + // Maximum number of messages to batch in a single write operation // Must be >= MAX_INITIAL_PER_BATCH in api_connection.h (enforced by static_assert there) static constexpr size_t MAX_MESSAGES_PER_BATCH = 34; diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 2aad732f7f..3ae35e9be8 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -194,16 +194,21 @@ APIError APINoiseFrameHelper::try_read_frame_() { uint16_t msg_size = (((uint16_t) rx_header_buf_[1]) << 8) | rx_header_buf_[2]; // Check against size limits to prevent OOM: MAX_HANDSHAKE_SIZE for handshake, MAX_MESSAGE_SIZE for data - uint16_t limit = (state_ == State::DATA) ? MAX_MESSAGE_SIZE : MAX_HANDSHAKE_SIZE; + bool is_data = (state_ == State::DATA); + uint16_t limit = is_data ? MAX_MESSAGE_SIZE : MAX_HANDSHAKE_SIZE; if (msg_size > limit) { state_ = State::FAILED; HELPER_LOG("Bad packet: message size %u exceeds maximum %u", msg_size, limit); - return (state_ == State::DATA) ? APIError::BAD_DATA_PACKET : APIError::BAD_HANDSHAKE_PACKET_LEN; + return is_data ? APIError::BAD_DATA_PACKET : APIError::BAD_HANDSHAKE_PACKET_LEN; } - // Reserve space for body - if (this->rx_buf_.size() != msg_size) { - this->rx_buf_.resize(msg_size); + // Reserve space for body (+ null terminator in DATA state so protobuf + // StringRef fields can be safely null-terminated in-place after decode. + // During handshake, rx_buf_.size() is used in prologue construction, so + // the buffer must be exactly msg_size to avoid prologue mismatch.) + uint16_t alloc_size = msg_size + (is_data ? RX_BUF_NULL_TERMINATOR : 0); + if (this->rx_buf_.size() != alloc_size) { + this->rx_buf_.resize(alloc_size); } if (rx_buf_len_ < msg_size) { @@ -407,7 +412,18 @@ APIError APINoiseFrameHelper::read_packet(ReadPacketBuffer *buffer) { NoiseBuffer mbuf; noise_buffer_init(mbuf); - noise_buffer_set_inout(mbuf, this->rx_buf_.data(), this->rx_buf_.size(), this->rx_buf_.size()); + // read_packet() must only be called in DATA state; the extra + // RX_BUF_NULL_TERMINATOR byte is only allocated in DATA state + // (see try_read_frame_), so calling this during handshake would + // underflow the size calculation below. +#ifdef ESPHOME_DEBUG_API + assert(this->state_ == State::DATA); +#endif + // rx_buf_ has RX_BUF_NULL_TERMINATOR extra byte for null termination + // (only added in DATA state — see try_read_frame_), so subtract it + // to get the actual encrypted data size for decryption. + size_t encrypted_size = this->rx_buf_.size() - RX_BUF_NULL_TERMINATOR; + noise_buffer_set_inout(mbuf, this->rx_buf_.data(), encrypted_size, encrypted_size); int err = noise_cipherstate_decrypt(this->recv_cipher_, &mbuf); APIError decrypt_err = handle_noise_error_(err, LOG_STR("noise_cipherstate_decrypt"), APIError::CIPHERSTATE_DECRYPT_FAILED); @@ -574,7 +590,9 @@ APIError APINoiseFrameHelper::init_handshake_() { } APIError APINoiseFrameHelper::check_handshake_finished_() { +#ifdef ESPHOME_DEBUG_API assert(state_ == State::HANDSHAKE); +#endif int action = noise_handshakestate_get_action(handshake_); if (action == NOISE_ACTION_READ_MESSAGE || action == NOISE_ACTION_WRITE_MESSAGE) diff --git a/esphome/components/api/api_frame_helper_plaintext.cpp b/esphome/components/api/api_frame_helper_plaintext.cpp index 5069dbf68b..e2bb56e0ac 100644 --- a/esphome/components/api/api_frame_helper_plaintext.cpp +++ b/esphome/components/api/api_frame_helper_plaintext.cpp @@ -163,9 +163,10 @@ APIError APIPlaintextFrameHelper::try_read_frame_() { } // header reading done - // Reserve space for body - if (this->rx_buf_.size() != this->rx_header_parsed_len_) { - this->rx_buf_.resize(this->rx_header_parsed_len_); + // Reserve space for body (+ null terminator so protobuf StringRef fields + // can be safely null-terminated in-place after decode) + if (this->rx_buf_.size() != this->rx_header_parsed_len_ + RX_BUF_NULL_TERMINATOR) { + this->rx_buf_.resize(this->rx_header_parsed_len_ + RX_BUF_NULL_TERMINATOR); } if (rx_buf_len_ < rx_header_parsed_len_) { diff --git a/esphome/components/api/user_services.cpp b/esphome/components/api/user_services.cpp index 9c2b4aa79a..28a43c656c 100644 --- a/esphome/components/api/user_services.cpp +++ b/esphome/components/api/user_services.cpp @@ -1,5 +1,6 @@ #include "user_services.h" #include "esphome/core/log.h" +#include "esphome/core/string_ref.h" namespace esphome::api { @@ -11,6 +12,8 @@ template<> int32_t get_execute_arg_value(const ExecuteServiceArgument & } template<> float get_execute_arg_value(const ExecuteServiceArgument &arg) { return arg.float_; } template<> std::string get_execute_arg_value(const ExecuteServiceArgument &arg) { return arg.string_; } +// Zero-copy StringRef version for YAML-generated services (string_ is null-terminated after decode) +template<> StringRef get_execute_arg_value(const ExecuteServiceArgument &arg) { return arg.string_; } // Legacy std::vector versions for external components using custom_api_device.h - optimized with reserve template<> std::vector get_execute_arg_value>(const ExecuteServiceArgument &arg) { @@ -61,6 +64,8 @@ template<> enums::ServiceArgType to_service_arg_type() { return enums::SER template<> enums::ServiceArgType to_service_arg_type() { return enums::SERVICE_ARG_TYPE_INT; } template<> enums::ServiceArgType to_service_arg_type() { return enums::SERVICE_ARG_TYPE_FLOAT; } template<> enums::ServiceArgType to_service_arg_type() { return enums::SERVICE_ARG_TYPE_STRING; } +// Zero-copy StringRef version for YAML-generated services +template<> enums::ServiceArgType to_service_arg_type() { return enums::SERVICE_ARG_TYPE_STRING; } // Legacy std::vector versions for external components using custom_api_device.h template<> enums::ServiceArgType to_service_arg_type>() { return enums::SERVICE_ARG_TYPE_BOOL_ARRAY; } diff --git a/esphome/components/button/__init__.py b/esphome/components/button/__init__.py index d2f143b97e..94816a0974 100644 --- a/esphome/components/button/__init__.py +++ b/esphome/components/button/__init__.py @@ -123,7 +123,9 @@ BUTTON_PRESS_SCHEMA = maybe_simple_id( ) -@automation.register_action("button.press", PressAction, BUTTON_PRESS_SCHEMA) +@automation.register_action( + "button.press", PressAction, BUTTON_PRESS_SCHEMA, synchronous=True +) async def button_press_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) diff --git a/esphome/components/cover/__init__.py b/esphome/components/cover/__init__.py index 648fe7decf..17095f41f6 100644 --- a/esphome/components/cover/__init__.py +++ b/esphome/components/cover/__init__.py @@ -248,25 +248,33 @@ COVER_ACTION_SCHEMA = maybe_simple_id( ) -@automation.register_action("cover.open", OpenAction, COVER_ACTION_SCHEMA) +@automation.register_action( + "cover.open", OpenAction, COVER_ACTION_SCHEMA, synchronous=True +) async def cover_open_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) -@automation.register_action("cover.close", CloseAction, COVER_ACTION_SCHEMA) +@automation.register_action( + "cover.close", CloseAction, COVER_ACTION_SCHEMA, synchronous=True +) async def cover_close_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) -@automation.register_action("cover.stop", StopAction, COVER_ACTION_SCHEMA) +@automation.register_action( + "cover.stop", StopAction, COVER_ACTION_SCHEMA, synchronous=True +) async def cover_stop_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) -@automation.register_action("cover.toggle", ToggleAction, COVER_ACTION_SCHEMA) +@automation.register_action( + "cover.toggle", ToggleAction, COVER_ACTION_SCHEMA, synchronous=True +) async def cover_toggle_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) @@ -283,7 +291,9 @@ COVER_CONTROL_ACTION_SCHEMA = cv.Schema( ) -@automation.register_action("cover.control", ControlAction, COVER_CONTROL_ACTION_SCHEMA) +@automation.register_action( + "cover.control", ControlAction, COVER_CONTROL_ACTION_SCHEMA, synchronous=True +) async def cover_control_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) var = cg.new_Pvariable(action_id, template_arg, paren) diff --git a/esphome/components/fan/__init__.py b/esphome/components/fan/__init__.py index 6010aa8ed4..e839df6aee 100644 --- a/esphome/components/fan/__init__.py +++ b/esphome/components/fan/__init__.py @@ -311,13 +311,17 @@ FAN_ACTION_SCHEMA = maybe_simple_id( ) -@automation.register_action("fan.toggle", ToggleAction, FAN_ACTION_SCHEMA) +@automation.register_action( + "fan.toggle", ToggleAction, FAN_ACTION_SCHEMA, synchronous=True +) async def fan_toggle_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) -@automation.register_action("fan.turn_off", TurnOffAction, FAN_ACTION_SCHEMA) +@automation.register_action( + "fan.turn_off", TurnOffAction, FAN_ACTION_SCHEMA, synchronous=True +) async def fan_turn_off_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) @@ -336,6 +340,7 @@ async def fan_turn_off_to_code(config, action_id, template_arg, args): ), } ), + synchronous=True, ) async def fan_turn_on_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) diff --git a/esphome/components/globals/__init__.py b/esphome/components/globals/__init__.py index fc400c5dd1..fe11a93a4b 100644 --- a/esphome/components/globals/__init__.py +++ b/esphome/components/globals/__init__.py @@ -102,6 +102,7 @@ async def to_code(config): cv.Required(CONF_VALUE): cv.templatable(cv.string_strict), } ), + synchronous=True, ) async def globals_set_to_code(config, action_id, template_arg, args): full_id, paren = await cg.get_variable_with_full_id(config[CONF_ID]) diff --git a/esphome/components/light/automation.py b/esphome/components/light/automation.py index e5aa8fa0e9..89b2fc0fb2 100644 --- a/esphome/components/light/automation.py +++ b/esphome/components/light/automation.py @@ -51,6 +51,7 @@ from .types import ( ), } ), + synchronous=True, ) async def light_toggle_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -111,13 +112,13 @@ LIGHT_TURN_ON_ACTION_SCHEMA = automation.maybe_simple_id( @automation.register_action( - "light.turn_off", LightControlAction, LIGHT_TURN_OFF_ACTION_SCHEMA + "light.turn_off", LightControlAction, LIGHT_TURN_OFF_ACTION_SCHEMA, synchronous=True ) @automation.register_action( - "light.turn_on", LightControlAction, LIGHT_TURN_ON_ACTION_SCHEMA + "light.turn_on", LightControlAction, LIGHT_TURN_ON_ACTION_SCHEMA, synchronous=True ) @automation.register_action( - "light.control", LightControlAction, LIGHT_CONTROL_ACTION_SCHEMA + "light.control", LightControlAction, LIGHT_CONTROL_ACTION_SCHEMA, synchronous=True ) async def light_control_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -193,7 +194,10 @@ LIGHT_DIM_RELATIVE_ACTION_SCHEMA = cv.Schema( @automation.register_action( - "light.dim_relative", DimRelativeAction, LIGHT_DIM_RELATIVE_ACTION_SCHEMA + "light.dim_relative", + DimRelativeAction, + LIGHT_DIM_RELATIVE_ACTION_SCHEMA, + synchronous=True, ) async def light_dim_relative_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index 264197c175..026b8aaf24 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -519,7 +519,9 @@ LOGGER_LOG_ACTION_SCHEMA = cv.All( ) -@automation.register_action(CONF_LOGGER_LOG, LambdaAction, LOGGER_LOG_ACTION_SCHEMA) +@automation.register_action( + CONF_LOGGER_LOG, LambdaAction, LOGGER_LOG_ACTION_SCHEMA, synchronous=True +) async def logger_log_action_to_code(config, action_id, template_arg, args): esp_log = LOG_LEVEL_TO_ESP_LOG[config[CONF_LEVEL]] args_ = [cg.RawExpression(str(x)) for x in config[CONF_ARGS]] diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index 44e8836487..c25c472038 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -492,7 +492,7 @@ MQTT_PUBLISH_ACTION_SCHEMA = cv.Schema( @automation.register_action( - "mqtt.publish", MQTTPublishAction, MQTT_PUBLISH_ACTION_SCHEMA + "mqtt.publish", MQTTPublishAction, MQTT_PUBLISH_ACTION_SCHEMA, synchronous=True ) async def mqtt_publish_action_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -521,7 +521,10 @@ MQTT_PUBLISH_JSON_ACTION_SCHEMA = cv.Schema( @automation.register_action( - "mqtt.publish_json", MQTTPublishJsonAction, MQTT_PUBLISH_JSON_ACTION_SCHEMA + "mqtt.publish_json", + MQTTPublishJsonAction, + MQTT_PUBLISH_JSON_ACTION_SCHEMA, + synchronous=True, ) async def mqtt_publish_json_action_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) diff --git a/esphome/components/number/__init__.py b/esphome/components/number/__init__.py index d12ec7463b..2238f2c037 100644 --- a/esphome/components/number/__init__.py +++ b/esphome/components/number/__init__.py @@ -352,6 +352,7 @@ OPERATION_BASE_SCHEMA = cv.Schema( cv.Required(CONF_VALUE): cv.templatable(cv.float_), } ), + synchronous=True, ) async def number_set_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -374,6 +375,7 @@ async def number_set_to_code(config, action_id, template_arg, args): } ) ), + synchronous=True, ) @automation.register_action( "number.decrement", @@ -388,6 +390,7 @@ async def number_set_to_code(config, action_id, template_arg, args): } ) ), + synchronous=True, ) @automation.register_action( "number.to_min", @@ -401,6 +404,7 @@ async def number_set_to_code(config, action_id, template_arg, args): } ) ), + synchronous=True, ) @automation.register_action( "number.to_max", @@ -414,6 +418,7 @@ async def number_set_to_code(config, action_id, template_arg, args): } ) ), + synchronous=True, ) @automation.register_action( "number.operation", @@ -426,6 +431,7 @@ async def number_set_to_code(config, action_id, template_arg, args): cv.Optional(CONF_CYCLE, default=True): cv.templatable(cv.boolean), } ), + synchronous=True, ) async def number_to_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) diff --git a/esphome/components/output/__init__.py b/esphome/components/output/__init__.py index bde106b085..a4c960927b 100644 --- a/esphome/components/output/__init__.py +++ b/esphome/components/output/__init__.py @@ -74,14 +74,16 @@ BINARY_OUTPUT_ACTION_SCHEMA = maybe_simple_id( ) -@automation.register_action("output.turn_on", TurnOnAction, BINARY_OUTPUT_ACTION_SCHEMA) +@automation.register_action( + "output.turn_on", TurnOnAction, BINARY_OUTPUT_ACTION_SCHEMA, synchronous=True +) async def output_turn_on_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) @automation.register_action( - "output.turn_off", TurnOffAction, BINARY_OUTPUT_ACTION_SCHEMA + "output.turn_off", TurnOffAction, BINARY_OUTPUT_ACTION_SCHEMA, synchronous=True ) async def output_turn_off_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -97,6 +99,7 @@ async def output_turn_off_to_code(config, action_id, template_arg, args): cv.Required(CONF_LEVEL): cv.templatable(cv.percentage), } ), + synchronous=True, ) async def output_set_level_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) diff --git a/esphome/components/script/__init__.py b/esphome/components/script/__init__.py index 8d69981db0..369cefad91 100644 --- a/esphome/components/script/__init__.py +++ b/esphome/components/script/__init__.py @@ -160,6 +160,7 @@ async def to_code(config): cv.Optional(validate_parameter_name): cv.templatable(cv.valid), }, ), + synchronous=True, ) async def script_execute_action_to_code(config, action_id, template_arg, args): def convert(type: str): @@ -208,6 +209,7 @@ async def script_execute_action_to_code(config, action_id, template_arg, args): "script.stop", ScriptStopAction, maybe_simple_id({cv.Required(CONF_ID): cv.use_id(Script)}), + synchronous=True, ) async def script_stop_action_to_code(config, action_id, template_arg, args): full_id, paren = await cg.get_variable_with_full_id(config[CONF_ID]) diff --git a/esphome/components/select/__init__.py b/esphome/components/select/__init__.py index 84ad591ba1..c114b140a9 100644 --- a/esphome/components/select/__init__.py +++ b/esphome/components/select/__init__.py @@ -145,6 +145,7 @@ OPERATION_BASE_SCHEMA = cv.Schema( cv.Required(CONF_OPTION): cv.templatable(cv.string_strict), } ), + synchronous=True, ) async def select_set_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -162,6 +163,7 @@ async def select_set_to_code(config, action_id, template_arg, args): cv.Required(CONF_INDEX): cv.templatable(cv.positive_int), } ), + synchronous=True, ) async def select_set_index_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -217,6 +219,7 @@ async def select_is_to_code(config, condition_id, template_arg, args): cv.Optional(CONF_CYCLE, default=True): cv.templatable(cv.boolean), } ), + synchronous=True, ) @automation.register_action( "select.next", @@ -229,6 +232,7 @@ async def select_is_to_code(config, condition_id, template_arg, args): } ) ), + synchronous=True, ) @automation.register_action( "select.previous", @@ -243,6 +247,7 @@ async def select_is_to_code(config, condition_id, template_arg, args): } ) ), + synchronous=True, ) @automation.register_action( "select.first", @@ -254,6 +259,7 @@ async def select_is_to_code(config, condition_id, template_arg, args): } ) ), + synchronous=True, ) @automation.register_action( "select.last", @@ -265,6 +271,7 @@ async def select_is_to_code(config, condition_id, template_arg, args): } ) ), + synchronous=True, ) async def select_operation_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) diff --git a/esphome/components/switch/__init__.py b/esphome/components/switch/__init__.py index cfc5e2b6e8..6f1be7d53d 100644 --- a/esphome/components/switch/__init__.py +++ b/esphome/components/switch/__init__.py @@ -204,7 +204,7 @@ SWITCH_CONTROL_ACTION_SCHEMA = automation.maybe_simple_id( @automation.register_action( - "switch.control", ControlAction, SWITCH_CONTROL_ACTION_SCHEMA + "switch.control", ControlAction, SWITCH_CONTROL_ACTION_SCHEMA, synchronous=True ) async def switch_control_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) @@ -214,9 +214,15 @@ async def switch_control_to_code(config, action_id, template_arg, args): return var -@automation.register_action("switch.toggle", ToggleAction, SWITCH_ACTION_SCHEMA) -@automation.register_action("switch.turn_off", TurnOffAction, SWITCH_ACTION_SCHEMA) -@automation.register_action("switch.turn_on", TurnOnAction, SWITCH_ACTION_SCHEMA) +@automation.register_action( + "switch.toggle", ToggleAction, SWITCH_ACTION_SCHEMA, synchronous=True +) +@automation.register_action( + "switch.turn_off", TurnOffAction, SWITCH_ACTION_SCHEMA, synchronous=True +) +@automation.register_action( + "switch.turn_on", TurnOnAction, SWITCH_ACTION_SCHEMA, synchronous=True +) async def switch_toggle_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(action_id, template_arg, paren) diff --git a/esphome/components/text/__init__.py b/esphome/components/text/__init__.py index 9ceea0dfdf..61f7119cad 100644 --- a/esphome/components/text/__init__.py +++ b/esphome/components/text/__init__.py @@ -164,6 +164,7 @@ OPERATION_BASE_SCHEMA = cv.Schema( cv.Required(CONF_VALUE): cv.templatable(cv.string_strict), } ), + synchronous=True, ) async def text_set_to_code(config, action_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) diff --git a/esphome/core/string_ref.h b/esphome/core/string_ref.h index d502c4d27f..6047202753 100644 --- a/esphome/core/string_ref.h +++ b/esphome/core/string_ref.h @@ -81,6 +81,20 @@ class StringRef { operator std::string() const { return str(); } + /// Compare (compatible with std::string::compare) + int compare(const StringRef &other) const { + int result = std::memcmp(base_, other.base_, std::min(len_, other.len_)); + if (result != 0) + return result; + if (len_ < other.len_) + return -1; + if (len_ > other.len_) + return 1; + return 0; + } + int compare(const char *s) const { return compare(StringRef(s)); } + int compare(const std::string &s) const { return compare(StringRef(s)); } + /// Find first occurrence of substring, returns std::string::npos if not found. /// Note: Requires the underlying string to be null-terminated. size_type find(const char *s, size_type pos = 0) const { diff --git a/esphome/util.py b/esphome/util.py index 7b896de27e..686aa74306 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -24,11 +24,14 @@ class RegistryEntry: fun: Callable[..., Any], type_id: "MockObjClass", schema: "Schema", + *, + synchronous: bool = False, ): self.name = name self.fun = fun self.type_id = type_id self.raw_schema = schema + self.synchronous = synchronous @property def coroutine_fun(self): @@ -49,9 +52,18 @@ class Registry(dict[str, RegistryEntry]): self.base_schema = base_schema or {} self.type_id_key = type_id_key - def register(self, name: str, type_id: "MockObjClass", schema: "Schema"): + def register( + self, + name: str, + type_id: "MockObjClass", + schema: "Schema", + *, + synchronous: bool = False, + ): def decorator(fun: Callable[..., Any]): - self[name] = RegistryEntry(name, fun, type_id, schema) + self[name] = RegistryEntry( + name, fun, type_id, schema, synchronous=synchronous + ) return fun return decorator diff --git a/tests/unit_tests/test_automation.py b/tests/unit_tests/test_automation.py new file mode 100644 index 0000000000..61fef8201d --- /dev/null +++ b/tests/unit_tests/test_automation.py @@ -0,0 +1,177 @@ +"""Tests for esphome.automation module.""" + +from collections.abc import Generator +from unittest.mock import patch + +import pytest + +from esphome.automation import has_non_synchronous_actions +from esphome.util import RegistryEntry + + +def _make_registry(non_synchronous_actions: set[str]) -> dict[str, RegistryEntry]: + """Create a mock ACTION_REGISTRY with specified non-synchronous actions. + + Uses the default synchronous=False, matching the real registry behavior. + """ + registry: dict[str, RegistryEntry] = {} + for name in non_synchronous_actions: + registry[name] = RegistryEntry(name, lambda: None, None, None) + return registry + + +@pytest.fixture +def mock_registry() -> Generator[dict[str, RegistryEntry]]: + """Fixture that patches ACTION_REGISTRY with delay, wait_until, script.wait as non-synchronous.""" + registry: dict[str, RegistryEntry] = _make_registry( + {"delay", "wait_until", "script.wait"} + ) + registry["logger.log"] = RegistryEntry( + "logger.log", lambda: None, None, None, synchronous=True + ) + with patch("esphome.automation.ACTION_REGISTRY", registry): + yield registry + + +def test_has_non_synchronous_actions_empty_list( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert has_non_synchronous_actions([]) is False + + +def test_has_non_synchronous_actions_empty_dict( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert has_non_synchronous_actions({}) is False + + +def test_has_non_synchronous_actions_non_dict_non_list( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert has_non_synchronous_actions("string") is False + assert has_non_synchronous_actions(42) is False + assert has_non_synchronous_actions(None) is False + + +def test_has_non_synchronous_actions_delay( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert has_non_synchronous_actions([{"delay": "1s"}]) is True + + +def test_has_non_synchronous_actions_wait_until( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert has_non_synchronous_actions([{"wait_until": {"condition": {}}}]) is True + + +def test_has_non_synchronous_actions_script_wait( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert has_non_synchronous_actions([{"script.wait": "script_id"}]) is True + + +def test_has_non_synchronous_actions_synchronous( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert has_non_synchronous_actions([{"logger.log": "hello"}]) is False + + +def test_has_non_synchronous_actions_unknown_not_in_registry( + mock_registry: dict[str, RegistryEntry], +) -> None: + """Unknown actions not in registry are not flagged (only registered actions count).""" + assert has_non_synchronous_actions([{"unknown.action": "value"}]) is False + + +def test_has_non_synchronous_actions_default_non_synchronous( + mock_registry: dict[str, RegistryEntry], +) -> None: + """Actions registered without explicit synchronous=True default to non-synchronous.""" + mock_registry["some.action"] = RegistryEntry( + "some.action", lambda: None, None, None + ) + assert has_non_synchronous_actions([{"some.action": "value"}]) is True + + +def test_has_non_synchronous_actions_nested_in_then( + mock_registry: dict[str, RegistryEntry], +) -> None: + """Non-synchronous action nested inside a synchronous action's then block.""" + actions: list[dict[str, object]] = [ + { + "logger.log": "first", + "then": [{"delay": "1s"}], + } + ] + assert has_non_synchronous_actions(actions) is True + + +def test_has_non_synchronous_actions_deeply_nested( + mock_registry: dict[str, RegistryEntry], +) -> None: + """Non-synchronous action deeply nested in action structure.""" + actions: list[dict[str, object]] = [ + { + "if": { + "then": [ + {"logger.log": "hello"}, + {"delay": "500ms"}, + ] + } + } + ] + assert has_non_synchronous_actions(actions) is True + + +def test_has_non_synchronous_actions_none_in_nested( + mock_registry: dict[str, RegistryEntry], +) -> None: + """No non-synchronous actions even with nesting.""" + actions: list[dict[str, object]] = [ + { + "if": { + "then": [ + {"logger.log": "hello"}, + ] + } + } + ] + assert has_non_synchronous_actions(actions) is False + + +def test_has_non_synchronous_actions_multiple_one_non_synchronous( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert ( + has_non_synchronous_actions( + [ + {"logger.log": "first"}, + {"delay": "1s"}, + {"logger.log": "second"}, + ] + ) + is True + ) + + +def test_has_non_synchronous_actions_multiple_all_synchronous( + mock_registry: dict[str, RegistryEntry], +) -> None: + assert ( + has_non_synchronous_actions( + [ + {"logger.log": "first"}, + {"logger.log": "second"}, + ] + ) + is False + ) + + +def test_has_non_synchronous_actions_dict_input( + mock_registry: dict[str, RegistryEntry], +) -> None: + """Direct dict input (single action).""" + assert has_non_synchronous_actions({"delay": "1s"}) is True + assert has_non_synchronous_actions({"logger.log": "hello"}) is False From 3e7424b307b5763bba217d5f0d5d18130b67be4a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 1 Mar 2026 15:22:55 -1000 Subject: [PATCH 03/20] [preferences] Reduce heap churn with small inline buffer optimization (#13259) --- esphome/components/esp32/preferences.cpp | 33 ++++----- esphome/components/libretiny/preferences.cpp | 33 ++++----- esphome/core/helpers.h | 72 ++++++++++++++++++++ 3 files changed, 96 insertions(+), 42 deletions(-) diff --git a/esphome/components/esp32/preferences.cpp b/esphome/components/esp32/preferences.cpp index 8d6fdc86f6..a3ef10b21f 100644 --- a/esphome/components/esp32/preferences.cpp +++ b/esphome/components/esp32/preferences.cpp @@ -19,16 +19,7 @@ static constexpr size_t KEY_BUFFER_SIZE = 12; struct NVSData { uint32_t key; - std::unique_ptr data; - size_t len; - - void set_data(const uint8_t *src, size_t size) { - if (!this->data || this->len != size) { - this->data = std::make_unique(size); - this->len = size; - } - memcpy(this->data.get(), src, size); - } + SmallInlineBuffer<8> data; // Most prefs fit in 8 bytes (covers fan, cover, select, etc.) }; static std::vector s_pending_save; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) @@ -41,14 +32,14 @@ class ESP32PreferenceBackend : public ESPPreferenceBackend { // try find in pending saves and update that for (auto &obj : s_pending_save) { if (obj.key == this->key) { - obj.set_data(data, len); + obj.data.set(data, len); return true; } } NVSData save{}; save.key = this->key; - save.set_data(data, len); - s_pending_save.emplace_back(std::move(save)); + save.data.set(data, len); + s_pending_save.push_back(std::move(save)); ESP_LOGVV(TAG, "s_pending_save: key: %" PRIu32 ", len: %zu", this->key, len); return true; } @@ -56,11 +47,11 @@ class ESP32PreferenceBackend : public ESPPreferenceBackend { // try find in pending saves and load from that for (auto &obj : s_pending_save) { if (obj.key == this->key) { - if (obj.len != len) { + if (obj.data.size() != len) { // size mismatch return false; } - memcpy(data, obj.data.get(), len); + memcpy(data, obj.data.data(), len); return true; } } @@ -133,10 +124,10 @@ class ESP32Preferences : public ESPPreferences { snprintf(key_str, sizeof(key_str), "%" PRIu32, save.key); ESP_LOGVV(TAG, "Checking if NVS data %s has changed", key_str); if (this->is_changed_(this->nvs_handle, save, key_str)) { - esp_err_t err = nvs_set_blob(this->nvs_handle, key_str, save.data.get(), save.len); - ESP_LOGV(TAG, "sync: key: %s, len: %zu", key_str, save.len); + esp_err_t err = nvs_set_blob(this->nvs_handle, key_str, save.data.data(), save.data.size()); + ESP_LOGV(TAG, "sync: key: %s, len: %zu", key_str, save.data.size()); if (err != 0) { - ESP_LOGV(TAG, "nvs_set_blob('%s', len=%zu) failed: %s", key_str, save.len, esp_err_to_name(err)); + ESP_LOGV(TAG, "nvs_set_blob('%s', len=%zu) failed: %s", key_str, save.data.size(), esp_err_to_name(err)); failed++; last_err = err; last_key = save.key; @@ -144,7 +135,7 @@ class ESP32Preferences : public ESPPreferences { } written++; } else { - ESP_LOGV(TAG, "NVS data not changed skipping %" PRIu32 " len=%zu", save.key, save.len); + ESP_LOGV(TAG, "NVS data not changed skipping %" PRIu32 " len=%zu", save.key, save.data.size()); cached++; } } @@ -176,7 +167,7 @@ class ESP32Preferences : public ESPPreferences { return true; } // Check size first before allocating memory - if (actual_len != to_save.len) { + if (actual_len != to_save.data.size()) { return true; } // Most preferences are small, use stack buffer with heap fallback for large ones @@ -186,7 +177,7 @@ class ESP32Preferences : public ESPPreferences { ESP_LOGV(TAG, "nvs_get_blob('%s') failed: %s", key_str, esp_err_to_name(err)); return true; } - return memcmp(to_save.data.get(), stored_data.get(), to_save.len) != 0; + return memcmp(to_save.data.data(), stored_data.get(), to_save.data.size()) != 0; } bool reset() override { diff --git a/esphome/components/libretiny/preferences.cpp b/esphome/components/libretiny/preferences.cpp index 740c1a233a..1c101136e1 100644 --- a/esphome/components/libretiny/preferences.cpp +++ b/esphome/components/libretiny/preferences.cpp @@ -17,16 +17,7 @@ static constexpr size_t KEY_BUFFER_SIZE = 12; struct NVSData { uint32_t key; - std::unique_ptr data; - size_t len; - - void set_data(const uint8_t *src, size_t size) { - if (!this->data || this->len != size) { - this->data = std::make_unique(size); - this->len = size; - } - memcpy(this->data.get(), src, size); - } + SmallInlineBuffer<8> data; // Most prefs fit in 8 bytes (covers fan, cover, select, etc.) }; static std::vector s_pending_save; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) @@ -41,14 +32,14 @@ class LibreTinyPreferenceBackend : public ESPPreferenceBackend { // try find in pending saves and update that for (auto &obj : s_pending_save) { if (obj.key == this->key) { - obj.set_data(data, len); + obj.data.set(data, len); return true; } } NVSData save{}; save.key = this->key; - save.set_data(data, len); - s_pending_save.emplace_back(std::move(save)); + save.data.set(data, len); + s_pending_save.push_back(std::move(save)); ESP_LOGVV(TAG, "s_pending_save: key: %" PRIu32 ", len: %zu", this->key, len); return true; } @@ -57,11 +48,11 @@ class LibreTinyPreferenceBackend : public ESPPreferenceBackend { // try find in pending saves and load from that for (auto &obj : s_pending_save) { if (obj.key == this->key) { - if (obj.len != len) { + if (obj.data.size() != len) { // size mismatch return false; } - memcpy(data, obj.data.get(), len); + memcpy(data, obj.data.data(), len); return true; } } @@ -122,11 +113,11 @@ class LibreTinyPreferences : public ESPPreferences { snprintf(key_str, sizeof(key_str), "%" PRIu32, save.key); ESP_LOGVV(TAG, "Checking if FDB data %s has changed", key_str); if (this->is_changed_(&this->db, save, key_str)) { - ESP_LOGV(TAG, "sync: key: %s, len: %zu", key_str, save.len); - fdb_blob_make(&this->blob, save.data.get(), save.len); + ESP_LOGV(TAG, "sync: key: %s, len: %zu", key_str, save.data.size()); + fdb_blob_make(&this->blob, save.data.data(), save.data.size()); fdb_err_t err = fdb_kv_set_blob(&this->db, key_str, &this->blob); if (err != FDB_NO_ERR) { - ESP_LOGV(TAG, "fdb_kv_set_blob('%s', len=%zu) failed: %d", key_str, save.len, err); + ESP_LOGV(TAG, "fdb_kv_set_blob('%s', len=%zu) failed: %d", key_str, save.data.size(), err); failed++; last_err = err; last_key = save.key; @@ -134,7 +125,7 @@ class LibreTinyPreferences : public ESPPreferences { } written++; } else { - ESP_LOGD(TAG, "FDB data not changed; skipping %" PRIu32 " len=%zu", save.key, save.len); + ESP_LOGD(TAG, "FDB data not changed; skipping %" PRIu32 " len=%zu", save.key, save.data.size()); cached++; } } @@ -159,7 +150,7 @@ class LibreTinyPreferences : public ESPPreferences { } // Check size first - if different, data has changed - if (kv.value_len != to_save.len) { + if (kv.value_len != to_save.data.size()) { return true; } @@ -173,7 +164,7 @@ class LibreTinyPreferences : public ESPPreferences { } // Compare the actual data - return memcmp(to_save.data.get(), stored_data.get(), kv.value_len) != 0; + return memcmp(to_save.data.data(), stored_data.get(), kv.value_len) != 0; } bool reset() override { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index b606e68df3..65d590a5e6 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -133,6 +133,78 @@ template class ConstVector { size_t size_; }; +/// Small buffer optimization - stores data inline when small, heap-allocates for large data +/// This avoids heap fragmentation for common small allocations while supporting arbitrary sizes. +/// Memory management is encapsulated - callers just use set() and data(). +template class SmallInlineBuffer { + public: + SmallInlineBuffer() = default; + ~SmallInlineBuffer() { + if (!this->is_inline_()) + delete[] this->heap_; + } + + // Move constructor + SmallInlineBuffer(SmallInlineBuffer &&other) noexcept : len_(other.len_) { + if (other.is_inline_()) { + memcpy(this->inline_, other.inline_, this->len_); + } else { + this->heap_ = other.heap_; + other.heap_ = nullptr; + } + other.len_ = 0; + } + + // Move assignment + SmallInlineBuffer &operator=(SmallInlineBuffer &&other) noexcept { + if (this != &other) { + if (!this->is_inline_()) + delete[] this->heap_; + this->len_ = other.len_; + if (other.is_inline_()) { + memcpy(this->inline_, other.inline_, this->len_); + } else { + this->heap_ = other.heap_; + other.heap_ = nullptr; + } + other.len_ = 0; + } + return *this; + } + + // Disable copy (would need deep copy of heap data) + SmallInlineBuffer(const SmallInlineBuffer &) = delete; + SmallInlineBuffer &operator=(const SmallInlineBuffer &) = delete; + + /// Set buffer contents, allocating heap if needed + void set(const uint8_t *src, size_t size) { + // Free existing heap allocation if switching from heap to inline or different heap size + if (!this->is_inline_() && (size <= InlineSize || size != this->len_)) { + delete[] this->heap_; + this->heap_ = nullptr; // Defensive: prevent use-after-free if logic changes + } + // Allocate new heap buffer if needed + if (size > InlineSize && (this->is_inline_() || size != this->len_)) { + this->heap_ = new uint8_t[size]; // NOLINT(cppcoreguidelines-owning-memory) + } + this->len_ = size; + memcpy(this->data(), src, size); + } + + uint8_t *data() { return this->is_inline_() ? this->inline_ : this->heap_; } + const uint8_t *data() const { return this->is_inline_() ? this->inline_ : this->heap_; } + size_t size() const { return this->len_; } + + protected: + bool is_inline_() const { return this->len_ <= InlineSize; } + + size_t len_{0}; + union { + uint8_t inline_[InlineSize]{}; // Zero-init ensures clean initial state + uint8_t *heap_; + }; +}; + /// Minimal static vector - saves memory by avoiding std::vector overhead template class StaticVector { public: From 0e18e4461e8d82df75b3acbb61c7dcc8c9993311 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 1 Mar 2026 16:52:53 -1000 Subject: [PATCH 04/20] [time,api] Send pre-parsed timezone struct over protobuf (#14233) Co-authored-by: Claude Opus 4.6 --- esphome/components/api/api.proto | 28 ++++++++++ esphome/components/api/api_connection.cpp | 25 ++++++++- esphome/components/api/api_pb2.cpp | 54 ++++++++++++++++++ esphome/components/api/api_pb2.h | 38 ++++++++++++- esphome/components/api/api_pb2_dump.cpp | 39 +++++++++++++ esphome/components/time/__init__.py | 68 +++++++++++++++++++++-- 6 files changed, 245 insertions(+), 7 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index d7f32cd8d1..802e3e3ae2 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -834,6 +834,33 @@ message GetTimeRequest { option (source) = SOURCE_SERVER; } +enum DSTRuleType { + DST_RULE_TYPE_NONE = 0; + DST_RULE_TYPE_MONTH_WEEK_DAY = 1; + DST_RULE_TYPE_JULIAN_NO_LEAP = 2; + DST_RULE_TYPE_DAY_OF_YEAR = 3; +} + +message DSTRule { + option (source) = SOURCE_CLIENT; + + sint32 time_seconds = 1; + uint32 day = 2; + DSTRuleType type = 3; + uint32 month = 4; + uint32 week = 5; + uint32 day_of_week = 6; +} + +message ParsedTimezone { + option (source) = SOURCE_CLIENT; + + sint32 std_offset_seconds = 1; + sint32 dst_offset_seconds = 2; + DSTRule dst_start = 3; + DSTRule dst_end = 4; +} + message GetTimeResponse { option (id) = 37; option (source) = SOURCE_CLIENT; @@ -841,6 +868,7 @@ message GetTimeResponse { fixed32 epoch_seconds = 1; string timezone = 2; + ParsedTimezone parsed_timezone = 3; } // ==================== USER-DEFINES SERVICES ==================== diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 215af611db..90287ec2dd 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1123,7 +1123,30 @@ void APIConnection::on_get_time_response(const GetTimeResponse &value) { homeassistant::global_homeassistant_time->set_epoch_time(value.epoch_seconds); #ifdef USE_TIME_TIMEZONE if (!value.timezone.empty()) { - homeassistant::global_homeassistant_time->set_timezone(value.timezone.c_str(), value.timezone.size()); + // Check if the sender provided pre-parsed timezone data. + // If std_offset is non-zero or DST rules are present, the parsed data was populated. + // For UTC (all zeros), string parsing produces the same result, so the fallback is equivalent. + const auto &pt = value.parsed_timezone; + if (pt.std_offset_seconds != 0 || pt.dst_start.type != enums::DST_RULE_TYPE_NONE) { + time::ParsedTimezone tz{}; + tz.std_offset_seconds = pt.std_offset_seconds; + tz.dst_offset_seconds = pt.dst_offset_seconds; + tz.dst_start.time_seconds = pt.dst_start.time_seconds; + tz.dst_start.day = static_cast(pt.dst_start.day); + tz.dst_start.type = static_cast(pt.dst_start.type); + tz.dst_start.month = static_cast(pt.dst_start.month); + tz.dst_start.week = static_cast(pt.dst_start.week); + tz.dst_start.day_of_week = static_cast(pt.dst_start.day_of_week); + tz.dst_end.time_seconds = pt.dst_end.time_seconds; + tz.dst_end.day = static_cast(pt.dst_end.day); + tz.dst_end.type = static_cast(pt.dst_end.type); + tz.dst_end.month = static_cast(pt.dst_end.month); + tz.dst_end.week = static_cast(pt.dst_end.week); + tz.dst_end.day_of_week = static_cast(pt.dst_end.day_of_week); + time::set_global_tz(tz); + } else { + homeassistant::global_homeassistant_time->set_timezone(value.timezone.c_str(), value.timezone.size()); + } } #endif } diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 5c50a8aa5b..9e74d5ddc7 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -954,12 +954,66 @@ bool HomeAssistantStateResponse::decode_length(uint32_t field_id, ProtoLengthDel return true; } #endif +bool DSTRule::decode_varint(uint32_t field_id, ProtoVarInt value) { + switch (field_id) { + case 1: + this->time_seconds = value.as_sint32(); + break; + case 2: + this->day = value.as_uint32(); + break; + case 3: + this->type = static_cast(value.as_uint32()); + break; + case 4: + this->month = value.as_uint32(); + break; + case 5: + this->week = value.as_uint32(); + break; + case 6: + this->day_of_week = value.as_uint32(); + break; + default: + return false; + } + return true; +} +bool ParsedTimezone::decode_varint(uint32_t field_id, ProtoVarInt value) { + switch (field_id) { + case 1: + this->std_offset_seconds = value.as_sint32(); + break; + case 2: + this->dst_offset_seconds = value.as_sint32(); + break; + default: + return false; + } + return true; +} +bool ParsedTimezone::decode_length(uint32_t field_id, ProtoLengthDelimited value) { + switch (field_id) { + case 3: + value.decode_to_message(this->dst_start); + break; + case 4: + value.decode_to_message(this->dst_end); + break; + default: + return false; + } + return true; +} bool GetTimeResponse::decode_length(uint32_t field_id, ProtoLengthDelimited value) { switch (field_id) { case 2: { this->timezone = StringRef(reinterpret_cast(value.data()), value.size()); break; } + case 3: + value.decode_to_message(this->parsed_timezone); + break; default: return false; } diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 22dc3de995..033b789c18 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -63,6 +63,12 @@ enum LogLevel : uint32_t { LOG_LEVEL_VERBOSE = 6, LOG_LEVEL_VERY_VERBOSE = 7, }; +enum DSTRuleType : uint32_t { + DST_RULE_TYPE_NONE = 0, + DST_RULE_TYPE_MONTH_WEEK_DAY = 1, + DST_RULE_TYPE_JULIAN_NO_LEAP = 2, + DST_RULE_TYPE_DAY_OF_YEAR = 3, +}; #ifdef USE_API_USER_DEFINED_ACTIONS enum ServiceArgType : uint32_t { SERVICE_ARG_TYPE_BOOL = 0, @@ -1116,15 +1122,45 @@ class GetTimeRequest final : public ProtoMessage { protected: }; +class DSTRule final : public ProtoDecodableMessage { + public: + int32_t time_seconds{0}; + uint32_t day{0}; + enums::DSTRuleType type{}; + uint32_t month{0}; + uint32_t week{0}; + uint32_t day_of_week{0}; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: + bool decode_varint(uint32_t field_id, ProtoVarInt value) override; +}; +class ParsedTimezone final : public ProtoDecodableMessage { + public: + int32_t std_offset_seconds{0}; + int32_t dst_offset_seconds{0}; + DSTRule dst_start{}; + DSTRule dst_end{}; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: + bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override; + bool decode_varint(uint32_t field_id, ProtoVarInt value) override; +}; class GetTimeResponse final : public ProtoDecodableMessage { public: static constexpr uint8_t MESSAGE_TYPE = 37; - static constexpr uint8_t ESTIMATED_SIZE = 14; + static constexpr uint8_t ESTIMATED_SIZE = 31; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "get_time_response"; } #endif uint32_t epoch_seconds{0}; StringRef timezone{}; + ParsedTimezone parsed_timezone{}; #ifdef HAS_PROTO_MESSAGE_DUMP const char *dump_to(DumpBuffer &out) const override; #endif diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 52d2486410..4eec42e936 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -208,6 +208,20 @@ template<> const char *proto_enum_to_string(enums::LogLevel val return "UNKNOWN"; } } +template<> const char *proto_enum_to_string(enums::DSTRuleType value) { + switch (value) { + case enums::DST_RULE_TYPE_NONE: + return "DST_RULE_TYPE_NONE"; + case enums::DST_RULE_TYPE_MONTH_WEEK_DAY: + return "DST_RULE_TYPE_MONTH_WEEK_DAY"; + case enums::DST_RULE_TYPE_JULIAN_NO_LEAP: + return "DST_RULE_TYPE_JULIAN_NO_LEAP"; + case enums::DST_RULE_TYPE_DAY_OF_YEAR: + return "DST_RULE_TYPE_DAY_OF_YEAR"; + default: + return "UNKNOWN"; + } +} #ifdef USE_API_USER_DEFINED_ACTIONS template<> const char *proto_enum_to_string(enums::ServiceArgType value) { switch (value) { @@ -1254,10 +1268,35 @@ const char *GetTimeRequest::dump_to(DumpBuffer &out) const { out.append("GetTimeRequest {}"); return out.c_str(); } +const char *DSTRule::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, "DSTRule"); + dump_field(out, "time_seconds", this->time_seconds); + dump_field(out, "day", this->day); + dump_field(out, "type", static_cast(this->type)); + dump_field(out, "month", this->month); + dump_field(out, "week", this->week); + dump_field(out, "day_of_week", this->day_of_week); + return out.c_str(); +} +const char *ParsedTimezone::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, "ParsedTimezone"); + dump_field(out, "std_offset_seconds", this->std_offset_seconds); + dump_field(out, "dst_offset_seconds", this->dst_offset_seconds); + out.append(" dst_start: "); + this->dst_start.dump_to(out); + out.append("\n"); + out.append(" dst_end: "); + this->dst_end.dump_to(out); + out.append("\n"); + return out.c_str(); +} const char *GetTimeResponse::dump_to(DumpBuffer &out) const { MessageDumpHelper helper(out, "GetTimeResponse"); dump_field(out, "epoch_seconds", this->epoch_seconds); dump_field(out, "timezone", this->timezone); + out.append(" parsed_timezone: "); + this->parsed_timezone.dump_to(out); + out.append("\n"); return out.c_str(); } #ifdef USE_API_USER_DEFINED_ACTIONS diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index a20d79b857..7ffa408db9 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -1,6 +1,10 @@ from importlib import resources import logging +from aioesphomeapi.posix_tz import ( + DSTRuleType as PyDSTRuleType, + parse_posix_tz as parse_posix_tz_python, +) import tzlocal from esphome import automation @@ -39,6 +43,19 @@ CronTrigger = time_ns.class_("CronTrigger", automation.Trigger.template(), cg.Co SyncTrigger = time_ns.class_("SyncTrigger", automation.Trigger.template(), cg.Component) TimeHasTimeCondition = time_ns.class_("TimeHasTimeCondition", Condition) +# C++ types for pre-parsed timezone struct generation +DSTRuleType_cpp = time_ns.enum("DSTRuleType", is_class=True) +DSTRule_cpp = time_ns.struct("DSTRule") +ParsedTimezone_cpp = time_ns.struct("ParsedTimezone") + +# Map Python DSTRuleType enum values to C++ enum expressions +_DST_RULE_TYPE_MAP = { + PyDSTRuleType.NONE: DSTRuleType_cpp.NONE, + PyDSTRuleType.MONTH_WEEK_DAY: DSTRuleType_cpp.MONTH_WEEK_DAY, + PyDSTRuleType.JULIAN_NO_LEAP: DSTRuleType_cpp.JULIAN_NO_LEAP, + PyDSTRuleType.DAY_OF_YEAR: DSTRuleType_cpp.DAY_OF_YEAR, +} + def _load_tzdata(iana_key: str) -> bytes | None: # From https://tzdata.readthedocs.io/en/latest/#examples @@ -260,11 +277,17 @@ def validate_tz(value: str) -> str: value = cv.string_strict(value) tzfile = _load_tzdata(value) - if tzfile is None: - # Not a IANA key, probably a TZ string - return value + if tzfile is not None: + value = _extract_tz_string(tzfile) - return _extract_tz_string(tzfile) + # Validate that the POSIX TZ string is parseable (skip empty strings) + if value: + try: + parse_posix_tz_python(value) + except ValueError as e: + raise cv.Invalid(f"Invalid POSIX timezone string '{value}': {e}") from e + + return value TIME_SCHEMA = cv.Schema( @@ -305,11 +328,46 @@ TIME_SCHEMA = cv.Schema( ).extend(cv.polling_component_schema("15min")) +def _emit_dst_rule_fields(prefix, rule): + """Emit field-by-field assignments for a DSTRule to avoid rodata struct blob.""" + cg.add(cg.RawExpression(f"{prefix}.time_seconds = {rule.time_seconds}")) + cg.add(cg.RawExpression(f"{prefix}.day = {rule.day}")) + cg.add(cg.RawExpression(f"{prefix}.type = {_DST_RULE_TYPE_MAP[rule.type]}")) + cg.add(cg.RawExpression(f"{prefix}.month = {rule.month}")) + cg.add(cg.RawExpression(f"{prefix}.week = {rule.week}")) + cg.add(cg.RawExpression(f"{prefix}.day_of_week = {rule.day_of_week}")) + + +def _emit_parsed_timezone_fields(parsed): + """Emit field-by-field assignments for a local ParsedTimezone, then set_global_tz(). + + Uses individual assignments on a stack variable instead of a struct initializer + to keep constants as immediate operands in instructions (.irom0.text/flash) + rather than a const blob in .rodata (which maps to RAM on ESP8266). + Wrapped in a scope block to allow multiple time platforms in the same build. + """ + cg.add(cg.RawStatement("{")) + cg.add(cg.RawExpression("time::ParsedTimezone tz{}")) + cg.add(cg.RawExpression(f"tz.std_offset_seconds = {parsed.std_offset_seconds}")) + cg.add(cg.RawExpression(f"tz.dst_offset_seconds = {parsed.dst_offset_seconds}")) + _emit_dst_rule_fields("tz.dst_start", parsed.dst_start) + _emit_dst_rule_fields("tz.dst_end", parsed.dst_end) + cg.add(time_ns.set_global_tz(cg.RawExpression("tz"))) + cg.add(cg.RawStatement("}")) + + async def setup_time_core_(time_var, config): if timezone := config.get(CONF_TIMEZONE): - cg.add(time_var.set_timezone(timezone)) cg.add_define("USE_TIME_TIMEZONE") + if CORE.is_host: + # Host platform needs setenv("TZ")/tzset() for libc compatibility + cg.add(time_var.set_timezone(timezone)) + else: + # Embedded: pre-parse at codegen time, emit struct directly + parsed = parse_posix_tz_python(timezone) + _emit_parsed_timezone_fields(parsed) + for conf in config.get(CONF_ON_TIME, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], time_var) From 073ca63f60bb23a7c84106eb57a5ce06388b1295 Mon Sep 17 00:00:00 2001 From: Thomas Rupprecht Date: Mon, 2 Mar 2026 04:44:02 +0100 Subject: [PATCH 05/20] [rtttl] improve comments Part 2 (#13971) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/rtttl/rtttl.cpp | 189 ++++++++++++++++------------- esphome/components/rtttl/rtttl.h | 31 ++--- 2 files changed, 119 insertions(+), 101 deletions(-) diff --git a/esphome/components/rtttl/rtttl.cpp b/esphome/components/rtttl/rtttl.cpp index ab95067f45..4ccfc539ea 100644 --- a/esphome/components/rtttl/rtttl.cpp +++ b/esphome/components/rtttl/rtttl.cpp @@ -8,18 +8,26 @@ namespace esphome::rtttl { static const char *const TAG = "rtttl"; -// These values can also be found as constants in the Tone library (Tone.h) -static const uint16_t NOTES[] = {0, 262, 277, 294, 311, 330, 349, 370, 392, 415, 440, 466, 494, - 523, 554, 587, 622, 659, 698, 740, 784, 831, 880, 932, 988, 1047, - 1109, 1175, 1245, 1319, 1397, 1480, 1568, 1661, 1760, 1865, 1976, 2093, 2217, - 2349, 2489, 2637, 2794, 2960, 3136, 3322, 3520, 3729, 3951}; +static constexpr uint8_t SONG_NAME_LENGTH_LIMIT = 64; +static constexpr uint8_t SEMITONES_IN_OCTAVE = 12; -#if defined(USE_OUTPUT) || defined(USE_SPEAKER) -static const uint32_t DOUBLE_NOTE_GAP_MS = 10; -#endif // USE_OUTPUT || USE_SPEAKER +static constexpr uint8_t MIN_OCTAVE = 4; +static constexpr uint8_t MAX_OCTAVE = 7; + +static constexpr uint8_t DEFAULT_BPM = 63; // Default beats per minute + +// These values can also be found as constants in the Tone library (Tone.h) +static constexpr uint16_t NOTES[] = {0, 262, 277, 294, 311, 330, 349, 370, 392, 415, 440, 466, 494, + 523, 554, 587, 622, 659, 698, 740, 784, 831, 880, 932, 988, 1047, + 1109, 1175, 1245, 1319, 1397, 1480, 1568, 1661, 1760, 1865, 1976, 2093, 2217, + 2349, 2489, 2637, 2794, 2960, 3136, 3322, 3520, 3729, 3951}; +static constexpr uint8_t NOTES_COUNT = static_cast(sizeof(NOTES) / sizeof(NOTES[0])); + +static constexpr uint8_t REPEATING_NOTE_GAP_MS = 10; #ifdef USE_SPEAKER -static const size_t SAMPLE_BUFFER_SIZE = 2048; +static constexpr uint16_t SAMPLE_BUFFER_SIZE = 2048; +static constexpr uint16_t SAMPLE_RATE = 16000; struct SpeakerSample { int8_t left{0}; @@ -27,7 +35,7 @@ struct SpeakerSample { }; inline double deg2rad(double degrees) { - static const double PI_ON_180 = 4.0 * atan(1.0) / 180.0; + static constexpr double PI_ON_180 = M_PI / 180.0; return degrees * PI_ON_180; } #endif // USE_SPEAKER @@ -85,7 +93,7 @@ void Rtttl::loop() { } #ifdef USE_OUTPUT - if (this->output_ != nullptr && millis() - this->last_note_ < this->note_duration_) { + if (this->output_ != nullptr && millis() - this->last_note_start_time_ < this->note_duration_) { return; } #endif // USE_OUTPUT @@ -113,36 +121,34 @@ void Rtttl::loop() { } if (this->samples_sent_ != this->samples_count_) { SpeakerSample sample[SAMPLE_BUFFER_SIZE + 2]; - int x = 0; + uint16_t sample_index = 0; double rem = 0.0; while (true) { - // Try and send out the remainder of the existing note, one per loop() - - if (this->samples_per_wave_ != 0 && this->samples_sent_ >= this->samples_gap_) { // Play note// + // Try and send out the remainder of the existing note, one per `loop()` + if (this->samples_per_wave_ != 0 && this->samples_sent_ >= this->samples_gap_) { // Play note rem = ((this->samples_sent_ << 10) % this->samples_per_wave_) * (360.0 / this->samples_per_wave_); - int16_t val = (127 * this->gain_) * sin(deg2rad(rem)); // 16bit = 49152 - - sample[x].left = val; - sample[x].right = val; + int8_t val = (127 * this->gain_) * sin(deg2rad(rem)); + sample[sample_index].left = val; + sample[sample_index].right = val; } else { - sample[x].left = 0; - sample[x].right = 0; + sample[sample_index].left = 0; + sample[sample_index].right = 0; } - if (static_cast(x) >= SAMPLE_BUFFER_SIZE || this->samples_sent_ >= this->samples_count_) { + if (sample_index >= SAMPLE_BUFFER_SIZE || this->samples_sent_ >= this->samples_count_) { break; } this->samples_sent_++; - x++; + sample_index++; } - if (x > 0) { - size_t bytes_to_send = x * sizeof(SpeakerSample); + if (sample_index > 0) { + size_t bytes_to_send = sample_index * sizeof(SpeakerSample); size_t send = this->speaker_->play((uint8_t *) (&sample), bytes_to_send); if (send != bytes_to_send) { - this->samples_sent_ -= (x - (send / sizeof(SpeakerSample))); + this->samples_sent_ -= (sample_index - (send / sizeof(SpeakerSample))); } return; } @@ -155,83 +161,84 @@ void Rtttl::loop() { return; } - // align to note: most rtttl's out there does not add and space after the ',' separator but just in case... + // Align to note: most rtttl's out there does not add any space after the ',' separator but just in case while (this->rtttl_[this->position_] == ',' || this->rtttl_[this->position_] == ' ') { this->position_++; } - // first, get note duration, if available - uint8_t num = this->get_integer_(); + // First, get note duration, if available + uint8_t note_denominator = this->get_integer_(); - if (num) { - this->note_duration_ = this->wholenote_ / num; + if (note_denominator) { + this->note_duration_ = this->wholenote_duration_ / note_denominator; } else { - this->note_duration_ = - this->wholenote_ / this->default_duration_; // we will need to check if we are a dotted note after + // We will need to check if we are a dotted note after + this->note_duration_ = this->wholenote_duration_ / this->default_note_denominator_; } - uint8_t note = note_index_from_char(this->rtttl_[this->position_]); + uint8_t note_index_in_octave = note_index_from_char(this->rtttl_[this->position_]); this->position_++; - // now, get optional '#' sharp + // Now, get optional '#' sharp if (this->rtttl_[this->position_] == '#') { - note++; + note_index_in_octave++; this->position_++; } - // now, get scale + // Now, get scale uint8_t scale = this->get_integer_(); if (scale == 0) { scale = this->default_octave_; } - if (scale < 4 || scale > 7) { - ESP_LOGE(TAG, "Octave must be between 4 and 7 (it is %d)", scale); + if (scale < MIN_OCTAVE || scale > MAX_OCTAVE) { + ESP_LOGE(TAG, "Octave must be between %d and %d (it is %d)", MIN_OCTAVE, MAX_OCTAVE, scale); this->finish_(); return; } - // now, get optional '.' dotted note + // Now, get optional '.' dotted note if (this->rtttl_[this->position_] == '.') { - this->note_duration_ += this->note_duration_ / 2; + this->note_duration_ += this->note_duration_ / 2; // Duration +50% this->position_++; } - // Now play the note bool need_note_gap = false; - if (note) { - auto note_index = (scale - 4) * 12 + note; - if (note_index < 0 || note_index >= (int) (sizeof(NOTES) / sizeof(NOTES[0]))) { - ESP_LOGE(TAG, "Note out of range (note: %d, scale: %d, index: %d, max: %d)", note, scale, note_index, - (int) (sizeof(NOTES) / sizeof(NOTES[0]))); + + // Now play the note + if (note_index_in_octave == 0) { + this->output_freq_ = 0; + ESP_LOGVV(TAG, "Waiting: %dms", this->note_duration_); + } else { + uint8_t note_index = (scale - MIN_OCTAVE) * SEMITONES_IN_OCTAVE + note_index_in_octave; + if (note_index >= NOTES_COUNT) { + ESP_LOGE(TAG, "Note out of range (note: %d, scale: %d, index: %d, max: %d)", note_index_in_octave, scale, + note_index, NOTES_COUNT); this->finish_(); return; } - auto freq = NOTES[note_index]; + uint16_t freq = NOTES[note_index]; need_note_gap = freq == this->output_freq_; // Add small silence gap between same note this->output_freq_ = freq; - ESP_LOGVV(TAG, "playing note: %d for %dms", note, this->note_duration_); - } else { - ESP_LOGVV(TAG, "waiting: %dms", this->note_duration_); - this->output_freq_ = 0; + ESP_LOGVV(TAG, "Playing note: %d for %dms", note_index_in_octave, this->note_duration_); } #ifdef USE_OUTPUT if (this->output_ != nullptr) { - if (need_note_gap && this->note_duration_ > DOUBLE_NOTE_GAP_MS) { + if (this->output_freq_ == 0) { this->output_->set_level(0.0); - delay(DOUBLE_NOTE_GAP_MS); - this->note_duration_ -= DOUBLE_NOTE_GAP_MS; - } - if (this->output_freq_ != 0) { + } else { + if (need_note_gap && this->note_duration_ > REPEATING_NOTE_GAP_MS) { + this->output_->set_level(0.0); + delay(REPEATING_NOTE_GAP_MS); + this->note_duration_ -= REPEATING_NOTE_GAP_MS; + } this->output_->update_frequency(this->output_freq_); this->output_->set_level(this->gain_); - } else { - this->output_->set_level(0.0); } } #endif // USE_OUTPUT @@ -241,28 +248,26 @@ void Rtttl::loop() { this->samples_sent_ = 0; this->samples_gap_ = 0; this->samples_per_wave_ = 0; - this->samples_count_ = (this->sample_rate_ * this->note_duration_) / 1000; + this->samples_count_ = (SAMPLE_RATE * this->note_duration_) / 1000; if (need_note_gap) { - this->samples_gap_ = (this->sample_rate_ * DOUBLE_NOTE_GAP_MS) / 1000; + this->samples_gap_ = (SAMPLE_RATE * REPEATING_NOTE_GAP_MS) / 1000; } if (this->output_freq_ != 0) { - // make sure there is enough samples to add a full last sinus. - - uint16_t samples_wish = this->samples_count_; - this->samples_per_wave_ = (this->sample_rate_ << 10) / this->output_freq_; + // Make sure there is enough samples to add a full last sinus. + uint32_t samples_wish = this->samples_count_; + this->samples_per_wave_ = (SAMPLE_RATE << 10) / this->output_freq_; uint16_t division = ((this->samples_count_ << 10) / this->samples_per_wave_) + 1; - this->samples_count_ = (division * this->samples_per_wave_); - this->samples_count_ = this->samples_count_ >> 10; - ESP_LOGVV(TAG, "- Calc play time: wish: %d gets: %d (div: %d spw: %d)", samples_wish, this->samples_count_, - division, this->samples_per_wave_); + this->samples_count_ = (division * this->samples_per_wave_) >> 10; + ESP_LOGVV(TAG, "Calc play time: wish: %" PRIu32 " gets: %" PRIu32 " (div: %d spw: %" PRIu32 ")", samples_wish, + this->samples_count_, division, this->samples_per_wave_); } // Convert from frequency in Hz to high and low samples in fixed point } #endif // USE_SPEAKER - this->last_note_ = millis(); + this->last_note_start_time_ = millis(); } void Rtttl::play(std::string rtttl) { @@ -275,25 +280,28 @@ void Rtttl::play(std::string rtttl) { this->rtttl_ = std::move(rtttl); - this->default_duration_ = 4; - this->default_octave_ = 6; + this->default_note_denominator_ = DEFAULT_NOTE_DENOMINATOR; + this->default_octave_ = DEFAULT_OCTAVE; this->note_duration_ = 0; - int bpm = 63; - uint16_t num; + uint16_t bpm = DEFAULT_BPM; + uint16_t num; // Used for: default note-denominator, default octave, BPM // Get name this->position_ = this->rtttl_.find(':'); - // it's somewhat documented to be up to 10 characters but let's be a bit flexible here - if (this->position_ == std::string::npos || this->position_ > 15) { + if (this->position_ == std::string::npos) { ESP_LOGE(TAG, "Unable to determine name; missing ':'"); return; } - + if (this->position_ >= SONG_NAME_LENGTH_LIMIT) { + ESP_LOGE(TAG, "Name is too long: length=%u, limit=%u", static_cast(this->position_), + static_cast(SONG_NAME_LENGTH_LIMIT)); + return; + } ESP_LOGD(TAG, "Playing song %.*s", (int) this->position_, this->rtttl_.c_str()); - // get default duration + // Get default duration this->position_ = this->rtttl_.find("d=", this->position_); if (this->position_ == std::string::npos) { ESP_LOGE(TAG, "Missing 'd='"); @@ -301,11 +309,14 @@ void Rtttl::play(std::string rtttl) { } this->position_ += 2; num = this->get_integer_(); - if (num > 0) { - this->default_duration_ = num; + if (num == 1 || num == 2 || num == 4 || num == 8 || num == 16 || num == 32) { + this->default_note_denominator_ = num; + } else { + ESP_LOGE(TAG, "Invalid default duration: %d", num); + return; } - // get default octave + // Get default octave this->position_ = this->rtttl_.find("o=", this->position_); if (this->position_ == std::string::npos) { ESP_LOGE(TAG, "Missing 'o="); @@ -313,11 +324,14 @@ void Rtttl::play(std::string rtttl) { } this->position_ += 2; num = this->get_integer_(); - if (num >= 3 && num <= 7) { + if (num >= MIN_OCTAVE && num <= MAX_OCTAVE) { this->default_octave_ = num; + } else { + ESP_LOGE(TAG, "Invalid default octave: %d", num); + return; } - // get BPM + // Get BPM this->position_ = this->rtttl_.find("b=", this->position_); if (this->position_ == std::string::npos) { ESP_LOGE(TAG, "Missing b="); @@ -325,8 +339,11 @@ void Rtttl::play(std::string rtttl) { } this->position_ += 2; num = this->get_integer_(); - if (num != 0) { + if (num >= 4) { // Below 4 is not realistic and would cause a integer overflow bpm = num; + } else { + ESP_LOGE(TAG, "Invalid BPM: %d", num); + return; } this->position_ = this->rtttl_.find(':', this->position_); @@ -337,10 +354,10 @@ void Rtttl::play(std::string rtttl) { this->position_++; // BPM usually expresses the number of quarter notes per minute - this->wholenote_ = 60 * 1000L * 4 / bpm; // this is the time for whole note (in milliseconds) + this->wholenote_duration_ = 60 * 1000L * 4 / bpm; // This is the time for whole note (in milliseconds) this->output_freq_ = 0; - this->last_note_ = millis(); + this->last_note_start_time_ = millis(); this->note_duration_ = 1; #ifdef USE_OUTPUT diff --git a/esphome/components/rtttl/rtttl.h b/esphome/components/rtttl/rtttl.h index 4d4a652c51..e37cccae9e 100644 --- a/esphome/components/rtttl/rtttl.h +++ b/esphome/components/rtttl/rtttl.h @@ -13,6 +13,10 @@ namespace esphome::rtttl { +inline constexpr uint8_t DEFAULT_NOTE_DENOMINATOR = 4; // Default note-denominator (quarter note) +inline constexpr uint8_t DEFAULT_OCTAVE = + 6; // Default octave for a note (see: `MIN_OCTAVE`, `MAX_OCTAVE` in `rtttl.cpp`) + enum class State : uint8_t { STOPPED = 0, INIT, @@ -67,19 +71,18 @@ class Rtttl : public Component { std::string rtttl_{""}; /// The current position in the RTTTL string. size_t position_{0}; - /// The duration of a whole note in milliseconds. - uint16_t wholenote_; /// The default duration of a note (e.g. 4 for a quarter note). - uint16_t default_duration_; + uint8_t default_note_denominator_{DEFAULT_NOTE_DENOMINATOR}; /// The default octave for a note. - uint16_t default_octave_; - /// The time the last note was started. - uint32_t last_note_; + uint8_t default_octave_{DEFAULT_OCTAVE}; /// The duration of the current note in milliseconds. - uint16_t note_duration_; - + uint16_t note_duration_{0}; + /// The duration of a whole note in milliseconds. + uint16_t wholenote_duration_; + /// The time in milliseconds since microcontroller boot when the last note was started. + uint32_t last_note_start_time_; /// The frequency of the current note in Hz. - uint32_t output_freq_; + uint32_t output_freq_{0}; /// The gain of the output. float gain_{0.6f}; /// The current state of the RTTTL player. @@ -93,16 +96,14 @@ class Rtttl : public Component { #ifdef USE_SPEAKER /// The speaker to write the sound to. speaker::Speaker *speaker_{nullptr}; - /// The sample rate of the speaker. - int sample_rate_{16000}; /// The number of samples for one full cycle of a note's waveform, in Q10 fixed-point format. - int samples_per_wave_{0}; + uint32_t samples_per_wave_{0}; /// The number of samples sent. - int samples_sent_{0}; + uint32_t samples_sent_{0}; /// The total number of samples to send. - int samples_count_{0}; + uint32_t samples_count_{0}; /// The number of samples for the gap between notes. - int samples_gap_{0}; + uint32_t samples_gap_{0}; #endif // USE_SPEAKER /// The callback to call when playback is finished. From 6d3d8970a67dfeed99e48e36388b319b40f3ad79 Mon Sep 17 00:00:00 2001 From: schrob <83939986+schdro@users.noreply.github.com> Date: Mon, 2 Mar 2026 04:47:02 +0100 Subject: [PATCH 06/20] [openthread] Disable default enabled OT diag code (#14399) --- esphome/components/openthread/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/openthread/__init__.py b/esphome/components/openthread/__init__.py index 89d335c574..dde4987181 100644 --- a/esphome/components/openthread/__init__.py +++ b/esphome/components/openthread/__init__.py @@ -53,6 +53,9 @@ def set_sdkconfig_options(config): # There is a conflict if the logger's uart also uses the default UART, which is seen as a watchdog failure on "ot_cli" add_idf_sdkconfig_option("CONFIG_OPENTHREAD_CLI", False) + # Diag unused, if needed for lab/cert/etc tests then enable separately + add_idf_sdkconfig_option("CONFIG_OPENTHREAD_DIAG", False) + add_idf_sdkconfig_option("CONFIG_OPENTHREAD_ENABLED", True) if tlv := config.get(CONF_TLV): From f68a3ed15d441a90cd96415af05cc69ad2ab5411 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 1 Mar 2026 18:09:00 -1000 Subject: [PATCH 07/20] [api] Remove virtual destructor from ProtoMessage (#14393) --- esphome/components/api/api_pb2.h | 6 +++--- esphome/components/api/proto.h | 7 ++++++- script/api_protobuf/api_protobuf.py | 5 ++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 033b789c18..a97f6c0a76 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -322,7 +322,6 @@ enum ZWaveProxyRequestType : uint32_t { class InfoResponseProtoMessage : public ProtoMessage { public: - ~InfoResponseProtoMessage() override = default; StringRef object_id{}; uint32_t key{0}; StringRef name{}; @@ -336,28 +335,29 @@ class InfoResponseProtoMessage : public ProtoMessage { #endif protected: + ~InfoResponseProtoMessage() = default; }; class StateResponseProtoMessage : public ProtoMessage { public: - ~StateResponseProtoMessage() override = default; uint32_t key{0}; #ifdef USE_DEVICES uint32_t device_id{0}; #endif protected: + ~StateResponseProtoMessage() = default; }; class CommandProtoMessage : public ProtoDecodableMessage { public: - ~CommandProtoMessage() override = default; uint32_t key{0}; #ifdef USE_DEVICES uint32_t device_id{0}; #endif protected: + ~CommandProtoMessage() = default; }; class HelloRequest final : public ProtoDecodableMessage { public: diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index c34f7744e6..750fff0810 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -452,7 +452,6 @@ class DumpBuffer { class ProtoMessage { public: - virtual ~ProtoMessage() = default; // Default implementation for messages with no fields virtual void encode(ProtoWriteBuffer &buffer) const {} // Default implementation for messages with no fields @@ -463,6 +462,11 @@ class ProtoMessage { virtual const char *dump_to(DumpBuffer &out) const = 0; virtual const char *message_name() const { return "unknown"; } #endif + + protected: + // Non-virtual: messages are never deleted polymorphically. + // Protected prevents accidental `delete base_ptr` (compile error). + ~ProtoMessage() = default; }; // Base class for messages that support decoding @@ -482,6 +486,7 @@ class ProtoDecodableMessage : public ProtoMessage { static uint32_t count_repeated_field(const uint8_t *buffer, size_t length, uint32_t target_field_id); protected: + ~ProtoDecodableMessage() = default; virtual bool decode_varint(uint32_t field_id, ProtoVarInt value) { return false; } virtual bool decode_length(uint32_t field_id, ProtoLengthDelimited value) { return false; } virtual bool decode_32bit(uint32_t field_id, Proto32Bit value) { return false; } diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 350947a8d6..9c9cda4d36 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -2473,9 +2473,6 @@ def build_base_class( out = f"class {base_class_name} : public {parent_class} {{\n" out += " public:\n" - # Add destructor with override - public_content.insert(0, f"~{base_class_name}() override = default;") - # Base classes don't implement encode/decode/calculate_size # Derived classes handle these with their specific field numbers cpp = "" @@ -2483,6 +2480,8 @@ def build_base_class( out += indent("\n".join(public_content)) + "\n" out += "\n" out += " protected:\n" + # Non-virtual protected destructor prevents accidental polymorphic deletion + protected_content.insert(0, f"~{base_class_name}() = default;") out += indent("\n".join(protected_content)) if protected_content: out += "\n" From 80a2acca4f576b5c43fa0a87b5752a68e1d85b6c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 1 Mar 2026 18:19:32 -1000 Subject: [PATCH 08/20] [ld2410] Add UART mock integration test for LD2410 component (#14377) --- .../external_components/uart_mock/__init__.py | 86 ++++ .../uart_mock/uart_mock.cpp | 159 +++++++ .../external_components/uart_mock/uart_mock.h | 78 ++++ .../fixtures/uart_mock_ld2410.yaml | 145 +++++++ .../uart_mock_ld2410_engineering.yaml | 156 +++++++ tests/integration/test_uart_mock_ld2410.py | 392 ++++++++++++++++++ 6 files changed, 1016 insertions(+) create mode 100644 tests/integration/fixtures/external_components/uart_mock/__init__.py create mode 100644 tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp create mode 100644 tests/integration/fixtures/external_components/uart_mock/uart_mock.h create mode 100644 tests/integration/fixtures/uart_mock_ld2410.yaml create mode 100644 tests/integration/fixtures/uart_mock_ld2410_engineering.yaml create mode 100644 tests/integration/test_uart_mock_ld2410.py diff --git a/tests/integration/fixtures/external_components/uart_mock/__init__.py b/tests/integration/fixtures/external_components/uart_mock/__init__.py new file mode 100644 index 0000000000..5bd879ab12 --- /dev/null +++ b/tests/integration/fixtures/external_components/uart_mock/__init__.py @@ -0,0 +1,86 @@ +import esphome.codegen as cg +from esphome.components import uart +from esphome.components.uart import CONF_DATA_BITS, CONF_PARITY, CONF_STOP_BITS +import esphome.config_validation as cv +from esphome.const import CONF_BAUD_RATE, CONF_DATA, CONF_DELAY, CONF_ID, CONF_INTERVAL + +CODEOWNERS = ["@esphome/tests"] +MULTI_CONF = True + +uart_mock_ns = cg.esphome_ns.namespace("uart_mock") +MockUartComponent = uart_mock_ns.class_( + "MockUartComponent", uart.UARTComponent, cg.Component +) + +CONF_INJECTIONS = "injections" +CONF_RESPONSES = "responses" +CONF_INJECT_RX = "inject_rx" +CONF_EXPECT_TX = "expect_tx" +CONF_PERIODIC_RX = "periodic_rx" + +UART_PARITY_OPTIONS = { + "NONE": uart.UARTParityOptions.UART_CONFIG_PARITY_NONE, + "EVEN": uart.UARTParityOptions.UART_CONFIG_PARITY_EVEN, + "ODD": uart.UARTParityOptions.UART_CONFIG_PARITY_ODD, +} + +INJECTION_SCHEMA = cv.Schema( + { + cv.Required(CONF_INJECT_RX): [cv.hex_uint8_t], + cv.Optional(CONF_DELAY, default="0ms"): cv.positive_time_period_milliseconds, + } +) + +RESPONSE_SCHEMA = cv.Schema( + { + cv.Required(CONF_EXPECT_TX): [cv.hex_uint8_t], + cv.Required(CONF_INJECT_RX): [cv.hex_uint8_t], + } +) + +PERIODIC_RX_SCHEMA = cv.Schema( + { + cv.Required(CONF_DATA): [cv.hex_uint8_t], + cv.Required(CONF_INTERVAL): cv.positive_time_period_milliseconds, + } +) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(MockUartComponent), + cv.Required(CONF_BAUD_RATE): cv.int_range(min=1), + cv.Optional(CONF_STOP_BITS, default=1): cv.one_of(1, 2, int=True), + cv.Optional(CONF_DATA_BITS, default=8): cv.int_range(min=5, max=8), + cv.Optional(CONF_PARITY, default="NONE"): cv.enum( + UART_PARITY_OPTIONS, upper=True + ), + cv.Optional(CONF_INJECTIONS, default=[]): cv.ensure_list(INJECTION_SCHEMA), + cv.Optional(CONF_RESPONSES, default=[]): cv.ensure_list(RESPONSE_SCHEMA), + cv.Optional(CONF_PERIODIC_RX, default=[]): cv.ensure_list(PERIODIC_RX_SCHEMA), + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + cg.add(var.set_baud_rate(config[CONF_BAUD_RATE])) + cg.add(var.set_stop_bits(config[CONF_STOP_BITS])) + cg.add(var.set_data_bits(config[CONF_DATA_BITS])) + cg.add(var.set_parity(config[CONF_PARITY])) + + for injection in config[CONF_INJECTIONS]: + rx_data = injection[CONF_INJECT_RX] + delay_ms = injection[CONF_DELAY] + cg.add(var.add_injection(rx_data, delay_ms)) + + for response in config[CONF_RESPONSES]: + tx_data = response[CONF_EXPECT_TX] + rx_data = response[CONF_INJECT_RX] + cg.add(var.add_response(tx_data, rx_data)) + + for periodic in config[CONF_PERIODIC_RX]: + data = periodic[CONF_DATA] + interval = periodic[CONF_INTERVAL] + cg.add(var.add_periodic_rx(data, interval)) diff --git a/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp b/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp new file mode 100644 index 0000000000..a49d51ba10 --- /dev/null +++ b/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp @@ -0,0 +1,159 @@ +// Host-only test component — do not copy to production code. +// See uart_mock.h for details. + +#include "uart_mock.h" +#include "esphome/core/application.h" +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +namespace esphome::uart_mock { + +static const char *const TAG = "uart_mock"; + +void MockUartComponent::setup() { + ESP_LOGI(TAG, "Mock UART initialized with %zu injections, %zu responses, %zu periodic", this->injections_.size(), + this->responses_.size(), this->periodic_rx_.size()); +} + +void MockUartComponent::loop() { + uint32_t now = App.get_loop_component_start_time(); + + // Initialize scenario start time on first loop() call, after all components have + // finished setup(). This prevents injection delays from being consumed during setup. + if (!this->loop_started_) { + this->loop_started_ = true; + this->scenario_start_ms_ = now; + this->cumulative_delay_ms_ = 0; + ESP_LOGD(TAG, "Scenario started at %u ms", now); + } + + // Process at most ONE timed injection per loop iteration. + // This ensures each injection is in a separate loop cycle, giving the consuming + // component (e.g., LD2410) a chance to process each batch independently. + if (this->injection_index_ < this->injections_.size()) { + auto &injection = this->injections_[this->injection_index_]; + uint32_t target_time = this->scenario_start_ms_ + this->cumulative_delay_ms_ + injection.delay_ms; + if (now >= target_time) { + ESP_LOGD(TAG, "Injecting %zu RX bytes (injection %u)", injection.rx_data.size(), this->injection_index_); + this->inject_to_rx_buffer_(injection.rx_data); + this->cumulative_delay_ms_ += injection.delay_ms; + this->injection_index_++; + } + } + + // Process periodic RX + for (auto &periodic : this->periodic_rx_) { + if (now - periodic.last_inject_ms >= periodic.interval_ms) { + this->inject_to_rx_buffer_(periodic.data); + periodic.last_inject_ms = now; + } + } +} + +void MockUartComponent::dump_config() { + ESP_LOGCONFIG(TAG, + "Mock UART Component:\n" + " Baud Rate: %u\n" + " Injections: %zu\n" + " Responses: %zu\n" + " Periodic RX: %zu", + this->baud_rate_, this->injections_.size(), this->responses_.size(), this->periodic_rx_.size()); +} + +void MockUartComponent::write_array(const uint8_t *data, size_t len) { + this->tx_count_ += len; + this->tx_buffer_.insert(this->tx_buffer_.end(), data, data + len); + + // Log all TX data so tests can verify what the component sends + if (len > 0 && len <= 64) { + char hex_buf[format_hex_pretty_size(64)]; + ESP_LOGD(TAG, "TX %zu bytes: %s", len, format_hex_pretty_to(hex_buf, sizeof(hex_buf), data, len)); + } else if (len > 64) { + ESP_LOGD(TAG, "TX %zu bytes (too large to log)", len); + } + +#ifdef USE_UART_DEBUGGER + for (size_t i = 0; i < len; i++) { + this->debug_callback_.call(uart::UART_DIRECTION_TX, data[i]); + } +#endif + + this->try_match_response_(); +} + +bool MockUartComponent::peek_byte(uint8_t *data) { + if (this->rx_buffer_.empty()) { + return false; + } + *data = this->rx_buffer_.front(); + return true; +} + +bool MockUartComponent::read_array(uint8_t *data, size_t len) { + if (this->rx_buffer_.size() < len) { + return false; + } + for (size_t i = 0; i < len; i++) { + data[i] = this->rx_buffer_.front(); + this->rx_buffer_.pop_front(); + } + this->rx_count_ += len; + +#ifdef USE_UART_DEBUGGER + for (size_t i = 0; i < len; i++) { + this->debug_callback_.call(uart::UART_DIRECTION_RX, data[i]); + } +#endif + + return true; +} + +size_t MockUartComponent::available() { return this->rx_buffer_.size(); } + +void MockUartComponent::flush() { + // Nothing to flush in mock +} + +void MockUartComponent::add_injection(const std::vector &rx_data, uint32_t delay_ms) { + this->injections_.push_back({rx_data, delay_ms}); +} + +void MockUartComponent::add_response(const std::vector &expect_tx, const std::vector &inject_rx) { + this->responses_.push_back({expect_tx, inject_rx}); +} + +void MockUartComponent::add_periodic_rx(const std::vector &data, uint32_t interval_ms) { + this->periodic_rx_.push_back({data, interval_ms, 0}); +} + +void MockUartComponent::try_match_response_() { + for (auto &response : this->responses_) { + if (this->tx_buffer_.size() < response.expect_tx.size()) { + continue; + } + // Check if tx_buffer_ ends with expect_tx + size_t offset = this->tx_buffer_.size() - response.expect_tx.size(); + if (std::equal(response.expect_tx.begin(), response.expect_tx.end(), this->tx_buffer_.begin() + offset)) { + ESP_LOGD(TAG, "TX match found, injecting %zu RX bytes", response.inject_rx.size()); + this->inject_to_rx_buffer_(response.inject_rx); + this->tx_buffer_.clear(); + return; + } + } +} + +void MockUartComponent::inject_to_rx_buffer_(const std::vector &data) { + // Log injected RX data so tests can see what's being fed to the component + if (!data.empty() && data.size() <= 64) { + char hex_buf[format_hex_pretty_size(64)]; + ESP_LOGD(TAG, "RX inject %zu bytes: %s", data.size(), + format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); + } else if (data.size() > 64) { + ESP_LOGD(TAG, "RX inject %zu bytes (too large to log inline)", data.size()); + } + for (uint8_t byte : data) { + this->rx_buffer_.push_back(byte); + } +} + +} // namespace esphome::uart_mock diff --git a/tests/integration/fixtures/external_components/uart_mock/uart_mock.h b/tests/integration/fixtures/external_components/uart_mock/uart_mock.h new file mode 100644 index 0000000000..00f4bf2e2b --- /dev/null +++ b/tests/integration/fixtures/external_components/uart_mock/uart_mock.h @@ -0,0 +1,78 @@ +#pragma once + +// ============================================================================ +// HOST-ONLY TEST COMPONENT — DO NOT COPY TO PRODUCTION CODE +// +// This component runs exclusively on the host platform for integration testing. +// It intentionally uses std::vector, std::deque, and dynamic allocation which +// would be inappropriate for production embedded components. Do not use this +// code as a reference for writing ESPHome components targeting real hardware. +// ============================================================================ + +#include "esphome/core/component.h" +#include "esphome/components/uart/uart_component.h" +#include +#include + +namespace esphome::uart_mock { + +class MockUartComponent : public uart::UARTComponent, public Component { + public: + void setup() override; + void loop() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::BUS; } + + // UARTComponent interface + void write_array(const uint8_t *data, size_t len) override; + bool peek_byte(uint8_t *data) override; + bool read_array(uint8_t *data, size_t len) override; + size_t available() override; + void flush() override; + + // Scenario configuration - called from generated code + void add_injection(const std::vector &rx_data, uint32_t delay_ms); + void add_response(const std::vector &expect_tx, const std::vector &inject_rx); + void add_periodic_rx(const std::vector &data, uint32_t interval_ms); + + protected: + void check_logger_conflict() override {} + void try_match_response_(); + void inject_to_rx_buffer_(const std::vector &data); + + // Timed injections + struct Injection { + std::vector rx_data; + uint32_t delay_ms; + }; + std::vector injections_; + uint32_t injection_index_{0}; + uint32_t scenario_start_ms_{0}; + uint32_t cumulative_delay_ms_{0}; + bool loop_started_{false}; + + // TX-triggered responses + struct Response { + std::vector expect_tx; + std::vector inject_rx; + }; + std::vector responses_; + std::vector tx_buffer_; + + // RX buffer + std::deque rx_buffer_; + + // Periodic RX + struct PeriodicRx { + std::vector data; + uint32_t interval_ms; + uint32_t last_inject_ms{0}; + }; + std::vector periodic_rx_; + + // Observability + uint32_t tx_count_{0}; + uint32_t rx_count_{0}; +}; + +} // namespace esphome::uart_mock diff --git a/tests/integration/fixtures/uart_mock_ld2410.yaml b/tests/integration/fixtures/uart_mock_ld2410.yaml new file mode 100644 index 0000000000..9a81468263 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_ld2410.yaml @@ -0,0 +1,145 @@ +esphome: + name: uart-mock-ld2410-test + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy ld2410's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + id: mock_uart + baud_rate: 256000 + injections: + # Phase 1 (t=100ms): Valid LD2410 normal mode data frame - happy path + # The buffer is clean at this point, so this frame should parse correctly. + # Moving target: 100cm, energy 50 + # Still target: 120cm, energy 25 + # Detection distance: 300cm + # Target state: 0x03 (moving + still) + # + # Note: LD2410's two_byte_to_int() uses signed char, so low bytes must be + # <=127 to produce correct values. Values >127 wrap negative. + # + # Frame layout (24 bytes): + # [0-3] F4 F3 F2 F1 = data frame header + # [4-5] 0D 00 = length 13 + # [6] 02 = data type (normal) + # [7] AA = data header marker + # [8] 03 = target states (moving+still) + # [9-10] 64 00 = moving distance 100 (0x0064) + # [11] 32 = moving energy 50 + # [12-13] 78 00 = still distance 120 (0x0078) + # [14] 19 = still energy 25 + # [15-16] 2C 01 = detection distance 300 (0x012C) + # [17] 00 = padding + # [18] 55 = data footer marker + # [19] 00 = CRC/check + # [20-23] F8 F7 F6 F5 = data frame footer + - delay: 100ms + inject_rx: + [ + 0xF4, 0xF3, 0xF2, 0xF1, + 0x0D, 0x00, + 0x02, 0xAA, + 0x03, + 0x64, 0x00, + 0x32, + 0x78, 0x00, + 0x19, + 0x2C, 0x01, + 0x00, + 0x55, 0x00, + 0xF8, 0xF7, 0xF6, 0xF5, + ] + + # Phase 2 (t=200ms): Garbage bytes - corrupt the buffer + # These random bytes will accumulate in the LD2410's internal buffer. + # No footer will be found, so the buffer just grows. + - delay: 100ms + inject_rx: [0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0x11, 0x22] + + # Phase 3 (t=300ms): Truncated frame (header + partial data, no footer) + # More bytes accumulating in the buffer without a footer match. + - delay: 100ms + inject_rx: [0xF4, 0xF3, 0xF2, 0xF1, 0x0D, 0x00, 0x02, 0xAA] + + # Phase 4 (t=500ms): Overflow - inject 85 bytes of 0xFF (MAX_LINE_LENGTH=50) + # Buffer has 15 bytes from phases 2+3. + # Overflow math: need 35 bytes to trigger first overflow (pos 15->49), + # then 50 more to trigger second overflow (pos 0->49). Total = 85 bytes. + # After two overflows, buffer_pos_ = 0 with a completely clean buffer. + # The LD2410 logs "Max command length exceeded" at each overflow. + - delay: 200ms + inject_rx: + [ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + ] + + # Phase 5 (t=600ms): Valid frame after overflow - recovery test + # Buffer was reset by overflow. This valid frame should parse correctly. + # Moving target: 50cm, energy 100 + # Still target: 75cm, energy 80 + # Detection distance: 127cm + - delay: 100ms + inject_rx: + [ + 0xF4, 0xF3, 0xF2, 0xF1, + 0x0D, 0x00, + 0x02, 0xAA, + 0x03, + 0x32, 0x00, + 0x64, + 0x4B, 0x00, + 0x50, + 0x7F, 0x00, + 0x00, + 0x55, 0x00, + 0xF8, 0xF7, 0xF6, 0xF5, + ] + +ld2410: + id: ld2410_dev + uart_id: mock_uart + +sensor: + - platform: ld2410 + ld2410_id: ld2410_dev + moving_distance: + name: "Moving Distance" + still_distance: + name: "Still Distance" + moving_energy: + name: "Moving Energy" + still_energy: + name: "Still Energy" + detection_distance: + name: "Detection Distance" + +binary_sensor: + - platform: ld2410 + ld2410_id: ld2410_dev + has_target: + name: "Has Target" + has_moving_target: + name: "Has Moving Target" + has_still_target: + name: "Has Still Target" diff --git a/tests/integration/fixtures/uart_mock_ld2410_engineering.yaml b/tests/integration/fixtures/uart_mock_ld2410_engineering.yaml new file mode 100644 index 0000000000..3b730fc1f8 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_ld2410_engineering.yaml @@ -0,0 +1,156 @@ +esphome: + name: uart-mock-ld2410-eng-test + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy ld2410's DEPENDENCIES = ["uart"] +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + id: mock_uart + baud_rate: 256000 + injections: + # Phase 1 (t=100ms): Valid LD2410 engineering mode data frame + # Captured from a real Screek Human Presence Sensor 1U with LD2410 firmware 2.4.x + # + # Engineering mode frame layout (45 bytes): + # [0-3] F4 F3 F2 F1 = data frame header + # [4-5] 23 00 = length 35 + # [6] 01 = data type (engineering mode) + # [7] AA = data header marker + # [8] 03 = target states (moving+still) + # [9-10] 1E 00 = moving distance 30 (0x001E) + # [11] 64 = moving energy 100 + # [12-13] 1E 00 = still distance 30 (0x001E) + # [14] 64 = still energy 100 + # [15-16] 00 00 = detection distance 0 + # [17] 08 = max moving distance gate + # [18] 08 = max still distance gate + # [19-27] gate moving energies (gates 0-8) + # [28-36] gate still energies (gates 0-8) + # [37] 57 = light sensor value 87 + # [38] 01 = out pin presence (HIGH) + # [39] 55 = data footer marker + # [40] 00 = check + # [41-44] F8 F7 F6 F5 = data frame footer + - delay: 100ms + inject_rx: + [ + 0xF4, 0xF3, 0xF2, 0xF1, + 0x23, 0x00, + 0x01, 0xAA, + 0x03, + 0x1E, 0x00, + 0x64, + 0x1E, 0x00, + 0x64, + 0x00, 0x00, + 0x08, 0x08, + 0x64, 0x41, 0x06, 0x0E, 0x2B, 0x16, 0x03, 0x03, 0x07, + 0x00, 0x00, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x57, 0x01, + 0x55, 0x00, + 0xF8, 0xF7, 0xF6, 0xF5, + ] + + # Phase 2 (t=200ms): Second engineering mode frame with different values + # Real capture: moving at 73cm, still at 30cm, detection at 33cm + - delay: 100ms + inject_rx: + [ + 0xF4, 0xF3, 0xF2, 0xF1, + 0x23, 0x00, + 0x01, 0xAA, + 0x03, + 0x49, 0x00, + 0x64, + 0x1E, 0x00, + 0x64, + 0x21, 0x00, + 0x08, 0x08, + 0x11, 0x64, 0x05, 0x29, 0x39, 0x10, 0x03, 0x11, 0x0E, + 0x00, 0x00, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x57, 0x01, + 0x55, 0x00, + 0xF8, 0xF7, 0xF6, 0xF5, + ] + + # Phase 3 (t=300ms): Frame with still target at 291cm (multi-byte distance) + # This tests the two_byte_to_int function with high byte > 0 + # Note: low byte 0x2F < 0x80 so it avoids the signed char bug + - delay: 100ms + inject_rx: + [ + 0xF4, 0xF3, 0xF2, 0xF1, + 0x23, 0x00, + 0x01, 0xAA, + 0x03, + 0x2F, 0x00, + 0x36, + 0x23, 0x01, + 0x64, + 0x21, 0x00, + 0x08, 0x08, + 0x2F, 0x36, 0x09, 0x0D, 0x15, 0x0B, 0x06, 0x06, 0x08, + 0x00, 0x00, 0x64, 0x64, 0x64, 0x64, 0x64, 0x5A, 0x3D, + 0x57, 0x01, + 0x55, 0x00, + 0xF8, 0xF7, 0xF6, 0xF5, + ] + +ld2410: + id: ld2410_dev + uart_id: mock_uart + +sensor: + - platform: ld2410 + ld2410_id: ld2410_dev + moving_distance: + name: "Moving Distance" + still_distance: + name: "Still Distance" + moving_energy: + name: "Moving Energy" + still_energy: + name: "Still Energy" + detection_distance: + name: "Detection Distance" + light: + name: "Light" + g0: + move_energy: + name: "Gate 0 Move Energy" + still_energy: + name: "Gate 0 Still Energy" + g1: + move_energy: + name: "Gate 1 Move Energy" + still_energy: + name: "Gate 1 Still Energy" + g2: + move_energy: + name: "Gate 2 Move Energy" + still_energy: + name: "Gate 2 Still Energy" + +binary_sensor: + - platform: ld2410 + ld2410_id: ld2410_dev + has_target: + name: "Has Target" + has_moving_target: + name: "Has Moving Target" + has_still_target: + name: "Has Still Target" + out_pin_presence_status: + name: "Out Pin Presence" diff --git a/tests/integration/test_uart_mock_ld2410.py b/tests/integration/test_uart_mock_ld2410.py new file mode 100644 index 0000000000..e01d6ff8e8 --- /dev/null +++ b/tests/integration/test_uart_mock_ld2410.py @@ -0,0 +1,392 @@ +"""Integration test for LD2410 component with mock UART. + +Tests: +test_uart_mock_ld2410 (normal mode): + 1. Happy path - valid data frame publishes correct sensor values + 2. Garbage resilience - random bytes don't crash the component + 3. Truncated frame handling - partial frame doesn't corrupt state + 4. Buffer overflow recovery - overflow resets the parser + 5. Post-overflow parsing - next valid frame after overflow is parsed correctly + 6. TX logging - verifies LD2410 sends expected setup commands + +test_uart_mock_ld2410_engineering (engineering mode): + 1. Engineering mode frames with per-gate energy data and light sensor + 2. Multi-byte still distance (291cm) using high byte > 0 + 3. Out pin presence binary sensor + 4. Gate energy sensor values from real device captures +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from aioesphomeapi import ( + BinarySensorInfo, + BinarySensorState, + EntityState, + SensorInfo, + SensorState, +) +import pytest + +from .state_utils import InitialStateHelper, build_key_to_entity_mapping, find_entity +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_uart_mock_ld2410( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test LD2410 data parsing with happy path, garbage, overflow, and recovery.""" + # Replace external component path placeholder + external_components_path = str( + Path(__file__).parent / "fixtures" / "external_components" + ) + yaml_config = yaml_config.replace( + "EXTERNAL_COMPONENT_PATH", external_components_path + ) + + loop = asyncio.get_running_loop() + + # Track overflow warning in logs + overflow_seen = loop.create_future() + + # Track TX data logged by the mock for assertions + tx_log_lines: list[str] = [] + + def line_callback(line: str) -> None: + if "Max command length exceeded" in line and not overflow_seen.done(): + overflow_seen.set_result(True) + # Capture all TX log lines from uart_mock + if "uart_mock" in line and "TX " in line: + tx_log_lines.append(line) + + # Track sensor state updates (after initial state is swallowed) + sensor_states: dict[str, list[float]] = { + "moving_distance": [], + "still_distance": [], + "moving_energy": [], + "still_energy": [], + "detection_distance": [], + } + binary_states: dict[str, list[bool]] = { + "has_target": [], + "has_moving_target": [], + "has_still_target": [], + } + + # Signal when we see recovery frame values + recovery_received = loop.create_future() + + def on_state(state: EntityState) -> None: + if isinstance(state, SensorState) and not state.missing_state: + sensor_name = key_to_sensor.get(state.key) + if sensor_name and sensor_name in sensor_states: + sensor_states[sensor_name].append(state.state) + # Check if this is the recovery frame (moving_distance = 50) + if ( + sensor_name == "moving_distance" + and state.state == pytest.approx(50.0) + and not recovery_received.done() + ): + recovery_received.set_result(True) + elif isinstance(state, BinarySensorState): + sensor_name = key_to_sensor.get(state.key) + if sensor_name and sensor_name in binary_states: + binary_states[sensor_name].append(state.state) + + async with ( + run_compiled(yaml_config, line_callback=line_callback), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + + # Build key mappings for all sensor types + all_names = list(sensor_states.keys()) + list(binary_states.keys()) + key_to_sensor = build_key_to_entity_mapping(entities, all_names) + + # Set up initial state helper + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Phase 1 values are in the initial states (swallowed by InitialStateHelper). + # Verify them via initial_states dict. + moving_dist_entity = find_entity(entities, "moving_distance", SensorInfo) + assert moving_dist_entity is not None + initial_moving = initial_state_helper.initial_states.get(moving_dist_entity.key) + assert initial_moving is not None and isinstance(initial_moving, SensorState) + assert initial_moving.state == pytest.approx(100.0), ( + f"Initial moving distance should be 100, got {initial_moving.state}" + ) + + still_dist_entity = find_entity(entities, "still_distance", SensorInfo) + assert still_dist_entity is not None + initial_still = initial_state_helper.initial_states.get(still_dist_entity.key) + assert initial_still is not None and isinstance(initial_still, SensorState) + assert initial_still.state == pytest.approx(120.0), ( + f"Initial still distance should be 120, got {initial_still.state}" + ) + + moving_energy_entity = find_entity(entities, "moving_energy", SensorInfo) + assert moving_energy_entity is not None + initial_me = initial_state_helper.initial_states.get(moving_energy_entity.key) + assert initial_me is not None and isinstance(initial_me, SensorState) + assert initial_me.state == pytest.approx(50.0), ( + f"Initial moving energy should be 50, got {initial_me.state}" + ) + + still_energy_entity = find_entity(entities, "still_energy", SensorInfo) + assert still_energy_entity is not None + initial_se = initial_state_helper.initial_states.get(still_energy_entity.key) + assert initial_se is not None and isinstance(initial_se, SensorState) + assert initial_se.state == pytest.approx(25.0), ( + f"Initial still energy should be 25, got {initial_se.state}" + ) + + detect_dist_entity = find_entity(entities, "detection_distance", SensorInfo) + assert detect_dist_entity is not None + initial_dd = initial_state_helper.initial_states.get(detect_dist_entity.key) + assert initial_dd is not None and isinstance(initial_dd, SensorState) + assert initial_dd.state == pytest.approx(300.0), ( + f"Initial detection distance should be 300, got {initial_dd.state}" + ) + + # Wait for the recovery frame (Phase 5) to be parsed + # This proves the component survived garbage + truncated + overflow + try: + await asyncio.wait_for(recovery_received, timeout=15.0) + except TimeoutError: + pytest.fail( + f"Timeout waiting for recovery frame. Received sensor states:\n" + f" moving_distance: {sensor_states['moving_distance']}\n" + f" still_distance: {sensor_states['still_distance']}\n" + f" moving_energy: {sensor_states['moving_energy']}\n" + f" still_energy: {sensor_states['still_energy']}\n" + f" detection_distance: {sensor_states['detection_distance']}" + ) + + # Verify overflow warning was logged + assert overflow_seen.done(), ( + "Expected 'Max command length exceeded' warning in logs" + ) + + # Verify LD2410 sent setup commands (TX logging) + # LD2410 sends 7 commands during setup: FF (config on), A0 (version), + # A5 (MAC), AB (distance res), AE (light), 61 (params), FE (config off) + assert len(tx_log_lines) > 0, "Expected TX log lines from uart_mock" + tx_data = " ".join(tx_log_lines) + # Verify command frame header appears (FD:FC:FB:FA) + assert "FD:FC:FB:FA" in tx_data, ( + "Expected LD2410 command frame header FD:FC:FB:FA in TX log" + ) + # Verify command frame footer appears (04:03:02:01) + assert "04:03:02:01" in tx_data, ( + "Expected LD2410 command frame footer 04:03:02:01 in TX log" + ) + + # Recovery frame values (Phase 5, after overflow) + assert len(sensor_states["moving_distance"]) >= 1, ( + f"Expected recovery moving_distance, got: {sensor_states['moving_distance']}" + ) + # Find the recovery value (moving_distance = 50) + recovery_values = [ + v for v in sensor_states["moving_distance"] if v == pytest.approx(50.0) + ] + assert len(recovery_values) >= 1, ( + f"Expected moving_distance=50 in recovery, got: {sensor_states['moving_distance']}" + ) + + # Recovery frame: moving=50, still=75, energy=100/80, detect=127 + recovery_idx = next( + i + for i, v in enumerate(sensor_states["moving_distance"]) + if v == pytest.approx(50.0) + ) + assert sensor_states["still_distance"][recovery_idx] == pytest.approx(75.0), ( + f"Recovery still distance should be 75, got {sensor_states['still_distance'][recovery_idx]}" + ) + assert sensor_states["moving_energy"][recovery_idx] == pytest.approx(100.0), ( + f"Recovery moving energy should be 100, got {sensor_states['moving_energy'][recovery_idx]}" + ) + assert sensor_states["still_energy"][recovery_idx] == pytest.approx(80.0), ( + f"Recovery still energy should be 80, got {sensor_states['still_energy'][recovery_idx]}" + ) + assert sensor_states["detection_distance"][recovery_idx] == pytest.approx( + 127.0 + ), ( + f"Recovery detection distance should be 127, got {sensor_states['detection_distance'][recovery_idx]}" + ) + + # Verify binary sensors detected targets + # Binary sensors could be in initial states or forwarded states + has_target_entity = find_entity(entities, "has_target", BinarySensorInfo) + assert has_target_entity is not None + initial_ht = initial_state_helper.initial_states.get(has_target_entity.key) + assert initial_ht is not None and isinstance(initial_ht, BinarySensorState) + assert initial_ht.state is True, "Has target should be True" + + has_moving_entity = find_entity(entities, "has_moving_target", BinarySensorInfo) + assert has_moving_entity is not None + initial_hm = initial_state_helper.initial_states.get(has_moving_entity.key) + assert initial_hm is not None and isinstance(initial_hm, BinarySensorState) + assert initial_hm.state is True, "Has moving target should be True" + + has_still_entity = find_entity(entities, "has_still_target", BinarySensorInfo) + assert has_still_entity is not None + initial_hs = initial_state_helper.initial_states.get(has_still_entity.key) + assert initial_hs is not None and isinstance(initial_hs, BinarySensorState) + assert initial_hs.state is True, "Has still target should be True" + + +@pytest.mark.asyncio +async def test_uart_mock_ld2410_engineering( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test LD2410 engineering mode with per-gate energy, light, and multi-byte distance.""" + external_components_path = str( + Path(__file__).parent / "fixtures" / "external_components" + ) + yaml_config = yaml_config.replace( + "EXTERNAL_COMPONENT_PATH", external_components_path + ) + + loop = asyncio.get_running_loop() + + # Track sensor state updates (after initial state is swallowed) + sensor_states: dict[str, list[float]] = { + "moving_distance": [], + "still_distance": [], + "moving_energy": [], + "still_energy": [], + "detection_distance": [], + "light": [], + "gate_0_move_energy": [], + "gate_1_move_energy": [], + "gate_2_move_energy": [], + "gate_0_still_energy": [], + "gate_1_still_energy": [], + "gate_2_still_energy": [], + } + binary_states: dict[str, list[bool]] = { + "has_target": [], + "has_moving_target": [], + "has_still_target": [], + "out_pin_presence": [], + } + + # Signal when we see Phase 3 frame (still_distance = 291) + phase3_received = loop.create_future() + + def on_state(state: EntityState) -> None: + if isinstance(state, SensorState) and not state.missing_state: + sensor_name = key_to_sensor.get(state.key) + if sensor_name and sensor_name in sensor_states: + sensor_states[sensor_name].append(state.state) + if ( + sensor_name == "still_distance" + and state.state == pytest.approx(291.0) + and not phase3_received.done() + ): + phase3_received.set_result(True) + elif isinstance(state, BinarySensorState): + sensor_name = key_to_sensor.get(state.key) + if sensor_name and sensor_name in binary_states: + binary_states[sensor_name].append(state.state) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + + all_names = list(sensor_states.keys()) + list(binary_states.keys()) + key_to_sensor = build_key_to_entity_mapping(entities, all_names) + + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Phase 1 initial values (engineering mode frame): + # moving=30, energy=100, still=30, energy=100, detect=0 + moving_dist_entity = find_entity(entities, "moving_distance", SensorInfo) + assert moving_dist_entity is not None + initial_moving = initial_state_helper.initial_states.get(moving_dist_entity.key) + assert initial_moving is not None and isinstance(initial_moving, SensorState) + assert initial_moving.state == pytest.approx(30.0), ( + f"Initial moving distance should be 30, got {initial_moving.state}" + ) + + still_dist_entity = find_entity(entities, "still_distance", SensorInfo) + assert still_dist_entity is not None + initial_still = initial_state_helper.initial_states.get(still_dist_entity.key) + assert initial_still is not None and isinstance(initial_still, SensorState) + assert initial_still.state == pytest.approx(30.0), ( + f"Initial still distance should be 30, got {initial_still.state}" + ) + + # Verify engineering mode sensors from initial state + # Gate 0 moving energy = 0x64 = 100 + gate0_move_entity = find_entity(entities, "gate_0_move_energy", SensorInfo) + assert gate0_move_entity is not None + initial_g0m = initial_state_helper.initial_states.get(gate0_move_entity.key) + assert initial_g0m is not None and isinstance(initial_g0m, SensorState) + assert initial_g0m.state == pytest.approx(100.0), ( + f"Gate 0 move energy should be 100, got {initial_g0m.state}" + ) + + # Gate 1 moving energy = 0x41 = 65 + gate1_move_entity = find_entity(entities, "gate_1_move_energy", SensorInfo) + assert gate1_move_entity is not None + initial_g1m = initial_state_helper.initial_states.get(gate1_move_entity.key) + assert initial_g1m is not None and isinstance(initial_g1m, SensorState) + assert initial_g1m.state == pytest.approx(65.0), ( + f"Gate 1 move energy should be 65, got {initial_g1m.state}" + ) + + # Light sensor = 0x57 = 87 + light_entity = find_entity(entities, "light", SensorInfo) + assert light_entity is not None + initial_light = initial_state_helper.initial_states.get(light_entity.key) + assert initial_light is not None and isinstance(initial_light, SensorState) + assert initial_light.state == pytest.approx(87.0), ( + f"Light sensor should be 87, got {initial_light.state}" + ) + + # Out pin presence = 0x01 = True + out_pin_entity = find_entity(entities, "out_pin_presence", BinarySensorInfo) + assert out_pin_entity is not None + initial_out = initial_state_helper.initial_states.get(out_pin_entity.key) + assert initial_out is not None and isinstance(initial_out, BinarySensorState) + assert initial_out.state is True, "Out pin presence should be True" + + # Wait for Phase 3 frame (still_distance = 291cm, multi-byte) + try: + await asyncio.wait_for(phase3_received, timeout=15.0) + except TimeoutError: + pytest.fail( + f"Timeout waiting for Phase 3 frame. Received sensor states:\n" + f" still_distance: {sensor_states['still_distance']}\n" + f" moving_distance: {sensor_states['moving_distance']}" + ) + + # Phase 3: still distance = 0x0123 = 291cm (multi-byte distance test) + phase3_still = [ + v for v in sensor_states["still_distance"] if v == pytest.approx(291.0) + ] + assert len(phase3_still) >= 1, ( + f"Expected still_distance=291, got: {sensor_states['still_distance']}" + ) From 9d4357c619ee94ae9a934276525e1bfa0ac99b8b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 1 Mar 2026 18:20:14 -1000 Subject: [PATCH 09/20] [core] Wake main loop from ISR in enable_loop_soon_any_context() (#14383) --- esphome/components/socket/__init__.py | 7 ++++--- esphome/core/application.h | 8 +++++++- esphome/core/component.cpp | 8 +++++++- esphome/core/lwip_fast_select.c | 24 +++++++++++++++++++----- esphome/core/lwip_fast_select.h | 7 +++++++ 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index a83648979c..08cf3ea33c 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -189,9 +189,10 @@ async def to_code(config): cg.add_define("USE_SOCKET_IMPL_BSD_SOCKETS") cg.add_define("USE_SOCKET_SELECT_SUPPORT") # ESP32 and LibreTiny both have LwIP >= 2.1.3 with lwip_socket_dbg_get_socket() - # and FreeRTOS task notifications — enable fast select to bypass lwip_select() - if CORE.is_esp32 or CORE.is_libretiny: - cg.add_define("USE_LWIP_FAST_SELECT") + # and FreeRTOS task notifications — enable fast select to bypass lwip_select(). + # Only when not using lwip_tcp, which does not provide select() support. + if (CORE.is_esp32 or CORE.is_libretiny) and impl != IMPLEMENTATION_LWIP_TCP: + cg.add_build_flag("-DUSE_LWIP_FAST_SELECT") def FILTER_SOURCE_FILES() -> list[str]: diff --git a/esphome/core/application.h b/esphome/core/application.h index 13e0f63885..d2345e2b0b 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -501,7 +501,7 @@ class Application { void wake_loop_threadsafe(); #endif -#if defined(USE_WAKE_LOOP_THREADSAFE) && defined(USE_LWIP_FAST_SELECT) +#ifdef USE_LWIP_FAST_SELECT /// Wake the main event loop from an ISR. /// Uses vTaskNotifyGiveFromISR() — <1 us, ISR-safe. /// Only available on platforms with fast select (ESP32, LibreTiny). @@ -509,6 +509,12 @@ class Application { static void IRAM_ATTR wake_loop_isrsafe(int *px_higher_priority_task_woken) { esphome_lwip_wake_main_loop_from_isr(px_higher_priority_task_woken); } + +#ifdef USE_ESP32 + /// Wake the main event loop from any context (ISR, thread, or main loop). + /// Detects the calling context and uses the appropriate FreeRTOS API. + static void IRAM_ATTR wake_loop_any_context() { esphome_lwip_wake_main_loop_any_context(); } +#endif #endif #endif diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index fd4e0d2984..5afd901da2 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -315,7 +315,7 @@ void IRAM_ATTR HOT Component::enable_loop_soon_any_context() { // This method is thread and ISR-safe because: // 1. Only performs simple assignments to volatile variables (atomic on all platforms) // 2. No read-modify-write operations that could be interrupted - // 3. No memory allocation, object construction, or function calls + // 3. No memory allocation or object construction; on ESP32 the only call (wake_loop_any_context) is ISR-safe // 4. IRAM_ATTR ensures code is in IRAM, not flash (required for ISR execution) // 5. Components are never destroyed, so no use-after-free concerns // 6. App is guaranteed to be initialized before any ISR could fire @@ -323,6 +323,12 @@ void IRAM_ATTR HOT Component::enable_loop_soon_any_context() { // 8. Race condition with main loop is handled by clearing flag before processing this->pending_enable_loop_ = true; App.has_pending_enable_loop_requests_ = true; +#if defined(USE_LWIP_FAST_SELECT) && defined(USE_ESP32) + // Wake the main loop if sleeping in ulTaskNotifyTake(). Without this, + // the main loop would not wake until the select timeout expires (~16ms). + // Uses xPortInIsrContext() to choose the correct FreeRTOS notify API. + Application::wake_loop_any_context(); +#endif } void Component::reset_to_construction_state() { if ((this->component_state_ & COMPONENT_STATE_MASK) == COMPONENT_STATE_FAILED) { diff --git a/esphome/core/lwip_fast_select.c b/esphome/core/lwip_fast_select.c index da0f1f337a..989f66e9be 100644 --- a/esphome/core/lwip_fast_select.c +++ b/esphome/core/lwip_fast_select.c @@ -105,10 +105,9 @@ // critical sections). Multiple concurrent xTaskNotifyGive calls are safe — // the notification count simply increments. -// USE_ESP32 and USE_LIBRETINY are compiler -D flags, so they are always visible in this .c file. -// Feature macros like USE_LWIP_FAST_SELECT may come from generated headers that are not included here, -// so this implementation is enabled based on platform flags instead of USE_LWIP_FAST_SELECT. -#if defined(USE_ESP32) || defined(USE_LIBRETINY) +// USE_LWIP_FAST_SELECT is set via -D build flag (not cg.add_define) so it is +// visible in both .c and .cpp translation units. +#ifdef USE_LWIP_FAST_SELECT // LwIP headers must come first — they define netconn_callback, struct lwip_sock, etc. #include @@ -239,4 +238,19 @@ void IRAM_ATTR esphome_lwip_wake_main_loop_from_isr(int *px_higher_priority_task } } -#endif // defined(USE_ESP32) || defined(USE_LIBRETINY) +// Wake the main loop from any context (ISR, thread, or main loop). +// ESP32-only: uses xPortInIsrContext() to detect ISR context. +// LibreTiny is excluded because it lacks IRAM_ATTR support needed for ISR-safe paths. +#ifdef USE_ESP32 +void IRAM_ATTR esphome_lwip_wake_main_loop_any_context(void) { + if (xPortInIsrContext()) { + int px_higher_priority_task_woken = 0; + esphome_lwip_wake_main_loop_from_isr(&px_higher_priority_task_woken); + portYIELD_FROM_ISR(px_higher_priority_task_woken); + } else { + esphome_lwip_wake_main_loop(); + } +} +#endif + +#endif // USE_LWIP_FAST_SELECT diff --git a/esphome/core/lwip_fast_select.h b/esphome/core/lwip_fast_select.h index b7a70a8f9f..6fce34fd76 100644 --- a/esphome/core/lwip_fast_select.h +++ b/esphome/core/lwip_fast_select.h @@ -33,6 +33,13 @@ void esphome_lwip_wake_main_loop(void); /// @param px_higher_priority_task_woken Set to pdTRUE if a context switch is needed. void esphome_lwip_wake_main_loop_from_isr(int *px_higher_priority_task_woken); +/// Wake the main loop task from any context (ISR, thread, or main loop). +/// ESP32-only: uses xPortInIsrContext() to detect ISR context. +/// LibreTiny lacks IRAM_ATTR support needed for ISR-safe paths. +#ifdef USE_ESP32 +void esphome_lwip_wake_main_loop_any_context(void); +#endif + #ifdef __cplusplus } #endif From 590ee81f7a13356c75d5cafa2436771fe4b80f53 Mon Sep 17 00:00:00 2001 From: schrob <83939986+schdro@users.noreply.github.com> Date: Mon, 2 Mar 2026 07:16:36 +0100 Subject: [PATCH 10/20] [openthread] Disable default enabled OT console build (#14390) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/components/openthread/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/openthread/__init__.py b/esphome/components/openthread/__init__.py index dde4987181..5861c3db3f 100644 --- a/esphome/components/openthread/__init__.py +++ b/esphome/components/openthread/__init__.py @@ -52,6 +52,8 @@ def set_sdkconfig_options(config): # There is a conflict if the logger's uart also uses the default UART, which is seen as a watchdog failure on "ot_cli" add_idf_sdkconfig_option("CONFIG_OPENTHREAD_CLI", False) + # Console is the transport layer for CLI; disable it too since CLI is disabled + add_idf_sdkconfig_option("CONFIG_OPENTHREAD_CONSOLE_ENABLE", False) # Diag unused, if needed for lab/cert/etc tests then enable separately add_idf_sdkconfig_option("CONFIG_OPENTHREAD_DIAG", False) From 3160457ca620adfbe48980af20d89a5c062462ee Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Mon, 2 Mar 2026 00:51:27 -0800 Subject: [PATCH 11/20] Create integration tests for modbus (#14395) Co-authored-by: J. Nick Koston --- .../external_components/uart_mock/__init__.py | 83 +++++++++- .../uart_mock/automation.h | 51 ++++++ .../uart_mock/uart_mock.cpp | 25 ++- .../external_components/uart_mock/uart_mock.h | 10 +- .../fixtures/uart_mock_modbus.yaml | 40 +++++ .../fixtures/uart_mock_modbus_timing.yaml | 54 +++++++ tests/integration/test_uart_mock_modbus.py | 153 ++++++++++++++++++ 7 files changed, 409 insertions(+), 7 deletions(-) create mode 100644 tests/integration/fixtures/external_components/uart_mock/automation.h create mode 100644 tests/integration/fixtures/uart_mock_modbus.yaml create mode 100644 tests/integration/fixtures/uart_mock_modbus_timing.yaml create mode 100644 tests/integration/test_uart_mock_modbus.py diff --git a/tests/integration/fixtures/external_components/uart_mock/__init__.py b/tests/integration/fixtures/external_components/uart_mock/__init__.py index 5bd879ab12..dea8c38551 100644 --- a/tests/integration/fixtures/external_components/uart_mock/__init__.py +++ b/tests/integration/fixtures/external_components/uart_mock/__init__.py @@ -1,8 +1,28 @@ +from esphome import automation import esphome.codegen as cg from esphome.components import uart -from esphome.components.uart import CONF_DATA_BITS, CONF_PARITY, CONF_STOP_BITS +from esphome.components.uart import ( + CONF_RX_FULL_THRESHOLD, + CONF_RX_TIMEOUT, + debug_to_code, + maybe_empty_debug, + validate_raw_data, +) import esphome.config_validation as cv -from esphome.const import CONF_BAUD_RATE, CONF_DATA, CONF_DELAY, CONF_ID, CONF_INTERVAL +from esphome.const import ( + CONF_BAUD_RATE, + CONF_DATA, + CONF_DATA_BITS, + CONF_DEBUG, + CONF_DELAY, + CONF_ID, + CONF_INTERVAL, + CONF_PARITY, + CONF_RX_BUFFER_SIZE, + CONF_STOP_BITS, + CONF_TRIGGER_ID, +) +from esphome.core import ID CODEOWNERS = ["@esphome/tests"] MULTI_CONF = True @@ -11,12 +31,20 @@ uart_mock_ns = cg.esphome_ns.namespace("uart_mock") MockUartComponent = uart_mock_ns.class_( "MockUartComponent", uart.UARTComponent, cg.Component ) +MockUartInjectRXAction = uart_mock_ns.class_( + "MockUartInjectRXAction", automation.Action +) +MockUartTXTrigger = uart_mock_ns.class_( + "MockUartTXTrigger", + automation.Trigger.template(cg.std_vector.template(cg.uint8)), +) CONF_INJECTIONS = "injections" CONF_RESPONSES = "responses" CONF_INJECT_RX = "inject_rx" CONF_EXPECT_TX = "expect_tx" CONF_PERIODIC_RX = "periodic_rx" +CONF_ON_TX = "on_tx" UART_PARITY_OPTIONS = { "NONE": uart.UARTParityOptions.UART_CONFIG_PARITY_NONE, @@ -31,6 +59,15 @@ INJECTION_SCHEMA = cv.Schema( } ) +CONFIG_INJECT_RX_SCHEMA = cv.maybe_simple_value( + { + cv.GenerateID(): cv.use_id(MockUartComponent), + cv.Required("data"): cv.templatable(validate_raw_data), + }, + key=CONF_DATA, +) + + RESPONSE_SCHEMA = cv.Schema( { cv.Required(CONF_EXPECT_TX): [cv.hex_uint8_t], @@ -49,6 +86,9 @@ CONFIG_SCHEMA = cv.Schema( { cv.GenerateID(): cv.declare_id(MockUartComponent), cv.Required(CONF_BAUD_RATE): cv.int_range(min=1), + cv.Optional(CONF_RX_BUFFER_SIZE, default=256): cv.validate_bytes, + cv.Optional(CONF_RX_FULL_THRESHOLD, default=10): cv.int_range(min=1, max=120), + cv.Optional(CONF_RX_TIMEOUT, default=2): cv.int_range(min=0, max=92), cv.Optional(CONF_STOP_BITS, default=1): cv.one_of(1, 2, int=True), cv.Optional(CONF_DATA_BITS, default=8): cv.int_range(min=5, max=8), cv.Optional(CONF_PARITY, default="NONE"): cv.enum( @@ -57,15 +97,45 @@ CONFIG_SCHEMA = cv.Schema( cv.Optional(CONF_INJECTIONS, default=[]): cv.ensure_list(INJECTION_SCHEMA), cv.Optional(CONF_RESPONSES, default=[]): cv.ensure_list(RESPONSE_SCHEMA), cv.Optional(CONF_PERIODIC_RX, default=[]): cv.ensure_list(PERIODIC_RX_SCHEMA), + cv.Optional(CONF_ON_TX): automation.validate_automation( + { + cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(MockUartTXTrigger), + } + ), + cv.Optional(CONF_DEBUG): maybe_empty_debug, } ).extend(cv.COMPONENT_SCHEMA) +@automation.register_action( + "uart_mock.inject_rx", MockUartInjectRXAction, CONFIG_INJECT_RX_SCHEMA +) +async def inject_rx_to_code(config, action_id, template_arg, args): + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + data = config[CONF_DATA] + if isinstance(data, bytes): + data = list(data) + + if cg.is_template(data): + templ = await cg.templatable(data, args, cg.std_vector.template(cg.uint8)) + cg.add(var.set_data_template(templ)) + else: + # Generate static array in flash to avoid RAM copy + arr_id = ID(f"{action_id}_data", is_declaration=True, type=cg.uint8) + arr = cg.static_const_array(arr_id, cg.ArrayInitializer(*data)) + cg.add(var.set_data_static(arr, len(data))) + return var + + async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) cg.add(var.set_baud_rate(config[CONF_BAUD_RATE])) + cg.add(var.set_rx_buffer_size(config[CONF_RX_BUFFER_SIZE])) + cg.add(var.set_rx_full_threshold(config[CONF_RX_FULL_THRESHOLD])) + cg.add(var.set_rx_timeout(config[CONF_RX_TIMEOUT])) cg.add(var.set_stop_bits(config[CONF_STOP_BITS])) cg.add(var.set_data_bits(config[CONF_DATA_BITS])) cg.add(var.set_parity(config[CONF_PARITY])) @@ -84,3 +154,12 @@ async def to_code(config): data = periodic[CONF_DATA] interval = periodic[CONF_INTERVAL] cg.add(var.add_periodic_rx(data, interval)) + + for conf in config.get(CONF_ON_TX, []): + trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) + await automation.build_automation( + trigger, [(cg.std_vector.template(cg.uint8), "data")], conf + ) + + if CONF_DEBUG in config: + await debug_to_code(config[CONF_DEBUG], var) diff --git a/tests/integration/fixtures/external_components/uart_mock/automation.h b/tests/integration/fixtures/external_components/uart_mock/automation.h new file mode 100644 index 0000000000..83a057d3a0 --- /dev/null +++ b/tests/integration/fixtures/external_components/uart_mock/automation.h @@ -0,0 +1,51 @@ +#pragma once + +#include "esphome/core/component.h" +#include "esphome/core/automation.h" +#include "uart_mock.h" + +namespace esphome::uart_mock { + +// This pattern is similar to UARTWriteAction but calls inject_rx instead of write_array, and is parented to VirtualUART +// instead of UARTComponent +template class MockUartInjectRXAction : public Action, public Parented { + public: + void set_data_template(std::vector (*func)(Ts...)) { + // Stateless lambdas (generated by ESPHome) implicitly convert to function pointers + this->code_.func = func; + this->len_ = -1; // Sentinel value indicates template mode + } + + // Store pointer to static data in flash (no RAM copy) + void set_data_static(const uint8_t *data, size_t len) { + this->code_.data = data; + this->len_ = len; // Length >= 0 indicates static mode + } + + void play(const Ts &...x) override { + if (this->len_ >= 0) { + // Static mode: use pointer and length + this->parent_->inject_to_rx_buffer(this->code_.data, static_cast(this->len_)); + } else { + // Template mode: call function + auto val = this->code_.func(x...); + this->parent_->inject_to_rx_buffer(val); + } + } + + protected: + ssize_t len_{-1}; // -1 = template mode, >=0 = static mode with length + union Code { + std::vector (*func)(Ts...); // Function pointer (stateless lambdas) + const uint8_t *data; // Pointer to static data in flash + } code_; +}; + +class MockUartTXTrigger : public Trigger> { + public: + explicit MockUartTXTrigger(MockUartComponent *parent) { + parent->set_tx_hook([this](std::vector data) { this->trigger(data); }); + } +}; + +} // namespace esphome::uart_mock diff --git a/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp b/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp index a49d51ba10..a4a1c41234 100644 --- a/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp +++ b/tests/integration/fixtures/external_components/uart_mock/uart_mock.cpp @@ -35,7 +35,7 @@ void MockUartComponent::loop() { uint32_t target_time = this->scenario_start_ms_ + this->cumulative_delay_ms_ + injection.delay_ms; if (now >= target_time) { ESP_LOGD(TAG, "Injecting %zu RX bytes (injection %u)", injection.rx_data.size(), this->injection_index_); - this->inject_to_rx_buffer_(injection.rx_data); + this->inject_to_rx_buffer(injection.rx_data); this->cumulative_delay_ms_ += injection.delay_ms; this->injection_index_++; } @@ -44,7 +44,7 @@ void MockUartComponent::loop() { // Process periodic RX for (auto &periodic : this->periodic_rx_) { if (now - periodic.last_inject_ms >= periodic.interval_ms) { - this->inject_to_rx_buffer_(periodic.data); + this->inject_to_rx_buffer(periodic.data); periodic.last_inject_ms = now; } } @@ -79,6 +79,12 @@ void MockUartComponent::write_array(const uint8_t *data, size_t len) { #endif this->try_match_response_(); + + // This directly calls a tx_hook (lambda) as an alternative to the simpler match_response mechanism. + if (this->tx_hook_) { + std::vector buf(data, data + len); + this->tx_hook_(buf); + } } bool MockUartComponent::peek_byte(uint8_t *data) { @@ -114,6 +120,12 @@ void MockUartComponent::flush() { // Nothing to flush in mock } +void MockUartComponent::set_rx_full_threshold(size_t rx_full_threshold) { + this->rx_full_threshold_ = rx_full_threshold; +} + +void MockUartComponent::set_rx_timeout(size_t rx_timeout) { this->rx_timeout_ = rx_timeout; } + void MockUartComponent::add_injection(const std::vector &rx_data, uint32_t delay_ms) { this->injections_.push_back({rx_data, delay_ms}); } @@ -135,14 +147,19 @@ void MockUartComponent::try_match_response_() { size_t offset = this->tx_buffer_.size() - response.expect_tx.size(); if (std::equal(response.expect_tx.begin(), response.expect_tx.end(), this->tx_buffer_.begin() + offset)) { ESP_LOGD(TAG, "TX match found, injecting %zu RX bytes", response.inject_rx.size()); - this->inject_to_rx_buffer_(response.inject_rx); + this->inject_to_rx_buffer(response.inject_rx); this->tx_buffer_.clear(); return; } } } -void MockUartComponent::inject_to_rx_buffer_(const std::vector &data) { +void MockUartComponent::inject_to_rx_buffer(const uint8_t *data, size_t len) { + std::vector vec(data, data + len); + this->inject_to_rx_buffer(vec); +} + +void MockUartComponent::inject_to_rx_buffer(const std::vector &data) { // Log injected RX data so tests can see what's being fed to the component if (!data.empty() && data.size() <= 64) { char hex_buf[format_hex_pretty_size(64)]; diff --git a/tests/integration/fixtures/external_components/uart_mock/uart_mock.h b/tests/integration/fixtures/external_components/uart_mock/uart_mock.h index 00f4bf2e2b..5bbc3c1bf6 100644 --- a/tests/integration/fixtures/external_components/uart_mock/uart_mock.h +++ b/tests/integration/fixtures/external_components/uart_mock/uart_mock.h @@ -29,16 +29,21 @@ class MockUartComponent : public uart::UARTComponent, public Component { bool read_array(uint8_t *data, size_t len) override; size_t available() override; void flush() override; + void set_rx_full_threshold(size_t rx_full_threshold) override; + void set_rx_timeout(size_t rx_timeout) override; // Scenario configuration - called from generated code void add_injection(const std::vector &rx_data, uint32_t delay_ms); void add_response(const std::vector &expect_tx, const std::vector &inject_rx); void add_periodic_rx(const std::vector &data, uint32_t interval_ms); + void set_tx_hook(std::function &)> &&cb) { this->tx_hook_ = std::move(cb); } + void inject_to_rx_buffer(const std::vector &data); + void inject_to_rx_buffer(const uint8_t *data, size_t len); + protected: void check_logger_conflict() override {} void try_match_response_(); - void inject_to_rx_buffer_(const std::vector &data); // Timed injections struct Injection { @@ -73,6 +78,9 @@ class MockUartComponent : public uart::UARTComponent, public Component { // Observability uint32_t tx_count_{0}; uint32_t rx_count_{0}; + + // Direct TX hook for tests that want to bypass the response-matching logic + std::function &)> tx_hook_; }; } // namespace esphome::uart_mock diff --git a/tests/integration/fixtures/uart_mock_modbus.yaml b/tests/integration/fixtures/uart_mock_modbus.yaml new file mode 100644 index 0000000000..89b9b91861 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus.yaml @@ -0,0 +1,40 @@ +esphome: + name: uart-mock-modbus-test + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_dev + baud_rate: 9600 + rx_full_threshold: 120 + rx_timeout: 2 + debug: + responses: + - expect_tx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 1 on device 1 + inject_rx: [0x01, 0x03, 0x02, 0x01, 0x03, 0xF9, 0xD5] # Return value 0x0103 (hex) = 259 (dec) + +modbus: + uart_id: virtual_uart_dev + +modbus_controller: + address: 1 + +sensor: + - platform: modbus_controller + name: "basic_register" + address: 0x03 + register_type: holding diff --git a/tests/integration/fixtures/uart_mock_modbus_timing.yaml b/tests/integration/fixtures/uart_mock_modbus_timing.yaml new file mode 100644 index 0000000000..cd485ca394 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_timing.yaml @@ -0,0 +1,54 @@ +esphome: + name: uart-mock-modbus-test + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_dev + baud_rate: 9600 + rx_full_threshold: 120 + rx_timeout: 2 + debug: + on_tx: + - then: + - if: + condition: #Read 80 input registers on device 2, starting at address 0 (SDM meter request) + lambda: "return data == std::vector({0x02,0x04,0x00,0x00,0x00,0x50,0xF0,0x05});" + then: + - uart_mock.inject_rx: # Good response from SDM meter with CRC. Voltage is 243V + !lambda return {0x02,0x04,0xA0,0x43,0x73,0x19,0x9A,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3F,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; + - delay: 120ms # Simulate some delay for UART buffer to fill before next part of response (1ms = 1 byte at 9600 baud) + - uart_mock.inject_rx: # Rest of response (including CRC) + !lambda return{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x42,0x6F,0xCC,0xCD,0x43,0x7C,0xB8,0x10,0x3D,0x38,0x51,0xEC, + 0x43,0x81,0x1B,0xE7,0x3B,0x03,0x12,0x6F,0x50,0x1B}; + +modbus: + uart_id: virtual_uart_dev + +sensor: + - platform: sdm_meter + address: 2 + phase_a: + voltage: + name: sdm_voltage diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py new file mode 100644 index 0000000000..309cb56dc9 --- /dev/null +++ b/tests/integration/test_uart_mock_modbus.py @@ -0,0 +1,153 @@ +"""Integration test for modbus component with virtual UART. + +Tests: +test_uart_mock_modbus : + 1. Read a single register and parse successfully (basic_register) + 2. Read multiple registers from SDM meter and parse successfully (sdm_voltage), with some intermediate delay to simulate UART buffer time. + +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from aioesphomeapi import EntityState, SensorState +import pytest + +from .state_utils import InitialStateHelper, build_key_to_entity_mapping +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_uart_mock_modbus( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test basic modbus data parsing.""" + # Replace external component path placeholder + external_components_path = str( + Path(__file__).parent / "fixtures" / "external_components" + ) + yaml_config = yaml_config.replace( + "EXTERNAL_COMPONENT_PATH", external_components_path + ) + + loop = asyncio.get_running_loop() + + # Track sensor state updates (after initial state is swallowed) + sensor_states: dict[str, list[float]] = { + "basic_register": [], + } + + basic_register_changed = loop.create_future() + + def on_state(state: EntityState) -> None: + if isinstance(state, SensorState) and not state.missing_state: + sensor_name = key_to_sensor.get(state.key) + if sensor_name and sensor_name in sensor_states: + sensor_states[sensor_name].append(state.state) + if ( + sensor_name == "basic_register" + and state.state == 259.0 + and not basic_register_changed.done() + ): + basic_register_changed.set_result(True) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + + # Build key mappings for all sensor types + all_names = list(sensor_states.keys()) + key_to_sensor = build_key_to_entity_mapping(entities, all_names) + + # Set up initial state helper + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Wait for basic register to be updated with successful parse + try: + await asyncio.wait_for(basic_register_changed, timeout=15.0) + except TimeoutError: + pytest.fail( + f"Timeout waiting for Basic Register change. Received sensor states:\n" + f" basic_register: {sensor_states['basic_register']}\n" + ) + + +@pytest.mark.asyncio +@pytest.mark.xfail( + reason="There is a bug in UART which will timeout for long responses." +) +async def test_uart_mock_modbus_timing( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test basic modbus data parsing.""" + # Replace external component path placeholder + external_components_path = str( + Path(__file__).parent / "fixtures" / "external_components" + ) + yaml_config = yaml_config.replace( + "EXTERNAL_COMPONENT_PATH", external_components_path + ) + + loop = asyncio.get_running_loop() + + # Track sensor state updates (after initial state is swallowed) + sensor_states: dict[str, list[float]] = { + "sdm_voltage": [], + } + + voltage_changed = loop.create_future() + + def on_state(state: EntityState) -> None: + if isinstance(state, SensorState) and not state.missing_state: + sensor_name = key_to_sensor.get(state.key) + if sensor_name and sensor_name in sensor_states: + sensor_states[sensor_name].append(state.state) + # Check if this is a good voltage reading (243V) + if ( + sensor_name == "sdm_voltage" + and state.state > 200.0 + and not voltage_changed.done() + ): + voltage_changed.set_result(True) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + + # Build key mappings for all sensor types + all_names = list(sensor_states.keys()) + key_to_sensor = build_key_to_entity_mapping(entities, all_names) + + # Set up initial state helper + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Wait for voltage to be updated with successful parse + try: + await asyncio.wait_for(voltage_changed, timeout=15.0) + except TimeoutError: + pytest.fail( + f"Timeout waiting for SDM voltage change. Received sensor states:\n" + f" sdm_voltage: {sensor_states['sdm_voltage']}\n" + ) From 77a7cbcffd4b8ec1fa3bc77b32f9ef06dc96761d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:41:20 -0500 Subject: [PATCH 12/20] Use cached files on network errors in external_files (#14055) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: jesserockz <3060199+jesserockz@users.noreply.github.com> --- esphome/external_files.py | 15 +++++++-- tests/unit_tests/test_external_files.py | 44 +++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/esphome/external_files.py b/esphome/external_files.py index 80b54ebb2f..72a3f33fdc 100644 --- a/esphome/external_files.py +++ b/esphome/external_files.py @@ -55,10 +55,12 @@ def has_remote_file_changed(url: str, local_file_path: Path) -> bool: _LOGGER.debug("has_remote_file_changed: File modified") return True except requests.exceptions.RequestException as e: - raise cv.Invalid( - f"Could not check if {url} has changed, please check if file exists " - f"({e})" + _LOGGER.warning( + "Could not check if %s has changed due to network error (%s), using cached file", + url, + e, ) + return False _LOGGER.debug("has_remote_file_changed: File doesn't exists at %s", local_file_path) return True @@ -98,6 +100,13 @@ def download_content(url: str, path: Path, timeout=NETWORK_TIMEOUT) -> bytes: ) req.raise_for_status() except requests.exceptions.RequestException as e: + if path.exists(): + _LOGGER.warning( + "Could not download from %s due to network error (%s), using cached file", + url, + e, + ) + return path.read_bytes() raise cv.Invalid(f"Could not download from {url}: {e}") path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/unit_tests/test_external_files.py b/tests/unit_tests/test_external_files.py index 05e0bd3523..a319fae83d 100644 --- a/tests/unit_tests/test_external_files.py +++ b/tests/unit_tests/test_external_files.py @@ -144,16 +144,16 @@ def test_has_remote_file_changed_no_local_file(setup_core: Path) -> None: def test_has_remote_file_changed_network_error( mock_head: MagicMock, setup_core: Path ) -> None: - """Test has_remote_file_changed handles network errors gracefully.""" + """Test has_remote_file_changed returns False on network error when file is cached.""" test_file = setup_core / "cached.txt" test_file.write_text("cached content") mock_head.side_effect = requests.exceptions.RequestException("Network error") url = "https://example.com/file.txt" + result = external_files.has_remote_file_changed(url, test_file) - with pytest.raises(Invalid, match="Could not check if.*Network error"): - external_files.has_remote_file_changed(url, test_file) + assert result is False @patch("esphome.external_files.requests.head") @@ -198,3 +198,41 @@ def test_is_file_recent_handles_float_seconds(setup_core: Path) -> None: result = external_files.is_file_recent(test_file, refresh) assert result is True + + +@patch("esphome.external_files.requests.get") +@patch("esphome.external_files.has_remote_file_changed") +def test_download_content_with_network_error_uses_cache( + mock_has_changed: MagicMock, mock_get: MagicMock, setup_core: Path +) -> None: + """Test download_content uses cached file when network fails.""" + test_file = setup_core / "cached.txt" + cached_content = b"cached content" + test_file.write_bytes(cached_content) + + # Simulate file has changed, so it tries to download + mock_has_changed.return_value = True + mock_get.side_effect = requests.exceptions.RequestException("Network error") + + url = "https://example.com/file.txt" + result = external_files.download_content(url, test_file) + + assert result == cached_content + + +@patch("esphome.external_files.requests.get") +@patch("esphome.external_files.has_remote_file_changed") +def test_download_content_with_network_error_no_cache_fails( + mock_has_changed: MagicMock, mock_get: MagicMock, setup_core: Path +) -> None: + """Test download_content raises error when network fails and no cache exists.""" + test_file = setup_core / "nonexistent.txt" + + # Simulate file has changed (doesn't exist), so it tries to download + mock_has_changed.return_value = True + mock_get.side_effect = requests.exceptions.RequestException("Network error") + + url = "https://example.com/file.txt" + + with pytest.raises(Invalid, match="Could not download from.*Network error"): + external_files.download_content(url, test_file) From 1c5fd8bbd46fca739ec2bf7743f87ea433c98ad6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:56:51 -1000 Subject: [PATCH 13/20] [core] Move millis_64 rollover tracking out of Scheduler (#14360) Co-authored-by: Claude Opus 4.6 --- esphome/components/esp32/__init__.py | 1 + esphome/components/esp8266/core.cpp | 4 +- esphome/components/host/__init__.py | 1 + esphome/components/libretiny/core.cpp | 4 +- esphome/components/rp2040/__init__.py | 1 + esphome/components/zephyr/__init__.py | 1 + esphome/core/config.py | 6 + esphome/core/defines.h | 5 + esphome/core/scheduler.cpp | 182 +--------------------- esphome/core/scheduler.h | 47 +----- esphome/core/time_64.cpp | 207 ++++++++++++++++++++++++++ esphome/core/time_64.h | 24 +++ 12 files changed, 257 insertions(+), 226 deletions(-) create mode 100644 esphome/core/time_64.cpp create mode 100644 esphome/core/time_64.h diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index dd9e394fd2..a14d3af69e 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1438,6 +1438,7 @@ async def to_code(config): cg.set_cpp_standard("gnu++20") cg.add_build_flag("-DUSE_ESP32") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_build_flag("-Wl,-z,noexecstack") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) variant = config[CONF_VARIANT] diff --git a/esphome/components/esp8266/core.cpp b/esphome/components/esp8266/core.cpp index 497e99b61f..7136cf41cb 100644 --- a/esphome/components/esp8266/core.cpp +++ b/esphome/components/esp8266/core.cpp @@ -3,7 +3,7 @@ #include "core.h" #include "esphome/core/defines.h" #include "esphome/core/hal.h" -#include "esphome/core/application.h" +#include "esphome/core/time_64.h" #include "esphome/core/helpers.h" #include "preferences.h" #include @@ -17,7 +17,7 @@ namespace esphome { void HOT yield() { ::yield(); } uint32_t IRAM_ATTR HOT millis() { return ::millis(); } -uint64_t millis_64() { return App.scheduler.millis_64_impl_(::millis()); } +uint64_t millis_64() { return Millis64Impl::compute(::millis()); } void HOT delay(uint32_t ms) { ::delay(ms); } uint32_t IRAM_ATTR HOT micros() { return ::micros(); } void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { delay_microseconds_safe(us); } diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index ba05e497c8..8adbfb02ec 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -41,6 +41,7 @@ CONFIG_SCHEMA = cv.All( async def to_code(config): cg.add_build_flag("-DUSE_HOST") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_define("USE_ESPHOME_HOST_MAC_ADDRESS", config[CONF_MAC_ADDRESS].parts) cg.add_build_flag("-std=gnu++20") cg.add_define("ESPHOME_BOARD", "host") diff --git a/esphome/components/libretiny/core.cpp b/esphome/components/libretiny/core.cpp index 6cbc81938d..88952692a7 100644 --- a/esphome/components/libretiny/core.cpp +++ b/esphome/components/libretiny/core.cpp @@ -3,7 +3,7 @@ #include "core.h" #include "esphome/core/defines.h" #include "esphome/core/hal.h" -#include "esphome/core/application.h" +#include "esphome/core/time_64.h" #include "esphome/core/helpers.h" #include "preferences.h" @@ -14,7 +14,7 @@ namespace esphome { void HOT yield() { ::yield(); } uint32_t IRAM_ATTR HOT millis() { return ::millis(); } -uint64_t millis_64() { return App.scheduler.millis_64_impl_(::millis()); } +uint64_t millis_64() { return Millis64Impl::compute(::millis()); } uint32_t IRAM_ATTR HOT micros() { return ::micros(); } void HOT delay(uint32_t ms) { ::delay(ms); } void IRAM_ATTR HOT delayMicroseconds(uint32_t us) { ::delayMicroseconds(us); } diff --git a/esphome/components/rp2040/__init__.py b/esphome/components/rp2040/__init__.py index 23f12e651f..ea269a47c5 100644 --- a/esphome/components/rp2040/__init__.py +++ b/esphome/components/rp2040/__init__.py @@ -169,6 +169,7 @@ async def to_code(config): cg.add_platformio_option("lib_compat_mode", "strict") cg.add_platformio_option("board", config[CONF_BOARD]) cg.add_build_flag("-DUSE_RP2040") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) cg.add_define("ESPHOME_VARIANT", "RP2040") diff --git a/esphome/components/zephyr/__init__.py b/esphome/components/zephyr/__init__.py index 43d5cebebb..4cc71bddca 100644 --- a/esphome/components/zephyr/__init__.py +++ b/esphome/components/zephyr/__init__.py @@ -112,6 +112,7 @@ def add_extra_script(stage: str, filename: str, path: Path) -> None: def zephyr_to_code(config): cg.add_build_flag("-DUSE_ZEPHYR") + cg.add_define("USE_NATIVE_64BIT_TIME") cg.set_cpp_standard("gnu++20") # build is done by west so bypass board checking in platformio cg.add_platformio_option("boards_dir", CORE.relative_build_path("boards")) diff --git a/esphome/core/config.py b/esphome/core/config.py index 215432835a..593b402230 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -650,6 +650,12 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( PlatformFramework.ESP32_ARDUINO, PlatformFramework.ESP32_IDF, }, + "time_64.cpp": { + PlatformFramework.ESP8266_ARDUINO, + PlatformFramework.BK72XX_ARDUINO, + PlatformFramework.RTL87XX_ARDUINO, + PlatformFramework.LN882X_ARDUINO, + }, # Note: lock_free_queue.h and event_pool.h are header files and don't need to be filtered # as they are only included when needed by the preprocessor } diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 181425c162..1f2597dd98 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -178,6 +178,11 @@ #define USE_I2S_LEGACY #endif +// Platforms with native 64-bit time sources (no rollover tracking needed) +#if defined(USE_ESP32) || defined(USE_HOST) || defined(USE_ZEPHYR) || defined(USE_RP2040) +#define USE_NATIVE_64BIT_TIME +#endif + // ESP32-specific feature flags #ifdef USE_ESP32 #define USE_MQTT_IDF_ENQUEUE diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 2c10e7e2da..ca560e8250 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -9,7 +9,6 @@ #include #include #include -#include namespace esphome { @@ -28,10 +27,6 @@ static constexpr size_t MAX_POOL_SIZE = 5; // Set to 5 to match the pool size - when we have as many cancelled items as our // pool can hold, it's time to clean up and recycle them. static constexpr uint32_t MAX_LOGICALLY_DELETED_ITEMS = 5; -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) -// Half the 32-bit range - used to detect rollovers vs normal time progression -static constexpr uint32_t HALF_MAX_UINT32 = std::numeric_limits::max() / 2; -#endif // max delay to start an interval sequence static constexpr uint32_t MAX_INTERVAL_DELAY = 5000; @@ -152,9 +147,6 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type return; } - // Get fresh 64-bit timestamp BEFORE taking lock - const uint64_t now_64 = millis_64(); - // Take lock early to protect scheduler_item_pool_ access LockGuard guard{this->lock_}; @@ -181,6 +173,9 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type } else #endif /* not ESPHOME_THREAD_SINGLE */ { + // Only non-defer items need a timestamp for scheduling + const uint64_t now_64 = millis_64(); + // Type-specific setup if (type == SchedulerItem::INTERVAL) { item->interval = delay; @@ -475,19 +470,8 @@ void HOT Scheduler::call(uint32_t now) { if (now_64 - last_print > 2000) { last_print = now_64; std::vector old_items; -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) && \ - defined(ESPHOME_THREAD_MULTI_ATOMICS) - const auto last_dbg = this->last_millis_.load(std::memory_order_relaxed); - const auto major_dbg = this->millis_major_.load(std::memory_order_relaxed); - ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64 " (%" PRIu16 ", %" PRIu32 ")", this->items_.size(), - this->scheduler_item_pool_.size(), now_64, major_dbg, last_dbg); -#elif !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) - ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64 " (%" PRIu16 ", %" PRIu32 ")", this->items_.size(), - this->scheduler_item_pool_.size(), now_64, this->millis_major_, this->last_millis_); -#else ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64, this->items_.size(), this->scheduler_item_pool_.size(), now_64); -#endif // Cleanup before debug output this->cleanup_(); while (!this->items_.empty()) { @@ -715,166 +699,6 @@ bool HOT Scheduler::cancel_item_locked_(Component *component, NameType name_type return total_cancelled > 0; } -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) -uint64_t Scheduler::millis_64_impl_(uint32_t now) { - // THREAD SAFETY NOTE: - // This function has three implementations, based on the precompiler flags - // - ESPHOME_THREAD_SINGLE - Runs on single-threaded platforms (ESP8266, RP2040, etc.) - // - ESPHOME_THREAD_MULTI_NO_ATOMICS - Runs on multi-threaded platforms without atomics (LibreTiny BK72xx) - // - ESPHOME_THREAD_MULTI_ATOMICS - Runs on multi-threaded platforms with atomics (ESP32, HOST, LibreTiny - // RTL87xx/LN882x, etc.) - // - // Make sure all changes are synchronized if you edit this function. - // - // IMPORTANT: Always pass fresh millis() values to this function. The implementation - // handles out-of-order timestamps between threads, but minimizing time differences - // helps maintain accuracy. - // - -#ifdef ESPHOME_THREAD_SINGLE - // This is the single core implementation. - // - // Single-core platforms have no concurrency, so this is a simple implementation - // that just tracks 32-bit rollover (every 49.7 days) without any locking or atomics. - - uint16_t major = this->millis_major_; - uint32_t last = this->last_millis_; - - // Check for rollover - if (now < last && (last - now) > HALF_MAX_UINT32) { - this->millis_major_++; - major++; - this->last_millis_ = now; -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); -#endif /* ESPHOME_DEBUG_SCHEDULER */ - } else if (now > last) { - // Only update if time moved forward - this->last_millis_ = now; - } - - // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time - return now + (static_cast(major) << 32); - -#elif defined(ESPHOME_THREAD_MULTI_NO_ATOMICS) - // This is the multi core no atomics implementation. - // - // Without atomics, this implementation uses locks more aggressively: - // 1. Always locks when near the rollover boundary (within 10 seconds) - // 2. Always locks when detecting a large backwards jump - // 3. Updates without lock in normal forward progression (accepting minor races) - // This is less efficient but necessary without atomic operations. - uint16_t major = this->millis_major_; - uint32_t last = this->last_millis_; - - // Define a safe window around the rollover point (10 seconds) - // This covers any reasonable scheduler delays or thread preemption - static constexpr uint32_t ROLLOVER_WINDOW = 10000; // 10 seconds in milliseconds - - // Check if we're near the rollover boundary (close to std::numeric_limits::max() or just past 0) - bool near_rollover = (last > (std::numeric_limits::max() - ROLLOVER_WINDOW)) || (now < ROLLOVER_WINDOW); - - if (near_rollover || (now < last && (last - now) > HALF_MAX_UINT32)) { - // Near rollover or detected a rollover - need lock for safety - LockGuard guard{this->lock_}; - // Re-read with lock held - last = this->last_millis_; - - if (now < last && (last - now) > HALF_MAX_UINT32) { - // True rollover detected (happens every ~49.7 days) - this->millis_major_++; - major++; -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); -#endif /* ESPHOME_DEBUG_SCHEDULER */ - } - // Update last_millis_ while holding lock - this->last_millis_ = now; - } else if (now > last) { - // Normal case: Not near rollover and time moved forward - // Update without lock. While this may cause minor races (microseconds of - // backwards time movement), they're acceptable because: - // 1. The scheduler operates at millisecond resolution, not microsecond - // 2. We've already prevented the critical rollover race condition - // 3. Any backwards movement is orders of magnitude smaller than scheduler delays - this->last_millis_ = now; - } - // If now <= last and we're not near rollover, don't update - // This minimizes backwards time movement - - // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time - return now + (static_cast(major) << 32); - -#elif defined(ESPHOME_THREAD_MULTI_ATOMICS) - // This is the multi core with atomics implementation. - // - // Uses atomic operations with acquire/release semantics to ensure coherent - // reads of millis_major_ and last_millis_ across cores. Features: - // 1. Epoch-coherency retry loop to handle concurrent updates - // 2. Lock only taken for actual rollover detection and update - // 3. Lock-free CAS updates for normal forward time progression - // 4. Memory ordering ensures cores see consistent time values - - for (;;) { - uint16_t major = this->millis_major_.load(std::memory_order_acquire); - - /* - * Acquire so that if we later decide **not** to take the lock we still - * observe a `millis_major_` value coherent with the loaded `last_millis_`. - * The acquire load ensures any later read of `millis_major_` sees its - * corresponding increment. - */ - uint32_t last = this->last_millis_.load(std::memory_order_acquire); - - // If we might be near a rollover (large backwards jump), take the lock for the entire operation - // This ensures rollover detection and last_millis_ update are atomic together - if (now < last && (last - now) > HALF_MAX_UINT32) { - // Potential rollover - need lock for atomic rollover detection + update - LockGuard guard{this->lock_}; - // Re-read with lock held; mutex already provides ordering - last = this->last_millis_.load(std::memory_order_relaxed); - - if (now < last && (last - now) > HALF_MAX_UINT32) { - // True rollover detected (happens every ~49.7 days) - this->millis_major_.fetch_add(1, std::memory_order_relaxed); - major++; -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); -#endif /* ESPHOME_DEBUG_SCHEDULER */ - } - /* - * Update last_millis_ while holding the lock to prevent races - * Publish the new low-word *after* bumping `millis_major_` (done above) - * so readers never see a mismatched pair. - */ - this->last_millis_.store(now, std::memory_order_release); - } else { - // Normal case: Try lock-free update, but only allow forward movement within same epoch - // This prevents accidentally moving backwards across a rollover boundary - while (now > last && (now - last) < HALF_MAX_UINT32) { - if (this->last_millis_.compare_exchange_weak(last, now, - std::memory_order_release, // success - std::memory_order_relaxed)) { // failure - break; - } - // CAS failure means no data was published; relaxed is fine - // last is automatically updated by compare_exchange_weak if it fails - } - } - uint16_t major_end = this->millis_major_.load(std::memory_order_relaxed); - if (major_end == major) - return now + (static_cast(major) << 32); - } - // Unreachable - the loop always returns when major_end == major - __builtin_unreachable(); - -#else -#error \ - "No platform threading model defined. One of ESPHOME_THREAD_SINGLE, ESPHOME_THREAD_MULTI_NO_ATOMICS, or ESPHOME_THREAD_MULTI_ATOMICS must be defined." -#endif -} -#endif // !USE_ESP32 && !USE_HOST && !USE_ZEPHYR && !USE_RP2040 - bool HOT Scheduler::SchedulerItem::cmp(const SchedulerItemPtr &a, const SchedulerItemPtr &b) { // High bits are almost always equal (change only on 32-bit rollover ~49 days) // Optimize for common case: check low bits first when high bits are equal diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index d52cf5147d..cefbdd1b22 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -12,6 +12,7 @@ #include "esphome/core/component.h" #include "esphome/core/hal.h" #include "esphome/core/helpers.h" +#include "esphome/core/time_64.h" namespace esphome { @@ -284,23 +285,16 @@ class Scheduler { bool cancel_retry_(Component *component, NameType name_type, const char *static_name, uint32_t hash_or_id); // Extend a 32-bit millis() value to 64-bit. Use when the caller already has a fresh now. - // On ESP32, Host, Zephyr, and RP2040, ignores now and uses the native 64-bit time source via millis_64(). + // On platforms with native 64-bit time, ignores now and uses millis_64() directly. // On other platforms, extends now to 64-bit using rollover tracking. uint64_t millis_64_from_(uint32_t now) { -#if defined(USE_ESP32) || defined(USE_HOST) || defined(USE_ZEPHYR) || defined(USE_RP2040) +#ifdef USE_NATIVE_64BIT_TIME (void) now; return millis_64(); #else - return this->millis_64_impl_(now); + return Millis64Impl::compute(now); #endif } - -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) - // On platforms without native 64-bit time, millis_64() HAL function delegates to this - // method which tracks 32-bit millis() rollover using millis_major_ and last_millis_. - friend uint64_t millis_64(); - uint64_t millis_64_impl_(uint32_t now); -#endif // Cleanup logically deleted items from the scheduler // Returns the number of items remaining after cleanup // IMPORTANT: This method should only be called from the main thread (loop task). @@ -566,39 +560,6 @@ class Scheduler { // can stall the entire system, causing timing issues and dropped events for any components that need // to synchronize between tasks (see https://github.com/esphome/backlog/issues/52) std::vector scheduler_item_pool_; - -#if !defined(USE_ESP32) && !defined(USE_HOST) && !defined(USE_ZEPHYR) && !defined(USE_RP2040) - // On platforms with native 64-bit time (ESP32, Host, Zephyr, RP2040), no rollover tracking needed. - // On other platforms, these fields track 32-bit millis() rollover for millis_64_impl_(). -#ifdef ESPHOME_THREAD_MULTI_ATOMICS - /* - * Multi-threaded platforms with atomic support: last_millis_ needs atomic for lock-free updates - * - * MEMORY-ORDERING NOTE - * -------------------- - * `last_millis_` and `millis_major_` form a single 64-bit timestamp split in half. - * Writers publish `last_millis_` with memory_order_release and readers use - * memory_order_acquire. This ensures that once a reader sees the new low word, - * it also observes the corresponding increment of `millis_major_`. - */ - std::atomic last_millis_{0}; -#else /* not ESPHOME_THREAD_MULTI_ATOMICS */ - // Platforms without atomic support or single-threaded platforms - uint32_t last_millis_{0}; -#endif /* else ESPHOME_THREAD_MULTI_ATOMICS */ - - /* - * Upper 16 bits of the 64-bit millis counter. Incremented only while holding - * `lock_`; read concurrently. Atomic (relaxed) avoids a formal data race. - * Ordering relative to `last_millis_` is provided by its release store and the - * corresponding acquire loads. - */ -#ifdef ESPHOME_THREAD_MULTI_ATOMICS - std::atomic millis_major_{0}; -#else /* not ESPHOME_THREAD_MULTI_ATOMICS */ - uint16_t millis_major_{0}; -#endif /* else ESPHOME_THREAD_MULTI_ATOMICS */ -#endif /* !USE_ESP32 && !USE_HOST && !USE_ZEPHYR && !USE_RP2040 */ }; } // namespace esphome diff --git a/esphome/core/time_64.cpp b/esphome/core/time_64.cpp new file mode 100644 index 0000000000..db5df25eb9 --- /dev/null +++ b/esphome/core/time_64.cpp @@ -0,0 +1,207 @@ +#include "esphome/core/defines.h" + +#ifndef USE_NATIVE_64BIT_TIME + +#include "time_64.h" + +#include "esphome/core/helpers.h" +#ifdef ESPHOME_DEBUG_SCHEDULER +#include "esphome/core/log.h" +#include +#endif +#ifdef ESPHOME_THREAD_MULTI_ATOMICS +#include +#endif +#include + +namespace esphome { + +#ifdef ESPHOME_DEBUG_SCHEDULER +static const char *const TAG = "time_64"; +#endif + +uint64_t Millis64Impl::compute(uint32_t now) { + // Half the 32-bit range - used to detect rollovers vs normal time progression + static constexpr uint32_t HALF_MAX_UINT32 = std::numeric_limits::max() / 2; + + // State variables for rollover tracking - static to persist across calls +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + // Mutex for rollover serialization (taken only every ~49.7 days). + // A spinlock would be smaller (~1 byte vs ~80-100 bytes) but is unsafe on + // preemptive single-core RTOS platforms due to priority inversion: a high-priority + // task spinning would prevent the lock holder from running to release it. + static Mutex lock; + /* + * Multi-threaded platforms with atomic support: last_millis needs atomic for lock-free updates. + * Writers publish last_millis with memory_order_release and readers use memory_order_acquire. + * This ensures that once a reader sees the new low word, it also observes the corresponding + * increment of millis_major. + */ + static std::atomic last_millis{0}; + /* + * Upper 16 bits of the 64-bit millis counter. Incremented only while holding lock; + * read concurrently. Atomic (relaxed) avoids a formal data race. Ordering relative + * to last_millis is provided by its release store and the corresponding acquire loads. + */ + static std::atomic millis_major{0}; +#elif !defined(ESPHOME_THREAD_SINGLE) /* ESPHOME_THREAD_MULTI_NO_ATOMICS */ + static Mutex lock; + static uint32_t last_millis{0}; + static uint16_t millis_major{0}; +#else /* ESPHOME_THREAD_SINGLE */ + static uint32_t last_millis{0}; + static uint16_t millis_major{0}; +#endif + + // THREAD SAFETY NOTE: + // This function has three implementations, based on the precompiler flags + // - ESPHOME_THREAD_SINGLE - Runs on single-threaded platforms (ESP8266, etc.) + // - ESPHOME_THREAD_MULTI_NO_ATOMICS - Runs on multi-threaded platforms without atomics (LibreTiny BK72xx) + // - ESPHOME_THREAD_MULTI_ATOMICS - Runs on multi-threaded platforms with atomics (LibreTiny RTL87xx/LN882x, etc.) + // + // Make sure all changes are synchronized if you edit this function. + // + // IMPORTANT: Always pass fresh millis() values to this function. The implementation + // handles out-of-order timestamps between threads, but minimizing time differences + // helps maintain accuracy. + +#ifdef ESPHOME_THREAD_SINGLE + // Single-core platforms have no concurrency, so this is a simple implementation + // that just tracks 32-bit rollover (every 49.7 days) without any locking or atomics. + + uint16_t major = millis_major; + uint32_t last = last_millis; + + // Check for rollover + if (now < last && (last - now) > HALF_MAX_UINT32) { + millis_major++; + major++; + last_millis = now; +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); +#endif /* ESPHOME_DEBUG_SCHEDULER */ + } else if (now > last) { + // Only update if time moved forward + last_millis = now; + } + + // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time + return now + (static_cast(major) << 32); + +#elif defined(ESPHOME_THREAD_MULTI_NO_ATOMICS) + // Without atomics, this implementation uses locks more aggressively: + // 1. Always locks when near the rollover boundary (within 10 seconds) + // 2. Always locks when detecting a large backwards jump + // 3. Updates without lock in normal forward progression (accepting minor races) + // This is less efficient but necessary without atomic operations. + uint16_t major = millis_major; + uint32_t last = last_millis; + + // Define a safe window around the rollover point (10 seconds) + // This covers any reasonable scheduler delays or thread preemption + static constexpr uint32_t ROLLOVER_WINDOW = 10000; // 10 seconds in milliseconds + + // Check if we're near the rollover boundary (close to std::numeric_limits::max() or just past 0) + bool near_rollover = (last > (std::numeric_limits::max() - ROLLOVER_WINDOW)) || (now < ROLLOVER_WINDOW); + + if (near_rollover || (now < last && (last - now) > HALF_MAX_UINT32)) { + // Near rollover or detected a rollover - need lock for safety + LockGuard guard{lock}; + // Re-read with lock held + last = last_millis; + + if (now < last && (last - now) > HALF_MAX_UINT32) { + // True rollover detected (happens every ~49.7 days) + millis_major++; + major++; +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); +#endif /* ESPHOME_DEBUG_SCHEDULER */ + } + // Update last_millis while holding lock + last_millis = now; + } else if (now > last) { + // Normal case: Not near rollover and time moved forward + // Update without lock. While this may cause minor races (microseconds of + // backwards time movement), they're acceptable because: + // 1. The scheduler operates at millisecond resolution, not microsecond + // 2. We've already prevented the critical rollover race condition + // 3. Any backwards movement is orders of magnitude smaller than scheduler delays + last_millis = now; + } + // If now <= last and we're not near rollover, don't update + // This minimizes backwards time movement + + // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time + return now + (static_cast(major) << 32); + +#elif defined(ESPHOME_THREAD_MULTI_ATOMICS) + // Uses atomic operations with acquire/release semantics to ensure coherent + // reads of millis_major and last_millis across cores. Features: + // 1. Epoch-coherency retry loop to handle concurrent updates + // 2. Lock only taken for actual rollover detection and update + // 3. Lock-free CAS updates for normal forward time progression + // 4. Memory ordering ensures cores see consistent time values + + for (;;) { + uint16_t major = millis_major.load(std::memory_order_acquire); + + /* + * Acquire so that if we later decide **not** to take the lock we still + * observe a millis_major value coherent with the loaded last_millis. + * The acquire load ensures any later read of millis_major sees its + * corresponding increment. + */ + uint32_t last = last_millis.load(std::memory_order_acquire); + + // If we might be near a rollover (large backwards jump), take the lock + // This ensures rollover detection and last_millis update are atomic together + if (now < last && (last - now) > HALF_MAX_UINT32) { + // Potential rollover - need lock for atomic rollover detection + update + LockGuard guard{lock}; + // Re-read with lock held; mutex already provides ordering + last = last_millis.load(std::memory_order_relaxed); + + if (now < last && (last - now) > HALF_MAX_UINT32) { + // True rollover detected (happens every ~49.7 days) + millis_major.fetch_add(1, std::memory_order_relaxed); + major++; +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); +#endif /* ESPHOME_DEBUG_SCHEDULER */ + } + /* + * Update last_millis while holding the lock to prevent races. + * Publish the new low-word *after* bumping millis_major (done above) + * so readers never see a mismatched pair. + */ + last_millis.store(now, std::memory_order_release); + } else { + // Normal case: Try lock-free update, but only allow forward movement within same epoch + // This prevents accidentally moving backwards across a rollover boundary + while (now > last && (now - last) < HALF_MAX_UINT32) { + if (last_millis.compare_exchange_weak(last, now, + std::memory_order_release, // success + std::memory_order_relaxed)) { // failure + break; + } + // CAS failure means no data was published; relaxed is fine + // last is automatically updated by compare_exchange_weak if it fails + } + } + uint16_t major_end = millis_major.load(std::memory_order_relaxed); + if (major_end == major) + return now + (static_cast(major) << 32); + } + // Unreachable - the loop always returns when major_end == major + __builtin_unreachable(); + +#else +#error \ + "No platform threading model defined. One of ESPHOME_THREAD_SINGLE, ESPHOME_THREAD_MULTI_NO_ATOMICS, or ESPHOME_THREAD_MULTI_ATOMICS must be defined." +#endif +} + +} // namespace esphome + +#endif // !USE_NATIVE_64BIT_TIME diff --git a/esphome/core/time_64.h b/esphome/core/time_64.h new file mode 100644 index 0000000000..42d4b041e5 --- /dev/null +++ b/esphome/core/time_64.h @@ -0,0 +1,24 @@ +#pragma once +#include "esphome/core/defines.h" + +#ifndef USE_NATIVE_64BIT_TIME + +#include + +namespace esphome { + +class Scheduler; + +/// Extends 32-bit millis() to 64-bit using rollover tracking. +/// Access restricted to platform HAL (millis_64()) and Scheduler. +/// All other code should call millis_64() from hal.h instead. +class Millis64Impl { + friend uint64_t millis_64(); + friend class Scheduler; + + static uint64_t compute(uint32_t now); +}; + +} // namespace esphome + +#endif // !USE_NATIVE_64BIT_TIME From 82da4935b603624bf90a1429309e3d23d749bd46 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:57:08 -1000 Subject: [PATCH 14/20] [core] Auto-wrap static strings in PROGMEM on ESP8266 via TemplatableValue (#13885) --- esphome/codegen.py | 1 + esphome/components/api/__init__.py | 34 +++++--- .../components/api/homeassistant_service.h | 46 ++++++++++- esphome/core/automation.h | 56 +++++++++++-- esphome/cpp_generator.py | 26 +++++++ tests/unit_tests/test_cpp_generator.py | 78 +++++++++++++++++++ 6 files changed, 223 insertions(+), 18 deletions(-) diff --git a/esphome/codegen.py b/esphome/codegen.py index c5283f4967..30e3135360 100644 --- a/esphome/codegen.py +++ b/esphome/codegen.py @@ -11,6 +11,7 @@ from esphome.cpp_generator import ( # noqa: F401 ArrayInitializer, Expression, + FlashStringLiteral, LineComment, LogStringLiteral, MockObj, diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index d7b6bec357..dd99862cc2 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -535,24 +535,31 @@ async def homeassistant_service_to_code( cg.add_define("USE_API_HOMEASSISTANT_SERVICES") serv = await cg.get_variable(config[CONF_ID]) var = cg.new_Pvariable(action_id, template_arg, serv, False) - templ = await cg.templatable(config[CONF_ACTION], args, None) + templ = await cg.templatable(config[CONF_ACTION], args, cg.std_string) cg.add(var.set_service(templ)) # Initialize FixedVectors with exact sizes from config cg.add(var.init_data(len(config[CONF_DATA]))) for key, value in config[CONF_DATA].items(): + # output_type=None because lambdas can return non-string types (int, + # float, char*) that TemplatableStringValue converts via to_string. + # Static strings are manually wrapped for PROGMEM on ESP8266. templ = await cg.templatable(value, args, None) - cg.add(var.add_data(key, templ)) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) + cg.add(var.add_data(cg.FlashStringLiteral(key), templ)) cg.add(var.init_data_template(len(config[CONF_DATA_TEMPLATE]))) for key, value in config[CONF_DATA_TEMPLATE].items(): templ = await cg.templatable(value, args, None) - cg.add(var.add_data_template(key, templ)) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) + cg.add(var.add_data_template(cg.FlashStringLiteral(key), templ)) cg.add(var.init_variables(len(config[CONF_VARIABLES]))) for key, value in config[CONF_VARIABLES].items(): templ = await cg.templatable(value, args, None) - cg.add(var.add_variable(key, templ)) + cg.add(var.add_variable(cg.FlashStringLiteral(key), templ)) if on_error := config.get(CONF_ON_ERROR): cg.add_define("USE_API_HOMEASSISTANT_ACTION_RESPONSES") @@ -621,24 +628,31 @@ async def homeassistant_event_to_code(config, action_id, template_arg, args): cg.add_define("USE_API_HOMEASSISTANT_SERVICES") serv = await cg.get_variable(config[CONF_ID]) var = cg.new_Pvariable(action_id, template_arg, serv, True) - templ = await cg.templatable(config[CONF_EVENT], args, None) + templ = await cg.templatable(config[CONF_EVENT], args, cg.std_string) cg.add(var.set_service(templ)) # Initialize FixedVectors with exact sizes from config cg.add(var.init_data(len(config[CONF_DATA]))) for key, value in config[CONF_DATA].items(): + # output_type=None because lambdas can return non-string types (int, + # float, char*) that TemplatableStringValue converts via to_string. + # Static strings are manually wrapped for PROGMEM on ESP8266. templ = await cg.templatable(value, args, None) - cg.add(var.add_data(key, templ)) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) + cg.add(var.add_data(cg.FlashStringLiteral(key), templ)) cg.add(var.init_data_template(len(config[CONF_DATA_TEMPLATE]))) for key, value in config[CONF_DATA_TEMPLATE].items(): templ = await cg.templatable(value, args, None) - cg.add(var.add_data_template(key, templ)) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) + cg.add(var.add_data_template(cg.FlashStringLiteral(key), templ)) cg.add(var.init_variables(len(config[CONF_VARIABLES]))) for key, value in config[CONF_VARIABLES].items(): templ = await cg.templatable(value, args, None) - cg.add(var.add_variable(key, templ)) + cg.add(var.add_variable(cg.FlashStringLiteral(key), templ)) return var @@ -662,11 +676,11 @@ async def homeassistant_tag_scanned_to_code(config, action_id, template_arg, arg cg.add_define("USE_API_HOMEASSISTANT_SERVICES") serv = await cg.get_variable(config[CONF_ID]) var = cg.new_Pvariable(action_id, template_arg, serv, True) - cg.add(var.set_service("esphome.tag_scanned")) + cg.add(var.set_service(cg.FlashStringLiteral("esphome.tag_scanned"))) # Initialize FixedVector with exact size (1 data field) cg.add(var.init_data(1)) templ = await cg.templatable(config[CONF_TAG], args, cg.std_string) - cg.add(var.add_data("tag_id", templ)) + cg.add(var.add_data(cg.FlashStringLiteral("tag_id"), templ)) return var diff --git a/esphome/components/api/homeassistant_service.h b/esphome/components/api/homeassistant_service.h index 340699e1a6..9d14061d07 100644 --- a/esphome/components/api/homeassistant_service.h +++ b/esphome/components/api/homeassistant_service.h @@ -130,6 +130,20 @@ template class HomeAssistantServiceCallAction : public Actionadd_kv_(this->variables_, key, std::forward(value)); } +#ifdef USE_ESP8266 + // On ESP8266, ESPHOME_F() returns __FlashStringHelper* (PROGMEM pointer). + // Store as const char* — populate_service_map copies from PROGMEM at play() time. + template void add_data(const __FlashStringHelper *key, V &&value) { + this->add_kv_(this->data_, reinterpret_cast(key), std::forward(value)); + } + template void add_data_template(const __FlashStringHelper *key, V &&value) { + this->add_kv_(this->data_template_, reinterpret_cast(key), std::forward(value)); + } + template void add_variable(const __FlashStringHelper *key, V &&value) { + this->add_kv_(this->variables_, reinterpret_cast(key), std::forward(value)); + } +#endif + #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES template void set_response_template(T response_template) { this->response_template_ = response_template; @@ -221,7 +235,32 @@ template class HomeAssistantServiceCallAction : public Action class HomeAssistantServiceCallAction : public Action #include @@ -56,6 +57,16 @@ template class TemplatableValue { this->static_str_ = str; } +#ifdef USE_ESP8266 + // On ESP8266, __FlashStringHelper* is a distinct type from const char*. + // ESPHOME_F(s) expands to F(s) which returns __FlashStringHelper* pointing to PROGMEM. + // Store as FLASH_STRING — value()/is_empty()/ref_or_copy_to() use _P functions + // to access the PROGMEM pointer safely. + TemplatableValue(const __FlashStringHelper *str) requires std::same_as : type_(FLASH_STRING) { + this->static_str_ = reinterpret_cast(str); + } +#endif + template TemplatableValue(F value) requires(!std::invocable) : type_(VALUE) { if constexpr (USE_HEAP_STORAGE) { this->value_ = new T(std::move(value)); @@ -89,7 +100,7 @@ template class TemplatableValue { this->f_ = new std::function(*other.f_); } else if (this->type_ == STATELESS_LAMBDA) { this->stateless_f_ = other.stateless_f_; - } else if (this->type_ == STATIC_STRING) { + } else if (this->type_ == STATIC_STRING || this->type_ == FLASH_STRING) { this->static_str_ = other.static_str_; } } @@ -108,7 +119,7 @@ template class TemplatableValue { other.f_ = nullptr; } else if (this->type_ == STATELESS_LAMBDA) { this->stateless_f_ = other.stateless_f_; - } else if (this->type_ == STATIC_STRING) { + } else if (this->type_ == STATIC_STRING || this->type_ == FLASH_STRING) { this->static_str_ = other.static_str_; } other.type_ = NONE; @@ -141,7 +152,7 @@ template class TemplatableValue { } else if (this->type_ == LAMBDA) { delete this->f_; } - // STATELESS_LAMBDA/STATIC_STRING/NONE: no cleanup needed (pointers, not heap-allocated) + // STATELESS_LAMBDA/STATIC_STRING/FLASH_STRING/NONE: no cleanup needed (pointers, not heap-allocated) } bool has_value() const { return this->type_ != NONE; } @@ -165,6 +176,17 @@ template class TemplatableValue { return std::string(this->static_str_); } __builtin_unreachable(); +#ifdef USE_ESP8266 + case FLASH_STRING: + // PROGMEM pointer — must use _P functions to access on ESP8266 + if constexpr (std::same_as) { + size_t len = strlen_P(this->static_str_); + std::string result(len, '\0'); + memcpy_P(result.data(), this->static_str_, len); + return result; + } + __builtin_unreachable(); +#endif case NONE: default: return T{}; @@ -186,9 +208,12 @@ template class TemplatableValue { } /// Check if this holds a static string (const char* stored without allocation) + /// The pointer is always directly readable (RAM or flash-mapped). + /// Returns false for FLASH_STRING (PROGMEM on ESP8266, requires _P functions). bool is_static_string() const { return this->type_ == STATIC_STRING; } /// Get the static string pointer (only valid if is_static_string() returns true) + /// The pointer is always directly readable — FLASH_STRING uses a separate type. const char *get_static_string() const { return this->static_str_; } /// Check if the string value is empty without allocating (for std::string specialization). @@ -200,6 +225,12 @@ template class TemplatableValue { return true; case STATIC_STRING: return this->static_str_ == nullptr || this->static_str_[0] == '\0'; +#ifdef USE_ESP8266 + case FLASH_STRING: + // PROGMEM pointer — must use progmem_read_byte on ESP8266 + return this->static_str_ == nullptr || + progmem_read_byte(reinterpret_cast(this->static_str_)) == '\0'; +#endif case VALUE: return this->value_->empty(); default: // LAMBDA/STATELESS_LAMBDA - must call value() @@ -209,8 +240,9 @@ template class TemplatableValue { /// Get a StringRef to the string value without heap allocation when possible. /// For STATIC_STRING/VALUE, returns reference to existing data (no allocation). + /// For FLASH_STRING (ESP8266 PROGMEM), copies to provided buffer via _P functions. /// For LAMBDA/STATELESS_LAMBDA, calls value(), copies to provided buffer, returns ref to buffer. - /// @param lambda_buf Buffer used only for lambda case (must remain valid while StringRef is used). + /// @param lambda_buf Buffer used only for copy cases (must remain valid while StringRef is used). /// @param lambda_buf_size Size of the buffer. /// @return StringRef pointing to the string data. StringRef ref_or_copy_to(char *lambda_buf, size_t lambda_buf_size) const requires std::same_as { @@ -221,6 +253,19 @@ template class TemplatableValue { if (this->static_str_ == nullptr) return StringRef(); return StringRef(this->static_str_, strlen(this->static_str_)); +#ifdef USE_ESP8266 + case FLASH_STRING: + if (this->static_str_ == nullptr) + return StringRef(); + { + // PROGMEM pointer — copy to buffer via _P functions + size_t len = strlen_P(this->static_str_); + size_t copy_len = std::min(len, lambda_buf_size - 1); + memcpy_P(lambda_buf, this->static_str_, copy_len); + lambda_buf[copy_len] = '\0'; + return StringRef(lambda_buf, copy_len); + } +#endif case VALUE: return StringRef(this->value_->data(), this->value_->size()); default: { // LAMBDA/STATELESS_LAMBDA - must call value() and copy @@ -239,6 +284,7 @@ template class TemplatableValue { LAMBDA, STATELESS_LAMBDA, STATIC_STRING, // For const char* when T is std::string - avoids heap allocation + FLASH_STRING, // PROGMEM pointer on ESP8266; never set on other platforms } type_; // For std::string, use heap pointer to minimize union size (4 bytes vs 12+). // For other types, store value inline as before. @@ -247,7 +293,7 @@ template class TemplatableValue { ValueStorage value_; // T for inline storage, T* for heap storage std::function *f_; T (*stateless_f_)(X...); - const char *static_str_; // For STATIC_STRING type + const char *static_str_; // For STATIC_STRING and FLASH_STRING types }; }; diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index fe666bdd6e..5457485d25 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -247,6 +247,23 @@ class LogStringLiteral(Literal): return f"LOG_STR({cpp_string_escape(self.string)})" +class FlashStringLiteral(Literal): + """A string literal wrapped in ESPHOME_F() for PROGMEM storage on ESP8266. + + On ESP8266, ESPHOME_F(s) expands to F(s) which stores the string in flash (PROGMEM). + On other platforms, ESPHOME_F(s) expands to plain s (no-op). + """ + + __slots__ = ("string",) + + def __init__(self, string: str) -> None: + super().__init__() + self.string = string + + def __str__(self) -> str: + return f"ESPHOME_F({cpp_string_escape(self.string)})" + + class IntLiteral(Literal): __slots__ = ("i",) @@ -761,6 +778,15 @@ async def templatable( if is_template(value): return await process_lambda(value, args, return_type=output_type) if to_exp is None: + # Automatically wrap static strings in ESPHOME_F() for PROGMEM storage on ESP8266. + # On other platforms ESPHOME_F() is a no-op returning const char*. + # Lazy import to avoid circular dependency (cpp_generator <-> cpp_types). + # Identity check (is) avoids brittle string comparison. + if isinstance(value, str) and output_type is not None: + from esphome.cpp_types import std_string + + if output_type is std_string: + return FlashStringLiteral(value) return value if isinstance(to_exp, dict): return to_exp[value] diff --git a/tests/unit_tests/test_cpp_generator.py b/tests/unit_tests/test_cpp_generator.py index 049d21027f..bdc31cdef8 100644 --- a/tests/unit_tests/test_cpp_generator.py +++ b/tests/unit_tests/test_cpp_generator.py @@ -248,6 +248,12 @@ class TestLiterals: (cg.FloatLiteral(4.2), "4.2f"), (cg.FloatLiteral(1.23456789), "1.23456789f"), (cg.FloatLiteral(math.nan), "NAN"), + (cg.FlashStringLiteral("hello"), 'ESPHOME_F("hello")'), + (cg.FlashStringLiteral(""), 'ESPHOME_F("")'), + ( + cg.FlashStringLiteral('quote"here'), + 'ESPHOME_F("quote\\042here")', + ), ), ) def test_str__simple(self, target: cg.Literal, expected: str): @@ -624,3 +630,75 @@ class TestProcessLambda: # Test invalid tuple format (single element) with pytest.raises(AssertionError): await cg.process_lambda(lambda_obj, [(int,)]) + + +@pytest.mark.asyncio +async def test_templatable__string_with_std_string_returns_flash_literal() -> None: + """Static string with std::string output_type returns FlashStringLiteral.""" + result = await cg.templatable("hello", [], ct.std_string) + + assert isinstance(result, cg.FlashStringLiteral) + assert str(result) == 'ESPHOME_F("hello")' + + +@pytest.mark.asyncio +async def test_templatable__empty_string_with_std_string() -> None: + """Empty static string with std::string output_type returns FlashStringLiteral.""" + result = await cg.templatable("", [], ct.std_string) + + assert isinstance(result, cg.FlashStringLiteral) + assert str(result) == 'ESPHOME_F("")' + + +@pytest.mark.asyncio +async def test_templatable__string_with_none_output_type() -> None: + """Static string with output_type=None returns raw string (no wrapping).""" + result = await cg.templatable("hello", [], None) + + assert isinstance(result, str) + assert result == "hello" + + +@pytest.mark.asyncio +async def test_templatable__int_with_std_string() -> None: + """Non-string value with std::string output_type returns raw value.""" + result = await cg.templatable(42, [], ct.std_string) + + assert result == 42 + + +@pytest.mark.asyncio +async def test_templatable__string_with_non_string_output_type() -> None: + """Static string with non-std::string output_type returns raw string.""" + result = await cg.templatable("hello", [], ct.bool_) + + assert isinstance(result, str) + assert result == "hello" + + +@pytest.mark.asyncio +async def test_templatable__with_to_exp_callable() -> None: + """When to_exp is provided, it is applied to non-template values.""" + result = await cg.templatable(42, [], None, to_exp=lambda x: x * 2) + + assert result == 84 + + +@pytest.mark.asyncio +async def test_templatable__with_to_exp_dict() -> None: + """When to_exp is a dict, value is looked up.""" + mapping: dict[str, int] = {"on": 1, "off": 0} + result = await cg.templatable("on", [], None, to_exp=mapping) + + assert result == 1 + + +@pytest.mark.asyncio +async def test_templatable__lambda_with_std_string() -> None: + """Lambda value returns LambdaExpression, not FlashStringLiteral.""" + from esphome.core import Lambda + + lambda_obj = Lambda('return "hello";') + result = await cg.templatable(lambda_obj, [], ct.std_string) + + assert isinstance(result, cg.LambdaExpression) From 00242443e1e67366712840b9fe1b47831d2e9fea Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:57:23 -1000 Subject: [PATCH 15/20] [light] Replace powf gamma with pre-computed lookup tables (LUT) (#14123) Co-authored-by: Claude Opus 4.6 --- esphome/components/esp32/core.cpp | 1 + esphome/components/esp8266/core.cpp | 3 + esphome/components/host/core.cpp | 1 + esphome/components/libretiny/core.cpp | 1 + esphome/components/light/__init__.py | 40 +++++++++- esphome/components/light/addressable_light.h | 4 +- .../light/addressable_light_wrapper.h | 9 +-- .../components/light/esp_color_correction.cpp | 36 ++++----- .../components/light/esp_color_correction.h | 45 +++++++---- esphome/components/light/light_call.cpp | 5 +- esphome/components/light/light_color_values.h | 48 +++++------- esphome/components/light/light_state.cpp | 78 ++++++++++++++++--- esphome/components/light/light_state.h | 26 ++++++- esphome/components/lvgl/light/lvgl_light.h | 2 +- esphome/components/rgb/rgb_light_output.h | 2 +- esphome/components/rgbw/rgbw_light_output.h | 2 +- esphome/components/rp2040/core.cpp | 1 + esphome/components/zephyr/core.cpp | 1 + esphome/core/defines.h | 1 + esphome/core/hal.h | 1 + esphome/core/helpers.h | 4 + 21 files changed, 228 insertions(+), 83 deletions(-) diff --git a/esphome/components/esp32/core.cpp b/esphome/components/esp32/core.cpp index b9ae871abf..59b791da40 100644 --- a/esphome/components/esp32/core.cpp +++ b/esphome/components/esp32/core.cpp @@ -48,6 +48,7 @@ void arch_init() { void HOT arch_feed_wdt() { esp_task_wdt_reset(); } uint8_t progmem_read_byte(const uint8_t *addr) { return *addr; } +uint16_t progmem_read_uint16(const uint16_t *addr) { return *addr; } uint32_t arch_get_cpu_cycle_count() { return esp_cpu_get_cycle_count(); } uint32_t arch_get_cpu_freq_hz() { uint32_t freq = 0; diff --git a/esphome/components/esp8266/core.cpp b/esphome/components/esp8266/core.cpp index 7136cf41cb..b665124d66 100644 --- a/esphome/components/esp8266/core.cpp +++ b/esphome/components/esp8266/core.cpp @@ -34,6 +34,9 @@ void HOT arch_feed_wdt() { system_soft_wdt_feed(); } uint8_t progmem_read_byte(const uint8_t *addr) { return pgm_read_byte(addr); // NOLINT } +uint16_t progmem_read_uint16(const uint16_t *addr) { + return pgm_read_word(addr); // NOLINT +} uint32_t IRAM_ATTR HOT arch_get_cpu_cycle_count() { return esp_get_cycle_count(); } uint32_t arch_get_cpu_freq_hz() { return F_CPU; } diff --git a/esphome/components/host/core.cpp b/esphome/components/host/core.cpp index 9af85bec58..cb2b2e19d7 100644 --- a/esphome/components/host/core.cpp +++ b/esphome/components/host/core.cpp @@ -59,6 +59,7 @@ void HOT arch_feed_wdt() { } uint8_t progmem_read_byte(const uint8_t *addr) { return *addr; } +uint16_t progmem_read_uint16(const uint16_t *addr) { return *addr; } uint32_t arch_get_cpu_cycle_count() { struct timespec spec; clock_gettime(CLOCK_MONOTONIC, &spec); diff --git a/esphome/components/libretiny/core.cpp b/esphome/components/libretiny/core.cpp index 88952692a7..74b33a30a0 100644 --- a/esphome/components/libretiny/core.cpp +++ b/esphome/components/libretiny/core.cpp @@ -36,6 +36,7 @@ void HOT arch_feed_wdt() { lt_wdt_feed(); } uint32_t arch_get_cpu_cycle_count() { return lt_cpu_get_cycle_count(); } uint32_t arch_get_cpu_freq_hz() { return lt_cpu_get_freq(); } uint8_t progmem_read_byte(const uint8_t *addr) { return *addr; } +uint16_t progmem_read_uint16(const uint16_t *addr) { return *addr; } } // namespace esphome diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index f1089ad64f..40382bbda7 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass, field import enum import esphome.automation as auto @@ -37,7 +38,7 @@ from esphome.const import ( CONF_WEB_SERVER, CONF_WHITE, ) -from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.core import CORE, ID, CoroPriority, HexInt, coroutine_with_priority from esphome.core.entity_helpers import entity_duplicate_validator, setup_entity from esphome.cpp_generator import MockObjClass @@ -66,6 +67,40 @@ from .types import ( # noqa CODEOWNERS = ["@esphome/core"] IS_PLATFORM_COMPONENT = True +DOMAIN = "light" + + +@dataclass +class LightData: + gamma_tables: dict = field(default_factory=dict) # gamma_value -> fwd_arr + + +def _get_data() -> LightData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = LightData() + return CORE.data[DOMAIN] + + +def _get_or_create_gamma_table(gamma_correct): + data = _get_data() + if gamma_correct in data.gamma_tables: + return data.gamma_tables[gamma_correct] + + if gamma_correct > 0: + forward = [ + HexInt(min(65535, int(round((i / 255.0) ** gamma_correct * 65535)))) + for i in range(256) + ] + else: + forward = [HexInt(int(round(i / 255.0 * 65535))) for i in range(256)] + + gamma_str = f"{gamma_correct}".replace(".", "_") + fwd_id = ID(f"gamma_{gamma_str}_fwd", is_declaration=True, type=cg.uint16) + fwd_arr = cg.progmem_array(fwd_id, forward) + data.gamma_tables[gamma_correct] = fwd_arr + return fwd_arr + + LightRestoreMode = light_ns.enum("LightRestoreMode") RESTORE_MODES = { "RESTORE_DEFAULT_OFF": LightRestoreMode.LIGHT_RESTORE_DEFAULT_OFF, @@ -239,6 +274,9 @@ async def setup_light_core_(light_var, output_var, config): cg.add(light_var.set_flash_transition_length(flash_transition_length)) if (gamma_correct := config.get(CONF_GAMMA_CORRECT)) is not None: cg.add(light_var.set_gamma_correct(gamma_correct)) + fwd_arr = _get_or_create_gamma_table(gamma_correct) + cg.add(light_var.set_gamma_table(fwd_arr)) + cg.add_define("USE_LIGHT_GAMMA_LUT") effects = await cg.build_registry_list( EFFECTS_REGISTRY, config.get(CONF_EFFECTS, []) ) diff --git a/esphome/components/light/addressable_light.h b/esphome/components/light/addressable_light.h index fcaf07f578..17cdb7d6f6 100644 --- a/esphome/components/light/addressable_light.h +++ b/esphome/components/light/addressable_light.h @@ -66,7 +66,9 @@ class AddressableLight : public LightOutput, public Component { Color(to_uint8_scale(red), to_uint8_scale(green), to_uint8_scale(blue), to_uint8_scale(white))); } void setup_state(LightState *state) override { - this->correction_.calculate_gamma_table(state->get_gamma_correct()); +#ifdef USE_LIGHT_GAMMA_LUT + this->correction_.set_gamma_table(state->get_gamma_table()); +#endif this->state_parent_ = state; } void update_state(LightState *state) override; diff --git a/esphome/components/light/addressable_light_wrapper.h b/esphome/components/light/addressable_light_wrapper.h index cd83482248..cc6aa57905 100644 --- a/esphome/components/light/addressable_light_wrapper.h +++ b/esphome/components/light/addressable_light_wrapper.h @@ -74,11 +74,10 @@ class AddressableLightWrapper : public light::AddressableLight { return; } - float gamma = this->light_state_->get_gamma_correct(); - float r = gamma_uncorrect(this->wrapper_state_[0] / 255.0f, gamma); - float g = gamma_uncorrect(this->wrapper_state_[1] / 255.0f, gamma); - float b = gamma_uncorrect(this->wrapper_state_[2] / 255.0f, gamma); - float w = gamma_uncorrect(this->wrapper_state_[3] / 255.0f, gamma); + float r = this->light_state_->gamma_uncorrect_lut(this->wrapper_state_[0] / 255.0f); + float g = this->light_state_->gamma_uncorrect_lut(this->wrapper_state_[1] / 255.0f); + float b = this->light_state_->gamma_uncorrect_lut(this->wrapper_state_[2] / 255.0f); + float w = this->light_state_->gamma_uncorrect_lut(this->wrapper_state_[3] / 255.0f); auto call = this->light_state_->make_call(); diff --git a/esphome/components/light/esp_color_correction.cpp b/esphome/components/light/esp_color_correction.cpp index 1b511a94b2..9d731a2bd5 100644 --- a/esphome/components/light/esp_color_correction.cpp +++ b/esphome/components/light/esp_color_correction.cpp @@ -1,25 +1,25 @@ #include "esp_color_correction.h" -#include "light_color_values.h" -#include "esphome/core/log.h" namespace esphome::light { -void ESPColorCorrection::calculate_gamma_table(float gamma) { - for (uint16_t i = 0; i < 256; i++) { - // corrected = val ^ gamma - auto corrected = to_uint8_scale(gamma_correct(i / 255.0f, gamma)); - this->gamma_table_[i] = corrected; - } - if (gamma == 0.0f) { - for (uint16_t i = 0; i < 256; i++) - this->gamma_reverse_table_[i] = i; - return; - } - for (uint16_t i = 0; i < 256; i++) { - // val = corrected ^ (1/gamma) - auto uncorrected = to_uint8_scale(powf(i / 255.0f, 1.0f / gamma)); - this->gamma_reverse_table_[i] = uncorrected; - } +uint8_t ESPColorCorrection::gamma_correct_(uint8_t value) const { + if (this->gamma_table_ == nullptr) + return value; + return static_cast((progmem_read_uint16(&this->gamma_table_[value]) + 128) / 257); +} + +uint8_t ESPColorCorrection::gamma_uncorrect_(uint8_t value) const { + if (this->gamma_table_ == nullptr) + return value; + if (value == 0) + return 0; + uint16_t target = value * 257; // Scale 0-255 to 0-65535 + uint8_t lo = gamma_table_reverse_search(this->gamma_table_, target); + if (lo >= 255) + return 255; + uint16_t a = progmem_read_uint16(&this->gamma_table_[lo]); + uint16_t b = progmem_read_uint16(&this->gamma_table_[lo + 1]); + return (target - a <= b - target) ? lo : lo + 1; } } // namespace esphome::light diff --git a/esphome/components/light/esp_color_correction.h b/esphome/components/light/esp_color_correction.h index d275e045b7..48ecc46364 100644 --- a/esphome/components/light/esp_color_correction.h +++ b/esphome/components/light/esp_color_correction.h @@ -1,15 +1,30 @@ #pragma once #include "esphome/core/color.h" +#include "esphome/core/hal.h" namespace esphome::light { +/// Binary search a monotonically increasing uint16[256] PROGMEM table. +/// Returns the largest index where table[index] <= target. +inline uint8_t gamma_table_reverse_search(const uint16_t *table, uint16_t target) { + uint8_t lo = 0, hi = 255; + while (lo < hi) { + uint8_t mid = (lo + hi + 1) / 2; + if (progmem_read_uint16(&table[mid]) <= target) { + lo = mid; + } else { + hi = mid - 1; + } + } + return lo; +} + class ESPColorCorrection { public: - ESPColorCorrection() : max_brightness_(255, 255, 255, 255) {} void set_max_brightness(const Color &max_brightness) { this->max_brightness_ = max_brightness; } void set_local_brightness(uint8_t local_brightness) { this->local_brightness_ = local_brightness; } - void calculate_gamma_table(float gamma); + void set_gamma_table(const uint16_t *table) { this->gamma_table_ = table; } inline Color color_correct(Color color) const ESPHOME_ALWAYS_INLINE { // corrected = (uncorrected * max_brightness * local_brightness) ^ gamma return Color(this->color_correct_red(color.red), this->color_correct_green(color.green), @@ -17,19 +32,19 @@ class ESPColorCorrection { } inline uint8_t color_correct_red(uint8_t red) const ESPHOME_ALWAYS_INLINE { uint8_t res = esp_scale8_twice(red, this->max_brightness_.red, this->local_brightness_); - return this->gamma_table_[res]; + return this->gamma_correct_(res); } inline uint8_t color_correct_green(uint8_t green) const ESPHOME_ALWAYS_INLINE { uint8_t res = esp_scale8_twice(green, this->max_brightness_.green, this->local_brightness_); - return this->gamma_table_[res]; + return this->gamma_correct_(res); } inline uint8_t color_correct_blue(uint8_t blue) const ESPHOME_ALWAYS_INLINE { uint8_t res = esp_scale8_twice(blue, this->max_brightness_.blue, this->local_brightness_); - return this->gamma_table_[res]; + return this->gamma_correct_(res); } inline uint8_t color_correct_white(uint8_t white) const ESPHOME_ALWAYS_INLINE { uint8_t res = esp_scale8_twice(white, this->max_brightness_.white, this->local_brightness_); - return this->gamma_table_[res]; + return this->gamma_correct_(res); } inline Color color_uncorrect(Color color) const ESPHOME_ALWAYS_INLINE { // uncorrected = corrected^(1/gamma) / (max_brightness * local_brightness) @@ -39,36 +54,40 @@ class ESPColorCorrection { inline uint8_t color_uncorrect_red(uint8_t red) const ESPHOME_ALWAYS_INLINE { if (this->max_brightness_.red == 0 || this->local_brightness_ == 0) return 0; - uint16_t uncorrected = this->gamma_reverse_table_[red] * 255UL; + uint16_t uncorrected = this->gamma_uncorrect_(red) * 255UL; uint16_t res = ((uncorrected / this->max_brightness_.red) * 255UL) / this->local_brightness_; return (uint8_t) std::min(res, uint16_t(255)); } inline uint8_t color_uncorrect_green(uint8_t green) const ESPHOME_ALWAYS_INLINE { if (this->max_brightness_.green == 0 || this->local_brightness_ == 0) return 0; - uint16_t uncorrected = this->gamma_reverse_table_[green] * 255UL; + uint16_t uncorrected = this->gamma_uncorrect_(green) * 255UL; uint16_t res = ((uncorrected / this->max_brightness_.green) * 255UL) / this->local_brightness_; return (uint8_t) std::min(res, uint16_t(255)); } inline uint8_t color_uncorrect_blue(uint8_t blue) const ESPHOME_ALWAYS_INLINE { if (this->max_brightness_.blue == 0 || this->local_brightness_ == 0) return 0; - uint16_t uncorrected = this->gamma_reverse_table_[blue] * 255UL; + uint16_t uncorrected = this->gamma_uncorrect_(blue) * 255UL; uint16_t res = ((uncorrected / this->max_brightness_.blue) * 255UL) / this->local_brightness_; return (uint8_t) std::min(res, uint16_t(255)); } inline uint8_t color_uncorrect_white(uint8_t white) const ESPHOME_ALWAYS_INLINE { if (this->max_brightness_.white == 0 || this->local_brightness_ == 0) return 0; - uint16_t uncorrected = this->gamma_reverse_table_[white] * 255UL; + uint16_t uncorrected = this->gamma_uncorrect_(white) * 255UL; uint16_t res = ((uncorrected / this->max_brightness_.white) * 255UL) / this->local_brightness_; return (uint8_t) std::min(res, uint16_t(255)); } protected: - uint8_t gamma_table_[256]; - uint8_t gamma_reverse_table_[256]; - Color max_brightness_; + /// Forward gamma: read uint16 PROGMEM table, convert to uint8 + uint8_t gamma_correct_(uint8_t value) const; + /// Reverse gamma: binary search the forward PROGMEM table + uint8_t gamma_uncorrect_(uint8_t value) const; + + const uint16_t *gamma_table_{nullptr}; + Color max_brightness_{255, 255, 255, 255}; uint8_t local_brightness_{255}; }; diff --git a/esphome/components/light/light_call.cpp b/esphome/components/light/light_call.cpp index 0291b2c3c6..14cd0e92f6 100644 --- a/esphome/components/light/light_call.cpp +++ b/esphome/components/light/light_call.cpp @@ -389,9 +389,8 @@ void LightCall::transform_parameters_() { const float ww_fraction = (color_temp - min_mireds) / range; const float cw_fraction = 1.0f - ww_fraction; const float max_cw_ww = std::max(ww_fraction, cw_fraction); - const float gamma = this->parent_->get_gamma_correct(); - this->cold_white_ = gamma_uncorrect(cw_fraction / max_cw_ww, gamma); - this->warm_white_ = gamma_uncorrect(ww_fraction / max_cw_ww, gamma); + this->cold_white_ = this->parent_->gamma_uncorrect_lut(cw_fraction / max_cw_ww); + this->warm_white_ = this->parent_->gamma_uncorrect_lut(ww_fraction / max_cw_ww); this->set_flag_(FLAG_HAS_COLD_WHITE); this->set_flag_(FLAG_HAS_WARM_WHITE); } diff --git a/esphome/components/light/light_color_values.h b/esphome/components/light/light_color_values.h index dc23263312..3a9ca8c8c2 100644 --- a/esphome/components/light/light_color_values.h +++ b/esphome/components/light/light_color_values.h @@ -111,60 +111,54 @@ class LightColorValues { } } - // Note that method signature of as_* methods is kept as-is for compatibility reasons, so not all parameters - // are always used or necessary. Methods will be deprecated later. - /// Convert these light color values to a binary representation and write them to binary. void as_binary(bool *binary) const { *binary = this->state_ == 1.0f; } /// Convert these light color values to a brightness-only representation and write them to brightness. - void as_brightness(float *brightness, float gamma = 0) const { - *brightness = gamma_correct(this->state_ * this->brightness_, gamma); - } + void as_brightness(float *brightness) const { *brightness = this->state_ * this->brightness_; } /// Convert these light color values to an RGB representation and write them to red, green, blue. - void as_rgb(float *red, float *green, float *blue, float gamma = 0, bool color_interlock = false) const { + void as_rgb(float *red, float *green, float *blue) const { if (this->color_mode_ & ColorCapability::RGB) { float brightness = this->state_ * this->brightness_ * this->color_brightness_; - *red = gamma_correct(brightness * this->red_, gamma); - *green = gamma_correct(brightness * this->green_, gamma); - *blue = gamma_correct(brightness * this->blue_, gamma); + *red = brightness * this->red_; + *green = brightness * this->green_; + *blue = brightness * this->blue_; } else { *red = *green = *blue = 0; } } /// Convert these light color values to an RGBW representation and write them to red, green, blue, white. - void as_rgbw(float *red, float *green, float *blue, float *white, float gamma = 0, - bool color_interlock = false) const { - this->as_rgb(red, green, blue, gamma); + void as_rgbw(float *red, float *green, float *blue, float *white) const { + this->as_rgb(red, green, blue); if (this->color_mode_ & ColorCapability::WHITE) { - *white = gamma_correct(this->state_ * this->brightness_ * this->white_, gamma); + *white = this->state_ * this->brightness_ * this->white_; } else { *white = 0; } } /// Convert these light color values to an RGBWW representation with the given parameters. - void as_rgbww(float *red, float *green, float *blue, float *cold_white, float *warm_white, float gamma = 0, + void as_rgbww(float *red, float *green, float *blue, float *cold_white, float *warm_white, bool constant_brightness = false) const { - this->as_rgb(red, green, blue, gamma); - this->as_cwww(cold_white, warm_white, gamma, constant_brightness); + this->as_rgb(red, green, blue); + this->as_cwww(cold_white, warm_white, constant_brightness); } /// Convert these light color values to an RGB+CT+BR representation with the given parameters. void as_rgbct(float color_temperature_cw, float color_temperature_ww, float *red, float *green, float *blue, - float *color_temperature, float *white_brightness, float gamma = 0) const { - this->as_rgb(red, green, blue, gamma); - this->as_ct(color_temperature_cw, color_temperature_ww, color_temperature, white_brightness, gamma); + float *color_temperature, float *white_brightness) const { + this->as_rgb(red, green, blue); + this->as_ct(color_temperature_cw, color_temperature_ww, color_temperature, white_brightness); } /// Convert these light color values to an CWWW representation with the given parameters. - void as_cwww(float *cold_white, float *warm_white, float gamma = 0, bool constant_brightness = false) const { + void as_cwww(float *cold_white, float *warm_white, bool constant_brightness = false) const { if (this->color_mode_ & ColorCapability::COLD_WARM_WHITE) { - const float cw_level = gamma_correct(this->cold_white_, gamma); - const float ww_level = gamma_correct(this->warm_white_, gamma); - const float white_level = gamma_correct(this->state_ * this->brightness_, gamma); + const float cw_level = this->cold_white_; + const float ww_level = this->warm_white_; + const float white_level = this->state_ * this->brightness_; if (!constant_brightness) { *cold_white = white_level * cw_level; *warm_white = white_level * ww_level; @@ -184,13 +178,13 @@ class LightColorValues { } /// Convert these light color values to a CT+BR representation with the given parameters. - void as_ct(float color_temperature_cw, float color_temperature_ww, float *color_temperature, float *white_brightness, - float gamma = 0) const { + void as_ct(float color_temperature_cw, float color_temperature_ww, float *color_temperature, + float *white_brightness) const { const float white_level = this->color_mode_ & ColorCapability::RGB ? this->white_ : 1; if (this->color_mode_ & ColorCapability::COLOR_TEMPERATURE) { *color_temperature = (this->color_temperature_ - color_temperature_cw) / (color_temperature_ww - color_temperature_cw); - *white_brightness = gamma_correct(this->state_ * this->brightness_ * white_level, gamma); + *white_brightness = this->state_ * this->brightness_ * white_level; } else { // Probably won't get here but put this here anyway. *white_brightness = 0; } diff --git a/esphome/components/light/light_state.cpp b/esphome/components/light/light_state.cpp index ed86bf58da..161092532a 100644 --- a/esphome/components/light/light_state.cpp +++ b/esphome/components/light/light_state.cpp @@ -1,4 +1,5 @@ #include "light_state.h" +#include "esp_color_correction.h" #include "esphome/core/defines.h" #include "esphome/core/controller_registry.h" #include "esphome/core/log.h" @@ -204,33 +205,90 @@ void LightState::add_effects(const std::initializer_list &effects void LightState::current_values_as_binary(bool *binary) { this->current_values.as_binary(binary); } void LightState::current_values_as_brightness(float *brightness) { - this->current_values.as_brightness(brightness, this->gamma_correct_); + this->current_values.as_brightness(brightness); + *brightness = this->gamma_correct_lut(*brightness); } -void LightState::current_values_as_rgb(float *red, float *green, float *blue, bool color_interlock) { - this->current_values.as_rgb(red, green, blue, this->gamma_correct_, false); +void LightState::current_values_as_rgb(float *red, float *green, float *blue) { + this->current_values.as_rgb(red, green, blue); + *red = this->gamma_correct_lut(*red); + *green = this->gamma_correct_lut(*green); + *blue = this->gamma_correct_lut(*blue); } -void LightState::current_values_as_rgbw(float *red, float *green, float *blue, float *white, bool color_interlock) { - this->current_values.as_rgbw(red, green, blue, white, this->gamma_correct_, false); +void LightState::current_values_as_rgbw(float *red, float *green, float *blue, float *white) { + this->current_values.as_rgbw(red, green, blue, white); + *red = this->gamma_correct_lut(*red); + *green = this->gamma_correct_lut(*green); + *blue = this->gamma_correct_lut(*blue); + *white = this->gamma_correct_lut(*white); } void LightState::current_values_as_rgbww(float *red, float *green, float *blue, float *cold_white, float *warm_white, bool constant_brightness) { - this->current_values.as_rgbww(red, green, blue, cold_white, warm_white, this->gamma_correct_, constant_brightness); + this->current_values.as_rgbww(red, green, blue, cold_white, warm_white, constant_brightness); + *red = this->gamma_correct_lut(*red); + *green = this->gamma_correct_lut(*green); + *blue = this->gamma_correct_lut(*blue); + *cold_white = this->gamma_correct_lut(*cold_white); + *warm_white = this->gamma_correct_lut(*warm_white); } void LightState::current_values_as_rgbct(float *red, float *green, float *blue, float *color_temperature, float *white_brightness) { auto traits = this->get_traits(); this->current_values.as_rgbct(traits.get_min_mireds(), traits.get_max_mireds(), red, green, blue, color_temperature, - white_brightness, this->gamma_correct_); + white_brightness); + *red = this->gamma_correct_lut(*red); + *green = this->gamma_correct_lut(*green); + *blue = this->gamma_correct_lut(*blue); + *white_brightness = this->gamma_correct_lut(*white_brightness); } void LightState::current_values_as_cwww(float *cold_white, float *warm_white, bool constant_brightness) { - this->current_values.as_cwww(cold_white, warm_white, this->gamma_correct_, constant_brightness); + this->current_values.as_cwww(cold_white, warm_white, constant_brightness); + *cold_white = this->gamma_correct_lut(*cold_white); + *warm_white = this->gamma_correct_lut(*warm_white); } void LightState::current_values_as_ct(float *color_temperature, float *white_brightness) { auto traits = this->get_traits(); - this->current_values.as_ct(traits.get_min_mireds(), traits.get_max_mireds(), color_temperature, white_brightness, - this->gamma_correct_); + this->current_values.as_ct(traits.get_min_mireds(), traits.get_max_mireds(), color_temperature, white_brightness); + *white_brightness = this->gamma_correct_lut(*white_brightness); } +#ifdef USE_LIGHT_GAMMA_LUT +float LightState::gamma_correct_lut(float value) const { + if (value <= 0.0f) + return 0.0f; + if (value >= 1.0f) + return 1.0f; + if (this->gamma_table_ == nullptr) + return value; + float scaled = value * 255.0f; + auto idx = static_cast(scaled); + if (idx >= 255) + return progmem_read_uint16(&this->gamma_table_[255]) / 65535.0f; + float frac = scaled - idx; + float a = progmem_read_uint16(&this->gamma_table_[idx]); + float b = progmem_read_uint16(&this->gamma_table_[idx + 1]); + return (a + frac * (b - a)) / 65535.0f; +} +float LightState::gamma_uncorrect_lut(float value) const { + if (value <= 0.0f) + return 0.0f; + if (value >= 1.0f) + return 1.0f; + if (this->gamma_table_ == nullptr) + return value; + uint16_t target = static_cast(value * 65535.0f); + uint8_t lo = gamma_table_reverse_search(this->gamma_table_, target); + if (lo >= 255) + return 1.0f; + // Interpolate between lo and lo+1 + uint16_t a = progmem_read_uint16(&this->gamma_table_[lo]); + uint16_t b = progmem_read_uint16(&this->gamma_table_[lo + 1]); + if (b == a) + return lo / 255.0f; + float frac = static_cast(target - a) / static_cast(b - a); + return (lo + frac) / 255.0f; +} +#endif // USE_LIGHT_GAMMA_LUT + bool LightState::is_transformer_active() { return this->is_transformer_active_; } void LightState::start_effect_(uint32_t effect_index) { diff --git a/esphome/components/light/light_state.h b/esphome/components/light/light_state.h index 83b9226d03..528f5bd911 100644 --- a/esphome/components/light/light_state.h +++ b/esphome/components/light/light_state.h @@ -11,6 +11,7 @@ #include "light_traits.h" #include "light_transformer.h" +#include "esphome/core/hal.h" #include "esphome/core/helpers.h" #include #include @@ -166,6 +167,23 @@ class LightState : public EntityBase, public Component { void set_gamma_correct(float gamma_correct); float get_gamma_correct() const { return this->gamma_correct_; } +#ifdef USE_LIGHT_GAMMA_LUT + /// Set pre-computed gamma forward lookup table (256-entry uint16 PROGMEM array) + void set_gamma_table(const uint16_t *forward) { this->gamma_table_ = forward; } + + /// Get the forward gamma lookup table + const uint16_t *get_gamma_table() const { return this->gamma_table_; } + + /// Apply gamma correction using the pre-computed forward LUT + float gamma_correct_lut(float value) const; + /// Reverse gamma correction by binary-searching the forward LUT + float gamma_uncorrect_lut(float value) const; +#else + /// No gamma LUT — passthrough + float gamma_correct_lut(float value) const { return value; } + float gamma_uncorrect_lut(float value) const { return value; } +#endif // USE_LIGHT_GAMMA_LUT + /// Set the restore mode of this light void set_restore_mode(LightRestoreMode restore_mode); @@ -224,9 +242,9 @@ class LightState : public EntityBase, public Component { void current_values_as_brightness(float *brightness); - void current_values_as_rgb(float *red, float *green, float *blue, bool color_interlock = false); + void current_values_as_rgb(float *red, float *green, float *blue); - void current_values_as_rgbw(float *red, float *green, float *blue, float *white, bool color_interlock = false); + void current_values_as_rgbw(float *red, float *green, float *blue, float *white); void current_values_as_rgbww(float *red, float *green, float *blue, float *cold_white, float *warm_white, bool constant_brightness = false); @@ -297,6 +315,10 @@ class LightState : public EntityBase, public Component { uint32_t flash_transition_length_{}; /// Gamma correction factor for the light. float gamma_correct_{}; +#ifdef USE_LIGHT_GAMMA_LUT + const uint16_t *gamma_table_{nullptr}; +#endif // USE_LIGHT_GAMMA_LUT + /// Whether the light value should be written in the next cycle. bool next_write_{true}; // for effects, true if a transformer (transition) is active. diff --git a/esphome/components/lvgl/light/lvgl_light.h b/esphome/components/lvgl/light/lvgl_light.h index 50ae4c5327..569f9a03c0 100644 --- a/esphome/components/lvgl/light/lvgl_light.h +++ b/esphome/components/lvgl/light/lvgl_light.h @@ -16,7 +16,7 @@ class LVLight : public light::LightOutput { } void write_state(light::LightState *state) override { float red, green, blue; - state->current_values_as_rgb(&red, &green, &blue, false); + state->current_values_as_rgb(&red, &green, &blue); auto color = lv_color_make(red * 255, green * 255, blue * 255); if (this->obj_ != nullptr) { this->set_value_(color); diff --git a/esphome/components/rgb/rgb_light_output.h b/esphome/components/rgb/rgb_light_output.h index ef53c8042d..783187667a 100644 --- a/esphome/components/rgb/rgb_light_output.h +++ b/esphome/components/rgb/rgb_light_output.h @@ -20,7 +20,7 @@ class RGBLightOutput : public light::LightOutput { } void write_state(light::LightState *state) override { float red, green, blue; - state->current_values_as_rgb(&red, &green, &blue, false); + state->current_values_as_rgb(&red, &green, &blue); this->red_->set_level(red); this->green_->set_level(green); this->blue_->set_level(blue); diff --git a/esphome/components/rgbw/rgbw_light_output.h b/esphome/components/rgbw/rgbw_light_output.h index a2ab17b75d..140726a43c 100644 --- a/esphome/components/rgbw/rgbw_light_output.h +++ b/esphome/components/rgbw/rgbw_light_output.h @@ -25,7 +25,7 @@ class RGBWLightOutput : public light::LightOutput { } void write_state(light::LightState *state) override { float red, green, blue, white; - state->current_values_as_rgbw(&red, &green, &blue, &white, this->color_interlock_); + state->current_values_as_rgbw(&red, &green, &blue, &white); this->red_->set_level(red); this->green_->set_level(green); this->blue_->set_level(blue); diff --git a/esphome/components/rp2040/core.cpp b/esphome/components/rp2040/core.cpp index 8b86de4be1..6386d53292 100644 --- a/esphome/components/rp2040/core.cpp +++ b/esphome/components/rp2040/core.cpp @@ -34,6 +34,7 @@ void HOT arch_feed_wdt() { watchdog_update(); } uint8_t progmem_read_byte(const uint8_t *addr) { return pgm_read_byte(addr); // NOLINT } +uint16_t progmem_read_uint16(const uint16_t *addr) { return *addr; } uint32_t HOT arch_get_cpu_cycle_count() { return ulMainGetRunTimeCounterValue(); } uint32_t arch_get_cpu_freq_hz() { return RP2040::f_cpu(); } diff --git a/esphome/components/zephyr/core.cpp b/esphome/components/zephyr/core.cpp index f0772a4422..cf3ea70245 100644 --- a/esphome/components/zephyr/core.cpp +++ b/esphome/components/zephyr/core.cpp @@ -60,6 +60,7 @@ void arch_restart() { sys_reboot(SYS_REBOOT_COLD); } uint32_t arch_get_cpu_cycle_count() { return k_cycle_get_32(); } uint32_t arch_get_cpu_freq_hz() { return sys_clock_hw_cycles_per_sec(); } uint8_t progmem_read_byte(const uint8_t *addr) { return *addr; } +uint16_t progmem_read_uint16(const uint16_t *addr) { return *addr; } Mutex::Mutex() { auto *mutex = new k_mutex(); diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 1f2597dd98..8c78afa7d4 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -61,6 +61,7 @@ #define USE_IR_RF #define USE_JSON #define USE_LIGHT +#define USE_LIGHT_GAMMA_LUT #define USE_LOCK #define USE_LOGGER #define USE_LOGGER_LEVEL_LISTENERS diff --git a/esphome/core/hal.h b/esphome/core/hal.h index fa5ca646f2..ef45be629d 100644 --- a/esphome/core/hal.h +++ b/esphome/core/hal.h @@ -42,5 +42,6 @@ void arch_feed_wdt(); uint32_t arch_get_cpu_cycle_count(); uint32_t arch_get_cpu_freq_hz(); uint8_t progmem_read_byte(const uint8_t *addr); +uint16_t progmem_read_uint16(const uint16_t *addr); } // namespace esphome diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 65d590a5e6..c68cb549bb 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -1437,8 +1437,12 @@ bool base64_decode_int32_vector(const std::string &base64, std::vector ///@{ /// Applies gamma correction of \p gamma to \p value. +// Remove before 2026.9.0 +ESPDEPRECATED("Use LightState::gamma_correct_lut() instead. Removed in 2026.9.0.", "2026.3.0") float gamma_correct(float value, float gamma); /// Reverts gamma correction of \p gamma to \p value. +// Remove before 2026.9.0 +ESPDEPRECATED("Use LightState::gamma_uncorrect_lut() instead. Removed in 2026.9.0.", "2026.3.0") float gamma_uncorrect(float value, float gamma); /// Convert \p red, \p green and \p blue (all 0-1) values to \p hue (0-360), \p saturation (0-1) and \p value (0-1). From 54f410901fa09bd025bc370be3753346c2430b9d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:57:39 -1000 Subject: [PATCH 16/20] [web_server] Avoid temporary std::string allocations in request parameter parsing (#14366) Co-authored-by: Claude Opus 4.6 --- .../alarm_control_panel_call.cpp | 9 ++++- .../alarm_control_panel_call.h | 3 +- esphome/components/api/api_connection.cpp | 4 +- esphome/components/climate/climate.cpp | 18 ++++++--- esphome/components/climate/climate.h | 4 ++ esphome/components/text/text_call.cpp | 5 +++ esphome/components/text/text_call.h | 1 + .../components/water_heater/water_heater.cpp | 21 +++++----- .../components/water_heater/water_heater.h | 3 +- esphome/components/web_server/web_server.cpp | 38 ++++++++++++++----- esphome/components/web_server/web_server.h | 8 ++-- 11 files changed, 82 insertions(+), 32 deletions(-) diff --git a/esphome/components/alarm_control_panel/alarm_control_panel_call.cpp b/esphome/components/alarm_control_panel/alarm_control_panel_call.cpp index ba58ee3904..0c43cd555a 100644 --- a/esphome/components/alarm_control_panel/alarm_control_panel_call.cpp +++ b/esphome/components/alarm_control_panel/alarm_control_panel_call.cpp @@ -12,7 +12,14 @@ AlarmControlPanelCall::AlarmControlPanelCall(AlarmControlPanel *parent) : parent AlarmControlPanelCall &AlarmControlPanelCall::set_code(const char *code) { if (code != nullptr) { - this->code_ = std::string(code); + return this->set_code(code, strlen(code)); + } + return *this; +} + +AlarmControlPanelCall &AlarmControlPanelCall::set_code(const char *code, size_t len) { + if (code != nullptr) { + this->code_ = std::string(code, len); } return *this; } diff --git a/esphome/components/alarm_control_panel/alarm_control_panel_call.h b/esphome/components/alarm_control_panel/alarm_control_panel_call.h index 58764ea166..6e39a0a413 100644 --- a/esphome/components/alarm_control_panel/alarm_control_panel_call.h +++ b/esphome/components/alarm_control_panel/alarm_control_panel_call.h @@ -15,7 +15,8 @@ class AlarmControlPanelCall { AlarmControlPanelCall(AlarmControlPanel *parent); AlarmControlPanelCall &set_code(const char *code); - AlarmControlPanelCall &set_code(const std::string &code) { return this->set_code(code.c_str()); } + AlarmControlPanelCall &set_code(const char *code, size_t len); + AlarmControlPanelCall &set_code(const std::string &code) { return this->set_code(code.c_str(), code.size()); } AlarmControlPanelCall &arm_away(); AlarmControlPanelCall &arm_home(); AlarmControlPanelCall &arm_night(); diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 90287ec2dd..738dd1ef05 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -889,7 +889,7 @@ uint16_t APIConnection::try_send_text_info(EntityBase *entity, APIConnection *co } void APIConnection::on_text_command_request(const TextCommandRequest &msg) { ENTITY_COMMAND_MAKE_CALL(text::Text, text, text) - call.set_value(msg.state); + call.set_value(msg.state.c_str(), msg.state.size()); call.perform(); } #endif @@ -1360,7 +1360,7 @@ void APIConnection::on_alarm_control_panel_command_request(const AlarmControlPan call.pending(); break; } - call.set_code(msg.code); + call.set_code(msg.code.c_str(), msg.code.size()); call.perform(); } #endif diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index ba6de4ff61..43d25effa3 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -173,14 +173,17 @@ ClimateCall &ClimateCall::set_mode(ClimateMode mode) { return *this; } -ClimateCall &ClimateCall::set_mode(const std::string &mode) { +ClimateCall &ClimateCall::set_mode(const std::string &mode) { return this->set_mode(mode.c_str(), mode.size()); } + +ClimateCall &ClimateCall::set_mode(const char *mode, size_t len) { + StringRef mode_ref(mode, len); for (const auto &mode_entry : CLIMATE_MODES_BY_STR) { - if (str_equals_case_insensitive(mode, mode_entry.str)) { + if (str_equals_case_insensitive(mode_ref, mode_entry.str)) { this->set_mode(static_cast(mode_entry.value)); return *this; } } - ESP_LOGW(TAG, "'%s' - Unrecognized mode %s", this->parent_->get_name().c_str(), mode.c_str()); + ESP_LOGW(TAG, "'%s' - Unrecognized mode %.*s", this->parent_->get_name().c_str(), (int) len, mode); return *this; } @@ -266,13 +269,18 @@ ClimateCall &ClimateCall::set_swing_mode(ClimateSwingMode swing_mode) { } ClimateCall &ClimateCall::set_swing_mode(const std::string &swing_mode) { + return this->set_swing_mode(swing_mode.c_str(), swing_mode.size()); +} + +ClimateCall &ClimateCall::set_swing_mode(const char *swing_mode, size_t len) { + StringRef mode_ref(swing_mode, len); for (const auto &mode_entry : CLIMATE_SWING_MODES_BY_STR) { - if (str_equals_case_insensitive(swing_mode, mode_entry.str)) { + if (str_equals_case_insensitive(mode_ref, mode_entry.str)) { this->set_swing_mode(static_cast(mode_entry.value)); return *this; } } - ESP_LOGW(TAG, "'%s' - Unrecognized swing mode %s", this->parent_->get_name().c_str(), swing_mode.c_str()); + ESP_LOGW(TAG, "'%s' - Unrecognized swing mode %.*s", this->parent_->get_name().c_str(), (int) len, swing_mode); return *this; } diff --git a/esphome/components/climate/climate.h b/esphome/components/climate/climate.h index 6fac254502..aa9ca91bc2 100644 --- a/esphome/components/climate/climate.h +++ b/esphome/components/climate/climate.h @@ -41,6 +41,8 @@ class ClimateCall { ClimateCall &set_mode(optional mode); /// Set the mode of the climate device based on a string. ClimateCall &set_mode(const std::string &mode); + /// Set the mode of the climate device based on a C string. + ClimateCall &set_mode(const char *mode, size_t len); /// Set the target temperature of the climate device. ClimateCall &set_target_temperature(float target_temperature); /// Set the target temperature of the climate device. @@ -87,6 +89,8 @@ class ClimateCall { ClimateCall &set_swing_mode(optional swing_mode); /// Set the swing mode of the climate device based on a string. ClimateCall &set_swing_mode(const std::string &swing_mode); + /// Set the swing mode of the climate device based on a C string. + ClimateCall &set_swing_mode(const char *swing_mode, size_t len); /// Set the preset of the climate device. ClimateCall &set_preset(ClimatePreset preset); /// Set the preset of the climate device. diff --git a/esphome/components/text/text_call.cpp b/esphome/components/text/text_call.cpp index 8a1630c5ca..b7aed098c7 100644 --- a/esphome/components/text/text_call.cpp +++ b/esphome/components/text/text_call.cpp @@ -11,6 +11,11 @@ TextCall &TextCall::set_value(const std::string &value) { return *this; } +TextCall &TextCall::set_value(const char *value, size_t len) { + this->value_ = std::string(value, len); + return *this; +} + void TextCall::validate_() { const auto *name = this->parent_->get_name().c_str(); diff --git a/esphome/components/text/text_call.h b/esphome/components/text/text_call.h index 532fae34b2..5a2b257ab2 100644 --- a/esphome/components/text/text_call.h +++ b/esphome/components/text/text_call.h @@ -13,6 +13,7 @@ class TextCall { void perform(); TextCall &set_value(const std::string &value); + TextCall &set_value(const char *value, size_t len); protected: Text *const parent_; diff --git a/esphome/components/water_heater/water_heater.cpp b/esphome/components/water_heater/water_heater.cpp index 9d7ae0cbc0..3989230d2d 100644 --- a/esphome/components/water_heater/water_heater.cpp +++ b/esphome/components/water_heater/water_heater.cpp @@ -5,6 +5,7 @@ #include "esphome/core/progmem.h" #include +#include namespace esphome::water_heater { @@ -23,23 +24,25 @@ WaterHeaterCall &WaterHeaterCall::set_mode(WaterHeaterMode mode) { return *this; } -WaterHeaterCall &WaterHeaterCall::set_mode(const char *mode) { - if (ESPHOME_strcasecmp_P(mode, ESPHOME_PSTR("OFF")) == 0) { +WaterHeaterCall &WaterHeaterCall::set_mode(const char *mode) { return this->set_mode(mode, strlen(mode)); } + +WaterHeaterCall &WaterHeaterCall::set_mode(const char *mode, size_t len) { + if (len == 3 && ESPHOME_strncasecmp_P(mode, ESPHOME_PSTR("OFF"), 3) == 0) { this->set_mode(WATER_HEATER_MODE_OFF); - } else if (ESPHOME_strcasecmp_P(mode, ESPHOME_PSTR("ECO")) == 0) { + } else if (len == 3 && ESPHOME_strncasecmp_P(mode, ESPHOME_PSTR("ECO"), 3) == 0) { this->set_mode(WATER_HEATER_MODE_ECO); - } else if (ESPHOME_strcasecmp_P(mode, ESPHOME_PSTR("ELECTRIC")) == 0) { + } else if (len == 8 && ESPHOME_strncasecmp_P(mode, ESPHOME_PSTR("ELECTRIC"), 8) == 0) { this->set_mode(WATER_HEATER_MODE_ELECTRIC); - } else if (ESPHOME_strcasecmp_P(mode, ESPHOME_PSTR("PERFORMANCE")) == 0) { + } else if (len == 11 && ESPHOME_strncasecmp_P(mode, ESPHOME_PSTR("PERFORMANCE"), 11) == 0) { this->set_mode(WATER_HEATER_MODE_PERFORMANCE); - } else if (ESPHOME_strcasecmp_P(mode, ESPHOME_PSTR("HIGH_DEMAND")) == 0) { + } else if (len == 11 && ESPHOME_strncasecmp_P(mode, ESPHOME_PSTR("HIGH_DEMAND"), 11) == 0) { this->set_mode(WATER_HEATER_MODE_HIGH_DEMAND); - } else if (ESPHOME_strcasecmp_P(mode, ESPHOME_PSTR("HEAT_PUMP")) == 0) { + } else if (len == 9 && ESPHOME_strncasecmp_P(mode, ESPHOME_PSTR("HEAT_PUMP"), 9) == 0) { this->set_mode(WATER_HEATER_MODE_HEAT_PUMP); - } else if (ESPHOME_strcasecmp_P(mode, ESPHOME_PSTR("GAS")) == 0) { + } else if (len == 3 && ESPHOME_strncasecmp_P(mode, ESPHOME_PSTR("GAS"), 3) == 0) { this->set_mode(WATER_HEATER_MODE_GAS); } else { - ESP_LOGW(TAG, "'%s' - Unrecognized mode %s", this->parent_->get_name().c_str(), mode); + ESP_LOGW(TAG, "'%s' - Unrecognized mode %.*s", this->parent_->get_name().c_str(), (int) len, mode); } return *this; } diff --git a/esphome/components/water_heater/water_heater.h b/esphome/components/water_heater/water_heater.h index 070ae99575..a1e1ca10a6 100644 --- a/esphome/components/water_heater/water_heater.h +++ b/esphome/components/water_heater/water_heater.h @@ -76,7 +76,8 @@ class WaterHeaterCall { WaterHeaterCall &set_mode(WaterHeaterMode mode); WaterHeaterCall &set_mode(const char *mode); - WaterHeaterCall &set_mode(const std::string &mode) { return this->set_mode(mode.c_str()); } + WaterHeaterCall &set_mode(const char *mode, size_t len); + WaterHeaterCall &set_mode(const std::string &mode) { return this->set_mode(mode.c_str(), mode.size()); } WaterHeaterCall &set_target_temperature(float temperature); WaterHeaterCall &set_target_temperature_low(float temperature); WaterHeaterCall &set_target_temperature_high(float temperature); diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp index 4824e33dcd..47e427c0d1 100644 --- a/esphome/components/web_server/web_server.cpp +++ b/esphome/components/web_server/web_server.cpp @@ -969,7 +969,9 @@ void WebServer::handle_light_request(AsyncWebServerRequest *request, const UrlMa parse_light_param_uint_(request, ESPHOME_F("transition"), call, &decltype(call)::set_transition_length, 1000); if (is_on) { - parse_string_param_(request, ESPHOME_F("effect"), call, &decltype(call)::set_effect); + parse_cstr_param_( + request, ESPHOME_F("effect"), call, + static_cast(&decltype(call)::set_effect)); } DEFER_ACTION(call, call.perform()); @@ -1368,7 +1370,9 @@ void WebServer::handle_text_request(AsyncWebServerRequest *request, const UrlMat } auto call = obj->make_call(); - parse_string_param_(request, ESPHOME_F("value"), call, &decltype(call)::set_value); + parse_cstr_param_( + request, ESPHOME_F("value"), call, + static_cast(&decltype(call)::set_value)); DEFER_ACTION(call, call.perform()); request->send(200); @@ -1426,7 +1430,9 @@ void WebServer::handle_select_request(AsyncWebServerRequest *request, const UrlM } auto call = obj->make_call(); - parse_string_param_(request, ESPHOME_F("option"), call, &decltype(call)::set_option); + parse_cstr_param_( + request, ESPHOME_F("option"), call, + static_cast(&decltype(call)::set_option)); DEFER_ACTION(call, call.perform()); request->send(200); @@ -1487,10 +1493,18 @@ void WebServer::handle_climate_request(AsyncWebServerRequest *request, const Url auto call = obj->make_call(); // Parse string mode parameters - parse_string_param_(request, ESPHOME_F("mode"), call, &decltype(call)::set_mode); - parse_string_param_(request, ESPHOME_F("fan_mode"), call, &decltype(call)::set_fan_mode); - parse_string_param_(request, ESPHOME_F("swing_mode"), call, &decltype(call)::set_swing_mode); - parse_string_param_(request, ESPHOME_F("preset"), call, &decltype(call)::set_preset); + parse_cstr_param_( + request, ESPHOME_F("mode"), call, + static_cast(&decltype(call)::set_mode)); + parse_cstr_param_(request, ESPHOME_F("fan_mode"), call, + static_cast( + &decltype(call)::set_fan_mode)); + parse_cstr_param_(request, ESPHOME_F("swing_mode"), call, + static_cast( + &decltype(call)::set_swing_mode)); + parse_cstr_param_(request, ESPHOME_F("preset"), call, + static_cast( + &decltype(call)::set_preset)); // Parse temperature parameters // static_cast needed to disambiguate overloaded setters (float vs optional) @@ -1804,7 +1818,10 @@ void WebServer::handle_alarm_control_panel_request(AsyncWebServerRequest *reques } auto call = obj->make_call(); - parse_string_param_(request, ESPHOME_F("code"), call, &decltype(call)::set_code); + parse_cstr_param_( + request, ESPHOME_F("code"), call, + static_cast(&decltype(call)::set_code)); // Lookup table for alarm control panel methods static const struct { @@ -1892,7 +1909,10 @@ void WebServer::handle_water_heater_request(AsyncWebServerRequest *request, cons water_heater::WaterHeaterCall &base_call = call; // Parse mode parameter - parse_string_param_(request, ESPHOME_F("mode"), base_call, &water_heater::WaterHeaterCall::set_mode); + parse_cstr_param_( + request, ESPHOME_F("mode"), base_call, + static_cast( + &water_heater::WaterHeaterCall::set_mode)); // Parse temperature parameters parse_num_param_(request, ESPHOME_F("target_temperature"), base_call, diff --git a/esphome/components/web_server/web_server.h b/esphome/components/web_server/web_server.h index 64c492f82b..6152dfbfd3 100644 --- a/esphome/components/web_server/web_server.h +++ b/esphome/components/web_server/web_server.h @@ -533,13 +533,13 @@ class WebServer final : public Controller, public Component, public AsyncWebHand } } - // Generic helper to parse and apply a string parameter + // Generic helper to parse and apply a string parameter using const char* setter (avoids std::string allocation) template - void parse_string_param_(AsyncWebServerRequest *request, ParamNameType param_name, T &call, - Ret (T::*setter)(const std::string &)) { + void parse_cstr_param_(AsyncWebServerRequest *request, ParamNameType param_name, T &call, + Ret (T::*setter)(const char *, size_t)) { if (request->hasArg(param_name)) { const auto &value = request->arg(param_name); - (call.*setter)(std::string(value.c_str(), value.length())); + (call.*setter)(value.c_str(), value.length()); } } From f278250740330b6b4e990ae19b0bc91d846ec924 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:57:59 -1000 Subject: [PATCH 17/20] [core] Deduplicate ControllerRegistry notify dispatch loop (#14394) --- esphome/core/controller_registry.cpp | 21 +++++++++++++-------- esphome/core/controller_registry.h | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/esphome/core/controller_registry.cpp b/esphome/core/controller_registry.cpp index 13b505e8e9..255efa86ba 100644 --- a/esphome/core/controller_registry.cpp +++ b/esphome/core/controller_registry.cpp @@ -10,21 +10,26 @@ StaticVector ControllerRegistry::controll void ControllerRegistry::register_controller(Controller *controller) { controllers.push_back(controller); } +void ControllerRegistry::notify(void *obj, DispatchFunc dispatch) { + for (auto *controller : controllers) { + dispatch(controller, obj); + } +} + // Macro for standard registry notification dispatch - calls on__update() +// Each wrapper passes a small trampoline lambda that calls the correct virtual method. +// NOLINTBEGIN(bugprone-macro-parentheses) #define CONTROLLER_REGISTRY_NOTIFY(entity_type, entity_name) \ - void ControllerRegistry::notify_##entity_name##_update(entity_type *obj) { /* NOLINT(bugprone-macro-parentheses) */ \ - for (auto *controller : controllers) { \ - controller->on_##entity_name##_update(obj); \ - } \ + void ControllerRegistry::notify_##entity_name##_update(entity_type *obj) { \ + notify(obj, [](Controller *c, void *o) { c->on_##entity_name##_update(static_cast(o)); }); \ } // Macro for entities where controller method has no "_update" suffix (Event, Update) #define CONTROLLER_REGISTRY_NOTIFY_NO_UPDATE_SUFFIX(entity_type, entity_name) \ - void ControllerRegistry::notify_##entity_name(entity_type *obj) { /* NOLINT(bugprone-macro-parentheses) */ \ - for (auto *controller : controllers) { \ - controller->on_##entity_name(obj); \ - } \ + void ControllerRegistry::notify_##entity_name(entity_type *obj) { \ + notify(obj, [](Controller *c, void *o) { c->on_##entity_name(static_cast(o)); }); \ } +// NOLINTEND(bugprone-macro-parentheses) #ifdef USE_BINARY_SENSOR CONTROLLER_REGISTRY_NOTIFY(binary_sensor::BinarySensor, binary_sensor) diff --git a/esphome/core/controller_registry.h b/esphome/core/controller_registry.h index d6452d8827..15e3b4ba83 100644 --- a/esphome/core/controller_registry.h +++ b/esphome/core/controller_registry.h @@ -247,6 +247,21 @@ class ControllerRegistry { #endif protected: + /** Type-erased dispatch function pointer. + * + * Each notify method passes a small trampoline that calls the + * correct virtual method on Controller. The shared notify() loop + * iterates controllers once, calling the trampoline for each. + */ + using DispatchFunc = void (*)(Controller *, void *); + + /** Shared dispatch loop - iterates controllers and calls dispatch for each. + * + * Marked noinline to ensure only one copy of the loop exists in flash, + * rather than being duplicated into each notify_*_update wrapper. + */ + static void __attribute__((noinline)) notify(void *obj, DispatchFunc dispatch); + static StaticVector controllers; }; From 39572d9628d80ecf5f74f99f54c536ca26b357c8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:58:15 -1000 Subject: [PATCH 18/20] [light] Resolve effect names to indices at codegen time (#14265) --- esphome/components/light/automation.h | 2 +- esphome/components/light/automation.py | 50 ++++++++++++++++++++++++-- esphome/components/light/light_state.h | 15 ++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/esphome/components/light/automation.h b/esphome/components/light/automation.h index c90d71c5df..2854bc62d9 100644 --- a/esphome/components/light/automation.h +++ b/esphome/components/light/automation.h @@ -41,7 +41,7 @@ template class LightControlAction : public Action { TEMPLATABLE_VALUE(float, color_temperature) TEMPLATABLE_VALUE(float, cold_white) TEMPLATABLE_VALUE(float, warm_white) - TEMPLATABLE_VALUE(std::string, effect) + TEMPLATABLE_VALUE(uint32_t, effect) void play(const Ts &...x) override { auto call = this->parent_->make_call(); diff --git a/esphome/components/light/automation.py b/esphome/components/light/automation.py index 89b2fc0fb2..08fd26a937 100644 --- a/esphome/components/light/automation.py +++ b/esphome/components/light/automation.py @@ -10,12 +10,14 @@ from esphome.const import ( CONF_COLOR_MODE, CONF_COLOR_TEMPERATURE, CONF_EFFECT, + CONF_EFFECTS, CONF_FLASH_LENGTH, CONF_GREEN, CONF_ID, CONF_LIMIT_MODE, CONF_MAX_BRIGHTNESS, CONF_MIN_BRIGHTNESS, + CONF_NAME, CONF_RANGE_FROM, CONF_RANGE_TO, CONF_RED, @@ -24,6 +26,9 @@ from esphome.const import ( CONF_WARM_WHITE, CONF_WHITE, ) +from esphome.core import CORE, Lambda +from esphome.cpp_generator import LambdaExpression +from esphome.types import ConfigType from .types import ( COLOR_MODES, @@ -111,6 +116,26 @@ LIGHT_TURN_ON_ACTION_SCHEMA = automation.maybe_simple_id( ) +def _resolve_effect_index(config: ConfigType) -> int: + """Resolve a static effect name to its 1-based index at codegen time. + + Effect index 0 means "None" (no effect). Effects are 1-indexed matching + the C++ convention in LightState. + """ + original_name = config[CONF_EFFECT] + effect_name = original_name.lower() + if effect_name == "none": + return 0 + light_id = config[CONF_ID] + light_path = CORE.config.get_path_for_id(light_id)[:-1] + light_config = CORE.config.get_config_for_path(light_path) + for i, effect_conf in enumerate(light_config.get(CONF_EFFECTS, [])): + key = next(iter(effect_conf)) + if effect_conf[key][CONF_NAME].lower() == effect_name: + return i + 1 + raise ValueError(f"Effect '{original_name}' not found in light '{light_id}'") + + @automation.register_action( "light.turn_off", LightControlAction, LIGHT_TURN_OFF_ACTION_SCHEMA, synchronous=True ) @@ -165,8 +190,29 @@ async def light_control_to_code(config, action_id, template_arg, args): template_ = await cg.templatable(config[CONF_WARM_WHITE], args, float) cg.add(var.set_warm_white(template_)) if CONF_EFFECT in config: - template_ = await cg.templatable(config[CONF_EFFECT], args, cg.std_string) - cg.add(var.set_effect(template_)) + if isinstance(config[CONF_EFFECT], Lambda): + # Lambda returns a string — wrap in a C++ lambda that resolves + # the effect name to its uint32_t index at runtime + inner_lambda = await cg.process_lambda( + config[CONF_EFFECT], args, return_type=cg.std_string + ) + fwd_args = ", ".join(n for _, n in args) + # capture="" is correct: paren is a global variable name + # string-interpolated into the body at codegen time, not a + # C++ runtime capture. + wrapper = LambdaExpression( + f"auto __effect_s = ({inner_lambda})({fwd_args});\n" + f"return {paren}->get_effect_index(" + f"__effect_s.c_str(), __effect_s.size());", + args, + capture="", + return_type=cg.uint32, + ) + cg.add(var.set_effect(wrapper)) + else: + # Static string — resolve effect name to index at codegen time + effect_index = _resolve_effect_index(config) + cg.add(var.set_effect(effect_index)) return var diff --git a/esphome/components/light/light_state.h b/esphome/components/light/light_state.h index 528f5bd911..b8d72cc832 100644 --- a/esphome/components/light/light_state.h +++ b/esphome/components/light/light_state.h @@ -13,6 +13,7 @@ #include "esphome/core/hal.h" #include "esphome/core/helpers.h" +#include "esphome/core/progmem.h" #include #include @@ -218,6 +219,20 @@ class LightState : public EntityBase, public Component { return 0; // Effect not found } + /// Get effect index by name (const char* overload, avoids std::string construction). + uint32_t get_effect_index(const char *name, size_t len) const { + if (len == 4 && ESPHOME_strncasecmp_P(name, ESPHOME_PSTR("none"), 4) == 0) { + return 0; + } + StringRef ref(name, len); + for (size_t i = 0; i < this->effects_.size(); i++) { + if (str_equals_case_insensitive(ref, this->effects_[i]->get_name())) { + return i + 1; + } + } + return 0; + } + /// Get effect by index. Returns nullptr if index is invalid. LightEffect *get_effect_by_index(uint32_t index) const { if (index == 0 || index > this->effects_.size()) { From 585e195044af6a7415bf56108d4073c60a4a3e3b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:58:31 -1000 Subject: [PATCH 19/20] [socket] Devirtualize socket abstraction layer (#14398) --- esphome/components/api/api_server.h | 2 +- .../captive_portal/dns_server_esp32_idf.h | 2 +- esphome/components/esphome/ota/ota_esphome.h | 2 +- .../components/socket/bsd_sockets_impl.cpp | 172 +-- esphome/components/socket/bsd_sockets_impl.h | 114 ++ esphome/components/socket/headers.h | 17 + .../components/socket/lwip_raw_tcp_impl.cpp | 1150 ++++++++--------- esphome/components/socket/lwip_raw_tcp_impl.h | 200 +++ .../components/socket/lwip_sockets_impl.cpp | 139 +- esphome/components/socket/lwip_sockets_impl.h | 80 ++ esphome/components/socket/socket.cpp | 32 +- esphome/components/socket/socket.h | 110 +- esphome/core/application.h | 9 +- 13 files changed, 1153 insertions(+), 876 deletions(-) create mode 100644 esphome/components/socket/bsd_sockets_impl.h create mode 100644 esphome/components/socket/lwip_raw_tcp_impl.h create mode 100644 esphome/components/socket/lwip_sockets_impl.h diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 3abf68358c..6eff2005f8 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -257,7 +257,7 @@ class APIServer : public Component, } void socket_failed_(const LogString *msg); // Pointers and pointer-like types first (4 bytes each) - socket::Socket *socket_{nullptr}; + socket::ListenSocket *socket_{nullptr}; #ifdef USE_API_CLIENT_CONNECTED_TRIGGER Trigger client_connected_trigger_; #endif diff --git a/esphome/components/captive_portal/dns_server_esp32_idf.h b/esphome/components/captive_portal/dns_server_esp32_idf.h index f8e4cfec84..b30856c204 100644 --- a/esphome/components/captive_portal/dns_server_esp32_idf.h +++ b/esphome/components/captive_portal/dns_server_esp32_idf.h @@ -22,7 +22,7 @@ class DNSServer { } static constexpr size_t DNS_BUFFER_SIZE = 192; - socket::Socket *socket_{nullptr}; + socket::ListenSocket *socket_{nullptr}; network::IPAddress server_ip_; uint8_t buffer_[DNS_BUFFER_SIZE]; }; diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index 53715cfe6a..08edacad92 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -84,7 +84,7 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD - socket::Socket *server_{nullptr}; + socket::ListenSocket *server_{nullptr}; std::unique_ptr client_; std::unique_ptr backend_; diff --git a/esphome/components/socket/bsd_sockets_impl.cpp b/esphome/components/socket/bsd_sockets_impl.cpp index c96713f376..92ecfc692b 100644 --- a/esphome/components/socket/bsd_sockets_impl.cpp +++ b/esphome/components/socket/bsd_sockets_impl.cpp @@ -1,135 +1,81 @@ -#include "socket.h" #include "esphome/core/defines.h" #include "esphome/core/helpers.h" +#include "socket.h" #ifdef USE_SOCKET_IMPL_BSD_SOCKETS #include #include "esphome/core/application.h" -#ifdef USE_ESP32 -#include -#include -#endif - namespace esphome::socket { -class BSDSocketImpl final : public Socket { - public: - BSDSocketImpl(int fd, bool monitor_loop = false) { - this->fd_ = fd; - // Register new socket with the application for select() if monitoring requested - if (monitor_loop && this->fd_ >= 0) { - // Only set loop_monitored_ to true if registration succeeds - this->loop_monitored_ = App.register_socket_fd(this->fd_); - } - } - ~BSDSocketImpl() override { - if (!this->closed_) { - this->close(); // NOLINT(clang-analyzer-optin.cplusplus.VirtualCall) - } - } - int connect(const struct sockaddr *addr, socklen_t addrlen) override { return ::connect(this->fd_, addr, addrlen); } - std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { - int fd = ::accept(this->fd_, addr, addrlen); - if (fd == -1) - return {}; - return make_unique(fd, false); - } - std::unique_ptr accept_loop_monitored(struct sockaddr *addr, socklen_t *addrlen) override { - int fd = ::accept(this->fd_, addr, addrlen); - if (fd == -1) - return {}; - return make_unique(fd, true); +BSDSocketImpl::BSDSocketImpl(int fd, bool monitor_loop) { + this->fd_ = fd; + // Register new socket with the application for select() if monitoring requested + if (monitor_loop && this->fd_ >= 0) { + // Only set loop_monitored_ to true if registration succeeds + this->loop_monitored_ = App.register_socket_fd(this->fd_); } +} - int bind(const struct sockaddr *addr, socklen_t addrlen) override { return ::bind(this->fd_, addr, addrlen); } - int close() override { - if (!this->closed_) { - // Unregister from select() before closing if monitored - if (this->loop_monitored_) { - App.unregister_socket_fd(this->fd_); - } - int ret = ::close(this->fd_); - this->closed_ = true; - return ret; +BSDSocketImpl::~BSDSocketImpl() { + if (!this->closed_) { + this->close(); + } +} + +int BSDSocketImpl::close() { + if (!this->closed_) { + // Unregister from select() before closing if monitored + if (this->loop_monitored_) { + App.unregister_socket_fd(this->fd_); } + int ret = ::close(this->fd_); + this->closed_ = true; + return ret; + } + return 0; +} + +int BSDSocketImpl::setblocking(bool blocking) { + int fl = ::fcntl(this->fd_, F_GETFL, 0); + if (blocking) { + fl &= ~O_NONBLOCK; + } else { + fl |= O_NONBLOCK; + } + ::fcntl(this->fd_, F_SETFL, fl); + return 0; +} + +bool BSDSocketImpl::ready() const { return socket_ready_fd(this->fd_, this->loop_monitored_); } + +size_t BSDSocketImpl::getpeername_to(std::span buf) { + struct sockaddr_storage storage; + socklen_t len = sizeof(storage); + if (this->getpeername(reinterpret_cast(&storage), &len) != 0) { + buf[0] = '\0'; return 0; } - int shutdown(int how) override { return ::shutdown(this->fd_, how); } + return format_sockaddr_to(reinterpret_cast(&storage), len, buf); +} - int getpeername(struct sockaddr *addr, socklen_t *addrlen) override { - return ::getpeername(this->fd_, addr, addrlen); - } - int getsockname(struct sockaddr *addr, socklen_t *addrlen) override { - return ::getsockname(this->fd_, addr, addrlen); - } - int getsockopt(int level, int optname, void *optval, socklen_t *optlen) override { - return ::getsockopt(this->fd_, level, optname, optval, optlen); - } - int setsockopt(int level, int optname, const void *optval, socklen_t optlen) override { - return ::setsockopt(this->fd_, level, optname, optval, optlen); - } - int listen(int backlog) override { return ::listen(this->fd_, backlog); } - ssize_t read(void *buf, size_t len) override { -#ifdef USE_ESP32 - return ::lwip_read(this->fd_, buf, len); -#else - return ::read(this->fd_, buf, len); -#endif - } - ssize_t recvfrom(void *buf, size_t len, sockaddr *addr, socklen_t *addr_len) override { -#if defined(USE_ESP32) || defined(USE_HOST) - return ::recvfrom(this->fd_, buf, len, 0, addr, addr_len); -#else - return ::lwip_recvfrom(this->fd_, buf, len, 0, addr, addr_len); -#endif - } - ssize_t readv(const struct iovec *iov, int iovcnt) override { -#if defined(USE_ESP32) - return ::lwip_readv(this->fd_, iov, iovcnt); -#else - return ::readv(this->fd_, iov, iovcnt); -#endif - } - ssize_t write(const void *buf, size_t len) override { -#ifdef USE_ESP32 - return ::lwip_write(this->fd_, buf, len); -#else - return ::write(this->fd_, buf, len); -#endif - } - ssize_t send(void *buf, size_t len, int flags) { return ::send(this->fd_, buf, len, flags); } - ssize_t writev(const struct iovec *iov, int iovcnt) override { -#if defined(USE_ESP32) - return ::lwip_writev(this->fd_, iov, iovcnt); -#else - return ::writev(this->fd_, iov, iovcnt); -#endif - } - - ssize_t sendto(const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) override { - return ::sendto(this->fd_, buf, len, flags, to, tolen); // NOLINT(readability-suspicious-call-argument) - } - - int setblocking(bool blocking) override { - int fl = ::fcntl(this->fd_, F_GETFL, 0); - if (blocking) { - fl &= ~O_NONBLOCK; - } else { - fl |= O_NONBLOCK; - } - ::fcntl(this->fd_, F_SETFL, fl); +size_t BSDSocketImpl::getsockname_to(std::span buf) { + struct sockaddr_storage storage; + socklen_t len = sizeof(storage); + if (this->getsockname(reinterpret_cast(&storage), &len) != 0) { + buf[0] = '\0'; return 0; } -}; + return format_sockaddr_to(reinterpret_cast(&storage), len, buf); +} // Helper to create a socket with optional monitoring -static std::unique_ptr create_socket(int domain, int type, int protocol, bool loop_monitored = false) { +static std::unique_ptr create_socket(int domain, int type, int protocol, bool loop_monitored = false) { int ret = ::socket(domain, type, protocol); if (ret == -1) return nullptr; - return std::unique_ptr{new BSDSocketImpl(ret, loop_monitored)}; + return make_unique(ret, loop_monitored); } std::unique_ptr socket(int domain, int type, int protocol) { @@ -140,6 +86,14 @@ std::unique_ptr socket_loop_monitored(int domain, int type, int protocol return create_socket(domain, type, protocol, true); } +std::unique_ptr socket_listen(int domain, int type, int protocol) { + return create_socket(domain, type, protocol, false); +} + +std::unique_ptr socket_listen_loop_monitored(int domain, int type, int protocol) { + return create_socket(domain, type, protocol, true); +} + } // namespace esphome::socket #endif // USE_SOCKET_IMPL_BSD_SOCKETS diff --git a/esphome/components/socket/bsd_sockets_impl.h b/esphome/components/socket/bsd_sockets_impl.h new file mode 100644 index 0000000000..edee39f315 --- /dev/null +++ b/esphome/components/socket/bsd_sockets_impl.h @@ -0,0 +1,114 @@ +#pragma once +#include "esphome/core/defines.h" + +#ifdef USE_SOCKET_IMPL_BSD_SOCKETS + +#include +#include + +#include "esphome/core/helpers.h" +#include "headers.h" + +#ifdef USE_ESP32 +#include +#endif + +namespace esphome::socket { + +class BSDSocketImpl { + public: + BSDSocketImpl(int fd, bool monitor_loop = false); + ~BSDSocketImpl(); + BSDSocketImpl(const BSDSocketImpl &) = delete; + BSDSocketImpl &operator=(const BSDSocketImpl &) = delete; + + int connect(const struct sockaddr *addr, socklen_t addrlen) { return ::connect(this->fd_, addr, addrlen); } + std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) { + int fd = ::accept(this->fd_, addr, addrlen); + if (fd == -1) + return {}; + return make_unique(fd, false); + } + std::unique_ptr accept_loop_monitored(struct sockaddr *addr, socklen_t *addrlen) { + int fd = ::accept(this->fd_, addr, addrlen); + if (fd == -1) + return {}; + return make_unique(fd, true); + } + + int bind(const struct sockaddr *addr, socklen_t addrlen) { return ::bind(this->fd_, addr, addrlen); } + int close(); + int shutdown(int how) { return ::shutdown(this->fd_, how); } + + int getpeername(struct sockaddr *addr, socklen_t *addrlen) { return ::getpeername(this->fd_, addr, addrlen); } + int getsockname(struct sockaddr *addr, socklen_t *addrlen) { return ::getsockname(this->fd_, addr, addrlen); } + + /// Format peer address into a fixed-size buffer (no heap allocation) + size_t getpeername_to(std::span buf); + /// Format local address into a fixed-size buffer (no heap allocation) + size_t getsockname_to(std::span buf); + + int getsockopt(int level, int optname, void *optval, socklen_t *optlen) { + return ::getsockopt(this->fd_, level, optname, optval, optlen); + } + int setsockopt(int level, int optname, const void *optval, socklen_t optlen) { + return ::setsockopt(this->fd_, level, optname, optval, optlen); + } + int listen(int backlog) { return ::listen(this->fd_, backlog); } + ssize_t read(void *buf, size_t len) { +#ifdef USE_ESP32 + return ::lwip_read(this->fd_, buf, len); +#else + return ::read(this->fd_, buf, len); +#endif + } + ssize_t recvfrom(void *buf, size_t len, sockaddr *addr, socklen_t *addr_len) { +#if defined(USE_ESP32) || defined(USE_HOST) + return ::recvfrom(this->fd_, buf, len, 0, addr, addr_len); +#else + return ::lwip_recvfrom(this->fd_, buf, len, 0, addr, addr_len); +#endif + } + ssize_t readv(const struct iovec *iov, int iovcnt) { +#if defined(USE_ESP32) + return ::lwip_readv(this->fd_, iov, iovcnt); +#else + return ::readv(this->fd_, iov, iovcnt); +#endif + } + ssize_t write(const void *buf, size_t len) { +#ifdef USE_ESP32 + return ::lwip_write(this->fd_, buf, len); +#else + return ::write(this->fd_, buf, len); +#endif + } + ssize_t send(void *buf, size_t len, int flags) { return ::send(this->fd_, buf, len, flags); } + ssize_t writev(const struct iovec *iov, int iovcnt) { +#if defined(USE_ESP32) + return ::lwip_writev(this->fd_, iov, iovcnt); +#else + return ::writev(this->fd_, iov, iovcnt); +#endif + } + + ssize_t sendto(const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) { + return ::sendto(this->fd_, buf, len, flags, to, tolen); // NOLINT(readability-suspicious-call-argument) + } + + int setblocking(bool blocking); + int loop() { return 0; } + + bool ready() const; + + int get_fd() const { return this->fd_; } + + protected: + int fd_{-1}; + bool closed_{false}; + bool loop_monitored_{false}; +}; + +} // namespace esphome::socket + +#endif // USE_SOCKET_IMPL_BSD_SOCKETS diff --git a/esphome/components/socket/headers.h b/esphome/components/socket/headers.h index 032892072d..16e4d23d3b 100644 --- a/esphome/components/socket/headers.h +++ b/esphome/components/socket/headers.h @@ -183,3 +183,20 @@ using socklen_t = uint32_t; #endif #endif // USE_SOCKET_IMPL_BSD_SOCKETS + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +namespace esphome::socket { + +// Maximum length for formatted socket address string (IP address without port) +// IPv4: "255.255.255.255" = 15 chars + null = 16 +// IPv6: full address = 45 chars + null = 46 +#if USE_NETWORK_IPV6 +static constexpr size_t SOCKADDR_STR_LEN = 46; // INET6_ADDRSTRLEN +#else +static constexpr size_t SOCKADDR_STR_LEN = 16; // INET_ADDRSTRLEN +#endif + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index 6e95f5bc7a..6556979fe0 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -3,13 +3,8 @@ #ifdef USE_SOCKET_IMPL_LWIP_TCP -#include "lwip/ip.h" -#include "lwip/netif.h" -#include "lwip/opt.h" -#include "lwip/tcp.h" #include #include -#include #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -56,628 +51,596 @@ static const char *const TAG = "socket.lwip"; #define LWIP_LOG(msg, ...) #endif -class LWIPRawImpl : public Socket { - public: - LWIPRawImpl(sa_family_t family, struct tcp_pcb *pcb) : pcb_(pcb), family_(family) {} - ~LWIPRawImpl() override { - if (pcb_ != nullptr) { - LWIP_LOG("tcp_abort(%p)", pcb_); - tcp_abort(pcb_); - pcb_ = nullptr; - } - } +// ---- LWIPRawCommon methods ---- - void init() { - LWIP_LOG("init(%p)", pcb_); - tcp_arg(pcb_, this); - tcp_recv(pcb_, LWIPRawImpl::s_recv_fn); - tcp_err(pcb_, LWIPRawImpl::s_err_fn); +LWIPRawCommon::~LWIPRawCommon() { + if (this->pcb_ != nullptr) { + LWIP_LOG("tcp_abort(%p)", this->pcb_); + tcp_abort(this->pcb_); + this->pcb_ = nullptr; } +} - std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { - // Non-listening sockets return error +int LWIPRawCommon::bind(const struct sockaddr *name, socklen_t addrlen) { + if (this->pcb_ == nullptr) { + errno = EBADF; + return -1; + } + if (name == nullptr) { errno = EINVAL; - return nullptr; + return 0; } - int bind(const struct sockaddr *name, socklen_t addrlen) final { - if (pcb_ == nullptr) { - errno = EBADF; - return -1; - } - if (name == nullptr) { - errno = EINVAL; - return 0; - } - ip_addr_t ip; - in_port_t port; + ip_addr_t ip; + in_port_t port; #if LWIP_IPV6 - if (family_ == AF_INET) { - if (addrlen < sizeof(sockaddr_in)) { - errno = EINVAL; - return -1; - } - auto *addr4 = reinterpret_cast(name); - port = ntohs(addr4->sin_port); - ip.type = IPADDR_TYPE_V4; - ip.u_addr.ip4.addr = addr4->sin_addr.s_addr; - LWIP_LOG("tcp_bind(%p ip=%s port=%u)", pcb_, ip4addr_ntoa(&ip.u_addr.ip4), port); - } else if (family_ == AF_INET6) { - if (addrlen < sizeof(sockaddr_in6)) { - errno = EINVAL; - return -1; - } - auto *addr6 = reinterpret_cast(name); - port = ntohs(addr6->sin6_port); - ip.type = IPADDR_TYPE_ANY; - memcpy(&ip.u_addr.ip6.addr, &addr6->sin6_addr.un.u8_addr, 16); - LWIP_LOG("tcp_bind(%p ip=%s port=%u)", pcb_, ip6addr_ntoa(&ip.u_addr.ip6), port); - } else { - errno = EINVAL; - return -1; - } -#else - if (family_ != AF_INET) { + if (this->family_ == AF_INET) { + if (addrlen < sizeof(sockaddr_in)) { errno = EINVAL; return -1; } auto *addr4 = reinterpret_cast(name); port = ntohs(addr4->sin_port); - ip.addr = addr4->sin_addr.s_addr; - LWIP_LOG("tcp_bind(%p ip=%u port=%u)", pcb_, ip.addr, port); + ip.type = IPADDR_TYPE_V4; + ip.u_addr.ip4.addr = addr4->sin_addr.s_addr; + LWIP_LOG("tcp_bind(%p ip=%s port=%u)", this->pcb_, ip4addr_ntoa(&ip.u_addr.ip4), port); + } else if (this->family_ == AF_INET6) { + if (addrlen < sizeof(sockaddr_in6)) { + errno = EINVAL; + return -1; + } + auto *addr6 = reinterpret_cast(name); + port = ntohs(addr6->sin6_port); + ip.type = IPADDR_TYPE_ANY; + memcpy(&ip.u_addr.ip6.addr, &addr6->sin6_addr.un.u8_addr, 16); + LWIP_LOG("tcp_bind(%p ip=%s port=%u)", this->pcb_, ip6addr_ntoa(&ip.u_addr.ip6), port); + } else { + errno = EINVAL; + return -1; + } +#else + if (this->family_ != AF_INET) { + errno = EINVAL; + return -1; + } + auto *addr4 = reinterpret_cast(name); + port = ntohs(addr4->sin_port); + ip.addr = addr4->sin_addr.s_addr; + LWIP_LOG("tcp_bind(%p ip=%u port=%u)", this->pcb_, ip.addr, port); #endif - err_t err = tcp_bind(pcb_, &ip, port); - if (err == ERR_USE) { - LWIP_LOG(" -> err ERR_USE"); - errno = EADDRINUSE; - return -1; - } - if (err == ERR_VAL) { - LWIP_LOG(" -> err ERR_VAL"); + err_t err = tcp_bind(this->pcb_, &ip, port); + if (err == ERR_USE) { + LWIP_LOG(" -> err ERR_USE"); + errno = EADDRINUSE; + return -1; + } + if (err == ERR_VAL) { + LWIP_LOG(" -> err ERR_VAL"); + errno = EINVAL; + return -1; + } + if (err != ERR_OK) { + LWIP_LOG(" -> err %d", err); + errno = EIO; + return -1; + } + return 0; +} + +int LWIPRawCommon::close() { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + LWIP_LOG("tcp_close(%p)", this->pcb_); + err_t err = tcp_close(this->pcb_); + if (err != ERR_OK) { + LWIP_LOG(" -> err %d", err); + tcp_abort(this->pcb_); + this->pcb_ = nullptr; + errno = err == ERR_MEM ? ENOMEM : EIO; + return -1; + } + this->pcb_ = nullptr; + return 0; +} + +int LWIPRawCommon::shutdown(int how) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + bool shut_rx = false, shut_tx = false; + if (how == SHUT_RD) { + shut_rx = true; + } else if (how == SHUT_WR) { + shut_tx = true; + } else if (how == SHUT_RDWR) { + shut_rx = shut_tx = true; + } else { + errno = EINVAL; + return -1; + } + LWIP_LOG("tcp_shutdown(%p shut_rx=%d shut_tx=%d)", this->pcb_, shut_rx ? 1 : 0, shut_tx ? 1 : 0); + err_t err = tcp_shutdown(this->pcb_, shut_rx, shut_tx); + if (err != ERR_OK) { + LWIP_LOG(" -> err %d", err); + errno = err == ERR_MEM ? ENOMEM : EIO; + return -1; + } + return 0; +} + +int LWIPRawCommon::getpeername(struct sockaddr *name, socklen_t *addrlen) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + if (name == nullptr || addrlen == nullptr) { + errno = EINVAL; + return -1; + } + return this->ip2sockaddr_(&this->pcb_->remote_ip, this->pcb_->remote_port, name, addrlen); +} + +int LWIPRawCommon::getsockname(struct sockaddr *name, socklen_t *addrlen) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + if (name == nullptr || addrlen == nullptr) { + errno = EINVAL; + return -1; + } + return this->ip2sockaddr_(&this->pcb_->local_ip, this->pcb_->local_port, name, addrlen); +} + +size_t LWIPRawCommon::getpeername_to(std::span buf) { + struct sockaddr_storage storage; + socklen_t len = sizeof(storage); + if (this->getpeername(reinterpret_cast(&storage), &len) != 0) { + buf[0] = '\0'; + return 0; + } + return format_sockaddr_to(reinterpret_cast(&storage), len, buf); +} + +size_t LWIPRawCommon::getsockname_to(std::span buf) { + struct sockaddr_storage storage; + socklen_t len = sizeof(storage); + if (this->getsockname(reinterpret_cast(&storage), &len) != 0) { + buf[0] = '\0'; + return 0; + } + return format_sockaddr_to(reinterpret_cast(&storage), len, buf); +} + +int LWIPRawCommon::getsockopt(int level, int optname, void *optval, socklen_t *optlen) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + if (optlen == nullptr || optval == nullptr) { + errno = EINVAL; + return -1; + } + if (level == SOL_SOCKET && optname == SO_REUSEADDR) { + if (*optlen < 4) { errno = EINVAL; return -1; } - if (err != ERR_OK) { - LWIP_LOG(" -> err %d", err); - errno = EIO; - return -1; - } + // lwip doesn't seem to have this feature. Don't send an error + // to prevent warnings + *reinterpret_cast(optval) = 1; + *optlen = 4; return 0; } - int close() final { - if (pcb_ == nullptr) { - errno = ECONNRESET; + if (level == IPPROTO_TCP && optname == TCP_NODELAY) { + if (*optlen < 4) { + errno = EINVAL; return -1; } - LWIP_LOG("tcp_close(%p)", pcb_); - err_t err = tcp_close(pcb_); - if (err != ERR_OK) { - LWIP_LOG(" -> err %d", err); - tcp_abort(pcb_); - pcb_ = nullptr; - errno = err == ERR_MEM ? ENOMEM : EIO; - return -1; - } - pcb_ = nullptr; + *reinterpret_cast(optval) = this->nodelay_; + *optlen = 4; return 0; } - int shutdown(int how) final { - if (pcb_ == nullptr) { - errno = ECONNRESET; + + errno = EINVAL; + return -1; +} + +int LWIPRawCommon::setsockopt(int level, int optname, const void *optval, socklen_t optlen) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + if (level == SOL_SOCKET && optname == SO_REUSEADDR) { + if (optlen != 4) { + errno = EINVAL; return -1; } - bool shut_rx = false, shut_tx = false; - if (how == SHUT_RD) { - shut_rx = true; - } else if (how == SHUT_WR) { - shut_tx = true; - } else if (how == SHUT_RDWR) { - shut_rx = shut_tx = true; + // lwip doesn't seem to have this feature. Don't send an error + // to prevent warnings + return 0; + } + if (level == IPPROTO_TCP && optname == TCP_NODELAY) { + if (optlen != 4) { + errno = EINVAL; + return -1; + } + int val = *reinterpret_cast(optval); + this->nodelay_ = val; + return 0; + } + + errno = EINVAL; + return -1; +} + +int LWIPRawCommon::ip2sockaddr_(ip_addr_t *ip, uint16_t port, struct sockaddr *name, socklen_t *addrlen) { + if (this->family_ == AF_INET) { + if (*addrlen < sizeof(struct sockaddr_in)) { + errno = EINVAL; + return -1; + } + + struct sockaddr_in *addr = reinterpret_cast(name); + addr->sin_family = AF_INET; + *addrlen = addr->sin_len = sizeof(struct sockaddr_in); + addr->sin_port = port; + inet_addr_from_ip4addr(&addr->sin_addr, ip_2_ip4(ip)); + return 0; + } +#if LWIP_IPV6 + else if (this->family_ == AF_INET6) { + if (*addrlen < sizeof(struct sockaddr_in6)) { + errno = EINVAL; + return -1; + } + + struct sockaddr_in6 *addr = reinterpret_cast(name); + addr->sin6_family = AF_INET6; + *addrlen = addr->sin6_len = sizeof(struct sockaddr_in6); + addr->sin6_port = port; + + // AF_INET6 sockets are bound to IPv4 as well, so we may encounter IPv4 addresses that must be converted to IPv6. + if (IP_IS_V4(ip)) { + ip_addr_t mapped; + ip4_2_ipv4_mapped_ipv6(ip_2_ip6(&mapped), ip_2_ip4(ip)); + inet6_addr_from_ip6addr(&addr->sin6_addr, ip_2_ip6(&mapped)); } else { - errno = EINVAL; - return -1; - } - LWIP_LOG("tcp_shutdown(%p shut_rx=%d shut_tx=%d)", pcb_, shut_rx ? 1 : 0, shut_tx ? 1 : 0); - err_t err = tcp_shutdown(pcb_, shut_rx, shut_tx); - if (err != ERR_OK) { - LWIP_LOG(" -> err %d", err); - errno = err == ERR_MEM ? ENOMEM : EIO; - return -1; + inet6_addr_from_ip6addr(&addr->sin6_addr, ip_2_ip6(ip)); } return 0; } +#endif + return -1; +} - int getpeername(struct sockaddr *name, socklen_t *addrlen) final { - if (pcb_ == nullptr) { - errno = ECONNRESET; - return -1; - } - if (name == nullptr || addrlen == nullptr) { - errno = EINVAL; - return -1; - } - return this->ip2sockaddr_(&pcb_->remote_ip, pcb_->remote_port, name, addrlen); +// ---- LWIPRawImpl methods ---- + +LWIPRawImpl::~LWIPRawImpl() { + // Base class destructor handles pcb_ cleanup via tcp_abort +} + +void LWIPRawImpl::init() { + LWIP_LOG("init(%p)", this->pcb_); + tcp_arg(this->pcb_, this); + tcp_recv(this->pcb_, LWIPRawImpl::s_recv_fn); + tcp_err(this->pcb_, LWIPRawImpl::s_err_fn); +} + +void LWIPRawImpl::s_err_fn(void *arg, err_t err) { + // "If a connection is aborted because of an error, the application is alerted of this event by + // the err callback." + // pcb is already freed when this callback is called + // ERR_RST: connection was reset by remote host + // ERR_ABRT: aborted through tcp_abort or TCP timer + auto *arg_this = reinterpret_cast(arg); + ESP_LOGVV(TAG, "socket %p: err(err=%d)", arg_this, err); + arg_this->pcb_ = nullptr; +} + +err_t LWIPRawImpl::s_recv_fn(void *arg, struct tcp_pcb *pcb, struct pbuf *pb, err_t err) { + auto *arg_this = reinterpret_cast(arg); + return arg_this->recv_fn(pb, err); +} + +err_t LWIPRawImpl::recv_fn(struct pbuf *pb, err_t err) { + LWIP_LOG("recv(pb=%p err=%d)", pb, err); + if (err != 0) { + // "An error code if there has been an error receiving Only return ERR_ABRT if you have + // called tcp_abort from within the callback function!" + this->rx_closed_ = true; + return ERR_OK; } - int getsockname(struct sockaddr *name, socklen_t *addrlen) final { - if (pcb_ == nullptr) { - errno = ECONNRESET; - return -1; - } - if (name == nullptr || addrlen == nullptr) { - errno = EINVAL; - return -1; - } - return this->ip2sockaddr_(&pcb_->local_ip, pcb_->local_port, name, addrlen); + if (pb == nullptr) { + this->rx_closed_ = true; + return ERR_OK; } - int getsockopt(int level, int optname, void *optval, socklen_t *optlen) final { - if (pcb_ == nullptr) { - errno = ECONNRESET; - return -1; - } - if (optlen == nullptr || optval == nullptr) { - errno = EINVAL; - return -1; - } - if (level == SOL_SOCKET && optname == SO_REUSEADDR) { - if (*optlen < 4) { - errno = EINVAL; - return -1; - } + if (this->rx_buf_ == nullptr) { + // no need to copy because lwIP gave control of it to us + this->rx_buf_ = pb; + this->rx_buf_offset_ = 0; + } else { + pbuf_cat(this->rx_buf_, pb); + } +#ifdef USE_ESP8266 + // Wake the main loop immediately so it can process the received data. + socket_wake(); +#endif + return ERR_OK; +} - // lwip doesn't seem to have this feature. Don't send an error - // to prevent warnings - *reinterpret_cast(optval) = 1; - *optlen = 4; - return 0; - } - if (level == IPPROTO_TCP && optname == TCP_NODELAY) { - if (*optlen < 4) { - errno = EINVAL; - return -1; - } - *reinterpret_cast(optval) = nodelay_; - *optlen = 4; - return 0; - } - - errno = EINVAL; +ssize_t LWIPRawImpl::read(void *buf, size_t len) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; return -1; } - int setsockopt(int level, int optname, const void *optval, socklen_t optlen) final { - if (pcb_ == nullptr) { - errno = ECONNRESET; - return -1; - } - if (level == SOL_SOCKET && optname == SO_REUSEADDR) { - if (optlen != 4) { - errno = EINVAL; - return -1; - } - - // lwip doesn't seem to have this feature. Don't send an error - // to prevent warnings - return 0; - } - if (level == IPPROTO_TCP && optname == TCP_NODELAY) { - if (optlen != 4) { - errno = EINVAL; - return -1; - } - int val = *reinterpret_cast(optval); - nodelay_ = val; - return 0; - } - - errno = EINVAL; + if (this->rx_closed_ && this->rx_buf_ == nullptr) { + return 0; + } + if (len == 0) { + return 0; + } + if (this->rx_buf_ == nullptr) { + errno = EWOULDBLOCK; return -1; } - int listen(int backlog) override { - // Regular sockets can't be converted to listening - this shouldn't happen - // as listen() should only be called on sockets created for listening + + size_t read = 0; + uint8_t *buf8 = reinterpret_cast(buf); + while (len && this->rx_buf_ != nullptr) { + size_t pb_len = this->rx_buf_->len; + size_t pb_left = pb_len - this->rx_buf_offset_; + if (pb_left == 0) + break; + size_t copysize = std::min(len, pb_left); + memcpy(buf8, reinterpret_cast(this->rx_buf_->payload) + this->rx_buf_offset_, copysize); + + if (pb_left == copysize) { + // full pb copied, free it + if (this->rx_buf_->next == nullptr) { + // last buffer in chain + pbuf_free(this->rx_buf_); + this->rx_buf_ = nullptr; + this->rx_buf_offset_ = 0; + } else { + auto *old_buf = this->rx_buf_; + this->rx_buf_ = this->rx_buf_->next; + pbuf_ref(this->rx_buf_); + pbuf_free(old_buf); + this->rx_buf_offset_ = 0; + } + } else { + this->rx_buf_offset_ += copysize; + } + LWIP_LOG("tcp_recved(%p %u)", this->pcb_, copysize); + tcp_recved(this->pcb_, copysize); + + buf8 += copysize; + len -= copysize; + read += copysize; + } + + if (read == 0) { + errno = EWOULDBLOCK; + return -1; + } + + return read; +} + +ssize_t LWIPRawImpl::readv(const struct iovec *iov, int iovcnt) { + ssize_t ret = 0; + for (int i = 0; i < iovcnt; i++) { + ssize_t err = this->read(reinterpret_cast(iov[i].iov_base), iov[i].iov_len); + if (err == -1) { + if (ret != 0) { + // if we already read some don't return an error + break; + } + return err; + } + ret += err; + if ((size_t) err != iov[i].iov_len) + break; + } + return ret; +} + +ssize_t LWIPRawImpl::internal_write_(const void *buf, size_t len) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + if (len == 0) + return 0; + if (buf == nullptr) { + errno = EINVAL; + return 0; + } + auto space = tcp_sndbuf(this->pcb_); + if (space == 0) { + errno = EWOULDBLOCK; + return -1; + } + size_t to_send = std::min((size_t) space, len); + LWIP_LOG("tcp_write(%p buf=%p %u)", this->pcb_, buf, to_send); + err_t err = tcp_write(this->pcb_, buf, to_send, TCP_WRITE_FLAG_COPY); + if (err == ERR_MEM) { + LWIP_LOG(" -> err ERR_MEM"); + errno = EWOULDBLOCK; + return -1; + } + if (err != ERR_OK) { + LWIP_LOG(" -> err %d", err); + errno = ECONNRESET; + return -1; + } + return to_send; +} + +int LWIPRawImpl::internal_output_() { + LWIP_LOG("tcp_output(%p)", this->pcb_); + err_t err = tcp_output(this->pcb_); + if (err == ERR_ABRT) { + // sometimes lwip returns ERR_ABRT for no apparent reason + // the connection works fine afterwards, and back with ESPAsyncTCP we + // indirectly also ignored this error + // FIXME: figure out where this is returned and what it means in this context + LWIP_LOG(" -> err ERR_ABRT"); + return 0; + } + if (err != ERR_OK) { + LWIP_LOG(" -> err %d", err); + errno = ECONNRESET; + return -1; + } + return 0; +} + +ssize_t LWIPRawImpl::write(const void *buf, size_t len) { + ssize_t written = this->internal_write_(buf, len); + if (written == -1) + return -1; + if (written == 0) { + // no need to output if nothing written + return 0; + } + if (this->nodelay_) { + int err = this->internal_output_(); + if (err == -1) + return -1; + } + return written; +} + +ssize_t LWIPRawImpl::writev(const struct iovec *iov, int iovcnt) { + ssize_t written = 0; + for (int i = 0; i < iovcnt; i++) { + ssize_t err = this->internal_write_(reinterpret_cast(iov[i].iov_base), iov[i].iov_len); + if (err == -1) { + if (written != 0) { + // if we already read some don't return an error + break; + } + return err; + } + written += err; + if ((size_t) err != iov[i].iov_len) + break; + } + if (written == 0) { + // no need to output if nothing written + return 0; + } + if (this->nodelay_) { + int err = this->internal_output_(); + if (err == -1) + return -1; + } + return written; +} + +// ---- LWIPRawListenImpl methods ---- + +LWIPRawListenImpl::~LWIPRawListenImpl() { + // Base class destructor handles pcb_ cleanup via tcp_abort +} + +void LWIPRawListenImpl::init() { + LWIP_LOG("init(%p)", this->pcb_); + tcp_arg(this->pcb_, this); + tcp_accept(this->pcb_, LWIPRawListenImpl::s_accept_fn); + tcp_err(this->pcb_, LWIPRawListenImpl::s_err_fn); +} + +void LWIPRawListenImpl::s_err_fn(void *arg, err_t err) { + auto *arg_this = reinterpret_cast(arg); + ESP_LOGVV(TAG, "socket %p: err(err=%d)", arg_this, err); + arg_this->pcb_ = nullptr; +} + +err_t LWIPRawListenImpl::s_accept_fn(void *arg, struct tcp_pcb *newpcb, err_t err) { + auto *arg_this = reinterpret_cast(arg); + return arg_this->accept_fn_(newpcb, err); +} + +std::unique_ptr LWIPRawListenImpl::accept(struct sockaddr *addr, socklen_t *addrlen) { + if (this->pcb_ == nullptr) { + errno = EBADF; + return nullptr; + } + if (this->accepted_socket_count_ == 0) { + errno = EWOULDBLOCK; + return nullptr; + } + // Take from front for FIFO ordering + std::unique_ptr sock = std::move(this->accepted_sockets_[0]); + // Shift remaining sockets forward + for (uint8_t i = 1; i < this->accepted_socket_count_; i++) { + this->accepted_sockets_[i - 1] = std::move(this->accepted_sockets_[i]); + } + this->accepted_socket_count_--; + LWIP_LOG("Connection accepted by application, queue size: %d", this->accepted_socket_count_); + if (addr != nullptr) { + sock->getpeername(addr, addrlen); + } + LWIP_LOG("accept(%p)", sock.get()); + return sock; +} + +int LWIPRawListenImpl::listen(int backlog) { + if (this->pcb_ == nullptr) { + errno = EBADF; + return -1; + } + LWIP_LOG("tcp_listen_with_backlog(%p backlog=%d)", this->pcb_, backlog); + struct tcp_pcb *listen_pcb = tcp_listen_with_backlog(this->pcb_, backlog); + if (listen_pcb == nullptr) { + tcp_abort(this->pcb_); + this->pcb_ = nullptr; errno = EOPNOTSUPP; return -1; } - ssize_t read(void *buf, size_t len) final { - if (pcb_ == nullptr) { - errno = ECONNRESET; - return -1; - } - if (rx_closed_ && rx_buf_ == nullptr) { - return 0; - } - if (len == 0) { - return 0; - } - if (rx_buf_ == nullptr) { - errno = EWOULDBLOCK; - return -1; - } + // tcp_listen reallocates the pcb, replace ours + this->pcb_ = listen_pcb; + // set callbacks on new pcb + LWIP_LOG("tcp_arg(%p)", this->pcb_); + tcp_arg(this->pcb_, this); + tcp_accept(this->pcb_, LWIPRawListenImpl::s_accept_fn); + return 0; +} - size_t read = 0; - uint8_t *buf8 = reinterpret_cast(buf); - while (len && rx_buf_ != nullptr) { - size_t pb_len = rx_buf_->len; - size_t pb_left = pb_len - rx_buf_offset_; - if (pb_left == 0) - break; - size_t copysize = std::min(len, pb_left); - memcpy(buf8, reinterpret_cast(rx_buf_->payload) + rx_buf_offset_, copysize); - - if (pb_left == copysize) { - // full pb copied, free it - if (rx_buf_->next == nullptr) { - // last buffer in chain - pbuf_free(rx_buf_); - rx_buf_ = nullptr; - rx_buf_offset_ = 0; - } else { - auto *old_buf = rx_buf_; - rx_buf_ = rx_buf_->next; - pbuf_ref(rx_buf_); - pbuf_free(old_buf); - rx_buf_offset_ = 0; - } - } else { - rx_buf_offset_ += copysize; - } - LWIP_LOG("tcp_recved(%p %u)", pcb_, copysize); - tcp_recved(pcb_, copysize); - - buf8 += copysize; - len -= copysize; - read += copysize; - } - - if (read == 0) { - errno = EWOULDBLOCK; - return -1; - } - - return read; - } - ssize_t readv(const struct iovec *iov, int iovcnt) final { - ssize_t ret = 0; - for (int i = 0; i < iovcnt; i++) { - ssize_t err = read(reinterpret_cast(iov[i].iov_base), iov[i].iov_len); - if (err == -1) { - if (ret != 0) { - // if we already read some don't return an error - break; - } - return err; - } - ret += err; - if ((size_t) err != iov[i].iov_len) - break; - } - return ret; - } - - ssize_t recvfrom(void *buf, size_t len, sockaddr *addr, socklen_t *addr_len) final { - errno = ENOTSUP; - return -1; - } - - ssize_t internal_write(const void *buf, size_t len) { - if (pcb_ == nullptr) { - errno = ECONNRESET; - return -1; - } - if (len == 0) - return 0; - if (buf == nullptr) { - errno = EINVAL; - return 0; - } - auto space = tcp_sndbuf(pcb_); - if (space == 0) { - errno = EWOULDBLOCK; - return -1; - } - size_t to_send = std::min((size_t) space, len); - LWIP_LOG("tcp_write(%p buf=%p %u)", pcb_, buf, to_send); - err_t err = tcp_write(pcb_, buf, to_send, TCP_WRITE_FLAG_COPY); - if (err == ERR_MEM) { - LWIP_LOG(" -> err ERR_MEM"); - errno = EWOULDBLOCK; - return -1; - } - if (err != ERR_OK) { - LWIP_LOG(" -> err %d", err); - errno = ECONNRESET; - return -1; - } - return to_send; - } - int internal_output() { - LWIP_LOG("tcp_output(%p)", pcb_); - err_t err = tcp_output(pcb_); - if (err == ERR_ABRT) { - LWIP_LOG(" -> err ERR_ABRT"); - // sometimes lwip returns ERR_ABRT for no apparent reason - // the connection works fine afterwards, and back with ESPAsyncTCP we - // indirectly also ignored this error - // FIXME: figure out where this is returned and what it means in this context - return 0; - } - if (err != ERR_OK) { - LWIP_LOG(" -> err %d", err); - errno = ECONNRESET; - return -1; - } - return 0; - } - ssize_t write(const void *buf, size_t len) final { - ssize_t written = internal_write(buf, len); - if (written == -1) - return -1; - if (written == 0) { - // no need to output if nothing written - return 0; - } - if (nodelay_) { - int err = internal_output(); - if (err == -1) - return -1; - } - return written; - } - ssize_t writev(const struct iovec *iov, int iovcnt) final { - ssize_t written = 0; - for (int i = 0; i < iovcnt; i++) { - ssize_t err = internal_write(reinterpret_cast(iov[i].iov_base), iov[i].iov_len); - if (err == -1) { - if (written != 0) { - // if we already read some don't return an error - break; - } - return err; - } - written += err; - if ((size_t) err != iov[i].iov_len) - break; - } - if (written == 0) { - // no need to output if nothing written - return 0; - } - if (nodelay_) { - int err = internal_output(); - if (err == -1) - return -1; - } - return written; - } - ssize_t sendto(const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) final { - // return ::sendto(fd_, buf, len, flags, to, tolen); - errno = ENOSYS; - return -1; - } - bool ready() const override { return this->rx_buf_ != nullptr || this->rx_closed_ || this->pcb_ == nullptr; } - - int setblocking(bool blocking) final { - if (pcb_ == nullptr) { - errno = ECONNRESET; - return -1; - } - if (blocking) { - // blocking operation not supported - errno = EINVAL; - return -1; - } - return 0; - } - - void err_fn(err_t err) { - LWIP_LOG("err(err=%d)", err); - // "If a connection is aborted because of an error, the application is alerted of this event by - // the err callback." - // pcb is already freed when this callback is called - // ERR_RST: connection was reset by remote host - // ERR_ABRT: aborted through tcp_abort or TCP timer - pcb_ = nullptr; - } - err_t recv_fn(struct pbuf *pb, err_t err) { - LWIP_LOG("recv(pb=%p err=%d)", pb, err); - if (err != 0) { - // "An error code if there has been an error receiving Only return ERR_ABRT if you have - // called tcp_abort from within the callback function!" - rx_closed_ = true; - return ERR_OK; - } - if (pb == nullptr) { - rx_closed_ = true; - return ERR_OK; - } - if (rx_buf_ == nullptr) { - // no need to copy because lwIP gave control of it to us - rx_buf_ = pb; - rx_buf_offset_ = 0; - } else { - pbuf_cat(rx_buf_, pb); - } -#ifdef USE_ESP8266 - // Wake the main loop immediately so it can process the received data. - socket_wake(); -#endif +err_t LWIPRawListenImpl::accept_fn_(struct tcp_pcb *newpcb, err_t err) { + LWIP_LOG("accept(newpcb=%p err=%d)", newpcb, err); + if (err != ERR_OK || newpcb == nullptr) { + // "An error code if there has been an error accepting. Only return ERR_ABRT if you have + // called tcp_abort from within the callback function!" + // https://www.nongnu.org/lwip/2_1_x/tcp_8h.html#a00517abce6856d6c82f0efebdafb734d + // nothing to do here, we just don't push it to the queue return ERR_OK; } - - static void s_err_fn(void *arg, err_t err) { - LWIPRawImpl *arg_this = reinterpret_cast(arg); - arg_this->err_fn(err); + // Check if we've reached the maximum accept queue size + if (this->accepted_socket_count_ >= MAX_ACCEPTED_SOCKETS) { + LWIP_LOG("Rejecting connection, queue full (%d)", this->accepted_socket_count_); + // Abort the connection when queue is full + tcp_abort(newpcb); + // Must return ERR_ABRT since we called tcp_abort() + return ERR_ABRT; } - - static err_t s_recv_fn(void *arg, struct tcp_pcb *pcb, struct pbuf *pb, err_t err) { - LWIPRawImpl *arg_this = reinterpret_cast(arg); - return arg_this->recv_fn(pb, err); - } - - protected: - int ip2sockaddr_(ip_addr_t *ip, uint16_t port, struct sockaddr *name, socklen_t *addrlen) { - if (family_ == AF_INET) { - if (*addrlen < sizeof(struct sockaddr_in)) { - errno = EINVAL; - return -1; - } - - struct sockaddr_in *addr = reinterpret_cast(name); - addr->sin_family = AF_INET; - *addrlen = addr->sin_len = sizeof(struct sockaddr_in); - addr->sin_port = port; - inet_addr_from_ip4addr(&addr->sin_addr, ip_2_ip4(ip)); - return 0; - } -#if LWIP_IPV6 - else if (family_ == AF_INET6) { - if (*addrlen < sizeof(struct sockaddr_in6)) { - errno = EINVAL; - return -1; - } - - struct sockaddr_in6 *addr = reinterpret_cast(name); - addr->sin6_family = AF_INET6; - *addrlen = addr->sin6_len = sizeof(struct sockaddr_in6); - addr->sin6_port = port; - - // AF_INET6 sockets are bound to IPv4 as well, so we may encounter IPv4 addresses that must be converted to IPv6. - if (IP_IS_V4(ip)) { - ip_addr_t mapped; - ip4_2_ipv4_mapped_ipv6(ip_2_ip6(&mapped), ip_2_ip4(ip)); - inet6_addr_from_ip6addr(&addr->sin6_addr, ip_2_ip6(&mapped)); - } else { - inet6_addr_from_ip6addr(&addr->sin6_addr, ip_2_ip6(ip)); - } - return 0; - } -#endif - return -1; - } - - // Member ordering optimized to minimize padding on 32-bit systems - // Largest members first (4 bytes), then smaller members (1 byte each) - struct tcp_pcb *pcb_; - pbuf *rx_buf_ = nullptr; - size_t rx_buf_offset_ = 0; - bool rx_closed_ = false; - // don't use lwip nodelay flag, it sometimes causes reconnect - // instead use it for determining whether to call lwip_output - bool nodelay_ = false; - sa_family_t family_ = 0; -}; - -// Listening socket class - only allocates accept queue when needed (for bind+listen sockets) -// This saves 16 bytes (12 bytes array + 1 byte count + 3 bytes padding) for regular connected sockets on ESP8266/RP2040 -class LWIPRawListenImpl final : public LWIPRawImpl { - public: - LWIPRawListenImpl(sa_family_t family, struct tcp_pcb *pcb) : LWIPRawImpl(family, pcb) {} - - void init() { - LWIP_LOG("init(%p)", pcb_); - tcp_arg(pcb_, this); - tcp_accept(pcb_, LWIPRawListenImpl::s_accept_fn); - tcp_err(pcb_, LWIPRawImpl::s_err_fn); // Use base class error handler - } - - bool ready() const override { return this->accepted_socket_count_ > 0; } - - std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { - if (pcb_ == nullptr) { - errno = EBADF; - return nullptr; - } - if (accepted_socket_count_ == 0) { - errno = EWOULDBLOCK; - return nullptr; - } - // Take from front for FIFO ordering - std::unique_ptr sock = std::move(accepted_sockets_[0]); - // Shift remaining sockets forward - for (uint8_t i = 1; i < accepted_socket_count_; i++) { - accepted_sockets_[i - 1] = std::move(accepted_sockets_[i]); - } - accepted_socket_count_--; - LWIP_LOG("Connection accepted by application, queue size: %d", accepted_socket_count_); - if (addr != nullptr) { - sock->getpeername(addr, addrlen); - } - LWIP_LOG("accept(%p)", sock.get()); - return std::unique_ptr(std::move(sock)); - } - - int listen(int backlog) override { - if (pcb_ == nullptr) { - errno = EBADF; - return -1; - } - LWIP_LOG("tcp_listen_with_backlog(%p backlog=%d)", pcb_, backlog); - struct tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb_, backlog); - if (listen_pcb == nullptr) { - tcp_abort(pcb_); - pcb_ = nullptr; - errno = EOPNOTSUPP; - return -1; - } - // tcp_listen reallocates the pcb, replace ours - pcb_ = listen_pcb; - // set callbacks on new pcb - LWIP_LOG("tcp_arg(%p)", pcb_); - tcp_arg(pcb_, this); - tcp_accept(pcb_, LWIPRawListenImpl::s_accept_fn); - return 0; - } - - private: - err_t accept_fn_(struct tcp_pcb *newpcb, err_t err) { - LWIP_LOG("accept(newpcb=%p err=%d)", newpcb, err); - if (err != ERR_OK || newpcb == nullptr) { - // "An error code if there has been an error accepting. Only return ERR_ABRT if you have - // called tcp_abort from within the callback function!" - // https://www.nongnu.org/lwip/2_1_x/tcp_8h.html#a00517abce6856d6c82f0efebdafb734d - // nothing to do here, we just don't push it to the queue - return ERR_OK; - } - // Check if we've reached the maximum accept queue size - if (accepted_socket_count_ >= MAX_ACCEPTED_SOCKETS) { - LWIP_LOG("Rejecting connection, queue full (%d)", accepted_socket_count_); - // Abort the connection when queue is full - tcp_abort(newpcb); - // Must return ERR_ABRT since we called tcp_abort() - return ERR_ABRT; - } - auto sock = make_unique(family_, newpcb); - sock->init(); - accepted_sockets_[accepted_socket_count_++] = std::move(sock); - LWIP_LOG("Accepted connection, queue size: %d", accepted_socket_count_); + auto sock = make_unique(this->family_, newpcb); + sock->init(); + this->accepted_sockets_[this->accepted_socket_count_++] = std::move(sock); + LWIP_LOG("Accepted connection, queue size: %d", this->accepted_socket_count_); #ifdef USE_ESP8266 - // Wake the main loop immediately so it can accept the new connection. - socket_wake(); + // Wake the main loop immediately so it can accept the new connection. + socket_wake(); #endif - return ERR_OK; - } + return ERR_OK; +} - static err_t s_accept_fn(void *arg, struct tcp_pcb *newpcb, err_t err) { - LWIPRawListenImpl *arg_this = reinterpret_cast(arg); - return arg_this->accept_fn_(newpcb, err); - } - - // Accept queue - holds incoming connections briefly until the event loop calls accept() - // This is NOT a connection pool - just a temporary queue between LWIP callbacks and the main loop - // 3 slots is plenty since connections are pulled out quickly by the event loop - // - // Memory analysis: std::array<3> vs original std::queue implementation: - // - std::queue uses std::deque internally which on 32-bit systems needs: - // 24 bytes (deque object) + 32+ bytes (map array) + heap allocations - // Total: ~56+ bytes minimum, plus heap fragmentation - // - std::array<3>: 12 bytes fixed (3 pointers × 4 bytes) - // Saves ~44+ bytes RAM per listening socket + avoids ALL heap allocations - // Used on ESP8266 and RP2040 (platforms using LWIP_TCP implementation) - // - // By using a separate listening socket class, regular connected sockets save - // 16 bytes (12 bytes array + 1 byte count + 3 bytes padding) of memory overhead on 32-bit systems - static constexpr size_t MAX_ACCEPTED_SOCKETS = 3; - std::array, MAX_ACCEPTED_SOCKETS> accepted_sockets_; - uint8_t accepted_socket_count_ = 0; // Number of sockets currently in queue -}; +// ---- Factory functions ---- std::unique_ptr socket(int domain, int type, int protocol) { if (type != SOCK_STREAM) { @@ -688,9 +651,7 @@ std::unique_ptr socket(int domain, int type, int protocol) { auto *pcb = tcp_new(); if (pcb == nullptr) return nullptr; - // Create listening socket implementation since user sockets typically bind+listen - // Accepted connections are created directly as LWIPRawImpl in the accept callback - auto *sock = new LWIPRawListenImpl((sa_family_t) domain, pcb); // NOLINT(cppcoreguidelines-owning-memory) + auto *sock = new LWIPRawImpl((sa_family_t) domain, pcb); // NOLINT(cppcoreguidelines-owning-memory) sock->init(); return std::unique_ptr{sock}; } @@ -700,6 +661,25 @@ std::unique_ptr socket_loop_monitored(int domain, int type, int protocol return socket(domain, type, protocol); } +std::unique_ptr socket_listen(int domain, int type, int protocol) { + if (type != SOCK_STREAM) { + ESP_LOGE(TAG, "UDP sockets not supported on this platform, use WiFiUDP"); + errno = EPROTOTYPE; + return nullptr; + } + auto *pcb = tcp_new(); + if (pcb == nullptr) + return nullptr; + auto *sock = new LWIPRawListenImpl((sa_family_t) domain, pcb); // NOLINT(cppcoreguidelines-owning-memory) + sock->init(); + return std::unique_ptr{sock}; +} + +std::unique_ptr socket_listen_loop_monitored(int domain, int type, int protocol) { + // LWIPRawImpl doesn't use file descriptors, so monitoring is not applicable + return socket_listen(domain, type, protocol); +} + } // namespace esphome::socket #endif // USE_SOCKET_IMPL_LWIP_TCP diff --git a/esphome/components/socket/lwip_raw_tcp_impl.h b/esphome/components/socket/lwip_raw_tcp_impl.h new file mode 100644 index 0000000000..c171e0537f --- /dev/null +++ b/esphome/components/socket/lwip_raw_tcp_impl.h @@ -0,0 +1,200 @@ +#pragma once +#include "esphome/core/defines.h" + +#ifdef USE_SOCKET_IMPL_LWIP_TCP + +#include +#include +#include +#include +#include + +#include "esphome/core/helpers.h" +#include "headers.h" +#include "lwip/ip.h" +#include "lwip/netif.h" +#include "lwip/opt.h" +#include "lwip/tcp.h" + +namespace esphome::socket { + +// Forward declaration +class LWIPRawImpl; + +/// Non-virtual common base for LWIP raw TCP sockets. +/// Provides shared fields and methods for both connected and listening sockets. +/// No virtual methods — pure code sharing. +class LWIPRawCommon { + public: + LWIPRawCommon(sa_family_t family, struct tcp_pcb *pcb) : pcb_(pcb), family_(family) {} + ~LWIPRawCommon(); + LWIPRawCommon(const LWIPRawCommon &) = delete; + LWIPRawCommon &operator=(const LWIPRawCommon &) = delete; + + int bind(const struct sockaddr *name, socklen_t addrlen); + int close(); + int shutdown(int how); + + int getpeername(struct sockaddr *name, socklen_t *addrlen); + int getsockname(struct sockaddr *name, socklen_t *addrlen); + + /// Format peer address into a fixed-size buffer (no heap allocation) + size_t getpeername_to(std::span buf); + /// Format local address into a fixed-size buffer (no heap allocation) + size_t getsockname_to(std::span buf); + + int getsockopt(int level, int optname, void *optval, socklen_t *optlen); + int setsockopt(int level, int optname, const void *optval, socklen_t optlen); + + int get_fd() const { return -1; } + + protected: + int ip2sockaddr_(ip_addr_t *ip, uint16_t port, struct sockaddr *name, socklen_t *addrlen); + + // Member ordering optimized to minimize padding on 32-bit systems + struct tcp_pcb *pcb_; + // don't use lwip nodelay flag, it sometimes causes reconnect + // instead use it for determining whether to call lwip_output + bool nodelay_ = false; + sa_family_t family_ = 0; +}; + +/// Connected socket implementation for LWIP raw TCP. +/// No virtual methods — callers always use the concrete type. +class LWIPRawImpl : public LWIPRawCommon { + public: + using LWIPRawCommon::LWIPRawCommon; + ~LWIPRawImpl(); + + void init(); + + // Non-listening sockets return error + std::unique_ptr accept(struct sockaddr *, socklen_t *) { + errno = EINVAL; + return nullptr; + } + std::unique_ptr accept_loop_monitored(struct sockaddr *addr, socklen_t *addrlen) { + return this->accept(addr, addrlen); + } + // Regular sockets can't be converted to listening - this shouldn't happen + // as listen() should only be called on sockets created for listening + int listen(int) { + errno = EOPNOTSUPP; + return -1; + } + ssize_t read(void *buf, size_t len); + ssize_t readv(const struct iovec *iov, int iovcnt); + ssize_t recvfrom(void *, size_t, sockaddr *, socklen_t *) { + errno = ENOTSUP; + return -1; + } + ssize_t write(const void *buf, size_t len); + ssize_t writev(const struct iovec *iov, int iovcnt); + ssize_t sendto(const void *, size_t, int, const struct sockaddr *, socklen_t) { + // return ::sendto(fd_, buf, len, flags, to, tolen); + errno = ENOSYS; + return -1; + } + bool ready() const { return this->rx_buf_ != nullptr || this->rx_closed_ || this->pcb_ == nullptr; } + + int setblocking(bool blocking) { + if (this->pcb_ == nullptr) { + errno = ECONNRESET; + return -1; + } + if (blocking) { + // blocking operation not supported + errno = EINVAL; + return -1; + } + return 0; + } + int loop() { return 0; } + + err_t recv_fn(struct pbuf *pb, err_t err); + + static void s_err_fn(void *arg, err_t err); + static err_t s_recv_fn(void *arg, struct tcp_pcb *pcb, struct pbuf *pb, err_t err); + + protected: + ssize_t internal_write_(const void *buf, size_t len); + int internal_output_(); + + pbuf *rx_buf_ = nullptr; + size_t rx_buf_offset_ = 0; + bool rx_closed_ = false; +}; + +/// Listening socket implementation for LWIP raw TCP. +/// Separate from LWIPRawImpl — no virtual dispatch needed. +class LWIPRawListenImpl : public LWIPRawCommon { + public: + using LWIPRawCommon::LWIPRawCommon; + ~LWIPRawListenImpl(); + + void init(); + + bool ready() const { return this->accepted_socket_count_ > 0; } + + std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen); + std::unique_ptr accept_loop_monitored(struct sockaddr *addr, socklen_t *addrlen) { + return this->accept(addr, addrlen); + } + int listen(int backlog); + + // Listening sockets don't do I/O + ssize_t read(void *, size_t) { + errno = ENOTSUP; + return -1; + } + ssize_t write(const void *, size_t) { + errno = ENOTSUP; + return -1; + } + ssize_t readv(const struct iovec *, int) { + errno = ENOTSUP; + return -1; + } + ssize_t writev(const struct iovec *, int) { + errno = ENOTSUP; + return -1; + } + ssize_t recvfrom(void *, size_t, sockaddr *, socklen_t *) { + errno = ENOTSUP; + return -1; + } + ssize_t sendto(const void *, size_t, int, const struct sockaddr *, socklen_t) { + errno = ENOTSUP; + return -1; + } + int setblocking(bool) { return 0; } + int loop() { return 0; } + + static void s_err_fn(void *arg, err_t err); + + private: + err_t accept_fn_(struct tcp_pcb *newpcb, err_t err); + static err_t s_accept_fn(void *arg, struct tcp_pcb *newpcb, err_t err); + + // Accept queue - holds incoming connections briefly until the event loop calls accept() + // This is NOT a connection pool - just a temporary queue between LWIP callbacks and the main loop + // 3 slots is plenty since connections are pulled out quickly by the event loop + // + // Memory analysis: std::array<3> vs original std::queue implementation: + // - std::queue uses std::deque internally which on 32-bit systems needs: + // 24 bytes (deque object) + 32+ bytes (map array) + heap allocations + // Total: ~56+ bytes minimum, plus heap fragmentation + // - std::array<3>: 12 bytes fixed (3 pointers × 4 bytes) + // Saves ~44+ bytes RAM per listening socket + avoids ALL heap allocations + // Used on ESP8266 and RP2040 (platforms using LWIP_TCP implementation) + // + // By using a separate listening socket class, regular connected sockets save + // 16 bytes (12 bytes array + 1 byte count + 3 bytes padding) of memory overhead on 32-bit systems + static constexpr size_t MAX_ACCEPTED_SOCKETS = 3; + std::array, MAX_ACCEPTED_SOCKETS> accepted_sockets_; + uint8_t accepted_socket_count_ = 0; // Number of sockets currently in queue +}; + +} // namespace esphome::socket + +#endif // USE_SOCKET_IMPL_LWIP_TCP diff --git a/esphome/components/socket/lwip_sockets_impl.cpp b/esphome/components/socket/lwip_sockets_impl.cpp index 79d68e085a..0322820ef4 100644 --- a/esphome/components/socket/lwip_sockets_impl.cpp +++ b/esphome/components/socket/lwip_sockets_impl.cpp @@ -1,6 +1,6 @@ -#include "socket.h" #include "esphome/core/defines.h" #include "esphome/core/helpers.h" +#include "socket.h" #ifdef USE_SOCKET_IMPL_LWIP_SOCKETS @@ -9,94 +9,73 @@ namespace esphome::socket { -class LwIPSocketImpl final : public Socket { - public: - LwIPSocketImpl(int fd, bool monitor_loop = false) { - this->fd_ = fd; - // Register new socket with the application for select() if monitoring requested - if (monitor_loop && this->fd_ >= 0) { - // Only set loop_monitored_ to true if registration succeeds - this->loop_monitored_ = App.register_socket_fd(this->fd_); - } - } - ~LwIPSocketImpl() override { - if (!this->closed_) { - this->close(); // NOLINT(clang-analyzer-optin.cplusplus.VirtualCall) - } - } - int connect(const struct sockaddr *addr, socklen_t addrlen) override { - return lwip_connect(this->fd_, addr, addrlen); - } - std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { - int fd = lwip_accept(this->fd_, addr, addrlen); - if (fd == -1) - return {}; - return make_unique(fd, false); - } - std::unique_ptr accept_loop_monitored(struct sockaddr *addr, socklen_t *addrlen) override { - int fd = lwip_accept(this->fd_, addr, addrlen); - if (fd == -1) - return {}; - return make_unique(fd, true); +LwIPSocketImpl::LwIPSocketImpl(int fd, bool monitor_loop) { + this->fd_ = fd; + // Register new socket with the application for select() if monitoring requested + if (monitor_loop && this->fd_ >= 0) { + // Only set loop_monitored_ to true if registration succeeds + this->loop_monitored_ = App.register_socket_fd(this->fd_); } +} - int bind(const struct sockaddr *addr, socklen_t addrlen) override { return lwip_bind(this->fd_, addr, addrlen); } - int close() override { - if (!this->closed_) { - // Unregister from select() before closing if monitored - if (this->loop_monitored_) { - App.unregister_socket_fd(this->fd_); - } - int ret = lwip_close(this->fd_); - this->closed_ = true; - return ret; +LwIPSocketImpl::~LwIPSocketImpl() { + if (!this->closed_) { + this->close(); + } +} + +int LwIPSocketImpl::close() { + if (!this->closed_) { + // Unregister from select() before closing if monitored + if (this->loop_monitored_) { + App.unregister_socket_fd(this->fd_); } + int ret = lwip_close(this->fd_); + this->closed_ = true; + return ret; + } + return 0; +} + +int LwIPSocketImpl::setblocking(bool blocking) { + int fl = lwip_fcntl(this->fd_, F_GETFL, 0); + if (blocking) { + fl &= ~O_NONBLOCK; + } else { + fl |= O_NONBLOCK; + } + lwip_fcntl(this->fd_, F_SETFL, fl); + return 0; +} + +bool LwIPSocketImpl::ready() const { return socket_ready_fd(this->fd_, this->loop_monitored_); } + +size_t LwIPSocketImpl::getpeername_to(std::span buf) { + struct sockaddr_storage storage; + socklen_t len = sizeof(storage); + if (this->getpeername(reinterpret_cast(&storage), &len) != 0) { + buf[0] = '\0'; return 0; } - int shutdown(int how) override { return lwip_shutdown(this->fd_, how); } + return format_sockaddr_to(reinterpret_cast(&storage), len, buf); +} - int getpeername(struct sockaddr *addr, socklen_t *addrlen) override { - return lwip_getpeername(this->fd_, addr, addrlen); - } - int getsockname(struct sockaddr *addr, socklen_t *addrlen) override { - return lwip_getsockname(this->fd_, addr, addrlen); - } - int getsockopt(int level, int optname, void *optval, socklen_t *optlen) override { - return lwip_getsockopt(this->fd_, level, optname, optval, optlen); - } - int setsockopt(int level, int optname, const void *optval, socklen_t optlen) override { - return lwip_setsockopt(this->fd_, level, optname, optval, optlen); - } - int listen(int backlog) override { return lwip_listen(this->fd_, backlog); } - ssize_t read(void *buf, size_t len) override { return lwip_read(this->fd_, buf, len); } - ssize_t recvfrom(void *buf, size_t len, sockaddr *addr, socklen_t *addr_len) override { - return lwip_recvfrom(this->fd_, buf, len, 0, addr, addr_len); - } - ssize_t readv(const struct iovec *iov, int iovcnt) override { return lwip_readv(this->fd_, iov, iovcnt); } - ssize_t write(const void *buf, size_t len) override { return lwip_write(this->fd_, buf, len); } - ssize_t send(void *buf, size_t len, int flags) { return lwip_send(this->fd_, buf, len, flags); } - ssize_t writev(const struct iovec *iov, int iovcnt) override { return lwip_writev(this->fd_, iov, iovcnt); } - ssize_t sendto(const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) override { - return lwip_sendto(this->fd_, buf, len, flags, to, tolen); - } - int setblocking(bool blocking) override { - int fl = lwip_fcntl(this->fd_, F_GETFL, 0); - if (blocking) { - fl &= ~O_NONBLOCK; - } else { - fl |= O_NONBLOCK; - } - lwip_fcntl(this->fd_, F_SETFL, fl); +size_t LwIPSocketImpl::getsockname_to(std::span buf) { + struct sockaddr_storage storage; + socklen_t len = sizeof(storage); + if (this->getsockname(reinterpret_cast(&storage), &len) != 0) { + buf[0] = '\0'; return 0; } -}; + return format_sockaddr_to(reinterpret_cast(&storage), len, buf); +} // Helper to create a socket with optional monitoring -static std::unique_ptr create_socket(int domain, int type, int protocol, bool loop_monitored = false) { +static std::unique_ptr create_socket(int domain, int type, int protocol, bool loop_monitored = false) { int ret = lwip_socket(domain, type, protocol); if (ret == -1) return nullptr; - return std::unique_ptr{new LwIPSocketImpl(ret, loop_monitored)}; + return make_unique(ret, loop_monitored); } std::unique_ptr socket(int domain, int type, int protocol) { @@ -107,6 +86,14 @@ std::unique_ptr socket_loop_monitored(int domain, int type, int protocol return create_socket(domain, type, protocol, true); } +std::unique_ptr socket_listen(int domain, int type, int protocol) { + return create_socket(domain, type, protocol, false); +} + +std::unique_ptr socket_listen_loop_monitored(int domain, int type, int protocol) { + return create_socket(domain, type, protocol, true); +} + } // namespace esphome::socket #endif // USE_SOCKET_IMPL_LWIP_SOCKETS diff --git a/esphome/components/socket/lwip_sockets_impl.h b/esphome/components/socket/lwip_sockets_impl.h new file mode 100644 index 0000000000..2e319fcc4d --- /dev/null +++ b/esphome/components/socket/lwip_sockets_impl.h @@ -0,0 +1,80 @@ +#pragma once +#include "esphome/core/defines.h" + +#ifdef USE_SOCKET_IMPL_LWIP_SOCKETS + +#include +#include + +#include "esphome/core/helpers.h" +#include "headers.h" + +namespace esphome::socket { + +class LwIPSocketImpl { + public: + LwIPSocketImpl(int fd, bool monitor_loop = false); + ~LwIPSocketImpl(); + LwIPSocketImpl(const LwIPSocketImpl &) = delete; + LwIPSocketImpl &operator=(const LwIPSocketImpl &) = delete; + + int connect(const struct sockaddr *addr, socklen_t addrlen) { return lwip_connect(this->fd_, addr, addrlen); } + std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) { + int fd = lwip_accept(this->fd_, addr, addrlen); + if (fd == -1) + return {}; + return make_unique(fd, false); + } + std::unique_ptr accept_loop_monitored(struct sockaddr *addr, socklen_t *addrlen) { + int fd = lwip_accept(this->fd_, addr, addrlen); + if (fd == -1) + return {}; + return make_unique(fd, true); + } + + int bind(const struct sockaddr *addr, socklen_t addrlen) { return lwip_bind(this->fd_, addr, addrlen); } + int close(); + int shutdown(int how) { return lwip_shutdown(this->fd_, how); } + + int getpeername(struct sockaddr *addr, socklen_t *addrlen) { return lwip_getpeername(this->fd_, addr, addrlen); } + int getsockname(struct sockaddr *addr, socklen_t *addrlen) { return lwip_getsockname(this->fd_, addr, addrlen); } + + /// Format peer address into a fixed-size buffer (no heap allocation) + size_t getpeername_to(std::span buf); + /// Format local address into a fixed-size buffer (no heap allocation) + size_t getsockname_to(std::span buf); + + int getsockopt(int level, int optname, void *optval, socklen_t *optlen) { + return lwip_getsockopt(this->fd_, level, optname, optval, optlen); + } + int setsockopt(int level, int optname, const void *optval, socklen_t optlen) { + return lwip_setsockopt(this->fd_, level, optname, optval, optlen); + } + int listen(int backlog) { return lwip_listen(this->fd_, backlog); } + ssize_t read(void *buf, size_t len) { return lwip_read(this->fd_, buf, len); } + ssize_t recvfrom(void *buf, size_t len, sockaddr *addr, socklen_t *addr_len) { + return lwip_recvfrom(this->fd_, buf, len, 0, addr, addr_len); + } + ssize_t readv(const struct iovec *iov, int iovcnt) { return lwip_readv(this->fd_, iov, iovcnt); } + ssize_t write(const void *buf, size_t len) { return lwip_write(this->fd_, buf, len); } + ssize_t send(void *buf, size_t len, int flags) { return lwip_send(this->fd_, buf, len, flags); } + ssize_t writev(const struct iovec *iov, int iovcnt) { return lwip_writev(this->fd_, iov, iovcnt); } + ssize_t sendto(const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) { + return lwip_sendto(this->fd_, buf, len, flags, to, tolen); + } + int setblocking(bool blocking); + int loop() { return 0; } + + bool ready() const; + + int get_fd() const { return this->fd_; } + + protected: + int fd_{-1}; + bool closed_{false}; + bool loop_monitored_{false}; +}; + +} // namespace esphome::socket + +#endif // USE_SOCKET_IMPL_LWIP_SOCKETS diff --git a/esphome/components/socket/socket.cpp b/esphome/components/socket/socket.cpp index 6154c497e0..c04671c7ee 100644 --- a/esphome/components/socket/socket.cpp +++ b/esphome/components/socket/socket.cpp @@ -8,10 +8,10 @@ namespace esphome::socket { -Socket::~Socket() {} - #ifdef USE_SOCKET_SELECT_SUPPORT -bool Socket::ready() const { return !this->loop_monitored_ || App.is_socket_ready_(this->fd_); } +// Shared ready() implementation for fd-based socket implementations (BSD and LWIP sockets). +// Checks if the Application's select() loop has marked this fd as ready. +bool socket_ready_fd(int fd, bool loop_monitored) { return !loop_monitored || App.is_socket_ready_(fd); } #endif // Platform-specific inet_ntop wrappers @@ -81,26 +81,6 @@ size_t format_sockaddr_to(const struct sockaddr *addr_ptr, socklen_t len, std::s return 0; } -size_t Socket::getpeername_to(std::span buf) { - struct sockaddr_storage storage; - socklen_t len = sizeof(storage); - if (this->getpeername(reinterpret_cast(&storage), &len) != 0) { - buf[0] = '\0'; - return 0; - } - return format_sockaddr_to(reinterpret_cast(&storage), len, buf); -} - -size_t Socket::getsockname_to(std::span buf) { - struct sockaddr_storage storage; - socklen_t len = sizeof(storage); - if (this->getsockname(reinterpret_cast(&storage), &len) != 0) { - buf[0] = '\0'; - return 0; - } - return format_sockaddr_to(reinterpret_cast(&storage), len, buf); -} - std::unique_ptr socket_ip(int type, int protocol) { #if USE_NETWORK_IPV6 return socket(AF_INET6, type, protocol); @@ -109,11 +89,11 @@ std::unique_ptr socket_ip(int type, int protocol) { #endif /* USE_NETWORK_IPV6 */ } -std::unique_ptr socket_ip_loop_monitored(int type, int protocol) { +std::unique_ptr socket_ip_loop_monitored(int type, int protocol) { #if USE_NETWORK_IPV6 - return socket_loop_monitored(AF_INET6, type, protocol); + return socket_listen_loop_monitored(AF_INET6, type, protocol); #else - return socket_loop_monitored(AF_INET, type, protocol); + return socket_listen_loop_monitored(AF_INET, type, protocol); #endif /* USE_NETWORK_IPV6 */ } diff --git a/esphome/components/socket/socket.h b/esphome/components/socket/socket.h index a771e2fe1a..86a4f0cba9 100644 --- a/esphome/components/socket/socket.h +++ b/esphome/components/socket/socket.h @@ -7,87 +7,41 @@ #include "headers.h" #if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +// Include only the active implementation's header. +// SOCKADDR_STR_LEN is defined in headers.h. +#ifdef USE_SOCKET_IMPL_BSD_SOCKETS +#include "bsd_sockets_impl.h" +#elif defined(USE_SOCKET_IMPL_LWIP_SOCKETS) +#include "lwip_sockets_impl.h" +#elif defined(USE_SOCKET_IMPL_LWIP_TCP) +#include "lwip_raw_tcp_impl.h" +#endif + namespace esphome::socket { -// Maximum length for formatted socket address string (IP address without port) -// IPv4: "255.255.255.255" = 15 chars + null = 16 -// IPv6: full address = 45 chars + null = 46 -#if USE_NETWORK_IPV6 -static constexpr size_t SOCKADDR_STR_LEN = 46; // INET6_ADDRSTRLEN -#else -static constexpr size_t SOCKADDR_STR_LEN = 16; // INET_ADDRSTRLEN +// Type aliases — only one implementation is active per build. +// Socket is the concrete type for connected sockets. +// ListenSocket is the concrete type for listening/server sockets. +// On BSD and LWIP_SOCKETS, both aliases resolve to the same type. +// On LWIP_TCP, they are different types (no virtual dispatch between them). +#ifdef USE_SOCKET_IMPL_BSD_SOCKETS +using Socket = BSDSocketImpl; +using ListenSocket = BSDSocketImpl; +#elif defined(USE_SOCKET_IMPL_LWIP_SOCKETS) +using Socket = LwIPSocketImpl; +using ListenSocket = LwIPSocketImpl; +#elif defined(USE_SOCKET_IMPL_LWIP_TCP) +using Socket = LWIPRawImpl; +using ListenSocket = LWIPRawListenImpl; #endif -class Socket { - public: - Socket() = default; - virtual ~Socket(); - Socket(const Socket &) = delete; - Socket &operator=(const Socket &) = delete; - - virtual std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) = 0; - /// Accept a connection and monitor it in the main loop - /// NOTE: This function is NOT thread-safe and must only be called from the main loop - virtual std::unique_ptr accept_loop_monitored(struct sockaddr *addr, socklen_t *addrlen) { - return accept(addr, addrlen); // Default implementation for backward compatibility - } - virtual int bind(const struct sockaddr *addr, socklen_t addrlen) = 0; - virtual int close() = 0; - // not supported yet: - // virtual int connect(const std::string &address) = 0; -#if defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) - virtual int connect(const struct sockaddr *addr, socklen_t addrlen) = 0; -#endif - virtual int shutdown(int how) = 0; - - virtual int getpeername(struct sockaddr *addr, socklen_t *addrlen) = 0; - virtual int getsockname(struct sockaddr *addr, socklen_t *addrlen) = 0; - - /// Format peer address into a fixed-size buffer (no heap allocation) - /// Non-virtual wrapper around getpeername() - can be optimized away if unused - /// Returns number of characters written (excluding null terminator), or 0 on error - size_t getpeername_to(std::span buf); - /// Format local address into a fixed-size buffer (no heap allocation) - /// Non-virtual wrapper around getsockname() - can be optimized away if unused - size_t getsockname_to(std::span buf); - virtual int getsockopt(int level, int optname, void *optval, socklen_t *optlen) = 0; - virtual int setsockopt(int level, int optname, const void *optval, socklen_t optlen) = 0; - virtual int listen(int backlog) = 0; - virtual ssize_t read(void *buf, size_t len) = 0; - virtual ssize_t recvfrom(void *buf, size_t len, sockaddr *addr, socklen_t *addr_len) = 0; - virtual ssize_t readv(const struct iovec *iov, int iovcnt) = 0; - virtual ssize_t write(const void *buf, size_t len) = 0; - virtual ssize_t writev(const struct iovec *iov, int iovcnt) = 0; - virtual ssize_t sendto(const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) = 0; - - virtual int setblocking(bool blocking) = 0; - virtual int loop() { return 0; }; - - /// Get the underlying file descriptor (returns -1 if not supported) - /// Non-virtual: only one socket implementation is active per build. #ifdef USE_SOCKET_SELECT_SUPPORT - int get_fd() const { return this->fd_; } -#else - int get_fd() const { return -1; } +/// Shared ready() helper for fd-based socket implementations. +/// Checks if the Application's select() loop has marked this fd as ready. +bool socket_ready_fd(int fd, bool loop_monitored); #endif - /// Check if socket has data ready to read. Must only be called from the main loop thread. - /// For select()-based sockets: non-virtual, checks Application's select() results - /// For LWIP raw TCP sockets: virtual, checks internal buffer state -#ifdef USE_SOCKET_SELECT_SUPPORT - bool ready() const; -#else - virtual bool ready() const { return true; } -#endif - - protected: -#ifdef USE_SOCKET_SELECT_SUPPORT - int fd_{-1}; - bool closed_{false}; - bool loop_monitored_{false}; -#endif -}; - /// Create a socket of the given domain, type and protocol. std::unique_ptr socket(int domain, int type, int protocol); /// Create a socket in the newest available IP domain (IPv6 or IPv4) of the given type and protocol. @@ -100,7 +54,13 @@ std::unique_ptr socket_ip(int type, int protocol); /// NOTE: On ESP platforms, FD_SETSIZE is typically 10, limiting the number of monitored sockets. /// File descriptors >= FD_SETSIZE will not be monitored and will log an error. std::unique_ptr socket_loop_monitored(int domain, int type, int protocol); -std::unique_ptr socket_ip_loop_monitored(int type, int protocol); + +/// Create a listening socket of the given domain, type and protocol. +std::unique_ptr socket_listen(int domain, int type, int protocol); +/// Create a listening socket and monitor it for data in the main loop. +std::unique_ptr socket_listen_loop_monitored(int domain, int type, int protocol); +/// Create a listening socket in the newest available IP domain and monitor it. +std::unique_ptr socket_ip_loop_monitored(int type, int protocol); /// Set a sockaddr to the specified address and port for the IP version used by socket_ip(). /// @param addr Destination sockaddr structure diff --git a/esphome/core/application.h b/esphome/core/application.h index d2345e2b0b..659b60222d 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -105,7 +105,10 @@ #endif namespace esphome::socket { -class Socket; +#ifdef USE_SOCKET_SELECT_SUPPORT +/// Shared ready() helper for fd-based socket implementations. +bool socket_ready_fd(int fd, bool loop_monitored); // NOLINT(readability-redundant-declaration) +#endif } // namespace esphome::socket // Forward declarations for friend access from codegen-generated setup() @@ -520,7 +523,9 @@ class Application { protected: friend Component; - friend class socket::Socket; +#ifdef USE_SOCKET_SELECT_SUPPORT + friend bool socket::socket_ready_fd(int fd, bool loop_monitored); +#endif friend void ::setup(); friend void ::original_setup(); From a1d91ac779458545b0a6e5f014f57bc6d31d87ca Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 2 Mar 2026 06:59:23 -1000 Subject: [PATCH 20/20] [core] Compile-time detection of loop() overrides (#14405) --- esphome/core/application.cpp | 19 +++++++-------- esphome/core/application.h | 10 +++++++- esphome/core/component.cpp | 12 ---------- esphome/core/component.h | 7 ++++-- esphome/core/config.py | 36 ++++++++++++++++++++++++++++ esphome/cpp_helpers.py | 5 ++++ tests/unit_tests/test_cpp_helpers.py | 12 ++++++++-- 7 files changed, 73 insertions(+), 28 deletions(-) diff --git a/esphome/core/application.cpp b/esphome/core/application.cpp index c977fd66b3..26cd670629 100644 --- a/esphome/core/application.cpp +++ b/esphome/core/application.cpp @@ -79,7 +79,12 @@ static void insertion_sort_by_priority(Iterator first, Iterator last) { } } -void Application::register_component_(Component *comp) { this->components_.push_back(comp); } +void Application::register_component_impl_(Component *comp, bool has_loop) { + if (has_loop) { + comp->component_state_ |= COMPONENT_HAS_LOOP; + } + this->components_.push_back(comp); +} void Application::setup() { ESP_LOGI(TAG, "Running through setup()"); ESP_LOGV(TAG, "Sorting components by setup priority"); @@ -382,16 +387,8 @@ void Application::teardown_components(uint32_t timeout_ms) { } void Application::calculate_looping_components_() { - // Count total components that need looping - size_t total_looping = 0; - for (auto *obj : this->components_) { - if (obj->has_overridden_loop()) { - total_looping++; - } - } - - // Initialize FixedVector with exact size - no reallocation possible - this->looping_components_.init(total_looping); + // FixedVector capacity was pre-initialized by codegen with the exact count + // of components that override loop(), computed at C++ compile time. // Add all components with loop override that aren't already LOOP_DONE // Some components (like logger) may call disable_loop() during initialization diff --git a/esphome/core/application.h b/esphome/core/application.h index 659b60222d..44e8de7ee9 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "esphome/core/component.h" #include "esphome/core/defines.h" @@ -542,7 +543,14 @@ class Application { #endif #endif - void register_component_(Component *comp); + /// Register a component, detecting loop() override at compile time. + /// The template resolves &T::loop vs &Component::loop as a constexpr bool + /// and forwards it to register_component_impl_ which stores it in component_state_. + template void register_component_(T *comp) { + this->register_component_impl_(comp, !std::is_same_v); + } + + void register_component_impl_(Component *comp, bool has_loop); void calculate_looping_components_(); void add_looping_components_by_state_(bool match_loop_done); diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index 5afd901da2..a71aa8b3a3 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -496,18 +496,6 @@ void Component::set_setup_priority(float priority) { } #endif -bool Component::has_overridden_loop() const { -#if defined(USE_HOST) || defined(CLANG_TIDY) - return true; -#else -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpmf-conversions" - bool loop_overridden = (void *) (this->*(&Component::loop)) != (void *) (&Component::loop); -#pragma GCC diagnostic pop - return loop_overridden; -#endif -} - PollingComponent::PollingComponent(uint32_t update_interval) : update_interval_(update_interval) {} void PollingComponent::call_setup() { diff --git a/esphome/core/component.h b/esphome/core/component.h index 6b920da290..d8102ea670 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -76,6 +76,8 @@ inline constexpr uint8_t STATUS_LED_MASK = 0x18; inline constexpr uint8_t STATUS_LED_OK = 0x00; inline constexpr uint8_t STATUS_LED_WARNING = 0x08; inline constexpr uint8_t STATUS_LED_ERROR = 0x10; +// Component loop override flag uses bit 5 (set at registration time) +inline constexpr uint8_t COMPONENT_HAS_LOOP = 0x20; // Remove before 2026.8.0 enum class RetryResult { DONE, RETRY }; @@ -271,7 +273,7 @@ class Component { */ void status_momentary_error(const char *name, uint32_t length = 5000); - bool has_overridden_loop() const; + bool has_overridden_loop() const { return (this->component_state_ & COMPONENT_HAS_LOOP) != 0; } /** Set where this component was loaded from for some debug messages. * @@ -510,7 +512,8 @@ class Component { /// Bits 0-2: Component state (0x00=CONSTRUCTION, 0x01=SETUP, 0x02=LOOP, 0x03=FAILED, 0x04=LOOP_DONE) /// Bit 3: STATUS_LED_WARNING /// Bit 4: STATUS_LED_ERROR - /// Bits 5-7: Unused - reserved for future expansion + /// Bit 5: Has overridden loop() (set at registration time) + /// Bits 6-7: Unused - reserved for future expansion uint8_t component_state_{0x00}; volatile bool pending_enable_loop_{false}; ///< ISR-safe flag for enable_loop_soon_any_context }; diff --git a/esphome/core/config.py b/esphome/core/config.py index 593b402230..3835fd3875 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections import Counter import logging import os from pathlib import Path @@ -504,6 +505,40 @@ async def _add_controller_registry_define() -> None: cg.add_define("CONTROLLER_REGISTRY_MAX", controller_count) +@coroutine_with_priority(CoroPriority.FINAL) +async def _add_looping_components() -> None: + # Emit a constexpr that computes the looping component count at C++ compile time + # and pre-init the FixedVector with the exact capacity. Uses std::is_same_v to + # detect loop() overrides. The constexpr goes in main.cpp's global section where + # all component types are in scope. calculate_looping_components_() then skips + # the counting pass and only does the two population passes. + entries = CORE.data.get("looping_component_entries", []) + if not entries: + return + + # Build constexpr sum for the exact count, deduplicating by type + type_counts = Counter(entries) + terms = [ + f"({count} * !std::is_same_v)" + for cpp_type, count in type_counts.items() + ] + constexpr_expr = " + \\\n ".join(terms) + cg.add_global( + cg.RawStatement( + f"static constexpr size_t ESPHOME_LOOPING_COMPONENT_COUNT = \\\n" + f" {constexpr_expr};" + ) + ) + + # Pre-init FixedVector with exact capacity so calculate_looping_components_() + # can skip the counting pass + cg.add( + cg.RawExpression( + "App.looping_components_.init(ESPHOME_LOOPING_COMPONENT_COUNT)" + ) + ) + + @coroutine_with_priority(CoroPriority.CORE) async def to_code(config: ConfigType) -> None: cg.add_global(cg.global_ns.namespace("esphome").using) @@ -527,6 +562,7 @@ async def to_code(config: ConfigType) -> None: CORE.add_job(_add_platform_defines) CORE.add_job(_add_controller_registry_define) + CORE.add_job(_add_looping_components) CORE.add_job(_add_automations, config) diff --git a/esphome/cpp_helpers.py b/esphome/cpp_helpers.py index b673eaa7e1..8f8c693140 100644 --- a/esphome/cpp_helpers.py +++ b/esphome/cpp_helpers.py @@ -80,6 +80,11 @@ async def register_component(var, config): add(var.set_component_source(LogStringLiteral(name))) add(App.register_component_(var)) + + # Collect C++ type for compile-time looping component count + comp_entries = CORE.data.setdefault("looping_component_entries", []) + comp_entries.append(str(var.base.type)) + return var diff --git a/tests/unit_tests/test_cpp_helpers.py b/tests/unit_tests/test_cpp_helpers.py index 82ded409c7..5b6eed156f 100644 --- a/tests/unit_tests/test_cpp_helpers.py +++ b/tests/unit_tests/test_cpp_helpers.py @@ -14,7 +14,11 @@ async def test_gpio_pin_expression__conf_is_none(monkeypatch): @pytest.mark.asyncio async def test_register_component(monkeypatch): - var = Mock(base="foo.bar") + base_mock = Mock() + base_mock.__str__ = lambda self: "foo.bar" + base_mock.type = Mock() + base_mock.type.__str__ = lambda self: "foo::Bar" + var = Mock(base=base_mock) app_mock = Mock(register_component_=Mock(return_value=var)) monkeypatch.setattr(ch, "App", app_mock) @@ -46,7 +50,11 @@ async def test_register_component__no_component_id(monkeypatch): @pytest.mark.asyncio async def test_register_component__with_setup_priority(monkeypatch): - var = Mock(base="foo.bar") + base_mock = Mock() + base_mock.__str__ = lambda self: "foo.bar" + base_mock.type = Mock() + base_mock.type.__str__ = lambda self: "foo::Bar" + var = Mock(base=base_mock) app_mock = Mock(register_component_=Mock(return_value=var)) monkeypatch.setattr(ch, "App", app_mock)