diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 4b3df62ec4..88af5957e7 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -19,6 +19,7 @@ service APIConnection { rpc device_info (DeviceInfoRequest) returns (DeviceInfoResponse) { option (needs_authentication) = false; } + rpc device_capabilities (DeviceCapabilitiesRequest) returns (DeviceCapabilitiesResponse) {} rpc list_entities (ListEntitiesRequest) returns (void) {} rpc subscribe_states (SubscribeStatesRequest) returns (void) {} rpc subscribe_logs (SubscribeLogsRequest) returns (void) {} @@ -243,6 +244,12 @@ message SerialProxyInfo { // model = 127 (core/config.BOARD_MAX_LENGTH, validated in platform schemas) // project_name/project_version = 127 (core/config.PROJECT_MAX_LENGTH) // suggested_area = 120 (core/config.FRIENDLY_NAME_MAX_LEN via AREA_SCHEMA) +// +// Some fields below are marked "Superseded by DeviceCapabilitiesResponse". They +// have moved to that message as of API 1.15, but are still sent here so that +// older clients keep working. Do NOT mark them (deprecated) until the removal +// release: in this repo (deprecated) makes the generator drop the field +// entirely, so the device would stop sending it. message DeviceInfoResponse { option (id) = 10; option (source) = SOURCE_SERVER; @@ -280,6 +287,8 @@ message DeviceInfoResponse { // Deprecated in API version 1.9 uint32 legacy_bluetooth_proxy_version = 11 [deprecated=true, (field_ifdef) = "USE_BLUETOOTH_PROXY"]; + + // Superseded by DeviceCapabilitiesResponse.bluetooth_proxy as of API 1.15. uint32 bluetooth_proxy_feature_flags = 15 [(field_ifdef) = "USE_BLUETOOTH_PROXY"]; string manufacturer = 12 [(max_data_length) = 20, (force) = true]; @@ -288,11 +297,14 @@ message DeviceInfoResponse { // Deprecated in API version 1.10 uint32 legacy_voice_assistant_version = 14 [deprecated=true, (field_ifdef) = "USE_VOICE_ASSISTANT"]; + + // Superseded by DeviceCapabilitiesResponse.voice_assistant as of API 1.15. uint32 voice_assistant_feature_flags = 17 [(field_ifdef) = "USE_VOICE_ASSISTANT"]; string suggested_area = 16 [(max_data_length) = 120, (force) = true, (field_ifdef) = "USE_AREAS"]; // The Bluetooth mac address of the device. For example "AC:BC:32:89:0E:AA" + // Superseded by DeviceCapabilitiesResponse.bluetooth_proxy.mac_address as of API 1.15. string bluetooth_mac_address = 18 [(max_data_length) = 17, (force) = true, (field_ifdef) = "USE_BLUETOOTH_PROXY"]; // Supports receiving and saving api encryption key @@ -305,10 +317,13 @@ message DeviceInfoResponse { AreaInfo area = 22 [(field_ifdef) = "USE_AREAS"]; // Indicates if Z-Wave proxy support is available and features supported + // Superseded by DeviceCapabilitiesResponse.zwave_proxy as of API 1.15. uint32 zwave_proxy_feature_flags = 23 [(field_ifdef) = "USE_ZWAVE_PROXY"]; + // Superseded by DeviceCapabilitiesResponse.zwave_proxy as of API 1.15. uint32 zwave_home_id = 24 [(field_ifdef) = "USE_ZWAVE_PROXY"]; // Serial proxy instance metadata + // Superseded by DeviceCapabilitiesResponse.serial_proxies as of API 1.15. repeated SerialProxyInfo serial_proxies = 25 [(field_ifdef) = "USE_SERIAL_PROXY", (fixed_array_size_define) = "SERIAL_PROXY_COUNT"]; // Device is unprovisioned and accepts Noise handshakes with the well-known @@ -317,6 +332,63 @@ message DeviceInfoResponse { bool api_encryption_provisionable = 26 [(field_ifdef) = "USE_API_NOISE"]; } +// ==================== DEVICE CAPABILITIES ==================== + +// Asks the device which optional features it supports. +// +// This message exists so that DeviceInfoResponse does not have to keep growing +// a flat list of feature flags. DeviceInfoResponse is served before +// authentication, so it is limited to identity information. Capabilities are +// only served on an authenticated connection (encrypted as well, when +// encryption is configured). +// +// Clients that see api_version >= 1.15 should read these values from +// DeviceCapabilitiesResponse and ignore the matching DeviceInfoResponse fields. +// Older clients keep reading DeviceInfoResponse, which still carries the same +// values, so this is not a breaking change. +message DeviceCapabilitiesRequest { + option (id) = 149; + option (source) = SOURCE_CLIENT; + // Empty +} + +// Each feature gets its own sub-message so that it can gain fields over time +// without crowding the top-level field numbering. +// +// Note: a sub-message whose fields are all at their default value is not sent +// at all, so the presence of a sub-message is not a reliable test for "this +// feature is compiled in". Clients should test a value inside it, for example +// a non-zero feature_flags, exactly as they do today with DeviceInfoResponse. + +message BluetoothProxyCapabilities { + // Bitmask of the features this proxy supports + uint32 feature_flags = 1; + // The Bluetooth mac address of the device. For example "AC:BC:32:89:0E:AA" + string mac_address = 2 [(max_data_length) = 17, (force) = true]; +} + +message VoiceAssistantCapabilities { + // Bitmask of the features this voice assistant supports + uint32 feature_flags = 1; +} + +message ZWaveProxyCapabilities { + // Bitmask of the features this proxy supports + uint32 feature_flags = 1; + uint32 home_id = 2; +} + +message DeviceCapabilitiesResponse { + option (id) = 150; + option (source) = SOURCE_SERVER; + + BluetoothProxyCapabilities bluetooth_proxy = 1 [(field_ifdef) = "USE_BLUETOOTH_PROXY"]; + VoiceAssistantCapabilities voice_assistant = 2 [(field_ifdef) = "USE_VOICE_ASSISTANT"]; + ZWaveProxyCapabilities zwave_proxy = 3 [(field_ifdef) = "USE_ZWAVE_PROXY"]; + repeated SerialProxyInfo serial_proxies = 4 + [(field_ifdef) = "USE_SERIAL_PROXY", (fixed_array_size_define) = "SERIAL_PROXY_COUNT"]; +} + message ListEntitiesRequest { option (id) = 11; option (source) = SOURCE_CLIENT; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 2d03052d63..18eb2592ff 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1736,7 +1736,7 @@ bool APIConnection::send_hello_response_(const HelloRequest &msg) { HelloResponse resp; resp.api_version_major = 1; - resp.api_version_minor = 14; + resp.api_version_minor = 15; // Send only the version string - the client only logs this for debugging and doesn't use it otherwise resp.server_info = ESPHOME_VERSION_REF; resp.name = StringRef(App.get_name()); @@ -1904,6 +1904,35 @@ bool APIConnection::send_device_info_response_() { return this->send_message(resp); } +bool APIConnection::send_device_capabilities_response_() { + // These are the same values DeviceInfoResponse still reports for older clients. Keep the blocks + // below in sync with send_device_info_response_() until those copies are removed. + DeviceCapabilitiesResponse resp; +#ifdef USE_BLUETOOTH_PROXY + resp.bluetooth_proxy.feature_flags = bluetooth_proxy::global_bluetooth_proxy->get_feature_flags(); + char bluetooth_mac[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + bluetooth_proxy::global_bluetooth_proxy->get_bluetooth_mac_address_pretty(bluetooth_mac); + resp.bluetooth_proxy.mac_address = StringRef(bluetooth_mac); +#endif +#ifdef USE_VOICE_ASSISTANT + resp.voice_assistant.feature_flags = voice_assistant::global_voice_assistant->get_feature_flags(); +#endif +#ifdef USE_ZWAVE_PROXY + resp.zwave_proxy.feature_flags = zwave_proxy::global_zwave_proxy->get_feature_flags(); + resp.zwave_proxy.home_id = zwave_proxy::global_zwave_proxy->get_home_id(); +#endif +#ifdef USE_SERIAL_PROXY + size_t serial_proxy_index = 0; + for (auto const &proxy : App.get_serial_proxies()) { + if (serial_proxy_index >= SERIAL_PROXY_COUNT) + break; + auto &info = resp.serial_proxies[serial_proxy_index++]; + info.name = StringRef(proxy->get_name()); + info.port_type = proxy->get_port_type(); + } +#endif + return this->send_message(resp); +} void APIConnection::on_hello_request(const HelloRequest &msg) { if (!this->send_hello_response_(msg)) { this->on_fatal_error(); @@ -1925,6 +1954,11 @@ void APIConnection::on_device_info_request() { this->on_fatal_error(); } } +void APIConnection::on_device_capabilities_request() { + if (!this->send_device_capabilities_response_()) { + this->on_fatal_error(); + } +} #ifdef USE_API_HOMEASSISTANT_STATES void APIConnection::on_home_assistant_state_response(const HomeAssistantStateResponse &msg) { diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 7df7ea1429..9ca1b8b6a4 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -266,6 +266,7 @@ class APIConnection final : public APIServerConnectionBase { void on_disconnect_request(const DisconnectRequest &msg); void on_ping_request(); void on_device_info_request(); + void on_device_capabilities_request(); void on_list_entities_request() { this->begin_iterator_(ActiveIterator::LIST_ENTITIES); } void on_subscribe_states_request() { this->flags_.state_subscription = true; @@ -385,6 +386,7 @@ class APIConnection final : public APIServerConnectionBase { bool send_disconnect_response_(); bool send_ping_response_(); bool send_device_info_response_(); + bool send_device_capabilities_response_(); #ifdef USE_API_NOISE bool send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg); #endif diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h index 9cae6ba92e..9c49956bbd 100644 --- a/esphome/components/api/api_frame_helper.h +++ b/esphome/components/api/api_frame_helper.h @@ -149,7 +149,7 @@ class APIFrameHelper { // holding data too long waiting for Nagle's timer causes buffer exhaustion // and dropped messages. // - // ESP32 (TCP_SND_BUF=4×MSS+) / RP2040 (8×MSS) / LibreTiny (4×MSS): 4 logs per cycle + // ESP32 (TCP_SND_BUF=4×MSS+) / RP2040 (4×MSS) / LibreTiny (4×MSS): 4 logs per cycle // ESP8266 (2×MSS): 3 logs per cycle (tightest buffers) // // Flow (ESP32/RP2040/LT): Log 1 (Nagle on) -> Log 2 -> Log 3 -> Log 4 (NODELAY, flush) @@ -312,7 +312,7 @@ class APIFrameHelper { // Values 1..LOG_NAGLE_COUNT count log messages in the current Nagle batch. // After LOG_NAGLE_COUNT logs, we flush by re-enabling NODELAY and resetting to 0. // ESP8266 has the tightest TCP send buffer (2×MSS) and needs conservative batching. - // ESP32 (4×MSS+), RP2040 (8×MSS), and LibreTiny (4×MSS) can coalesce more. + // ESP32 (4×MSS+), RP2040 (4×MSS), and LibreTiny (4×MSS) can coalesce more. #ifdef USE_ESP8266 static constexpr uint8_t LOG_NAGLE_COUNT = 2; #else diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 190bd32425..5776ec5c62 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -241,6 +241,82 @@ uint32_t DeviceInfoResponse::calculate_size() const { #endif return size; } +#ifdef USE_BLUETOOTH_PROXY +uint8_t *BluetoothProxyCapabilities::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { + uint8_t *__restrict__ pos = buffer.get_pos(); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 1, this->feature_flags); + ProtoEncode::encode_short_string_force(pos PROTO_ENCODE_DEBUG_ARG, 18, this->mac_address); + return pos; +} +uint32_t BluetoothProxyCapabilities::calculate_size() const { + uint32_t size = 0; + size += ProtoSize::calc_uint32(1, this->feature_flags); + size += 2 + this->mac_address.size(); + return size; +} +#endif +#ifdef USE_VOICE_ASSISTANT +uint8_t *VoiceAssistantCapabilities::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { + uint8_t *__restrict__ pos = buffer.get_pos(); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 1, this->feature_flags); + return pos; +} +uint32_t VoiceAssistantCapabilities::calculate_size() const { + uint32_t size = 0; + size += ProtoSize::calc_uint32(1, this->feature_flags); + return size; +} +#endif +#ifdef USE_ZWAVE_PROXY +uint8_t *ZWaveProxyCapabilities::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { + uint8_t *__restrict__ pos = buffer.get_pos(); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 1, this->feature_flags); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 2, this->home_id); + return pos; +} +uint32_t ZWaveProxyCapabilities::calculate_size() const { + uint32_t size = 0; + size += ProtoSize::calc_uint32(1, this->feature_flags); + size += ProtoSize::calc_uint32(1, this->home_id); + return size; +} +#endif +uint8_t *DeviceCapabilitiesResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { + uint8_t *__restrict__ pos = buffer.get_pos(); +#ifdef USE_BLUETOOTH_PROXY + ProtoEncode::encode_optional_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 1, this->bluetooth_proxy); +#endif +#ifdef USE_VOICE_ASSISTANT + ProtoEncode::encode_optional_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 2, this->voice_assistant); +#endif +#ifdef USE_ZWAVE_PROXY + ProtoEncode::encode_optional_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 3, this->zwave_proxy); +#endif +#ifdef USE_SERIAL_PROXY + for (const auto &it : this->serial_proxies) { + ProtoEncode::encode_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 4, it); + } +#endif + return pos; +} +uint32_t DeviceCapabilitiesResponse::calculate_size() const { + uint32_t size = 0; +#ifdef USE_BLUETOOTH_PROXY + size += ProtoSize::calc_message(1, this->bluetooth_proxy.calculate_size()); +#endif +#ifdef USE_VOICE_ASSISTANT + size += ProtoSize::calc_message(1, this->voice_assistant.calculate_size()); +#endif +#ifdef USE_ZWAVE_PROXY + size += ProtoSize::calc_message(1, this->zwave_proxy.calculate_size()); +#endif +#ifdef USE_SERIAL_PROXY + for (const auto &it : this->serial_proxies) { + size += ProtoSize::calc_message_force(1, it.calculate_size()); + } +#endif + return size; +} #ifdef USE_BINARY_SENSOR uint8_t *ListEntitiesBinarySensorResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { uint8_t *__restrict__ pos = buffer.get_pos(); diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 4d5866da0b..f35f551060 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -600,6 +600,74 @@ class DeviceInfoResponse final : public ProtoMessage { protected: }; +#ifdef USE_BLUETOOTH_PROXY +class BluetoothProxyCapabilities final : public ProtoMessage { + public: + uint32_t feature_flags{0}; + StringRef mac_address{}; + uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; + uint32_t calculate_size() const; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: +}; +#endif +#ifdef USE_VOICE_ASSISTANT +class VoiceAssistantCapabilities final : public ProtoMessage { + public: + uint32_t feature_flags{0}; + uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; + uint32_t calculate_size() const; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: +}; +#endif +#ifdef USE_ZWAVE_PROXY +class ZWaveProxyCapabilities final : public ProtoMessage { + public: + uint32_t feature_flags{0}; + uint32_t home_id{0}; + uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; + uint32_t calculate_size() const; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: +}; +#endif +class DeviceCapabilitiesResponse final : public ProtoMessage { + public: + static constexpr uint8_t MESSAGE_TYPE = 150; + static constexpr uint8_t ESTIMATED_SIZE = 102; +#ifdef HAS_PROTO_MESSAGE_DUMP + const LogString *message_name() const override { return LOG_STR("device_capabilities_response"); } +#endif +#ifdef USE_BLUETOOTH_PROXY + BluetoothProxyCapabilities bluetooth_proxy{}; +#endif +#ifdef USE_VOICE_ASSISTANT + VoiceAssistantCapabilities voice_assistant{}; +#endif +#ifdef USE_ZWAVE_PROXY + ZWaveProxyCapabilities zwave_proxy{}; +#endif +#ifdef USE_SERIAL_PROXY + std::array serial_proxies{}; +#endif + uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; + uint32_t calculate_size() const; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: +}; class ListEntitiesDoneResponse final : public ProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 19; diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 09570b09e4..17ce7fba45 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -988,6 +988,55 @@ const char *DeviceInfoResponse::dump_to(DumpBuffer &out) const { #endif return out.c_str(); } +#ifdef USE_BLUETOOTH_PROXY +const char *BluetoothProxyCapabilities::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("BluetoothProxyCapabilities")); + dump_field(out, ESPHOME_PSTR("feature_flags"), this->feature_flags); + dump_field(out, ESPHOME_PSTR("mac_address"), this->mac_address); + return out.c_str(); +} +#endif +#ifdef USE_VOICE_ASSISTANT +const char *VoiceAssistantCapabilities::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("VoiceAssistantCapabilities")); + dump_field(out, ESPHOME_PSTR("feature_flags"), this->feature_flags); + return out.c_str(); +} +#endif +#ifdef USE_ZWAVE_PROXY +const char *ZWaveProxyCapabilities::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("ZWaveProxyCapabilities")); + dump_field(out, ESPHOME_PSTR("feature_flags"), this->feature_flags); + dump_field(out, ESPHOME_PSTR("home_id"), this->home_id); + return out.c_str(); +} +#endif +const char *DeviceCapabilitiesResponse::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("DeviceCapabilitiesResponse")); +#ifdef USE_BLUETOOTH_PROXY + out.append(2, ' ').append_p(ESPHOME_PSTR("bluetooth_proxy")).append(": "); + this->bluetooth_proxy.dump_to(out); + out.append("\n"); +#endif +#ifdef USE_VOICE_ASSISTANT + out.append(2, ' ').append_p(ESPHOME_PSTR("voice_assistant")).append(": "); + this->voice_assistant.dump_to(out); + out.append("\n"); +#endif +#ifdef USE_ZWAVE_PROXY + out.append(2, ' ').append_p(ESPHOME_PSTR("zwave_proxy")).append(": "); + this->zwave_proxy.dump_to(out); + out.append("\n"); +#endif +#ifdef USE_SERIAL_PROXY + for (const auto &it : this->serial_proxies) { + out.append(4, ' ').append_p(ESPHOME_PSTR("serial_proxies")).append(": "); + it.dump_to(out); + out.append("\n"); + } +#endif + return out.c_str(); +} const char *ListEntitiesDoneResponse::dump_to(DumpBuffer &out) const { out.append_p(ESPHOME_PSTR("ListEntitiesDoneResponse {}")); return out.c_str(); diff --git a/esphome/components/api/api_pb2_service.cpp b/esphome/components/api/api_pb2_service.cpp index 5c9df433dd..19dcbfb77c 100644 --- a/esphome/components/api/api_pb2_service.cpp +++ b/esphome/components/api/api_pb2_service.cpp @@ -705,6 +705,13 @@ void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const ui break; } #endif + case 149 /* DeviceCapabilitiesRequest is empty */: { +#ifdef HAS_PROTO_MESSAGE_DUMP + this->log_receive_message_(LOG_STR("on_device_capabilities_request")); +#endif + this->on_device_capabilities_request(); + break; + } default: break; } diff --git a/esphome/components/api/api_pb2_service.h b/esphome/components/api/api_pb2_service.h index d1b51f4846..5ed78b3385 100644 --- a/esphome/components/api/api_pb2_service.h +++ b/esphome/components/api/api_pb2_service.h @@ -27,6 +27,8 @@ class APIServerConnectionBase { void on_ping_response(){}; void on_device_info_request(){}; + void on_device_capabilities_request(){}; + void on_list_entities_request(){}; void on_subscribe_states_request(){}; diff --git a/esphome/components/rp2/__init__.py b/esphome/components/rp2/__init__.py index 1bf01e6828..ea37134e27 100644 --- a/esphome/components/rp2/__init__.py +++ b/esphome/components/rp2/__init__.py @@ -351,6 +351,11 @@ async def to_code(config): ], ) + # newlib-nano is the default libc for the arduino-pico toolchain and its + # printf silently drops %f unless _printf_float is force-linked. Components + # use %f widely in logging, so pull it in. + cg.add_build_flag("-Wl,-u,_printf_float") + # Wrap FILE*-based printf functions to eliminate newlib's _vfprintf_r # (~9.2 KB). See printf_stubs.cpp for implementation. if config.get(CONF_ENABLE_FULL_PRINTF): @@ -402,25 +407,39 @@ def _configure_lwip() -> None: ──────────────────────────────────────────────────────────────── TCP_SND_BUF 2×MSS 4×MSS 8×MSS 4×MSS TCP_WND 4×MSS 4×MSS 8×MSS 4×MSS + TCP_SND_QUEUELEN ~8 17 32 17 MEM_LIBC_MALLOC 1 1 0 0* MEMP_MEM_MALLOC 1 1 0 0** - MEM_SIZE N/A*** N/A*** 16KB 16KB + MEM_SIZE N/A*** N/A*** 16KB 32KB PBUF_POOL_SIZE 10 16 24 16 - MEMP_NUM_TCP_SEG 10 16 32 17 + MEMP_NUM_TCP_SEG 10 16 32 34**** MEMP_NUM_TCP_PCB 5 16 5 dynamic - MEMP_NUM_TCP_PCB_LISTEN 4 16 8**** dynamic + MEMP_NUM_TCP_PCB_LISTEN 4 16 8***** dynamic MEMP_NUM_UDP_PCB 4 16 7 dynamic - TCP_SND_QUEUELEN ~8 17 32 17 * MEM_LIBC_MALLOC must stay 0: arduino-pico uses PICO_CYW43_ARCH_THREADSAFE_BACKGROUND which runs lwIP callbacks from a low-priority pendsv IRQ. The pico-sdk explicitly blocks MEM_LIBC_MALLOC=1 because libc malloc uses mutexes (unsafe in IRQ). - ** MEMP_MEM_MALLOC must stay 0: the dedicated lwIP heap (MEM_SIZE=16KB) - is too small to hold all pools dynamically. The PBUF_POOL alone needs - ~24KB (16 × 1524 bytes). Increasing MEM_SIZE would negate BSS savings. - *** ESP8266/ESP32 use MEM_LIBC_MALLOC=1 (system heap, no dedicated pool). - **** opt.h default; arduino-pico doesn't override MEMP_NUM_TCP_PCB_LISTEN. + ** MEMP_MEM_MALLOC must stay 0, and the reason is IRQ safety, not size. + memp_malloc() pops the pool free list inside SYS_ARCH_PROTECT, but + lwIP's heap picks its protection from LWIP_ALLOW_MEM_FREE_FROM_OTHER_ + CONTEXT (default 0), so under NO_SYS=1 mem_malloc()/mem_free() run + with no protection at all. Setting MEMP_MEM_MALLOC=1 also moves the + allocation outside the guard entirely (memp.c calls mem_malloc before + SYS_ARCH_PROTECT). RX pbufs would then be allocated from the pendsv + IRQ on the same unguarded free list the main loop uses for + tcp_write(), which corrupts the heap. Tried on hardware; it faults + within seconds on CYW43. Ethernet survives only because that path + polls from the main loop and never runs a second context. + *** ESP8266/ESP32 use MEM_LIBC_MALLOC=1 (system heap, no dedicated + pool), so their MEMP_NUM_* and PBUF_POOL_SIZE values are not caps. + Ours are hard limits; do not copy their numbers without that in mind. + **** MEMP_NUM_TCP_SEG is a *global* pool while TCP_SND_QUEUELEN is + *per-PCB*, so it must not be sized to the per-PCB value: one busy + connection would drain it for every other PCB. 2× covers two PCBs at + full depth, and MEM_SIZE is the real backing limit past that. + ***** opt.h default; arduino-pico doesn't override MEMP_NUM_TCP_PCB_LISTEN. "dynamic" = auto-calculated from component socket registrations via socket.get_socket_counts() with minimums of 8 TCP / 6 UDP / 2 TCP_LISTEN. """ @@ -446,35 +465,42 @@ def _configure_lwip() -> None: # TCP_WND: receive window. 4×MSS matches ESP32. Down from arduino-pico's 8×MSS. tcp_wnd = "(4*TCP_MSS)" - # TCP_SND_QUEUELEN: max pbufs queued for send buffer + # TCP_SND_QUEUELEN: max pbufs queued per PCB for the send buffer # ESP-IDF formula: (4 * TCP_SND_BUF + (TCP_MSS - 1)) / TCP_MSS # With 4×MSS: (4*5840 + 1459) / 1460 = 17 — match ESP32 tcp_snd_queuelen = 17 - # MEMP_NUM_TCP_SEG: segment pool, must be >= TCP_SND_QUEUELEN (lwIP sanity check) - memp_num_tcp_seg = tcp_snd_queuelen + # MEMP_NUM_TCP_SEG: segment pool shared by every PCB, so it cannot be the + # per-PCB queue length — lwIP's sanity check only demands >=, which is the + # floor for a single connection. 2× lets two PCBs queue fully before the + # rest start seeing ERR_MEM. Each entry is ~16 bytes. + memp_num_tcp_seg = 2 * tcp_snd_queuelen # PBUF_POOL_SIZE: RP2040 has 264KB RAM, more generous than LibreTiny. - # 16 matches ESP32 (vs arduino-pico's 24). With MEMP_MEM_MALLOC=1, - # this is a max count (allocated on demand from heap). + # 16 matches ESP32 (vs arduino-pico's 24). Receive side only; the send + # path copies into PBUF_RAM out of MEM_SIZE. pbuf_pool_size = 16 + # MEM_SIZE: the lwIP heap that backs PBUF_RAM, which is where tcp_write() + # copies outgoing data. TCP_OVERSIZE defaults to TCP_MSS, so one PCB + # holding a full TCP_SND_BUF pins ~4 × 1.5KB. At arduino-pico's 16KB, two + # busy connections exhausted it while api's max_connections on rp2 is 4; + # that surfaces as ERR_MEM from tcp_write(), then a refused send_message(). + # 32KB is arduino-pico's own next tier (its __LWIP_MEMMULT=2 boards). + # Must stay under 64000 or lwIP widens mem_size_t to u32_t. + mem_size = 32768 + # Build the lwIP override defines for the Jinja2 template. # The template uses #include_next to chain to the framework's original # lwipopts.h, then #undef/#define only the values we need to change. # - # Note: MEMP_MEM_MALLOC stays 0 (framework default). While the memp - # allocations use the dedicated lwIP heap (IRQ-safe), the 16KB MEM_SIZE - # is too small to hold all pools dynamically under stress. The PBUF_POOL - # alone needs ~24KB (16 × 1524 bytes). Increasing MEM_SIZE would negate - # the BSS savings. - # - # MEM_LIBC_MALLOC stays 0 (framework default): arduino-pico uses - # PICO_CYW43_ARCH_THREADSAFE_BACKGROUND which runs lwIP callbacks from - # a low-priority pendsv IRQ where libc malloc (mutex-based) is unsafe. + # Both malloc flags stay 0 (framework defaults); see the docstring. The + # static pools are the only IRQ-safe allocator on this platform, so the + # fix is to size them correctly rather than to make them dynamic. lwip_defines: dict[str, str] = { "TCP_SND_BUF": tcp_snd_buf, "TCP_WND": tcp_wnd, "TCP_SND_QUEUELEN": str(tcp_snd_queuelen), + "MEM_SIZE": str(mem_size), "MEMP_NUM_TCP_SEG": str(memp_num_tcp_seg), "PBUF_POOL_SIZE": str(pbuf_pool_size), "MEMP_NUM_TCP_PCB": str(tcp_sockets), @@ -495,7 +521,8 @@ def _configure_lwip() -> None: udp_min = " (min)" if udp_sockets > sc.udp else "" listen_min = " (min)" if listening_tcp > sc.tcp_listen else "" _LOGGER.info( - "Configuring lwIP: TCP=%d%s [%s], UDP=%d%s [%s], TCP_LISTEN=%d%s [%s]", + "Configuring lwIP: %d byte heap; TCP=%d%s [%s], UDP=%d%s [%s], TCP_LISTEN=%d%s [%s]", + mem_size, tcp_sockets, tcp_min, sc.tcp_details, diff --git a/esphome/components/rp2/lwipopts.h.jinja b/esphome/components/rp2/lwipopts.h.jinja index 36d7d4da14..6563489d2e 100644 --- a/esphome/components/rp2/lwipopts.h.jinja +++ b/esphome/components/rp2/lwipopts.h.jinja @@ -20,13 +20,22 @@ #undef TCP_WND #define TCP_WND {{ TCP_WND }} -// Queued segment limits: derived from 4xMSS buffer size, matching ESP32 +// Per-PCB send queue: derived from 4xMSS buffer size, matching ESP32 #undef TCP_SND_QUEUELEN #define TCP_SND_QUEUELEN {{ TCP_SND_QUEUELEN }} +// Segment pool: global across every PCB, so it is sized above the per-PCB +// queue length rather than equal to it. lwIP's sanity check only requires +// >= TCP_SND_QUEUELEN, which is the floor for a single connection. #undef MEMP_NUM_TCP_SEG #define MEMP_NUM_TCP_SEG {{ MEMP_NUM_TCP_SEG }} +// lwIP heap backing PBUF_RAM, which is what tcp_write() copies into. +// Raised from arduino-pico's 16KB: TCP_OVERSIZE is TCP_MSS, so a single PCB +// at a full TCP_SND_BUF pins about 6KB and two connections exhausted it. +#undef MEM_SIZE +#define MEM_SIZE {{ MEM_SIZE }} + // Packet buffer pool: 16 matches ESP32 (down from 24) #undef PBUF_POOL_SIZE #define PBUF_POOL_SIZE {{ PBUF_POOL_SIZE }} diff --git a/tests/unit_tests/components/api/test_api_proto.py b/tests/unit_tests/components/api/test_api_proto.py new file mode 100644 index 0000000000..35aa5ff529 --- /dev/null +++ b/tests/unit_tests/components/api/test_api_proto.py @@ -0,0 +1,371 @@ +"""Invariant tests for esphome/components/api/api.proto and its generated code. + +These guard the DeviceCapabilitiesRequest/DeviceCapabilitiesResponse addition +(API 1.15) against regressions that protoc-based codegen would not catch on +its own, without requiring protoc to be installed at test time: + +* script/api_protobuf/api_protobuf.py skips any field marked + `[deprecated = true]` completely -- it generates no C++ for it at all, so + the device silently stops sending that value. Six DeviceInfoResponse fields + were superseded by DeviceCapabilitiesResponse but must keep being sent for + backward compatibility with clients older than API 1.15. If a future edit + "tidies up" by marking one of them deprecated, this file breaks that field + for every existing client with nothing else in CI noticing. +* Field numbers are the wire protocol, not the field names. Renaming a field + is harmless; renumbering it is a silent breaking change, because an old + client still decodes by number. This file pins the field number of each of + the six superseded DeviceInfoResponse fields and of every field on the new + DeviceCapabilitiesResponse/BluetoothProxyCapabilities/ + VoiceAssistantCapabilities/ZWaveProxyCapabilities sub-messages, so a + well-intentioned reshuffle of api.proto gets caught here instead of on a + device in the field. +* Message wire ids must be unique, and the new capabilities RPC must stay + authenticated-only. + +Group A below asserts on the checked-in generated files (api_pb2.h / +api_pb2.cpp), since "the field is present in the generated C++" is exactly +equivalent to "the device still sends it". Group B parses api.proto as plain +text (no protoc). Group C checks the advertised API minor version. +""" + +from __future__ import annotations + +from pathlib import Path +import re + +import esphome + +API_DIR = Path(esphome.__file__).parent / "components" / "api" + +PROTO_TEXT = (API_DIR / "api.proto").read_text(encoding="utf-8") +HEADER_TEXT = (API_DIR / "api_pb2.h").read_text(encoding="utf-8") +CPP_TEXT = (API_DIR / "api_pb2.cpp").read_text(encoding="utf-8") +API_CONNECTION_TEXT = (API_DIR / "api_connection.cpp").read_text(encoding="utf-8") + +# Fields on DeviceInfoResponse that were superseded by DeviceCapabilitiesResponse +# as of API 1.15 but must still be generated (and therefore still sent) for +# backward compatibility with older clients. +SUPERSEDED_FIELDS: dict[str, int] = { + "bluetooth_proxy_feature_flags": 15, + "voice_assistant_feature_flags": 17, + "bluetooth_mac_address": 18, + "zwave_proxy_feature_flags": 23, + "zwave_home_id": 24, + "serial_proxies": 25, +} + +# Field numbers on the new capability messages. These are a frozen wire +# contract from the moment they ship: an old client decodes a sub-message +# field purely by number, so renumbering any of these -- even without +# touching a name -- silently corrupts what every already-deployed client +# reads. Keyed by message name so the next capability sub-message is a +# data-only addition here. +NEW_CAPABILITY_FIELDS: dict[str, dict[str, int]] = { + "DeviceCapabilitiesResponse": { + "bluetooth_proxy": 1, + "voice_assistant": 2, + "zwave_proxy": 3, + "serial_proxies": 4, + }, + "BluetoothProxyCapabilities": { + "feature_flags": 1, + "mac_address": 2, + }, + "VoiceAssistantCapabilities": { + "feature_flags": 1, + }, + "ZWaveProxyCapabilities": { + "feature_flags": 1, + "home_id": 2, + }, +} + +# Fields that are genuinely dead and are expected to carry `deprecated=true`. +# Used to prove the deprecated-detection logic below actually detects +# deprecation rather than trivially passing. +GENUINELY_DEPRECATED_FIELDS: tuple[str, ...] = ( + "legacy_bluetooth_proxy_version", + "legacy_voice_assistant_version", +) + +DEPRECATED_FIELD_TRAP = ( + "script/api_protobuf/api_protobuf.py skips fields marked `[deprecated = " + "true]` completely, generating no C++ for them at all. Marking this field " + "deprecated would silently stop the device from ever sending it, breaking " + "every existing client that still reads it from DeviceInfoResponse." +) + + +def _extract_braced_region(text: str, anchor_pattern: str) -> str: + """Return the region of `text` starting at the first match of + `anchor_pattern` up to the matching closing brace (inclusive), using + brace-depth counting so nested braces (e.g. a `for (...) { ... }` loop + inside a function body) don't cause a premature stop. + """ + anchor_match = re.search(anchor_pattern, text) + if anchor_match is None: + raise AssertionError(f"could not find a match for {anchor_pattern!r}") + start = anchor_match.start() + open_brace = text.index("{", start) + depth = 0 + for i in range(open_brace, len(text)): + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + return text[start : i + 1] + raise AssertionError(f"unbalanced braces while scanning after {anchor_pattern!r}") + + +def _extract_class_body(header_text: str, class_name: str) -> str: + """Return the body of a generated C++ class, scoped so a field name that + also happens to exist on some other class cannot satisfy the assertion. + """ + return _extract_braced_region(header_text, rf"class {re.escape(class_name)}\b") + + +def _extract_function_body(cpp_text: str, qualified_name: str) -> str: + """Return the body of a generated `Class::method(...)` definition.""" + return _extract_braced_region(cpp_text, rf"{re.escape(qualified_name)}\(") + + +def _extract_proto_message(proto_text: str, message_name: str) -> str: + """Return the body of a top-level `message Name { ... }` block from the + .proto source. Proto message bodies here contain no nested `{`/`}` of + their own (options use parens, not braces), so a non-greedy match up to + the first line that is just `}` is sufficient and keeps the parsing + simple. + """ + match = re.search( + rf"^message {re.escape(message_name)}\s*\{{(.*?)^\}}", + proto_text, + re.MULTILINE | re.DOTALL, + ) + if match is None: + raise AssertionError(f"could not find `message {message_name}` in api.proto") + return match.group(1) + + +def _extract_rpc_body(proto_text: str, rpc_name: str) -> str: + """Return the option body of an `rpc name (...) returns (...) { ... }` + declaration from the APIConnection service, robust to it being written + on one line (`{}`) or spread across several with options inside. + """ + match = re.search( + rf"rpc\s+{re.escape(rpc_name)}\s*\([^)]*\)\s*returns\s*\([^)]*\)\s*\{{(.*?)\}}", + proto_text, + re.DOTALL, + ) + if match is None: + raise AssertionError(f"could not find `rpc {rpc_name}` in api.proto") + return match.group(1) + + +def _field_declaration_line(message_body: str, field_name: str) -> str: + """Return the single source line declaring `field_name` inside a proto + message body (all fields here are declared on one line). + """ + for line in message_body.splitlines(): + if re.search(rf"\b{re.escape(field_name)}\s*=\s*\d+", line): + return line + raise AssertionError( + f"could not find a field declaration for {field_name!r} in the given message body" + ) + + +# ==================== Group A: generated files ==================== + + +def test_superseded_device_info_fields_still_declared_in_header() -> None: + """Each superseded field must still be a real member of DeviceInfoResponse + in api_pb2.h -- not merely present somewhere in the file. Several of these + names (e.g. serial_proxies) also exist on DeviceCapabilitiesResponse, so an + unscoped substring search over the whole header would pass even if the + field were removed from DeviceInfoResponse. + """ + class_body = _extract_class_body(HEADER_TEXT, "DeviceInfoResponse") + for field_name in SUPERSEDED_FIELDS: + assert re.search(rf"\b{field_name}\b", class_body), ( + f"{field_name} is missing from the DeviceInfoResponse class body in " + f"api_pb2.h. {DEPRECATED_FIELD_TRAP}" + ) + + +def test_superseded_device_info_fields_still_encoded_and_sized() -> None: + """Each superseded field must still be touched by DeviceInfoResponse's + generated encode() and calculate_size(), i.e. it is still put on the wire. + """ + encode_body = _extract_function_body(CPP_TEXT, "DeviceInfoResponse::encode") + size_body = _extract_function_body(CPP_TEXT, "DeviceInfoResponse::calculate_size") + for field_name in SUPERSEDED_FIELDS: + assert f"this->{field_name}" in encode_body, ( + f"DeviceInfoResponse::encode() no longer references {field_name}. " + f"{DEPRECATED_FIELD_TRAP}" + ) + assert f"this->{field_name}" in size_body, ( + f"DeviceInfoResponse::calculate_size() no longer references " + f"{field_name}. {DEPRECATED_FIELD_TRAP}" + ) + + +def test_new_capability_classes_present_in_header() -> None: + """The new response message and its capability sub-messages must exist as + generated classes. + """ + for class_name in ( + "DeviceCapabilitiesResponse", + "BluetoothProxyCapabilities", + "VoiceAssistantCapabilities", + "ZWaveProxyCapabilities", + ): + assert re.search(rf"class {re.escape(class_name)}\b", HEADER_TEXT), ( + f"expected a generated class named {class_name} in api_pb2.h" + ) + + +# ==================== Group B: api.proto source text ==================== + + +def test_all_message_ids_are_unique() -> None: + """Every `option (id) = N;` in api.proto must be unique. Two messages + sharing a wire id would make the client and server misinterpret each + other's messages -- nothing else currently checks this. + """ + ids = [int(value) for value in re.findall(r"option \(id\) = (\d+);", PROTO_TEXT)] + assert ids, "did not find any `option (id) = N;` declarations in api.proto" + duplicates = sorted({value for value in ids if ids.count(value) > 1}) + assert not duplicates, ( + f"Duplicate `option (id)` values found in api.proto: {duplicates}. Each " + "message must have a unique wire id." + ) + + +def test_device_capabilities_request_has_id_149() -> None: + body = _extract_proto_message(PROTO_TEXT, "DeviceCapabilitiesRequest") + match = re.search(r"option \(id\) = (\d+);", body) + assert match is not None, "DeviceCapabilitiesRequest is missing `option (id)`" + assert int(match.group(1)) == 149, ( + f"DeviceCapabilitiesRequest has id {match.group(1)}, expected 149. " + "Message ids are part of the wire protocol and must not change once " + "assigned." + ) + + +def test_device_capabilities_response_has_id_150() -> None: + body = _extract_proto_message(PROTO_TEXT, "DeviceCapabilitiesResponse") + match = re.search(r"option \(id\) = (\d+);", body) + assert match is not None, "DeviceCapabilitiesResponse is missing `option (id)`" + assert int(match.group(1)) == 150, ( + f"DeviceCapabilitiesResponse has id {match.group(1)}, expected 150. " + "Message ids are part of the wire protocol and must not change once " + "assigned." + ) + + +def test_superseded_fields_are_not_marked_deprecated_in_proto() -> None: + """The six superseded fields must not carry `[deprecated = true]` in + api.proto, or the generator drops them and old clients stop receiving + them (see module docstring). The second half of this test proves the + deprecated-detection itself works: two genuinely dead fields + (legacy_bluetooth_proxy_version, legacy_voice_assistant_version) must + still be detected as deprecated, so the first half isn't vacuously true. + """ + body = _extract_proto_message(PROTO_TEXT, "DeviceInfoResponse") + + for field_name in SUPERSEDED_FIELDS: + line = _field_declaration_line(body, field_name) + assert "deprecated" not in line, ( + f"{field_name} in DeviceInfoResponse is marked deprecated in " + f"api.proto ({line.strip()!r}). {DEPRECATED_FIELD_TRAP}" + ) + + for field_name in GENUINELY_DEPRECATED_FIELDS: + line = _field_declaration_line(body, field_name) + assert "deprecated" in line, ( + f"expected {field_name} to still carry `deprecated=true` in " + f"api.proto ({line.strip()!r}). If this fails, the deprecated " + "detection used above is broken, and the sibling assertion that " + "the superseded fields are NOT deprecated is not testing anything." + ) + + +def test_superseded_fields_keep_their_wire_numbers() -> None: + """Each superseded field must stay on the field number recorded in + SUPERSEDED_FIELDS. Old clients decode DeviceInfoResponse purely by field + number, so renumbering one of these -- even without touching its name -- + would make an old client read a completely different value out of the + wire, with nothing else in CI noticing. + """ + body = _extract_proto_message(PROTO_TEXT, "DeviceInfoResponse") + + for field_name, field_number in SUPERSEDED_FIELDS.items(): + line = _field_declaration_line(body, field_name) + assert re.search(rf"\b{field_name}\s*=\s*{field_number}\b", line), ( + f"{field_name} in DeviceInfoResponse is no longer declared at " + f"field number {field_number} ({line.strip()!r}). Field numbers " + "are the wire protocol -- renumbering this field silently breaks " + "every existing client that still decodes DeviceInfoResponse by " + "the old numbering." + ) + + +def test_capability_message_fields_keep_their_wire_numbers() -> None: + """Every field on DeviceCapabilitiesResponse and its three capability + sub-messages must stay on the field number recorded in + NEW_CAPABILITY_FIELDS. These messages are brand new as of API 1.15, but + the moment a device ships with them, their field numbers are a frozen + wire contract -- a client decodes a sub-message field purely by number, + so a later "cleanup" that renumbers one of these would silently corrupt + what every already-deployed client reads, with nothing else in CI + noticing. + """ + for message_name, fields in NEW_CAPABILITY_FIELDS.items(): + body = _extract_proto_message(PROTO_TEXT, message_name) + for field_name, field_number in fields.items(): + line = _field_declaration_line(body, field_name) + assert re.search(rf"\b{field_name}\s*=\s*{field_number}\b", line), ( + f"{field_name} on {message_name} is no longer declared at " + f"field number {field_number} ({line.strip()!r}). Field " + "numbers are the wire protocol -- renumbering this field " + "silently breaks every existing client that decodes this " + "message by the old numbering." + ) + + +def test_device_capabilities_rpc_requires_authentication() -> None: + """The `device_capabilities` RPC must not set + `option (needs_authentication) = false;` (or set it to anything at all). + Leaving it unset makes it inherit needs_authentication = true, keeping + capability data behind authentication (and encryption, when configured). + """ + body = _extract_rpc_body(PROTO_TEXT, "device_capabilities") + assert "needs_authentication" not in body, ( + "rpc device_capabilities sets a `needs_authentication` option in " + "api.proto. It must stay unset so it inherits needs_authentication = " + "true; otherwise device capability data could be requested over an " + "unauthenticated connection." + ) + + +# ==================== Group C: advertised API version ==================== + + +def test_api_version_minor_is_at_least_15() -> None: + """Clients gate sending DeviceCapabilitiesRequest on seeing + api_version >= 1.15 in HelloResponse. Regressing api_version_minor below + 15 would make every client believe capabilities are unsupported even + though the RPC exists, so this must never go backwards. Use >= rather + than == so the next unrelated minor-version bump doesn't need to touch + this test. + """ + match = re.search(r"resp\.api_version_minor\s*=\s*(\d+);", API_CONNECTION_TEXT) + assert match is not None, ( + "could not find `resp.api_version_minor = N;` in api_connection.cpp" + ) + minor = int(match.group(1)) + assert minor >= 15, ( + f"api_version_minor is {minor}, but device_capabilities requires " + "clients to see api_version >= 1.15 in HelloResponse before they will " + "ever request it." + )