From 555804db143d035796cf16f3f9e165bc64fa3f05 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 22:03:26 -1000 Subject: [PATCH] no gap --- esphome/components/api/api_connection.cpp | 34 +++++--- esphome/components/api/api_connection.h | 9 +- esphome/components/api/api_frame_helper.h | 20 ++++- .../api/api_frame_helper_plaintext.cpp | 86 +++++++------------ esphome/components/api/proto.h | 11 +++ 5 files changed, 86 insertions(+), 74 deletions(-) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 0f456ecd0c..42a1615e1d 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2009,11 +2009,16 @@ uint16_t APIConnection::encode_to_buffer(uint32_t calculated_size, MessageEncode } #endif // Cache frame sizes to avoid repeated virtual calls - const uint8_t header_padding = conn->helper_->frame_header_padding(); const uint8_t footer_size = conn->helper_->frame_footer_size(); - // Calculate total size with padding for buffer allocation - size_t total_calculated_size = calculated_size + header_padding + footer_size; + // Use actual header size for all batch messages (eliminates gaps between messages). + const uint8_t header_size = conn->helper_->frame_header_size(calculated_size, conn->batch_message_type_); + + // Store for process_batch_multi_ to pass into MessageInfo + conn->batch_header_size_ = header_size; + + // Calculate total size with header for buffer allocation + size_t total_calculated_size = calculated_size + header_size + footer_size; // Check if it fits if (total_calculated_size > remaining_size) @@ -2023,12 +2028,13 @@ uint16_t APIConnection::encode_to_buffer(uint32_t calculated_size, MessageEncode size_t to_add; if (conn->flags_.batch_first_message) { - // First message - buffer already prepared by caller, just clear flag + // First message - buffer already prepared by caller with max header padding. + // We only add payload bytes since padding is already in the buffer. + // The unused leading bytes (max_padding - actual_header) are skipped at write time. conn->flags_.batch_first_message = false; to_add = calculated_size; } else { - // Batch message second or later - // Reserve for full message, resize to include footer gap + header padding + payload + // Subsequent batch messages use exact header size — no gaps to_add = total_calculated_size; } @@ -2164,17 +2170,15 @@ void APIConnection::process_batch_multi_(APIBuffer &shared_buf, size_t num_items "MessageInfo must remain trivially destructible with this placement-new approach"); const size_t messages_to_process = std::min(num_items, MAX_MESSAGES_PER_BATCH); - const uint8_t frame_overhead = header_padding + footer_size; // Stack-allocated array for message info alignas(MessageInfo) char message_info_storage[MAX_MESSAGES_PER_BATCH * sizeof(MessageInfo)]; MessageInfo *message_info = reinterpret_cast(message_info_storage); size_t items_processed = 0; uint16_t remaining_size = std::numeric_limits::max(); - // Track where each message's header padding begins in the buffer - // For plaintext: this is where the 6-byte header padding starts - // For noise: this is where the 7-byte header padding starts - // The actual message data follows after the header padding + // Track where each message's header begins in the buffer + // First message: offset 0 (max padding, may have unused leading bytes) + // Subsequent messages: offset points to exact header start (no gaps) uint32_t current_offset = 0; // Process items and encode directly to buffer (up to our limit) @@ -2190,13 +2194,14 @@ void APIConnection::process_batch_multi_(APIBuffer &shared_buf, size_t num_items } // Message was encoded successfully - // payload_size is header_padding + actual payload size + footer_size - uint16_t proto_payload_size = payload_size - frame_overhead; + // payload_size = header_size + proto_payload_size + footer_size + uint16_t proto_payload_size = payload_size - this->batch_header_size_ - footer_size; // Use placement new to construct MessageInfo in pre-allocated stack array // This avoids default-constructing all MAX_MESSAGES_PER_BATCH elements // Explicit destruction is not needed because MessageInfo is trivially destructible, // as ensured by the static_assert in its definition. - new (&message_info[items_processed++]) MessageInfo(item.message_type, current_offset, proto_payload_size); + new (&message_info[items_processed++]) + MessageInfo(item.message_type, current_offset, proto_payload_size, this->batch_header_size_); // After first message, set remaining size to MAX_BATCH_PACKET_SIZE to avoid fragmentation if (items_processed == 1) { remaining_size = MAX_BATCH_PACKET_SIZE; @@ -2246,6 +2251,7 @@ void APIConnection::process_batch_multi_(APIBuffer &shared_buf, size_t num_items uint16_t APIConnection::dispatch_message_(const DeferredBatch::BatchItem &item, uint32_t remaining_size, bool batch_first) { this->flags_.batch_first_message = batch_first; + this->batch_message_type_ = item.message_type; #ifdef USE_EVENT // Events need aux_data_index to look up event type from entity if (item.message_type == EventResponse::MESSAGE_TYPE) { diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 13d5273ecb..d0bccf418f 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -726,9 +726,14 @@ class APIConnection final : public APIServerConnectionBase { // 2-byte types immediately after flags_ (no padding between them) uint16_t client_api_version_major_{0}; uint16_t client_api_version_minor_{0}; - // 1-byte type to fill padding + // 1-byte types to fill remaining space before next 4-byte boundary ActiveIterator active_iterator_{ActiveIterator::NONE}; - // Total: 2 (flags) + 2 + 2 + 1 = 7 bytes, then 1 byte padding to next 4-byte boundary + uint8_t batch_message_type_{0}; // Current message type during batch encoding + // Total: 2 (flags) + 2 + 2 + 1 + 1 = 8 bytes, aligned to 4-byte boundary + + // Actual header size used by encode_to_buffer for the current message. + // Read by process_batch_multi_ to pass into MessageInfo. + uint8_t batch_header_size_{0}; uint32_t get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } // Message will use 8 more bytes than the minimum size, and typical diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h index e22b959909..3dfe6f0a7b 100644 --- a/esphome/components/api/api_frame_helper.h +++ b/esphome/components/api/api_frame_helper.h @@ -53,8 +53,10 @@ struct MessageInfo { uint16_t offset; // Offset in buffer where message starts uint16_t payload_size; // Size of the message payload uint8_t message_type; // Message type (0-255) + uint8_t header_size; // Actual header size used (avoids recomputation in write path) - MessageInfo(uint8_t type, uint16_t off, uint16_t size) : offset(off), payload_size(size), message_type(type) {} + MessageInfo(uint8_t type, uint16_t off, uint16_t size, uint8_t hdr) + : offset(off), payload_size(size), message_type(type), header_size(hdr) {} }; enum class APIError : uint16_t { @@ -169,8 +171,22 @@ class APIFrameHelper { // messages contains (message_type, offset, length) for each message in the buffer. // The buffer contains all messages with appropriate padding before each. virtual APIError write_protobuf_messages(ProtoWriteBuffer buffer, std::span messages) = 0; - // Get the frame header padding required by this protocol + // Get the maximum frame header padding required by this protocol (worst case) uint8_t frame_header_padding() const { return frame_header_padding_; } + // Get the actual frame header size for a specific message. + // For noise: always returns frame_header_padding_ (fixed 7-byte header). + // For plaintext: computes actual size from varint lengths (3-6 bytes). + uint8_t frame_header_size(uint16_t payload_size, uint8_t message_type) const { +#if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT) + return this->frame_footer_size_ + ? this->frame_header_padding_ + : static_cast(1 + ProtoSize::varint16(payload_size) + ProtoSize::varint8(message_type)); +#elif defined(USE_API_NOISE) + return this->frame_header_padding_; +#else // USE_API_PLAINTEXT only + return static_cast(1 + ProtoSize::varint16(payload_size) + ProtoSize::varint8(message_type)); +#endif + } // Get the frame footer size required by this protocol uint8_t frame_footer_size() const { return frame_footer_size_; } // Check if socket has data ready to read diff --git a/esphome/components/api/api_frame_helper_plaintext.cpp b/esphome/components/api/api_frame_helper_plaintext.cpp index c905bdf2c9..d263aac3f6 100644 --- a/esphome/components/api/api_frame_helper_plaintext.cpp +++ b/esphome/components/api/api_frame_helper_plaintext.cpp @@ -227,15 +227,6 @@ APIError APIPlaintextFrameHelper::read_packet(ReadPacketBuffer *buffer) { buffer->type = this->rx_header_parsed_type_; return APIError::OK; } -// Compute varint encoded length for a 16-bit value (1, 2, or 3 bytes). -ESPHOME_ALWAYS_INLINE static inline uint8_t varint_encoded_length_16(uint16_t value) { - return value < ProtoSize::VARINT_THRESHOLD_1_BYTE ? 1 : (value < ProtoSize::VARINT_THRESHOLD_2_BYTE ? 2 : 3); -} - -// Compute varint encoded length for an 8-bit value (1 or 2 bytes). -ESPHOME_ALWAYS_INLINE static inline uint8_t varint_encoded_length_8(uint8_t value) { - return value < ProtoSize::VARINT_THRESHOLD_1_BYTE ? 1 : 2; -} // Encode a 16-bit varint (1-3 bytes) using pre-computed length. ESPHOME_ALWAYS_INLINE static inline void encode_varint_16(uint16_t value, uint8_t varint_len, uint8_t *p) { @@ -261,39 +252,26 @@ ESPHOME_ALWAYS_INLINE static inline void encode_varint_8(uint8_t value, uint8_t } // Write plaintext header into pre-allocated padding before payload. +// padding_size: bytes reserved before payload (HEADER_PADDING for first/single msg, +// actual header size for contiguous batch messages). // Returns the total header length (indicator + varints). ESPHOME_ALWAYS_INLINE static inline uint8_t write_plaintext_header(uint8_t *buf_start, uint16_t payload_size, - uint8_t message_type) { - uint8_t size_varint_len = varint_encoded_length_16(payload_size); - uint8_t type_varint_len = varint_encoded_length_8(message_type); + uint8_t message_type, uint8_t padding_size) { + uint8_t size_varint_len = ProtoSize::varint16(payload_size); + uint8_t type_varint_len = ProtoSize::varint8(message_type); uint8_t total_header_len = 1 + size_varint_len + type_varint_len; - // Calculate where to start writing the header - // The header starts at the latest possible position to minimize unused padding + // The header is right-justified within the padding so it sits immediately before payload. // - // Example 1 (small values): total_header_len = 3, header_offset = 6 - 3 = 3 - // [0-2] - Unused padding - // [3] - 0x00 indicator byte - // [4] - Payload size varint (1 byte, for sizes 0-127) - // [5] - Message type varint (1 byte, for types 0-127) - // [6...] - Actual payload data + // Single/first message (padding_size = HEADER_PADDING = 6): + // Example (small, header=3): [0-2] unused | [3] 0x00 | [4] size | [5] type | [6...] payload + // Example (medium, header=4): [0-1] unused | [2] 0x00 | [3-4] size | [5] type | [6...] payload + // Example (large, header=6): [0] 0x00 | [1-3] size | [4-5] type | [6...] payload // - // Example 2 (medium values): total_header_len = 4, header_offset = 6 - 4 = 2 - // [0-1] - Unused padding - // [2] - 0x00 indicator byte - // [3-4] - Payload size varint (2 bytes, for sizes 128-16383) - // [5] - Message type varint (1 byte, for types 0-127) - // [6...] - Actual payload data - // - // Example 3 (large values): total_header_len = 6, header_offset = 6 - 6 = 0 - // [0] - 0x00 indicator byte - // [1-3] - Payload size varint (3 bytes, for sizes 16384-65535) - // [4-5] - Message type varint (2 bytes, for types 128-16383) - // [6...] - Actual payload data - // - // The message starts at offset + frame_header_padding - // So we write the header starting at offset + HEADER_PADDING - total_header_len - uint32_t header_offset = APIPlaintextFrameHelper::HEADER_PADDING - total_header_len; + // Batch messages 2+ (padding_size = actual header size, no unused bytes): + // Example (small, header=3): [0] 0x00 | [1] size | [2] type | [3...] payload + // Example (medium, header=4): [0] 0x00 | [1-2] size | [3] type | [4...] payload + uint32_t header_offset = padding_size - total_header_len; // Write the plaintext header buf_start[header_offset] = 0x00; // indicator @@ -312,7 +290,7 @@ APIError APIPlaintextFrameHelper::write_protobuf_packet(uint8_t type, ProtoWrite uint16_t payload_size = static_cast(buffer.get_buffer()->size() - HEADER_PADDING); uint8_t *buffer_data = buffer.get_buffer()->data(); - uint8_t header_len = write_plaintext_header(buffer_data, payload_size, type); + uint8_t header_len = write_plaintext_header(buffer_data, payload_size, type, HEADER_PADDING); return this->write_raw_fast_buf_(buffer_data + HEADER_PADDING - header_len, static_cast(header_len + payload_size)); } @@ -324,28 +302,24 @@ APIError APIPlaintextFrameHelper::write_protobuf_messages(ProtoWriteBuffer buffe assert(!messages.empty()); #endif uint8_t *buffer_data = buffer.get_buffer()->data(); - uint8_t *write_start = nullptr; - uint8_t *write_end = nullptr; - // Write headers and compact messages to close 0-3 byte varint padding gaps. - // First iteration records start position via continue; subsequent iterations - // memmove to close gaps between messages. - for (const auto &msg : messages) { - uint8_t header_len = write_plaintext_header(buffer_data + msg.offset, msg.payload_size, msg.message_type); - uint8_t *src = buffer_data + msg.offset + HEADER_PADDING - header_len; - uint16_t msg_len = header_len + msg.payload_size; - if (write_start == nullptr) { - write_start = src; - write_end = src + msg_len; - continue; - } - if (src != write_end) { - memmove(write_end, src, msg_len); - } - write_end += msg_len; + // First message has max padding (HEADER_PADDING), may have unused leading bytes. + // Subsequent messages were encoded with exact header sizes, so they are contiguous. + const auto &first = messages[0]; + uint8_t header_len = + write_plaintext_header(buffer_data + first.offset, first.payload_size, first.message_type, HEADER_PADDING); + uint8_t *write_start = buffer_data + first.offset + HEADER_PADDING - header_len; + uint16_t total_len = header_len + first.payload_size; + + // Write headers for remaining messages — padding equals actual header size + // (stored in msg.header_size), so write_plaintext_header writes at offset 0 with no gap. + for (size_t i = 1; i < messages.size(); i++) { + const auto &msg = messages[i]; + write_plaintext_header(buffer_data + msg.offset, msg.payload_size, msg.message_type, msg.header_size); + total_len += msg.header_size + msg.payload_size; } - return this->write_raw_fast_buf_(write_start, static_cast(write_end - write_start)); + return this->write_raw_fast_buf_(write_start, total_len); } } // namespace esphome::api diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index b629018a91..4e122ea71e 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -518,6 +518,17 @@ class ProtoSize { static constexpr uint32_t VARINT_THRESHOLD_3_BYTE = 1 << 21; // 2097152 static constexpr uint32_t VARINT_THRESHOLD_4_BYTE = 1 << 28; // 268435456 + // Varint encoded length for a 16-bit value (1, 2, or 3 bytes). + // Fully inline — no slow path call for values >= 128. + static constexpr inline uint8_t ESPHOME_ALWAYS_INLINE varint16(uint16_t value) { + return value < VARINT_THRESHOLD_1_BYTE ? 1 : (value < VARINT_THRESHOLD_2_BYTE ? 2 : 3); + } + + // Varint encoded length for an 8-bit value (1 or 2 bytes). + static constexpr inline uint8_t ESPHOME_ALWAYS_INLINE varint8(uint8_t value) { + return value < VARINT_THRESHOLD_1_BYTE ? 1 : 2; + } + /** * @brief Calculates the size in bytes needed to encode a uint32_t value as a varint *