From e35aa729f3ec6ff6640b3035cb20241766bc8514 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 12:21:37 -1000 Subject: [PATCH 1/7] [api] Add max_value proto option for constant-size varint codegen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a max_value field option to api_options.proto that tells the code generator the maximum value a field can have. When max_value < 128, the generated calculate_size() uses constant arithmetic instead of calling varint size functions, and encode() uses direct byte writes instead of varint encoding. Also optimize FixedArrayBytesType: when fixed_array_size < 128, the length varint is always 1 byte, so calculate_size() uses constant arithmetic and encode() uses write_raw_byte for the length. Applied to BluetoothLERawAdvertisement.address_type (max_value=4). Measured on ESP32 (upstairsdesk89proxy): - BluetoothLERawAdvertisement::calculate_size: 88 → 71 bytes (-19%) - BluetoothLERawAdvertisement::encode: 199 → 179 bytes (-10%) - Total BLE proxy hot path: 1807 → 1770 bytes (-37 bytes) --- esphome/components/api/api.proto | 2 +- esphome/components/api/api_options.proto | 6 +++ esphome/components/api/api_pb2.cpp | 6 +-- script/api_protobuf/api_protobuf.py | 47 ++++++++++++++++++++++-- 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 96ee2fb920..1e03675999 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1606,7 +1606,7 @@ message BluetoothLEAdvertisementResponse { message BluetoothLERawAdvertisement { uint64 address = 1 [(force) = true]; sint32 rssi = 2 [(force) = true]; - uint32 address_type = 3; + uint32 address_type = 3 [(max_value) = 4]; bytes data = 4 [(fixed_array_size) = 62, (force) = true]; } diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 02600f0977..0aa9e814cf 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -96,4 +96,10 @@ extend google.protobuf.FieldOptions { // variant of the calc_ method. Use on fields that are almost always non-default // to eliminate dead branches on hot paths. optional bool force = 50016 [default=false]; + + // max_value: Maximum value a field can have. + // When max_value < 128, the code generator emits constant-size calculations + // and direct byte writes instead of varint branching, since the encoded varint + // is guaranteed to be 1 byte. + optional uint32 max_value = 50017; } diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index ae2cd2bae8..f25d269e8f 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -2255,15 +2255,15 @@ void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer &buffer) const { buffer.encode_varint_raw(encode_zigzag32(this->rssi)); buffer.encode_uint32(3, this->address_type); buffer.write_raw_byte(34); - buffer.encode_varint_raw(this->data_len); + buffer.write_raw_byte(static_cast(this->data_len)); buffer.encode_raw(this->data, this->data_len); } uint32_t BluetoothLERawAdvertisement::calculate_size() const { uint32_t size = 0; size += ProtoSize::calc_uint64_force(1, this->address); size += ProtoSize::calc_sint32_force(1, this->rssi); - size += ProtoSize::calc_uint32(1, this->address_type); - size += ProtoSize::calc_length_force(1, this->data_len); + size += this->address_type ? 2 : 0; + size += 2 + this->data_len; return size; } void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer &buffer) const { diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index f2a11141af..a8870f0f53 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -156,6 +156,11 @@ class TypeInfo(ABC): """Check if this field should always be encoded (skip zero/empty check).""" return get_field_opt(self._field, pb.force, False) + @property + def max_value(self) -> int | None: + """Get the max_value option for this field, or None if not set.""" + return get_field_opt(self._field, pb.max_value, None) + @property def wire_type(self) -> WireType: """Get the wire type for the field.""" @@ -240,32 +245,55 @@ class TypeInfo(ABC): Returns the raw encode string if the tag is a single byte and the encode_func has a known raw equivalent, or None otherwise. + When max_value < 128, uses direct byte write instead of varint encoding. """ if not self.force: return None tag = self.calculate_tag() if tag >= 128: return None + # When max_value < 128, varint is always 1 byte - use direct byte write + max_val = self.max_value + if ( + max_val is not None + and max_val < 128 + and self.encode_func + in ( + "encode_uint32", + "encode_uint64", + ) + ): + return ( + f"buffer.write_raw_byte({tag});\n" + f"buffer.write_raw_byte(static_cast({value_expr}));" + ) raw_expr = self.RAW_ENCODE_MAP.get(self.encode_func) if raw_expr is None: return None return f"buffer.write_raw_byte({tag});\n{raw_expr.format(value=value_expr)}" def _encode_bytes_with_precomputed_tag( - self, data_expr: str, len_expr: str + self, data_expr: str, len_expr: str, max_len: int | None = None ) -> str | None: """Try to emit a precomputed-tag encode for a forced bytes/string field. Returns the raw encode string if the tag is a single byte, or None. + When max_len < 128, uses direct byte write for the length varint. """ if not self.force: return None tag = self.calculate_tag() if tag >= 128: return None + # When max_len < 128, length varint is always 1 byte + len_encode = ( + f"buffer.write_raw_byte(static_cast({len_expr}));" + if max_len is not None and max_len < 128 + else f"buffer.encode_varint_raw({len_expr});" + ) return ( f"buffer.write_raw_byte({tag});\n" - f"buffer.encode_varint_raw({len_expr});\n" + f"{len_encode}\n" f"buffer.encode_raw({data_expr}, {len_expr});" ) @@ -1191,8 +1219,9 @@ class FixedArrayBytesType(TypeInfo): @property def encode_content(self) -> str: + max_len = self.array_size if isinstance(self.array_size, int) else None if result := self._encode_bytes_with_precomputed_tag( - f"this->{self.field_name}", f"this->{self.field_name}_len" + f"this->{self.field_name}", f"this->{self.field_name}_len", max_len=max_len ): return result if self.force: @@ -1214,6 +1243,12 @@ class FixedArrayBytesType(TypeInfo): length_field = f"this->{self.field_name}_len" field_id_size = self.calculate_field_id_size() + # When array_size < 128, length varint is always 1 byte + if isinstance(self.array_size, int) and self.array_size < 128: + if force: + return f"size += {field_id_size + 1} + {length_field};" + return f"size += {length_field} ? {field_id_size + 1} + {length_field} : 0;" + if force: # For repeated fields, always calculate size (no zero check) return f"size += ProtoSize::calc_length_force({field_id_size}, {length_field});" @@ -1245,6 +1280,12 @@ class UInt32Type(TypeInfo): return o def get_size_calculation(self, name: str, force: bool = False) -> str: + max_val = self.max_value + if max_val is not None and max_val < 128: + field_id_size = self.calculate_field_id_size() + if force: + return f"size += {field_id_size + 1};" + return f"size += {name} ? {field_id_size + 1} : 0;" return self._get_simple_size_calculation(name, force, "uint32") def get_estimated_size(self) -> int: From 1e68a90ac9364cdef366aa20940212127692b58c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 12:26:51 -1000 Subject: [PATCH 2/7] [api] Extract _get_single_byte_varint_size helper in codegen Refactor the constant-size varint pattern into a reusable helper method on the TypeInfo base class, used by both UInt32Type (max_value) and FixedArrayBytesType (fixed_array_size < 128). --- script/api_protobuf/api_protobuf.py | 35 ++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index a8870f0f53..66e3182375 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -374,6 +374,28 @@ class TypeInfo(ABC): value = value_expr or name return f"size += ProtoSize::{method}({field_id_size}, {value});" + def _get_single_byte_varint_size( + self, name: str, force: bool, extra_expr: str | None = None + ) -> str: + """Size calculation when the varint is guaranteed to be 1 byte. + + Used when max_value < 128 or fixed_array_size < 128. + The fixed part is field_id_size + 1 (tag + 1-byte varint). + + Args: + name: Expression to check for zero (non-force only) + force: Whether to skip the zero check + extra_expr: Additional variable expression to add (e.g., data length) + """ + fixed = self.calculate_field_id_size() + 1 + if extra_expr: + if force: + return f"size += {fixed} + {extra_expr};" + return f"size += {name} ? {fixed} + {extra_expr} : 0;" + if force: + return f"size += {fixed};" + return f"size += {name} ? {fixed} : 0;" + @abstractmethod def get_size_calculation(self, name: str, force: bool = False) -> str: """Calculate the size needed for encoding this field. @@ -1241,14 +1263,14 @@ class FixedArrayBytesType(TypeInfo): def get_size_calculation(self, name: str, force: bool = False) -> str: # Use the actual length stored in the _len field length_field = f"this->{self.field_name}_len" - field_id_size = self.calculate_field_id_size() # When array_size < 128, length varint is always 1 byte if isinstance(self.array_size, int) and self.array_size < 128: - if force: - return f"size += {field_id_size + 1} + {length_field};" - return f"size += {length_field} ? {field_id_size + 1} + {length_field} : 0;" + return self._get_single_byte_varint_size( + length_field, force, extra_expr=length_field + ) + field_id_size = self.calculate_field_id_size() if force: # For repeated fields, always calculate size (no zero check) return f"size += ProtoSize::calc_length_force({field_id_size}, {length_field});" @@ -1282,10 +1304,7 @@ class UInt32Type(TypeInfo): def get_size_calculation(self, name: str, force: bool = False) -> str: max_val = self.max_value if max_val is not None and max_val < 128: - field_id_size = self.calculate_field_id_size() - if force: - return f"size += {field_id_size + 1};" - return f"size += {name} ? {field_id_size + 1} : 0;" + return self._get_single_byte_varint_size(name, force) return self._get_simple_size_calculation(name, force, "uint32") def get_estimated_size(self) -> int: From 363d811cc1f9818c394f3b5f3b8db74699042dea Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 12:27:55 -1000 Subject: [PATCH 3/7] [api] Use _get_simple_size_calculation for FixedArrayBytesType fallback Replace inline calc_length/calc_length_force formatting with the existing helper method. --- script/api_protobuf/api_protobuf.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 66e3182375..4012db336e 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1270,12 +1270,7 @@ class FixedArrayBytesType(TypeInfo): length_field, force, extra_expr=length_field ) - field_id_size = self.calculate_field_id_size() - if force: - # For repeated fields, always calculate size (no zero check) - return f"size += ProtoSize::calc_length_force({field_id_size}, {length_field});" - # For non-repeated fields, length already checks for zero - return f"size += ProtoSize::calc_length({field_id_size}, {length_field});" + return self._get_simple_size_calculation(length_field, force, "length") def get_estimated_size(self) -> int: # Estimate based on typical BLE advertisement size From 02b424388e4ef62207bb1d71ef160ccaf114673c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 12:28:51 -1000 Subject: [PATCH 4/7] [api] Use RAW_ENCODE_SMALL_MAP for max_value encode optimization Replace inline encode_func check with a lookup table, matching the existing RAW_ENCODE_MAP pattern. --- script/api_protobuf/api_protobuf.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 4012db336e..1381d5586f 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -240,6 +240,12 @@ class TypeInfo(ABC): "encode_bool": "buffer.write_raw_byte({value} ? 0x01 : 0x00);", } + # When max_value < 128, the varint is always 1 byte — use a direct byte write + RAW_ENCODE_SMALL_MAP: dict[str, str] = { + "encode_uint32": "buffer.write_raw_byte(static_cast({value}));", + "encode_uint64": "buffer.write_raw_byte(static_cast({value}));", + } + def _encode_with_precomputed_tag(self, value_expr: str) -> str | None: """Try to emit a precomputed-tag encode for a forced field. @@ -252,22 +258,12 @@ class TypeInfo(ABC): tag = self.calculate_tag() if tag >= 128: return None - # When max_value < 128, varint is always 1 byte - use direct byte write max_val = self.max_value - if ( - max_val is not None - and max_val < 128 - and self.encode_func - in ( - "encode_uint32", - "encode_uint64", - ) - ): - return ( - f"buffer.write_raw_byte({tag});\n" - f"buffer.write_raw_byte(static_cast({value_expr}));" - ) - raw_expr = self.RAW_ENCODE_MAP.get(self.encode_func) + raw_expr = ( + self.RAW_ENCODE_SMALL_MAP.get(self.encode_func) + if max_val is not None and max_val < 128 + else self.RAW_ENCODE_MAP.get(self.encode_func) + ) if raw_expr is None: return None return f"buffer.write_raw_byte({tag});\n{raw_expr.format(value=value_expr)}" From f3238a857e2d2d356fd387ffa20ac158ed747091 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 12:31:01 -1000 Subject: [PATCH 5/7] [api] Select encode map before lookup --- script/api_protobuf/api_protobuf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 1381d5586f..83aee17cf9 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -259,11 +259,12 @@ class TypeInfo(ABC): if tag >= 128: return None max_val = self.max_value - raw_expr = ( - self.RAW_ENCODE_SMALL_MAP.get(self.encode_func) + encode_map = ( + self.RAW_ENCODE_SMALL_MAP if max_val is not None and max_val < 128 - else self.RAW_ENCODE_MAP.get(self.encode_func) + else self.RAW_ENCODE_MAP ) + raw_expr = encode_map.get(self.encode_func) if raw_expr is None: return None return f"buffer.write_raw_byte({tag});\n{raw_expr.format(value=value_expr)}" From fa80caf0ffc3a9d252ac8e789dbfbb4ca7817362 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 12:31:44 -1000 Subject: [PATCH 6/7] [api] Fall back to RAW_ENCODE_MAP when SMALL_MAP has no entry --- script/api_protobuf/api_protobuf.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 83aee17cf9..cef4067ea3 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -259,12 +259,11 @@ class TypeInfo(ABC): if tag >= 128: return None max_val = self.max_value - encode_map = ( - self.RAW_ENCODE_SMALL_MAP - if max_val is not None and max_val < 128 - else self.RAW_ENCODE_MAP - ) - raw_expr = encode_map.get(self.encode_func) + raw_expr = None + if max_val is not None and max_val < 128: + raw_expr = self.RAW_ENCODE_SMALL_MAP.get(self.encode_func) + if raw_expr is None: + raw_expr = self.RAW_ENCODE_MAP.get(self.encode_func) if raw_expr is None: return None return f"buffer.write_raw_byte({tag});\n{raw_expr.format(value=value_expr)}" From 9ece286a26f8c3aaa10a8e7025934866e589efc7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 3 Apr 2026 12:32:32 -1000 Subject: [PATCH 7/7] [api] Simplify _get_single_byte_varint_size --- script/api_protobuf/api_protobuf.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index cef4067ea3..c17f16412c 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -384,13 +384,10 @@ class TypeInfo(ABC): extra_expr: Additional variable expression to add (e.g., data length) """ fixed = self.calculate_field_id_size() + 1 - if extra_expr: - if force: - return f"size += {fixed} + {extra_expr};" - return f"size += {name} ? {fixed} + {extra_expr} : 0;" + size_expr = f"{fixed} + {extra_expr}" if extra_expr else str(fixed) if force: - return f"size += {fixed};" - return f"size += {name} ? {fixed} : 0;" + return f"size += {size_expr};" + return f"size += {name} ? {size_expr} : 0;" @abstractmethod def get_size_calculation(self, name: str, force: bool = False) -> str: