From 65f68a412ae268771435f055709296cc585aba51 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 24 Apr 2026 19:58:39 -0500 Subject: [PATCH 1/8] [api] Add 48-bit MAC address varint fast path for BLE advertisements Adds a (mac_address) field option that switches uint64 fields holding 48-bit MAC addresses to a specialized varint encoder. The fast path emits exactly 7 bytes when bits [42..47] are non-zero (the common case for real MACs, since OUIs occupy the top 24 bits) -- one bounds check and 7 independent stores instead of the 7-iteration shift+branch loop. calc_uint64_48bit_force mirrors the same fast path in size calculation. Applied to BluetoothLERawAdvertisement.address, the per-advertisement address encode in BluetoothLERawAdvertisementsResponse drops from a serialized per-byte loop to straight-line code. --- esphome/components/api/api.proto | 2 +- esphome/components/api/api_options.proto | 6 ++++++ esphome/components/api/api_pb2.cpp | 4 ++-- esphome/components/api/proto.h | 27 ++++++++++++++++++++++++ script/api_protobuf/api_protobuf.py | 19 +++++++++++++++++ 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index c3e4c38633..e385aeee35 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1628,7 +1628,7 @@ message BluetoothLEAdvertisementResponse { message BluetoothLERawAdvertisement { option (inline_encode) = true; - uint64 address = 1 [(force) = true]; + uint64 address = 1 [(force) = true, (mac_address) = true]; sint32 rssi = 2 [(force) = true]; uint32 address_type = 3 [(max_value) = 4]; diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index d5d0b37e8d..ac9c4e59cc 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -110,4 +110,10 @@ extend google.protobuf.FieldOptions { // length varint calculations and direct byte writes, since the length // varint is guaranteed to be 1 byte. optional uint32 max_data_length = 50018; + + // mac_address: Field is a 48-bit MAC address stored in a uint64. + // Emits encode_varint_raw_48bit which has a 7-byte fast path that avoids + // the per-byte loop when the upper bits are non-zero (the common case + // for real MAC addresses, since OUIs occupy the top 24 bits). + optional bool mac_address = 50019 [default=false]; } diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 3d12453939..1866937858 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -2348,7 +2348,7 @@ BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCO uint8_t *len_pos = pos; ProtoEncode::reserve_byte(pos PROTO_ENCODE_DEBUG_ARG); ProtoEncode::write_raw_byte(pos PROTO_ENCODE_DEBUG_ARG, 8); - ProtoEncode::encode_varint_raw_64(pos PROTO_ENCODE_DEBUG_ARG, sub_msg.address); + ProtoEncode::encode_varint_raw_48bit(pos PROTO_ENCODE_DEBUG_ARG, sub_msg.address); ProtoEncode::write_raw_byte(pos PROTO_ENCODE_DEBUG_ARG, 16); ProtoEncode::encode_varint_raw_short(pos PROTO_ENCODE_DEBUG_ARG, encode_zigzag32(sub_msg.rssi)); if (sub_msg.address_type) { @@ -2369,7 +2369,7 @@ BluetoothLERawAdvertisementsResponse::calculate_size() const { for (uint16_t i = 0; i < this->advertisements_len; i++) { auto &sub_msg = this->advertisements[i]; size += 2; - size += ProtoSize::calc_uint64_force(1, sub_msg.address); + size += ProtoSize::calc_uint64_48bit_force(1, sub_msg.address); size += ProtoSize::calc_sint32_force(1, sub_msg.rssi); size += sub_msg.address_type ? 2 : 0; size += 2 + sub_msg.data_len; diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 8cac7fff3b..d274c89537 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -342,6 +342,26 @@ class ProtoEncode { } encode_varint_raw_loop(pos PROTO_ENCODE_DEBUG_ARG, value); } + /// Encode a 48-bit MAC address (stored in a uint64) as varint. + /// Real MAC addresses occupy the full 48 bits (OUI in upper 24), so the + /// fast path -- any non-zero bit in bits [42..47] -- emits exactly 7 bytes + /// with no per-byte branch. Falls back to the general loop otherwise. + static inline void ESPHOME_ALWAYS_INLINE encode_varint_raw_48bit(uint8_t *__restrict__ &pos PROTO_ENCODE_DEBUG_PARAM, + uint64_t value) { + if (value >= (1ULL << 42)) [[likely]] { + PROTO_ENCODE_CHECK_BOUNDS(pos, 7); + pos[0] = static_cast(value | 0x80); + pos[1] = static_cast((value >> 7) | 0x80); + pos[2] = static_cast((value >> 14) | 0x80); + pos[3] = static_cast((value >> 21) | 0x80); + pos[4] = static_cast((value >> 28) | 0x80); + pos[5] = static_cast((value >> 35) | 0x80); + pos[6] = static_cast(value >> 42); + pos += 7; + return; + } + encode_varint_raw_64(pos PROTO_ENCODE_DEBUG_ARG, value); + } static inline void ESPHOME_ALWAYS_INLINE encode_field_raw(uint8_t *__restrict__ &pos PROTO_ENCODE_DEBUG_PARAM, uint32_t field_id, uint32_t type) { encode_varint_raw(pos PROTO_ENCODE_DEBUG_ARG, (field_id << 3) | type); @@ -817,6 +837,13 @@ class ProtoSize { static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_uint64_force(uint32_t field_id_size, uint64_t value) { return field_id_size + varint(value); } + /// 48-bit MAC address variant: matches encode_varint_raw_48bit's fast path. + /// When value uses any of bits [42..47] the encoded varint is 7 bytes; + /// otherwise fall back to the general size calculation. + static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_uint64_48bit_force(uint32_t field_id_size, + uint64_t value) { + return field_id_size + (value >= (1ULL << 42) ? 7 : varint(value)); + } static constexpr uint32_t calc_length(uint32_t field_id_size, size_t len) { return len ? field_id_size + varint(static_cast(len)) + static_cast(len) : 0; } diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index c10479a726..a0dd26b1fe 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -184,6 +184,11 @@ class TypeInfo(ABC): """Check if this field should always be encoded (skip zero/empty check).""" return get_field_opt(self._field, pb.force, False) + @property + def mac_address(self) -> bool: + """Check if this uint64 field is a 48-bit MAC address (use 7-byte fast path).""" + return get_field_opt(self._field, pb.mac_address, False) + @property def max_value(self) -> int | None: """Get the max_value option for this field, or None if not set.""" @@ -665,8 +670,22 @@ class UInt64Type(VarintTypeMixin, TypeInfo): return o def get_size_calculation(self, name: str, force: bool = False) -> str: + if self.mac_address and force: + field_id_size = self.calculate_field_id_size() + return ( + f"size += ProtoSize::calc_uint64_48bit_force({field_id_size}, {name});" + ) return self._get_simple_size_calculation(name, force, "uint64") + @property + def RAW_ENCODE_MAP(self) -> dict[str, str]: # noqa: N802 + if self.mac_address: + return { + **TypeInfo.RAW_ENCODE_MAP, + "encode_uint64": "ProtoEncode::encode_varint_raw_48bit(pos, {value});", + } + return TypeInfo.RAW_ENCODE_MAP + def get_estimated_size(self) -> int: return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint From c27f9e512b63ced298e3634eb2cc17be92d521b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Apr 2026 02:28:04 +0000 Subject: [PATCH 2/8] Bump aioesphomeapi from 44.21.0 to 44.22.0 (#15989) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 71db6d3444..c1838d3b51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.3 esphome-dashboard==20260408.1 -aioesphomeapi==44.21.0 +aioesphomeapi==44.22.0 zeroconf==0.148.0 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import From ba362a7c95e9cffce5b6a39da32b95f1c9f3bd3e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 24 Apr 2026 21:43:13 -0500 Subject: [PATCH 3/8] [api] Narrow 48-bit MAC varint fast path to values < 2^48 Per Copilot review: encode_varint_raw_48bit and calc_uint64_48bit_force would silently truncate bits 48..63 if ever called with a uint64 that doesn't fit in 48 bits. Real MAC addresses always fit, but since the helpers are exposed and the (mac_address) option is generic, narrow the fast path to [1<<42, 1<<48) so values outside that range fall back to the general encoder/size helpers. Re-verified byte-identical output and decoder round-trip across the 1<<48 boundary and all bit positions up to 63. --- esphome/components/api/proto.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index d274c89537..30efdfbb61 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -344,11 +344,13 @@ class ProtoEncode { } /// Encode a 48-bit MAC address (stored in a uint64) as varint. /// Real MAC addresses occupy the full 48 bits (OUI in upper 24), so the - /// fast path -- any non-zero bit in bits [42..47] -- emits exactly 7 bytes - /// with no per-byte branch. Falls back to the general loop otherwise. + /// fast path -- any non-zero bit in bits [42..47] with value fitting in + /// 48 bits -- emits exactly 7 bytes with no per-byte branch. Values outside + /// [1<<42, 1<<48) fall back to the general encoder to preserve correctness + /// if this helper is ever misapplied to a non-MAC uint64. static inline void ESPHOME_ALWAYS_INLINE encode_varint_raw_48bit(uint8_t *__restrict__ &pos PROTO_ENCODE_DEBUG_PARAM, uint64_t value) { - if (value >= (1ULL << 42)) [[likely]] { + if (value >= (1ULL << 42) && value < (1ULL << 48)) [[likely]] { PROTO_ENCODE_CHECK_BOUNDS(pos, 7); pos[0] = static_cast(value | 0x80); pos[1] = static_cast((value >> 7) | 0x80); @@ -838,11 +840,12 @@ class ProtoSize { return field_id_size + varint(value); } /// 48-bit MAC address variant: matches encode_varint_raw_48bit's fast path. - /// When value uses any of bits [42..47] the encoded varint is 7 bytes; - /// otherwise fall back to the general size calculation. + /// When value is in [1<<42, 1<<48) the encoded varint is 7 bytes; values + /// outside that range fall back to the general size calculation so the + /// result stays correct if this helper is ever misapplied. static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_uint64_48bit_force(uint32_t field_id_size, uint64_t value) { - return field_id_size + (value >= (1ULL << 42) ? 7 : varint(value)); + return field_id_size + ((value >= (1ULL << 42) && value < (1ULL << 48)) ? 7 : varint(value)); } static constexpr uint32_t calc_length(uint32_t field_id_size, size_t len) { return len ? field_id_size + varint(static_cast(len)) + static_cast(len) : 0; From 7ab701e16232abe1c272a4b86192cda6e8933922 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 25 Apr 2026 04:13:30 -0500 Subject: [PATCH 4/8] [api] Gate 48-bit MAC range check behind ESPHOME_DEBUG_API Copilot flagged that encode_varint_raw_48bit/calc_uint64_48bit_force would silently truncate for uint64 values >= 2^48. In practice the (mac_address) option is only applied to fields populated by the BLE stack, which always fits in 48 bits -- so the runtime upper-bound check added in ba362a7c95 regressed CodSpeed by up to 8.7pp on CalculateSize_BLERawAdvs12 for a scenario that can't happen. Move the value-fits-in-48-bits check to a debug assert guarded by ESPHOME_DEBUG_API, and express 48 via MAC_ADDRESS_SIZE * 8 so the threshold tracks the existing MAC size constant. Release builds are back to the original fast path; debug builds catch misuse. --- esphome/components/api/proto.h | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 30efdfbb61..3ff65029e1 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -344,13 +344,17 @@ class ProtoEncode { } /// Encode a 48-bit MAC address (stored in a uint64) as varint. /// Real MAC addresses occupy the full 48 bits (OUI in upper 24), so the - /// fast path -- any non-zero bit in bits [42..47] with value fitting in - /// 48 bits -- emits exactly 7 bytes with no per-byte branch. Values outside - /// [1<<42, 1<<48) fall back to the general encoder to preserve correctness - /// if this helper is ever misapplied to a non-MAC uint64. + /// fast path -- any non-zero bit in the top 6 of 48 -- emits exactly 7 bytes + /// with no per-byte branch. Falls back to the general loop otherwise. + /// Caller must guarantee value fits in 48 bits (checked in debug builds). static inline void ESPHOME_ALWAYS_INLINE encode_varint_raw_48bit(uint8_t *__restrict__ &pos PROTO_ENCODE_DEBUG_PARAM, uint64_t value) { - if (value >= (1ULL << 42) && value < (1ULL << 48)) [[likely]] { +#ifdef ESPHOME_DEBUG_API + assert(value < (1ULL << (MAC_ADDRESS_SIZE * 8)) && "encode_varint_raw_48bit: value exceeds 48 bits"); +#endif + // 7-byte varint holds 49 bits (7 * 7), so a 48-bit value needs all 7 bytes + // whenever bit 42 or higher is set (i.e. value >= 1 << (48 - 6)). + if (value >= (1ULL << (MAC_ADDRESS_SIZE * 8 - 6))) [[likely]] { PROTO_ENCODE_CHECK_BOUNDS(pos, 7); pos[0] = static_cast(value | 0x80); pos[1] = static_cast((value >> 7) | 0x80); @@ -840,12 +844,12 @@ class ProtoSize { return field_id_size + varint(value); } /// 48-bit MAC address variant: matches encode_varint_raw_48bit's fast path. - /// When value is in [1<<42, 1<<48) the encoded varint is 7 bytes; values - /// outside that range fall back to the general size calculation so the - /// result stays correct if this helper is ever misapplied. + /// When any of the top 6 of 48 bits is set the encoded varint is 7 bytes; + /// otherwise fall back to the general size calculation. + /// Caller must guarantee value fits in 48 bits (encoder asserts in debug). static constexpr inline uint32_t ESPHOME_ALWAYS_INLINE calc_uint64_48bit_force(uint32_t field_id_size, uint64_t value) { - return field_id_size + ((value >= (1ULL << 42) && value < (1ULL << 48)) ? 7 : varint(value)); + return field_id_size + (value >= (1ULL << (MAC_ADDRESS_SIZE * 8 - 6)) ? 7 : varint(value)); } static constexpr uint32_t calc_length(uint32_t field_id_size, size_t len) { return len ? field_id_size + varint(static_cast(len)) + static_cast(len) : 0; From 946af918214da2cac8988ffc441bd18d103ccd6d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 25 Apr 2026 04:22:39 -0500 Subject: [PATCH 5/8] [api] Add unit tests for 48-bit MAC varint encoder Verifies encode_varint_raw_48bit and calc_uint64_48bit_force for the 8 corner-case MAC addresses requested in review: 00:00:00:00:00:00, 11:00:00:00:00:00, 00:AA:00:00:00:00, 00:00:BB:00:00:00, 00:00:00:CC:00:00, 00:00:00:00:DD:00, 00:00:00:00:00:EE, FF:FF:FF:FF:FF:FF For each value the test asserts byte-identical output to the reference encode_varint_raw_64 loop, the expected encoded byte length, agreement with calc_uint64_48bit_force, and round-trip through a generic varint decoder. Adds a 100-value deterministic-random sample across the full 48-bit space for additional coverage. --- .../components/api/test_proto_mac_varint.cpp | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/components/api/test_proto_mac_varint.cpp diff --git a/tests/components/api/test_proto_mac_varint.cpp b/tests/components/api/test_proto_mac_varint.cpp new file mode 100644 index 0000000000..7597c89806 --- /dev/null +++ b/tests/components/api/test_proto_mac_varint.cpp @@ -0,0 +1,119 @@ +#include + +#include +#include + +#include "esphome/components/api/api_buffer.h" +#include "esphome/components/api/proto.h" + +namespace esphome::api::testing { + +// Generic varint decoder, used to verify the encoded bytes round-trip back to +// the original 48-bit MAC value, independent of the specialized encoder under +// test. +static uint64_t decode_varint(const uint8_t *buf, size_t len, size_t *consumed) { + uint64_t value = 0; + int shift = 0; + for (size_t i = 0; i < len; i++) { + value |= static_cast(buf[i] & 0x7F) << shift; + if ((buf[i] & 0x80) == 0) { + *consumed = i + 1; + return value; + } + shift += 7; + } + *consumed = 0; + return 0; +} + +// Reference encoder mirroring ProtoEncode::encode_varint_raw_64. +static size_t reference_encode(uint64_t value, uint8_t *out) { + uint8_t *p = out; + if (value < 128) { + *p++ = static_cast(value); + return p - out; + } + do { + *p++ = static_cast(value | 0x80); + value >>= 7; + } while (value > 0x7F); + *p++ = static_cast(value); + return p - out; +} + +// Encode `mac` via the 48-bit fast path and verify: +// - byte-identical output to the reference loop +// - encoded byte length matches `expected_bytes` +// - calc_uint64_48bit_force agrees on the size +// - the bytes round-trip through a generic varint decoder +static void verify_mac(uint64_t mac, size_t expected_bytes) { + ASSERT_LT(mac, 1ULL << 48) << "test fixture mac exceeds 48 bits"; + + uint8_t ref_buf[16] = {0}; + size_t ref_len = reference_encode(mac, ref_buf); + + APIBuffer api_buf; + api_buf.resize(16); + uint8_t *pos = api_buf.data(); + PROTO_ENCODE_DEBUG_INIT(&api_buf); + ProtoEncode::encode_varint_raw_48bit(pos PROTO_ENCODE_DEBUG_ARG, mac); + size_t new_len = pos - api_buf.data(); + + EXPECT_EQ(new_len, expected_bytes) << "mac=0x" << std::hex << mac << std::dec; + EXPECT_EQ(ref_len, expected_bytes) << "reference disagrees on length for mac=0x" << std::hex << mac << std::dec; + + for (size_t i = 0; i < new_len; i++) { + EXPECT_EQ(api_buf.data()[i], ref_buf[i]) + << "byte " << i << " differs for mac=0x" << std::hex << mac << " (got 0x" << static_cast(api_buf.data()[i]) + << ", expected 0x" << static_cast(ref_buf[i]) << ")" << std::dec; + } + + size_t consumed = 0; + uint64_t decoded = decode_varint(api_buf.data(), new_len, &consumed); + EXPECT_EQ(consumed, new_len) << "decoder did not consume all bytes for mac=0x" << std::hex << mac << std::dec; + EXPECT_EQ(decoded, mac) << "round-trip mismatch for mac=0x" << std::hex << mac << std::dec; + + // Verify the size helper agrees. field_id_size = 1 (typical 1-byte tag). + uint32_t calc_size = ProtoSize::calc_uint64_48bit_force(1, mac); + EXPECT_EQ(calc_size, 1 + expected_bytes) + << "calc_uint64_48bit_force size mismatch for mac=0x" << std::hex << mac << std::dec; +} + +// Compute the canonical varint byte length for a value < 1<<48. +static size_t expected_varint_len(uint64_t v) { + if (v < (1ULL << 7)) + return 1; + if (v < (1ULL << 14)) + return 2; + if (v < (1ULL << 21)) + return 3; + if (v < (1ULL << 28)) + return 4; + if (v < (1ULL << 35)) + return 5; + if (v < (1ULL << 42)) + return 6; + return 7; +} + +// --- Specific MACs requested for verification --- + +TEST(ProtoMacVarint, AllZeros) { verify_mac(0x000000000000ULL, 1); } // 00:00:00:00:00:00 +TEST(ProtoMacVarint, FirstByteOnly) { verify_mac(0x110000000000ULL, 7); } // 11:00:00:00:00:00 +TEST(ProtoMacVarint, SecondByteOnly) { verify_mac(0x00AA00000000ULL, 6); } // 00:AA:00:00:00:00 +TEST(ProtoMacVarint, ThirdByteOnly) { verify_mac(0x0000BB000000ULL, 5); } // 00:00:BB:00:00:00 +TEST(ProtoMacVarint, FourthByteOnly) { verify_mac(0x000000CC0000ULL, 4); } // 00:00:00:CC:00:00 +TEST(ProtoMacVarint, FifthByteOnly) { verify_mac(0x00000000DD00ULL, 3); } // 00:00:00:00:DD:00 +TEST(ProtoMacVarint, SixthByteOnly) { verify_mac(0x0000000000EEULL, 2); } // 00:00:00:00:00:EE +TEST(ProtoMacVarint, AllOnes) { verify_mac(0xFFFFFFFFFFFFULL, 7); } // FF:FF:FF:FF:FF:FF + +// 100 deterministic-random 48-bit MACs to catch regressions across the space. +TEST(ProtoMacVarint, RandomSample) { + std::mt19937_64 rng(0xC0FFEE); + for (int i = 0; i < 100; i++) { + uint64_t mac = rng() & 0xFFFFFFFFFFFFULL; + verify_mac(mac, expected_varint_len(mac)); + } +} + +} // namespace esphome::api::testing From 68ffd3b22125ab0c9ed75e14893bf633dbeff0c3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 25 Apr 2026 04:36:32 -0500 Subject: [PATCH 6/8] [api] Fix CI errors in MAC varint unit tests - Test file: declare proto_debug_end_ locally instead of misusing PROTO_ENCODE_DEBUG_INIT (which expands to a comma+expression for appending to a function call, not a standalone statement). Add NOLINTNEXTLINE on the deterministic mt19937_64 seed so clang-tidy cert-msc32-c stops failing the build (the seed is intentional for reproducible test runs). - socket FILTER_SOURCE_FILES: tolerate non-dict CORE.config['socket'] (e.g. C++ unit-test builds where socket isn't validated as a mapping). Returning [] is safe -- all impl files are guarded by USE_SOCKET_IMPL_* defines so only the selected one contributes code. --- esphome/components/socket/__init__.py | 9 ++++++++- tests/components/api/test_proto_mac_varint.cpp | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index abbbb0f056..c7b2fec35f 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -177,7 +177,14 @@ async def to_code(config): def FILTER_SOURCE_FILES() -> list[str]: """Return list of socket implementation files that aren't selected by the user.""" - impl = CORE.config["socket"][CONF_IMPLEMENTATION] + socket_config = CORE.config.get("socket") + if not isinstance(socket_config, dict): + # Config has not been validated yet (or socket isn't configured as a + # mapping for this build, e.g. a C++ unit test build that skips config + # validation). Don't filter -- all impl files are guarded by USE_* + # defines so only the selected implementation contributes code anyway. + return [] + impl = socket_config[CONF_IMPLEMENTATION] # Build list of files to exclude based on selected implementation excluded = [] diff --git a/tests/components/api/test_proto_mac_varint.cpp b/tests/components/api/test_proto_mac_varint.cpp index 7597c89806..317a6fb9d6 100644 --- a/tests/components/api/test_proto_mac_varint.cpp +++ b/tests/components/api/test_proto_mac_varint.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include "esphome/components/api/api_buffer.h" @@ -55,7 +56,9 @@ static void verify_mac(uint64_t mac, size_t expected_bytes) { APIBuffer api_buf; api_buf.resize(16); uint8_t *pos = api_buf.data(); - PROTO_ENCODE_DEBUG_INIT(&api_buf); +#ifdef ESPHOME_DEBUG_API + uint8_t *proto_debug_end_ = api_buf.data() + api_buf.size(); +#endif ProtoEncode::encode_varint_raw_48bit(pos PROTO_ENCODE_DEBUG_ARG, mac); size_t new_len = pos - api_buf.data(); @@ -109,6 +112,7 @@ TEST(ProtoMacVarint, AllOnes) { verify_mac(0xFFFFFFFFFFFFULL, 7); } // F // 100 deterministic-random 48-bit MACs to catch regressions across the space. TEST(ProtoMacVarint, RandomSample) { + // NOLINTNEXTLINE(cert-msc32-c,cert-msc51-cpp) -- intentional fixed seed for reproducibility. std::mt19937_64 rng(0xC0FFEE); for (int i = 0; i < 100; i++) { uint64_t mac = rng() & 0xFFFFFFFFFFFFULL; From 45525c8a82c7ad1ee7f7e5bcfc6d27e7dde72481 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 25 Apr 2026 04:41:17 -0500 Subject: [PATCH 7/8] [api] Make api a buildable cpp unit test target Three fixes so 'script/cpp_unit_test.py api' actually compiles instead of crashing in build setup: 1. script/build_helpers.py: when adding transitive component dependencies to the post-validation config, use {} (dict) instead of [] (list) for non-MULTI_CONF components. socket's FILTER_SOURCE_FILES (and any other code that subscripts CORE.config[component] with a string key) was crashing because socket got config = [] from setdefault. 2. esphome/components/api/api_pb2_service.cpp + the codegen in script/api_protobuf/api_protobuf.py: wrap the generated APIConnection::read_message_ definition in #ifdef USE_API. The class itself is only declared inside #ifdef USE_API in api_connection.h, so without the guard the .cpp fails to compile in any build that pulls in the api source files without setting USE_API (e.g. cpp unit tests of api dependencies). --- esphome/components/api/api_pb2_service.cpp | 2 ++ esphome/components/socket/__init__.py | 9 +-------- script/api_protobuf/api_protobuf.py | 8 +++++++- script/build_helpers.py | 9 +++++++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/esphome/components/api/api_pb2_service.cpp b/esphome/components/api/api_pb2_service.cpp index 6ae2a3e369..0ba2961a13 100644 --- a/esphome/components/api/api_pb2_service.cpp +++ b/esphome/components/api/api_pb2_service.cpp @@ -21,6 +21,7 @@ void APIServerConnectionBase::log_receive_message_(const LogString *name) { } #endif +#ifdef USE_API void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const uint8_t *msg_data) { // Check authentication/connection requirements switch (msg_type) { @@ -706,5 +707,6 @@ void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const ui break; } } +#endif // USE_API } // namespace esphome::api diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index c7b2fec35f..abbbb0f056 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -177,14 +177,7 @@ async def to_code(config): def FILTER_SOURCE_FILES() -> list[str]: """Return list of socket implementation files that aren't selected by the user.""" - socket_config = CORE.config.get("socket") - if not isinstance(socket_config, dict): - # Config has not been validated yet (or socket isn't configured as a - # mapping for this build, e.g. a C++ unit test build that skips config - # validation). Don't filter -- all impl files are guarded by USE_* - # defines so only the selected implementation contributes code anyway. - return [] - impl = socket_config[CONF_IMPLEMENTATION] + impl = CORE.config["socket"][CONF_IMPLEMENTATION] # Build list of files to exclude based on selected implementation excluded = [] diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index a0dd26b1fe..bf672d0567 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -3577,8 +3577,13 @@ static const char *const TAG = "api.service"; # Generate read_message_ as APIConnection method (not base class) so the compiler # can devirtualize and inline the on_* handler calls within the same class. # APIConnection declares this method in api_connection.h. + # Guard with #ifdef USE_API since APIConnection itself is only defined when + # USE_API is set; without this, builds that compile this .cpp without + # USE_API (e.g. C++ unit tests for api dependencies) fail to find the + # class declaration. - out = "void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const uint8_t *msg_data) {\n" + out = "#ifdef USE_API\n" + out += "void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const uint8_t *msg_data) {\n" # Auth check block before dispatch switch out += " // Check authentication/connection requirements\n" @@ -3623,6 +3628,7 @@ static const char *const TAG = "api.service"; out += " break;\n" out += " }\n" out += "}\n" + out += "#endif // USE_API\n" cpp += out hpp += "};\n" diff --git a/script/build_helpers.py b/script/build_helpers.py index 1cfae51fca..59dded66dd 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -324,8 +324,13 @@ def compile_and_get_binary( domain_list.append({CONF_PLATFORM: component}) # Skip "core" — it's a pseudo-component handled by the build # system, not a real loadable component (get_component returns None) - elif get_component(component_name) is not None: - config.setdefault(component_name, []) + elif (component := get_component(component_name)) is not None: + # MULTI_CONF components store their config as a list of dicts, + # everything else stores a single dict. Using the wrong shape + # breaks code paths that subscript CORE.config[component] with + # a string key (e.g. socket.FILTER_SOURCE_FILES). + default = [] if component.multi_conf else {} + config.setdefault(component_name, default) # Register platforms from the extra config (benchmark.yaml) so # USE_SENSOR, USE_LIGHT, etc. defines are emitted without needing From 5f8e991ed8cc4725a3b64ec958bdf0a0885d4e66 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 25 Apr 2026 04:44:35 -0500 Subject: [PATCH 8/8] [api] Populate schema defaults for transitive cpp test deps; add json override - script/build_helpers.py: when injecting a non-MULTI_CONF component into the post-validation config, run its CONFIG_SCHEMA with {} so defaults are populated. Without this, socket got config = {} and socket.FILTER_SOURCE_FILES crashed with KeyError on 'implementation' (the schema's defaulted key was never filled in). Falls back to {} if the schema can't validate empty input. - tests/components/json/__init__.py: enable codegen for json so its to_code runs during cpp unit test builds, registering the ArduinoJson library. Required for any api dep test, since json_util.cpp #includes . Locally verified 'script/cpp_unit_test.py api' now compiles and runs; ProtoMacVarint test suite (9 cases) passes. --- script/build_helpers.py | 20 +++++++++++++++----- tests/components/json/__init__.py | 9 +++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 tests/components/json/__init__.py diff --git a/script/build_helpers.py b/script/build_helpers.py index 59dded66dd..4cf2f93fbb 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -326,11 +326,21 @@ def compile_and_get_binary( # system, not a real loadable component (get_component returns None) elif (component := get_component(component_name)) is not None: # MULTI_CONF components store their config as a list of dicts, - # everything else stores a single dict. Using the wrong shape - # breaks code paths that subscript CORE.config[component] with - # a string key (e.g. socket.FILTER_SOURCE_FILES). - default = [] if component.multi_conf else {} - config.setdefault(component_name, default) + # everything else stores a single dict. Run the component's + # schema with {} so defaults get populated -- code paths like + # socket.FILTER_SOURCE_FILES expect a fully-populated mapping. + if component.multi_conf: + config.setdefault(component_name, []) + elif component_name not in config: + schema = component.config_schema + try: + config[component_name] = schema({}) if schema is not None else {} + except Exception: # noqa: BLE001 + # Schema requires explicit input we can't synthesize; fall + # back to an empty mapping so subscripting at least returns + # KeyError on missing keys rather than crashing on the + # wrong type. + config[component_name] = {} # Register platforms from the extra config (benchmark.yaml) so # USE_SENSOR, USE_LIGHT, etc. defines are emitted without needing diff --git a/tests/components/json/__init__.py b/tests/components/json/__init__.py new file mode 100644 index 0000000000..40ec1f996e --- /dev/null +++ b/tests/components/json/__init__.py @@ -0,0 +1,9 @@ +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + # json's to_code calls cg.add_library("bblanchon/ArduinoJson", ...). C++ + # unit test builds that pull json in transitively (e.g. api) need that + # library registration to happen, otherwise json_util.cpp fails to find + # ArduinoJson.h. + manifest.enable_codegen()