Import the service-table materializer from #18198

The Bluedroid on-demand materializer, the neutral lookup helpers' host
tests, and the USE_BLE_GATT_SERVICE_TABLE define move here, next to
their first consumer (the neutral engine's table resolution and the
service_table codegen flag). One behavior fix rides along: hitting
MAX_DESCRIPTORS_PER_CHARACTERISTIC now fails the walk like every other
inconsistency instead of truncating the table silently.

esphome/components/ble_device_base/ble_gatt_client.cpp

@@ -0,0 +1,43 @@
+#include "ble_gatt_client.h"
+
+#ifdef USE_BLE_GATT_CLIENT
+
+#include "esphome/core/log.h"
+
+namespace esphome::ble_device_base {
+
+static const char *const TAG = "ble_gatt_client";
+
+const GattCharacteristic *find_characteristic(const GattServiceTable &table, const GattService &service,
+                                              const ESPBTUUID &uuid) {
+  // 32-bit range math: a corrupt first/count pair cannot wrap past the check.
+  uint32_t end = uint32_t(service.first_characteristic) + service.characteristic_count;
+  if (end > table.characteristic_count) {
+    ESP_LOGW(TAG, "characteristic range out of bounds");
+    return nullptr;
+  }
+  for (uint32_t i = service.first_characteristic; i < end; i++) {
+    if (table.characteristics[i].uuid == uuid)
+      return &table.characteristics[i];
+  }
+  return nullptr;
+}
+
+uint16_t find_cccd(const GattServiceTable &table, const GattCharacteristic &characteristic) {
+  uint32_t end = uint32_t(characteristic.first_descriptor) + characteristic.descriptor_count;
+  if (end > table.descriptor_count) {
+    // Corrupt range, not a missing CCCD.
+    ESP_LOGW(TAG, "descriptor range out of bounds");
+    return 0;
+  }
+  const ESPBTUUID cccd_uuid = ESPBTUUID::from_uint16(CCCD_UUID);
+  for (uint32_t i = characteristic.first_descriptor; i < end; i++) {
+    if (table.descriptors[i].uuid == cccd_uuid)
+      return table.descriptors[i].handle;
+  }
+  return 0;
+}
+
+}  // namespace esphome::ble_device_base
+
+#endif  // USE_BLE_GATT_CLIENT

esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp

@@ -33,6 +33,10 @@ using esp32_ble_tracker::ClientState;
 using esp32_ble_tracker::ConnectionType;
 
 static constexpr uint16_t UNSET_CONN_ID = 0xFFFF;
+// Bounds one characteristic's descriptor walk against a stack that never
+// reports end-of-range.
+static constexpr uint16_t MAX_DESCRIPTORS_PER_CHARACTERISTIC = 64;
+
 // ---- tracker surface ----
 
 void BluedroidGattClient::connect() { this->tracker_connect_(); }
@@ -289,6 +293,9 @@ int BluedroidGattClient::update_connection_params(uint16_t min_interval, uint16_
 
 void BluedroidGattClient::release_services() {
   this->service_total_ = 0;
+#ifdef USE_BLE_GATT_SERVICE_TABLE
+  this->free_service_table_();
+#endif
   // Always set: terminates any in-flight stream on every cache config.
   this->services_released_ = true;
 #ifndef CONFIG_BT_GATTC_CACHE_NVS_FLASH
@@ -296,6 +303,188 @@ void BluedroidGattClient::release_services() {
 #endif
 }
 
+#ifdef USE_BLE_GATT_SERVICE_TABLE
+ble_device_base::GattServiceTable BluedroidGattClient::get_service_table() {
+  if (this->table_storage_ == nullptr &&
+      (this->services_released_ || this->service_total_ == 0 || !this->build_service_table_())) {
+    return {};
+  }
+  return this->table_view_();
+}
+
+// The view is carved from the storage block and the counts on each call
+// (a cold path) rather than cached, saving a per-instance table member.
+ble_device_base::GattServiceTable BluedroidGattClient::table_view_() const {
+  size_t svc_bytes = this->service_total_ * sizeof(ble_device_base::GattService);
+  size_t char_bytes = this->table_char_total_ * sizeof(ble_device_base::GattCharacteristic);
+  return {reinterpret_cast<const ble_device_base::GattService *>(this->table_storage_),
+          reinterpret_cast<const ble_device_base::GattCharacteristic *>(this->table_storage_ + svc_bytes),
+          reinterpret_cast<const ble_device_base::GattDescriptor *>(this->table_storage_ + svc_bytes + char_bytes),
+          this->service_total_,
+          this->table_char_total_,
+          this->table_desc_total_};
+}
+
+void BluedroidGattClient::free_service_table_() {
+  if (this->table_storage_ == nullptr) {
+    return;
+  }
+  RAMAllocator<uint8_t> allocator(RAMAllocator<uint8_t>::ALLOC_INTERNAL);
+  allocator.deallocate(this->table_storage_, 0);
+  this->table_storage_ = nullptr;
+  this->table_char_total_ = 0;
+  this->table_desc_total_ = 0;
+}
+
+template<typename ServiceFn, typename CharFn, typename DescFn>
+bool BluedroidGattClient::walk_database_(ServiceFn &&on_service, CharFn &&on_char, DescFn &&on_desc) {
+  // Shared enumeration for both table-build passes: an identical walk order
+  // is what lets the counting pass size the block the filling pass fills.
+  // INVALID_OFFSET/NOT_FOUND mean end-of-range; anything else is a failure.
+  for (uint16_t s = 0; s < this->service_total_; s++) {
+    esp_gattc_service_elem_t svc;
+    uint16_t svc_count = 1;
+    if (esp_ble_gattc_get_service(this->gattc_if_, this->conn_id_, nullptr, &svc, &svc_count, s) != ESP_GATT_OK ||
+        svc_count == 0) {
+      return false;
+    }
+    if (!on_service(s, svc)) {
+      return false;
+    }
+    uint16_t svc_chars = 0;
+    if (esp_ble_gattc_get_attr_count(this->gattc_if_, this->conn_id_, ESP_GATT_DB_CHARACTERISTIC, svc.start_handle,
+                                     svc.end_handle, 0, &svc_chars) != ESP_GATT_OK) {
+      return false;
+    }
+    for (uint16_t c = 0; c < svc_chars; c++) {
+      esp_gattc_char_elem_t chr;
+      uint16_t char_count = 1;
+      auto status = esp_ble_gattc_get_all_char(this->gattc_if_, this->conn_id_, svc.start_handle, svc.end_handle, &chr,
+                                               &char_count, c);
+      if (status != ESP_GATT_OK || char_count == 0) {
+        // An early terminator contradicts svc_chars from the same cache;
+        // never build a silently truncated table.
+        return false;
+      }
+      if (!on_char(svc, chr)) {
+        return false;
+      }
+      for (uint16_t d = 0;; d++) {
+        if (d == MAX_DESCRIPTORS_PER_CHARACTERISTIC) {
+          // A stack that never reports end-of-range; fail like every other
+          // inconsistency instead of truncating the table silently.
+          return false;
+        }
+        esp_gattc_descr_elem_t desc;
+        uint16_t desc_count = 1;
+        auto desc_status =
+            esp_ble_gattc_get_all_descr(this->gattc_if_, this->conn_id_, chr.char_handle, &desc, &desc_count, d);
+        if (desc_status == ESP_GATT_INVALID_OFFSET || desc_status == ESP_GATT_NOT_FOUND) {
+          break;
+        }
+        if (desc_status != ESP_GATT_OK || desc_count == 0) {
+          return false;
+        }
+        if (!on_desc(chr, desc)) {
+          return false;
+        }
+      }
+    }
+  }
+  return true;
+}
+
+bool BluedroidGattClient::build_service_table_() {
+  // Pass 1: count, so one exact-size block holds the whole table.
+  uint16_t char_total = 0;
+  uint16_t desc_total = 0;
+  bool counted = this->walk_database_([](uint16_t, const esp_gattc_service_elem_t &) { return true; },
+                                      [&](const esp_gattc_service_elem_t &, const esp_gattc_char_elem_t &) {
+                                        char_total++;
+                                        return true;
+                                      },
+                                      [&](const esp_gattc_char_elem_t &, const esp_gattc_descr_elem_t &) {
+                                        desc_total++;
+                                        return true;
+                                      });
+  if (!counted) {
+    ESP_LOGW(TAG, "[%d] Service table walk failed during count", this->connection_index_);
+    return false;
+  }
+
+  // The arrays share one block; carving stays aligned because each struct's
+  // strictest member is the UUID and array sizes are multiples of it.
+  size_t svc_bytes = this->service_total_ * sizeof(ble_device_base::GattService);
+  size_t char_bytes = char_total * sizeof(ble_device_base::GattCharacteristic);
+  size_t total_bytes = svc_bytes + char_bytes + desc_total * sizeof(ble_device_base::GattDescriptor);
+  RAMAllocator<uint8_t> allocator(RAMAllocator<uint8_t>::ALLOC_INTERNAL);
+  this->table_storage_ = allocator.allocate(total_bytes);
+  if (this->table_storage_ == nullptr) {
+    ESP_LOGW(TAG, "[%d] Service table allocation failed (%u bytes)", this->connection_index_,
+             static_cast<unsigned>(total_bytes));
+    return false;
+  }
+  auto *services = reinterpret_cast<ble_device_base::GattService *>(this->table_storage_);
+  auto *characteristics = reinterpret_cast<ble_device_base::GattCharacteristic *>(this->table_storage_ + svc_bytes);
+  auto *descriptors =
+      reinterpret_cast<ble_device_base::GattDescriptor *>(this->table_storage_ + svc_bytes + char_bytes);
+
+  // Pass 2: fill, bounded by the pass-1 totals. A bound trip or a shortfall
+  // means the cached database changed between the passes; fail the build
+  // rather than serve an inconsistent table (the consumer retries).
+  uint16_t char_index = 0;
+  uint16_t desc_index = 0;
+  ble_device_base::GattService *cur_service = nullptr;
+  ble_device_base::GattCharacteristic *cur_char = nullptr;
+  bool filled = this->walk_database_(
+      [&](uint16_t s, const esp_gattc_service_elem_t &svc) {
+        cur_service = &services[s];
+        cur_service->uuid = ble_device_base::ESPBTUUID::from_uuid(svc.uuid);
+        cur_service->start_handle = svc.start_handle;
+        cur_service->end_handle = svc.end_handle;
+        cur_service->first_characteristic = char_index;
+        cur_service->characteristic_count = 0;
+        return true;
+      },
+      [&](const esp_gattc_service_elem_t &svc, const esp_gattc_char_elem_t &chr) {
+        if (char_index >= char_total) {
+          return false;
+        }
+        cur_char = &characteristics[char_index++];
+        cur_char->uuid = ble_device_base::ESPBTUUID::from_uuid(chr.uuid);
+        cur_char->value_handle = chr.char_handle;
+        // Bluedroid addresses descriptors by characteristic handle, so the
+        // table's end_handle only needs the service-bounded upper bound.
+        cur_char->end_handle = svc.end_handle;
+        cur_char->properties = chr.properties;
+        cur_char->first_descriptor = desc_index;
+        cur_char->descriptor_count = 0;
+        cur_service->characteristic_count++;
+        return true;
+      },
+      [&](const esp_gattc_char_elem_t &, const esp_gattc_descr_elem_t &desc) {
+        if (desc_index >= desc_total) {
+          return false;
+        }
+        descriptors[desc_index].uuid = ble_device_base::ESPBTUUID::from_uuid(desc.uuid);
+        descriptors[desc_index].handle = desc.handle;
+        desc_index++;
+        cur_char->descriptor_count++;
+        return true;
+      });
+  if (!filled || char_index != char_total || desc_index != desc_total) {
+    // Walk error or the database changed between passes; better an empty
+    // table than a corrupt one.
+    ESP_LOGW(TAG, "[%d] Service table walk mismatch, discarding", this->connection_index_);
+    this->free_service_table_();
+    return false;
+  }
+  this->table_char_total_ = char_total;
+  this->table_desc_total_ = desc_total;
+  return true;
+}
+#endif  // USE_BLE_GATT_SERVICE_TABLE
+
 // ---- internals ----
 
 bool BluedroidGattClient::check_addr_(const esp_bd_addr_t &addr) const {
@@ -342,6 +531,11 @@ void BluedroidGattClient::log_gattc_warning_(const char *operation, int code) {
 // ---- service streaming ----
 
 int BluedroidGattClient::handle_search_cmpl_(esp_gatt_status_t status) {
+#ifdef USE_BLE_GATT_SERVICE_TABLE
+  // Re-discovery moves the counts table_view_() derives offsets from; free
+  // the stale table.
+  this->free_service_table_();
+#endif
   // Step down from the fast discovery params.
   this->update_conn_params_(MEDIUM_MIN_CONN_INTERVAL, MEDIUM_MAX_CONN_INTERVAL, 0, MEDIUM_CONN_TIMEOUT, "medium");
   if (status != ESP_GATT_OK) {

esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h

@@ -71,9 +71,14 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public
   int notify_characteristic(uint16_t handle, bool enable);
   int pair();
   int update_connection_params(uint16_t min_interval, uint16_t max_interval, uint16_t latency, uint16_t timeout);
-  // Contract stub: the proxy streams in place; the on-demand materializer
-  // for direct consumers lands with #18205.
+  // On-demand table for direct consumers; the proxy streams instead, so the
+  // materializer compiles only under USE_BLE_GATT_SERVICE_TABLE (emitted by
+  // direct-consumer codegen, never by the proxy).
+#ifdef USE_BLE_GATT_SERVICE_TABLE
+  ble_device_base::GattServiceTable get_service_table();
+#else
   ble_device_base::GattServiceTable get_service_table() { return {}; }
+#endif
   void release_services();
 
 #ifdef BLUETOOTH_CONNECTION_SERVES_PROXY
@@ -99,9 +104,21 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public
                                 const char *param_type);
   int check_and_log_error_(const char *operation, esp_err_t err);
   void log_gattc_warning_(const char *operation, int code);
+#ifdef USE_BLE_GATT_SERVICE_TABLE
+  template<typename ServiceFn, typename CharFn, typename DescFn>
+  bool walk_database_(ServiceFn &&on_service, CharFn &&on_char, DescFn &&on_desc);
+  bool build_service_table_();
+  void free_service_table_();
+  ble_device_base::GattServiceTable table_view_() const;
+#endif
 
   // Group 1: pointers / composed objects
   ble_device_base::GattClientListener *listener_{nullptr};
+#ifdef USE_BLE_GATT_SERVICE_TABLE
+  // One exact-size block carved into the three arrays; the view is rebuilt
+  // per (cold) call instead of cached.
+  uint8_t *table_storage_{nullptr};
+#endif
   // Group 2: 4-byte types
   uint32_t disconnecting_started_{0};
 
@@ -111,6 +128,11 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public
   // Group 4: 2-byte types
   uint16_t conn_id_{0xFFFF};
   uint16_t service_total_{0};
+#ifdef USE_BLE_GATT_SERVICE_TABLE
+  // Filled element counts of the materialized table (0 when none).
+  uint16_t table_char_total_{0};
+  uint16_t table_desc_total_{0};
+#endif
 
   // Group 5: 1-byte types
   esp_gatt_if_t gattc_if_{ESP_GATT_IF_NONE};  // uint8_t width keeps the object at 48 bytes

esphome/core/defines.h

@@ -307,6 +307,7 @@
 #define USE_ESP32_BLE_SERVER_ON_DISCONNECT
 #define USE_ESP32_BLE_TRACKER
 #define USE_BLE_GATT_CLIENT
+#define USE_BLE_GATT_SERVICE_TABLE
 #define ESPHOME_BLE_GATT_CLIENT_COUNT 1
 #define ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT 1
 #define ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT 1

tests/components/ble_device_base/test_gatt_client_contract.cpp

@@ -78,4 +78,54 @@ TEST(BleGattClientContract, MinimalImplementerCompilesAndRoutesEvents) {
   EXPECT_EQ(table.descriptor_count, 0);
 }
 
+// A radon_eye_rd200-shaped table: two services, the second holding a
+// notifying characteristic with a CCCD and a bare write characteristic.
+class ServiceTableLookup : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    this->services_[0] = {ESPBTUUID::from_uint16(0x1800), 0x0001, 0x0005, 0, 1};
+    this->services_[1] = {ESPBTUUID::from_uint16(0x1523), 0x0010, 0x0020, 1, 2};
+    this->characteristics_[0] = {ESPBTUUID::from_uint16(0x2A00), 0x0003, 0x0003, 0x02, 0, 0};
+    this->characteristics_[1] = {ESPBTUUID::from_uint16(0x1525), 0x0012, 0x0014, 0x10, 0, 1};
+    this->characteristics_[2] = {ESPBTUUID::from_uint16(0x1524), 0x0016, 0x0016, 0x04, 1, 0};
+    this->descriptors_[0] = {ESPBTUUID::from_uint16(0x2902), 0x0013};
+    this->table_ = {this->services_, this->characteristics_, this->descriptors_, 2, 3, 1};
+  }
+
+  GattService services_[2];
+  GattCharacteristic characteristics_[3];
+  GattDescriptor descriptors_[1];
+  GattServiceTable table_;
+};
+
+TEST_F(ServiceTableLookup, FindsServicesAndCharacteristicsByUuid) {
+  const GattService *service = find_service(this->table_, ESPBTUUID::from_uint16(0x1523));
+  ASSERT_NE(service, nullptr);
+  EXPECT_EQ(service->start_handle, 0x0010);
+  EXPECT_EQ(find_service(this->table_, ESPBTUUID::from_uint16(0xFFFF)), nullptr);
+
+  const GattCharacteristic *characteristic =
+      find_characteristic(this->table_, *service, ESPBTUUID::from_uint16(0x1525));
+  ASSERT_NE(characteristic, nullptr);
+  EXPECT_EQ(characteristic->value_handle, 0x0012);
+  // The lookup is scoped to the service: 0x2A00 lives in the other service.
+  EXPECT_EQ(find_characteristic(this->table_, *service, ESPBTUUID::from_uint16(0x2A00)), nullptr);
+}
+
+TEST_F(ServiceTableLookup, FindsTheCccdAndReportsItsAbsence) {
+  const GattService *service = find_service(this->table_, ESPBTUUID::from_uint16(0x1523));
+  const GattCharacteristic *notify_char = find_characteristic(this->table_, *service, ESPBTUUID::from_uint16(0x1525));
+  EXPECT_EQ(find_cccd(this->table_, *notify_char), 0x0013);
+  const GattCharacteristic *write_char = find_characteristic(this->table_, *service, ESPBTUUID::from_uint16(0x1524));
+  EXPECT_EQ(find_cccd(this->table_, *write_char), 0);
+}
+
+TEST_F(ServiceTableLookup, RejectsRangesThatOverrunTheTable) {
+  // A corrupt index range must fail the lookup, not walk out of bounds.
+  GattService bad_service = {ESPBTUUID::from_uint16(0x1523), 0x0010, 0x0020, 2, 5};
+  EXPECT_EQ(find_characteristic(this->table_, bad_service, ESPBTUUID::from_uint16(0x1524)), nullptr);
+  GattCharacteristic bad_char = {ESPBTUUID::from_uint16(0x1525), 0x0012, 0x0014, 0x10, 0, 9};
+  EXPECT_EQ(find_cccd(this->table_, bad_char), 0);
+}
+
 }  // namespace esphome::ble_device_base::testing