[core] Add FixedVector::try_init so callers can handle an exhausted heap (#19253)

esphome/core/helpers.h

@@ -7,6 +7,7 @@
 #include <cstdarg>
 #include <cstdint>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <functional>
 #include <iterator>
@@ -38,6 +39,7 @@
 #endif
 
 #ifdef USE_ESP32
+#include <esp_system.h>
 #include <esp_heap_caps.h>
 #endif
 
@@ -539,7 +541,15 @@ template<typename T, size_t N> inline void init_array_from(std::array<T, N> &des
   }
 }
 
-/// Fixed-capacity vector - allocates once at runtime, never reallocates
+// Abort with a reason that reaches the panic output on ESP32. Elsewhere the literal is dropped
+// before it can land in rodata, which is RAM on ESP8266
+#ifdef USE_ESP32
+#define ESPHOME_ABORT_WITH_REASON(reason) esp_system_abort(reason)
+#else
+#define ESPHOME_ABORT_WITH_REASON(reason) abort()
+#endif
+
+/// Fixed-capacity vector - sized once through init() or try_init(); push_back never reallocates
 /// This avoids std::vector template overhead (_M_realloc_insert, _M_default_append)
 /// when size is known at initialization but not at compile time
 template<typename T> class FixedVector {
@@ -562,8 +572,7 @@ template<typename T> class FixedVector {
   void cleanup_() {
     if (data_ != nullptr) {
       destroy_elements_();
-      // Free raw memory
-      ::operator delete(data_);
+      free(data_);  // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-no-malloc)
     }
   }
 
@@ -632,16 +641,27 @@ template<typename T> class FixedVector {
   // Allocate capacity - can be called multiple times to reinit
   // IMPORTANT: After calling init(), you MUST use push_back() to add elements.
   // Direct assignment via operator[] does NOT update the size counter.
+  // Aborts on exhaustion; use try_init() to handle failure.
   void init(size_t n) {
+    if (!try_init(n))
+      ESPHOME_ABORT_WITH_REASON("FixedVector: out of memory");
+  }
+
+  // Same as init(), but returns false when memory is exhausted; the previous storage is freed either way
+  bool try_init(size_t n) {
     cleanup_();
     reset_();
-    if (n > 0) {
-      // Allocate raw memory without calling constructors
-      // sizeof(T) is correct here for any type T (value types, pointers, etc.)
-      // NOLINTNEXTLINE(bugprone-sizeof-expression)
-      data_ = static_cast<T *>(::operator new(n * sizeof(T)));
-      capacity_ = n;
-    }
+    if (n == 0)
+      return true;
+    if (n > SIZE_MAX / sizeof(T))
+      return false;  // the byte count would wrap into a small block
+    // sizeof(T) is correct here for any type T (value types, pointers, etc.)
+    // NOLINTNEXTLINE(bugprone-sizeof-expression,cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory)
+    data_ = static_cast<T *>(malloc(n * sizeof(T)));
+    if (data_ == nullptr)
+      return false;
+    capacity_ = n;
+    return true;
   }
 
   // Clear the vector (destroy all elements, reset size to 0, keep capacity)
@@ -738,14 +758,22 @@ template<typename T> class FixedVector {
 template<size_t STACK_SIZE, typename T = uint8_t> class SmallBufferWithHeapFallback {
  public:
   explicit SmallBufferWithHeapFallback(size_t size) {
+    static_assert(std::is_trivially_default_constructible_v<T> && std::is_trivially_destructible_v<T>,
+                  "the heap fallback leaves elements unconstructed");
     if (size <= STACK_SIZE) {
       this->buffer_ = this->stack_buffer_;
     } else {
-      this->heap_buffer_ = new T[size];
+      if (size <= SIZE_MAX / sizeof(T)) {
+        // NOLINTNEXTLINE(bugprone-sizeof-expression,cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory)
+        this->heap_buffer_ = static_cast<T *>(malloc(size * sizeof(T)));
+      }
+      // Callers write through get() unchecked, so exhaustion aborts like the new[] it replaces
+      if (this->heap_buffer_ == nullptr)
+        ESPHOME_ABORT_WITH_REASON("SmallBufferWithHeapFallback: out of memory");
       this->buffer_ = this->heap_buffer_;
     }
   }
-  ~SmallBufferWithHeapFallback() { delete[] this->heap_buffer_; }
+  ~SmallBufferWithHeapFallback() { free(this->heap_buffer_); }  // NOLINT(cppcoreguidelines-no-malloc)
 
   // Delete copy and move operations to prevent double-delete
   SmallBufferWithHeapFallback(const SmallBufferWithHeapFallback &) = delete;

script/cpp_unit_test.py

@@ -36,7 +36,8 @@ PLATFORMIO_OPTIONS = {
 
 
 def run_tests(selected_components: list[str]) -> int:
-    os.environ["ASAN_OPTIONS"] = "detect_leaks=0"
+    # allocator_may_return_null: an oversized request must come back empty, not abort the run
+    os.environ["ASAN_OPTIONS"] = "detect_leaks=0:allocator_may_return_null=1"
     return build_and_run(
         selected_components=selected_components,
         tests_dir=COMPONENTS_TESTS_DIR,

tests/components/core/test_helpers.cpp

@@ -348,4 +348,23 @@ TEST(StepToAccuracyDecimals, NonFiniteAndZero) {
   EXPECT_EQ(step_to_accuracy_decimals(-INFINITY), 0);
 }
 
+// --- FixedVector::try_init() ---
+
+// Keeps the block observable, else the compiler may drop the malloc and free pair and fold the check
+static void escape(const void *p) { asm volatile("" : : "g"(p) : "memory"); }
+
+TEST(FixedVectorTryInit, ReportsExhaustionAndStaysEmpty) {
+  FixedVector<uint32_t> v;
+  const bool ok = v.try_init(SIZE_MAX / sizeof(uint32_t));
+  escape(&v);
+  EXPECT_FALSE(ok);
+  EXPECT_EQ(v.capacity(), 0u);
+  EXPECT_FALSE(v.try_init(SIZE_MAX / sizeof(uint32_t) + 1));  // byte count would wrap
+  EXPECT_EQ(v.capacity(), 0u);
+  EXPECT_TRUE(v.try_init(0));
+  EXPECT_TRUE(v.try_init(4));
+  v.push_back(7);
+  EXPECT_EQ(v.size(), 1u);
+}
+
 }  // namespace esphome::core::testing