[api] Add a progmem option for bytes fields kept in flash

The generated encode and dump of a pointer_to_buffer bytes field marked
(progmem) copy the data with progmem_memcpy, so ESP8266 can send it from
flash. The device wizard uses it in place of its hand written encoder.

esphome/components/api/api.proto

@@ -468,11 +468,9 @@ message DeviceWizardResponse {
   option (id) = 157;
   option (source) = SOURCE_SERVER;
   option (ifdef) = "USE_API_WIZARD";
-  // Not logged: the data is in flash, which ESP8266 cannot read for a dump
-  option (log) = false;
 
   // The data is in flash, and the device sends it from there
-  bytes data = 1 [(pointer_to_buffer) = true];
+  bytes data = 1 [(pointer_to_buffer) = true, (progmem) = true];
 }
 
 // Sets the entity id of a wizard input (see WizardInputField.key). Clients send

esphome/components/api/api_connection.h

@@ -423,9 +423,6 @@ class APIConnection final : public APIServerConnectionBase {
   bool send_ping_response_();
   bool send_device_info_response_();
   bool send_device_capabilities_response_();
-#ifdef USE_API_WIZARD
-  bool send_device_wizard_response_();
-#endif
 #ifdef USE_API_NOISE
   bool send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg);
 #endif

esphome/components/api/api_options.proto

@@ -122,4 +122,9 @@ extend google.protobuf.FieldOptions {
     // to true when the field arrives, so an all-default submessage can be told apart
     // from an absent one (e.g. a UTC ParsedTimezone, which is all zeros).
     optional bool track_presence = 50020 [default=false];
+
+    // progmem: The pointer_to_buffer bytes field points into flash (PROGMEM).
+    // The generated encode and dump copy it with progmem_memcpy, as ESP8266
+    // cannot read flash with ordinary loads.
+    optional bool progmem = 50021 [default=false];
 }

esphome/components/api/api_pb2.cpp

@@ -352,7 +352,7 @@ uint32_t DeviceCapabilitiesResponse::calc_size_msg(const void *self) {
 #ifdef USE_API_WIZARD
 uint8_t *DeviceWizardResponse::encode_msg(const void *self, uint8_t *__restrict__ pos PROTO_ENCODE_DEBUG_PARAM) {
   const auto &msg = *static_cast<const DeviceWizardResponse *>(self);
-  pos = ProtoEncode::encode_bytes(pos PROTO_ENCODE_DEBUG_ARG, 1, msg.data, msg.data_len);
+  pos = ProtoEncode::encode_progmem_bytes(pos PROTO_ENCODE_DEBUG_ARG, 1, msg.data, msg.data_len);
   return pos;
 }
 uint32_t DeviceWizardResponse::calc_size_msg(const void *self) {

esphome/components/api/api_pb2_dump.cpp

@@ -123,6 +123,15 @@ static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint
   format_hex_pretty_to(hex_buf, data, len);
   out.append(hex_buf).append("\n");
 }
+
+// Helper for bytes fields in flash: copies the shown bytes out with progmem_memcpy first
+static void dump_progmem_bytes_field(DumpBuffer &out, const char *field_name, const uint8_t *data, size_t len,
+                                     int indent = 2) {
+  uint8_t data_buf[160];
+  len = std::min(len, sizeof(data_buf));
+  progmem_memcpy(data_buf, data, len);
+  dump_bytes_field(out, field_name, data_buf, len, indent);
+}
 #pragma GCC diagnostic pop
 
 template<> const char *proto_enum_to_string<enums::DisconnectReason>(enums::DisconnectReason value) {
@@ -1119,7 +1128,7 @@ const char *DeviceCapabilitiesResponse::dump_to(DumpBuffer &out) const {
 #ifdef USE_API_WIZARD
 const char *DeviceWizardResponse::dump_to(DumpBuffer &out) const {
   MessageDumpHelper helper(out, ESPHOME_PSTR("DeviceWizardResponse"));
-  dump_bytes_field(out, ESPHOME_PSTR("data"), this->data, this->data_len);
+  dump_progmem_bytes_field(out, ESPHOME_PSTR("data"), this->data, this->data_len);
   return out.c_str();
 }
 #endif

esphome/components/api/api_wizard.cpp

@@ -13,17 +13,6 @@ namespace esphome::api {
 
 static const char *const TAG = "api.wizard";
 
-uint8_t *wizard_encode_response(const void *self, uint8_t *__restrict__ pos PROTO_ENCODE_DEBUG_PARAM) {
-  const auto &msg = *static_cast<const DeviceWizardResponse *>(self);
-  if (msg.data_len == 0)
-    return pos;
-  pos = ProtoEncode::encode_field_raw(pos PROTO_ENCODE_DEBUG_ARG, 1, 2);  // type 2: Length-delimited
-  pos = ProtoEncode::encode_varint_raw(pos PROTO_ENCODE_DEBUG_ARG, msg.data_len);
-  PROTO_ENCODE_CHECK_BOUNDS(pos, msg.data_len);
-  progmem_memcpy(pos, msg.data, msg.data_len);
-  return pos + msg.data_len;
-}
-
 #ifdef USE_API_WIZARD_INPUTS
 static bool wizard_entity_id_valid(const char *entity_id, size_t length) {
   return length > 0 && length < WIZARD_ENTITY_ID_BUFFER_SIZE && memchr(entity_id, '.', length) != nullptr;
@@ -49,17 +38,11 @@ const char *wizard_set_input(const WizardInputSetRequest &msg) {
 }
 #endif  // USE_API_WIZARD_INPUTS
 
-bool APIConnection::send_device_wizard_response_() {
+void APIConnection::on_device_wizard_request() {
   DeviceWizardResponse resp;
   resp.data = API_WIZARD_DATA;
   resp.data_len = API_WIZARD_DATA_SIZE;
-  // Not send_message: the data is in flash, so wizard_encode_response copies it out
-  return this->send_message_(DeviceWizardResponse::calc_size_msg(&resp), DeviceWizardResponse::MESSAGE_TYPE,
-                             &wizard_encode_response, &resp);
-}
-
-void APIConnection::on_device_wizard_request() {
-  if (!this->send_device_wizard_response_()) {
+  if (!this->send_message(resp)) {
     this->on_fatal_error();
   }
 }

esphome/components/api/api_wizard.h

@@ -22,10 +22,6 @@ static constexpr size_t WIZARD_ENTITY_ID_BUFFER_SIZE = 256;
 /// (components/api/wizard.py) and kept in flash. API_WIZARD_DATA_SIZE bytes long.
 extern const uint8_t API_WIZARD_DATA[] PROGMEM;
 
-/// Encodes a DeviceWizardResponse like the generated encoder would. The data is in flash, which ESP8266 can only read
-/// with progmem_memcpy, so the generated encoder (a plain memcpy) cannot be used. Plain memcpy elsewhere.
-uint8_t *wizard_encode_response(const void *self, uint8_t *pos PROTO_ENCODE_DEBUG_PARAM);
-
 #ifdef USE_API_WIZARD_INPUTS
 /// Where the entity id of an input is kept, found by the key the client uses for it.
 struct WizardInputEntry {

esphome/components/api/proto.h

@@ -3,6 +3,7 @@
 #include "api_pb2_defines.h"
 #include "api_buffer.h"
 #include "esphome/core/component.h"
+#include "esphome/core/hal.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 #include "esphome/core/progmem.h"
@@ -465,6 +466,21 @@ class ProtoEncode {
                                                           uint32_t field_id, const uint8_t *data, size_t len) {
     return encode_string_force(pos PROTO_ENCODE_DEBUG_ARG, field_id, reinterpret_cast<const char *>(data), len);
   }
+  /// encode_bytes for data in flash (PROGMEM), which ESP8266 can only read with progmem_memcpy.
+  [[nodiscard]] static inline uint8_t *encode_progmem_bytes(uint8_t *__restrict__ pos PROTO_ENCODE_DEBUG_PARAM,
+                                                            uint32_t field_id, const uint8_t *data, size_t len) {
+    if (len == 0)
+      return pos;
+    return encode_progmem_bytes_force(pos PROTO_ENCODE_DEBUG_ARG, field_id, data, len);
+  }
+  [[nodiscard]] static inline uint8_t *encode_progmem_bytes_force(uint8_t *__restrict__ pos PROTO_ENCODE_DEBUG_PARAM,
+                                                                  uint32_t field_id, const uint8_t *data, size_t len) {
+    pos = encode_field_raw(pos PROTO_ENCODE_DEBUG_ARG, field_id, 2);  // type 2: Length-delimited
+    pos = encode_varint_raw(pos PROTO_ENCODE_DEBUG_ARG, len);
+    PROTO_ENCODE_CHECK_BOUNDS(pos, len);
+    progmem_memcpy(pos, data, len);
+    return pos + len;
+  }
   [[nodiscard]] static inline uint8_t *encode_uint32_force(uint8_t *__restrict__ pos PROTO_ENCODE_DEBUG_PARAM,
                                                            uint32_t field_id, uint32_t value) {
     pos = encode_field_raw(pos PROTO_ENCODE_DEBUG_ARG, field_id, 0);

script/api_protobuf/api_protobuf.py

@@ -1158,6 +1158,14 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):
     reference_type = "const uint8_t*"
     const_reference_type = "const uint8_t*"
 
+    @property
+    def progmem(self) -> bool:
+        """Whether the data is in flash, so encode and dump copy it with progmem_memcpy."""
+        progmem_opt = getattr(pb, "progmem", None)
+        return progmem_opt is not None and get_field_opt(
+            self._field, progmem_opt, False
+        )
+
     @property
     def public_content(self) -> list[str]:
         # Use uint16_t for length - max packet size is well below 65535
@@ -1168,6 +1176,14 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):
 
     @property
     def encode_content(self) -> str:
+        if self.progmem:
+            return _encode_call(
+                "encode_progmem_bytes",
+                str(self.number),
+                f"this->{self.field_name}",
+                f"this->{self.field_name}_len",
+                force=self.force,
+            )
         if result := self._encode_bytes_with_precomputed_tag(
             f"this->{self.field_name}", f"this->{self.field_name}_len"
         ):
@@ -1194,8 +1210,9 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):
 
     @property
     def dump_content(self) -> str:
+        dump_fn = "dump_progmem_bytes_field" if self.progmem else "dump_bytes_field"
         return (
-            f'dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
+            f'{dump_fn}(out, ESPHOME_PSTR("{self.name}"), '
             f"this->{self.field_name}, this->{self.field_name}_len);"
         )
 
@@ -3328,6 +3345,15 @@ static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint
   format_hex_pretty_to(hex_buf, data, len);
   out.append(hex_buf).append("\\n");
 }
+
+// Helper for bytes fields in flash: copies the shown bytes out with progmem_memcpy first
+static void dump_progmem_bytes_field(DumpBuffer &out, const char *field_name, const uint8_t *data, size_t len,
+                                     int indent = 2) {
+  uint8_t data_buf[160];
+  len = std::min(len, sizeof(data_buf));
+  progmem_memcpy(data_buf, data, len);
+  dump_bytes_field(out, field_name, data_buf, len, indent);
+}
 #pragma GCC diagnostic pop
 
 """

tests/components/api/test_wizard.cpp

@@ -68,7 +68,7 @@ TEST(DeviceWizard, ResponseSendsTheDataUnchanged) {
   Bytes expected{0x0a, 0xc8, 0x01};
   expected.insert(expected.end(), API_WIZARD_DATA, API_WIZARD_DATA + API_WIZARD_DATA_SIZE);
 
-  EXPECT_EQ(encode(resp, &DeviceWizardResponse::calc_size_msg, &wizard_encode_response), expected);
+  EXPECT_EQ(encode(resp, &DeviceWizardResponse::calc_size_msg, &DeviceWizardResponse::encode_msg), expected);
 }
 
 TEST(DeviceWizard, ShortDataHasAOneByteLength) {
@@ -77,13 +77,13 @@ TEST(DeviceWizard, ShortDataHasAOneByteLength) {
   resp.data_len = 3;
 
   Bytes expected{0x0a, 0x03, API_WIZARD_DATA[0], API_WIZARD_DATA[1], API_WIZARD_DATA[2]};
-  EXPECT_EQ(encode(resp, &DeviceWizardResponse::calc_size_msg, &wizard_encode_response), expected);
+  EXPECT_EQ(encode(resp, &DeviceWizardResponse::calc_size_msg, &DeviceWizardResponse::encode_msg), expected);
 }
 
 TEST(DeviceWizard, NoDataEncodesNothing) {
   DeviceWizardResponse resp;
   EXPECT_EQ(resp.calculate_size(), 0u);
-  EXPECT_TRUE(encode(resp, &DeviceWizardResponse::calc_size_msg, &wizard_encode_response).empty());
+  EXPECT_TRUE(encode(resp, &DeviceWizardResponse::calc_size_msg, &DeviceWizardResponse::encode_msg).empty());
 }
 
 TEST(DeviceWizard, CapabilitiesAnnounceTheWizard) {