[api] Add a progmem option for bytes fields kept in flash (#20318)

script/api_protobuf/api_protobuf.py

@@ -536,6 +536,16 @@ def create_field_type_info(
             f"track_presence on field '{field.name}' has no effect; it requires "
             "a non-repeated message field in a message that is decoded"
         )
+    if get_field_opt(field, pb.progmem, False) and (
+        field.label == FieldDescriptorProto.LABEL_REPEATED
+        or field.type != 12
+        or not get_field_opt(field, pb.pointer_to_buffer, False)
+        or get_field_opt(field, pb.fixed_array_size) is not None
+    ):
+        raise ValueError(
+            f"progmem on field '{field.name}' requires a non-repeated bytes field "
+            "with pointer_to_buffer"
+        )
     if field.label == FieldDescriptorProto.LABEL_REPEATED:
         # Check if this is a packed_buffer field (zero-copy packed repeated)
         if get_field_opt(field, pb.packed_buffer, False):
@@ -1158,6 +1168,11 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):
     reference_type = "const uint8_t*"
     const_reference_type = "const uint8_t*"
 
+    @property
+    def progmem(self) -> bool:
+        """Whether the data is in flash, so encode and dump copy it with progmem_memcpy."""
+        return get_field_opt(self._field, pb.progmem, False)
+
     @property
     def public_content(self) -> list[str]:
         # Use uint16_t for length - max packet size is well below 65535
@@ -1168,6 +1183,14 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):
 
     @property
     def encode_content(self) -> str:
+        if self.progmem:
+            return _encode_call(
+                "encode_progmem_bytes",
+                str(self.number),
+                f"this->{self.field_name}",
+                f"this->{self.field_name}_len",
+                force=self.force,
+            )
         if result := self._encode_bytes_with_precomputed_tag(
             f"this->{self.field_name}", f"this->{self.field_name}_len"
         ):
@@ -1194,8 +1217,9 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):
 
     @property
     def dump_content(self) -> str:
+        dump_fn = "dump_progmem_bytes_field" if self.progmem else "dump_bytes_field"
         return (
-            f'dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
+            f'{dump_fn}(out, ESPHOME_PSTR("{self.name}"), '
             f"this->{self.field_name}, this->{self.field_name}_len);"
         )
 
@@ -2609,6 +2633,11 @@ def build_message_type(
         ):
             fixed_vector_fields.append((field.name, field.number))
 
+        if needs_decode and get_field_opt(field, pb.progmem, False):
+            raise ValueError(
+                f"progmem on field '{field.name}' of {desc.name} requires a message "
+                "that is only encoded; received data is never in flash"
+            )
         ti = create_field_type_info(field, needs_decode, needs_encode)
 
         # Skip field declarations for fields that are in the base class
@@ -3319,15 +3348,26 @@ template<typename T> static void dump_field(DumpBuffer &out, const char *field_n
   out.append("\\n");
 }
 
+// Bytes shown by a bytes field dump: 160 bytes is 480 chars with separators, to fit a typical log buffer
+static constexpr size_t DUMP_BYTES_MAX = 160;
+
 // Helper for bytes fields - uses stack buffer to avoid heap allocation
-// Buffer sized for 160 bytes of data (480 chars with separators) to fit typical log buffer
 // field_name is a PROGMEM pointer (flash on ESP8266, regular pointer on other platforms)
 static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint8_t *data, size_t len, int indent = 2) {
-  char hex_buf[format_hex_pretty_size(160)];
+  char hex_buf[format_hex_pretty_size(DUMP_BYTES_MAX)];
   append_field_prefix(out, field_name, indent);
   format_hex_pretty_to(hex_buf, data, len);
   out.append(hex_buf).append("\\n");
 }
+
+// Helper for bytes fields in flash: copies the shown bytes out with progmem_memcpy first
+static void dump_progmem_bytes_field(DumpBuffer &out, const char *field_name, const uint8_t *data, size_t len,
+                                     int indent = 2) {
+  uint8_t data_buf[DUMP_BYTES_MAX];
+  len = std::min(len, sizeof(data_buf));
+  progmem_memcpy(data_buf, data, len);
+  dump_bytes_field(out, field_name, data_buf, len, indent);
+}
 #pragma GCC diagnostic pop
 
 """