[store_yaml] Embed user YAML in firmware for recovery

Adds a new opt-in component that compresses the on-disk YAML files with zstd
at codegen time and stores them in PROGMEM. The native API exposes a chunked
GetYaml RPC so a lost configuration can be retrieved from a running device.
Decompression happens client-side; no decompressor is shipped on-device.

CODEOWNERS

@@ -501,6 +501,7 @@ esphome/components/st7735/* @SenexCrenshaw
 esphome/components/st7789v/* @kbx81
 esphome/components/st7920/* @marsjan155
 esphome/components/statsd/* @Links2004
+esphome/components/store_yaml/* @bdraco
 esphome/components/stts22h/* @B48D81EFCC
 esphome/components/substitutions/* @esphome/core
 esphome/components/sun/* @OttoWinter

esphome/components/api/api.proto

@@ -76,6 +76,8 @@ service APIConnection {
   rpc serial_proxy_set_modem_pins(SerialProxySetModemPinsRequest) returns (void) {}
   rpc serial_proxy_get_modem_pins(SerialProxyGetModemPinsRequest) returns (void) {}
   rpc serial_proxy_request(SerialProxyRequest) returns (void) {}
+
+  rpc get_yaml(GetYamlRequest) returns (void) {}
 }
 
 
@@ -2733,3 +2735,27 @@ message BluetoothSetConnectionParamsResponse {
   uint64 address = 1;
   int32 error = 2;
 }
+
+// ==================== STORE YAML ====================
+// Embed the user's YAML in firmware and stream it back over the API so a lost
+// config can be recovered from a running device. The device only stores the
+// compressed bytes; decompression happens client-side.
+message GetYamlRequest {
+  option (id) = 149;
+  option (source) = SOURCE_CLIENT;
+  option (ifdef) = "USE_STORE_YAML";
+  option (no_delay) = true;
+}
+
+message GetYamlResponse {
+  option (id) = 150;
+  option (source) = SOURCE_SERVER;
+  option (ifdef) = "USE_STORE_YAML";
+
+  bytes data = 1;
+  bool done = 2;
+  // total compressed size, populated on the first chunk
+  uint32 total_size = 3;
+  // compression encoding, populated on the first chunk ("zstd")
+  string encoding = 4 [(max_data_length) = 8];
+}

esphome/components/api/api_connection.cpp

@@ -52,6 +52,9 @@
 #ifdef USE_RADIO_FREQUENCY
 #include "esphome/components/radio_frequency/radio_frequency.h"
 #endif
+#ifdef USE_STORE_YAML
+#include "esphome/components/store_yaml/store_yaml.h"
+#endif
 
 namespace esphome::api {
 
@@ -310,6 +313,10 @@ void APIConnection::loop() {
   // (missing a frame is fine, missing a state update is not)
   this->try_send_camera_image_();
 #endif
+
+#ifdef USE_STORE_YAML
+  this->try_send_store_yaml_();
+#endif
 }
 
 void APIConnection::check_keepalive_(uint32_t now) {
@@ -1164,6 +1171,71 @@ void APIConnection::on_camera_image_request(const CameraImageRequest &msg) {
 }
 #endif
 
+#ifdef USE_STORE_YAML
+// Chunk size per GetYamlResponse. Small enough to leave room for the protobuf frame
+// inside the 65535-byte API limit and friendly to TCP MSS.
+static constexpr size_t STORE_YAML_CHUNK_SIZE = 512;
+// Scratch buffer used to copy a chunk from PROGMEM (needed on ESP8266; harmless elsewhere).
+static uint8_t store_yaml_chunk_buf[STORE_YAML_CHUNK_SIZE];
+
+void APIConnection::on_get_yaml_request() {
+  if (store_yaml::global_store_yaml == nullptr || store_yaml::global_store_yaml->get_data() == nullptr) {
+    // No blob — send a single done=true response so the client doesn't hang.
+    GetYamlResponse resp;
+    resp.done = true;
+    this->send_message(resp);
+    return;
+  }
+  this->store_yaml_pos_ = 0;
+  this->try_send_store_yaml_();
+}
+
+void APIConnection::try_send_store_yaml_() {
+  if (this->store_yaml_pos_ == std::numeric_limits<size_t>::max())
+    return;
+  auto *comp = store_yaml::global_store_yaml;
+  if (comp == nullptr)
+    return;
+
+  const uint8_t *data = comp->get_data();
+  const size_t total = comp->get_size();
+
+  while (this->store_yaml_pos_ < total || this->store_yaml_pos_ == 0) {
+    if (!this->helper_->can_write_without_blocking())
+      return;
+
+    const size_t remaining = total - this->store_yaml_pos_;
+    const size_t to_send = remaining < STORE_YAML_CHUNK_SIZE ? remaining : STORE_YAML_CHUNK_SIZE;
+
+    // Copy from PROGMEM into a stack buffer. progmem_read_byte is a no-op except on ESP8266.
+    for (size_t i = 0; i < to_send; i++) {
+      store_yaml_chunk_buf[i] = progmem_read_byte(&data[this->store_yaml_pos_ + i]);
+    }
+
+    GetYamlResponse resp;
+    resp.set_data(store_yaml_chunk_buf, to_send);
+    const bool first = this->store_yaml_pos_ == 0;
+    if (first) {
+      resp.total_size = static_cast<uint32_t>(total);
+      resp.encoding = StringRef(store_yaml::ENCODING);
+    }
+    this->store_yaml_pos_ += to_send;
+    const bool done = this->store_yaml_pos_ >= total;
+    resp.done = done;
+
+    if (!this->send_message(resp)) {
+      // Send failed; rewind so we retry this chunk next loop.
+      this->store_yaml_pos_ -= to_send;
+      return;
+    }
+    if (done) {
+      this->store_yaml_pos_ = std::numeric_limits<size_t>::max();
+      return;
+    }
+  }
+}
+#endif
+
 #ifdef USE_HOMEASSISTANT_TIME
 void APIConnection::on_get_time_response(const GetTimeResponse &value) {
   if (homeassistant::global_homeassistant_time != nullptr) {

esphome/components/api/api_connection.h

@@ -120,6 +120,9 @@ class APIConnection final : public APIServerConnectionBase {
   void set_camera_state(std::shared_ptr<camera::CameraImage> image);
   void on_camera_image_request(const CameraImageRequest &msg);
 #endif
+#ifdef USE_STORE_YAML
+  void on_get_yaml_request();
+#endif
 #ifdef USE_CLIMATE
   bool send_climate_state(climate::Climate *climate);
   void on_climate_command_request(const ClimateCommandRequest &msg);
@@ -394,6 +397,12 @@ class APIConnection final : public APIServerConnectionBase {
   void try_send_camera_image_();
 #endif
 
+#ifdef USE_STORE_YAML
+  void try_send_store_yaml_();
+  // Streaming offset into the PROGMEM blob; max() means "not streaming".
+  size_t store_yaml_pos_{std::numeric_limits<size_t>::max()};
+#endif
+
 #ifdef USE_API_HOMEASSISTANT_STATES
   void process_state_subscriptions_();
 #endif

esphome/components/api/api_pb2.cpp

@@ -4155,5 +4155,23 @@ uint32_t BluetoothSetConnectionParamsResponse::calculate_size() const {
   return size;
 }
 #endif
+#ifdef USE_STORE_YAML
+uint8_t *GetYamlResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const {
+  uint8_t *__restrict__ pos = buffer.get_pos();
+  ProtoEncode::encode_bytes(pos PROTO_ENCODE_DEBUG_ARG, 1, this->data_ptr_, this->data_len_);
+  ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 2, this->done);
+  ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 3, this->total_size);
+  ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 4, this->encoding);
+  return pos;
+}
+uint32_t GetYamlResponse::calculate_size() const {
+  uint32_t size = 0;
+  size += ProtoSize::calc_length(1, this->data_len_);
+  size += ProtoSize::calc_bool(1, this->done);
+  size += ProtoSize::calc_uint32(1, this->total_size);
+  size += !this->encoding.empty() ? 2 + this->encoding.size() : 0;
+  return size;
+}
+#endif
 
 }  // namespace esphome::api

esphome/components/api/api_pb2.h

@@ -3317,5 +3317,31 @@ class BluetoothSetConnectionParamsResponse final : public ProtoMessage {
  protected:
 };
 #endif
+#ifdef USE_STORE_YAML
+class GetYamlResponse final : public ProtoMessage {
+ public:
+  static constexpr uint8_t MESSAGE_TYPE = 150;
+  static constexpr uint8_t ESTIMATED_SIZE = 34;
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  const LogString *message_name() const override { return LOG_STR("get_yaml_response"); }
+#endif
+  const uint8_t *data_ptr_{nullptr};
+  size_t data_len_{0};
+  void set_data(const uint8_t *data, size_t len) {
+    this->data_ptr_ = data;
+    this->data_len_ = len;
+  }
+  bool done{false};
+  uint32_t total_size{0};
+  StringRef encoding{};
+  uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const;
+  uint32_t calculate_size() const;
+#ifdef HAS_PROTO_MESSAGE_DUMP
+  const char *dump_to(DumpBuffer &out) const override;
+#endif
+
+ protected:
+};
+#endif
 
 }  // namespace esphome::api

esphome/components/api/api_pb2_dump.cpp

@@ -2718,6 +2718,16 @@ const char *BluetoothSetConnectionParamsResponse::dump_to(DumpBuffer &out) const
   return out.c_str();
 }
 #endif
+#ifdef USE_STORE_YAML
+const char *GetYamlResponse::dump_to(DumpBuffer &out) const {
+  MessageDumpHelper helper(out, ESPHOME_PSTR("GetYamlResponse"));
+  dump_bytes_field(out, ESPHOME_PSTR("data"), this->data_ptr_, this->data_len_);
+  dump_field(out, ESPHOME_PSTR("done"), this->done);
+  dump_field(out, ESPHOME_PSTR("total_size"), this->total_size);
+  dump_field(out, ESPHOME_PSTR("encoding"), this->encoding);
+  return out.c_str();
+}
+#endif
 
 }  // namespace esphome::api
 

esphome/components/api/api_pb2_service.cpp

@@ -702,6 +702,15 @@ void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const ui
       this->on_bluetooth_set_connection_params_request(msg);
       break;
     }
+#endif
+#ifdef USE_STORE_YAML
+    case 149 /* GetYamlRequest is empty */: {
+#ifdef HAS_PROTO_MESSAGE_DUMP
+      this->log_receive_message_(LOG_STR("on_get_yaml_request"));
+#endif
+      this->on_get_yaml_request();
+      break;
+    }
 #endif
     default:
       break;

esphome/components/api/api_pb2_service.h

@@ -236,6 +236,10 @@ class APIServerConnectionBase {
 #ifdef USE_BLUETOOTH_PROXY
   void on_bluetooth_set_connection_params_request(const BluetoothSetConnectionParamsRequest &value){};
 #endif
+
+#ifdef USE_STORE_YAML
+  void on_get_yaml_request(){};
+#endif
 };
 
 }  // namespace esphome::api

esphome/components/store_yaml/__init__.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+import struct
+
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ID, CONF_RAW_DATA_ID
+from esphome.core import CORE, EsphomeError, HexInt
+
+_LOGGER = logging.getLogger(__name__)
+
+CODEOWNERS = ["@bdraco"]
+DEPENDENCIES = ["api"]
+
+CONF_INCLUDE_SECRETS = "include_secrets"
+
+store_yaml_ns = cg.esphome_ns.namespace("store_yaml")
+StoreYamlComponent = store_yaml_ns.class_("StoreYamlComponent", cg.Component)
+
+# Compression level for zstd; 22 is the max and gives ~70-90% reduction on YAML.
+ZSTD_LEVEL = 22
+# Envelope magic: "EHY1" = ESPHome YAML, version 1.
+ENVELOPE_MAGIC = b"EHY1"
+# Replacement content when secrets are not included.
+REDACTED_PLACEHOLDER = b"# redacted\n"
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(StoreYamlComponent),
+        cv.GenerateID(CONF_RAW_DATA_ID): cv.declare_id(cg.uint8),
+        cv.Optional(CONF_INCLUDE_SECRETS, default=False): cv.boolean,
+    }
+).extend(cv.COMPONENT_SCHEMA)
+
+
+def _import_zstd():
+    try:
+        from compression import zstd  # noqa: PLC0415 — Python 3.14+ stdlib
+    except ImportError:
+        try:
+            from backports import zstd  # noqa: PLC0415
+        except ImportError as err:
+            raise EsphomeError(
+                "store_yaml requires zstd compression. Install backports.zstd for "
+                "Python < 3.14 or upgrade to Python 3.14+."
+            ) from err
+    return zstd
+
+
+def _gather_files(include_secrets: bool) -> list[tuple[str, bytes]]:
+    """Read each YAML file the config loader touched, return (relative_path, content) pairs."""
+    sources = CORE.data.get("yaml_sources")
+    if not sources:
+        raise EsphomeError(
+            "store_yaml could not find any tracked YAML files; the config loader "
+            "did not populate CORE.data['yaml_sources']."
+        )
+
+    config_path = Path(CORE.config_path).resolve()
+    root = config_path.parent
+
+    seen: set[Path] = set()
+    files: list[tuple[str, bytes]] = []
+    for path in sources:
+        resolved = Path(path).resolve()
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+
+        is_secret = resolved.name == "secrets.yaml"
+        if is_secret and not include_secrets:
+            content = REDACTED_PLACEHOLDER
+        else:
+            try:
+                content = resolved.read_bytes()
+            except OSError as err:
+                _LOGGER.warning(
+                    "store_yaml: skipping unreadable %s (%s)", resolved, err
+                )
+                continue
+
+        try:
+            rel = resolved.relative_to(root)
+            rel_str = rel.as_posix()
+        except ValueError:
+            # Outside the project root (e.g. secrets.yaml in $HOME); store basename only.
+            rel_str = resolved.name
+
+        files.append((rel_str, content))
+
+    return files
+
+
+def _pack_envelope(files: list[tuple[str, bytes]]) -> bytes:
+    """Pack files into the EHY1 envelope.
+
+    Layout: magic (4) | u32 file_count | repeat { u16 path_len | path_utf8 | u32 content_len | content_bytes }
+    All integers are little-endian.
+    """
+    parts: list[bytes] = [ENVELOPE_MAGIC, struct.pack("<I", len(files))]
+    for path, content in files:
+        path_bytes = path.encode("utf-8")
+        if len(path_bytes) > 0xFFFF:
+            raise EsphomeError(
+                f"store_yaml: path too long ({len(path_bytes)} bytes): {path}"
+            )
+        parts.append(struct.pack("<H", len(path_bytes)))
+        parts.append(path_bytes)
+        parts.append(struct.pack("<I", len(content)))
+        parts.append(content)
+    return b"".join(parts)
+
+
+async def to_code(config):
+    cg.add_define("USE_STORE_YAML")
+
+    zstd = _import_zstd()
+
+    files = _gather_files(config[CONF_INCLUDE_SECRETS])
+    envelope = _pack_envelope(files)
+    compressed = zstd.compress(envelope, level=ZSTD_LEVEL)
+
+    _LOGGER.info(
+        "store_yaml: embedding %d file(s) as %d bytes (%d uncompressed, %.1f%% ratio)",
+        len(files),
+        len(compressed),
+        len(envelope),
+        100.0 * len(compressed) / max(1, len(envelope)),
+    )
+
+    rhs = [HexInt(b) for b in compressed]
+    prog_arr = cg.progmem_array(config[CONF_RAW_DATA_ID], rhs)
+
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)
+    cg.add(var.set_data(prog_arr, len(compressed), len(envelope)))

esphome/components/store_yaml/store_yaml.cpp

@@ -0,0 +1,25 @@
+#include "store_yaml.h"
+
+#ifdef USE_STORE_YAML
+
+#include "esphome/core/log.h"
+
+namespace esphome::store_yaml {
+
+static const char *const TAG = "store_yaml";
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+StoreYamlComponent *global_store_yaml = nullptr;
+
+void StoreYamlComponent::setup() { global_store_yaml = this; }
+
+void StoreYamlComponent::dump_config() {
+  ESP_LOGCONFIG(TAG, "YAML:");
+  ESP_LOGCONFIG(TAG, "  Compressed size: %zu bytes", this->size_);
+  ESP_LOGCONFIG(TAG, "  Uncompressed size: %zu bytes", this->uncompressed_size_);
+  ESP_LOGCONFIG(TAG, "  Encoding: %s", ENCODING);
+}
+
+}  // namespace esphome::store_yaml
+
+#endif  // USE_STORE_YAML

esphome/components/store_yaml/store_yaml.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "esphome/core/defines.h"
+#ifdef USE_STORE_YAML
+
+#include "esphome/core/component.h"
+#include "esphome/core/hal.h"
+
+namespace esphome::store_yaml {
+
+// "zstd" — published in GetYamlResponse.encoding so clients know how to decompress.
+constexpr const char *ENCODING = "zstd";
+
+class StoreYamlComponent : public Component {
+ public:
+  void setup() override;
+  void dump_config() override;
+  float get_setup_priority() const override { return setup_priority::DATA; }
+
+  // Called once from codegen with the PROGMEM blob.
+  void set_data(const uint8_t *data, size_t size, size_t uncompressed_size) {
+    this->data_ = data;
+    this->size_ = size;
+    this->uncompressed_size_ = uncompressed_size;
+  }
+  const uint8_t *get_data() const { return this->data_; }
+  size_t get_size() const { return this->size_; }
+  size_t get_uncompressed_size() const { return this->uncompressed_size_; }
+
+ protected:
+  const uint8_t *data_{nullptr};
+  size_t size_{0};
+  size_t uncompressed_size_{0};
+};
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+extern StoreYamlComponent *global_store_yaml;
+
+}  // namespace esphome::store_yaml
+
+#endif  // USE_STORE_YAML

esphome/config.py

@@ -1192,7 +1192,11 @@ def _load_config(
 ) -> Config:
     """Load the configuration file."""
     try:
-        config = yaml_util.load_yaml(CORE.config_path)
+        with yaml_util.track_yaml_loads() as loaded_files:
+            config = yaml_util.load_yaml(CORE.config_path)
+        # Stash for components that want the on-disk YAML at codegen time
+        # (e.g. store_yaml embeds them into firmware for recovery).
+        CORE.data["yaml_sources"] = loaded_files
     except EsphomeError as e:
         raise InvalidYAMLError(e) from e
 

esphome/core/defines.h

@@ -195,6 +195,7 @@
 #define USE_API_CUSTOM_SERVICES
 #define USE_API_USER_DEFINED_ACTION_RESPONSES
 #define USE_API_USER_DEFINED_ACTION_RESPONSES_JSON
+#define USE_STORE_YAML
 #define API_MAX_SEND_QUEUE 8
 #define MAX_API_CONNECTIONS 6
 #define USE_MD5

requirements.txt

@@ -26,6 +26,9 @@ bleak==2.1.1
 smpclient==6.0.0
 requests==2.34.1
 
+# zstd compression for store_yaml component (stdlib in 3.14+)
+backports.zstd==1.5.0; python_version < "3.14"
+
 # esp-idf >= 5.0 requires this
 pyparsing >= 3.3.2
 

tests/components/store_yaml/common.yaml

@@ -0,0 +1,3 @@
+api:
+
+store_yaml:

tests/components/store_yaml/test.bk72xx-ard.yaml

@@ -0,0 +1,5 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+<<: !include common.yaml

tests/components/store_yaml/test.esp32-ard.yaml

@@ -0,0 +1,5 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+<<: !include common.yaml

tests/components/store_yaml/test.esp32-idf.yaml

@@ -0,0 +1,5 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+<<: !include common.yaml

tests/components/store_yaml/test.esp8266-ard.yaml

@@ -0,0 +1,5 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+<<: !include common.yaml

tests/components/store_yaml/test.host.yaml

@@ -0,0 +1,3 @@
+<<: !include common.yaml
+
+network:

tests/components/store_yaml/test.ln882x-ard.yaml

@@ -0,0 +1,5 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+<<: !include common.yaml

tests/components/store_yaml/test.rp2040-ard.yaml

@@ -0,0 +1,5 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+<<: !include common.yaml

tests/components/store_yaml/test.rtl87xx-ard.yaml

@@ -0,0 +1,5 @@
+wifi:
+  ssid: MySSID
+  password: password1
+
+<<: !include common.yaml