[globals] Add RTC memory support (#13954)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: J. Nick Koston <nick@home-assistant.io>
Co-authored-by: J. Nick Koston <nick@koston.org>

esphome/components/globals/__init__.py

@@ -1,10 +1,11 @@
 import hashlib
 
-from esphome import automation, codegen as cg, config_validation as cv
+from esphome import automation, codegen as cg, config_validation as cv, preferences
 from esphome.const import (
     CONF_ID,
     CONF_INITIAL_VALUE,
     CONF_RESTORE_VALUE,
+    CONF_STORAGE,
     CONF_TYPE,
     CONF_VALUE,
 )
@@ -21,6 +22,7 @@ RestoringGlobalsComponent = globals_ns.class_(
 RestoringGlobalStringComponent = globals_ns.class_(
     "RestoringGlobalStringComponent", cg.PollingComponent
 )
+Storage = globals_ns.enum("Storage", is_class=True)
 
 CONF_MAX_RESTORE_DATA_LENGTH = "max_restore_data_length"
 
@@ -37,6 +39,9 @@ _NON_RESTORING_SCHEMA = cv.Schema(
     {
         **_BASE_SCHEMA,
         cv.Optional(CONF_RESTORE_VALUE, default=False): cv.boolean,
+        cv.Optional(CONF_STORAGE): cv.invalid(
+            f"'{CONF_STORAGE}' needs '{CONF_RESTORE_VALUE}: true'"
+        ),
     }
 ).extend(cv.COMPONENT_SCHEMA)
 
@@ -45,6 +50,9 @@ _RESTORING_SCHEMA = cv.Schema(
     {
         **_BASE_SCHEMA,
         cv.Optional(CONF_RESTORE_VALUE, default=True): cv.boolean,
+        # No default: when omitted the global keeps the platform's own default backend,
+        # which on ESP8266 follows restore_from_flash
+        cv.Optional(CONF_STORAGE): preferences.validate_storage,
     }
 ).extend(cv.polling_component_schema("1s"))
 
@@ -68,13 +76,17 @@ async def to_code(config: ConfigType) -> None:
 
     # Special casing the strings to their own class with a different save/restore mechanism
     if str(type_) == "std::string" and restore:
-        template_args = cg.TemplateArguments(
-            type_, config.get(CONF_MAX_RESTORE_DATA_LENGTH, 63) + 1
-        )
+        args = [type_, config.get(CONF_MAX_RESTORE_DATA_LENGTH, 63) + 1]
         type = RestoringGlobalStringComponent
     else:
-        template_args = cg.TemplateArguments(type_)
+        args = [type_]
         type = RestoringGlobalsComponent if restore else GlobalsComponent
+    in_flash = None
+    if (storage := config.get(CONF_STORAGE)) is not None:
+        # is_in_flash() also requests the RTC backend when rtc is chosen
+        in_flash = preferences.is_in_flash(storage)
+        args.append(Storage.STORAGE_FLASH if in_flash else Storage.STORAGE_RTC)
+    template_args = cg.TemplateArguments(*args)
 
     res_type = type.template(template_args)
     initial_value = None
@@ -91,6 +103,10 @@ async def to_code(config: ConfigType) -> None:
             value = value.encode()
         hash_ = int(hashlib.md5(value).hexdigest()[:8], 16)
         cg.add(glob.set_name_hash(hash_))
+        if type is RestoringGlobalStringComponent:
+            # The ESP8266 adds up every saved value to check that they fit its store; only the
+            # string's size is known here
+            preferences.reserve_preferences(1, args[1], in_flash)
 
 
 # A lambda returns the global's own type so `return 42;` reduces to a cast; the value

esphome/components/globals/globals_component.h

@@ -25,7 +25,11 @@ template<typename T> class GlobalsComponent final : public Component {
   T value_{};
 };
 
-template<typename T> class RestoringGlobalsComponent : public PollingComponent {
+// Where a restoring global saves its value; STORAGE_PLATFORM keeps the platform default
+enum class Storage : uint8_t { STORAGE_PLATFORM, STORAGE_FLASH, STORAGE_RTC };
+
+template<typename T, Storage STORAGE = Storage::STORAGE_PLATFORM>
+class RestoringGlobalsComponent : public PollingComponent {
  public:
   using value_type = T;
   explicit RestoringGlobalsComponent() : PollingComponent(1000) {}
@@ -39,7 +43,12 @@ template<typename T> class RestoringGlobalsComponent : public PollingComponent {
   T &value() { return this->value_; }
 
   void setup() override {
-    this->rtc_ = global_preferences->make_preference<T>(1944399030U ^ this->name_hash_);
+    if constexpr (STORAGE == Storage::STORAGE_PLATFORM) {
+      this->rtc_ = global_preferences->make_preference<T>(1944399030U ^ this->name_hash_);
+    } else {
+      this->rtc_ =
+          global_preferences->make_preference<T>(1944399030U ^ this->name_hash_, STORAGE == Storage::STORAGE_FLASH);
+    }
     this->rtc_.load(&this->value_);
     memcpy(&this->last_checked_value_, &this->value_, sizeof(T));
   }
@@ -68,7 +77,8 @@ template<typename T> class RestoringGlobalsComponent : public PollingComponent {
 };
 
 // Use with string or subclasses of strings
-template<typename T, uint8_t SZ> class RestoringGlobalStringComponent : public PollingComponent {
+template<typename T, uint8_t SZ, Storage STORAGE = Storage::STORAGE_PLATFORM>
+class RestoringGlobalStringComponent : public PollingComponent {
  public:
   using value_type = T;
   explicit RestoringGlobalStringComponent() : PollingComponent(1000) {}
@@ -83,7 +93,12 @@ template<typename T, uint8_t SZ> class RestoringGlobalStringComponent : public P
 
   void setup() override {
     char temp[SZ];
-    this->rtc_ = global_preferences->make_preference<uint8_t[SZ]>(1944399030U ^ this->name_hash_);
+    if constexpr (STORAGE == Storage::STORAGE_PLATFORM) {
+      this->rtc_ = global_preferences->make_preference<uint8_t[SZ]>(1944399030U ^ this->name_hash_);
+    } else {
+      this->rtc_ = global_preferences->make_preference<uint8_t[SZ]>(1944399030U ^ this->name_hash_,
+                                                                    STORAGE == Storage::STORAGE_FLASH);
+    }
     bool hasdata = this->rtc_.load(&temp);
     if (hasdata) {
       this->value_.assign(temp + 1, static_cast<uint8_t>(temp[0]));
@@ -130,7 +145,9 @@ template<typename T, uint8_t SZ> class RestoringGlobalStringComponent : public P
 };
 
 template<typename T> T &id(GlobalsComponent<T> *value) { return value->value(); }
-template<typename T> T &id(RestoringGlobalsComponent<T> *value) { return value->value(); }
-template<typename T, uint8_t SZ> T &id(RestoringGlobalStringComponent<T, SZ> *value) { return value->value(); }
+template<typename T, Storage STORAGE> T &id(RestoringGlobalsComponent<T, STORAGE> *value) { return value->value(); }
+template<typename T, uint8_t SZ, Storage STORAGE> T &id(RestoringGlobalStringComponent<T, SZ, STORAGE> *value) {
+  return value->value();
+}
 
 }  // namespace esphome::globals

tests/component_tests/globals/config/globals_esp8266_storage_test.yaml

@@ -0,0 +1,13 @@
+esphome:
+  name: test
+
+esp8266:
+  board: d1_mini
+
+globals:
+  - id: rtc_string
+    type: std::string
+    restore_value: true
+    storage: rtc
+    max_restore_data_length: 15
+    initial_value: '"boot"'

tests/component_tests/globals/config/globals_esp8266_test.yaml

@@ -0,0 +1,12 @@
+esphome:
+  name: test
+
+esp8266:
+  board: d1_mini
+  restore_from_flash: true
+
+globals:
+  - id: my_restoring_int
+    type: int
+    restore_value: true
+    initial_value: "8"

tests/component_tests/globals/config/globals_storage_test.yaml

@@ -0,0 +1,17 @@
+esphome:
+  name: test
+
+esp32:
+  board: esp32dev
+
+globals:
+  - id: rtc_int
+    type: int
+    restore_value: true
+    storage: rtc
+    initial_value: "7"
+  - id: flash_string
+    type: std::string
+    restore_value: true
+    storage: flash
+    max_restore_data_length: 15

tests/component_tests/globals/config/globals_test.yaml

@@ -14,3 +14,7 @@ globals:
   - id: my_global_bool
     type: bool
     initial_value: "true"
+  - id: my_restoring_int
+    type: int
+    restore_value: true
+    initial_value: "8"

tests/component_tests/globals/test_storage.py

@@ -0,0 +1,103 @@
+"""Tests for the globals storage option."""
+
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+from esphome import config_validation as cv
+from esphome.components import globals as globals_component
+from esphome.components.esp32 import KEY_VARIANT, VARIANT_ESP32, VARIANT_ESP32C2
+from esphome.components.esp8266.const import (
+    KEY_ESP8266,
+    KEY_PREFERENCES,
+    preference_words,
+)
+from esphome.const import (
+    CONF_ID,
+    CONF_RESTORE_VALUE,
+    CONF_STORAGE,
+    CONF_TYPE,
+    PlatformFramework,
+)
+from esphome.core import CORE
+from esphome.types import ConfigType
+from tests.component_tests.types import SetCoreConfigCallable
+
+
+def _validate(**extra: object) -> ConfigType:
+    return globals_component.CONFIG_SCHEMA(
+        {CONF_ID: "my_global", CONF_TYPE: "int", **extra}
+    )
+
+
+def _define_names() -> set[str]:
+    return {define.name for define in CORE.defines}
+
+
+@pytest.mark.parametrize(
+    ("platform_framework", "platform_data"),
+    [
+        (PlatformFramework.ESP32_IDF, {KEY_VARIANT: VARIANT_ESP32C2}),
+        (PlatformFramework.RP2040_ARDUINO, None),
+    ],
+)
+def test_rtc_rejected_without_rtc_memory(
+    set_core_config: SetCoreConfigCallable,
+    platform_framework: PlatformFramework,
+    platform_data: ConfigType | None,
+) -> None:
+    set_core_config(platform_framework, platform_data=platform_data)
+    with pytest.raises(cv.Invalid, match="not supported on this platform"):
+        _validate(**{CONF_RESTORE_VALUE: True, CONF_STORAGE: "rtc"})
+
+
+@pytest.mark.parametrize("extra", [{}, {CONF_RESTORE_VALUE: False}])
+def test_storage_rejected_without_restore_value(
+    set_core_config: SetCoreConfigCallable, extra: ConfigType
+) -> None:
+    set_core_config(
+        PlatformFramework.ESP32_IDF, platform_data={KEY_VARIANT: VARIANT_ESP32}
+    )
+    with pytest.raises(cv.Invalid, match="'storage' needs 'restore_value: true'"):
+        _validate(**extra, **{CONF_STORAGE: "rtc"})
+
+
+def test_storage_generates_template_argument(
+    generate_main: Callable[[str | Path], str],
+    component_config_path: Callable[[str], Path],
+) -> None:
+    main_cpp = generate_main(component_config_path("globals_storage_test.yaml"))
+    assert (
+        "globals::RestoringGlobalsComponent<int, globals::Storage::STORAGE_RTC>"
+        in main_cpp
+    )
+    assert (
+        "globals::RestoringGlobalStringComponent<std::string, 16, "
+        "globals::Storage::STORAGE_FLASH>" in main_cpp
+    )
+    assert "USE_ESP32_RTC_PREFERENCES" in _define_names()
+
+
+# On the ESP8266 the omitted form must stay the two-argument call so restore_from_flash keeps deciding
+@pytest.mark.parametrize("config", ["globals_test.yaml", "globals_esp8266_test.yaml"])
+def test_omitted_storage_keeps_platform_default(
+    generate_main: Callable[[str | Path], str],
+    component_config_path: Callable[[str], Path],
+    config: str,
+) -> None:
+    main_cpp = generate_main(component_config_path(config))
+    assert (
+        "new(my_restoring_int) globals::RestoringGlobalsComponent<int>(8)" in main_cpp
+    )
+    assert "globals::Storage" not in main_cpp
+    assert "USE_ESP32_RTC_PREFERENCES" not in _define_names()
+
+
+def test_restored_string_reserves_esp8266_store(
+    generate_main: Callable[[str | Path], str],
+    component_config_path: Callable[[str], Path],
+) -> None:
+    generate_main(component_config_path("globals_esp8266_storage_test.yaml"))
+    # 15 characters plus the length byte, in RTC memory
+    assert CORE.data[KEY_ESP8266][KEY_PREFERENCES] == [(preference_words(16), False)]

tests/components/globals/storage.yaml

@@ -0,0 +1,16 @@
+globals:
+  - id: glob_rtc_int
+    type: int
+    restore_value: true
+    storage: rtc
+    initial_value: "0"
+  - id: glob_rtc_string
+    type: std::string
+    restore_value: true
+    storage: rtc
+    max_restore_data_length: 15
+  - id: glob_flash_float
+    type: float
+    restore_value: true
+    storage: flash
+    initial_value: "0.0f"

tests/components/globals/test.esp32-idf.yaml

@@ -1 +1,3 @@
-<<: !include common.yaml
+packages:
+  globals: !include common.yaml
+  storage: !include storage.yaml

tests/components/globals/test.esp8266-ard.yaml

@@ -1 +1,3 @@
-<<: !include common.yaml
+packages:
+  globals: !include common.yaml
+  storage: !include storage.yaml

tests/components/globals/test.rp2040-ard.yaml

@@ -1 +1,2 @@
-<<: !include common.yaml
+packages:
+  globals: !include common.yaml