Merge branch 'dev' into light-control-action-compact

2026-09-16 01:28:39 +00:00 · 2026-03-23 20:04:23 -10:00
parent 04c6e7e770 793813790a
commit 8c02b90b87
44 changed files with 2675 additions and 1728 deletions
@@ -2249,10 +2249,14 @@ bool SubscribeBluetoothLEAdvertisementsRequest::decode_varint(uint32_t field_id,
  return true;
 }
 void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer &buffer) const {
-  buffer.encode_uint64(1, this->address, true);
-  buffer.encode_sint32(2, this->rssi, true);
+  buffer.write_raw_byte(8);
+  buffer.encode_varint_raw_64(this->address);
+  buffer.write_raw_byte(16);
+  buffer.encode_varint_raw(encode_zigzag32(this->rssi));
  buffer.encode_uint32(3, this->address_type);
-  buffer.encode_bytes(4, this->data, this->data_len, true);
+  buffer.write_raw_byte(34);
+  buffer.encode_varint_raw(this->data_len);
+  buffer.encode_raw(this->data, this->data_len);
 }
 uint32_t BluetoothLERawAdvertisement::calculate_size() const {
  uint32_t size = 0;
@@ -9,8 +9,8 @@ namespace esphome::api {
 static const char *const TAG = "api.service";

 #ifdef HAS_PROTO_MESSAGE_DUMP
-void APIServerConnectionBase::log_send_message_(const char *name, const char *dump) {
-  ESP_LOGVV(TAG, "send_message %s: %s", name, dump);
+void APIServerConnectionBase::log_send_message_(const LogString *name, const char *dump) {
+  ESP_LOGVV(TAG, "send_message %s: %s", LOG_STR_ARG(name), dump);
 }
 void APIServerConnectionBase::log_receive_message_(const LogString *name, const ProtoMessage &msg) {
  DumpBuffer dump_buf;
@@ -12,7 +12,7 @@ class APIServerConnectionBase {
 public:
 #ifdef HAS_PROTO_MESSAGE_DUMP
 protected:
-  void log_send_message_(const char *name, const char *dump);
+  void log_send_message_(const LogString *name, const char *dump);
  void log_receive_message_(const LogString *name, const ProtoMessage &msg);
  void log_receive_message_(const LogString *name);

@@ -46,10 +46,8 @@ void APIServer::setup() {

 #ifndef USE_API_NOISE_PSK_FROM_YAML
  // Only load saved PSK if not set from YAML
-  SavedNoisePsk noise_pref_saved{};
-  if (this->noise_pref_.load(&noise_pref_saved)) {
+  if (this->load_and_apply_noise_psk_()) {
    ESP_LOGD(TAG, "Loaded saved Noise PSK");
-    this->set_noise_psk(noise_pref_saved.psk);
  }
 #endif
 #endif
@@ -514,7 +512,7 @@ void APIServer::set_reboot_timeout(uint32_t reboot_timeout) { this->reboot_timeo

 #ifdef USE_API_NOISE
 bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg,
-                                  const LogString *fail_log_msg, const psk_t &active_psk, bool make_active) {
+                                  const LogString *fail_log_msg, bool make_active) {
  if (!this->noise_pref_.save(&new_psk)) {
    ESP_LOGW(TAG, "%s", LOG_STR_ARG(fail_log_msg));
    return false;
@@ -526,9 +524,14 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString
  }
  ESP_LOGD(TAG, "%s", LOG_STR_ARG(save_log_msg));
  if (make_active) {
-    this->set_timeout(100, [this, active_psk]() {
+    this->set_timeout(100, [this]() {
+      // Re-read the PSK from preferences rather than capturing the 32-byte array
+      // in the lambda (which would exceed std::function SBO and heap-allocate).
+      if (!this->load_and_apply_noise_psk_()) {
+        ESP_LOGW(TAG, "Failed to load saved PSK for activation");
+        return;
+      }
      ESP_LOGW(TAG, "Disconnecting all clients to reset PSK");
-      this->set_noise_psk(active_psk);
      for (auto &c : this->clients_) {
        DisconnectRequest req;
        c->send_message(req);
@@ -538,6 +541,14 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString
  return true;
 }

+bool APIServer::load_and_apply_noise_psk_() {
+  SavedNoisePsk saved{};
+  if (!this->noise_pref_.load(&saved))
+    return false;
+  this->set_noise_psk(saved.psk);
+  return true;
+}
+
 bool APIServer::save_noise_psk(psk_t psk, bool make_active) {
 #ifdef USE_API_NOISE_PSK_FROM_YAML
  // When PSK is set from YAML, this function should never be called
@@ -552,7 +563,7 @@ bool APIServer::save_noise_psk(psk_t psk, bool make_active) {
  }

  SavedNoisePsk new_saved_psk{psk};
-  return this->update_noise_psk_(new_saved_psk, LOG_STR("Noise PSK saved"), LOG_STR("Failed to save Noise PSK"), psk,
+  return this->update_noise_psk_(new_saved_psk, LOG_STR("Noise PSK saved"), LOG_STR("Failed to save Noise PSK"),
                                 make_active);
 #endif
 }
@@ -564,8 +575,7 @@ bool APIServer::clear_noise_psk(bool make_active) {
  return false;
 #else
  SavedNoisePsk empty_psk{};
-  psk_t empty{};
-  return this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"), empty,
+  return this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"),
                                 make_active);
 #endif
 }
@@ -239,7 +239,9 @@ class APIServer final : public Component,

 #ifdef USE_API_NOISE
  bool update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg,
-                         const psk_t &active_psk, bool make_active);
+                         bool make_active);
+  // Load saved PSK from preferences and apply it. Returns true on success.
+  bool load_and_apply_noise_psk_();
 #endif  // USE_API_NOISE
 #ifdef USE_API_HOMEASSISTANT_STATES
  // Helper methods to reduce code duplication
@@ -5,6 +5,7 @@
 #include "esphome/core/component.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
+#include "esphome/core/progmem.h"
 #include "esphome/core/string_ref.h"

 #include <cassert>
@@ -228,6 +229,17 @@ class ProtoWriteBuffer {
   * Following https://protobuf.dev/programming-guides/encoding/#structure
   */
  void encode_field_raw(uint32_t field_id, uint32_t type) { this->encode_varint_raw((field_id << 3) | type); }
+  /// Write a single precomputed tag byte. Tag must be < 128.
+  inline void write_raw_byte(uint8_t b) ESPHOME_ALWAYS_INLINE {
+    this->debug_check_bounds_(1);
+    *this->pos_++ = b;
+  }
+  /// Write raw bytes to the buffer (no tag, no length prefix).
+  inline void encode_raw(const void *data, size_t len) ESPHOME_ALWAYS_INLINE {
+    this->debug_check_bounds_(len);
+    std::memcpy(this->pos_, data, len);
+    this->pos_ += len;
+  }
  /// Write a precomputed tag byte + 32-bit value in one operation.
  /// Tag must be a single-byte varint (< 128). No zero check.
  inline void write_tag_and_fixed32(uint8_t tag, uint32_t value) ESPHOME_ALWAYS_INLINE {
@@ -400,6 +412,23 @@ class DumpBuffer {
    return *this;
  }

+  /// Append a PROGMEM string (flash-safe on ESP8266, regular append on other platforms)
+  DumpBuffer &append_p(const char *str) {
+    if (str) {
+#ifdef USE_ESP8266
+      append_p_esp8266(str);
+#else
+      append_impl_(str, strlen(str));
+#endif
+    }
+    return *this;
+  }
+
+#ifdef USE_ESP8266
+  /// Out-of-line ESP8266 PROGMEM append to avoid inlining strlen_P/memcpy_P at every call site
+  void append_p_esp8266(const char *str);
+#endif
+
  const char *c_str() const { return buf_; }
  size_t size() const { return pos_; }

@@ -445,7 +474,7 @@ class ProtoMessage {
  uint32_t calculate_size() const { return 0; }
 #ifdef HAS_PROTO_MESSAGE_DUMP
  virtual const char *dump_to(DumpBuffer &out) const = 0;
-  virtual const char *message_name() const { return "unknown"; }
+  virtual const LogString *message_name() const { return LOG_STR("unknown"); }
 #endif

 #ifndef USE_HOST
@@ -97,6 +97,7 @@ CONF_ENABLE_LWIP_ASSERT = "enable_lwip_assert"
 CONF_EXECUTE_FROM_PSRAM = "execute_from_psram"
 CONF_MINIMUM_CHIP_REVISION = "minimum_chip_revision"
 CONF_RELEASE = "release"
+CONF_SRAM1_AS_IRAM = "sram1_as_iram"
 CONF_SUBTYPE = "subtype"

 ARDUINO_FRAMEWORK_NAME = "framework-arduinoespressif32"
@@ -884,6 +885,13 @@ def final_validate(config):
                path=[CONF_FRAMEWORK, CONF_ADVANCED, CONF_MINIMUM_CHIP_REVISION],
            )
        )
+    if config[CONF_VARIANT] != VARIANT_ESP32 and advanced[CONF_SRAM1_AS_IRAM]:
+        errs.append(
+            cv.Invalid(
+                f"'{CONF_SRAM1_AS_IRAM}' is only supported on {VARIANT_ESP32}",
+                path=[CONF_FRAMEWORK, CONF_ADVANCED, CONF_SRAM1_AS_IRAM],
+            )
+        )
    if (
        config[CONF_VARIANT] != VARIANT_ESP32P4
        and config.get(CONF_ENGINEERING_SAMPLE) is not None
@@ -1131,6 +1139,7 @@ FRAMEWORK_SCHEMA = cv.Schema(
                cv.Optional(CONF_MINIMUM_CHIP_REVISION): cv.one_of(
                    *ESP32_CHIP_REVISIONS
                ),
+                cv.Optional(CONF_SRAM1_AS_IRAM, default=False): cv.boolean,
                # DHCP server is needed for WiFi AP mode. When WiFi component is used,
                # it will handle disabling DHCP server when AP is not configured.
                # Default to false (disabled) when WiFi is not used.
@@ -1655,6 +1664,16 @@ async def to_code(config):
            for rev, flag in ESP32_CHIP_REVISIONS.items():
                add_idf_sdkconfig_option(flag, rev == min_rev)
            cg.add_define("USE_ESP32_MIN_CHIP_REVISION_SET")
+
+    # Use SRAM1 region as IRAM on ESP32 (original) variant
+    # This provides an additional 40KB of IRAM by using SRAM1 memory that was previously
+    # reserved for bootloader DRAM. Requires a bootloader from ESP-IDF v5.1 or later.
+    # WARNING: If the device has an old bootloader (pre-v5.1), the app will fail to boot.
+    # A USB flash will update the bootloader automatically. OTA updates do not.
+    # See: https://docs.espressif.com/projects/esp-idf/en/stable/esp32/api-guides/performance/ram-usage.html
+    if variant == VARIANT_ESP32 and conf[CONF_ADVANCED][CONF_SRAM1_AS_IRAM]:
+        add_idf_sdkconfig_option("CONFIG_ESP_SYSTEM_ESP32_SRAM1_REGION_AS_IRAM", True)
+        cg.add_define("USE_ESP32_SRAM1_AS_IRAM")
    add_idf_sdkconfig_option("CONFIG_PARTITION_TABLE_SINGLE_APP", False)
    add_idf_sdkconfig_option("CONFIG_PARTITION_TABLE_CUSTOM", True)
    add_idf_sdkconfig_option("CONFIG_PARTITION_TABLE_CUSTOM_FILENAME", "partitions.csv")
@@ -23,9 +23,8 @@ static const LogString *gpio_mode_to_string(bool use_interrupt) {

 void IRAM_ATTR GPIOBinarySensorStore::gpio_intr(GPIOBinarySensorStore *arg) {
  bool new_state = arg->isr_pin_.digital_read();
-  if (new_state != arg->last_state_) {
+  if (new_state != arg->state_) {
    arg->state_ = new_state;
-    arg->last_state_ = new_state;
    arg->changed_ = true;
    // Wake up the component from its disabled loop state
    if (arg->component_ != nullptr) {
@@ -34,28 +33,27 @@ void IRAM_ATTR GPIOBinarySensorStore::gpio_intr(GPIOBinarySensorStore *arg) {
  }
 }

-void GPIOBinarySensorStore::setup(InternalGPIOPin *pin, gpio::InterruptType type, Component *component) {
+void GPIOBinarySensorStore::setup(InternalGPIOPin *pin, Component *component) {
  pin->setup();
  this->isr_pin_ = pin->to_isr();
  this->component_ = component;

  // Read initial state
-  this->last_state_ = pin->digital_read();
-  this->state_ = this->last_state_;
+  this->state_ = pin->digital_read();

  // Attach interrupt - from this point on, any changes will be caught by the interrupt
-  pin->attach_interrupt(&GPIOBinarySensorStore::gpio_intr, this, type);
+  pin->attach_interrupt(&GPIOBinarySensorStore::gpio_intr, this, this->interrupt_type_);
 }

 void GPIOBinarySensor::setup() {
-  if (this->use_interrupt_ && !this->pin_->is_internal()) {
+  if (this->store_.use_interrupt_ && !this->pin_->is_internal()) {
    ESP_LOGD(TAG, "GPIO is not internal, falling back to polling mode");
-    this->use_interrupt_ = false;
+    this->store_.use_interrupt_ = false;
  }

-  if (this->use_interrupt_) {
+  if (this->store_.use_interrupt_) {
    auto *internal_pin = static_cast<InternalGPIOPin *>(this->pin_);
-    this->store_.setup(internal_pin, this->interrupt_type_, this);
+    this->store_.setup(internal_pin, this);
    this->publish_initial_state(this->store_.get_state());
  } else {
    this->pin_->setup();
@@ -66,14 +64,14 @@ void GPIOBinarySensor::setup() {
 void GPIOBinarySensor::dump_config() {
  LOG_BINARY_SENSOR("", "GPIO Binary Sensor", this);
  LOG_PIN("  Pin: ", this->pin_);
-  ESP_LOGCONFIG(TAG, "  Mode: %s", LOG_STR_ARG(gpio_mode_to_string(this->use_interrupt_)));
-  if (this->use_interrupt_) {
-    ESP_LOGCONFIG(TAG, "  Interrupt Type: %s", LOG_STR_ARG(interrupt_type_to_string(this->interrupt_type_)));
+  ESP_LOGCONFIG(TAG, "  Mode: %s", LOG_STR_ARG(gpio_mode_to_string(this->store_.use_interrupt_)));
+  if (this->store_.use_interrupt_) {
+    ESP_LOGCONFIG(TAG, "  Interrupt Type: %s", LOG_STR_ARG(interrupt_type_to_string(this->store_.interrupt_type_)));
  }
 }

 void GPIOBinarySensor::loop() {
-  if (this->use_interrupt_) {
+  if (this->store_.use_interrupt_) {
    if (this->store_.is_changed()) {
      // Clear the flag immediately to minimize the window where we might miss changes
      this->store_.clear_changed();
@@ -8,10 +8,10 @@
 namespace esphome {
 namespace gpio {

-// Store class for ISR data (no vtables, ISR-safe)
+// Store class for ISR data and configuration (no vtables, ISR-safe)
 class GPIOBinarySensorStore {
 public:
-  void setup(InternalGPIOPin *pin, gpio::InterruptType type, Component *component);
+  void setup(InternalGPIOPin *pin, Component *component);

  static void gpio_intr(GPIOBinarySensorStore *arg);

@@ -32,11 +32,13 @@ class GPIOBinarySensorStore {
  }

 protected:
+  friend class GPIOBinarySensor;
  ISRInternalGPIOPin isr_pin_;
-  volatile bool state_{false};
-  volatile bool last_state_{false};
-  volatile bool changed_{false};
  Component *component_{nullptr};  // Pointer to the component for enable_loop_soon_any_context()
+  volatile bool state_{false};
+  volatile bool changed_{false};
+  bool use_interrupt_{true};
+  gpio::InterruptType interrupt_type_{gpio::INTERRUPT_ANY_EDGE};
 };

 class GPIOBinarySensor final : public binary_sensor::BinarySensor, public Component {
@@ -44,9 +46,9 @@ class GPIOBinarySensor final : public binary_sensor::BinarySensor, public Compon
  // No destructor needed: ESPHome components are created at boot and live forever.
  // Interrupts are only detached on reboot when memory is cleared anyway.

-  void set_pin(GPIOPin *pin) { pin_ = pin; }
-  void set_use_interrupt(bool use_interrupt) { use_interrupt_ = use_interrupt; }
-  void set_interrupt_type(gpio::InterruptType type) { interrupt_type_ = type; }
+  void set_pin(GPIOPin *pin) { this->pin_ = pin; }
+  void set_use_interrupt(bool use_interrupt) { this->store_.use_interrupt_ = use_interrupt; }
+  void set_interrupt_type(gpio::InterruptType type) { this->store_.interrupt_type_ = type; }
  // ========== INTERNAL METHODS ==========
  // (In most use cases you won't need these)
  /// Setup pin
@@ -59,8 +61,6 @@ class GPIOBinarySensor final : public binary_sensor::BinarySensor, public Compon

 protected:
  GPIOPin *pin_;
-  bool use_interrupt_{true};
-  gpio::InterruptType interrupt_type_{gpio::INTERRUPT_ANY_EDGE};
  GPIOBinarySensorStore store_;
 };

@@ -748,7 +748,7 @@ void HonClimate::update_sub_sensor_(SubSensorType type, float value) {
  if (type < SubSensorType::SUB_SENSOR_TYPE_COUNT) {
    size_t index = (size_t) type;
    if ((this->sub_sensors_[index] != nullptr) &&
-        ((!this->sub_sensors_[index]->has_state()) || (this->sub_sensors_[index]->raw_state != value)))
+        ((!this->sub_sensors_[index]->has_state()) || (this->sub_sensors_[index]->get_raw_state() != value)))
      this->sub_sensors_[index]->publish_state(value);
  }
 }
@@ -322,22 +322,6 @@ class LightState : public EntityBase, public Component {
  FixedVector<LightEffect *> effects_;
  /// Object used to store the persisted values of the light.
  ESPPreferenceObject rtc_;
-  /// Value for storing the index of the currently active effect. 0 if no effect is active
-  uint32_t active_effect_index_{};
-  /// Default transition length for all transitions in ms.
-  uint32_t default_transition_length_{};
-  /// Transition length to use for flash transitions.
-  uint32_t flash_transition_length_{};
-  /// Gamma correction factor for the light.
-  float gamma_correct_{};
-#ifdef USE_LIGHT_GAMMA_LUT
-  const uint16_t *gamma_table_{nullptr};
-#endif  // USE_LIGHT_GAMMA_LUT
-
-  /// Whether the light value should be written in the next cycle.
-  bool next_write_{true};
-  // for effects, true if a transformer (transition) is active.
-  bool is_transformer_active_ = false;

  /** Listeners for remote values changes.
   *
@@ -361,6 +345,22 @@ class LightState : public EntityBase, public Component {
  /// Initial state of the light.
  optional<LightStateRTCState> initial_state_{};

+  /// Value for storing the index of the currently active effect. 0 if no effect is active
+  uint32_t active_effect_index_{};
+  /// Default transition length for all transitions in ms.
+  uint32_t default_transition_length_{};
+  /// Transition length to use for flash transitions.
+  uint32_t flash_transition_length_{};
+  /// Gamma correction factor for the light.
+  float gamma_correct_{};
+#ifdef USE_LIGHT_GAMMA_LUT
+  const uint16_t *gamma_table_{nullptr};
+#endif  // USE_LIGHT_GAMMA_LUT
+
+  /// Whether the light value should be written in the next cycle.
+  bool next_write_{true};
+  // for effects, true if a transformer (transition) is active.
+  bool is_transformer_active_{false};
  /// Restore mode of the light.
  LightRestoreMode restore_mode_;
 };
@@ -85,10 +85,6 @@ void NextionSensor::set_state(float state, bool publish, bool send_to_nextion) {
      }

      this->publish_state(published_state);
-    } else {
-      this->raw_state = state;
-      this->state = state;
-      this->set_has_state(true);
    }
  }
  this->update_component_settings();
@@ -0,0 +1,25 @@
+import esphome.codegen as cg
+from esphome.components import sensor
+import esphome.config_validation as cv
+from esphome.const import CONF_SOURCE_ID
+
+from .. import Number, number_ns
+
+NumberSensor = number_ns.class_("NumberSensor", sensor.Sensor, cg.Component)
+
+
+CONFIG_SCHEMA = (
+    sensor.sensor_schema(NumberSensor)
+    .extend(
+        {
+            cv.Required(CONF_SOURCE_ID): cv.use_id(Number),
+        }
+    )
+    .extend(cv.COMPONENT_SCHEMA)
+)
+
+
+async def to_code(config):
+    source = await cg.get_variable(config[CONF_SOURCE_ID])
+    var = await sensor.new_sensor(config, source)
+    await cg.register_component(var, config)
@@ -0,0 +1,16 @@
+#include "number_sensor.h"
+#include "esphome/core/log.h"
+
+namespace esphome::number {
+
+static const char *const TAG = "number.sensor";
+
+void NumberSensor::setup() {
+  this->source_->add_on_state_callback([this](float value) { this->publish_state(value); });
+  if (this->source_->has_state())
+    this->publish_state(this->source_->state);
+}
+
+void NumberSensor::dump_config() { LOG_SENSOR("", "Number Sensor", this); }
+
+}  // namespace esphome::number
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "../number.h"
+#include "esphome/core/component.h"
+#include "esphome/components/sensor/sensor.h"
+
+namespace esphome::number {
+
+class NumberSensor : public sensor::Sensor, public Component {
+ public:
+  explicit NumberSensor(Number *source) : source_(source) {}
+  void setup() override;
+  void dump_config() override;
+
+ protected:
+  Number *source_;
+};
+
+}  // namespace esphome::number
@@ -226,7 +226,7 @@ def _process_remote_package(config: dict, skip_update: bool = False) -> dict:
                        raise cv.Invalid(
                            f"Current ESPHome Version is too old to use this package: {ESPHOME_VERSION} < {min_version}"
                        )
-                new_yaml = yaml_util.substitute_vars(new_yaml, vars)
+                new_yaml = yaml_util.add_context(new_yaml, vars or None)
                packages[f"{filename}{idx}"] = new_yaml
            except EsphomeError as e:
                raise cv.Invalid(
@@ -296,6 +296,18 @@ def do_packages_pass(config: dict, skip_update: bool = False) -> dict:

    def process_package_callback(package_config: dict) -> dict:
        """This will be called for each package found in the config."""
+        if isinstance(package_config, yaml_util.ConfigContext):
+            context_vars = package_config.vars
+            if CONF_PACKAGES in package_config or CONF_URL in package_config:
+                # Remote package definition: eagerly resolve before PACKAGE_SCHEMA validation.
+                from esphome.components.substitutions import ContextVars, substitute
+
+                package_config = substitute(
+                    package_config,
+                    [],
+                    ContextVars(context_vars),
+                    strict_undefined=False,
+                )
        package_config = PACKAGE_SCHEMA(package_config)
        if isinstance(package_config, str):
            return package_config  # Jinja string, skip processing
@@ -40,7 +40,10 @@ const LogString *state_class_to_string(StateClass state_class) {
  return StateClassStrings::get_log_str(static_cast<uint8_t>(state_class), 0);
 }

+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 Sensor::Sensor() : state(NAN), raw_state(NAN) {}
+#pragma GCC diagnostic pop

 int8_t Sensor::get_accuracy_decimals() {
  if (this->sensor_flags_.has_accuracy_override)
@@ -63,8 +66,13 @@ StateClass Sensor::get_state_class() {
 }

 void Sensor::publish_state(float state) {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
  this->raw_state = state;
+#pragma GCC diagnostic pop
+#ifdef USE_SENSOR_FILTER
  this->raw_callback_.call(state);
+#endif

  ESP_LOGV(TAG, "'%s': Received new state %f", this->name_.c_str(), state);

@@ -110,8 +118,6 @@ void Sensor::clear_filters() {
  this->filter_list_ = nullptr;
 }
 #endif  // USE_SENSOR_FILTER
-float Sensor::get_state() const { return this->state; }
-float Sensor::get_raw_state() const { return this->raw_state; }

 void Sensor::internal_send_state_to_frontend(float state) {
  this->set_has_state(true);
@@ -95,9 +95,14 @@ class Sensor : public EntityBase {
 #endif

  /// Getter-syntax for .state.
-  float get_state() const;
+  float get_state() const { return this->state; }
  /// Getter-syntax for .raw_state
-  float get_raw_state() const;
+  float get_raw_state() const {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+    return this->raw_state;
+#pragma GCC diagnostic pop
+  }

  /** Publish a new state to the front-end.
   *
@@ -113,8 +118,14 @@ class Sensor : public EntityBase {
  /// Add a callback that will be called every time a filtered value arrives.
  template<typename F> void add_on_state_callback(F &&callback) { this->callback_.add(std::forward<F>(callback)); }
  /// Add a callback that will be called every time the sensor sends a raw value.
+  /// When USE_SENSOR_FILTER is not enabled, delegates to the regular callback
+  /// since raw state equals filtered state without filter support compiled in.
  template<typename F> void add_on_raw_state_callback(F &&callback) {
+#ifdef USE_SENSOR_FILTER
    this->raw_callback_.add(std::forward<F>(callback));
+#else
+    this->callback_.add(std::forward<F>(callback));
+#endif
  }

  /** This member variable stores the last state that has passed through all filters.
@@ -126,17 +137,20 @@ class Sensor : public EntityBase {
   */
  float state;

-  /** This member variable stores the current raw state of the sensor, without any filters applied.
-   *
-   * Unlike .state,this will be updated immediately when publish_state is called.
-   */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+  /// @deprecated Use get_raw_state() instead. This member will be removed in ESPHome 2026.10.0.
+  ESPDEPRECATED("Use get_raw_state() instead of .raw_state. Will be removed in 2026.10.0", "2026.4.0")
  float raw_state;
+#pragma GCC diagnostic pop

  void internal_send_state_to_frontend(float state);

 protected:
+#ifdef USE_SENSOR_FILTER
  LazyCallbackManager<void(float)> raw_callback_;  ///< Storage for raw state callbacks.
-  LazyCallbackManager<void(float)> callback_;      ///< Storage for filtered state callbacks.
+#endif
+  LazyCallbackManager<void(float)> callback_;  ///< Storage for filtered state callbacks.

 #ifdef USE_SENSOR_FILTER
  Filter *filter_list_{nullptr};  ///< Store all active filters.
@@ -1,31 +1,50 @@
+from collections import ChainMap
 import logging
-from re import Match
 from typing import Any

 from esphome import core
 from esphome.config_helpers import Extend, Remove, merge_config, merge_dicts_ordered
 import esphome.config_validation as cv
 from esphome.const import CONF_SUBSTITUTIONS, VALID_SUBSTITUTIONS_CHARACTERS
-from esphome.yaml_util import ESPHomeDataBase, ESPLiteralValue, make_data_base
+from esphome.types import ConfigType
+from esphome.util import OrderedDict
+from esphome.yaml_util import (
+    ConfigContext,
+    ESPHomeDataBase,
+    ESPLiteralValue,
+    make_data_base,
+)

-from .jinja import Jinja, JinjaError, JinjaStr, has_jinja
+from .jinja import Jinja, JinjaError, Missing, Resolver, UndefinedError, has_jinja

 CODEOWNERS = ["@esphome/core"]
 _LOGGER = logging.getLogger(__name__)

+ContextVars = ChainMap[str, Any]
+SubstitutionPath = list[int | str]
+ErrList = list[tuple[UndefinedError, SubstitutionPath, Any]]
+# Module-level instance is safe: context_vars is passed per-call, and context_trace
+# is stack-saved/restored within expand(). Not thread-safe — only use from one thread.
+jinja = Jinja()

-def validate_substitution_key(value):
+
+def validate_substitution_key(value: Any) -> str:
+    """Validate and normalize a substitution key, stripping a leading ``$`` if present."""
    value = cv.string(value)
    if not value:
        raise cv.Invalid("Substitution key must not be empty")
    if value[0] == "$":
        value = value[1:]
+    if not value:
+        raise cv.Invalid("Substitution key must not be empty")
    if value[0].isdigit():
        raise cv.Invalid("First character in substitutions cannot be a digit.")
    for char in value:
        if char not in VALID_SUBSTITUTIONS_CHARACTERS:
            raise cv.Invalid(
-                f"Substitution must only consist of upper/lowercase characters, the underscore and numbers. The character '{char}' cannot be used"
+                f"Substitution must only consist of upper/lowercase characters,"
+                f" the underscore and numbers."
+                f" The character '{char}' cannot be used"
            )
    return value

@@ -37,8 +56,8 @@ CONFIG_SCHEMA = cv.Schema(
 )


-async def to_code(config):
-    pass
+async def to_code(config: ConfigType) -> None:
+    """No runtime code generation needed — substitutions are resolved at config time."""


 def _restore_data_base(value: Any, orig_value: ESPHomeDataBase) -> ESPHomeDataBase:
@@ -62,91 +81,116 @@ def _restore_data_base(value: Any, orig_value: ESPHomeDataBase) -> ESPHomeDataBa
    return value


-def _expand_jinja(
-    value: str | JinjaStr,
-    orig_value: str | JinjaStr,
-    path,
-    jinja: Jinja,
-    ignore_missing: bool,
-) -> Any:
-    if has_jinja(value):
-        # If the original value passed in to this function is a JinjaStr, it means it contains an unresolved
-        # Jinja expression from a previous pass.
-        if isinstance(orig_value, JinjaStr):
-            # Rebuild the JinjaStr in case it was lost while replacing substitutions.
-            value = JinjaStr(value, orig_value.upvalues)
-        try:
-            # Invoke the jinja engine to evaluate the expression.
-            value, err = jinja.expand(value)
-            if err is not None and not ignore_missing and "password" not in path:
-                _LOGGER.warning(
-                    "Found '%s' (see %s) which looks like an expression,"
-                    " but could not resolve all the variables: %s",
-                    value,
-                    "->".join(str(x) for x in path),
-                    err.message,
-                )
-        except JinjaError as err:
-            raise cv.Invalid(
-                f"{err.error_name()} Error evaluating jinja expression '{value}': {str(err.parent())}."
-                f"\nEvaluation stack: (most recent evaluation last)\n{err.stack_trace_str()}"
-                f"\nRelevant context:\n{err.context_trace_str()}"
-                f"\nSee {'->'.join(str(x) for x in path)}",
-                path,
-            )
-        # If the original, unexpanded string, contained document metadata (ESPHomeDatabase),
-        # assign this same document metadata to the resulting value.
-        if isinstance(orig_value, ESPHomeDataBase):
-            value = _restore_data_base(value, orig_value)
+def _resolve_var(name: str, context_vars: ContextVars) -> Any:
+    """Look up a substitution variable, falling back to the resolver callback."""
+    sub = context_vars.get(name, Missing)
+    if sub is Missing:
+        resolver = context_vars.get(Resolver)
+        if resolver:
+            sub = resolver(name)
+    return sub

-    return value
+
+def _handle_undefined(
+    err: UndefinedError,
+    path: SubstitutionPath,
+    value: Any,
+    strict_undefined: bool,
+    errors: ErrList | None,
+) -> None:
+    """Handle an undefined variable.
+
+    In strict mode, raises immediately. Otherwise, appends to the errors
+    list for deferred warning at the end of the substitution pass.
+    """
+    if strict_undefined:
+        raise err
+    if errors is not None:
+        errors.append((err, path, value))


 def _expand_substitutions(
-    substitutions: dict, value: str, path, jinja: Jinja, ignore_missing: bool
+    value: str,
+    path: SubstitutionPath,
+    context_vars: ContextVars,
+    strict_undefined: bool,
+    errors: ErrList | None,
 ) -> Any:
+    """Expand ``$var``, ``${var}``, and Jinja expressions in a string.
+
+    Works in two phases:
+
+    1. **Simple substitution** — scan for ``$name`` / ``${name}`` tokens
+       and replace them with the value from *context_vars*.  If the token
+       spans the entire string, return the raw value (preserving type).
+    2. **Jinja evaluation** — if the result still contains Jinja syntax
+       (e.g. ``${a * b}``), render it through the Jinja engine with the
+       full *context_vars* as template variables.
+
+    Returns the expanded value (may be a non-string type) or the
+    original *value* unchanged if there is nothing to substitute.
+    """
    if "$" not in value:
        return value

    orig_value = value

-    i = 0
-    while True:
-        m: Match[str] = cv.VARIABLE_PROG.search(value, i)
-        if not m:
-            # No more variable substitutions found. See if the remainder looks like a jinja template
-            value = _expand_jinja(value, orig_value, path, jinja, ignore_missing)
-            break
-
-        i, j = m.span(0)
+    # Phase 1: Replace $var and ${var} references
+    search_pos = 0
+    while (m := cv.VARIABLE_PROG.search(value, search_pos)) is not None:
+        match_start, match_end = m.span(0)
        name: str = m.group(1)
        if name.startswith("{") and name.endswith("}"):
            name = name[1:-1]
-        if name not in substitutions:
-            if not ignore_missing and "password" not in path:
-                _LOGGER.warning(
-                    "Found '%s' (see %s) which looks like a substitution, but '%s' was "
-                    "not declared",
-                    orig_value,
-                    "->".join(str(x) for x in path),
-                    name,
-                )
-            i = j
+        sub = _resolve_var(name, context_vars)
+        if sub is Missing:
+            _handle_undefined(
+                err=UndefinedError(f"'{name}' is undefined"),
+                path=path,
+                value=value,
+                strict_undefined=strict_undefined,
+                errors=errors,
+            )
+            search_pos = match_end
            continue

-        sub: Any = substitutions[name]
-
-        if i == 0 and j == len(value):
-            # The variable spans the whole expression, e.g., "${varName}". Return its resolved value directly
-            # to conserve its type.
+        if match_start == 0 and match_end == len(value):
+            # The variable spans the whole expression, e.g., "${varName}".
+            # Return its resolved value directly to conserve its type.
            value = sub
            break

-        tail = value[j:]
-        value = value[:i] + str(sub)
-        i = len(value)
+        tail = value[match_end:]
+        value = value[:match_start] + str(sub)
+        search_pos = len(value)
        value += tail

+    # Phase 2: Evaluate any remaining jinja expressions (e.g., "${a * b}")
+    if isinstance(value, str) and has_jinja(value):
+        try:
+            value = jinja.expand(value, context_vars)
+        except UndefinedError as err:
+            _handle_undefined(
+                err=err,
+                path=path,
+                value=value,
+                strict_undefined=strict_undefined,
+                errors=errors,
+            )
+        except JinjaError as err:
+            raise cv.Invalid(
+                f"{err.error_name()} Error evaluating jinja expression"
+                f" '{value}': {str(err.parent())}."
+                f"\nEvaluation stack: (most recent evaluation last)"
+                f"\n{err.stack_trace_str()}"
+                f"\nRelevant context:\n{err.context_trace_str()}"
+                f"\nSee {'->'.join(str(x) for x in path)}",
+                path,
+            )
+        else:
+            if isinstance(orig_value, ESPHomeDataBase):
+                value = _restore_data_base(value, orig_value)
+
    # orig_value can also already be a lambda with esp_range info, and only
    # a plain string is sent in orig_value
    if isinstance(orig_value, ESPHomeDataBase):
@@ -157,83 +201,204 @@ def _expand_substitutions(
    return value


-def _substitute_item(
-    substitutions: dict,
+def _push_context(
+    local_vars: dict[str, Any],
+    parent_context: ContextVars,
+    errors: ErrList | None = None,
+) -> tuple[ContextVars, dict[str, Any]]:
+    """Resolve local_vars and layer them on top of parent_context.
+
+    Returns ``(child_context, resolved_vars)`` where *child_context* is a
+    new :class:`ChainMap` whose front map is *resolved_vars* (an
+    :class:`OrderedDict` of successfully-resolved variables).
+
+    Variables may reference each other (e.g. ``b: ${a + 1}``).
+    Dependencies are resolved recursively via a *resolver* callback
+    that Jinja invokes on cache-miss.  If vars are already in
+    dependency order, the loop iterates exactly once per variable.
+
+    The ChainMap stack used during resolution is::
+
+        resolver_context  →  resolved_vars  →  parent maps …
+              ↑                    ↑
+        holds Resolver       filled as vars
+        callback               are resolved
+    """
+    # Vars still waiting to be resolved — popped one-by-one by resolve().
+    unresolved_vars = local_vars.copy()
+    # Accumulates resolved values in dependency order; becomes the front
+    # map of the returned child context so later lookups find them first.
+    resolved_vars = OrderedDict()
+    # The context callees will search: resolved_vars (initially empty)
+    # shadowing whatever the parent already provides.
+    context_vars = parent_context.new_child(resolved_vars)
+
+    # Vars that failed resolution (missing or circular references).
+    # Maps name → (original_value, cause_error) for deferred warnings.
+    unresolvables: dict[str, tuple[Any, UndefinedError]] = {}
+
+    # One extra child layer so the Resolver callback lives in its own
+    # map and doesn't pollute resolved_vars.
+    resolver_context = context_vars.new_child()
+
+    def resolve(key: str) -> Any:
+        """Resolve a variable, recursively resolving any dependencies it references."""
+        value = unresolved_vars.pop(key, Missing)
+        if value is Missing:
+            return Missing
+        try:
+            value = substitute(value, [], resolver_context, True)
+        except UndefinedError as err:
+            unresolvables[key] = (value, err)
+            return Missing
+        resolved_vars[key] = value
+        return value
+
+    # Set up the resolver for use during substitution
+    resolver_context[Resolver] = resolve
+
+    # Resolve all variables, recursively resolving dependencies as needed.
+    # Each call to resolve() resolves that variable and any variables it depends on.
+    while unresolved_vars:
+        resolve(next(iter(unresolved_vars)))
+
+    for name, (value, cause) in unresolvables.items():
+        resolved_vars[name] = value
+        if errors is not None:
+            _handle_undefined(
+                err=UndefinedError(
+                    f"Could not resolve substitution variable '{name}': {cause}"
+                ),
+                path=["substitutions", name],
+                value=value,
+                strict_undefined=False,
+                errors=errors,
+            )
+
+    return context_vars, resolved_vars
+
+
+def push_context(
+    config_node: Any,
+    parent_context: ContextVars,
+    errors: ErrList | None = None,
+) -> ContextVars:
+    """Returns the context vars this config node must be evaluated with."""
+    if isinstance(config_node, ConfigContext):
+        return _push_context(config_node.vars, parent_context, errors)[0]
+
+    # This node does not define any vars itself, so just return parent context
+    return parent_context
+
+
+def substitute(
    item: Any,
-    path: list[int | str],
-    jinja: Jinja,
-    ignore_missing: bool,
-) -> Any | None:
+    path: SubstitutionPath,
+    parent_context: ContextVars,
+    strict_undefined: bool,
+    errors: ErrList | None = None,
+) -> Any:
+    """Returns a recursively substituted version of `item`."""
+
    if isinstance(item, ESPLiteralValue):
-        return None  # do not substitute inside literal blocks
+        return item  # do not substitute inside literal blocks
+
+    # Push the current item's context onto the context stack
+    context_vars = push_context(item, parent_context, errors)
+
+    result = item
+
    if isinstance(item, list):
-        for i, it in enumerate(item):
-            sub = _substitute_item(substitutions, it, path + [i], jinja, ignore_missing)
-            if sub is not None:
-                item[i] = sub
+        result = [
+            substitute(it, path + [i], context_vars, strict_undefined, errors)
+            for i, it in enumerate(item)
+        ]
+
    elif isinstance(item, dict):
-        replace_keys = []
+        result = OrderedDict()
        for k, v in item.items():
-            if path or k != CONF_SUBSTITUTIONS:
-                sub = _substitute_item(
-                    substitutions, k, path + [k], jinja, ignore_missing
-                )
-                if sub is not None:
-                    replace_keys.append((k, sub))
-            sub = _substitute_item(substitutions, v, path + [k], jinja, ignore_missing)
-            if sub is not None:
-                item[k] = sub
-        for old, new in replace_keys:
-            if str(new) == str(old):
-                item[new] = item[old]
-            else:
-                item[new] = merge_config(item.get(old), item.get(new))
-                del item[old]
+            v = substitute(v, path + [k], context_vars, strict_undefined, errors)
+            k = substitute(k, path + [k], context_vars, strict_undefined, errors)
+            result[k] = merge_config(result.get(k), v)
+
    elif isinstance(item, str):
-        sub = _expand_substitutions(substitutions, item, path, jinja, ignore_missing)
-        if isinstance(sub, JinjaStr) or sub != item:
-            return sub
-    elif isinstance(item, (core.Lambda, Extend, Remove)):
-        sub = _expand_substitutions(
-            substitutions, item.value, path, jinja, ignore_missing
+        result = _expand_substitutions(
+            item, path, context_vars, strict_undefined, errors
+        )
+
+    elif isinstance(item, (core.Lambda, Extend, Remove)) and item.value:
+        value = _expand_substitutions(
+            item.value, path, context_vars, strict_undefined, errors
+        )
+        if item.value != value:
+            result = type(item)(value)
+
+    if isinstance(item, ESPHomeDataBase):
+        result = make_data_base(result, item)
+    return result
+
+
+def _warn_unresolved_variables(errors: ErrList) -> None:
+    """Log warnings for unresolved substitution variables, skipping password fields."""
+    for err, path, expression in errors:
+        if "password" in path:
+            continue
+        location: str = "->".join(str(x) for x in path)
+        if isinstance(expression, ESPHomeDataBase) and expression.esp_range is not None:
+            location += f" in {str(expression.esp_range.start_mark)}"
+
+        _LOGGER.warning(
+            "The string '%s' looks like an expression,"
+            " but could not resolve all the variables: %s (see %s)",
+            expression,
+            err.message,
+            location,
        )
-        if sub != item:
-            item.value = sub
-    return None


 def do_substitution_pass(
-    config: dict, command_line_substitutions: dict, ignore_missing: bool = False
-) -> None:
-    if CONF_SUBSTITUTIONS not in config and not command_line_substitutions:
-        return
+    config: OrderedDict, command_line_substitutions: dict[str, Any] | None = None
+) -> OrderedDict:
+    """Run the substitution pass over the entire config.

-    # Merge substitutions in config, overriding with substitutions coming from command line:
+    Extracts the ``substitutions:`` block, merges in any command-line
+    overrides, resolves inter-variable dependencies, then walks the
+    config tree replacing all ``$var`` / ``${expr}`` references.
+    Returns a new config dict with resolved substitutions
+    restored at the front.
+    """
+    # Extract substitutions from config, overriding with substitutions coming from command line:
    # Use merge_dicts_ordered to preserve OrderedDict type for move_to_end()
-    substitutions = merge_dicts_ordered(
-        config.get(CONF_SUBSTITUTIONS, {}), command_line_substitutions or {}
-    )
-    with cv.prepend_path("substitutions"):
+    substitutions = config.pop(CONF_SUBSTITUTIONS, {})
+    with cv.prepend_path(CONF_SUBSTITUTIONS):
        if not isinstance(substitutions, dict):
            raise cv.Invalid(
                f"Substitutions must be a key to value mapping, got {type(substitutions)}"
            )
+        substitutions = merge_dicts_ordered(
+            substitutions, command_line_substitutions or {}
+        )

-        replace_keys = []
-        for key, value in substitutions.items():
+        replace_keys: list[tuple[str, str]] = []
+        for key in substitutions:
            with cv.prepend_path(key):
                sub = validate_substitution_key(key)
                if sub != key:
                    replace_keys.append((key, sub))
-                substitutions[key] = value
        for old, new in replace_keys:
            substitutions[new] = substitutions[old]
            del substitutions[old]

-    config[CONF_SUBSTITUTIONS] = substitutions
-    # Move substitutions to the first place to replace substitutions in them correctly
-    config.move_to_end(CONF_SUBSTITUTIONS, False)
+    errors: ErrList = []  # Collect undefined errors during substitution
+    parent_context, substitutions = _push_context(substitutions, ContextVars(), errors)

-    # Create a Jinja environment that will consider substitutions in scope:
-    jinja = Jinja(substitutions)
-    _substitute_item(substitutions, config, [], jinja, ignore_missing)
+    config = substitute(config, [], parent_context, False, errors)
+
+    if errors:
+        _warn_unresolved_variables(errors)
+
+    # Restore substitutions to front of dict for readability
+    if substitutions:
+        config[CONF_SUBSTITUTIONS] = substitutions
+        config.move_to_end(CONF_SUBSTITUTIONS, last=False)
+    return config
@@ -1,7 +1,6 @@
 from ast import literal_eval
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
 from itertools import chain, islice
-import logging
 import math
 import re
 from types import GeneratorType
@@ -9,16 +8,17 @@ from typing import Any

 import jinja2 as jinja
 from jinja2.nativetypes import NativeCodeGenerator, NativeTemplate
-
-from esphome.yaml_util import ESPLiteralValue
+from jinja2.runtime import missing as Missing

 TemplateError = jinja.TemplateError
 TemplateSyntaxError = jinja.TemplateSyntaxError
 TemplateRuntimeError = jinja.TemplateRuntimeError
 UndefinedError = jinja.UndefinedError
 Undefined = jinja.Undefined
+# Sentinel key for resolver callback in ContextVars.
+# Dots are invalid in substitution names so this can never collide with user keys.
+Resolver = ".resolver"

-_LOGGER = logging.getLogger(__name__)

 DETECT_JINJA = r"(\$\{)"
 detect_jinja_re = re.compile(
@@ -52,33 +52,6 @@ SAFE_GLOBALS = {
 }


-class JinjaStr(str):
-    """
-    Wraps a string containing an unresolved Jinja expression,
-    storing the variables visible to it when it failed to resolve.
-    For example, an expression inside a package, `${ A * B }` may fail
-    to resolve at package parsing time if `A` is a local package var
-    but `B` is a substitution defined in the root yaml.
-    Therefore, we store the value of `A` as an upvalue bound
-    to the original string so we may be able to resolve `${ A * B }`
-    later in the main substitutions pass.
-    """
-
-    Undefined = object()
-
-    def __new__(cls, value: str, upvalues=None):
-        if isinstance(value, JinjaStr):
-            base = str(value)
-            merged = {**value.upvalues, **(upvalues or {})}
-        else:
-            base = value
-            merged = dict(upvalues or {})
-        obj = super().__new__(cls, base)
-        obj.upvalues = merged
-        obj.result = JinjaStr.Undefined
-        return obj
-
-
 class JinjaError(Exception):
    def __init__(self, context_trace: dict, expr: str):
        self.context_trace = context_trace
@@ -106,9 +79,13 @@ class JinjaError(Exception):
 class TrackerContext(jinja.runtime.Context):
    def resolve_or_missing(self, key):
        val = super().resolve_or_missing(key)
-        if isinstance(val, JinjaStr):
-            self.environment.context_trace[key] = val
-            val, _ = self.environment.expand(val)
+        if val is Missing:
+            # Variable not in the template context — check if a resolver callback
+            # was registered (by _push_context) to lazily resolve dependencies
+            # between substitution variables in the same block.
+            resolver = super().resolve_or_missing(Resolver)
+            if resolver is not Missing:
+                val = resolver(key)
        self.environment.context_trace[key] = val
        return val

@@ -160,15 +137,13 @@ def _concat_nodes_override(values: Iterator[Any]) -> Any:


 class Jinja(jinja.Environment):
-    """
-    Wraps a Jinja environment
-    """
+    """Jinja environment configured for ESPHome substitution expressions."""

    # jinja environment customization overrides
    code_generator_class = NativeCodeGenerator
    concat = staticmethod(_concat_nodes_override)

-    def __init__(self, context_vars: dict):
+    def __init__(self) -> None:
        super().__init__(
            trim_blocks=True,
            lstrip_blocks=True,
@@ -183,49 +158,25 @@ class Jinja(jinja.Environment):
        self.context_class = TrackerContext
        self.add_extension("jinja2.ext.do")
        self.context_trace = {}
-        self.context_vars = {**context_vars}
-        for k, v in self.context_vars.items():
-            if isinstance(v, ESPLiteralValue):
-                continue
-            if isinstance(v, str) and not isinstance(v, JinjaStr) and has_jinja(v):
-                self.context_vars[k] = JinjaStr(v, self.context_vars)

-        self.globals = {
-            **self.globals,
-            **self.context_vars,
-            **SAFE_GLOBALS,
-        }
+        self.globals = {**self.globals, **SAFE_GLOBALS}

-    def expand(self, content_str: str | JinjaStr) -> Any:
+    def expand(self, content_str: str, context_vars: Mapping[str, Any]) -> Any:
        """
        Renders a string that may contain Jinja expressions or statements
        Returns the resulting value if all variables and expressions could be resolved.
-        Otherwise, it returns a tagged (JinjaStr) string that captures variables
-        in scope (upvalues), like a closure for later evaluation.
        """
        result = None
-        override_vars = {}
-        if isinstance(content_str, JinjaStr):
-            if content_str.result is not JinjaStr.Undefined:
-                return content_str.result, None
-            # If `value` is already a JinjaStr, it means we are trying to evaluate it again
-            # in a parent pass.
-            # Hopefully, all required variables are visible now.
-            override_vars = content_str.upvalues

        old_trace = self.context_trace
        self.context_trace = {}
        try:
            template = self.from_string(content_str)
-            result = template.render(override_vars)
+            result = template.render(context_vars)
            if isinstance(result, Undefined):
-                print("" + result)  # force a UndefinedError exception
-        except (TemplateSyntaxError, UndefinedError) as err:
-            # `content_str` contains a Jinja expression that refers to a variable that is undefined
-            # in this scope. Perhaps it refers to a root substitution that is not visible yet.
-            # Therefore, return `content_str` as a JinjaStr, which contains the variables
-            # that are actually visible to it at this point to postpone evaluation.
-            return JinjaStr(content_str, {**self.context_vars, **override_vars}), err
+                str(result)  # force a UndefinedError exception
+        except UndefinedError as err:
+            raise err
        except JinjaError as err:
            err.context_trace = {**self.context_trace, **err.context_trace}
            err.eval_stack.append(content_str)
@@ -242,10 +193,7 @@ class Jinja(jinja.Environment):
        finally:
            self.context_trace = old_trace

-        if isinstance(content_str, JinjaStr):
-            content_str.result = result
-
-        return result, None
+        return result


 class JinjaTemplate(NativeTemplate):
@@ -664,11 +664,22 @@ bool WiFiComponent::wifi_scan_start_(bool passive) {
  config.show_hidden = 1;
 #if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0)
  config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE;
+  // Use shorter dwell times for roaming scans - we only need to detect strong
+  // nearby APs, not do a thorough survey. This also reduces off-channel time
+  // which can cause Beacon Timeout disconnects on some APs.
+  // Roaming times match the ESP32 IDF scan defaults.
+  static constexpr uint32_t SCAN_PASSIVE_DEFAULT_MS = 500;
+  static constexpr uint32_t SCAN_PASSIVE_ROAMING_MS = 300;
+  static constexpr uint32_t SCAN_ACTIVE_MIN_DEFAULT_MS = 400;
+  static constexpr uint32_t SCAN_ACTIVE_MAX_DEFAULT_MS = 500;
+  static constexpr uint32_t SCAN_ACTIVE_MIN_ROAMING_MS = 100;
+  static constexpr uint32_t SCAN_ACTIVE_MAX_ROAMING_MS = 300;
+  bool roaming = this->roaming_state_ == RoamingState::SCANNING;
  if (passive) {
-    config.scan_time.passive = 500;
+    config.scan_time.passive = roaming ? SCAN_PASSIVE_ROAMING_MS : SCAN_PASSIVE_DEFAULT_MS;
  } else {
-    config.scan_time.active.min = 400;
-    config.scan_time.active.max = 500;
+    config.scan_time.active.min = roaming ? SCAN_ACTIVE_MIN_ROAMING_MS : SCAN_ACTIVE_MIN_DEFAULT_MS;
+    config.scan_time.active.max = roaming ? SCAN_ACTIVE_MAX_ROAMING_MS : SCAN_ACTIVE_MAX_DEFAULT_MS;
  }
 #endif
  bool ret = wifi_station_scan(&config, &WiFiComponent::s_wifi_scan_done_callback);
@@ -12,7 +12,8 @@ from typing import Any
 import voluptuous as vol

 from esphome import core, loader, pins, yaml_util
-from esphome.config_helpers import Extend, Remove, merge_config, merge_dicts_ordered
+from esphome.components.substitutions import do_substitution_pass
+from esphome.config_helpers import Extend, Remove, merge_config
 import esphome.config_validation as cv
 from esphome.const import (
    CONF_ESPHOME,
@@ -974,7 +975,7 @@ class PinUseValidationCheck(ConfigValidationStep):

 def validate_config(
    config: dict[str, Any],
-    command_line_substitutions: dict[str, Any],
+    command_line_substitutions: dict[str, Any] | None,
    skip_external_update: bool = False,
 ) -> Config:
    result = Config()
@@ -994,21 +995,15 @@ def validate_config(
            result.add_error(err)
            return result

-    CORE.raw_config = config
-
    # 1. Load substitutions
    if CONF_SUBSTITUTIONS in config or command_line_substitutions:
-        from esphome.components import substitutions
-
-        result[CONF_SUBSTITUTIONS] = merge_dicts_ordered(
-            config.get(CONF_SUBSTITUTIONS) or {}, command_line_substitutions
-        )
        result.add_output_path([CONF_SUBSTITUTIONS], CONF_SUBSTITUTIONS)
-        try:
-            substitutions.do_substitution_pass(config, command_line_substitutions)
-        except vol.Invalid as err:
-            result.add_error(err)
-            return result
+    try:
+        config = do_substitution_pass(config, command_line_substitutions)
+    except vol.Invalid as err:
+        CORE.raw_config = config
+        result.add_error(err)
+        return result

    # 1.1. Merge packages
    if CONF_PACKAGES in config:
@@ -1016,6 +1011,9 @@ def validate_config(

        config = merge_packages(config)

+    # Remove substitutions from config during validation to prevent
+    # re-substitution. Re-added to result at the end of this function.
+    substitutions = config.pop(CONF_SUBSTITUTIONS, None)
    CORE.raw_config = config

    # 1.2. Resolve !extend and !remove and check for REPLACEME
@@ -1089,6 +1087,10 @@ def validate_config(

    result.run_validation_steps()

+    if substitutions is not None:
+        result[CONF_SUBSTITUTIONS] = substitutions
+        result.move_to_end(CONF_SUBSTITUTIONS, last=False)
+
    return result


@@ -9,6 +9,8 @@
 #endif
 #ifdef USE_ESP32
 #include <esp_chip_info.h>
+#include <esp_ota_ops.h>
+#include <esp_bootloader_desc.h>
 #endif
 #ifdef USE_LWIP_FAST_SELECT
 #include "esphome/core/lwip_fast_select.h"
@@ -167,19 +169,49 @@ void Application::process_dump_config_() {
    esp_chip_info(&chip_info);
    ESP_LOGI(TAG, "ESP32 Chip: %s rev%d.%d, %d core(s)", ESPHOME_VARIANT, chip_info.revision / 100,
             chip_info.revision % 100, chip_info.cores);
-#if defined(USE_ESP32_VARIANT_ESP32) && !defined(USE_ESP32_MIN_CHIP_REVISION_SET)
-    // Suggest optimization for chips that don't need the PSRAM cache workaround
-    if (chip_info.revision >= 300) {
-#ifdef USE_PSRAM
-      ESP_LOGW(TAG, "Set minimum_chip_revision: \"%d.%d\" to save ~10KB IRAM", chip_info.revision / 100,
-               chip_info.revision % 100);
-#else
-      ESP_LOGW(TAG, "Set minimum_chip_revision: \"%d.%d\" to reduce binary size", chip_info.revision / 100,
-               chip_info.revision % 100);
+#if defined(USE_ESP32_VARIANT_ESP32) && (!defined(USE_ESP32_MIN_CHIP_REVISION_SET) || !defined(USE_ESP32_SRAM1_AS_IRAM))
+    static const char *const ESP32_ADVANCED_PATH = "under esp32 > framework > advanced";
 #endif
+#if defined(USE_ESP32_VARIANT_ESP32) && !defined(USE_ESP32_MIN_CHIP_REVISION_SET)
+    {
+      // Suggest optimization for chips that don't need the PSRAM cache workaround
+      if (chip_info.revision >= 300) {
+#ifdef USE_PSRAM
+        ESP_LOGW(TAG, "Chip rev >= 3.0 detected. Set minimum_chip_revision: \"%d.%d\" %s to save ~10KB IRAM",
+                 chip_info.revision / 100, chip_info.revision % 100, ESP32_ADVANCED_PATH);
+#else
+        ESP_LOGW(TAG, "Chip rev >= 3.0 detected. Set minimum_chip_revision: \"%d.%d\" %s to reduce binary size",
+                 chip_info.revision / 100, chip_info.revision % 100, ESP32_ADVANCED_PATH);
+#endif
+      }
    }
 #endif
+    {
+      // esp_bootloader_desc_t is available in ESP-IDF >= 5.2; if readable the bootloader is modern.
+      //
+      // Design decision: We intentionally do NOT mention sram1_as_iram when the bootloader is too old.
+      // Enabling sram1_as_iram with an old bootloader causes a hard brick (device fails to boot,
+      // requires USB reflash to recover). Users don't always read warnings carefully, so we only
+      // suggest the option once we've confirmed the bootloader can handle it. In practice this
+      // means a user with an old bootloader may need to flash twice: once via USB to update the
+      // bootloader (they'll see the suggestion on next boot), then OTA with sram1_as_iram: true.
+      // Two flashes is a better outcome than a bricked device.
+      esp_bootloader_desc_t boot_desc;
+      if (esp_ota_get_bootloader_description(nullptr, &boot_desc) != ESP_OK) {
+#ifdef USE_ESP32_VARIANT_ESP32
+        ESP_LOGW(TAG, "Bootloader too old for OTA rollback and SRAM1 as IRAM (+40KB). "
+                      "Flash via USB once to update the bootloader");
+#else
+        ESP_LOGW(TAG, "Bootloader too old for OTA rollback. Flash via USB once to update the bootloader");
 #endif
+      }
+#if defined(USE_ESP32_VARIANT_ESP32) && !defined(USE_ESP32_SRAM1_AS_IRAM)
+      else {
+        ESP_LOGW(TAG, "Bootloader supports SRAM1 as IRAM (+40KB). Set sram1_as_iram: true %s", ESP32_ADVANCED_PATH);
+      }
+#endif
+    }
+#endif  // USE_ESP32
  }

  this->components_[this->dump_config_at_]->call_dump_config_();
@@ -202,6 +202,7 @@
 #define USE_ESPHOME_TASK_LOG_BUFFER
 #define USE_OTA_ROLLBACK
 #define USE_ESP32_MIN_CHIP_REVISION_SET
+#define USE_ESP32_SRAM1_AS_IRAM

 #define USE_BLUETOOTH_PROXY
 #define BLUETOOTH_PROXY_MAX_CONNECTIONS 3
@@ -325,9 +325,7 @@ class ESPHomeLoaderMixin:
        return val

    @_add_data_ref
-    def construct_include(
-        self, node: yaml.Node
-    ) -> dict[str, Any] | OrderedDict[str, Any]:
+    def construct_include(self, node: yaml.Node) -> Any:
        from esphome.const import CONF_VARS

        def extract_file_vars(node):
@@ -344,9 +342,7 @@ class ESPHomeLoaderMixin:
            file, vars = node.value, None

        result = self.yaml_loader(self._rel_path(file))
-        if not vars:
-            vars = {}
-        return substitute_vars(result, vars)
+        return add_context(result, vars)

    @_add_data_ref
    def construct_include_dir_list(self, node: yaml.Node) -> list[dict[str, Any]]:
@@ -495,39 +491,6 @@ def parse_yaml(
        )


-def substitute_vars(config, vars):
-    from esphome.components import substitutions
-    from esphome.const import CONF_SUBSTITUTIONS
-
-    org_subs = None
-    result = config
-    if not isinstance(config, dict):
-        # when the included yaml contains a list or a scalar
-        # wrap it into an OrderedDict because do_substitution_pass expects it
-        result = OrderedDict([("yaml", config)])
-    elif CONF_SUBSTITUTIONS in result:
-        org_subs = result.pop(CONF_SUBSTITUTIONS)
-
-    defaults = {}
-    if CONF_DEFAULTS in result:
-        defaults = result.pop(CONF_DEFAULTS)
-
-    result[CONF_SUBSTITUTIONS] = vars
-    for k, v in defaults.items():
-        if k not in result[CONF_SUBSTITUTIONS]:
-            result[CONF_SUBSTITUTIONS][k] = v
-
-    # Ignore missing vars that refer to the top level substitutions
-    substitutions.do_substitution_pass(result, None, ignore_missing=True)
-    result.pop(CONF_SUBSTITUTIONS)
-
-    if not isinstance(config, dict):
-        result = result["yaml"]  # unwrap the result
-    elif org_subs:
-        result[CONF_SUBSTITUTIONS] = org_subs
-    return result
-
-
 def _load_yaml_internal_with_type(
    loader_type: type[ESPHomeLoader] | type[ESPHomePurePythonLoader],
    fname: Path,
@@ -221,8 +221,58 @@ class TypeInfo(ABC):

    decode_64bit = None

+    # Mapping from encode_func to raw encode expression template.
+    # When a forced field has a single-byte tag, the code generator emits
+    # write_raw_byte(tag) + raw encode instead of the full encode_* method,
+    # eliminating the zero-check branch and encode_field_raw indirection.
+    # {value} is replaced with the actual field expression.
+    RAW_ENCODE_MAP: dict[str, str] = {
+        "encode_uint32": "buffer.encode_varint_raw({value});",
+        "encode_uint64": "buffer.encode_varint_raw_64({value});",
+        "encode_sint32": "buffer.encode_varint_raw(encode_zigzag32({value}));",
+        "encode_sint64": "buffer.encode_varint_raw_64(encode_zigzag64({value}));",
+        "encode_int64": "buffer.encode_varint_raw_64(static_cast<uint64_t>({value}));",
+        "encode_bool": "buffer.write_raw_byte({value} ? 0x01 : 0x00);",
+    }
+
+    def _encode_with_precomputed_tag(self, value_expr: str) -> str | None:
+        """Try to emit a precomputed-tag encode for a forced field.
+
+        Returns the raw encode string if the tag is a single byte and the
+        encode_func has a known raw equivalent, or None otherwise.
+        """
+        if not self.force:
+            return None
+        tag = self.calculate_tag()
+        if tag >= 128:
+            return None
+        raw_expr = self.RAW_ENCODE_MAP.get(self.encode_func)
+        if raw_expr is None:
+            return None
+        return f"buffer.write_raw_byte({tag});\n{raw_expr.format(value=value_expr)}"
+
+    def _encode_bytes_with_precomputed_tag(
+        self, data_expr: str, len_expr: str
+    ) -> str | None:
+        """Try to emit a precomputed-tag encode for a forced bytes/string field.
+
+        Returns the raw encode string if the tag is a single byte, or None.
+        """
+        if not self.force:
+            return None
+        tag = self.calculate_tag()
+        if tag >= 128:
+            return None
+        return (
+            f"buffer.write_raw_byte({tag});\n"
+            f"buffer.encode_varint_raw({len_expr});\n"
+            f"buffer.encode_raw({data_expr}, {len_expr});"
+        )
+
    @property
    def encode_content(self) -> str:
+        if result := self._encode_with_precomputed_tag(f"this->{self.field_name}"):
+            return result
        if self.force:
            return f"buffer.{self.encode_func}({self.number}, this->{self.field_name}, true);"
        return f"buffer.{self.encode_func}({self.number}, this->{self.field_name});"
@@ -248,7 +298,7 @@ class TypeInfo(ABC):
    @property
    def dump_content(self) -> str:
        # Default implementation - subclasses can override if they need special handling
-        return f'dump_field(out, "{self.name}", {self.dump_field_value(f"this->{self.field_name}")});'
+        return f'dump_field(out, ESPHOME_PSTR("{self.name}"), {self.dump_field_value(f"this->{self.field_name}")});'

    @abstractmethod
    def dump(self, name: str) -> str:
@@ -635,6 +685,11 @@ class StringType(TypeInfo):
    @property
    def encode_content(self) -> str:
        # Use the StringRef
+        if result := self._encode_bytes_with_precomputed_tag(
+            f"this->{self.field_name}_ref_.c_str()",
+            f"this->{self.field_name}_ref_.size()",
+        ):
+            return result
        if self.force:
            return f"buffer.encode_string({self.number}, this->{self.field_name}_ref_, true);"
        return f"buffer.encode_string({self.number}, this->{self.field_name}_ref_);"
@@ -665,14 +720,14 @@ class StringType(TypeInfo):
    def dump_content(self) -> str:
        # For SOURCE_CLIENT only, use std::string
        if not self._needs_encode:
-            return f'dump_field(out, "{self.name}", this->{self.field_name});'
+            return f'dump_field(out, ESPHOME_PSTR("{self.name}"), this->{self.field_name});'

        # For SOURCE_SERVER, use StringRef with _ref_ suffix
        if not self._needs_decode:
-            return f'dump_field(out, "{self.name}", this->{self.field_name}_ref_);'
+            return f'dump_field(out, ESPHOME_PSTR("{self.name}"), this->{self.field_name}_ref_);'

        # For SOURCE_BOTH, we need custom logic
-        o = f'out.append("  {self.name}: ");\n'
+        o = f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n'
        o += self.dump(f"this->{self.field_name}") + "\n"
        o += 'out.append("\\n");'
        return o
@@ -745,7 +800,7 @@ class MessageType(TypeInfo):

    @property
    def dump_content(self) -> str:
-        o = f'out.append("  {self.name}: ");\n'
+        o = f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n'
        o += f"this->{self.field_name}.dump_to(out);\n"
        o += 'out.append("\\n");'
        return o
@@ -801,6 +856,10 @@ class BytesType(TypeInfo):

    @property
    def encode_content(self) -> str:
+        if result := self._encode_bytes_with_precomputed_tag(
+            f"this->{self.field_name}_ptr_", f"this->{self.field_name}_len_"
+        ):
+            return result
        if self.force:
            return f"buffer.encode_bytes({self.number}, this->{self.field_name}_ptr_, this->{self.field_name}_len_, true);"
        return f"buffer.encode_bytes({self.number}, this->{self.field_name}_ptr_, this->{self.field_name}_len_);"
@@ -831,7 +890,7 @@ class BytesType(TypeInfo):
        # For SOURCE_CLIENT only, always use std::string
        if not self._needs_encode:
            return (
-                f'dump_bytes_field(out, "{self.name}", '
+                f'dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
                f"reinterpret_cast<const uint8_t*>(this->{self.field_name}.data()), "
                f"this->{self.field_name}.size());"
            )
@@ -839,17 +898,17 @@ class BytesType(TypeInfo):
        # For SOURCE_SERVER, always use pointer/length
        if not self._needs_decode:
            return (
-                f'dump_bytes_field(out, "{self.name}", '
+                f'dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
                f"this->{self.field_name}_ptr_, this->{self.field_name}_len_);"
            )

        # For SOURCE_BOTH, check if pointer is set (sending) or use string (received)
        return (
            f"if (this->{self.field_name}_ptr_ != nullptr) {{\n"
-            f'  dump_bytes_field(out, "{self.name}", '
+            f'  dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
            f"this->{self.field_name}_ptr_, this->{self.field_name}_len_);\n"
            f"}} else {{\n"
-            f'  dump_bytes_field(out, "{self.name}", '
+            f'  dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
            f"reinterpret_cast<const uint8_t*>(this->{self.field_name}.data()), "
            f"this->{self.field_name}.size());\n"
            f"}}"
@@ -908,6 +967,10 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):

    @property
    def encode_content(self) -> str:
+        if result := self._encode_bytes_with_precomputed_tag(
+            f"this->{self.field_name}", f"this->{self.field_name}_len"
+        ):
+            return result
        if self.force:
            return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len, true);"
        return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);"
@@ -928,7 +991,7 @@ class PointerToBytesBufferType(PointerToBufferTypeBase):
    @property
    def dump_content(self) -> str:
        return (
-            f'dump_bytes_field(out, "{self.name}", '
+            f'dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
            f"this->{self.field_name}, this->{self.field_name}_len);"
        )

@@ -957,6 +1020,10 @@ class PointerToStringBufferType(PointerToBufferTypeBase):

    @property
    def encode_content(self) -> str:
+        if result := self._encode_bytes_with_precomputed_tag(
+            f"this->{self.field_name}.c_str()", f"this->{self.field_name}.size()"
+        ):
+            return result
        if self.force:
            return (
                f"buffer.encode_string({self.number}, this->{self.field_name}, true);"
@@ -976,7 +1043,7 @@ class PointerToStringBufferType(PointerToBufferTypeBase):

    @property
    def dump_content(self) -> str:
-        return f'dump_field(out, "{self.name}", this->{self.field_name});'
+        return f'dump_field(out, ESPHOME_PSTR("{self.name}"), this->{self.field_name});'

    def get_size_calculation(self, name: str, force: bool = False) -> str:
        return f"size += ProtoSize::calc_length({self.calculate_field_id_size()}, this->{self.field_name}.size());"
@@ -1036,12 +1103,12 @@ class PackedBufferTypeInfo(TypeInfo):
    def dump_content(self) -> str:
        """Dump shows buffer info but not decoded values."""
        return (
-            f'out.append("  {self.name}: ");\n'
-            + 'out.append("packed buffer [");\n'
+            f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n'
+            + 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n'
            + f"append_uint(out, this->{self.field_name}_count_);\n"
-            + 'out.append(" values, ");\n'
+            + 'out.append_p(ESPHOME_PSTR(" values, "));\n'
            + f"append_uint(out, this->{self.field_name}_length_);\n"
-            + 'out.append(" bytes]\\n");'
+            + 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));'
        )

    def dump(self, name: str) -> str:
@@ -1124,6 +1191,10 @@ class FixedArrayBytesType(TypeInfo):

    @property
    def encode_content(self) -> str:
+        if result := self._encode_bytes_with_precomputed_tag(
+            f"this->{self.field_name}", f"this->{self.field_name}_len"
+        ):
+            return result
        if self.force:
            return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len, true);"
        return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);"
@@ -1134,7 +1205,7 @@ class FixedArrayBytesType(TypeInfo):
    @property
    def dump_content(self) -> str:
        return (
-            f'dump_bytes_field(out, "{self.name}", '
+            f'dump_bytes_field(out, ESPHOME_PSTR("{self.name}"), '
            f"this->{self.field_name}, this->{self.field_name}_len);"
        )

@@ -1199,12 +1270,16 @@ class EnumType(TypeInfo):

    @property
    def encode_content(self) -> str:
+        if result := self._encode_with_precomputed_tag(
+            f"static_cast<uint32_t>(this->{self.field_name})"
+        ):
+            return result
        if self.force:
            return f"buffer.{self.encode_func}({self.number}, static_cast<uint32_t>(this->{self.field_name}), true);"
        return f"buffer.{self.encode_func}({self.number}, static_cast<uint32_t>(this->{self.field_name}));"

    def dump(self, name: str) -> str:
-        return f"out.append(proto_enum_to_string<{self.cpp_type}>({name}));"
+        return f"out.append_p(proto_enum_to_string<{self.cpp_type}>({name}));"

    def dump_field_value(self, value: str) -> str:
        # Enums need explicit cast for the template
@@ -1326,15 +1401,15 @@ def _generate_array_dump_content(
    # Check if underlying type can use dump_field
    if is_const_char_ptr:
        # Special case for const char* - use it directly
-        o += f'  dump_field(out, "{name}", it, 4);\n'
+        o += f'  dump_field(out, ESPHOME_PSTR("{name}"), it, 4);\n'
    elif ti.can_use_dump_field():
        # For types that have dump_field overloads, use them with extra indent
        # std::vector<bool> iterators return proxy objects, need explicit cast
        value_expr = "static_cast<bool>(it)" if is_bool else ti.dump_field_value("it")
-        o += f'  dump_field(out, "{name}", {value_expr}, 4);\n'
+        o += f'  dump_field(out, ESPHOME_PSTR("{name}"), {value_expr}, 4);\n'
    else:
        # For complex types (messages, bytes), use the old pattern
-        o += f'  out.append("  {name}: ");\n'
+        o += f'  out.append(4, \' \').append_p(ESPHOME_PSTR("{name}")).append(": ");\n'
        o += indent(ti.dump("it")) + "\n"
        o += '  out.append("\\n");\n'
    o += "}"
@@ -1543,9 +1618,9 @@ class FixedArrayWithLengthRepeatedType(FixedArrayRepeatedType):
        o = f"for (uint16_t i = 0; i < this->{self.field_name}_len; i++) {{\n"
        # Check if underlying type can use dump_field
        if self._ti.can_use_dump_field():
-            o += f'  dump_field(out, "{self.name}", {self._ti.dump_field_value(f"this->{self.field_name}[i]")}, 4);\n'
+            o += f'  dump_field(out, ESPHOME_PSTR("{self.name}"), {self._ti.dump_field_value(f"this->{self.field_name}[i]")}, 4);\n'
        else:
-            o += f'  out.append("  {self.name}: ");\n'
+            o += f'  out.append(4, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n'
            o += indent(self._ti.dump(f"this->{self.field_name}[i]")) + "\n"
            o += '  out.append("\\n");\n'
        o += "}"
@@ -2023,9 +2098,9 @@ def build_enum_type(desc, enum_ifdef_map) -> tuple[str, str, str]:
    dump_cpp += "  switch (value) {\n"
    for v in desc.value:
        dump_cpp += f"    case enums::{v.name}:\n"
-        dump_cpp += f'      return "{v.name}";\n'
+        dump_cpp += f'      return ESPHOME_PSTR("{v.name}");\n'
    dump_cpp += "    default:\n"
-    dump_cpp += '      return "UNKNOWN";\n'
+    dump_cpp += '      return ESPHOME_PSTR("UNKNOWN");\n'
    dump_cpp += "  }\n"
    dump_cpp += "}\n"

@@ -2107,7 +2182,7 @@ def build_message_type(
        public_content.append("#ifdef HAS_PROTO_MESSAGE_DUMP")
        snake_name = camel_to_snake(desc.name)
        public_content.append(
-            f'const char *message_name() const override {{ return "{snake_name}"; }}'
+            f'const LogString *message_name() const override {{ return LOG_STR("{snake_name}"); }}'
        )
        public_content.append("#endif")

@@ -2315,12 +2390,12 @@ def build_message_type(
    if dump:
        # Always use MessageDumpHelper for consistent output formatting
        dump_impl += "\n"
-        dump_impl += f'  MessageDumpHelper helper(out, "{desc.name}");\n'
+        dump_impl += f'  MessageDumpHelper helper(out, ESPHOME_PSTR("{desc.name}"));\n'
        dump_impl += indent("\n".join(dump)) + "\n"
        dump_impl += "  return out.c_str();\n"
    else:
        dump_impl += "\n"
-        dump_impl += f'  out.append("{desc.name} {{}}");\n'
+        dump_impl += f'  out.append_p(ESPHOME_PSTR("{desc.name} {{}}"));\n'
        dump_impl += "  return out.c_str();\n"
    dump_impl += "}\n"

@@ -2707,6 +2782,7 @@ namespace esphome::api {
    dump_cpp += """\
 #include "api_pb2.h"
 #include "esphome/core/helpers.h"
+#include "esphome/core/progmem.h"

 #include <cinttypes>

@@ -2714,6 +2790,21 @@ namespace esphome::api {

 namespace esphome::api {

+#ifdef USE_ESP8266
+// Out-of-line to avoid inlining strlen_P/memcpy_P at every call site
+void DumpBuffer::append_p_esp8266(const char *str) {
+  size_t len = strlen_P(str);
+  size_t space = CAPACITY - 1 - pos_;
+  if (len > space)
+    len = space;
+  if (len > 0) {
+    memcpy_P(buf_ + pos_, str, len);
+    pos_ += len;
+    buf_[pos_] = '\\0';
+  }
+}
+#endif
+
 // Helper function to append a quoted string, handling empty StringRef
 static inline void append_quoted_string(DumpBuffer &out, const StringRef &ref) {
  out.append("'");
@@ -2724,8 +2815,9 @@ static inline void append_quoted_string(DumpBuffer &out, const StringRef &ref) {
 }

 // Common helpers for dump_field functions
+// field_name is a PROGMEM pointer (flash on ESP8266, regular pointer on other platforms)
 static inline void append_field_prefix(DumpBuffer &out, const char *field_name, int indent) {
-  out.append(indent, ' ').append(field_name).append(": ");
+  out.append(indent, ' ').append_p(field_name).append(": ");
 }

 static inline void append_uint(DumpBuffer &out, uint32_t value) {
@@ -2733,10 +2825,11 @@ static inline void append_uint(DumpBuffer &out, uint32_t value) {
 }

 // RAII helper for message dump formatting
+// message_name is a PROGMEM pointer (flash on ESP8266, regular pointer on other platforms)
 class MessageDumpHelper {
 public:
  MessageDumpHelper(DumpBuffer &out, const char *message_name) : out_(out) {
-    out_.append(message_name);
+    out_.append_p(message_name);
    out_.append(" {\\n");
  }
  ~MessageDumpHelper() { out_.append(" }"); }
@@ -2746,6 +2839,10 @@ class MessageDumpHelper {
 };

 // Helper functions to reduce code duplication in dump methods
+// field_name parameters are PROGMEM pointers (flash on ESP8266, regular pointers on other platforms)
+// Not all overloads are used in every build (depends on enabled components)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
 static void dump_field(DumpBuffer &out, const char *field_name, int32_t value, int indent = 2) {
  append_field_prefix(out, field_name, indent);
  out.set_pos(buf_append_printf(out.data(), DumpBuffer::CAPACITY, out.pos(), "%" PRId32 "\\n", value));
@@ -2790,21 +2887,23 @@ static void dump_field(DumpBuffer &out, const char *field_name, const char *valu
  out.append("\\n");
 }

-template<typename T>
-static void dump_field(DumpBuffer &out, const char *field_name, T value, int indent = 2) {
+// proto_enum_to_string returns PROGMEM pointers, so use append_p
+template<typename T> static void dump_field(DumpBuffer &out, const char *field_name, T value, int indent = 2) {
  append_field_prefix(out, field_name, indent);
-  out.append(proto_enum_to_string<T>(value));
+  out.append_p(proto_enum_to_string<T>(value));
  out.append("\\n");
 }

 // Helper for bytes fields - uses stack buffer to avoid heap allocation
 // Buffer sized for 160 bytes of data (480 chars with separators) to fit typical log buffer
+// field_name is a PROGMEM pointer (flash on ESP8266, regular pointer on other platforms)
 static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint8_t *data, size_t len, int indent = 2) {
  char hex_buf[format_hex_pretty_size(160)];
  append_field_prefix(out, field_name, indent);
  format_hex_pretty_to(hex_buf, data, len);
  out.append(hex_buf).append("\\n");
 }
+#pragma GCC diagnostic pop

 """

@@ -2977,7 +3076,7 @@ static const char *const TAG = "api.service";
    # Add logging helper method declarations
    hpp += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
    hpp += " protected:\n"
-    hpp += "  void log_send_message_(const char *name, const char *dump);\n"
+    hpp += "  void log_send_message_(const LogString *name, const char *dump);\n"
    hpp += (
        "  void log_receive_message_(const LogString *name, const ProtoMessage &msg);\n"
    )
@@ -2990,10 +3089,8 @@ static const char *const TAG = "api.service";

    # Add logging helper method implementations to cpp
    cpp += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
-    cpp += (
-        f"void {class_name}::log_send_message_(const char *name, const char *dump) {{\n"
-    )
-    cpp += '  ESP_LOGVV(TAG, "send_message %s: %s", name, dump);\n'
+    cpp += f"void {class_name}::log_send_message_(const LogString *name, const char *dump) {{\n"
+    cpp += '  ESP_LOGVV(TAG, "send_message %s: %s", LOG_STR_ARG(name), dump);\n'
    cpp += "}\n"
    cpp += f"void {class_name}::log_receive_message_(const LogString *name, const ProtoMessage &msg) {{\n"
    cpp += "  DumpBuffer dump_buf;\n"
@@ -172,6 +172,135 @@ BENCHMARK(NoiseDecrypt_MediumMessage);
 static void NoiseDecrypt_LargeMessage(benchmark::State &state) { noise_decrypt_bench(state, 1024); }
 BENCHMARK(NoiseDecrypt_LargeMessage);

+// --- Full Noise_NNpsk0 handshake benchmark ---
+// Measures the complete handshake between initiator and responder:
+//   - Create handshake states for both sides
+//   - Set PSK and prologue
+//   - Exchange messages (initiator write -> responder read -> responder write -> initiator read)
+//   - Split to get cipher states
+// This is dominated by Curve25519 DH operations (expensive on ESP8266).
+// No inner iterations — each handshake is already expensive enough.
+
+static void NoiseHandshake_Full(benchmark::State &state) {
+  // Matching ESPHome's protocol: Noise_NNpsk0_25519_ChaChaPoly_SHA256
+  NoiseProtocolId nid;
+  memset(&nid, 0, sizeof(nid));
+  nid.pattern_id = NOISE_PATTERN_NN;
+  nid.cipher_id = NOISE_CIPHER_CHACHAPOLY;
+  nid.dh_id = NOISE_DH_CURVE25519;
+  nid.prefix_id = NOISE_PREFIX_STANDARD;
+  nid.hybrid_id = NOISE_DH_NONE;
+  nid.hash_id = NOISE_HASH_SHA256;
+  nid.modifier_ids[0] = NOISE_MODIFIER_PSK0;
+
+  // Dummy PSK (32 bytes) and prologue matching production setup
+  static constexpr uint8_t PSK[32] = {0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB,
+                                      0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB,
+                                      0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB};
+  static constexpr uint8_t PROLOGUE[] = "NoESPHome";
+
+  // Message buffer for handshake exchange (max handshake message ~96 bytes)
+  uint8_t msg_buf[128];
+
+  for (auto _ : state) {
+    NoiseHandshakeState *initiator = nullptr;
+    NoiseHandshakeState *responder = nullptr;
+    NoiseCipherState *init_send = nullptr, *init_recv = nullptr;
+    NoiseCipherState *resp_send = nullptr, *resp_recv = nullptr;
+    int err;
+
+    // Create both handshake states
+    err = noise_handshakestate_new_by_id(&initiator, &nid, NOISE_ROLE_INITIATOR);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Failed to create initiator");
+      return;
+    }
+    err = noise_handshakestate_new_by_id(&responder, &nid, NOISE_ROLE_RESPONDER);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Failed to create responder");
+      noise_handshakestate_free(initiator);
+      return;
+    }
+
+    // Set PSK and prologue on both sides
+    noise_handshakestate_set_pre_shared_key(initiator, PSK, sizeof(PSK));
+    noise_handshakestate_set_pre_shared_key(responder, PSK, sizeof(PSK));
+    noise_handshakestate_set_prologue(initiator, PROLOGUE, sizeof(PROLOGUE) - 1);
+    noise_handshakestate_set_prologue(responder, PROLOGUE, sizeof(PROLOGUE) - 1);
+
+    noise_handshakestate_start(initiator);
+    noise_handshakestate_start(responder);
+
+    // Message 1: Initiator -> Responder
+    NoiseBuffer write_buf, read_buf;
+    noise_buffer_set_output(write_buf, msg_buf, sizeof(msg_buf));
+    err = noise_handshakestate_write_message(initiator, &write_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Initiator write_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    noise_buffer_set_input(read_buf, msg_buf, write_buf.size);
+    err = noise_handshakestate_read_message(responder, &read_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Responder read_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    // Message 2: Responder -> Initiator
+    noise_buffer_set_output(write_buf, msg_buf, sizeof(msg_buf));
+    err = noise_handshakestate_write_message(responder, &write_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Responder write_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    noise_buffer_set_input(read_buf, msg_buf, write_buf.size);
+    err = noise_handshakestate_read_message(initiator, &read_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Initiator read_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    // Split to get cipher states
+    err = noise_handshakestate_split(initiator, &init_send, &init_recv);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Initiator split failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+    err = noise_handshakestate_split(responder, &resp_send, &resp_recv);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Responder split failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      noise_cipherstate_free(init_send);
+      noise_cipherstate_free(init_recv);
+      return;
+    }
+
+    benchmark::DoNotOptimize(init_send);
+
+    // Cleanup
+    noise_handshakestate_free(initiator);
+    noise_handshakestate_free(responder);
+    noise_cipherstate_free(init_send);
+    noise_cipherstate_free(init_recv);
+    noise_cipherstate_free(resp_send);
+    noise_cipherstate_free(resp_recv);
+  }
+}
+BENCHMARK(NoiseHandshake_Full);
+
 }  // namespace esphome::api::benchmarks

 #endif  // USE_API_NOISE
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch
 import pytest

 from esphome.components.packages import CONFIG_SCHEMA, do_packages_pass, merge_packages
+from esphome.components.substitutions import do_substitution_pass
 import esphome.config as config_module
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, Remove
@@ -71,6 +72,7 @@ def fixture_basic_esphome():
 def packages_pass(config):
    """Wrapper around packages_pass that also resolves Extend and Remove."""
    config = do_packages_pass(config)
+    config = do_substitution_pass(config)
    config = merge_packages(config)
    resolve_extend_remove(config)
    return config
@@ -19,6 +19,7 @@ esp32:
      disable_mbedtls_pkcs7: true
      disable_regi2c_in_iram: true
      disable_fatfs: true
+      sram1_as_iram: true

 wifi:
  ssid: MySSID
@@ -0,0 +1,13 @@
+number:
+  - platform: template
+    name: "Test Number"
+    id: test_number
+    optimistic: true
+    min_value: 0
+    max_value: 100
+    step: 1
+
+sensor:
+  - platform: number
+    name: "Test Number Value"
+    source_id: test_number
@@ -0,0 +1,2 @@
+packages:
+  common: !include common.yaml
@@ -0,0 +1,2 @@
+packages:
+  common: !include common.yaml
@@ -38,3 +38,20 @@ test_list:
  - '{ 79, 82 }'
  - a: 15 should be 15, overridden from command line
    b: 20 should stay as 20, not overridden
+  - aa:
+      - 1
+      - 2
+      - 3
+      - 4
+      - 5
+      - 6
+    bb:
+      - 7
+      - 8
+      - 9
+  - aa:
+      x: 1
+      y: 3
+      z: 4
+    bb:
+      w: 5
@@ -44,3 +44,13 @@ test_list:
  - '{ ${position.x}, ${position.y} }'
  - a: ${a} should be 15, overridden from command line
    b: ${b} should stay as 20, not overridden
+
+  # Test merging lists when substituted keys resolve to an existing key
+  - ${ "aa" }: [1, 2, 3]
+    ${ "a" + "a" }: [4, 5, 6]
+    ${ "bb" }: [7, 8, 9]
+
+  # Test merging dicts when substituted keys resolve to an existing key
+  - ${ "aa" }: {"x": 1, "y": 2}
+    ${ "a" + "a" }: {"y": 3, "z": 4}
+    ${ "bb" }: {"w": 5}
@@ -9,6 +9,11 @@ substitutions:
  numberOne: 1
  var1: 79
  double_width: 14
+  double_height: 16
+  y: ${x}
+  x: ${y}
+  b: 79
+  c: 80
 test_list:
  - The area is 56
  - 56
@@ -27,3 +32,4 @@ test_list:
  - chr(97) = a
  - len([1,2,3]) = 3
  - width = 7, double_width = 14
+  - a = ${a}
@@ -1,4 +1,7 @@
 substitutions:
+  y: ${x}  # Circular reference, expect to pass unresolved.
+  x: ${y}  # Circular reference, expect to pass unresolved.
+  double_height: ${height * 2}
  width: 7
  height: 8
  enabled: true
@@ -9,6 +12,8 @@ substitutions:
  numberOne: 1
  var1: 79
  double_width: ${width * 2}
+  c: ${b+1}
+  b: ${undefined_variable | default(79) }

 test_list:
  - "The area is ${width * height}"
@@ -25,3 +30,4 @@ test_list:
  - chr(97) = ${ chr(97) }
  - len([1,2,3]) = ${ len([1,2,3]) }
  - width = ${width}, double_width = ${double_width}
+  - a = ${a}
@@ -0,0 +1,46 @@
+fancy_component: &id001
+  - id: component9
+    value: 9
+some_component:
+  - id: component1
+    value: 1
+  - id: component2
+    value: 2
+  - id: component3
+    value: 3
+  - id: component4
+    value: 4
+  - id: component5
+    value: 79
+    power: 200
+  - id: component6
+    value: 6
+  - id: component7
+    value: 7
+switch: &id002
+  - platform: gpio
+    id: switch1
+    pin: 12
+  - platform: gpio
+    id: switch2
+    pin: 13
+display:
+  - platform: ili9xxx
+    dimensions:
+      width: 100
+      height: 480
+substitutions:
+  extended_component: component5
+  package_options:
+    alternative_package:
+      alternative_component:
+        - id: component8
+          value: 8
+    fancy_package:
+      substitutions:
+        fancy_subst: 42
+      fancy_component: *id001
+  pin: 12
+  some_switches: *id002
+  package_selection: fancy_package
+  fancy_subst: 42
@@ -0,0 +1,63 @@
+substitutions:
+  package_options:
+    alternative_package:
+      alternative_component:
+        - id: component8
+          value: 8
+    fancy_package:
+      substitutions:
+        fancy_subst: 42
+      fancy_component:
+        - id: component9
+          value: 9
+
+  pin: 12
+  some_switches:
+    - platform: gpio
+      id: switch1
+      pin: ${pin}
+    - platform: gpio
+      id: switch2
+      pin: ${pin+1}
+
+  package_selection: fancy_package
+
+packages:
+  - ${ package_options[package_selection] }
+  - some_component:
+      - id: component1
+        value: 1
+  - some_component:
+      - id: component2
+        value: 2
+  - switch: ${ some_switches }
+  - packages:
+      package_with_defaults: !include
+        file: display.yaml
+        vars:
+          native_width: 100
+          high_dpi: false
+      my_package:
+        packages:
+          - packages:
+              special_package:
+                substitutions:
+                  extended_component: component5
+                some_component:
+                  - id: component3
+                    value: 3
+        some_component:
+          - id: component4
+            value: 4
+          - id: !extend ${ extended_component }
+            power: 200
+            value: 79
+    some_component:
+      - id: component5
+        value: 5
+
+some_component:
+  - id: component6
+    value: 6
+  - id: component7
+    value: 7
@@ -0,0 +1,5 @@
+values:
+  - var1: $var1
+  - a: 10
+  - b: B-default
+  - c: The value of C is 79
@@ -0,0 +1,7 @@
+# Test that include_vars with vars works even when there are no substitutions key defined.
+packages:
+  - !include
+    file: inc1.yaml
+    vars:
+      a: 10
+      c: 79
@@ -10,9 +10,10 @@ from esphome import config as config_module, yaml_util
 from esphome.components import substitutions
 from esphome.components.packages import do_packages_pass, merge_packages
 from esphome.config import resolve_extend_remove
-from esphome.config_helpers import merge_config
+from esphome.config_helpers import Extend, merge_config
+import esphome.config_validation as cv
 from esphome.const import CONF_SUBSTITUTIONS
-from esphome.core import CORE
+from esphome.core import CORE, Lambda
 from esphome.util import OrderedDict

 _LOGGER = logging.getLogger(__name__)
@@ -144,7 +145,7 @@ def test_substitutions_fixtures(

    config = do_packages_pass(config)

-    substitutions.do_substitution_pass(config, command_line_substitutions)
+    config = substitutions.do_substitution_pass(config, command_line_substitutions)

    config = merge_packages(config)

@@ -206,7 +207,7 @@ def test_substitutions_with_command_line_maintains_ordered_dict() -> None:
    command_line_subs = {"var2": "override", "var3": "new_value"}

    # Call do_substitution_pass with command line substitutions
-    substitutions.do_substitution_pass(config, command_line_subs)
+    config = substitutions.do_substitution_pass(config, command_line_subs)

    # Verify that config is still an OrderedDict
    assert isinstance(config, OrderedDict), "Config should remain an OrderedDict"
@@ -234,7 +235,7 @@ def test_substitutions_without_command_line_maintains_ordered_dict() -> None:
    config["other_key"] = "other_value"

    # Call without command line substitutions
-    substitutions.do_substitution_pass(config, None)
+    config = substitutions.do_substitution_pass(config, None)

    # Verify that config is still an OrderedDict
    assert isinstance(config, OrderedDict), "Config should remain an OrderedDict"
@@ -268,7 +269,7 @@ def test_substitutions_after_merge_config_maintains_ordered_dict() -> None:
    )

    # Now try to run substitution pass on the merged config
-    substitutions.do_substitution_pass(merged_config, None)
+    merged_config = substitutions.do_substitution_pass(merged_config, None)

    # Should not raise AttributeError
    assert isinstance(merged_config, OrderedDict), (
@@ -279,7 +280,7 @@ def test_substitutions_after_merge_config_maintains_ordered_dict() -> None:


 def test_validate_config_with_command_line_substitutions_maintains_ordered_dict(
-    tmp_path,
+    tmp_path: Path,
 ) -> None:
    """Test that validate_config preserves OrderedDict when merging command-line substitutions.

@@ -288,7 +289,7 @@ def test_validate_config_with_command_line_substitutions_maintains_ordered_dict(
    """
    # Create a minimal valid config
    test_config = OrderedDict()
-    test_config["esphome"] = {"name": "test_device", "platform": "ESP32"}
+    test_config["esphome"] = {"name": "test_device"}
    test_config[CONF_SUBSTITUTIONS] = OrderedDict({"var1": "value1", "var2": "value2"})
    test_config["esp32"] = {"board": "esp32dev"}

@@ -314,17 +315,11 @@ def test_validate_config_with_command_line_substitutions_maintains_ordered_dict(
    assert result[CONF_SUBSTITUTIONS]["var3"] == "new_value"


-def test_validate_config_without_command_line_substitutions_maintains_ordered_dict(
-    tmp_path,
-) -> None:
-    """Test that validate_config preserves OrderedDict without command-line substitutions.
-
-    This tests the code path in config.py where result[CONF_SUBSTITUTIONS] is set
-    using merge_dicts_ordered() when command_line_substitutions is None.
-    """
+def _get_test_minimal_valid_config(tmp_path: Path) -> OrderedDict:
+    """Helper to create a minimal valid config for testing."""
    # Create a minimal valid config
    test_config = OrderedDict()
-    test_config["esphome"] = {"name": "test_device", "platform": "ESP32"}
+    test_config["esphome"] = {"name": "test_device"}
    test_config[CONF_SUBSTITUTIONS] = OrderedDict({"var1": "value1", "var2": "value2"})
    test_config["esp32"] = {"board": "esp32dev"}

@@ -332,6 +327,19 @@ def test_validate_config_without_command_line_substitutions_maintains_ordered_di
    test_yaml = tmp_path / "test.yaml"
    test_yaml.write_text("# test config")
    CORE.config_path = test_yaml
+    return test_config
+
+
+def test_validate_config_without_command_line_substitutions_maintains_ordered_dict(
+    tmp_path: Path,
+) -> None:
+    """Test that validate_config preserves OrderedDict without command-line substitutions.
+
+    This tests the code path in config.py where result[CONF_SUBSTITUTIONS] is set
+    using merge_dicts_ordered() when command_line_substitutions is None.
+    """
+
+    test_config = _get_test_minimal_valid_config(tmp_path)

    # Call validate_config without command line substitutions
    result = config_module.validate_config(test_config, None)
@@ -384,3 +392,239 @@ def test_merge_config_preserves_ordered_dict() -> None:
    assert not isinstance(result, OrderedDict), (
        "dict + dict should not return OrderedDict"
    )
+
+
+def test_substitution_pass_error_gets_captured(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """vol.Invalid from do_substitution_pass is captured by validate_config."""
+
+    # Patch the target: in config_module.do_substitution_pass (NOT where it's defined)
+    def fake_do_substitution_pass(*args, **kwargs):
+        raise cv.Invalid("Error in do_substitutions_pass!!")
+
+    monkeypatch.setattr(
+        config_module, "do_substitution_pass", fake_do_substitution_pass
+    )
+
+    # Prepare minimal config + no CLI substitutions
+    config = _get_test_minimal_valid_config(tmp_path)
+
+    # Call the function under test
+    result = config_module.validate_config(config, None)
+
+    # Now assert that add_error was called with the vol.Invalid
+
+    assert "Error in do_substitutions_pass!!" in str(result.get_error_for_path([]))
+
+
+@pytest.mark.parametrize(
+    "value", ["", "   ", "1foo", "9VAR", "0abc", "$1foo", "$9VAR", "$0abc"]
+)
+def test_validate_substitution_key_empty_raises(value: str) -> None:
+    """Empty (or all-whitespace) substitution keys are rejected."""
+    with pytest.raises(cv.Invalid):
+        substitutions.validate_substitution_key(value)
+
+
+@pytest.mark.parametrize(
+    "input_value, expected_output",
+    [
+        ("$FOO_bar9", "FOO_bar9"),  # Valid key with leading '$'
+        ("Foo_bar9", "Foo_bar9"),  # Normal valid key
+    ],
+)
+def test_validate_substitution_key_valid(
+    input_value: str, expected_output: str
+) -> None:
+    """Valid substitution keys are accepted with optional leading '$'."""
+    result = substitutions.validate_substitution_key(input_value)
+    assert result == expected_output
+
+
+def test_circular_dependency_warnings(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Circular substitution references produce warnings naming the cause."""
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: OrderedDict({"x": "${y}", "y": "${x}"}),
+            "key": "value",
+        }
+    )
+    with caplog.at_level(logging.WARNING):
+        substitutions.do_substitution_pass(config)
+
+    assert "Could not resolve substitution variable 'x'" in caplog.text
+    assert "'y' is undefined" in caplog.text
+    assert "Could not resolve substitution variable 'y'" in caplog.text
+    assert "'x' is undefined" in caplog.text
+    # Verify path includes location
+    assert "substitutions->x" in caplog.text
+    assert "substitutions->y" in caplog.text
+
+
+def test_missing_dependency_warning(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A substitution referencing an undefined variable warns with the cause."""
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: OrderedDict({"a": "${missing}"}),
+            "key": "value",
+        }
+    )
+    with caplog.at_level(logging.WARNING):
+        substitutions.do_substitution_pass(config)
+
+    assert "Could not resolve substitution variable 'a'" in caplog.text
+    assert "'missing' is undefined" in caplog.text
+    assert "substitutions->a" in caplog.text
+
+
+def test_undefined_variable_warning(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A reference to an undefined variable in config values produces a warning."""
+    config = OrderedDict(
+        {
+            "key": "${undefined_var}",
+        }
+    )
+    with caplog.at_level(logging.WARNING):
+        substitutions.do_substitution_pass(config)
+
+    assert "'undefined_var' is undefined" in caplog.text
+
+
+def test_password_field_warnings_suppressed(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Undefined variables in password fields should not produce warnings."""
+    config = OrderedDict(
+        {
+            "password": "${undefined_var}",
+        }
+    )
+    with caplog.at_level(logging.WARNING):
+        substitutions.do_substitution_pass(config)
+
+    assert caplog.text == ""
+
+
+def test_config_context_unresolvable_warns(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Unresolvable vars in a ConfigContext produce warnings via push_context."""
+    inner = OrderedDict({"key": "${a}"})
+    yaml_util.add_context(inner, {"a": "${undefined}"})
+    config = OrderedDict({"items": [inner]})
+    with caplog.at_level(logging.WARNING):
+        substitutions.do_substitution_pass(config)
+
+    assert "Could not resolve substitution variable 'a'" in caplog.text
+    assert "'undefined' is undefined" in caplog.text
+
+
+def test_non_string_substitution_value_warning(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Undefined vars in non-string contexts (e.g. dict keys) produce warnings."""
+    config = OrderedDict(
+        {
+            "items": {"${undefined_key}": "value"},
+        }
+    )
+    with caplog.at_level(logging.WARNING):
+        substitutions.do_substitution_pass(config)
+
+    assert "'undefined_key' is undefined" in caplog.text
+
+
+def test_lambda_substitution() -> None:
+    """Substitution inside a Lambda value should be expanded."""
+    lam = Lambda("return ${var};")
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: OrderedDict({"var": "42"}),
+            "lambda": lam,
+        }
+    )
+    config = substitutions.do_substitution_pass(config)
+    assert config["lambda"].value == "return 42;"
+
+
+def test_lambda_no_substitution_unchanged() -> None:
+    """A Lambda with no variable references should not be mutated."""
+    lam = Lambda("return 1;")
+    original_value = lam.value
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: OrderedDict({"var": "42"}),
+            "lambda": lam,
+        }
+    )
+    config = substitutions.do_substitution_pass(config)
+    assert config["lambda"].value is original_value
+
+
+def test_extend_substitution() -> None:
+    """Substitution inside an Extend value should be expanded."""
+    ext = Extend("${component_id}")
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: OrderedDict({"component_id": "my_sensor"}),
+            "sensor": ext,
+        }
+    )
+    config = substitutions.do_substitution_pass(config)
+    assert config["sensor"].value == "my_sensor"
+
+
+def test_substitute_does_not_mutate_input() -> None:
+    """substitute() must return a new tree without modifying the original."""
+    inner_list = ["${var}", "static"]
+    inner_dict = OrderedDict({"key": "${var}"})
+    lam = Lambda("return ${var};")
+    config = OrderedDict(
+        {
+            "a_list": inner_list,
+            "a_dict": inner_dict,
+            "a_lambda": lam,
+            "plain": "${var}",
+        }
+    )
+    context = substitutions.ContextVars({"var": "replaced"})
+    result = substitutions.substitute(config, [], context, strict_undefined=True)
+
+    # Result has substitutions applied
+    assert result["plain"] == "replaced"
+    assert result["a_list"] == ["replaced", "static"]
+    assert result["a_dict"]["key"] == "replaced"
+    assert result["a_lambda"].value == "return replaced;"
+
+    # Original input is untouched
+    assert config["plain"] == "${var}"
+    assert inner_list == ["${var}", "static"]
+    assert inner_dict["key"] == "${var}"
+    assert lam.value == "return ${var};"
+
+    # Containers are new objects, not the originals
+    assert result["a_list"] is not inner_list
+    assert result["a_dict"] is not inner_dict
+    assert result["a_lambda"] is not lam
+
+
+def test_do_substitution_pass_substitutions_must_be_mapping_from_config() -> None:
+    """Non-mapping substitutions raises cv.Invalid."""
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: ["not", "a", "mapping"],
+            "other": "value",
+        }
+    )
+
+    with pytest.raises(
+        cv.Invalid, match="Substitutions must be a key to value mapping"
+    ):
+        substitutions.do_substitution_pass(config)
@@ -25,7 +25,7 @@ def test_include_with_vars(fixture_path: Path) -> None:
    yaml_file = fixture_path / "yaml_util" / "includetest.yaml"

    actual = yaml_util.load_yaml(yaml_file)
-    substitutions.do_substitution_pass(actual, None)
+    actual = substitutions.do_substitution_pass(actual, None)
    assert actual["esphome"]["name"] == "original"
    assert actual["esphome"]["libraries"][0] == "Wire"
    assert actual["esp8266"]["board"] == "nodemcu"