Merge remote-tracking branch 'upstream/precompute-tag-forced-varint-fields' into integration

# Conflicts:
#	esphome/components/api/api_pb2_dump.cpp
#	esphome/components/sensor/sensor.h

esphome/components/packages/__init__.py

@@ -300,9 +300,14 @@ def do_packages_pass(config: dict, skip_update: bool = False) -> dict:
             context_vars = package_config.vars
             if CONF_PACKAGES in package_config or CONF_URL in package_config:
                 # Remote package definition: eagerly resolve before PACKAGE_SCHEMA validation.
-                from esphome.components.substitutions import substitute_context_vars
+                from esphome.components.substitutions import ContextVars, substitute
 
-                substitute_context_vars(package_config, context_vars)
+                package_config = substitute(
+                    package_config,
+                    [],
+                    ContextVars(context_vars),
+                    strict_undefined=False,
+                )
         package_config = PACKAGE_SCHEMA(package_config)
         if isinstance(package_config, str):
             return package_config  # Jinja string, skip processing

esphome/components/sensor/sensor.h

@@ -97,10 +97,12 @@ class Sensor : public EntityBase {
   /// Getter-syntax for .state.
   float get_state() const { return this->state; }
   /// Getter-syntax for .raw_state
+  float get_raw_state() const {
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-  float get_raw_state() const { return this->raw_state; }
+    return this->raw_state;
 #pragma GCC diagnostic pop
+  }
 
   /** Publish a new state to the front-end.
    *

esphome/components/substitutions/__init__.py

@@ -81,12 +81,6 @@ def _restore_data_base(value: Any, orig_value: ESPHomeDataBase) -> ESPHomeDataBa
     return value
 
 
-def _try_substitute(value: Any, context: ContextVars) -> Any:
-    """Substitute variables in value, returning the result or the original if unchanged."""
-    result = _substitute_item(value, [], context, strict_undefined=True)
-    return result if result is not None else value
-
-
 def _resolve_var(name: str, context_vars: ContextVars) -> Any:
     """Look up a substitution variable, falling back to the resolver callback."""
     sub = context_vars.get(name, Missing)
@@ -253,7 +247,7 @@ def _push_context(
         if value is Missing:
             return Missing
         try:
-            value = _try_substitute(value, resolver_context)
+            value = substitute(value, [], resolver_context, True)
         except UndefinedError as err:
             unresolvables[key] = (value, err)
             return Missing
@@ -297,68 +291,51 @@ def push_context(
     return parent_context
 
 
-def _substitute_item(
+def substitute(
     item: Any,
     path: SubstitutionPath,
     parent_context: ContextVars,
     strict_undefined: bool,
     errors: ErrList | None = None,
-) -> Any | None:
-    """Recursively substitute variables in a config item.
+) -> Any:
+    """Returns a recursively substituted version of `item`."""
 
-    Walks dicts, lists, strings, Lambdas, Extend, and Remove nodes,
-    replacing variable references with values from context_vars.
-    Mutates containers in-place; returns a replacement value for
-    strings/scalars, or None if the item was unchanged.
-    """
+    if isinstance(item, ESPLiteralValue):
+        return item  # do not substitute inside literal blocks
 
-    def _walk(item: Any, path: SubstitutionPath, parent_ctx: ContextVars) -> Any | None:
-        if isinstance(item, ESPLiteralValue):
-            return None  # do not substitute inside literal blocks
+    # Push the current item's context onto the context stack
+    context_vars = push_context(item, parent_context, errors)
 
-        ctx = push_context(item, parent_ctx, errors)
+    result = item
 
-        if isinstance(item, list):
-            for idx, it in enumerate(item):
-                sub = _walk(it, path + [idx], ctx)
-                if sub is not None:
-                    item[idx] = sub
-        elif isinstance(item, dict):
-            replace_keys: list[tuple[str, Any]] = []
-            for k, v in item.items():
-                if path or k != CONF_SUBSTITUTIONS:
-                    sub = _walk(k, path + [k], ctx)
-                    if sub is not None:
-                        replace_keys.append((k, sub))
-                sub = _walk(v, path + [k], ctx)
-                if sub is not None:
-                    item[k] = sub
-            for old, new in replace_keys:
-                if str(new) == str(old):
-                    item[new] = item[old]
-                else:
-                    item[new] = merge_config(item.get(new), item.get(old))
-                    del item[old]
-        elif isinstance(item, str):
-            sub = _expand_substitutions(item, path, ctx, strict_undefined, errors)
-            if not isinstance(sub, str) or sub != item:
-                return sub
-        elif isinstance(item, (core.Lambda, Extend, Remove)) and item.value:
-            sub = _expand_substitutions(item.value, path, ctx, strict_undefined, errors)
-            if sub != item.value:
-                item.value = sub
-        return None
+    if isinstance(item, list):
+        result = [
+            substitute(it, path + [i], context_vars, strict_undefined, errors)
+            for i, it in enumerate(item)
+        ]
 
-    return _walk(item, path, parent_context)
+    elif isinstance(item, dict):
+        result = OrderedDict()
+        for k, v in item.items():
+            v = substitute(v, path + [k], context_vars, strict_undefined, errors)
+            k = substitute(k, path + [k], context_vars, strict_undefined, errors)
+            result[k] = merge_config(result.get(k), v)
 
+    elif isinstance(item, str):
+        result = _expand_substitutions(
+            item, path, context_vars, strict_undefined, errors
+        )
 
-def substitute_context_vars(node: Any, context_vars: dict[str, Any]) -> None:
-    """Eagerly substitute context vars into a config node in-place.
+    elif isinstance(item, (core.Lambda, Extend, Remove)) and item.value:
+        value = _expand_substitutions(
+            item.value, path, context_vars, strict_undefined, errors
+        )
+        if item.value != value:
+            result = type(item)(value)
 
-    Undefined variables are silently ignored — this is used before
-    the main substitution pass when not all variables are visible yet.
-    """
-    _substitute_item(node, [], ContextVars(context_vars), strict_undefined=False)
+    if isinstance(item, ESPHomeDataBase):
+        result = make_data_base(result, item)
+    return result
 
 
 def _warn_unresolved_variables(errors: ErrList) -> None:
@@ -387,7 +364,7 @@ def do_substitution_pass(
     Extracts the ``substitutions:`` block, merges in any command-line
     overrides, resolves inter-variable dependencies, then walks the
     config tree replacing all ``$var`` / ``${expr}`` references.
-    Returns the (mutated) config dict with resolved substitutions
+    Returns a new config dict with resolved substitutions
     restored at the front.
     """
     # Extract substitutions from config, overriding with substitutions coming from command line:
@@ -415,7 +392,7 @@ def do_substitution_pass(
     errors: ErrList = []  # Collect undefined errors during substitution
     parent_context, substitutions = _push_context(substitutions, ContextVars(), errors)
 
-    _substitute_item(config, [], parent_context, False, errors)
+    config = substitute(config, [], parent_context, False, errors)
 
     if errors:
         _warn_unresolved_variables(errors)

tests/benchmarks/components/api/bench_noise_encrypt.cpp

@@ -172,6 +172,135 @@ BENCHMARK(NoiseDecrypt_MediumMessage);
 static void NoiseDecrypt_LargeMessage(benchmark::State &state) { noise_decrypt_bench(state, 1024); }
 BENCHMARK(NoiseDecrypt_LargeMessage);
 
+// --- Full Noise_NNpsk0 handshake benchmark ---
+// Measures the complete handshake between initiator and responder:
+//   - Create handshake states for both sides
+//   - Set PSK and prologue
+//   - Exchange messages (initiator write -> responder read -> responder write -> initiator read)
+//   - Split to get cipher states
+// This is dominated by Curve25519 DH operations (expensive on ESP8266).
+// No inner iterations — each handshake is already expensive enough.
+
+static void NoiseHandshake_Full(benchmark::State &state) {
+  // Matching ESPHome's protocol: Noise_NNpsk0_25519_ChaChaPoly_SHA256
+  NoiseProtocolId nid;
+  memset(&nid, 0, sizeof(nid));
+  nid.pattern_id = NOISE_PATTERN_NN;
+  nid.cipher_id = NOISE_CIPHER_CHACHAPOLY;
+  nid.dh_id = NOISE_DH_CURVE25519;
+  nid.prefix_id = NOISE_PREFIX_STANDARD;
+  nid.hybrid_id = NOISE_DH_NONE;
+  nid.hash_id = NOISE_HASH_SHA256;
+  nid.modifier_ids[0] = NOISE_MODIFIER_PSK0;
+
+  // Dummy PSK (32 bytes) and prologue matching production setup
+  static constexpr uint8_t PSK[32] = {0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB,
+                                      0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB,
+                                      0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB, 0xAB};
+  static constexpr uint8_t PROLOGUE[] = "NoESPHome";
+
+  // Message buffer for handshake exchange (max handshake message ~96 bytes)
+  uint8_t msg_buf[128];
+
+  for (auto _ : state) {
+    NoiseHandshakeState *initiator = nullptr;
+    NoiseHandshakeState *responder = nullptr;
+    NoiseCipherState *init_send = nullptr, *init_recv = nullptr;
+    NoiseCipherState *resp_send = nullptr, *resp_recv = nullptr;
+    int err;
+
+    // Create both handshake states
+    err = noise_handshakestate_new_by_id(&initiator, &nid, NOISE_ROLE_INITIATOR);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Failed to create initiator");
+      return;
+    }
+    err = noise_handshakestate_new_by_id(&responder, &nid, NOISE_ROLE_RESPONDER);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Failed to create responder");
+      noise_handshakestate_free(initiator);
+      return;
+    }
+
+    // Set PSK and prologue on both sides
+    noise_handshakestate_set_pre_shared_key(initiator, PSK, sizeof(PSK));
+    noise_handshakestate_set_pre_shared_key(responder, PSK, sizeof(PSK));
+    noise_handshakestate_set_prologue(initiator, PROLOGUE, sizeof(PROLOGUE) - 1);
+    noise_handshakestate_set_prologue(responder, PROLOGUE, sizeof(PROLOGUE) - 1);
+
+    noise_handshakestate_start(initiator);
+    noise_handshakestate_start(responder);
+
+    // Message 1: Initiator -> Responder
+    NoiseBuffer write_buf, read_buf;
+    noise_buffer_set_output(write_buf, msg_buf, sizeof(msg_buf));
+    err = noise_handshakestate_write_message(initiator, &write_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Initiator write_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    noise_buffer_set_input(read_buf, msg_buf, write_buf.size);
+    err = noise_handshakestate_read_message(responder, &read_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Responder read_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    // Message 2: Responder -> Initiator
+    noise_buffer_set_output(write_buf, msg_buf, sizeof(msg_buf));
+    err = noise_handshakestate_write_message(responder, &write_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Responder write_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    noise_buffer_set_input(read_buf, msg_buf, write_buf.size);
+    err = noise_handshakestate_read_message(initiator, &read_buf, nullptr);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Initiator read_message failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+
+    // Split to get cipher states
+    err = noise_handshakestate_split(initiator, &init_send, &init_recv);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Initiator split failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      return;
+    }
+    err = noise_handshakestate_split(responder, &resp_send, &resp_recv);
+    if (err != NOISE_ERROR_NONE) {
+      state.SkipWithError("Responder split failed");
+      noise_handshakestate_free(initiator);
+      noise_handshakestate_free(responder);
+      noise_cipherstate_free(init_send);
+      noise_cipherstate_free(init_recv);
+      return;
+    }
+
+    benchmark::DoNotOptimize(init_send);
+
+    // Cleanup
+    noise_handshakestate_free(initiator);
+    noise_handshakestate_free(responder);
+    noise_cipherstate_free(init_send);
+    noise_cipherstate_free(init_recv);
+    noise_cipherstate_free(resp_send);
+    noise_cipherstate_free(resp_recv);
+  }
+}
+BENCHMARK(NoiseHandshake_Full);
+
 }  // namespace esphome::api::benchmarks
 
 #endif  // USE_API_NOISE

tests/unit_tests/test_substitutions.py

@@ -550,8 +550,8 @@ def test_lambda_substitution() -> None:
             "lambda": lam,
         }
     )
-    substitutions.do_substitution_pass(config)
-    assert lam.value == "return 42;"
+    config = substitutions.do_substitution_pass(config)
+    assert config["lambda"].value == "return 42;"
 
 
 def test_lambda_no_substitution_unchanged() -> None:
@@ -564,8 +564,8 @@ def test_lambda_no_substitution_unchanged() -> None:
             "lambda": lam,
         }
     )
-    substitutions.do_substitution_pass(config)
-    assert lam.value is original_value
+    config = substitutions.do_substitution_pass(config)
+    assert config["lambda"].value is original_value
 
 
 def test_extend_substitution() -> None:
@@ -577,8 +577,42 @@ def test_extend_substitution() -> None:
             "sensor": ext,
         }
     )
-    substitutions.do_substitution_pass(config)
-    assert ext.value == "my_sensor"
+    config = substitutions.do_substitution_pass(config)
+    assert config["sensor"].value == "my_sensor"
+
+
+def test_substitute_does_not_mutate_input() -> None:
+    """substitute() must return a new tree without modifying the original."""
+    inner_list = ["${var}", "static"]
+    inner_dict = OrderedDict({"key": "${var}"})
+    lam = Lambda("return ${var};")
+    config = OrderedDict(
+        {
+            "a_list": inner_list,
+            "a_dict": inner_dict,
+            "a_lambda": lam,
+            "plain": "${var}",
+        }
+    )
+    context = substitutions.ContextVars({"var": "replaced"})
+    result = substitutions.substitute(config, [], context, strict_undefined=True)
+
+    # Result has substitutions applied
+    assert result["plain"] == "replaced"
+    assert result["a_list"] == ["replaced", "static"]
+    assert result["a_dict"]["key"] == "replaced"
+    assert result["a_lambda"].value == "return replaced;"
+
+    # Original input is untouched
+    assert config["plain"] == "${var}"
+    assert inner_list == ["${var}", "static"]
+    assert inner_dict["key"] == "${var}"
+    assert lam.value == "return ${var};"
+
+    # Containers are new objects, not the originals
+    assert result["a_list"] is not inner_list
+    assert result["a_dict"] is not inner_dict
+    assert result["a_lambda"] is not lam
 
 
 def test_do_substitution_pass_substitutions_must_be_mapping_from_config() -> None: