[core] Store the validated-config cache as JSON to drop YAML off the upload fast path (#18106)

esphome/compiled_config.py

@@ -1,48 +1,69 @@
 """Validated-config cache for the upload/logs fast path.
 
-compile dumps the validated config to <data_dir>/storage/<file>.validated.yaml;
+compile dumps the validated config to <data_dir>/storage/<file>.validated.json;
 the next upload/logs for that YAML reuses it instead of running the full
-read_config pipeline. YAML round-trip (yaml_util.dump/load_yaml) keeps
+read_config pipeline. The cache is deliberately lossy: only ``!lambda``
-!lambda/!include/IDs/paths intact; mtime gates staleness.
+bodies survive typed (``Lambda``); IDs, time periods, MAC/IP addresses,
+paths, UUIDs and enums store the same string form the YAML dumper
+produced for them. JSON additionally coerces non-str dict keys to
+strings; validated configs only use string keys (every schema key
+validator is ``cv.string``). mtime gates staleness.
 """
 
 from __future__ import annotations
 
+import json
 import logging
 from pathlib import Path
+from typing import Any
 
-from esphome.core import CORE
+from esphome.const import __version__ as ESPHOME_VERSION
+from esphome.core import CORE, Lambda
 from esphome.helpers import write_file
 from esphome.storage_json import StorageJSON, ext_storage_path
 from esphome.types import ConfigType
 
 _LOGGER = logging.getLogger(__name__)
 
+# Bump when the on-disk shape changes; a mismatched version falls back
+# to read_config. The envelope also stamps the writing esphome version:
+# after an upgrade the cache holds the previous release's validation, so
+# it falls back once and the re-save self-heals.
+_CACHE_VERSION = 1
+_LAMBDA_KEY = "__esphome_lambda__"
+
 
 def compiled_config_path(config_filename: str) -> Path:
     """Path to the cached validated config alongside the storage sidecar."""
-    return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml"
+    return CORE.data_dir / "storage" / f"{config_filename}.validated.json"
-
-
-def _cache_is_fresh(cache_path: Path, source_path: Path) -> bool:
-    """True iff the cache file exists and isn't older than the source."""
-    try:
-        return cache_path.stat().st_mtime >= source_path.stat().st_mtime
-    except OSError:
-        return False
 
 
 def save_compiled_config(config: ConfigType) -> None:
     """Write the validated-config cache. Always-write so mtime stays fresh.
 
-    Mode 0600 because show_secrets=True resolves !secret inline.
+    Mode 0600 because config validation resolved !secret inline.
     Failures are non-fatal: the fast path falls back to read_config.
     """
-    from esphome import yaml_util
-
     try:
-        rendered = yaml_util.dump(config, show_secrets=True)
+        # The legacy YAML cache holds inline-resolved secrets and nothing
+        # reads it anymore; drop it even when the write below fails. A
+        # failed removal leaves resolved secrets on disk, so it warns.
+        try:
+            _legacy_compiled_config_path(CORE.config_filename).unlink(missing_ok=True)
+        except OSError as err:
+            _LOGGER.warning(
+                "Could not remove the legacy validated-config cache: %s", err
+            )
+        rendered = json.dumps(
+            {"v": _CACHE_VERSION, "esphome": ESPHOME_VERSION, "config": config},
+            separators=(",", ":"),
+            default=_json_default,
+        )
         write_file(compiled_config_path(CORE.config_filename), rendered, private=True)
+    except TypeError as err:
+        # Structural, not transient: this config can never cache (e.g. a
+        # non-basic dict key), so every upload/logs pays the slow path.
+        _LOGGER.warning("Cannot cache the validated config: %s", err)
     except Exception as err:  # noqa: BLE001  # pylint: disable=broad-except
         _LOGGER.debug("Skipping compiled config cache write: %s", err)
 
@@ -51,25 +72,29 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None:
     """Load the cached validated config and apply storage metadata to CORE.
 
     Returns None (caller falls back to read_config) when the cache is
-    missing, older than the source YAML, unparseable, or the sidecar
+    missing, older than the source YAML, unparseable, a different cache
-    is incomplete.
+    version, or the sidecar is incomplete. The loaded config carries no
+    source ranges; callers must not feed it into read_config/write_cpp.
     """
     cache_path = compiled_config_path(conf_path.name)
     if not _cache_is_fresh(cache_path, conf_path):
         return None
 
-    from esphome import yaml_util
-
     try:
-        # Fast path never validates or generates code - no source ranges
+        envelope = json.loads(
-        # needed (see load_yaml). Callers must not feed this config into
+            cache_path.read_text(encoding="utf-8"), object_hook=_decode_object
-        # read_config/write_cpp: the esp_range consumers in config.py and
-        # cpp_generator.py are isinstance-guarded and would degrade
-        # silently (wrong error/lambda locations) instead of raising.
-        config = yaml_util.load_yaml(
-            cache_path, clear_secrets=False, track_document_range=False
         )
-    except Exception:  # noqa: BLE001  # pylint: disable=broad-except
+    except (OSError, ValueError) as err:
+        _LOGGER.debug("Ignoring unreadable compiled config cache: %s", err)
+        return None
+
+    if (
+        not isinstance(envelope, dict)
+        or envelope.get("v") != _CACHE_VERSION
+        or envelope.get("esphome") != ESPHOME_VERSION
+        or not isinstance(config := envelope.get("config"), dict)
+    ):
+        _LOGGER.debug("Ignoring compiled config cache with a foreign envelope")
         return None
 
     storage = StorageJSON.load(ext_storage_path(conf_path.name))
@@ -81,3 +106,38 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None:
         return None
     storage.apply_to_core()
     return config
+
+
+# Remove before 2027.8: by then every maintained install has saved the
+# JSON cache at least once and dropped its legacy YAML file.
+def _legacy_compiled_config_path(config_filename: str) -> Path:
+    """Path of the pre-JSON YAML cache; only ever removed."""
+    return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml"
+
+
+def _cache_is_fresh(cache_path: Path, source_path: Path) -> bool:
+    """True iff the cache file exists and isn't older than the source."""
+    try:
+        return cache_path.stat().st_mtime >= source_path.stat().st_mtime
+    except OSError:
+        return False
+
+
+def _json_default(value: Any) -> Any:
+    """Mirror ESPHomeDumper's representers: Lambda stays typed, the rest
+    stringify (IDs, time periods, MAC/IP addresses, paths, UUIDs, enums).
+
+    IncludeFile/Extend/Remove have no JSON mirror and would stringify
+    wrong, but none survive validation (config.py's packages merge and
+    the substitution pass consume them) so no guard is spent on them.
+    """
+    if isinstance(value, Lambda):
+        return {_LAMBDA_KEY: value.value}
+    return str(value)
+
+
+def _decode_object(obj: dict[str, Any]) -> Any:
+    """Revive the Lambda sentinel; every other mapping passes through."""
+    if len(obj) == 1 and isinstance(value := obj.get(_LAMBDA_KEY), str):
+        return Lambda(value)
+    return obj

esphome/core/__init__.py

@@ -321,14 +321,18 @@ LAMBDA_PROG = re.compile(r"\bid\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\)(\.?)")
 
 class Lambda:
     def __init__(self, value):
-        from esphome.cpp_generator import Expression, statement
-
         # pylint: disable=protected-access
         if isinstance(value, Lambda):
             self._value = value._value
-        elif isinstance(value, Expression):
+        elif isinstance(value, str):
-            self._value = str(statement(value))
+            # The validated-config cache revives Lambdas from strings on the
+            # upload/logs fast path; keep codegen off that path.
+            self._value = value
         else:
+            from esphome.cpp_generator import Expression, statement
+
+            if isinstance(value, Expression):
+                value = str(statement(value))
             self._value = value
         self._parts = None
         self._requires_ids = None

esphome/yaml_util.py

@@ -1349,6 +1349,8 @@ class ESPHomeDumper(yaml.SafeDumper):
         return super().increase_indent(flow, False)
 
 
+# Mirrored by compiled_config._json_default: a new representer that keeps a
+# type round-trippable (like Lambda's) needs a sentinel there too.
 ESPHomeDumper.add_multi_representer(
     dict, lambda dumper, value: dumper.represent_mapping("tag:yaml.org,2002:map", value)
 )

tests/benchmarks/python/test_compiled_config_bench.py

@@ -52,7 +52,7 @@ def _prime_cache(yaml_path: Path) -> None:
 
     Mirrors ``esphome compile``: ``read_config`` populates ``CORE.config``,
     then ``update_storage_json`` writes both the StorageJSON sidecar and
-    the ``.validated.yaml`` compiled-config cache.
+    the ``.validated.json`` compiled-config cache.
     """
     CORE.config_path = yaml_path
     config = read_config({}, skip_external_update=True)

tests/unit_tests/fixtures/lazy_imports/upload_command_fast_path.py

@@ -2,12 +2,13 @@
 
 Executed as a subprocess by test_lazy_imports.py: heavy module names come
 in on argv, the ones found in sys.modules afterwards go out on stdout.
-Covers both fast-path claims: the bundle suffix check in run_esphome reads
+Covers three fast-path claims: the bundle suffix check in run_esphome reads
-BUNDLE_EXTENSION from esphome.const without importing esphome.bundle, and
+BUNDLE_EXTENSION from esphome.const without importing esphome.bundle, the
-the real validated-config cache parse, include resolution included, stays
+validated-config cache parse stays voluptuous free, and the JSON cache
-voluptuous free.
+(lambda sentinel included) resolves without pyyaml or esphome.yaml_util.
 """
 
+import json
 import os
 from pathlib import Path
 import sys
@@ -16,7 +17,6 @@ from unittest.mock import patch
 
 from _leak_report import print_leaked_modules
 from _storage import make_storage
-import yaml
 
 # Everything imported past this point is the code under test; the pop
 # below must only drop what the setup itself preloaded, or it would
@@ -24,8 +24,10 @@ import yaml
 _FIXTURE_PRELOADED = frozenset(sys.modules)
 
 from esphome import __main__ as main_mod  # noqa: E402
+from esphome.const import __version__ as ESPHOME_VERSION  # noqa: E402
 
 CONFIG_TEXT = "esphome:\n  name: t\n"
+LAMBDA_BODY = 'ESP_LOGD("t", "x");'
 
 # An ambient data-dir override would relocate the storage tree away
 # from the tmp config dir this fixture builds.
@@ -39,13 +41,23 @@ with tempfile.TemporaryDirectory() as _td:
 
     storage_dir = tmp / ".esphome" / "storage"
     storage_dir.mkdir(parents=True)
-    # The cache is a top-level !include so loading it resolves an
+    # The cache carries a lambda sentinel so loading revives a real Lambda
-    # IncludeFile for real on the fast path. The sidecar is written to the
+    # on the fast path. The sidecar is written to the layout
-    # layout ext_storage_path resolves once run_esphome sets
+    # ext_storage_path resolves once run_esphome sets CORE.config_path;
-    # CORE.config_path; going through CORE here would be circular.
+    # going through CORE here would be circular.
-    (storage_dir / "inc.yaml").write_text(CONFIG_TEXT)
+    cache_path = storage_dir / "test.yaml.validated.json"
-    cache_path = storage_dir / "test.yaml.validated.yaml"
+    cache_path.write_text(
-    cache_path.write_text("!include inc.yaml\n")
+        json.dumps(
+            {
+                "v": 1,
+                "esphome": ESPHOME_VERSION,
+                "config": {
+                    "esphome": {"name": "t"},
+                    "script": [{"lambda": {"__esphome_lambda__": LAMBDA_BODY}}],
+                },
+            }
+        )
+    )
     os.utime(cache_path)  # keep the cache at least as fresh as the source
     make_storage().save(storage_dir / "test.yaml.json")
 
@@ -76,7 +88,13 @@ with tempfile.TemporaryDirectory() as _td:
     # asserts so PYTHONOPTIMIZE in the ambient environment can't strip them.
     if exit_code != 0:
         sys.exit(f"run_esphome exited {exit_code} before dispatching upload")
-    if dispatched.get("config") != yaml.safe_load(CONFIG_TEXT):
+    config = dispatched.get("config")
-        sys.exit(f"cache include did not resolve through the fast path: {dispatched!r}")
+    if config is None or config.get("esphome") != {"name": "t"}:
+        sys.exit(f"cache did not resolve through the fast path: {dispatched!r}")
+    from esphome.core import Lambda
+
+    revived = config["script"][0]["lambda"]
+    if not isinstance(revived, Lambda) or revived.value != LAMBDA_BODY:
+        sys.exit(f"lambda sentinel did not revive: {revived!r}")
 
     print_leaked_modules()

tests/unit_tests/test_compiled_config.py

@@ -2,15 +2,20 @@
 
 from __future__ import annotations
 
+from ipaddress import IPv4Address, IPv4Network
 import json
 import os
 from pathlib import Path
+from typing import Any
 from unittest.mock import patch
+from uuid import UUID
 
 import pytest
 
+from esphome import const, yaml_util
 from esphome.__main__ import run_esphome
 from esphome.compiled_config import (
+    _LAMBDA_KEY,
     compiled_config_path,
     load_compiled_config,
     save_compiled_config,
@@ -24,30 +29,26 @@ from esphome.const import (
     KEY_TARGET_FRAMEWORK,
     KEY_TARGET_PLATFORM,
     KEY_VARIANT,
+    Toolchain,
 )
-from esphome.core import CORE
+from esphome.core import CORE, ID, HexInt, Lambda, MACAddress, TimePeriodMilliseconds
-from esphome.yaml_util import ESPHomeDataBase
+from esphome.util import OrderedDict
 
-_VALIDATED_CONFIG_YAML = """\
+_VALIDATED_CONFIG = {
-esphome:
+    "esphome": {"name": "lite_test", "friendly_name": "Lite Test Device"},
-  name: lite_test
+    "esp32": {"board": "nodemcu-32s"},
-  friendly_name: Lite Test Device
+    "logger": {"baud_rate": 115200},
-esp32:
+    "api": {"port": 6053, "encryption": {"key": "6dGhpcyBpcyBhIHRlc3Q="}},
-  board: nodemcu-32s
+    "ota": [{"platform": "esphome", "port": 3232, "password": "secret"}],
-logger:
+    "wifi": {"ssid": "ssid", "use_address": "192.168.1.42"},
-  baud_rate: 115200
+}
-api:
+
-  port: 6053
+
-  encryption:
+def _cache_body(config: dict | None = None) -> str:
-    key: 6dGhpcyBpcyBhIHRlc3Q=
+    """Render the JSON envelope the production save writes."""
-ota:
+    return json.dumps(
-  - platform: esphome
+        {"v": 1, "esphome": const.__version__, "config": config or _VALIDATED_CONFIG}
-    port: 3232
+    )
-    password: secret
-wifi:
-  ssid: ssid
-  use_address: 192.168.1.42
-"""
 
 
 def _write_storage(
@@ -79,10 +80,10 @@ def _write_storage(
     storage_path.write_text(json.dumps(data), encoding="utf-8")
 
 
-def _write_cache(cache_path: Path, body: str = _VALIDATED_CONFIG_YAML) -> Path:
+def _write_cache(cache_path: Path, body: str | None = None) -> Path:
     """Write the cache file and return it."""
     cache_path.parent.mkdir(parents=True, exist_ok=True)
-    cache_path.write_text(body, encoding="utf-8")
+    cache_path.write_text(body if body is not None else _cache_body(), encoding="utf-8")
     return cache_path
 
 
@@ -96,24 +97,28 @@ def _set_cache_mtime(cache_path: Path, yaml_path: Path, *, offset: int) -> None:
 
 
 @pytest.fixture
-def fresh_cache_files(tmp_path: Path) -> Path:
+def primed_storage(tmp_path: Path) -> Path:
-    """YAML + StorageJSON + cache, all consistent and fresh."""
+    """YAML + StorageJSON sidecar, no cache yet."""
     yaml_path = tmp_path / "lite_test.yaml"
     yaml_path.write_text("esphome:\n  name: lite_test\n")
     CORE.config_path = yaml_path
-
+    _write_storage(tmp_path / ".esphome" / "storage" / "lite_test.yaml.json")
-    storage_dir = tmp_path / ".esphome" / "storage"
-    _write_storage(storage_dir / "lite_test.yaml.json")
-    cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
-    _set_cache_mtime(cache, yaml_path, offset=5)
-
     return yaml_path
 
 
+@pytest.fixture
+def fresh_cache_files(primed_storage: Path) -> Path:
+    """YAML + StorageJSON + cache, all consistent and fresh."""
+    storage_dir = primed_storage.parent / ".esphome" / "storage"
+    cache = _write_cache(storage_dir / "lite_test.yaml.validated.json")
+    _set_cache_mtime(cache, primed_storage, offset=5)
+    return primed_storage
+
+
 def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None:
     """The cache file shape is predictable from the YAML filename."""
     path = compiled_config_path("device.yaml")
-    assert path.name == "device.yaml.validated.yaml"
+    assert path.name == "device.yaml.validated.json"
     assert path.parent.name == "storage"
 
 
@@ -126,9 +131,8 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None:
     assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q="
     assert config["ota"][0]["password"] == "secret"
 
-    # The fast path loads without per-node source ranges (the full
+    # The fast path loads plain scalars; no per-node source ranges exist.
-    # contract lives in test_yaml_util; this checks the flag is wired up).
+    assert type(config[CONF_ESPHOME][CONF_NAME]) is str
-    assert not isinstance(config[CONF_ESPHOME][CONF_NAME], ESPHomeDataBase)
 
     # apply_to_core populated exactly what upload/logs read off CORE.
     assert CORE.name == "lite_test"
@@ -147,7 +151,7 @@ def test_load_compiled_config_populates_esp32_variant(tmp_path: Path) -> None:
 
     storage_dir = tmp_path / ".esphome" / "storage"
     _write_storage(storage_dir / "lite_test.yaml.json", esp_platform="ESP32S3")
-    cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
+    cache = _write_cache(storage_dir / "lite_test.yaml.validated.json")
     _set_cache_mtime(cache, yaml_path, offset=5)
 
     assert load_compiled_config(yaml_path) is not None
@@ -168,7 +172,7 @@ def test_load_compiled_config_skips_esp32_block_for_other_platforms(
         esp_platform="ESP8266",
         core_platform="esp8266",
     )
-    cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
+    cache = _write_cache(storage_dir / "lite_test.yaml.validated.json")
     _set_cache_mtime(cache, yaml_path, offset=5)
 
     assert load_compiled_config(yaml_path) is not None
@@ -185,7 +189,7 @@ def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None:
     yaml_path.write_text("esphome:\n  name: lite_test\n")
     CORE.config_path = yaml_path
     storage_dir = tmp_path / ".esphome" / "storage"
-    cache_path = storage_dir / "lite_test.yaml.validated.yaml"
+    cache_path = storage_dir / "lite_test.yaml.validated.json"
     sidecar_path = storage_dir / "lite_test.yaml.json"
 
     if scenario == "missing_cache":
@@ -196,7 +200,7 @@ def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None:
     elif scenario == "corrupt_cache":
         _write_storage(sidecar_path)
         _set_cache_mtime(
-            _write_cache(cache_path, "not: valid: yaml: ["), yaml_path, offset=5
+            _write_cache(cache_path, '{"v": 1, "config": {'), yaml_path, offset=5
         )
     elif scenario == "missing_sidecar":
         # Cache fresh + parseable, but no StorageJSON → can't populate CORE.
@@ -205,6 +209,108 @@ def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None:
     assert load_compiled_config(yaml_path) is None
 
 
+@pytest.mark.parametrize(
+    "body",
+    [
+        pytest.param(
+            json.dumps(
+                {"v": 999, "esphome": const.__version__, "config": {"esphome": {}}}
+            ),
+            id="wrong_version",
+        ),
+        pytest.param(
+            json.dumps({"esphome": const.__version__, "config": {"esphome": {}}}),
+            id="missing_version",
+        ),
+        pytest.param(
+            json.dumps({"v": 1, "esphome": "2020.1.0", "config": {"esphome": {}}}),
+            id="other_esphome_version",
+        ),
+        pytest.param(
+            json.dumps({"v": 1, "config": {"esphome": {}}}),
+            id="missing_esphome_version",
+        ),
+        pytest.param(
+            json.dumps(
+                {
+                    "v": 1,
+                    "esphome": const.__version__,
+                    "config": ["not", "a", "dict"],
+                }
+            ),
+            id="non_dict_config",
+        ),
+        pytest.param(
+            json.dumps({"v": 1, "esphome": const.__version__}), id="missing_config"
+        ),
+        pytest.param(json.dumps(["not", "an", "envelope"]), id="non_dict_envelope"),
+    ],
+)
+def test_load_compiled_config_rejects_bad_envelope(
+    primed_storage: Path, body: str
+) -> None:
+    """A foreign or future cache shape falls back instead of half-loading."""
+    storage_dir = primed_storage.parent / ".esphome" / "storage"
+    cache = _write_cache(storage_dir / "lite_test.yaml.validated.json", body)
+    _set_cache_mtime(cache, primed_storage, offset=5)
+
+    assert load_compiled_config(primed_storage) is None
+
+
+def test_load_ignores_legacy_yaml_cache(primed_storage: Path) -> None:
+    """A fresh pre-JSON ``.validated.yaml`` alone can't drive the fast path."""
+    storage_dir = primed_storage.parent / ".esphome" / "storage"
+    legacy = _write_cache(
+        storage_dir / "lite_test.yaml.validated.yaml", "esphome:\n  name: lite_test\n"
+    )
+    _set_cache_mtime(legacy, primed_storage, offset=5)
+
+    assert load_compiled_config(primed_storage) is None
+
+
+def test_save_removes_stale_legacy_yaml_cache(tmp_path: Path) -> None:
+    """A successful save leaves only the JSON cache behind."""
+    CORE.config_path = tmp_path / "lite_test.yaml"
+    legacy = tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.yaml"
+    legacy.parent.mkdir(parents=True, exist_ok=True)
+    legacy.write_text("esphome:\n  name: lite_test\n")
+
+    save_compiled_config({"esphome": {"name": "lite_test"}})
+
+    assert compiled_config_path("lite_test.yaml").is_file()
+    assert not legacy.exists()
+
+
+def test_save_removes_legacy_yaml_even_when_write_fails(tmp_path: Path) -> None:
+    """The secret-bearing legacy cache goes away regardless of write outcome."""
+    CORE.config_path = tmp_path / "lite_test.yaml"
+    legacy = tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.yaml"
+    legacy.parent.mkdir(parents=True, exist_ok=True)
+    legacy.write_text("esphome:\n  name: lite_test\n")
+
+    with patch("esphome.compiled_config.write_file", side_effect=RuntimeError("boom")):
+        save_compiled_config({"esphome": {"name": "lite_test"}})
+
+    assert not legacy.exists()
+    assert not compiled_config_path("lite_test.yaml").exists()
+
+
+def test_save_warns_when_legacy_cache_unremovable(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """A secret-bearing legacy file that won't unlink warns; the write proceeds."""
+    CORE.config_path = tmp_path / "lite_test.yaml"
+    legacy = tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.yaml"
+    legacy.parent.mkdir(parents=True, exist_ok=True)
+    legacy.mkdir()  # unlink() on a directory raises OSError
+
+    with caplog.at_level("WARNING", logger="esphome.compiled_config"):
+        save_compiled_config({"esphome": {"name": "lite_test"}})
+
+    assert "legacy validated-config cache" in caplog.text
+    assert compiled_config_path("lite_test.yaml").is_file()
+
+
 @pytest.mark.parametrize("command", ["upload", "logs"])
 def test_run_esphome_upload_and_logs_use_cache_when_fresh(
     command: str,
@@ -258,7 +364,7 @@ def test_run_esphome_upload_does_not_refresh_cache_without_sidecar(
 ) -> None:
     """Without a StorageJSON sidecar (no compile has run), the fallback
     skips the cache write -- load_compiled_config requires the sidecar,
-    so writing the rendered (secret-resolved) YAML would be inert and
+    so writing the rendered (secret-resolved) config would be inert and
     leak secrets to disk for nothing."""
     yaml_path = tmp_path / "lite_test.yaml"
     yaml_path.write_text("esphome:\n  name: lite_test\n")
@@ -293,7 +399,7 @@ def test_run_esphome_upload_and_logs_refresh_cache_on_fallback(
 
     storage_dir = tmp_path / ".esphome" / "storage"
     _write_storage(storage_dir / "lite_test.yaml.json")
-    cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
+    cache = _write_cache(storage_dir / "lite_test.yaml.validated.json")
     _set_cache_mtime(cache, yaml_path, offset=-60)  # stale
 
     fresh_config = {"esphome": {"name": "lite_test"}, "logger": {}}
@@ -386,28 +492,161 @@ def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None
 
 
 def test_save_compiled_config_writes_cache(tmp_path: Path) -> None:
-    """`save_compiled_config` writes the dumped YAML next to the sidecar."""
+    """`save_compiled_config` writes the JSON envelope next to the sidecar."""
     CORE.config_path = tmp_path / "lite_test.yaml"
     save_compiled_config({"esphome": {"name": "lite_test"}, "logger": {}})
 
     cache_path = compiled_config_path("lite_test.yaml")
     assert cache_path.is_file()
-    body = cache_path.read_text()
+    envelope = json.loads(cache_path.read_text())
-    assert "name: lite_test" in body
+    assert envelope["v"] == 1
-    assert "logger:" in body
+    assert envelope["esphome"] == const.__version__
+    assert envelope["config"] == {"esphome": {"name": "lite_test"}, "logger": {}}
 
 
-def test_save_compiled_config_swallows_dump_errors(
+def test_save_compiled_config_swallows_write_errors(
     tmp_path: Path, caplog: pytest.LogCaptureFixture
 ) -> None:
-    """Failures during the dump are non-fatal -- a bad cache just means
+    """Failures during the write are non-fatal -- a bad cache just means
     the next fast path falls back to read_config()."""
     CORE.config_path = tmp_path / "lite_test.yaml"
-    with patch("esphome.yaml_util.dump", side_effect=RuntimeError("boom")):
+    with patch("esphome.compiled_config.write_file", side_effect=RuntimeError("boom")):
         save_compiled_config({"esphome": {"name": "lite_test"}})
     assert not compiled_config_path("lite_test.yaml").exists()
 
 
+def test_save_stringifies_unknown_values(tmp_path: Path) -> None:
+    """A type with no dedicated encoding stores its string form."""
+
+    class Weird:
+        def __str__(self) -> str:
+            return "weird-str"
+
+    CORE.config_path = tmp_path / "lite_test.yaml"
+    save_compiled_config({"esphome": {"name": "lite_test", "weird": Weird()}})
+    envelope = json.loads(compiled_config_path("lite_test.yaml").read_text())
+    assert envelope["config"]["esphome"]["weird"] == "weird-str"
+
+
+def test_save_skips_cache_on_unserializable_key(tmp_path: Path) -> None:
+    """A non-basic dict key aborts the write; the fast path falls back."""
+    CORE.config_path = tmp_path / "lite_test.yaml"
+    save_compiled_config({"esphome": {("a", "b"): "lite_test"}})
+    assert not compiled_config_path("lite_test.yaml").exists()
+
+
+def _normalize(value: Any) -> Any:
+    """Make Lambda comparable; everything else compares by value already."""
+    if isinstance(value, Lambda):
+        return ("__lambda__", value.value)
+    if isinstance(value, dict):
+        return {k: _normalize(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_normalize(v) for v in value]
+    return value
+
+
+def _round_trip_config() -> OrderedDict:
+    """A post-validation shaped config exercising every representer type."""
+    return OrderedDict(
+        {
+            "esphome": OrderedDict(
+                {
+                    "name": "lite_test",
+                    "build_path": Path("/build/lite_test"),
+                    "on_boot": [
+                        OrderedDict(
+                            {
+                                "trigger_id": ID("trigger_1", type="Trigger"),
+                                "then": [{"lambda": Lambda('ESP_LOGD("t", "x");')}],
+                            }
+                        )
+                    ],
+                }
+            ),
+            "wifi": OrderedDict(
+                {
+                    "id": ID("wifi_id", type="WiFiComponent"),
+                    "reboot_timeout": TimePeriodMilliseconds(milliseconds=900000),
+                    "use_address": IPv4Address("192.168.1.42"),
+                    "subnet": IPv4Network("192.168.1.0/24"),
+                    "mac": MACAddress(0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0x01),
+                }
+            ),
+            "misc": OrderedDict(
+                {
+                    "uuid": UUID("12345678-1234-5678-1234-567812345678"),
+                    "toolchain": Toolchain.PLATFORMIO,
+                    "hex": HexInt(0x1234),
+                    "levels": (1, 2.5, True, None),
+                    "empty": {},
+                }
+            ),
+        }
+    )
+
+
+def test_cache_round_trip_matches_yaml_cache(primed_storage: Path) -> None:
+    """The JSON cache loads the same tree the YAML cache used to."""
+    config = _round_trip_config()
+    save_compiled_config(config)
+    from_json = load_compiled_config(primed_storage)
+    assert from_json is not None
+
+    yaml_cache = primed_storage.parent / "dumped.yaml"
+    yaml_cache.write_text(yaml_util.dump(config, show_secrets=True))
+    from_yaml = yaml_util.load_yaml(
+        yaml_cache, clear_secrets=False, track_document_range=False
+    )
+
+    assert _normalize(from_json) == _normalize(from_yaml)
+
+
+def test_lambda_sentinel_round_trips(primed_storage: Path) -> None:
+    """A !lambda body comes back as a Lambda with the same source."""
+    body = 'id(sensor_1).publish_state(42);\nreturn "multi\\nline";'
+    save_compiled_config(
+        {
+            "esphome": {"name": "lite_test"},
+            "script": [{"then": [{"lambda": Lambda(body)}]}],
+        }
+    )
+
+    config = load_compiled_config(primed_storage)
+    assert config is not None
+    revived = config["script"][0]["then"][0]["lambda"]
+    assert isinstance(revived, Lambda)
+    assert revived.value == body
+
+
+def test_object_hook_requires_exact_shape(primed_storage: Path) -> None:
+    """Only the exact one-key string-valued sentinel revives a Lambda."""
+    storage_dir = primed_storage.parent / ".esphome" / "storage"
+    config = {
+        "esphome": {"name": "lite_test"},
+        "extra_key": {_LAMBDA_KEY: "x", "y": 1},
+        "non_str": {_LAMBDA_KEY: 5},
+    }
+    cache = _write_cache(
+        storage_dir / "lite_test.yaml.validated.json", _cache_body(config)
+    )
+    _set_cache_mtime(cache, primed_storage, offset=5)
+
+    loaded = load_compiled_config(primed_storage)
+    assert loaded is not None
+    assert loaded["extra_key"] == {_LAMBDA_KEY: "x", "y": 1}
+    assert loaded["non_str"] == {_LAMBDA_KEY: 5}
+
+
+def test_int_keys_coerce_to_strings(primed_storage: Path) -> None:
+    """Non-str basic keys stringify; validated configs only use string keys."""
+    save_compiled_config({"esphome": {"name": "lite_test"}, "table": {1: "a", 2: "b"}})
+
+    config = load_compiled_config(primed_storage)
+    assert config is not None
+    assert config["table"] == {"1": "a", "2": "b"}
+
+
 def test_load_compiled_config_rejects_wizard_only_sidecar(tmp_path: Path) -> None:
     """A wizard-only sidecar (no compile -- no core_platform / target_platform)
     can't drive upload/logs, so the fast path falls back."""
@@ -426,7 +665,7 @@ def test_load_compiled_config_rejects_wizard_only_sidecar(tmp_path: Path) -> Non
         '"loaded_integrations": [], "loaded_platforms": [], "no_mdns": false, '
         '"framework": null, "core_platform": null}'
     )
-    cache_path = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
+    cache_path = _write_cache(storage_dir / "lite_test.yaml.validated.json")
     _set_cache_mtime(cache_path, yaml_path, offset=5)
 
     assert load_compiled_config(yaml_path) is None

tests/unit_tests/test_core.py

@@ -1,5 +1,7 @@
 import os
 from pathlib import Path
+import subprocess
+import sys
 from unittest.mock import patch
 
 from hypothesis import given
@@ -213,6 +215,31 @@ class TestLambda:
 
         assert str(target) is value.value
 
+    def test_init__expression_initializer(self):
+        from esphome.cpp_generator import RawExpression
+
+        target = core.Lambda(RawExpression("foo()"))
+
+        assert target.value == "foo();"
+
+    def test_init__other_initializer(self):
+        target = core.Lambda(123)
+
+        assert target.value == 123
+
+    def test_init_from_str_does_not_import_codegen(self):
+        """The validated-config cache revives Lambdas on the upload fast path."""
+        # sys.exit rather than assert so ambient PYTHONOPTIMIZE can't strip it.
+        check = (
+            "import sys; from esphome.core import Lambda; "
+            "Lambda('return 1;'); "
+            "sys.exit('codegen leaked' if 'esphome.cpp_generator' in sys.modules else 0)"
+        )
+        result = subprocess.run(
+            [sys.executable, "-c", check], capture_output=True, text=True, check=False
+        )
+        assert result.returncode == 0, result.stderr
+
     def test_parts(self):
         target = core.Lambda(SAMPLE_LAMBDA.strip())
 

tests/unit_tests/test_lazy_imports.py

@@ -46,6 +46,11 @@ API_HEAVY_MODULES = ("aioesphomeapi",)
 # never pays for the bundle machinery and its tarfile chain.
 BUNDLE_HEAVY_MODULES = ("esphome.bundle", "tarfile")
 
+# Heavy only for a cache-hit upload/logs run: the JSON cache parse must
+# not resolve pyyaml or the yaml_util chain (the read_config fallback
+# still uses both).
+CACHE_HIT_HEAVY_MODULES = ("esphome.yaml_util", "yaml")
+
 # Stdlib modules deferred out of the dispatch fast path: a cache-hit
 # upload/logs run never writes a file (tempfile), spawns a process
 # (subprocess), parses a URL (urllib.parse), or prints a serial
@@ -56,8 +61,6 @@ STDLIB_FAST_PATH_MODULES = (
     "tempfile",
     "subprocess",
     "getpass",
-    # Pins the module-level contract only: PyYAML's constructor loads
-    # datetime during the cache parse until the JSON cache lands.
     "datetime",
     *(("urllib.parse",) if sys.version_info >= (3, 13) else ()),
 )
@@ -108,6 +111,7 @@ def test_watched_heavy_modules_exist() -> None:
         FAST_PATH_HEAVY_MODULES
         + API_HEAVY_MODULES
         + BUNDLE_HEAVY_MODULES
+        + CACHE_HIT_HEAVY_MODULES
         + STDLIB_FAST_PATH_MODULES
     ):
         assert importlib.util.find_spec(module) is not None, (
@@ -270,13 +274,13 @@ def test_upload_command_path_does_not_import_heavy_modules(
     leaked = _leaked_from_fixture(
         fixture_path,
         "upload_command_fast_path.py",
-        extra=BUNDLE_HEAVY_MODULES + STDLIB_FAST_PATH_MODULES,
+        extra=BUNDLE_HEAVY_MODULES + CACHE_HIT_HEAVY_MODULES + STDLIB_FAST_PATH_MODULES,
     )
     assert not leaked, (
         f"the upload dispatch path pulls in heavy modules: {leaked}. "
         "An ordinary run only needs the bundle suffix constant, and the "
-        "cache parse must not resolve voluptuous; keep the esphome.bundle "
+        "JSON cache parse must not resolve voluptuous or pyyaml; keep the "
-        "import inside the branch that extracts one, the Invalid import "
+        "esphome.bundle import inside the branch that extracts one, the "
-        "inside the branch that raises it, and the deferred stdlib "
+        "yaml_util imports inside the read_config fallback, and the "
-        "imports inside the write/spawn/serial helpers that use them."
+        "deferred stdlib imports inside the write/spawn/serial helpers."
     )