[core] Extract cache helpers into esphome/compiled_config.py

storage_json.py is for the JSON sidecar; the validated-config cache
is YAML. Move it out into its own module.

Also consolidate the dispatcher's open-coded fast path (load cache +
load sidecar + apply_to_core) into a single load_compiled_config()
entry point, and collapse save_compiled_config()'s two try blocks
into one. mtime comparison gets its own _cache_is_fresh() helper so
the loader reads top to bottom.
This commit is contained in:
J. Nick Koston
2026-05-12 15:45:47 -05:00
parent 26d2d4f65d
commit aee2fc5761
5 changed files with 176 additions and 218 deletions
+14 -24
View File
@@ -2419,33 +2419,23 @@ def run_esphome(argv):
skip_external = args.command in ("logs", "clean")
command_line_substitutions = dict(args.substitution) if args.substitution else {}
# Fast path for `upload` and `logs`: the caller already has a binary
# on disk and only needs the CLI to ship bytes to a device or stream
# logs back. Re-running the full `read_config()` pipeline (parse +
# schema validate + final-validate + external component refresh) is
# dead work and produces a wall of "Reading configuration ..." log
# lines on every install. Reload the validated config the last
# compile cached alongside the StorageJSON sidecar; fall back to
# full validation if the cache is missing or older than the YAML so
# a cold cache never produces a worse outcome.
# Fast path for `upload` and `logs`: reuse the validated config the
# last compile cached on disk. Skips parse + schema validate +
# final-validate + external-component refresh, which is dead work
# for those subcommands -- the binary on disk is already known good
# and the caller already knows the device address. Falls back to a
# full `read_config()` when the cache is missing/stale/corrupt so a
# cold cache never produces a worse outcome than today.
config = None
if args.command in ("upload", "logs"):
from esphome.storage_json import (
StorageJSON,
ext_storage_path,
load_compiled_config,
)
from esphome.compiled_config import load_compiled_config
cached = load_compiled_config(conf_path)
if cached is not None:
storage = StorageJSON.load(ext_storage_path(conf_path.name))
if storage is not None:
storage.apply_to_core()
config = cached
_LOGGER.info(
"Loaded validated config cache for %s, skipping validation.",
conf_path.name,
)
config = load_compiled_config(conf_path)
if config is not None:
_LOGGER.info(
"Loaded validated config cache for %s, skipping validation.",
conf_path.name,
)
if config is None:
config = read_config(
+96
View File
@@ -0,0 +1,96 @@
"""Validated-config cache for the ``upload`` / ``logs`` fast path.
After every successful ``esphome compile``, the writer dumps the
validated config to ``<data_dir>/storage/<file>.validated.yaml``.
The next ``esphome upload`` / ``esphome logs`` for that YAML reuses
the cache instead of re-running the full ``read_config()`` pipeline
(parse + schema validate + final-validate + external-component
refresh), which is dead work once the binary on disk is known good.
The cache uses YAML (via ``yaml_util``) rather than JSON so all
esphome-specific tags (``!lambda``, ``!include``, ``ID`` instances,
paths) round-trip cleanly, and so the small ``StorageJSON`` sidecar
isn't bloated by configs that can grow past a megabyte. Staleness
is gated by an mtime check against the source YAML; the loader
falls back to ``None`` (and the caller to ``read_config()``) whenever
the cache is missing, stale, unparseable, or the companion
``StorageJSON`` sidecar can't be loaded.
"""
from __future__ import annotations
import logging
from pathlib import Path
from esphome.core import CORE
from esphome.helpers import write_file_if_changed
from esphome.storage_json import StorageJSON, ext_storage_path
from esphome.types import ConfigType
_LOGGER = logging.getLogger(__name__)
def compiled_config_path(config_filename: str) -> Path:
"""Path to the cached validated config alongside the storage sidecar."""
return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml"
def _cache_is_fresh(cache_path: Path, source_path: Path) -> bool:
"""True iff the cache file exists and isn't older than the source."""
try:
return cache_path.stat().st_mtime >= source_path.stat().st_mtime
except OSError:
return False
def save_compiled_config(config: ConfigType) -> None:
"""Dump the validated config to its sidecar YAML file.
Called from the writer after every successful compile so the next
``esphome upload`` / ``esphome logs`` for this YAML can skip
validation via :func:`load_compiled_config`. Failures here are
non-fatal: the worst case is that the fast path falls back to a
full ``read_config`` next time.
"""
from esphome import yaml_util
try:
# show_secrets=True keeps the cache self-contained; it lives
# in the same trust zone as the rest of .esphome/storage/.
rendered = yaml_util.dump(config, show_secrets=True)
write_file_if_changed(compiled_config_path(CORE.config_filename), rendered)
except Exception as err: # pylint: disable=broad-except
_LOGGER.debug("Skipping compiled config cache write: %s", err)
def load_compiled_config(conf_path: Path) -> ConfigType | None:
"""Load the cached validated config and apply storage metadata to CORE.
Single entry point for the ``upload`` / ``logs`` fast path: loads
the cache, applies the ``StorageJSON`` sidecar's platform / build
metadata to ``CORE``, and returns the config dict. Returns
``None`` (so the caller falls back to ``read_config``) when the
cache is missing, older than the source YAML, unparseable, or
when the sidecar can't be loaded. The mtime check catches the
common "user edited the YAML and forgot to recompile" case;
deeper drift (an edited ``!include`` whose parent YAML mtime
didn't change) is the user's responsibility.
"""
cache_path = compiled_config_path(conf_path.name)
if not _cache_is_fresh(cache_path, conf_path):
return None
from esphome import yaml_util
try:
# clear_secrets=False keeps in-flight secret state intact; the
# cache is self-contained and resolves no !secret references.
config = yaml_util.load_yaml(cache_path, clear_secrets=False)
except Exception: # pylint: disable=broad-except
return None
storage = StorageJSON.load(ext_storage_path(conf_path.name))
if storage is None:
return None
storage.apply_to_core()
return config
+1 -77
View File
@@ -11,7 +11,7 @@ from esphome import const
from esphome.const import CONF_DISABLED, CONF_MDNS
from esphome.core import CORE
from esphome.helpers import write_file_if_changed
from esphome.types import ConfigType, CoreType
from esphome.types import CoreType
_LOGGER = logging.getLogger(__name__)
@@ -56,82 +56,6 @@ def archive_storage_path() -> Path:
return CORE.relative_config_path("archive")
def compiled_config_path(config_filename: str) -> Path:
"""Path to the cached validated config alongside the storage sidecar.
Written after every successful compile from the validated config
dict (via ``yaml_util.dump``). Powers the dispatcher's
``--from-storage-json`` fast path in ``esphome upload`` and
``esphome logs`` so they can skip the full ``read_config()``
validation pipeline.
Lives next to the existing JSON sidecar (``<filename>.json``) but
in its own file so the small metadata sidecar isn't bloated by
configs that can reach a megabyte or more once packages and
substitutions are expanded.
"""
return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml"
def save_compiled_config(config: ConfigType) -> None:
"""Dump the validated config to its sidecar YAML file.
Called by the writer at the end of ``compile`` so the next call
to ``esphome upload --from-storage-json`` / ``esphome logs
--from-storage-json`` for this YAML can skip validation. Failures
here are non-fatal: the worst case is that the fast path falls
back to a full ``read_config`` next time.
"""
from esphome import yaml_util
try:
# show_secrets=True so the cache is self-contained; the file
# lives next to the binary it describes, in the same trust zone
# as the rest of .esphome/storage/.
rendered = yaml_util.dump(config, show_secrets=True)
except Exception as err: # pylint: disable=broad-except
_LOGGER.debug("Skipping compiled config cache write: %s", err)
return
try:
write_file_if_changed(compiled_config_path(CORE.config_filename), rendered)
except OSError as err:
_LOGGER.debug("Skipping compiled config cache write: %s", err)
def load_compiled_config(config_path: Path) -> ConfigType | None:
"""Load the cached validated config for ``--from-storage-json``.
Returns ``None`` (so the caller falls back to ``read_config``) if
the cache is missing or older than the source YAML. The mtime
check catches the common "user edited the YAML and forgot to
recompile" case; deeper drift (an edited ``!include`` whose
parent YAML mtime didn't change) is the user's responsibility —
this flag is opt-in and assumes the caller knows the binary on
disk matches the cache.
"""
cache_path = compiled_config_path(config_path.name)
try:
yaml_mtime = config_path.stat().st_mtime
except OSError:
return None
try:
cache_mtime = cache_path.stat().st_mtime
except OSError:
return None
if cache_mtime < yaml_mtime:
return None
from esphome import yaml_util
try:
# clear_secrets=False so we don't disturb any in-flight secret
# state; the cache is self-contained and resolves no !secret
# references.
return yaml_util.load_yaml(cache_path, clear_secrets=False)
except Exception: # pylint: disable=broad-except
return None
def _to_path_if_not_none(value: str | None) -> Path | None:
"""Convert a string to Path if it's not None."""
return Path(value) if value is not None else None
+2 -1
View File
@@ -7,6 +7,7 @@ import re
import time
from esphome import loader
from esphome.compiled_config import save_compiled_config
from esphome.config import iter_component_configs, iter_components
from esphome.const import (
HEADER_FILE_EXTENSIONS,
@@ -24,7 +25,7 @@ from esphome.helpers import (
walk_files,
write_file_if_changed,
)
from esphome.storage_json import StorageJSON, save_compiled_config, storage_path
from esphome.storage_json import StorageJSON, storage_path
_LOGGER = logging.getLogger(__name__)
@@ -1,4 +1,4 @@
"""Tests for the validated-config cache fast path used by upload/logs."""
"""Tests for the validated-config cache used by upload/logs."""
from __future__ import annotations
@@ -10,6 +10,7 @@ from unittest.mock import patch
import pytest
from esphome.__main__ import run_esphome
from esphome.compiled_config import compiled_config_path, load_compiled_config
from esphome.const import (
CONF_API,
CONF_ESPHOME,
@@ -19,7 +20,6 @@ from esphome.const import (
KEY_TARGET_PLATFORM,
)
from esphome.core import CORE
from esphome.storage_json import StorageJSON, compiled_config_path, load_compiled_config
_VALIDATED_CONFIG_YAML = """\
esphome:
@@ -67,36 +67,46 @@ def _write_storage(storage_path: Path) -> None:
storage_path.write_text(json.dumps(data))
def _write_cache(cache_path: Path, body: str = _VALIDATED_CONFIG_YAML) -> Path:
"""Write the cache file and return it."""
cache_path.parent.mkdir(parents=True, exist_ok=True)
cache_path.write_text(body)
return cache_path
def _set_cache_mtime(cache_path: Path, yaml_path: Path, *, offset: int) -> None:
"""Force the cache file's mtime relative to the source YAML.
Positive offset cache is fresh. Negative cache is stale.
"""
yaml_stat = yaml_path.stat()
os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + offset))
@pytest.fixture
def fresh_cache_files(tmp_path: Path) -> Path:
"""Set up a YAML + StorageJSON + validated-config cache.
Cache mtime is bumped 5s past the YAML so the staleness check
treats it as fresh.
"""
"""YAML + StorageJSON + cache, all consistent and fresh."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
storage_dir = tmp_path / ".esphome" / "storage"
_write_storage(storage_dir / "lite_test.yaml.json")
cache_path = storage_dir / "lite_test.yaml.validated.yaml"
cache_path.write_text(_VALIDATED_CONFIG_YAML)
yaml_stat = yaml_path.stat()
os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5))
cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
_set_cache_mtime(cache, yaml_path, offset=5)
return yaml_path
def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None:
"""The cache file shape is predictable from the YAML filename."""
path = compiled_config_path("device.yaml")
assert str(path).endswith("storage/device.yaml.validated.yaml")
assert str(compiled_config_path("device.yaml")).endswith(
"storage/device.yaml.validated.yaml"
)
def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None:
"""Fresh cache → returns the validated config dict."""
"""Fresh cache + sidecar → returns config and populates CORE."""
config = load_compiled_config(fresh_cache_files)
assert config is not None
@@ -104,78 +114,50 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None:
assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q="
assert config["ota"][0]["password"] == "secret"
def test_load_compiled_config_missing_cache(tmp_path: Path) -> None:
"""No cache file on disk → None so caller falls back."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
assert load_compiled_config(yaml_path) is None
def test_load_compiled_config_stale_cache(tmp_path: Path) -> None:
"""Cache older than the YAML → None (the YAML was edited)."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
storage_dir = tmp_path / ".esphome" / "storage"
storage_dir.mkdir(parents=True, exist_ok=True)
cache_path = storage_dir / "lite_test.yaml.validated.yaml"
cache_path.write_text(_VALIDATED_CONFIG_YAML)
yaml_stat = yaml_path.stat()
# Cache is one minute older than the YAML — the YAML's been edited
# since the last compile, so the cache no longer describes the
# binary on disk.
os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60))
assert load_compiled_config(yaml_path) is None
def test_load_compiled_config_corrupt_cache(tmp_path: Path) -> None:
"""Cache file is unparseable → None so caller falls back."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
storage_dir = tmp_path / ".esphome" / "storage"
storage_dir.mkdir(parents=True, exist_ok=True)
cache_path = storage_dir / "lite_test.yaml.validated.yaml"
cache_path.write_text("not: valid: yaml: [")
yaml_stat = yaml_path.stat()
os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5))
assert load_compiled_config(yaml_path) is None
def test_storage_json_apply_to_core_populates_target_platform(tmp_path: Path) -> None:
"""apply_to_core sets the CORE attributes upload / logs read."""
storage_path = tmp_path / "lite_test.yaml.json"
_write_storage(storage_path)
storage = StorageJSON.load(storage_path)
assert storage is not None
storage.apply_to_core()
# apply_to_core ran as part of the orchestration.
assert CORE.name == "lite_test"
assert CORE.friendly_name == "Lite Test Device"
assert CORE.build_path == Path("/build/lite_test")
assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32"
assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino"
assert "api" in CORE.loaded_integrations
@pytest.mark.parametrize(
"scenario",
["missing_cache", "stale_cache", "corrupt_cache", "missing_sidecar"],
)
def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None:
"""All non-happy cases return None so the caller falls back."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
storage_dir = tmp_path / ".esphome" / "storage"
cache_path = storage_dir / "lite_test.yaml.validated.yaml"
sidecar_path = storage_dir / "lite_test.yaml.json"
if scenario == "missing_cache":
pass # no cache, no sidecar
elif scenario == "stale_cache":
_write_storage(sidecar_path)
_set_cache_mtime(_write_cache(cache_path), yaml_path, offset=-60)
elif scenario == "corrupt_cache":
_write_storage(sidecar_path)
_set_cache_mtime(
_write_cache(cache_path, "not: valid: yaml: ["), yaml_path, offset=5
)
elif scenario == "missing_sidecar":
# Cache fresh + parseable, but no StorageJSON → can't populate CORE.
_set_cache_mtime(_write_cache(cache_path), yaml_path, offset=5)
assert load_compiled_config(yaml_path) is None
@pytest.mark.parametrize("command", ["upload", "logs"])
def test_run_esphome_upload_and_logs_use_cache_when_fresh(
command: str, fresh_cache_files: Path
) -> None:
"""When the cache is fresh, upload/logs skip read_config() entirely."""
yaml_path = fresh_cache_files
captured = {}
"""upload/logs skip read_config() when the cache is fresh."""
captured: dict = {}
def _stub(_args, config):
captured["config"] = config
@@ -185,11 +167,9 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh(
patch("esphome.__main__.read_config") as mock_read,
patch.dict("esphome.__main__.POST_CONFIG_ACTIONS", {command: _stub}),
):
result = run_esphome(["esphome", command, str(yaml_path)])
assert run_esphome(["esphome", command, str(fresh_cache_files)]) == 0
mock_read.assert_not_called()
assert result == 0
# The dispatcher hands the cached config dict through unchanged.
assert captured["config"][CONF_ESPHOME][CONF_NAME] == "lite_test"
assert captured["config"][CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q="
@@ -198,7 +178,7 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh(
def test_run_esphome_upload_and_logs_fall_back_when_no_cache(
tmp_path: Path, command: str
) -> None:
"""With no cache on disk, the dispatcher falls back to read_config()."""
"""Without a cache, the dispatcher falls back to read_config()."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
@@ -209,46 +189,13 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache(
{command: lambda args, config: 0},
),
):
result = run_esphome(["esphome", command, str(yaml_path)])
mock_read.assert_called_once()
assert result == 2
def test_run_esphome_upload_falls_back_when_cache_stale(tmp_path: Path) -> None:
"""If YAML mtime > cache mtime, dispatcher falls back to read_config()."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
storage_dir = tmp_path / ".esphome" / "storage"
_write_storage(storage_dir / "lite_test.yaml.json")
cache_path = storage_dir / "lite_test.yaml.validated.yaml"
cache_path.write_text(_VALIDATED_CONFIG_YAML)
yaml_stat = yaml_path.stat()
os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60))
with (
patch("esphome.__main__.read_config", return_value=None) as mock_read,
patch.dict(
"esphome.__main__.POST_CONFIG_ACTIONS",
{"upload": lambda args, config: 0},
),
):
run_esphome(["esphome", "upload", str(yaml_path)])
assert run_esphome(["esphome", command, str(yaml_path)]) == 2
mock_read.assert_called_once()
def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None:
"""`compile` always re-validates, even with a fresh cache on disk.
The fast path is only for upload / logs -- compile is what writes
the cache in the first place, and it needs a fully validated
config to drive code generation.
"""
yaml_path = fresh_cache_files
"""compile always re-validates -- it's what writes the cache."""
with (
patch("esphome.__main__.read_config", return_value=None) as mock_read,
patch.dict(
@@ -256,6 +203,6 @@ def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None
{"compile": lambda args, config: 0},
),
):
run_esphome(["esphome", "compile", str(yaml_path)])
run_esphome(["esphome", "compile", str(fresh_cache_files)])
mock_read.assert_called_once()