diff --git a/esphome/__main__.py b/esphome/__main__.py index 435931cd28..d68dcffaf2 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2419,33 +2419,23 @@ def run_esphome(argv): skip_external = args.command in ("logs", "clean") command_line_substitutions = dict(args.substitution) if args.substitution else {} - # Fast path for `upload` and `logs`: the caller already has a binary - # on disk and only needs the CLI to ship bytes to a device or stream - # logs back. Re-running the full `read_config()` pipeline (parse + - # schema validate + final-validate + external component refresh) is - # dead work and produces a wall of "Reading configuration ..." log - # lines on every install. Reload the validated config the last - # compile cached alongside the StorageJSON sidecar; fall back to - # full validation if the cache is missing or older than the YAML so - # a cold cache never produces a worse outcome. + # Fast path for `upload` and `logs`: reuse the validated config the + # last compile cached on disk. Skips parse + schema validate + + # final-validate + external-component refresh, which is dead work + # for those subcommands -- the binary on disk is already known good + # and the caller already knows the device address. Falls back to a + # full `read_config()` when the cache is missing/stale/corrupt so a + # cold cache never produces a worse outcome than today. config = None if args.command in ("upload", "logs"): - from esphome.storage_json import ( - StorageJSON, - ext_storage_path, - load_compiled_config, - ) + from esphome.compiled_config import load_compiled_config - cached = load_compiled_config(conf_path) - if cached is not None: - storage = StorageJSON.load(ext_storage_path(conf_path.name)) - if storage is not None: - storage.apply_to_core() - config = cached - _LOGGER.info( - "Loaded validated config cache for %s, skipping validation.", - conf_path.name, - ) + config = load_compiled_config(conf_path) + if config is not None: + _LOGGER.info( + "Loaded validated config cache for %s, skipping validation.", + conf_path.name, + ) if config is None: config = read_config( diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py new file mode 100644 index 0000000000..bb127a0de9 --- /dev/null +++ b/esphome/compiled_config.py @@ -0,0 +1,96 @@ +"""Validated-config cache for the ``upload`` / ``logs`` fast path. + +After every successful ``esphome compile``, the writer dumps the +validated config to ``/storage/.validated.yaml``. +The next ``esphome upload`` / ``esphome logs`` for that YAML reuses +the cache instead of re-running the full ``read_config()`` pipeline +(parse + schema validate + final-validate + external-component +refresh), which is dead work once the binary on disk is known good. + +The cache uses YAML (via ``yaml_util``) rather than JSON so all +esphome-specific tags (``!lambda``, ``!include``, ``ID`` instances, +paths) round-trip cleanly, and so the small ``StorageJSON`` sidecar +isn't bloated by configs that can grow past a megabyte. Staleness +is gated by an mtime check against the source YAML; the loader +falls back to ``None`` (and the caller to ``read_config()``) whenever +the cache is missing, stale, unparseable, or the companion +``StorageJSON`` sidecar can't be loaded. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from esphome.core import CORE +from esphome.helpers import write_file_if_changed +from esphome.storage_json import StorageJSON, ext_storage_path +from esphome.types import ConfigType + +_LOGGER = logging.getLogger(__name__) + + +def compiled_config_path(config_filename: str) -> Path: + """Path to the cached validated config alongside the storage sidecar.""" + return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml" + + +def _cache_is_fresh(cache_path: Path, source_path: Path) -> bool: + """True iff the cache file exists and isn't older than the source.""" + try: + return cache_path.stat().st_mtime >= source_path.stat().st_mtime + except OSError: + return False + + +def save_compiled_config(config: ConfigType) -> None: + """Dump the validated config to its sidecar YAML file. + + Called from the writer after every successful compile so the next + ``esphome upload`` / ``esphome logs`` for this YAML can skip + validation via :func:`load_compiled_config`. Failures here are + non-fatal: the worst case is that the fast path falls back to a + full ``read_config`` next time. + """ + from esphome import yaml_util + + try: + # show_secrets=True keeps the cache self-contained; it lives + # in the same trust zone as the rest of .esphome/storage/. + rendered = yaml_util.dump(config, show_secrets=True) + write_file_if_changed(compiled_config_path(CORE.config_filename), rendered) + except Exception as err: # pylint: disable=broad-except + _LOGGER.debug("Skipping compiled config cache write: %s", err) + + +def load_compiled_config(conf_path: Path) -> ConfigType | None: + """Load the cached validated config and apply storage metadata to CORE. + + Single entry point for the ``upload`` / ``logs`` fast path: loads + the cache, applies the ``StorageJSON`` sidecar's platform / build + metadata to ``CORE``, and returns the config dict. Returns + ``None`` (so the caller falls back to ``read_config``) when the + cache is missing, older than the source YAML, unparseable, or + when the sidecar can't be loaded. The mtime check catches the + common "user edited the YAML and forgot to recompile" case; + deeper drift (an edited ``!include`` whose parent YAML mtime + didn't change) is the user's responsibility. + """ + cache_path = compiled_config_path(conf_path.name) + if not _cache_is_fresh(cache_path, conf_path): + return None + + from esphome import yaml_util + + try: + # clear_secrets=False keeps in-flight secret state intact; the + # cache is self-contained and resolves no !secret references. + config = yaml_util.load_yaml(cache_path, clear_secrets=False) + except Exception: # pylint: disable=broad-except + return None + + storage = StorageJSON.load(ext_storage_path(conf_path.name)) + if storage is None: + return None + storage.apply_to_core() + return config diff --git a/esphome/storage_json.py b/esphome/storage_json.py index d760297150..a9a83f5896 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -11,7 +11,7 @@ from esphome import const from esphome.const import CONF_DISABLED, CONF_MDNS from esphome.core import CORE from esphome.helpers import write_file_if_changed -from esphome.types import ConfigType, CoreType +from esphome.types import CoreType _LOGGER = logging.getLogger(__name__) @@ -56,82 +56,6 @@ def archive_storage_path() -> Path: return CORE.relative_config_path("archive") -def compiled_config_path(config_filename: str) -> Path: - """Path to the cached validated config alongside the storage sidecar. - - Written after every successful compile from the validated config - dict (via ``yaml_util.dump``). Powers the dispatcher's - ``--from-storage-json`` fast path in ``esphome upload`` and - ``esphome logs`` so they can skip the full ``read_config()`` - validation pipeline. - - Lives next to the existing JSON sidecar (``.json``) but - in its own file so the small metadata sidecar isn't bloated by - configs that can reach a megabyte or more once packages and - substitutions are expanded. - """ - return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml" - - -def save_compiled_config(config: ConfigType) -> None: - """Dump the validated config to its sidecar YAML file. - - Called by the writer at the end of ``compile`` so the next call - to ``esphome upload --from-storage-json`` / ``esphome logs - --from-storage-json`` for this YAML can skip validation. Failures - here are non-fatal: the worst case is that the fast path falls - back to a full ``read_config`` next time. - """ - from esphome import yaml_util - - try: - # show_secrets=True so the cache is self-contained; the file - # lives next to the binary it describes, in the same trust zone - # as the rest of .esphome/storage/. - rendered = yaml_util.dump(config, show_secrets=True) - except Exception as err: # pylint: disable=broad-except - _LOGGER.debug("Skipping compiled config cache write: %s", err) - return - try: - write_file_if_changed(compiled_config_path(CORE.config_filename), rendered) - except OSError as err: - _LOGGER.debug("Skipping compiled config cache write: %s", err) - - -def load_compiled_config(config_path: Path) -> ConfigType | None: - """Load the cached validated config for ``--from-storage-json``. - - Returns ``None`` (so the caller falls back to ``read_config``) if - the cache is missing or older than the source YAML. The mtime - check catches the common "user edited the YAML and forgot to - recompile" case; deeper drift (an edited ``!include`` whose - parent YAML mtime didn't change) is the user's responsibility — - this flag is opt-in and assumes the caller knows the binary on - disk matches the cache. - """ - cache_path = compiled_config_path(config_path.name) - try: - yaml_mtime = config_path.stat().st_mtime - except OSError: - return None - try: - cache_mtime = cache_path.stat().st_mtime - except OSError: - return None - if cache_mtime < yaml_mtime: - return None - - from esphome import yaml_util - - try: - # clear_secrets=False so we don't disturb any in-flight secret - # state; the cache is self-contained and resolves no !secret - # references. - return yaml_util.load_yaml(cache_path, clear_secrets=False) - except Exception: # pylint: disable=broad-except - return None - - def _to_path_if_not_none(value: str | None) -> Path | None: """Convert a string to Path if it's not None.""" return Path(value) if value is not None else None diff --git a/esphome/writer.py b/esphome/writer.py index 8efefe68ba..54eb254b1b 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -7,6 +7,7 @@ import re import time from esphome import loader +from esphome.compiled_config import save_compiled_config from esphome.config import iter_component_configs, iter_components from esphome.const import ( HEADER_FILE_EXTENSIONS, @@ -24,7 +25,7 @@ from esphome.helpers import ( walk_files, write_file_if_changed, ) -from esphome.storage_json import StorageJSON, save_compiled_config, storage_path +from esphome.storage_json import StorageJSON, storage_path _LOGGER = logging.getLogger(__name__) diff --git a/tests/unit_tests/test_main_validated_config_cache.py b/tests/unit_tests/test_compiled_config.py similarity index 51% rename from tests/unit_tests/test_main_validated_config_cache.py rename to tests/unit_tests/test_compiled_config.py index a5ecbe59c9..d4736f38a3 100644 --- a/tests/unit_tests/test_main_validated_config_cache.py +++ b/tests/unit_tests/test_compiled_config.py @@ -1,4 +1,4 @@ -"""Tests for the validated-config cache fast path used by upload/logs.""" +"""Tests for the validated-config cache used by upload/logs.""" from __future__ import annotations @@ -10,6 +10,7 @@ from unittest.mock import patch import pytest from esphome.__main__ import run_esphome +from esphome.compiled_config import compiled_config_path, load_compiled_config from esphome.const import ( CONF_API, CONF_ESPHOME, @@ -19,7 +20,6 @@ from esphome.const import ( KEY_TARGET_PLATFORM, ) from esphome.core import CORE -from esphome.storage_json import StorageJSON, compiled_config_path, load_compiled_config _VALIDATED_CONFIG_YAML = """\ esphome: @@ -67,36 +67,46 @@ def _write_storage(storage_path: Path) -> None: storage_path.write_text(json.dumps(data)) +def _write_cache(cache_path: Path, body: str = _VALIDATED_CONFIG_YAML) -> Path: + """Write the cache file and return it.""" + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(body) + return cache_path + + +def _set_cache_mtime(cache_path: Path, yaml_path: Path, *, offset: int) -> None: + """Force the cache file's mtime relative to the source YAML. + + Positive offset → cache is fresh. Negative → cache is stale. + """ + yaml_stat = yaml_path.stat() + os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + offset)) + + @pytest.fixture def fresh_cache_files(tmp_path: Path) -> Path: - """Set up a YAML + StorageJSON + validated-config cache. - - Cache mtime is bumped 5s past the YAML so the staleness check - treats it as fresh. - """ + """YAML + StorageJSON + cache, all consistent and fresh.""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path storage_dir = tmp_path / ".esphome" / "storage" _write_storage(storage_dir / "lite_test.yaml.json") - - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text(_VALIDATED_CONFIG_YAML) - yaml_stat = yaml_path.stat() - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache, yaml_path, offset=5) return yaml_path def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None: """The cache file shape is predictable from the YAML filename.""" - path = compiled_config_path("device.yaml") - assert str(path).endswith("storage/device.yaml.validated.yaml") + assert str(compiled_config_path("device.yaml")).endswith( + "storage/device.yaml.validated.yaml" + ) def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: - """Fresh cache → returns the validated config dict.""" + """Fresh cache + sidecar → returns config and populates CORE.""" config = load_compiled_config(fresh_cache_files) assert config is not None @@ -104,78 +114,50 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" assert config["ota"][0]["password"] == "secret" - -def test_load_compiled_config_missing_cache(tmp_path: Path) -> None: - """No cache file on disk → None so caller falls back.""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - CORE.config_path = yaml_path - - assert load_compiled_config(yaml_path) is None - - -def test_load_compiled_config_stale_cache(tmp_path: Path) -> None: - """Cache older than the YAML → None (the YAML was edited).""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - CORE.config_path = yaml_path - - storage_dir = tmp_path / ".esphome" / "storage" - storage_dir.mkdir(parents=True, exist_ok=True) - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text(_VALIDATED_CONFIG_YAML) - - yaml_stat = yaml_path.stat() - # Cache is one minute older than the YAML — the YAML's been edited - # since the last compile, so the cache no longer describes the - # binary on disk. - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) - - assert load_compiled_config(yaml_path) is None - - -def test_load_compiled_config_corrupt_cache(tmp_path: Path) -> None: - """Cache file is unparseable → None so caller falls back.""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - CORE.config_path = yaml_path - - storage_dir = tmp_path / ".esphome" / "storage" - storage_dir.mkdir(parents=True, exist_ok=True) - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text("not: valid: yaml: [") - - yaml_stat = yaml_path.stat() - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) - - assert load_compiled_config(yaml_path) is None - - -def test_storage_json_apply_to_core_populates_target_platform(tmp_path: Path) -> None: - """apply_to_core sets the CORE attributes upload / logs read.""" - storage_path = tmp_path / "lite_test.yaml.json" - _write_storage(storage_path) - storage = StorageJSON.load(storage_path) - assert storage is not None - - storage.apply_to_core() - + # apply_to_core ran as part of the orchestration. assert CORE.name == "lite_test" - assert CORE.friendly_name == "Lite Test Device" assert CORE.build_path == Path("/build/lite_test") assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" assert "api" in CORE.loaded_integrations +@pytest.mark.parametrize( + "scenario", + ["missing_cache", "stale_cache", "corrupt_cache", "missing_sidecar"], +) +def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None: + """All non-happy cases return None so the caller falls back.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + storage_dir = tmp_path / ".esphome" / "storage" + cache_path = storage_dir / "lite_test.yaml.validated.yaml" + sidecar_path = storage_dir / "lite_test.yaml.json" + + if scenario == "missing_cache": + pass # no cache, no sidecar + elif scenario == "stale_cache": + _write_storage(sidecar_path) + _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=-60) + elif scenario == "corrupt_cache": + _write_storage(sidecar_path) + _set_cache_mtime( + _write_cache(cache_path, "not: valid: yaml: ["), yaml_path, offset=5 + ) + elif scenario == "missing_sidecar": + # Cache fresh + parseable, but no StorageJSON → can't populate CORE. + _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=5) + + assert load_compiled_config(yaml_path) is None + + @pytest.mark.parametrize("command", ["upload", "logs"]) def test_run_esphome_upload_and_logs_use_cache_when_fresh( command: str, fresh_cache_files: Path ) -> None: - """When the cache is fresh, upload/logs skip read_config() entirely.""" - yaml_path = fresh_cache_files - - captured = {} + """upload/logs skip read_config() when the cache is fresh.""" + captured: dict = {} def _stub(_args, config): captured["config"] = config @@ -185,11 +167,9 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh( patch("esphome.__main__.read_config") as mock_read, patch.dict("esphome.__main__.POST_CONFIG_ACTIONS", {command: _stub}), ): - result = run_esphome(["esphome", command, str(yaml_path)]) + assert run_esphome(["esphome", command, str(fresh_cache_files)]) == 0 mock_read.assert_not_called() - assert result == 0 - # The dispatcher hands the cached config dict through unchanged. assert captured["config"][CONF_ESPHOME][CONF_NAME] == "lite_test" assert captured["config"][CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" @@ -198,7 +178,7 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh( def test_run_esphome_upload_and_logs_fall_back_when_no_cache( tmp_path: Path, command: str ) -> None: - """With no cache on disk, the dispatcher falls back to read_config().""" + """Without a cache, the dispatcher falls back to read_config().""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") @@ -209,46 +189,13 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache( {command: lambda args, config: 0}, ), ): - result = run_esphome(["esphome", command, str(yaml_path)]) - - mock_read.assert_called_once() - assert result == 2 - - -def test_run_esphome_upload_falls_back_when_cache_stale(tmp_path: Path) -> None: - """If YAML mtime > cache mtime, dispatcher falls back to read_config().""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - - storage_dir = tmp_path / ".esphome" / "storage" - _write_storage(storage_dir / "lite_test.yaml.json") - - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text(_VALIDATED_CONFIG_YAML) - yaml_stat = yaml_path.stat() - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) - - with ( - patch("esphome.__main__.read_config", return_value=None) as mock_read, - patch.dict( - "esphome.__main__.POST_CONFIG_ACTIONS", - {"upload": lambda args, config: 0}, - ), - ): - run_esphome(["esphome", "upload", str(yaml_path)]) + assert run_esphome(["esphome", command, str(yaml_path)]) == 2 mock_read.assert_called_once() def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None: - """`compile` always re-validates, even with a fresh cache on disk. - - The fast path is only for upload / logs -- compile is what writes - the cache in the first place, and it needs a fully validated - config to drive code generation. - """ - yaml_path = fresh_cache_files - + """compile always re-validates -- it's what writes the cache.""" with ( patch("esphome.__main__.read_config", return_value=None) as mock_read, patch.dict( @@ -256,6 +203,6 @@ def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None {"compile": lambda args, config: 0}, ), ): - run_esphome(["esphome", "compile", str(yaml_path)]) + run_esphome(["esphome", "compile", str(fresh_cache_files)]) mock_read.assert_called_once()