From 76ce45c59ef453be351dca454ee4ac50b1b5783e Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 13 May 2026 06:28:39 +1200 Subject: [PATCH 01/16] [script] Preserve source order of enum options in language schema (#16371) --- script/build_language_schema.py | 10 ++- tests/script/test_build_language_schema.py | 98 ++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 tests/script/test_build_language_schema.py diff --git a/script/build_language_schema.py b/script/build_language_schema.py index a7142fa8b5e..921ee9d3d70 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -119,7 +119,15 @@ from esphome.util import Registry # noqa: E402 def sort_obj(obj): if isinstance(obj, dict): - return {k: sort_obj(v) for k, v in sorted(obj.items(), key=lambda x: str(x[0]))} + is_enum = obj.get(S_TYPE) == "enum" + result = {} + for k, v in sorted(obj.items(), key=lambda x: str(x[0])): + if is_enum and k == "values" and isinstance(v, dict): + # Preserve source order of enum options + result[k] = {vk: sort_obj(vv) for vk, vv in v.items()} + else: + result[k] = sort_obj(v) + return result if isinstance(obj, list): return [sort_obj(item) for item in obj] return obj diff --git a/tests/script/test_build_language_schema.py b/tests/script/test_build_language_schema.py new file mode 100644 index 00000000000..59b8c7484b2 --- /dev/null +++ b/tests/script/test_build_language_schema.py @@ -0,0 +1,98 @@ +"""Unit tests for script/build_language_schema.py.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SCRIPT_PATH = ( + Path(__file__).resolve().parent.parent.parent + / "script" + / "build_language_schema.py" +) + + +def _extract_sort_obj(): + # build_language_schema.py runs argparse, loads every component, and + # calls build_schema() at import time, so a plain import isn't viable + # in a unit test. Pull just the pure helper out via AST instead. + tree = ast.parse(SCRIPT_PATH.read_text()) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "sort_obj": + namespace: dict = {"S_TYPE": "type"} + module = ast.Module(body=[node], type_ignores=[]) + exec(compile(module, str(SCRIPT_PATH), "exec"), namespace) + return namespace["sort_obj"] + raise AssertionError("sort_obj not found in build_language_schema.py") + + +sort_obj = _extract_sort_obj() + + +def test_sort_obj_sorts_dict_keys() -> None: + result = sort_obj({"b": 1, "a": 2, "c": 3}) + assert list(result.keys()) == ["a", "b", "c"] + + +def test_sort_obj_sorts_nested_dicts() -> None: + result = sort_obj({"outer": {"z": 1, "a": 2}}) + assert list(result["outer"].keys()) == ["a", "z"] + + +def test_sort_obj_preserves_enum_values_order() -> None: + config = { + "type": "enum", + "values": { + "2MB": None, + "4MB": None, + "8MB": None, + "16MB": None, + "32MB": None, + }, + } + result = sort_obj(config) + assert list(result["values"].keys()) == ["2MB", "4MB", "8MB", "16MB", "32MB"] + + +def test_sort_obj_sorts_non_enum_values_key() -> None: + config = {"type": "schema", "values": {"z": 1, "a": 2}} + result = sort_obj(config) + assert list(result["values"].keys()) == ["a", "z"] + + +def test_sort_obj_sorts_other_keys_in_enum() -> None: + config = { + "type": "enum", + "default": "4MB", + "key": "Optional", + "values": {"2MB": None, "4MB": None}, + } + result = sort_obj(config) + assert list(result.keys()) == ["default", "key", "type", "values"] + assert list(result["values"].keys()) == ["2MB", "4MB"] + + +def test_sort_obj_recurses_into_enum_value_entries() -> None: + config = { + "type": "enum", + "values": { + "esp32": {"name": "ESP32", "docs": "Original"}, + "esp32-c3": {"name": "ESP32-C3", "docs": "RISC-V"}, + }, + } + result = sort_obj(config) + assert list(result["values"].keys()) == ["esp32", "esp32-c3"] + assert list(result["values"]["esp32"].keys()) == ["docs", "name"] + + +def test_sort_obj_handles_lists() -> None: + result = sort_obj([{"b": 1, "a": 2}, {"d": 3, "c": 4}]) + assert list(result[0].keys()) == ["a", "b"] + assert list(result[1].keys()) == ["c", "d"] + + +def test_sort_obj_passes_through_scalars() -> None: + assert sort_obj("hello") == "hello" + assert sort_obj(42) == 42 + assert sort_obj(None) is None + assert sort_obj(True) is True From c511dddf2a3ccaf26068fe0d9db043f2c326a225 Mon Sep 17 00:00:00 2001 From: tomaszduda23 Date: Tue, 12 May 2026 20:59:54 +0200 Subject: [PATCH 02/16] [core] allow defining run_compile in external_components (#16179) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/__main__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 988f28a55fe..561391708ee 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -737,7 +737,11 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: # If you change this format, update the regex in that script as well _LOGGER.info("Compiling app... Build path: %s", CORE.build_path) - if CORE.using_toolchain_esp_idf: + module = importlib.import_module("esphome.components." + CORE.target_platform) + platform_run_compile = getattr(module, "run_compile", None) + if platform_run_compile is not None and platform_run_compile(args, config): + pass + elif CORE.using_toolchain_esp_idf: from esphome.espidf import toolchain rc = toolchain.run_compile(config, CORE.verbose) From 57893a8eb1c64e0f58f54aedc3e475f1ddec8888 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 19:37:31 +0000 Subject: [PATCH 03/16] Bump aioesphomeapi from 44.23.0 to 44.24.2 (#16376) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dd539d4f867..c0350518900 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 -aioesphomeapi==44.23.0 +aioesphomeapi==44.24.2 zeroconf==0.148.0 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import From 34f69e0d6ef69fe27892e1f934a2e7859fd20622 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 14:42:23 -0500 Subject: [PATCH 04/16] [ci] Comment on PRs that touch the legacy dashboard (#16378) --- .../dashboard-deprecation-comment.yml | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 .github/workflows/dashboard-deprecation-comment.yml diff --git a/.github/workflows/dashboard-deprecation-comment.yml b/.github/workflows/dashboard-deprecation-comment.yml new file mode 100644 index 00000000000..e15c61df5ea --- /dev/null +++ b/.github/workflows/dashboard-deprecation-comment.yml @@ -0,0 +1,113 @@ +name: Add Dashboard Deprecation Comment + +on: + pull_request_target: + types: [opened, synchronize] + +# All API calls (pulls.listFiles + issues.{list,create,update}Comment) are performed with +# the App token minted below, so the workflow's GITHUB_TOKEN does not need any scopes. +permissions: {} + +jobs: + dashboard-deprecation-comment: + name: Dashboard deprecation comment + runs-on: ubuntu-latest + steps: + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} + # pulls.listFiles + issues.{list,create,update}Comment on PRs. For PR resources + # the issues.*Comment APIs require the pull-requests scope, not issues. + permission-pull-requests: write + + - name: Add dashboard deprecation comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ steps.generate-token.outputs.token }} + script: | + const commentMarker = ""; + + const commentBody = `Thanks for opening this PR! + + Heads up: the legacy ESPHome dashboard (\`esphome/dashboard/\` and \`tests/dashboard/\`) is **deprecated** and is being replaced by [ESPHome Device Builder](https://github.com/esphome/device-builder). We are not adding new features to the legacy dashboard and it will eventually be removed from this repository. + + What this means for your PR: + + - **New features / enhancements**: please port the change to [esphome/device-builder](https://github.com/esphome/device-builder) instead. We are unlikely to review or merge new dashboard features here. + - **Bug fixes**: small fixes may still be considered, but please check first whether the same issue exists in Device Builder, where the fix will have a longer life. + - **Security issues**: please do not file a public PR. Report privately via [GitHub security advisories](https://github.com/esphome/esphome/security/advisories/new) so we can coordinate a fix. + + We appreciate the contribution and apologize for the friction; flagging this early so your time isn't spent on a change that may not land. + + --- + (Added by the PR bot) + + ${commentMarker}`; + + async function getDashboardChanges(github, owner, repo, prNumber) { + const changedFiles = await github.paginate( + github.rest.pulls.listFiles, + { + owner: owner, + repo: repo, + pull_number: prNumber, + per_page: 100, + } + ); + + return changedFiles.filter(file => + file.filename.startsWith('esphome/dashboard/') || + file.filename.startsWith('tests/dashboard/') + ); + } + + async function findBotComment(github, owner, repo, prNumber) { + const comments = await github.paginate( + github.rest.issues.listComments, + { + owner: owner, + repo: repo, + issue_number: prNumber, + per_page: 100, + } + ); + + return comments.find(comment => + comment.body.includes(commentMarker) && comment.user.type === "Bot" + ); + } + + const prNumber = context.payload.pull_request.number; + const { owner, repo } = context.repo; + + const dashboardChanges = await getDashboardChanges(github, owner, repo, prNumber); + const existingComment = await findBotComment(github, owner, repo, prNumber); + + if (dashboardChanges.length === 0) { + // PR doesn't (or no longer) touches the legacy dashboard. If we previously + // commented (e.g. files were removed in a later push), leave the comment in + // place for history rather than thrash on edit/delete. + return; + } + + if (existingComment) { + if (existingComment.body === commentBody) { + return; + } + await github.rest.issues.updateComment({ + owner: owner, + repo: repo, + comment_id: existingComment.id, + body: commentBody, + }); + } else { + await github.rest.issues.createComment({ + owner: owner, + repo: repo, + issue_number: prNumber, + body: commentBody, + }); + } From eeaa8dee7b5da95f54a51c92cf4b11cf1157d835 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 15:01:37 -0500 Subject: [PATCH 05/16] [core] Add --from-storage-json flag to upload and logs When a downstream caller (typically a dashboard) already has a known-good firmware binary on disk and only needs the CLI to ship bytes to a device or stream logs back, running the full ``read_config()`` pipeline for every subcommand is dead work. For the device-builder REMOTE install flow that runs compile, upload, and logs back-to-back this means three "Reading configuration ..." passes for one user action. Add an opt-in ``--from-storage-json`` flag to ``upload`` and ``logs`` that sources platform / build metadata from the StorageJSON sidecar produced by the last successful compile and re-parses just enough of the YAML head (substitutions + packages, no schema validation) to recover the ``api:`` / ``logger:`` / ``ota:`` / network blocks the subcommands consult. The flag is opt-in and falls back to a full ``read_config()`` pass when the sidecar is missing or older than the YAML, so a cold cache never produces a worse outcome than today. ``compile`` / ``run`` / ``clean`` / ``bundle`` continue to validate as before. --- esphome/__main__.py | 57 ++++- esphome/lite_config.py | 197 +++++++++++++++ tests/unit_tests/test_main_lite_config.py | 280 ++++++++++++++++++++++ 3 files changed, 530 insertions(+), 4 deletions(-) create mode 100644 esphome/lite_config.py create mode 100644 tests/unit_tests/test_main_lite_config.py diff --git a/esphome/__main__.py b/esphome/__main__.py index 561391708ee..33f17276969 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2117,6 +2117,16 @@ def parse_args(argv): help="Upload as bootloader (OTA).", action="store_true", ) + parser_upload.add_argument( + "--from-storage-json", + action="store_true", + help=( + "Skip YAML schema validation by loading device metadata from " + "the .esphome/storage/.json sidecar produced by the last " + "successful compile. Falls back to a full validation pass when " + "the sidecar is missing or older than the YAML." + ), + ) parser_logs = subparsers.add_parser( "logs", @@ -2144,6 +2154,16 @@ def parse_args(argv): action="store_true", help="Do not show entity state changes in log output.", ) + parser_logs.add_argument( + "--from-storage-json", + action="store_true", + help=( + "Skip YAML schema validation by loading device metadata from " + "the .esphome/storage/.json sidecar produced by the last " + "successful compile. Falls back to a full validation pass when " + "the sidecar is missing or older than the YAML." + ), + ) parser_discover = subparsers.add_parser( "discover", @@ -2417,10 +2437,39 @@ def run_esphome(argv): # Commands that don't need fresh external components: logs just connects # to the device, and clean is about to delete the build directory. skip_external = args.command in ("logs", "clean") - config = read_config( - dict(args.substitution) if args.substitution else {}, - skip_external_update=skip_external, - ) + command_line_substitutions = dict(args.substitution) if args.substitution else {} + + # Fast path for `upload --from-storage-json` and `logs --from-storage-json`: + # the caller already has a binary on disk and only needs the CLI to ship + # bytes to a device or stream logs back. Re-running the full + # `read_config()` pipeline (parse + schema validate + final-validate + + # external component refresh) for those two subcommands is dead work and + # produces a wall of "Reading configuration ..." log lines for every + # remote install. Source the platform / build metadata from the + # StorageJSON sidecar instead; fall back to full validation if it's + # missing or stale so a cold cache never produces a worse outcome. + config = None + if getattr(args, "from_storage_json", False) and args.command in ( + "upload", + "logs", + ): + from esphome.lite_config import load_lite_config_from_storage + + config = load_lite_config_from_storage(conf_path, command_line_substitutions) + if config is None: + _LOGGER.warning( + "StorageJSON sidecar missing or stale for %s; falling back to " + "full config validation.", + conf_path, + ) + else: + _LOGGER.info("Loaded device metadata from StorageJSON sidecar.") + + if config is None: + config = read_config( + command_line_substitutions, + skip_external_update=skip_external, + ) if config is None: return 2 CORE.config = config diff --git a/esphome/lite_config.py b/esphome/lite_config.py new file mode 100644 index 00000000000..89816a00c71 --- /dev/null +++ b/esphome/lite_config.py @@ -0,0 +1,197 @@ +"""Lite-config loader for ``upload`` / ``logs`` fast paths. + +When the caller (typically a dashboard) already has a known-good +firmware binary on disk and only needs the CLI to ship bytes to a +device or stream logs back, running the full ``read_config()`` +pipeline is dead work. It re-parses every ``!include``, runs every +component's schema validator, executes all final-validate hooks, and +fetches/refreshes external components -- producing a fully validated +``Config`` object that the upload / logs subcommands only consult for a +handful of leaf keys. + +This module loads the per-config ``StorageJSON`` sidecar that was +written by the last successful ``compile`` run, populates ``CORE`` with +the platform / build metadata from it, and re-parses just enough of the +YAML head (substitutions + packages, no schema validation) to recover +``api:`` / ``logger:`` / ``ota:`` / ``mqtt:`` / network blocks for the +subcommand callers. + +If the sidecar is missing or older than the YAML, this returns ``None`` +so the dispatcher can fall back to the full ``read_config()`` path. +That makes the flag a pure optimisation: a cold cache never produces a +worse outcome than today. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from esphome.const import ( + CONF_API, + CONF_ESPHOME, + CONF_ETHERNET, + CONF_FRIENDLY_NAME, + CONF_LOGGER, + CONF_MQTT, + CONF_NAME, + CONF_OPENTHREAD, + CONF_OTA, + CONF_USE_ADDRESS, + CONF_WEB_SERVER, + CONF_WIFI, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, +) +from esphome.core import CORE +from esphome.storage_json import StorageJSON, ext_storage_path +from esphome.types import ConfigType + +_LOGGER = logging.getLogger(__name__) + +# Top-level keys we copy from the raw YAML into the lite config dict. +# Anything outside this list is irrelevant to ``upload`` / ``logs``. +_LITE_TOP_LEVEL_KEYS: tuple[str, ...] = ( + CONF_ESPHOME, + CONF_API, + CONF_LOGGER, + CONF_OTA, + CONF_MQTT, + CONF_WIFI, + CONF_ETHERNET, + CONF_OPENTHREAD, + CONF_WEB_SERVER, +) + + +def _parse_yaml_head( + conf_path: Path, command_line_substitutions: dict[str, Any] | None +) -> dict[str, Any] | None: + """Load and lightly process the YAML, without schema validation. + + Resolves ``!include`` / ``!secret`` (via ``yaml_util``), + substitutions, and ``packages:`` -- the three passes whose output + the upload / logs subcommands need. Returns ``None`` on any error + so the caller can fall back to the full ``read_config()`` path. + """ + from esphome import yaml_util + from esphome.components.packages import resolve_packages + from esphome.components.substitutions import do_substitution_pass + + try: + config = yaml_util.load_yaml(conf_path) + except Exception: # pylint: disable=broad-except + return None + + try: + config = do_substitution_pass(config, command_line_substitutions) + except Exception: # pylint: disable=broad-except + return None + + try: + config = resolve_packages( + config, command_line_substitutions=command_line_substitutions + ) + except Exception: # pylint: disable=broad-except + return None + + return config + + +def _build_lite_config(raw_config: dict[str, Any]) -> ConfigType: + """Project the raw YAML down to the keys upload / logs read.""" + return {key: raw_config[key] for key in _LITE_TOP_LEVEL_KEYS if key in raw_config} + + +def _populate_core_from_storage(storage: StorageJSON) -> None: + """Populate ``CORE`` fields the subcommands read off the platform.""" + CORE.name = storage.name + CORE.friendly_name = storage.friendly_name + CORE.build_path = storage.build_path + CORE.loaded_integrations = set(storage.loaded_integrations) + CORE.loaded_platforms = set(storage.loaded_platforms) + + core_platform = storage.core_platform or ( + storage.target_platform.lower() if storage.target_platform else None + ) + CORE.data.setdefault(KEY_CORE, {}) + if core_platform is not None: + CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = core_platform + if storage.framework is not None: + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = storage.framework + + +def _ensure_address_in_config(config: ConfigType, storage: StorageJSON) -> None: + """Make sure ``CORE.address`` resolves once the lite config is set. + + ``CORE.address`` reads ``use_address`` off the wifi / ethernet / + openthread block; if the dashboard rewrote the YAML between the + last compile and now those blocks may be missing from the lite + parse. As a backstop, lift the address from the StorageJSON when + we have one and the YAML doesn't supply one already. + """ + if storage.address is None: + return + for network_type in (CONF_WIFI, CONF_ETHERNET, CONF_OPENTHREAD): + if network_type in config and CONF_USE_ADDRESS in config[network_type]: + return + # Fall back to a synthetic wifi block so ``CORE.address`` resolves. + config.setdefault(CONF_WIFI, {})[CONF_USE_ADDRESS] = storage.address + + +def load_lite_config_from_storage( + conf_path: Path, command_line_substitutions: dict[str, Any] | None = None +) -> ConfigType | None: + """Build a minimal config + populate CORE from the StorageJSON sidecar. + + Returns the lite config dict on success, or ``None`` when the + dispatcher should fall back to ``read_config()`` (sidecar missing, + stale, unreadable, or required fields absent). + """ + storage_path = ext_storage_path(conf_path.name) + try: + yaml_stat = conf_path.stat() + except OSError: + return None + try: + storage_stat = storage_path.stat() + except OSError: + return None + + # The sidecar is written by `compile`; if the YAML has been edited + # since, the cached platform / loaded_integrations may no longer + # describe what the binary on disk was built from. Fall back to a + # full validation pass in that case. + if storage_stat.st_mtime < yaml_stat.st_mtime: + return None + + storage = StorageJSON.load(storage_path) + if storage is None: + return None + if not storage.target_platform and not storage.core_platform: + # An incomplete sidecar (e.g. from a wizard run that never + # compiled) can't drive the upload / logs subcommands. + return None + + raw_config = _parse_yaml_head(conf_path, command_line_substitutions) + if raw_config is None: + return None + if CONF_ESPHOME not in raw_config or CONF_NAME not in raw_config[CONF_ESPHOME]: + return None + + config = _build_lite_config(raw_config) + + # Backfill `esphome:` block fields from the sidecar so downstream + # callers that read `config["esphome"]["name"]` work even when the + # YAML uses substitutions that didn't survive the light parse. + esphome_block = config.setdefault(CONF_ESPHOME, {}) + esphome_block.setdefault(CONF_NAME, storage.name) + if storage.friendly_name is not None: + esphome_block.setdefault(CONF_FRIENDLY_NAME, storage.friendly_name) + + _populate_core_from_storage(storage) + _ensure_address_in_config(config, storage) + + return config diff --git a/tests/unit_tests/test_main_lite_config.py b/tests/unit_tests/test_main_lite_config.py new file mode 100644 index 00000000000..cf0fa3c086e --- /dev/null +++ b/tests/unit_tests/test_main_lite_config.py @@ -0,0 +1,280 @@ +"""Tests for the StorageJSON-backed lite config fast path.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.__main__ import run_esphome +from esphome.const import ( + CONF_API, + CONF_ESPHOME, + CONF_LOGGER, + CONF_NAME, + CONF_OTA, + CONF_USE_ADDRESS, + CONF_WIFI, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, +) +from esphome.core import CORE +from esphome.lite_config import load_lite_config_from_storage + +# A YAML the lite parser can load. Contains every block ``upload`` / +# ``logs`` cares about (api, logger, ota, wifi) plus a substitution so +# the tests exercise the substitution pass too. +_SAMPLE_YAML = """\ +substitutions: + device_name: lite_test + encryption_key: 6dGhpcyBpcyBhIHRlc3Q= + +esphome: + name: ${device_name} + friendly_name: Lite Test Device + +esp32: + board: nodemcu-32s + +logger: + +api: + encryption: + key: ${encryption_key} + +ota: + - platform: esphome + +wifi: + ssid: "ssid" + password: "password" + use_address: 192.168.1.42 +""" + + +def _write_storage( + storage_path: Path, + *, + name: str = "lite_test", + friendly_name: str | None = "Lite Test Device", + address: str | None = "192.168.1.42", + target_platform: str = "ESP32", + core_platform: str | None = "esp32", + framework: str | None = "arduino", + build_path: str | None = "/build/lite_test", + firmware_bin_path: str | None = "/build/lite_test/firmware.bin", + loaded_integrations: list[str] | None = None, + loaded_platforms: list[str] | None = None, +) -> None: + """Write a StorageJSON sidecar to ``storage_path`` for the tests.""" + storage_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "storage_version": 1, + "name": name, + "friendly_name": friendly_name, + "comment": None, + "esphome_version": "2026.1.0", + "src_version": 1, + "address": address, + "web_port": None, + "esp_platform": target_platform, + "build_path": build_path, + "firmware_bin_path": firmware_bin_path, + "loaded_integrations": loaded_integrations or ["api", "logger", "ota", "wifi"], + "loaded_platforms": loaded_platforms or [], + "no_mdns": False, + "framework": framework, + "core_platform": core_platform, + } + storage_path.write_text(json.dumps(data)) + + +@pytest.fixture +def lite_config_files(tmp_path: Path) -> tuple[Path, Path]: + """Create a YAML + matching StorageJSON for the lite-config tests. + + The YAML lives at ``/lite_test.yaml`` so ``CORE.data_dir`` + points into ``/.esphome``. The storage sidecar is written + after the YAML and then back-dated, so callers can rely on + ``storage.mtime >= yaml.mtime`` matching the production happy + path (compile writes the sidecar last). + """ + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text(_SAMPLE_YAML) + CORE.config_path = yaml_path + + storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" + _write_storage(storage_path) + # Storage is younger than the YAML by virtue of being created last, + # but make the relationship explicit for tests that depend on it. + yaml_stat = yaml_path.stat() + os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) + + return yaml_path, storage_path + + +def test_lite_config_happy_path(lite_config_files: tuple[Path, Path]) -> None: + """Storage + YAML in sync: lite config returns a populated dict and CORE.""" + yaml_path, _ = lite_config_files + + config = load_lite_config_from_storage(yaml_path, {}) + + assert config is not None + assert config[CONF_ESPHOME][CONF_NAME] == "lite_test" + assert CONF_LOGGER in config + assert CONF_API in config + assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" + assert CONF_OTA in config + assert config[CONF_OTA][0]["platform"] == "esphome" + # Lite config should NOT contain bulky non-essential keys like esp32. + assert "esp32" not in config + + # CORE populated from storage. + assert CORE.name == "lite_test" + assert CORE.friendly_name == "Lite Test Device" + assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" + assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" + assert CORE.build_path == Path("/build/lite_test") + assert "api" in CORE.loaded_integrations + + +def test_lite_config_missing_storage_returns_none(tmp_path: Path) -> None: + """No StorageJSON sidecar → return None so caller falls back.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text(_SAMPLE_YAML) + CORE.config_path = yaml_path + + assert load_lite_config_from_storage(yaml_path, {}) is None + + +def test_lite_config_stale_storage_returns_none(tmp_path: Path) -> None: + """StorageJSON older than YAML → return None. + + The YAML may have grown a new ``api:`` key since the binary was + built; falling back to ``read_config()`` catches that. + """ + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text(_SAMPLE_YAML) + CORE.config_path = yaml_path + + storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" + _write_storage(storage_path) + + yaml_stat = yaml_path.stat() + # Storage is one minute older than the YAML. + os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) + + assert load_lite_config_from_storage(yaml_path, {}) is None + + +def test_lite_config_address_backfilled_from_storage(tmp_path: Path) -> None: + """No wifi/ethernet block in YAML → address comes from storage. + + Some users gate their network blocks behind substitutions that + don't fire in the lite parse. The loader should fall back to the + sidecar's ``address`` so ``CORE.address`` still resolves. + """ + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text( + "esphome:\n name: lite_test\nesp32:\n board: nodemcu-32s\n" + "logger:\napi:\nota:\n - platform: esphome\n" + ) + CORE.config_path = yaml_path + + storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" + _write_storage(storage_path, address="10.0.0.5") + + yaml_stat = yaml_path.stat() + os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) + + config = load_lite_config_from_storage(yaml_path, {}) + assert config is not None + assert config[CONF_WIFI][CONF_USE_ADDRESS] == "10.0.0.5" + + +def test_lite_config_corrupt_storage_returns_none(tmp_path: Path) -> None: + """A malformed sidecar (StorageJSON.load returns None) → fall back.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text(_SAMPLE_YAML) + CORE.config_path = yaml_path + + storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" + storage_path.parent.mkdir(parents=True, exist_ok=True) + storage_path.write_text("not valid json{") + + yaml_stat = yaml_path.stat() + os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) + + assert load_lite_config_from_storage(yaml_path, {}) is None + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_from_storage_json_skips_read_config( + tmp_path: Path, + command: str, + lite_config_files: tuple[Path, Path], +) -> None: + """`--from-storage-json` makes the dispatcher skip read_config().""" + yaml_path, _ = lite_config_files + + # Stub out the subcommand handler so the test doesn't try to open + # a network connection or run platform-specific upload logic. + with ( + patch("esphome.__main__.read_config") as mock_read, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {command: lambda args, config: 0}, + ), + ): + result = run_esphome( + ["esphome", command, "--from-storage-json", str(yaml_path)] + ) + + mock_read.assert_not_called() + assert result == 0 + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_from_storage_json_falls_back_when_missing( + tmp_path: Path, command: str +) -> None: + """With no sidecar on disk, the dispatcher falls back to read_config().""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text(_SAMPLE_YAML) + + with ( + patch("esphome.__main__.read_config", return_value=None) as mock_read, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {command: lambda args, config: 0}, + ), + ): + result = run_esphome( + ["esphome", command, "--from-storage-json", str(yaml_path)] + ) + + mock_read.assert_called_once() + # read_config returned None → dispatcher exits with 2. + assert result == 2 + + +def test_run_esphome_without_flag_still_calls_read_config( + tmp_path: Path, lite_config_files: tuple[Path, Path] +) -> None: + """Sanity: omitting the flag preserves the current behaviour.""" + yaml_path, _ = lite_config_files + + with ( + patch("esphome.__main__.read_config", return_value=None) as mock_read, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "upload", str(yaml_path)]) + + mock_read.assert_called_once() From 6a8a1d525671a042d52aaadc18dc2b5a3576f54e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 15:34:07 -0500 Subject: [PATCH 06/16] [core] Cache validated config for --from-storage-json fast path Drop the StorageJSON `compiled_config` field and the `esphome/lite_config.py` module from the first iteration. The earlier shape stored the entire validated config inside the JSON sidecar, but configs can grow past a megabyte once packages and substitutions expand, and JSON can't round-trip the YAML-specific types (lambdas, ID instances, includes) the validation pipeline produces. Instead: dump the validated config to its own YAML file alongside the sidecar (`.validated.yaml`), using `yaml_util.dump` so all esphome-specific tags survive the round trip. The fast path loads it back with `yaml_util.load_yaml(clear_secrets=False)` -- no schema validation, no final-validate, no external-component refresh. Staleness is gated by mtime: if the source YAML mtime is newer than the cache, the dispatcher falls back to a full `read_config()` with a warning. The cache is refreshed on every successful compile via `writer.update_storage_json`, so the dashboard's compile -> upload -> logs flow always sees a fresh cache. Drift surface shrinks to "what does yaml_util.dump emit" rather than "which top-level keys does upload/logs read" -- when those subcommands gain a new dependency, the cache already carries it. --- esphome/__main__.py | 31 +- esphome/lite_config.py | 197 ------------ esphome/storage_json.py | 105 ++++++- esphome/writer.py | 10 +- .../unit_tests/test_main_from_storage_json.py | 264 +++++++++++++++++ tests/unit_tests/test_main_lite_config.py | 280 ------------------ 6 files changed, 399 insertions(+), 488 deletions(-) delete mode 100644 esphome/lite_config.py create mode 100644 tests/unit_tests/test_main_from_storage_json.py delete mode 100644 tests/unit_tests/test_main_lite_config.py diff --git a/esphome/__main__.py b/esphome/__main__.py index 33f17276969..4358cdf1a36 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2445,25 +2445,38 @@ def run_esphome(argv): # `read_config()` pipeline (parse + schema validate + final-validate + # external component refresh) for those two subcommands is dead work and # produces a wall of "Reading configuration ..." log lines for every - # remote install. Source the platform / build metadata from the - # StorageJSON sidecar instead; fall back to full validation if it's - # missing or stale so a cold cache never produces a worse outcome. + # remote install. Reload the validated config that the last compile + # cached alongside the StorageJSON sidecar; fall back to full + # validation if the cache is missing or older than the YAML so a + # cold cache never produces a worse outcome. config = None if getattr(args, "from_storage_json", False) and args.command in ( "upload", "logs", ): - from esphome.lite_config import load_lite_config_from_storage + from esphome.storage_json import ( + StorageJSON, + ext_storage_path, + load_compiled_config, + ) - config = load_lite_config_from_storage(conf_path, command_line_substitutions) + config = load_compiled_config(conf_path) + if config is not None: + storage = StorageJSON.load(ext_storage_path(conf_path.name)) + if storage is None: + config = None + else: + storage.apply_to_core() + _LOGGER.info( + "Loaded validated config cache for %s, skipping validation.", + conf_path.name, + ) if config is None: _LOGGER.warning( - "StorageJSON sidecar missing or stale for %s; falling back to " - "full config validation.", + "Validated config cache for %s is missing or older than the " + "YAML; falling back to full config validation.", conf_path, ) - else: - _LOGGER.info("Loaded device metadata from StorageJSON sidecar.") if config is None: config = read_config( diff --git a/esphome/lite_config.py b/esphome/lite_config.py deleted file mode 100644 index 89816a00c71..00000000000 --- a/esphome/lite_config.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Lite-config loader for ``upload`` / ``logs`` fast paths. - -When the caller (typically a dashboard) already has a known-good -firmware binary on disk and only needs the CLI to ship bytes to a -device or stream logs back, running the full ``read_config()`` -pipeline is dead work. It re-parses every ``!include``, runs every -component's schema validator, executes all final-validate hooks, and -fetches/refreshes external components -- producing a fully validated -``Config`` object that the upload / logs subcommands only consult for a -handful of leaf keys. - -This module loads the per-config ``StorageJSON`` sidecar that was -written by the last successful ``compile`` run, populates ``CORE`` with -the platform / build metadata from it, and re-parses just enough of the -YAML head (substitutions + packages, no schema validation) to recover -``api:`` / ``logger:`` / ``ota:`` / ``mqtt:`` / network blocks for the -subcommand callers. - -If the sidecar is missing or older than the YAML, this returns ``None`` -so the dispatcher can fall back to the full ``read_config()`` path. -That makes the flag a pure optimisation: a cold cache never produces a -worse outcome than today. -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Any - -from esphome.const import ( - CONF_API, - CONF_ESPHOME, - CONF_ETHERNET, - CONF_FRIENDLY_NAME, - CONF_LOGGER, - CONF_MQTT, - CONF_NAME, - CONF_OPENTHREAD, - CONF_OTA, - CONF_USE_ADDRESS, - CONF_WEB_SERVER, - CONF_WIFI, - KEY_CORE, - KEY_TARGET_FRAMEWORK, - KEY_TARGET_PLATFORM, -) -from esphome.core import CORE -from esphome.storage_json import StorageJSON, ext_storage_path -from esphome.types import ConfigType - -_LOGGER = logging.getLogger(__name__) - -# Top-level keys we copy from the raw YAML into the lite config dict. -# Anything outside this list is irrelevant to ``upload`` / ``logs``. -_LITE_TOP_LEVEL_KEYS: tuple[str, ...] = ( - CONF_ESPHOME, - CONF_API, - CONF_LOGGER, - CONF_OTA, - CONF_MQTT, - CONF_WIFI, - CONF_ETHERNET, - CONF_OPENTHREAD, - CONF_WEB_SERVER, -) - - -def _parse_yaml_head( - conf_path: Path, command_line_substitutions: dict[str, Any] | None -) -> dict[str, Any] | None: - """Load and lightly process the YAML, without schema validation. - - Resolves ``!include`` / ``!secret`` (via ``yaml_util``), - substitutions, and ``packages:`` -- the three passes whose output - the upload / logs subcommands need. Returns ``None`` on any error - so the caller can fall back to the full ``read_config()`` path. - """ - from esphome import yaml_util - from esphome.components.packages import resolve_packages - from esphome.components.substitutions import do_substitution_pass - - try: - config = yaml_util.load_yaml(conf_path) - except Exception: # pylint: disable=broad-except - return None - - try: - config = do_substitution_pass(config, command_line_substitutions) - except Exception: # pylint: disable=broad-except - return None - - try: - config = resolve_packages( - config, command_line_substitutions=command_line_substitutions - ) - except Exception: # pylint: disable=broad-except - return None - - return config - - -def _build_lite_config(raw_config: dict[str, Any]) -> ConfigType: - """Project the raw YAML down to the keys upload / logs read.""" - return {key: raw_config[key] for key in _LITE_TOP_LEVEL_KEYS if key in raw_config} - - -def _populate_core_from_storage(storage: StorageJSON) -> None: - """Populate ``CORE`` fields the subcommands read off the platform.""" - CORE.name = storage.name - CORE.friendly_name = storage.friendly_name - CORE.build_path = storage.build_path - CORE.loaded_integrations = set(storage.loaded_integrations) - CORE.loaded_platforms = set(storage.loaded_platforms) - - core_platform = storage.core_platform or ( - storage.target_platform.lower() if storage.target_platform else None - ) - CORE.data.setdefault(KEY_CORE, {}) - if core_platform is not None: - CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = core_platform - if storage.framework is not None: - CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = storage.framework - - -def _ensure_address_in_config(config: ConfigType, storage: StorageJSON) -> None: - """Make sure ``CORE.address`` resolves once the lite config is set. - - ``CORE.address`` reads ``use_address`` off the wifi / ethernet / - openthread block; if the dashboard rewrote the YAML between the - last compile and now those blocks may be missing from the lite - parse. As a backstop, lift the address from the StorageJSON when - we have one and the YAML doesn't supply one already. - """ - if storage.address is None: - return - for network_type in (CONF_WIFI, CONF_ETHERNET, CONF_OPENTHREAD): - if network_type in config and CONF_USE_ADDRESS in config[network_type]: - return - # Fall back to a synthetic wifi block so ``CORE.address`` resolves. - config.setdefault(CONF_WIFI, {})[CONF_USE_ADDRESS] = storage.address - - -def load_lite_config_from_storage( - conf_path: Path, command_line_substitutions: dict[str, Any] | None = None -) -> ConfigType | None: - """Build a minimal config + populate CORE from the StorageJSON sidecar. - - Returns the lite config dict on success, or ``None`` when the - dispatcher should fall back to ``read_config()`` (sidecar missing, - stale, unreadable, or required fields absent). - """ - storage_path = ext_storage_path(conf_path.name) - try: - yaml_stat = conf_path.stat() - except OSError: - return None - try: - storage_stat = storage_path.stat() - except OSError: - return None - - # The sidecar is written by `compile`; if the YAML has been edited - # since, the cached platform / loaded_integrations may no longer - # describe what the binary on disk was built from. Fall back to a - # full validation pass in that case. - if storage_stat.st_mtime < yaml_stat.st_mtime: - return None - - storage = StorageJSON.load(storage_path) - if storage is None: - return None - if not storage.target_platform and not storage.core_platform: - # An incomplete sidecar (e.g. from a wizard run that never - # compiled) can't drive the upload / logs subcommands. - return None - - raw_config = _parse_yaml_head(conf_path, command_line_substitutions) - if raw_config is None: - return None - if CONF_ESPHOME not in raw_config or CONF_NAME not in raw_config[CONF_ESPHOME]: - return None - - config = _build_lite_config(raw_config) - - # Backfill `esphome:` block fields from the sidecar so downstream - # callers that read `config["esphome"]["name"]` work even when the - # YAML uses substitutions that didn't survive the light parse. - esphome_block = config.setdefault(CONF_ESPHOME, {}) - esphome_block.setdefault(CONF_NAME, storage.name) - if storage.friendly_name is not None: - esphome_block.setdefault(CONF_FRIENDLY_NAME, storage.friendly_name) - - _populate_core_from_storage(storage) - _ensure_address_in_config(config, storage) - - return config diff --git a/esphome/storage_json.py b/esphome/storage_json.py index c6df16ce78d..d760297150b 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -11,7 +11,7 @@ from esphome import const from esphome.const import CONF_DISABLED, CONF_MDNS from esphome.core import CORE from esphome.helpers import write_file_if_changed -from esphome.types import CoreType +from esphome.types import ConfigType, CoreType _LOGGER = logging.getLogger(__name__) @@ -56,6 +56,82 @@ def archive_storage_path() -> Path: return CORE.relative_config_path("archive") +def compiled_config_path(config_filename: str) -> Path: + """Path to the cached validated config alongside the storage sidecar. + + Written after every successful compile from the validated config + dict (via ``yaml_util.dump``). Powers the dispatcher's + ``--from-storage-json`` fast path in ``esphome upload`` and + ``esphome logs`` so they can skip the full ``read_config()`` + validation pipeline. + + Lives next to the existing JSON sidecar (``.json``) but + in its own file so the small metadata sidecar isn't bloated by + configs that can reach a megabyte or more once packages and + substitutions are expanded. + """ + return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml" + + +def save_compiled_config(config: ConfigType) -> None: + """Dump the validated config to its sidecar YAML file. + + Called by the writer at the end of ``compile`` so the next call + to ``esphome upload --from-storage-json`` / ``esphome logs + --from-storage-json`` for this YAML can skip validation. Failures + here are non-fatal: the worst case is that the fast path falls + back to a full ``read_config`` next time. + """ + from esphome import yaml_util + + try: + # show_secrets=True so the cache is self-contained; the file + # lives next to the binary it describes, in the same trust zone + # as the rest of .esphome/storage/. + rendered = yaml_util.dump(config, show_secrets=True) + except Exception as err: # pylint: disable=broad-except + _LOGGER.debug("Skipping compiled config cache write: %s", err) + return + try: + write_file_if_changed(compiled_config_path(CORE.config_filename), rendered) + except OSError as err: + _LOGGER.debug("Skipping compiled config cache write: %s", err) + + +def load_compiled_config(config_path: Path) -> ConfigType | None: + """Load the cached validated config for ``--from-storage-json``. + + Returns ``None`` (so the caller falls back to ``read_config``) if + the cache is missing or older than the source YAML. The mtime + check catches the common "user edited the YAML and forgot to + recompile" case; deeper drift (an edited ``!include`` whose + parent YAML mtime didn't change) is the user's responsibility — + this flag is opt-in and assumes the caller knows the binary on + disk matches the cache. + """ + cache_path = compiled_config_path(config_path.name) + try: + yaml_mtime = config_path.stat().st_mtime + except OSError: + return None + try: + cache_mtime = cache_path.stat().st_mtime + except OSError: + return None + if cache_mtime < yaml_mtime: + return None + + from esphome import yaml_util + + try: + # clear_secrets=False so we don't disturb any in-flight secret + # state; the cache is self-contained and resolves no !secret + # references. + return yaml_util.load_yaml(cache_path, clear_secrets=False) + except Exception: # pylint: disable=broad-except + return None + + def _to_path_if_not_none(value: str | None) -> Path | None: """Convert a string to Path if it's not None.""" return Path(value) if value is not None else None @@ -256,6 +332,33 @@ class StorageJSON: except Exception: # pylint: disable=broad-except return None + def apply_to_core(self) -> None: + """Populate ``CORE`` from this sidecar. + + Used by the ``--from-storage-json`` fast path in + ``esphome upload`` / ``esphome logs``: those subcommands read + a handful of ``CORE`` attributes (``target_platform``, + ``build_path``, ``name``, ``loaded_integrations``) that the + normal flow populates during ``read_config``. Lifting them off + the sidecar lets us skip the validation pass entirely. + """ + from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM + + CORE.name = self.name + CORE.friendly_name = self.friendly_name + CORE.build_path = self.build_path + CORE.loaded_integrations = set(self.loaded_integrations) + CORE.loaded_platforms = set(self.loaded_platforms) + + core_platform = self.core_platform or ( + self.target_platform.lower() if self.target_platform else None + ) + CORE.data.setdefault(KEY_CORE, {}) + if core_platform is not None: + CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = core_platform + if self.framework is not None: + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = self.framework + def __eq__(self, o) -> bool: return isinstance(o, StorageJSON) and self.as_dict() == o.as_dict() diff --git a/esphome/writer.py b/esphome/writer.py index 2fa43fa5eb1..67df3d1f225 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -24,7 +24,7 @@ from esphome.helpers import ( walk_files, write_file_if_changed, ) -from esphome.storage_json import StorageJSON, storage_path +from esphome.storage_json import StorageJSON, save_compiled_config, storage_path _LOGGER = logging.getLogger(__name__) @@ -109,6 +109,14 @@ def update_storage_json() -> None: path = storage_path() old = StorageJSON.load(path) new = StorageJSON.from_esphome_core(CORE, old) + + # Always refresh the validated-config cache so `esphome upload + # --from-storage-json` and `esphome logs --from-storage-json` can + # skip re-validating after this compile. Lives in its own file + # next to the sidecar; mtime gates staleness on the read side. + if CORE.config is not None: + save_compiled_config(CORE.config) + if old == new: return diff --git a/tests/unit_tests/test_main_from_storage_json.py b/tests/unit_tests/test_main_from_storage_json.py new file mode 100644 index 00000000000..2433511f7fd --- /dev/null +++ b/tests/unit_tests/test_main_from_storage_json.py @@ -0,0 +1,264 @@ +"""Tests for the `--from-storage-json` validated-config cache fast path.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.__main__ import run_esphome +from esphome.const import ( + CONF_API, + CONF_ESPHOME, + CONF_NAME, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, +) +from esphome.core import CORE +from esphome.storage_json import StorageJSON, compiled_config_path, load_compiled_config + +_VALIDATED_CONFIG_YAML = """\ +esphome: + name: lite_test + friendly_name: Lite Test Device +esp32: + board: nodemcu-32s +logger: + baud_rate: 115200 +api: + port: 6053 + encryption: + key: 6dGhpcyBpcyBhIHRlc3Q= +ota: + - platform: esphome + port: 3232 + password: secret +wifi: + ssid: ssid + use_address: 192.168.1.42 +""" + + +def _write_storage(storage_path: Path) -> None: + """Write a vanilla StorageJSON sidecar for the cache tests.""" + storage_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "storage_version": 1, + "name": "lite_test", + "friendly_name": "Lite Test Device", + "comment": None, + "esphome_version": "2026.1.0", + "src_version": 1, + "address": "192.168.1.42", + "web_port": None, + "esp_platform": "ESP32", + "build_path": "/build/lite_test", + "firmware_bin_path": "/build/lite_test/firmware.bin", + "loaded_integrations": ["api", "logger", "ota", "wifi"], + "loaded_platforms": [], + "no_mdns": False, + "framework": "arduino", + "core_platform": "esp32", + } + storage_path.write_text(json.dumps(data)) + + +@pytest.fixture +def fresh_cache_files(tmp_path: Path) -> Path: + """Set up a YAML + StorageJSON + validated-config cache. + + Cache mtime is bumped 5s past the YAML so the staleness check + treats it as fresh. + """ + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json") + + cache_path = storage_dir / "lite_test.yaml.validated.yaml" + cache_path.write_text(_VALIDATED_CONFIG_YAML) + yaml_stat = yaml_path.stat() + os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) + + return yaml_path + + +def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None: + """The cache file shape is predictable from the YAML filename.""" + path = compiled_config_path("device.yaml") + assert str(path).endswith("storage/device.yaml.validated.yaml") + + +def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: + """Fresh cache → returns the validated config dict.""" + config = load_compiled_config(fresh_cache_files) + + assert config is not None + assert config[CONF_ESPHOME][CONF_NAME] == "lite_test" + assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" + assert config["ota"][0]["password"] == "secret" + + +def test_load_compiled_config_missing_cache(tmp_path: Path) -> None: + """No cache file on disk → None so caller falls back.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + assert load_compiled_config(yaml_path) is None + + +def test_load_compiled_config_stale_cache(tmp_path: Path) -> None: + """Cache older than the YAML → None (the YAML was edited).""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + storage_dir.mkdir(parents=True, exist_ok=True) + cache_path = storage_dir / "lite_test.yaml.validated.yaml" + cache_path.write_text(_VALIDATED_CONFIG_YAML) + + yaml_stat = yaml_path.stat() + # Cache is one minute older than the YAML — the YAML's been edited + # since the last compile, so the cache no longer describes the + # binary on disk. + os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) + + assert load_compiled_config(yaml_path) is None + + +def test_load_compiled_config_corrupt_cache(tmp_path: Path) -> None: + """Cache file is unparseable → None so caller falls back.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + storage_dir.mkdir(parents=True, exist_ok=True) + cache_path = storage_dir / "lite_test.yaml.validated.yaml" + cache_path.write_text("not: valid: yaml: [") + + yaml_stat = yaml_path.stat() + os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) + + assert load_compiled_config(yaml_path) is None + + +def test_storage_json_apply_to_core_populates_target_platform(tmp_path: Path) -> None: + """apply_to_core sets the CORE attributes upload / logs read.""" + storage_path = tmp_path / "lite_test.yaml.json" + _write_storage(storage_path) + storage = StorageJSON.load(storage_path) + assert storage is not None + + storage.apply_to_core() + + assert CORE.name == "lite_test" + assert CORE.friendly_name == "Lite Test Device" + assert CORE.build_path == Path("/build/lite_test") + assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" + assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" + assert "api" in CORE.loaded_integrations + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_from_storage_json_skips_read_config( + command: str, fresh_cache_files: Path +) -> None: + """`--from-storage-json` makes the dispatcher skip read_config().""" + yaml_path = fresh_cache_files + + captured = {} + + def _stub(_args, config): + captured["config"] = config + return 0 + + with ( + patch("esphome.__main__.read_config") as mock_read, + patch.dict("esphome.__main__.POST_CONFIG_ACTIONS", {command: _stub}), + ): + result = run_esphome( + ["esphome", command, "--from-storage-json", str(yaml_path)] + ) + + mock_read.assert_not_called() + assert result == 0 + # The dispatcher hands the cached config dict through unchanged. + assert captured["config"][CONF_ESPHOME][CONF_NAME] == "lite_test" + assert captured["config"][CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_from_storage_json_falls_back_when_missing( + tmp_path: Path, command: str +) -> None: + """With no cache on disk, the dispatcher falls back to read_config().""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + + with ( + patch("esphome.__main__.read_config", return_value=None) as mock_read, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {command: lambda args, config: 0}, + ), + ): + result = run_esphome( + ["esphome", command, "--from-storage-json", str(yaml_path)] + ) + + mock_read.assert_called_once() + assert result == 2 + + +def test_run_esphome_from_storage_json_falls_back_when_stale( + tmp_path: Path, +) -> None: + """If YAML mtime > cache mtime, dispatcher falls back to read_config().""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json") + + cache_path = storage_dir / "lite_test.yaml.validated.yaml" + cache_path.write_text(_VALIDATED_CONFIG_YAML) + yaml_stat = yaml_path.stat() + os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) + + with ( + patch("esphome.__main__.read_config", return_value=None) as mock_read, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "upload", "--from-storage-json", str(yaml_path)]) + + mock_read.assert_called_once() + + +def test_run_esphome_without_flag_still_calls_read_config( + fresh_cache_files: Path, +) -> None: + """Sanity: omitting the flag preserves the current behaviour.""" + yaml_path = fresh_cache_files + + with ( + patch("esphome.__main__.read_config", return_value=None) as mock_read, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "upload", str(yaml_path)]) + + mock_read.assert_called_once() diff --git a/tests/unit_tests/test_main_lite_config.py b/tests/unit_tests/test_main_lite_config.py deleted file mode 100644 index cf0fa3c086e..00000000000 --- a/tests/unit_tests/test_main_lite_config.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Tests for the StorageJSON-backed lite config fast path.""" - -from __future__ import annotations - -import json -import os -from pathlib import Path -from unittest.mock import patch - -import pytest - -from esphome.__main__ import run_esphome -from esphome.const import ( - CONF_API, - CONF_ESPHOME, - CONF_LOGGER, - CONF_NAME, - CONF_OTA, - CONF_USE_ADDRESS, - CONF_WIFI, - KEY_CORE, - KEY_TARGET_FRAMEWORK, - KEY_TARGET_PLATFORM, -) -from esphome.core import CORE -from esphome.lite_config import load_lite_config_from_storage - -# A YAML the lite parser can load. Contains every block ``upload`` / -# ``logs`` cares about (api, logger, ota, wifi) plus a substitution so -# the tests exercise the substitution pass too. -_SAMPLE_YAML = """\ -substitutions: - device_name: lite_test - encryption_key: 6dGhpcyBpcyBhIHRlc3Q= - -esphome: - name: ${device_name} - friendly_name: Lite Test Device - -esp32: - board: nodemcu-32s - -logger: - -api: - encryption: - key: ${encryption_key} - -ota: - - platform: esphome - -wifi: - ssid: "ssid" - password: "password" - use_address: 192.168.1.42 -""" - - -def _write_storage( - storage_path: Path, - *, - name: str = "lite_test", - friendly_name: str | None = "Lite Test Device", - address: str | None = "192.168.1.42", - target_platform: str = "ESP32", - core_platform: str | None = "esp32", - framework: str | None = "arduino", - build_path: str | None = "/build/lite_test", - firmware_bin_path: str | None = "/build/lite_test/firmware.bin", - loaded_integrations: list[str] | None = None, - loaded_platforms: list[str] | None = None, -) -> None: - """Write a StorageJSON sidecar to ``storage_path`` for the tests.""" - storage_path.parent.mkdir(parents=True, exist_ok=True) - data = { - "storage_version": 1, - "name": name, - "friendly_name": friendly_name, - "comment": None, - "esphome_version": "2026.1.0", - "src_version": 1, - "address": address, - "web_port": None, - "esp_platform": target_platform, - "build_path": build_path, - "firmware_bin_path": firmware_bin_path, - "loaded_integrations": loaded_integrations or ["api", "logger", "ota", "wifi"], - "loaded_platforms": loaded_platforms or [], - "no_mdns": False, - "framework": framework, - "core_platform": core_platform, - } - storage_path.write_text(json.dumps(data)) - - -@pytest.fixture -def lite_config_files(tmp_path: Path) -> tuple[Path, Path]: - """Create a YAML + matching StorageJSON for the lite-config tests. - - The YAML lives at ``/lite_test.yaml`` so ``CORE.data_dir`` - points into ``/.esphome``. The storage sidecar is written - after the YAML and then back-dated, so callers can rely on - ``storage.mtime >= yaml.mtime`` matching the production happy - path (compile writes the sidecar last). - """ - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text(_SAMPLE_YAML) - CORE.config_path = yaml_path - - storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" - _write_storage(storage_path) - # Storage is younger than the YAML by virtue of being created last, - # but make the relationship explicit for tests that depend on it. - yaml_stat = yaml_path.stat() - os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) - - return yaml_path, storage_path - - -def test_lite_config_happy_path(lite_config_files: tuple[Path, Path]) -> None: - """Storage + YAML in sync: lite config returns a populated dict and CORE.""" - yaml_path, _ = lite_config_files - - config = load_lite_config_from_storage(yaml_path, {}) - - assert config is not None - assert config[CONF_ESPHOME][CONF_NAME] == "lite_test" - assert CONF_LOGGER in config - assert CONF_API in config - assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" - assert CONF_OTA in config - assert config[CONF_OTA][0]["platform"] == "esphome" - # Lite config should NOT contain bulky non-essential keys like esp32. - assert "esp32" not in config - - # CORE populated from storage. - assert CORE.name == "lite_test" - assert CORE.friendly_name == "Lite Test Device" - assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" - assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" - assert CORE.build_path == Path("/build/lite_test") - assert "api" in CORE.loaded_integrations - - -def test_lite_config_missing_storage_returns_none(tmp_path: Path) -> None: - """No StorageJSON sidecar → return None so caller falls back.""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text(_SAMPLE_YAML) - CORE.config_path = yaml_path - - assert load_lite_config_from_storage(yaml_path, {}) is None - - -def test_lite_config_stale_storage_returns_none(tmp_path: Path) -> None: - """StorageJSON older than YAML → return None. - - The YAML may have grown a new ``api:`` key since the binary was - built; falling back to ``read_config()`` catches that. - """ - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text(_SAMPLE_YAML) - CORE.config_path = yaml_path - - storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" - _write_storage(storage_path) - - yaml_stat = yaml_path.stat() - # Storage is one minute older than the YAML. - os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) - - assert load_lite_config_from_storage(yaml_path, {}) is None - - -def test_lite_config_address_backfilled_from_storage(tmp_path: Path) -> None: - """No wifi/ethernet block in YAML → address comes from storage. - - Some users gate their network blocks behind substitutions that - don't fire in the lite parse. The loader should fall back to the - sidecar's ``address`` so ``CORE.address`` still resolves. - """ - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text( - "esphome:\n name: lite_test\nesp32:\n board: nodemcu-32s\n" - "logger:\napi:\nota:\n - platform: esphome\n" - ) - CORE.config_path = yaml_path - - storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" - _write_storage(storage_path, address="10.0.0.5") - - yaml_stat = yaml_path.stat() - os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) - - config = load_lite_config_from_storage(yaml_path, {}) - assert config is not None - assert config[CONF_WIFI][CONF_USE_ADDRESS] == "10.0.0.5" - - -def test_lite_config_corrupt_storage_returns_none(tmp_path: Path) -> None: - """A malformed sidecar (StorageJSON.load returns None) → fall back.""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text(_SAMPLE_YAML) - CORE.config_path = yaml_path - - storage_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.json" - storage_path.parent.mkdir(parents=True, exist_ok=True) - storage_path.write_text("not valid json{") - - yaml_stat = yaml_path.stat() - os.utime(storage_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) - - assert load_lite_config_from_storage(yaml_path, {}) is None - - -@pytest.mark.parametrize("command", ["upload", "logs"]) -def test_run_esphome_from_storage_json_skips_read_config( - tmp_path: Path, - command: str, - lite_config_files: tuple[Path, Path], -) -> None: - """`--from-storage-json` makes the dispatcher skip read_config().""" - yaml_path, _ = lite_config_files - - # Stub out the subcommand handler so the test doesn't try to open - # a network connection or run platform-specific upload logic. - with ( - patch("esphome.__main__.read_config") as mock_read, - patch.dict( - "esphome.__main__.POST_CONFIG_ACTIONS", - {command: lambda args, config: 0}, - ), - ): - result = run_esphome( - ["esphome", command, "--from-storage-json", str(yaml_path)] - ) - - mock_read.assert_not_called() - assert result == 0 - - -@pytest.mark.parametrize("command", ["upload", "logs"]) -def test_run_esphome_from_storage_json_falls_back_when_missing( - tmp_path: Path, command: str -) -> None: - """With no sidecar on disk, the dispatcher falls back to read_config().""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text(_SAMPLE_YAML) - - with ( - patch("esphome.__main__.read_config", return_value=None) as mock_read, - patch.dict( - "esphome.__main__.POST_CONFIG_ACTIONS", - {command: lambda args, config: 0}, - ), - ): - result = run_esphome( - ["esphome", command, "--from-storage-json", str(yaml_path)] - ) - - mock_read.assert_called_once() - # read_config returned None → dispatcher exits with 2. - assert result == 2 - - -def test_run_esphome_without_flag_still_calls_read_config( - tmp_path: Path, lite_config_files: tuple[Path, Path] -) -> None: - """Sanity: omitting the flag preserves the current behaviour.""" - yaml_path, _ = lite_config_files - - with ( - patch("esphome.__main__.read_config", return_value=None) as mock_read, - patch.dict( - "esphome.__main__.POST_CONFIG_ACTIONS", - {"upload": lambda args, config: 0}, - ), - ): - run_esphome(["esphome", "upload", str(yaml_path)]) - - mock_read.assert_called_once() From 26d2d4f65d2a8c83e5dfaac0fb58a7c9394b486d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 15:38:57 -0500 Subject: [PATCH 07/16] [core] Make validated-config cache automatic, no flag Drop the `--from-storage-json` CLI flag. `esphome upload` and `esphome logs` now always try the validated-config cache and transparently fall back to `read_config()` when it's missing, stale (YAML mtime > cache mtime), or corrupt. No caller change is required to benefit, and a cold cache never produces a worse outcome than today. `compile` continues to write the cache unconditionally so the next upload / logs against this YAML can skip validation; it itself always re-validates since code generation needs the fully validated config. --- esphome/__main__.py | 59 +++++-------------- esphome/writer.py | 8 +-- ...py => test_main_validated_config_cache.py} | 37 ++++++------ 3 files changed, 35 insertions(+), 69 deletions(-) rename tests/unit_tests/{test_main_from_storage_json.py => test_main_validated_config_cache.py} (89%) diff --git a/esphome/__main__.py b/esphome/__main__.py index 4358cdf1a36..435931cd285 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2117,16 +2117,6 @@ def parse_args(argv): help="Upload as bootloader (OTA).", action="store_true", ) - parser_upload.add_argument( - "--from-storage-json", - action="store_true", - help=( - "Skip YAML schema validation by loading device metadata from " - "the .esphome/storage/.json sidecar produced by the last " - "successful compile. Falls back to a full validation pass when " - "the sidecar is missing or older than the YAML." - ), - ) parser_logs = subparsers.add_parser( "logs", @@ -2154,16 +2144,6 @@ def parse_args(argv): action="store_true", help="Do not show entity state changes in log output.", ) - parser_logs.add_argument( - "--from-storage-json", - action="store_true", - help=( - "Skip YAML schema validation by loading device metadata from " - "the .esphome/storage/.json sidecar produced by the last " - "successful compile. Falls back to a full validation pass when " - "the sidecar is missing or older than the YAML." - ), - ) parser_discover = subparsers.add_parser( "discover", @@ -2439,44 +2419,33 @@ def run_esphome(argv): skip_external = args.command in ("logs", "clean") command_line_substitutions = dict(args.substitution) if args.substitution else {} - # Fast path for `upload --from-storage-json` and `logs --from-storage-json`: - # the caller already has a binary on disk and only needs the CLI to ship - # bytes to a device or stream logs back. Re-running the full - # `read_config()` pipeline (parse + schema validate + final-validate + - # external component refresh) for those two subcommands is dead work and - # produces a wall of "Reading configuration ..." log lines for every - # remote install. Reload the validated config that the last compile - # cached alongside the StorageJSON sidecar; fall back to full - # validation if the cache is missing or older than the YAML so a - # cold cache never produces a worse outcome. + # Fast path for `upload` and `logs`: the caller already has a binary + # on disk and only needs the CLI to ship bytes to a device or stream + # logs back. Re-running the full `read_config()` pipeline (parse + + # schema validate + final-validate + external component refresh) is + # dead work and produces a wall of "Reading configuration ..." log + # lines on every install. Reload the validated config the last + # compile cached alongside the StorageJSON sidecar; fall back to + # full validation if the cache is missing or older than the YAML so + # a cold cache never produces a worse outcome. config = None - if getattr(args, "from_storage_json", False) and args.command in ( - "upload", - "logs", - ): + if args.command in ("upload", "logs"): from esphome.storage_json import ( StorageJSON, ext_storage_path, load_compiled_config, ) - config = load_compiled_config(conf_path) - if config is not None: + cached = load_compiled_config(conf_path) + if cached is not None: storage = StorageJSON.load(ext_storage_path(conf_path.name)) - if storage is None: - config = None - else: + if storage is not None: storage.apply_to_core() + config = cached _LOGGER.info( "Loaded validated config cache for %s, skipping validation.", conf_path.name, ) - if config is None: - _LOGGER.warning( - "Validated config cache for %s is missing or older than the " - "YAML; falling back to full config validation.", - conf_path, - ) if config is None: config = read_config( diff --git a/esphome/writer.py b/esphome/writer.py index 67df3d1f225..8efefe68baf 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -110,10 +110,10 @@ def update_storage_json() -> None: old = StorageJSON.load(path) new = StorageJSON.from_esphome_core(CORE, old) - # Always refresh the validated-config cache so `esphome upload - # --from-storage-json` and `esphome logs --from-storage-json` can - # skip re-validating after this compile. Lives in its own file - # next to the sidecar; mtime gates staleness on the read side. + # Refresh the validated-config cache. `esphome upload` and + # `esphome logs` look for it next time they run and skip the full + # validation pipeline when it's present and fresh (mtime >= YAML + # mtime). Cheap to write, big win to read. if CORE.config is not None: save_compiled_config(CORE.config) diff --git a/tests/unit_tests/test_main_from_storage_json.py b/tests/unit_tests/test_main_validated_config_cache.py similarity index 89% rename from tests/unit_tests/test_main_from_storage_json.py rename to tests/unit_tests/test_main_validated_config_cache.py index 2433511f7fd..a5ecbe59c95 100644 --- a/tests/unit_tests/test_main_from_storage_json.py +++ b/tests/unit_tests/test_main_validated_config_cache.py @@ -1,4 +1,4 @@ -"""Tests for the `--from-storage-json` validated-config cache fast path.""" +"""Tests for the validated-config cache fast path used by upload/logs.""" from __future__ import annotations @@ -169,10 +169,10 @@ def test_storage_json_apply_to_core_populates_target_platform(tmp_path: Path) -> @pytest.mark.parametrize("command", ["upload", "logs"]) -def test_run_esphome_from_storage_json_skips_read_config( +def test_run_esphome_upload_and_logs_use_cache_when_fresh( command: str, fresh_cache_files: Path ) -> None: - """`--from-storage-json` makes the dispatcher skip read_config().""" + """When the cache is fresh, upload/logs skip read_config() entirely.""" yaml_path = fresh_cache_files captured = {} @@ -185,9 +185,7 @@ def test_run_esphome_from_storage_json_skips_read_config( patch("esphome.__main__.read_config") as mock_read, patch.dict("esphome.__main__.POST_CONFIG_ACTIONS", {command: _stub}), ): - result = run_esphome( - ["esphome", command, "--from-storage-json", str(yaml_path)] - ) + result = run_esphome(["esphome", command, str(yaml_path)]) mock_read.assert_not_called() assert result == 0 @@ -197,7 +195,7 @@ def test_run_esphome_from_storage_json_skips_read_config( @pytest.mark.parametrize("command", ["upload", "logs"]) -def test_run_esphome_from_storage_json_falls_back_when_missing( +def test_run_esphome_upload_and_logs_fall_back_when_no_cache( tmp_path: Path, command: str ) -> None: """With no cache on disk, the dispatcher falls back to read_config().""" @@ -211,17 +209,13 @@ def test_run_esphome_from_storage_json_falls_back_when_missing( {command: lambda args, config: 0}, ), ): - result = run_esphome( - ["esphome", command, "--from-storage-json", str(yaml_path)] - ) + result = run_esphome(["esphome", command, str(yaml_path)]) mock_read.assert_called_once() assert result == 2 -def test_run_esphome_from_storage_json_falls_back_when_stale( - tmp_path: Path, -) -> None: +def test_run_esphome_upload_falls_back_when_cache_stale(tmp_path: Path) -> None: """If YAML mtime > cache mtime, dispatcher falls back to read_config().""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") @@ -241,24 +235,27 @@ def test_run_esphome_from_storage_json_falls_back_when_stale( {"upload": lambda args, config: 0}, ), ): - run_esphome(["esphome", "upload", "--from-storage-json", str(yaml_path)]) + run_esphome(["esphome", "upload", str(yaml_path)]) mock_read.assert_called_once() -def test_run_esphome_without_flag_still_calls_read_config( - fresh_cache_files: Path, -) -> None: - """Sanity: omitting the flag preserves the current behaviour.""" +def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None: + """`compile` always re-validates, even with a fresh cache on disk. + + The fast path is only for upload / logs -- compile is what writes + the cache in the first place, and it needs a fully validated + config to drive code generation. + """ yaml_path = fresh_cache_files with ( patch("esphome.__main__.read_config", return_value=None) as mock_read, patch.dict( "esphome.__main__.POST_CONFIG_ACTIONS", - {"upload": lambda args, config: 0}, + {"compile": lambda args, config: 0}, ), ): - run_esphome(["esphome", "upload", str(yaml_path)]) + run_esphome(["esphome", "compile", str(yaml_path)]) mock_read.assert_called_once() From aee2fc5761b93bf0c4e3c6b72914f0c21b11b02c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 15:45:47 -0500 Subject: [PATCH 08/16] [core] Extract cache helpers into esphome/compiled_config.py storage_json.py is for the JSON sidecar; the validated-config cache is YAML. Move it out into its own module. Also consolidate the dispatcher's open-coded fast path (load cache + load sidecar + apply_to_core) into a single load_compiled_config() entry point, and collapse save_compiled_config()'s two try blocks into one. mtime comparison gets its own _cache_is_fresh() helper so the loader reads top to bottom. --- esphome/__main__.py | 38 ++-- esphome/compiled_config.py | 96 ++++++++++ esphome/storage_json.py | 78 +------- esphome/writer.py | 3 +- ...onfig_cache.py => test_compiled_config.py} | 179 ++++++------------ 5 files changed, 176 insertions(+), 218 deletions(-) create mode 100644 esphome/compiled_config.py rename tests/unit_tests/{test_main_validated_config_cache.py => test_compiled_config.py} (51%) diff --git a/esphome/__main__.py b/esphome/__main__.py index 435931cd285..d68dcffaf22 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2419,33 +2419,23 @@ def run_esphome(argv): skip_external = args.command in ("logs", "clean") command_line_substitutions = dict(args.substitution) if args.substitution else {} - # Fast path for `upload` and `logs`: the caller already has a binary - # on disk and only needs the CLI to ship bytes to a device or stream - # logs back. Re-running the full `read_config()` pipeline (parse + - # schema validate + final-validate + external component refresh) is - # dead work and produces a wall of "Reading configuration ..." log - # lines on every install. Reload the validated config the last - # compile cached alongside the StorageJSON sidecar; fall back to - # full validation if the cache is missing or older than the YAML so - # a cold cache never produces a worse outcome. + # Fast path for `upload` and `logs`: reuse the validated config the + # last compile cached on disk. Skips parse + schema validate + + # final-validate + external-component refresh, which is dead work + # for those subcommands -- the binary on disk is already known good + # and the caller already knows the device address. Falls back to a + # full `read_config()` when the cache is missing/stale/corrupt so a + # cold cache never produces a worse outcome than today. config = None if args.command in ("upload", "logs"): - from esphome.storage_json import ( - StorageJSON, - ext_storage_path, - load_compiled_config, - ) + from esphome.compiled_config import load_compiled_config - cached = load_compiled_config(conf_path) - if cached is not None: - storage = StorageJSON.load(ext_storage_path(conf_path.name)) - if storage is not None: - storage.apply_to_core() - config = cached - _LOGGER.info( - "Loaded validated config cache for %s, skipping validation.", - conf_path.name, - ) + config = load_compiled_config(conf_path) + if config is not None: + _LOGGER.info( + "Loaded validated config cache for %s, skipping validation.", + conf_path.name, + ) if config is None: config = read_config( diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py new file mode 100644 index 00000000000..bb127a0de98 --- /dev/null +++ b/esphome/compiled_config.py @@ -0,0 +1,96 @@ +"""Validated-config cache for the ``upload`` / ``logs`` fast path. + +After every successful ``esphome compile``, the writer dumps the +validated config to ``/storage/.validated.yaml``. +The next ``esphome upload`` / ``esphome logs`` for that YAML reuses +the cache instead of re-running the full ``read_config()`` pipeline +(parse + schema validate + final-validate + external-component +refresh), which is dead work once the binary on disk is known good. + +The cache uses YAML (via ``yaml_util``) rather than JSON so all +esphome-specific tags (``!lambda``, ``!include``, ``ID`` instances, +paths) round-trip cleanly, and so the small ``StorageJSON`` sidecar +isn't bloated by configs that can grow past a megabyte. Staleness +is gated by an mtime check against the source YAML; the loader +falls back to ``None`` (and the caller to ``read_config()``) whenever +the cache is missing, stale, unparseable, or the companion +``StorageJSON`` sidecar can't be loaded. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from esphome.core import CORE +from esphome.helpers import write_file_if_changed +from esphome.storage_json import StorageJSON, ext_storage_path +from esphome.types import ConfigType + +_LOGGER = logging.getLogger(__name__) + + +def compiled_config_path(config_filename: str) -> Path: + """Path to the cached validated config alongside the storage sidecar.""" + return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml" + + +def _cache_is_fresh(cache_path: Path, source_path: Path) -> bool: + """True iff the cache file exists and isn't older than the source.""" + try: + return cache_path.stat().st_mtime >= source_path.stat().st_mtime + except OSError: + return False + + +def save_compiled_config(config: ConfigType) -> None: + """Dump the validated config to its sidecar YAML file. + + Called from the writer after every successful compile so the next + ``esphome upload`` / ``esphome logs`` for this YAML can skip + validation via :func:`load_compiled_config`. Failures here are + non-fatal: the worst case is that the fast path falls back to a + full ``read_config`` next time. + """ + from esphome import yaml_util + + try: + # show_secrets=True keeps the cache self-contained; it lives + # in the same trust zone as the rest of .esphome/storage/. + rendered = yaml_util.dump(config, show_secrets=True) + write_file_if_changed(compiled_config_path(CORE.config_filename), rendered) + except Exception as err: # pylint: disable=broad-except + _LOGGER.debug("Skipping compiled config cache write: %s", err) + + +def load_compiled_config(conf_path: Path) -> ConfigType | None: + """Load the cached validated config and apply storage metadata to CORE. + + Single entry point for the ``upload`` / ``logs`` fast path: loads + the cache, applies the ``StorageJSON`` sidecar's platform / build + metadata to ``CORE``, and returns the config dict. Returns + ``None`` (so the caller falls back to ``read_config``) when the + cache is missing, older than the source YAML, unparseable, or + when the sidecar can't be loaded. The mtime check catches the + common "user edited the YAML and forgot to recompile" case; + deeper drift (an edited ``!include`` whose parent YAML mtime + didn't change) is the user's responsibility. + """ + cache_path = compiled_config_path(conf_path.name) + if not _cache_is_fresh(cache_path, conf_path): + return None + + from esphome import yaml_util + + try: + # clear_secrets=False keeps in-flight secret state intact; the + # cache is self-contained and resolves no !secret references. + config = yaml_util.load_yaml(cache_path, clear_secrets=False) + except Exception: # pylint: disable=broad-except + return None + + storage = StorageJSON.load(ext_storage_path(conf_path.name)) + if storage is None: + return None + storage.apply_to_core() + return config diff --git a/esphome/storage_json.py b/esphome/storage_json.py index d760297150b..a9a83f58962 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -11,7 +11,7 @@ from esphome import const from esphome.const import CONF_DISABLED, CONF_MDNS from esphome.core import CORE from esphome.helpers import write_file_if_changed -from esphome.types import ConfigType, CoreType +from esphome.types import CoreType _LOGGER = logging.getLogger(__name__) @@ -56,82 +56,6 @@ def archive_storage_path() -> Path: return CORE.relative_config_path("archive") -def compiled_config_path(config_filename: str) -> Path: - """Path to the cached validated config alongside the storage sidecar. - - Written after every successful compile from the validated config - dict (via ``yaml_util.dump``). Powers the dispatcher's - ``--from-storage-json`` fast path in ``esphome upload`` and - ``esphome logs`` so they can skip the full ``read_config()`` - validation pipeline. - - Lives next to the existing JSON sidecar (``.json``) but - in its own file so the small metadata sidecar isn't bloated by - configs that can reach a megabyte or more once packages and - substitutions are expanded. - """ - return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml" - - -def save_compiled_config(config: ConfigType) -> None: - """Dump the validated config to its sidecar YAML file. - - Called by the writer at the end of ``compile`` so the next call - to ``esphome upload --from-storage-json`` / ``esphome logs - --from-storage-json`` for this YAML can skip validation. Failures - here are non-fatal: the worst case is that the fast path falls - back to a full ``read_config`` next time. - """ - from esphome import yaml_util - - try: - # show_secrets=True so the cache is self-contained; the file - # lives next to the binary it describes, in the same trust zone - # as the rest of .esphome/storage/. - rendered = yaml_util.dump(config, show_secrets=True) - except Exception as err: # pylint: disable=broad-except - _LOGGER.debug("Skipping compiled config cache write: %s", err) - return - try: - write_file_if_changed(compiled_config_path(CORE.config_filename), rendered) - except OSError as err: - _LOGGER.debug("Skipping compiled config cache write: %s", err) - - -def load_compiled_config(config_path: Path) -> ConfigType | None: - """Load the cached validated config for ``--from-storage-json``. - - Returns ``None`` (so the caller falls back to ``read_config``) if - the cache is missing or older than the source YAML. The mtime - check catches the common "user edited the YAML and forgot to - recompile" case; deeper drift (an edited ``!include`` whose - parent YAML mtime didn't change) is the user's responsibility — - this flag is opt-in and assumes the caller knows the binary on - disk matches the cache. - """ - cache_path = compiled_config_path(config_path.name) - try: - yaml_mtime = config_path.stat().st_mtime - except OSError: - return None - try: - cache_mtime = cache_path.stat().st_mtime - except OSError: - return None - if cache_mtime < yaml_mtime: - return None - - from esphome import yaml_util - - try: - # clear_secrets=False so we don't disturb any in-flight secret - # state; the cache is self-contained and resolves no !secret - # references. - return yaml_util.load_yaml(cache_path, clear_secrets=False) - except Exception: # pylint: disable=broad-except - return None - - def _to_path_if_not_none(value: str | None) -> Path | None: """Convert a string to Path if it's not None.""" return Path(value) if value is not None else None diff --git a/esphome/writer.py b/esphome/writer.py index 8efefe68baf..54eb254b1b8 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -7,6 +7,7 @@ import re import time from esphome import loader +from esphome.compiled_config import save_compiled_config from esphome.config import iter_component_configs, iter_components from esphome.const import ( HEADER_FILE_EXTENSIONS, @@ -24,7 +25,7 @@ from esphome.helpers import ( walk_files, write_file_if_changed, ) -from esphome.storage_json import StorageJSON, save_compiled_config, storage_path +from esphome.storage_json import StorageJSON, storage_path _LOGGER = logging.getLogger(__name__) diff --git a/tests/unit_tests/test_main_validated_config_cache.py b/tests/unit_tests/test_compiled_config.py similarity index 51% rename from tests/unit_tests/test_main_validated_config_cache.py rename to tests/unit_tests/test_compiled_config.py index a5ecbe59c95..d4736f38a35 100644 --- a/tests/unit_tests/test_main_validated_config_cache.py +++ b/tests/unit_tests/test_compiled_config.py @@ -1,4 +1,4 @@ -"""Tests for the validated-config cache fast path used by upload/logs.""" +"""Tests for the validated-config cache used by upload/logs.""" from __future__ import annotations @@ -10,6 +10,7 @@ from unittest.mock import patch import pytest from esphome.__main__ import run_esphome +from esphome.compiled_config import compiled_config_path, load_compiled_config from esphome.const import ( CONF_API, CONF_ESPHOME, @@ -19,7 +20,6 @@ from esphome.const import ( KEY_TARGET_PLATFORM, ) from esphome.core import CORE -from esphome.storage_json import StorageJSON, compiled_config_path, load_compiled_config _VALIDATED_CONFIG_YAML = """\ esphome: @@ -67,36 +67,46 @@ def _write_storage(storage_path: Path) -> None: storage_path.write_text(json.dumps(data)) +def _write_cache(cache_path: Path, body: str = _VALIDATED_CONFIG_YAML) -> Path: + """Write the cache file and return it.""" + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(body) + return cache_path + + +def _set_cache_mtime(cache_path: Path, yaml_path: Path, *, offset: int) -> None: + """Force the cache file's mtime relative to the source YAML. + + Positive offset → cache is fresh. Negative → cache is stale. + """ + yaml_stat = yaml_path.stat() + os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + offset)) + + @pytest.fixture def fresh_cache_files(tmp_path: Path) -> Path: - """Set up a YAML + StorageJSON + validated-config cache. - - Cache mtime is bumped 5s past the YAML so the staleness check - treats it as fresh. - """ + """YAML + StorageJSON + cache, all consistent and fresh.""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path storage_dir = tmp_path / ".esphome" / "storage" _write_storage(storage_dir / "lite_test.yaml.json") - - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text(_VALIDATED_CONFIG_YAML) - yaml_stat = yaml_path.stat() - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache, yaml_path, offset=5) return yaml_path def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None: """The cache file shape is predictable from the YAML filename.""" - path = compiled_config_path("device.yaml") - assert str(path).endswith("storage/device.yaml.validated.yaml") + assert str(compiled_config_path("device.yaml")).endswith( + "storage/device.yaml.validated.yaml" + ) def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: - """Fresh cache → returns the validated config dict.""" + """Fresh cache + sidecar → returns config and populates CORE.""" config = load_compiled_config(fresh_cache_files) assert config is not None @@ -104,78 +114,50 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" assert config["ota"][0]["password"] == "secret" - -def test_load_compiled_config_missing_cache(tmp_path: Path) -> None: - """No cache file on disk → None so caller falls back.""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - CORE.config_path = yaml_path - - assert load_compiled_config(yaml_path) is None - - -def test_load_compiled_config_stale_cache(tmp_path: Path) -> None: - """Cache older than the YAML → None (the YAML was edited).""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - CORE.config_path = yaml_path - - storage_dir = tmp_path / ".esphome" / "storage" - storage_dir.mkdir(parents=True, exist_ok=True) - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text(_VALIDATED_CONFIG_YAML) - - yaml_stat = yaml_path.stat() - # Cache is one minute older than the YAML — the YAML's been edited - # since the last compile, so the cache no longer describes the - # binary on disk. - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) - - assert load_compiled_config(yaml_path) is None - - -def test_load_compiled_config_corrupt_cache(tmp_path: Path) -> None: - """Cache file is unparseable → None so caller falls back.""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - CORE.config_path = yaml_path - - storage_dir = tmp_path / ".esphome" / "storage" - storage_dir.mkdir(parents=True, exist_ok=True) - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text("not: valid: yaml: [") - - yaml_stat = yaml_path.stat() - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + 5)) - - assert load_compiled_config(yaml_path) is None - - -def test_storage_json_apply_to_core_populates_target_platform(tmp_path: Path) -> None: - """apply_to_core sets the CORE attributes upload / logs read.""" - storage_path = tmp_path / "lite_test.yaml.json" - _write_storage(storage_path) - storage = StorageJSON.load(storage_path) - assert storage is not None - - storage.apply_to_core() - + # apply_to_core ran as part of the orchestration. assert CORE.name == "lite_test" - assert CORE.friendly_name == "Lite Test Device" assert CORE.build_path == Path("/build/lite_test") assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" assert "api" in CORE.loaded_integrations +@pytest.mark.parametrize( + "scenario", + ["missing_cache", "stale_cache", "corrupt_cache", "missing_sidecar"], +) +def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None: + """All non-happy cases return None so the caller falls back.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + storage_dir = tmp_path / ".esphome" / "storage" + cache_path = storage_dir / "lite_test.yaml.validated.yaml" + sidecar_path = storage_dir / "lite_test.yaml.json" + + if scenario == "missing_cache": + pass # no cache, no sidecar + elif scenario == "stale_cache": + _write_storage(sidecar_path) + _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=-60) + elif scenario == "corrupt_cache": + _write_storage(sidecar_path) + _set_cache_mtime( + _write_cache(cache_path, "not: valid: yaml: ["), yaml_path, offset=5 + ) + elif scenario == "missing_sidecar": + # Cache fresh + parseable, but no StorageJSON → can't populate CORE. + _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=5) + + assert load_compiled_config(yaml_path) is None + + @pytest.mark.parametrize("command", ["upload", "logs"]) def test_run_esphome_upload_and_logs_use_cache_when_fresh( command: str, fresh_cache_files: Path ) -> None: - """When the cache is fresh, upload/logs skip read_config() entirely.""" - yaml_path = fresh_cache_files - - captured = {} + """upload/logs skip read_config() when the cache is fresh.""" + captured: dict = {} def _stub(_args, config): captured["config"] = config @@ -185,11 +167,9 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh( patch("esphome.__main__.read_config") as mock_read, patch.dict("esphome.__main__.POST_CONFIG_ACTIONS", {command: _stub}), ): - result = run_esphome(["esphome", command, str(yaml_path)]) + assert run_esphome(["esphome", command, str(fresh_cache_files)]) == 0 mock_read.assert_not_called() - assert result == 0 - # The dispatcher hands the cached config dict through unchanged. assert captured["config"][CONF_ESPHOME][CONF_NAME] == "lite_test" assert captured["config"][CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" @@ -198,7 +178,7 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh( def test_run_esphome_upload_and_logs_fall_back_when_no_cache( tmp_path: Path, command: str ) -> None: - """With no cache on disk, the dispatcher falls back to read_config().""" + """Without a cache, the dispatcher falls back to read_config().""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") @@ -209,46 +189,13 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache( {command: lambda args, config: 0}, ), ): - result = run_esphome(["esphome", command, str(yaml_path)]) - - mock_read.assert_called_once() - assert result == 2 - - -def test_run_esphome_upload_falls_back_when_cache_stale(tmp_path: Path) -> None: - """If YAML mtime > cache mtime, dispatcher falls back to read_config().""" - yaml_path = tmp_path / "lite_test.yaml" - yaml_path.write_text("esphome:\n name: lite_test\n") - - storage_dir = tmp_path / ".esphome" / "storage" - _write_storage(storage_dir / "lite_test.yaml.json") - - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - cache_path.write_text(_VALIDATED_CONFIG_YAML) - yaml_stat = yaml_path.stat() - os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime - 60)) - - with ( - patch("esphome.__main__.read_config", return_value=None) as mock_read, - patch.dict( - "esphome.__main__.POST_CONFIG_ACTIONS", - {"upload": lambda args, config: 0}, - ), - ): - run_esphome(["esphome", "upload", str(yaml_path)]) + assert run_esphome(["esphome", command, str(yaml_path)]) == 2 mock_read.assert_called_once() def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None: - """`compile` always re-validates, even with a fresh cache on disk. - - The fast path is only for upload / logs -- compile is what writes - the cache in the first place, and it needs a fully validated - config to drive code generation. - """ - yaml_path = fresh_cache_files - + """compile always re-validates -- it's what writes the cache.""" with ( patch("esphome.__main__.read_config", return_value=None) as mock_read, patch.dict( @@ -256,6 +203,6 @@ def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None {"compile": lambda args, config: 0}, ), ): - run_esphome(["esphome", "compile", str(yaml_path)]) + run_esphome(["esphome", "compile", str(fresh_cache_files)]) mock_read.assert_called_once() From cc6245b87675b1156b3a3f4bbf45d0165b604edd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 15:53:46 -0500 Subject: [PATCH 09/16] [core] Derive CORE state from cached config dict, drop apply_to_core `StorageJSON.apply_to_core` was the inverse of `from_esphome_core`: two enumerations of the same fields that had to stay in sync. When a new CORE attribute joined the validation pipeline, both lists needed an update. Replace it with `_populate_core_from_validated_config(config)` in `compiled_config.py` that reads the cached config dict directly -- same source `read_config` writes into during validation, no separate sidecar schema. The fast path no longer touches StorageJSON at all; the sidecar stays in place for the dashboard's existing use. Drift surface shrinks to: name/friendly_name/build_path under the esphome block (config schema is stable) + target_platform discovered via `_is_target_platform` (the canonical detector) + framework via the `.framework.type` convention every platform component already follows. --- esphome/compiled_config.py | 81 ++++++++++++++++++++---- esphome/storage_json.py | 27 -------- tests/unit_tests/test_compiled_config.py | 61 ++++++------------ 3 files changed, 85 insertions(+), 84 deletions(-) diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py index bb127a0de98..b0c8f563192 100644 --- a/esphome/compiled_config.py +++ b/esphome/compiled_config.py @@ -13,8 +13,7 @@ paths) round-trip cleanly, and so the small ``StorageJSON`` sidecar isn't bloated by configs that can grow past a megabyte. Staleness is gated by an mtime check against the source YAML; the loader falls back to ``None`` (and the caller to ``read_config()``) whenever -the cache is missing, stale, unparseable, or the companion -``StorageJSON`` sidecar can't be loaded. +the cache is missing, stale, unparseable, or structurally incomplete. """ from __future__ import annotations @@ -22,14 +21,63 @@ from __future__ import annotations import logging from pathlib import Path +from esphome.const import ( + CONF_BUILD_PATH, + CONF_ESPHOME, + CONF_FRAMEWORK, + CONF_FRIENDLY_NAME, + CONF_NAME, + CONF_TYPE, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, +) from esphome.core import CORE from esphome.helpers import write_file_if_changed -from esphome.storage_json import StorageJSON, ext_storage_path from esphome.types import ConfigType _LOGGER = logging.getLogger(__name__) +def _populate_core_from_validated_config(config: ConfigType) -> None: + """Set up ``CORE`` from an already-validated config dict. + + Reads from the same config keys ``preload_core_config`` and the + target-platform component validator would write into during a full + ``read_config`` -- using the cached config dict as the canonical + source instead of duplicating the field list in a separate sidecar + schema. ``CORE.address`` and ``CORE.web_port`` aren't set here: + they're already properties that derive themselves from + ``CORE.config`` on access, which the dispatcher assigns next. + """ + from esphome.core.config import _is_target_platform + + esphome_block = config[CONF_ESPHOME] + CORE.name = esphome_block[CONF_NAME] + CORE.friendly_name = esphome_block.get(CONF_FRIENDLY_NAME) + if (build_path := esphome_block.get(CONF_BUILD_PATH)) is not None: + CORE.build_path = CORE.data_dir / build_path + + CORE.data.setdefault(KEY_CORE, {}) + for domain, sub in config.items(): + if not isinstance(domain, str) or not _is_target_platform(domain): + continue + CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = domain + # Every platform component follows the + # `.framework.type` convention to express which + # framework the binary was built against (esp-idf, arduino, + # zephyr). A platform without a framework block (host) just + # leaves KEY_TARGET_FRAMEWORK unset, matching what + # `read_config` would do. + if ( + isinstance(sub, dict) + and isinstance((framework := sub.get(CONF_FRAMEWORK)), dict) + and (framework_type := framework.get(CONF_TYPE)) is not None + ): + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = framework_type + break + + def compiled_config_path(config_filename: str) -> Path: """Path to the cached validated config alongside the storage sidecar.""" return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml" @@ -64,17 +112,19 @@ def save_compiled_config(config: ConfigType) -> None: def load_compiled_config(conf_path: Path) -> ConfigType | None: - """Load the cached validated config and apply storage metadata to CORE. + """Load the cached validated config and set up ``CORE``. - Single entry point for the ``upload`` / ``logs`` fast path: loads - the cache, applies the ``StorageJSON`` sidecar's platform / build - metadata to ``CORE``, and returns the config dict. Returns + Single entry point for the ``upload`` / ``logs`` fast path. Returns ``None`` (so the caller falls back to ``read_config``) when the cache is missing, older than the source YAML, unparseable, or - when the sidecar can't be loaded. The mtime check catches the - common "user edited the YAML and forgot to recompile" case; - deeper drift (an edited ``!include`` whose parent YAML mtime - didn't change) is the user's responsibility. + structurally incomplete. The mtime check catches the common "user + edited the YAML and forgot to recompile" case; deeper drift (an + edited ``!include`` whose parent YAML mtime didn't change) is the + user's responsibility. + + CORE state is derived from the cached config dict directly -- + same source ``read_config`` uses -- so there's no separate sidecar + schema to keep in sync. """ cache_path = compiled_config_path(conf_path.name) if not _cache_is_fresh(cache_path, conf_path): @@ -89,8 +139,11 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None: except Exception: # pylint: disable=broad-except return None - storage = StorageJSON.load(ext_storage_path(conf_path.name)) - if storage is None: + if not isinstance(config, dict) or CONF_ESPHOME not in config: + return None + + try: + _populate_core_from_validated_config(config) + except (KeyError, TypeError): return None - storage.apply_to_core() return config diff --git a/esphome/storage_json.py b/esphome/storage_json.py index a9a83f58962..c6df16ce78d 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -256,33 +256,6 @@ class StorageJSON: except Exception: # pylint: disable=broad-except return None - def apply_to_core(self) -> None: - """Populate ``CORE`` from this sidecar. - - Used by the ``--from-storage-json`` fast path in - ``esphome upload`` / ``esphome logs``: those subcommands read - a handful of ``CORE`` attributes (``target_platform``, - ``build_path``, ``name``, ``loaded_integrations``) that the - normal flow populates during ``read_config``. Lifting them off - the sidecar lets us skip the validation pass entirely. - """ - from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM - - CORE.name = self.name - CORE.friendly_name = self.friendly_name - CORE.build_path = self.build_path - CORE.loaded_integrations = set(self.loaded_integrations) - CORE.loaded_platforms = set(self.loaded_platforms) - - core_platform = self.core_platform or ( - self.target_platform.lower() if self.target_platform else None - ) - CORE.data.setdefault(KEY_CORE, {}) - if core_platform is not None: - CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = core_platform - if self.framework is not None: - CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = self.framework - def __eq__(self, o) -> bool: return isinstance(o, StorageJSON) and self.as_dict() == o.as_dict() diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index d4736f38a35..4efba4e0c7a 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os from pathlib import Path from unittest.mock import patch @@ -25,8 +24,11 @@ _VALIDATED_CONFIG_YAML = """\ esphome: name: lite_test friendly_name: Lite Test Device + build_path: build/lite_test esp32: board: nodemcu-32s + framework: + type: arduino logger: baud_rate: 115200 api: @@ -43,30 +45,6 @@ wifi: """ -def _write_storage(storage_path: Path) -> None: - """Write a vanilla StorageJSON sidecar for the cache tests.""" - storage_path.parent.mkdir(parents=True, exist_ok=True) - data = { - "storage_version": 1, - "name": "lite_test", - "friendly_name": "Lite Test Device", - "comment": None, - "esphome_version": "2026.1.0", - "src_version": 1, - "address": "192.168.1.42", - "web_port": None, - "esp_platform": "ESP32", - "build_path": "/build/lite_test", - "firmware_bin_path": "/build/lite_test/firmware.bin", - "loaded_integrations": ["api", "logger", "ota", "wifi"], - "loaded_platforms": [], - "no_mdns": False, - "framework": "arduino", - "core_platform": "esp32", - } - storage_path.write_text(json.dumps(data)) - - def _write_cache(cache_path: Path, body: str = _VALIDATED_CONFIG_YAML) -> Path: """Write the cache file and return it.""" cache_path.parent.mkdir(parents=True, exist_ok=True) @@ -85,14 +63,14 @@ def _set_cache_mtime(cache_path: Path, yaml_path: Path, *, offset: int) -> None: @pytest.fixture def fresh_cache_files(tmp_path: Path) -> Path: - """YAML + StorageJSON + cache, all consistent and fresh.""" + """YAML + cache, both consistent and fresh.""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path - storage_dir = tmp_path / ".esphome" / "storage" - _write_storage(storage_dir / "lite_test.yaml.json") - cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + cache = _write_cache( + tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.yaml" + ) _set_cache_mtime(cache, yaml_path, offset=5) return yaml_path @@ -106,7 +84,7 @@ def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None: def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: - """Fresh cache + sidecar → returns config and populates CORE.""" + """Fresh cache → returns the config and populates CORE from it.""" config = load_compiled_config(fresh_cache_files) assert config is not None @@ -114,40 +92,37 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" assert config["ota"][0]["password"] == "secret" - # apply_to_core ran as part of the orchestration. + # CORE state derives from the cached config dict -- same source + # `read_config` uses, no separate sidecar schema. assert CORE.name == "lite_test" - assert CORE.build_path == Path("/build/lite_test") + assert CORE.friendly_name == "Lite Test Device" + assert CORE.build_path == CORE.data_dir / "build/lite_test" assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" - assert "api" in CORE.loaded_integrations @pytest.mark.parametrize( "scenario", - ["missing_cache", "stale_cache", "corrupt_cache", "missing_sidecar"], + ["missing_cache", "stale_cache", "corrupt_cache", "missing_esphome_block"], ) def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None: """All non-happy cases return None so the caller falls back.""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path - storage_dir = tmp_path / ".esphome" / "storage" - cache_path = storage_dir / "lite_test.yaml.validated.yaml" - sidecar_path = storage_dir / "lite_test.yaml.json" + cache_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.yaml" if scenario == "missing_cache": - pass # no cache, no sidecar + pass # no cache elif scenario == "stale_cache": - _write_storage(sidecar_path) _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=-60) elif scenario == "corrupt_cache": - _write_storage(sidecar_path) _set_cache_mtime( _write_cache(cache_path, "not: valid: yaml: ["), yaml_path, offset=5 ) - elif scenario == "missing_sidecar": - # Cache fresh + parseable, but no StorageJSON → can't populate CORE. - _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=5) + elif scenario == "missing_esphome_block": + # Parseable YAML but no esphome: block -- can't populate CORE. + _set_cache_mtime(_write_cache(cache_path, "logger:\n"), yaml_path, offset=5) assert load_compiled_config(yaml_path) is None From 5709ade5775459fca510892cc3751526673b92e8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 15:54:23 -0500 Subject: [PATCH 10/16] Revert "[core] Derive CORE state from cached config dict, drop apply_to_core" This reverts commit cc6245b87675b1156b3a3f4bbf45d0165b604edd. --- esphome/compiled_config.py | 81 ++++-------------------- esphome/storage_json.py | 27 ++++++++ tests/unit_tests/test_compiled_config.py | 61 ++++++++++++------ 3 files changed, 84 insertions(+), 85 deletions(-) diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py index b0c8f563192..bb127a0de98 100644 --- a/esphome/compiled_config.py +++ b/esphome/compiled_config.py @@ -13,7 +13,8 @@ paths) round-trip cleanly, and so the small ``StorageJSON`` sidecar isn't bloated by configs that can grow past a megabyte. Staleness is gated by an mtime check against the source YAML; the loader falls back to ``None`` (and the caller to ``read_config()``) whenever -the cache is missing, stale, unparseable, or structurally incomplete. +the cache is missing, stale, unparseable, or the companion +``StorageJSON`` sidecar can't be loaded. """ from __future__ import annotations @@ -21,63 +22,14 @@ from __future__ import annotations import logging from pathlib import Path -from esphome.const import ( - CONF_BUILD_PATH, - CONF_ESPHOME, - CONF_FRAMEWORK, - CONF_FRIENDLY_NAME, - CONF_NAME, - CONF_TYPE, - KEY_CORE, - KEY_TARGET_FRAMEWORK, - KEY_TARGET_PLATFORM, -) from esphome.core import CORE from esphome.helpers import write_file_if_changed +from esphome.storage_json import StorageJSON, ext_storage_path from esphome.types import ConfigType _LOGGER = logging.getLogger(__name__) -def _populate_core_from_validated_config(config: ConfigType) -> None: - """Set up ``CORE`` from an already-validated config dict. - - Reads from the same config keys ``preload_core_config`` and the - target-platform component validator would write into during a full - ``read_config`` -- using the cached config dict as the canonical - source instead of duplicating the field list in a separate sidecar - schema. ``CORE.address`` and ``CORE.web_port`` aren't set here: - they're already properties that derive themselves from - ``CORE.config`` on access, which the dispatcher assigns next. - """ - from esphome.core.config import _is_target_platform - - esphome_block = config[CONF_ESPHOME] - CORE.name = esphome_block[CONF_NAME] - CORE.friendly_name = esphome_block.get(CONF_FRIENDLY_NAME) - if (build_path := esphome_block.get(CONF_BUILD_PATH)) is not None: - CORE.build_path = CORE.data_dir / build_path - - CORE.data.setdefault(KEY_CORE, {}) - for domain, sub in config.items(): - if not isinstance(domain, str) or not _is_target_platform(domain): - continue - CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = domain - # Every platform component follows the - # `.framework.type` convention to express which - # framework the binary was built against (esp-idf, arduino, - # zephyr). A platform without a framework block (host) just - # leaves KEY_TARGET_FRAMEWORK unset, matching what - # `read_config` would do. - if ( - isinstance(sub, dict) - and isinstance((framework := sub.get(CONF_FRAMEWORK)), dict) - and (framework_type := framework.get(CONF_TYPE)) is not None - ): - CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = framework_type - break - - def compiled_config_path(config_filename: str) -> Path: """Path to the cached validated config alongside the storage sidecar.""" return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml" @@ -112,19 +64,17 @@ def save_compiled_config(config: ConfigType) -> None: def load_compiled_config(conf_path: Path) -> ConfigType | None: - """Load the cached validated config and set up ``CORE``. + """Load the cached validated config and apply storage metadata to CORE. - Single entry point for the ``upload`` / ``logs`` fast path. Returns + Single entry point for the ``upload`` / ``logs`` fast path: loads + the cache, applies the ``StorageJSON`` sidecar's platform / build + metadata to ``CORE``, and returns the config dict. Returns ``None`` (so the caller falls back to ``read_config``) when the cache is missing, older than the source YAML, unparseable, or - structurally incomplete. The mtime check catches the common "user - edited the YAML and forgot to recompile" case; deeper drift (an - edited ``!include`` whose parent YAML mtime didn't change) is the - user's responsibility. - - CORE state is derived from the cached config dict directly -- - same source ``read_config`` uses -- so there's no separate sidecar - schema to keep in sync. + when the sidecar can't be loaded. The mtime check catches the + common "user edited the YAML and forgot to recompile" case; + deeper drift (an edited ``!include`` whose parent YAML mtime + didn't change) is the user's responsibility. """ cache_path = compiled_config_path(conf_path.name) if not _cache_is_fresh(cache_path, conf_path): @@ -139,11 +89,8 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None: except Exception: # pylint: disable=broad-except return None - if not isinstance(config, dict) or CONF_ESPHOME not in config: - return None - - try: - _populate_core_from_validated_config(config) - except (KeyError, TypeError): + storage = StorageJSON.load(ext_storage_path(conf_path.name)) + if storage is None: return None + storage.apply_to_core() return config diff --git a/esphome/storage_json.py b/esphome/storage_json.py index c6df16ce78d..a9a83f58962 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -256,6 +256,33 @@ class StorageJSON: except Exception: # pylint: disable=broad-except return None + def apply_to_core(self) -> None: + """Populate ``CORE`` from this sidecar. + + Used by the ``--from-storage-json`` fast path in + ``esphome upload`` / ``esphome logs``: those subcommands read + a handful of ``CORE`` attributes (``target_platform``, + ``build_path``, ``name``, ``loaded_integrations``) that the + normal flow populates during ``read_config``. Lifting them off + the sidecar lets us skip the validation pass entirely. + """ + from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM + + CORE.name = self.name + CORE.friendly_name = self.friendly_name + CORE.build_path = self.build_path + CORE.loaded_integrations = set(self.loaded_integrations) + CORE.loaded_platforms = set(self.loaded_platforms) + + core_platform = self.core_platform or ( + self.target_platform.lower() if self.target_platform else None + ) + CORE.data.setdefault(KEY_CORE, {}) + if core_platform is not None: + CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = core_platform + if self.framework is not None: + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = self.framework + def __eq__(self, o) -> bool: return isinstance(o, StorageJSON) and self.as_dict() == o.as_dict() diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 4efba4e0c7a..d4736f38a35 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os from pathlib import Path from unittest.mock import patch @@ -24,11 +25,8 @@ _VALIDATED_CONFIG_YAML = """\ esphome: name: lite_test friendly_name: Lite Test Device - build_path: build/lite_test esp32: board: nodemcu-32s - framework: - type: arduino logger: baud_rate: 115200 api: @@ -45,6 +43,30 @@ wifi: """ +def _write_storage(storage_path: Path) -> None: + """Write a vanilla StorageJSON sidecar for the cache tests.""" + storage_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "storage_version": 1, + "name": "lite_test", + "friendly_name": "Lite Test Device", + "comment": None, + "esphome_version": "2026.1.0", + "src_version": 1, + "address": "192.168.1.42", + "web_port": None, + "esp_platform": "ESP32", + "build_path": "/build/lite_test", + "firmware_bin_path": "/build/lite_test/firmware.bin", + "loaded_integrations": ["api", "logger", "ota", "wifi"], + "loaded_platforms": [], + "no_mdns": False, + "framework": "arduino", + "core_platform": "esp32", + } + storage_path.write_text(json.dumps(data)) + + def _write_cache(cache_path: Path, body: str = _VALIDATED_CONFIG_YAML) -> Path: """Write the cache file and return it.""" cache_path.parent.mkdir(parents=True, exist_ok=True) @@ -63,14 +85,14 @@ def _set_cache_mtime(cache_path: Path, yaml_path: Path, *, offset: int) -> None: @pytest.fixture def fresh_cache_files(tmp_path: Path) -> Path: - """YAML + cache, both consistent and fresh.""" + """YAML + StorageJSON + cache, all consistent and fresh.""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path - cache = _write_cache( - tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.yaml" - ) + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json") + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") _set_cache_mtime(cache, yaml_path, offset=5) return yaml_path @@ -84,7 +106,7 @@ def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None: def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: - """Fresh cache → returns the config and populates CORE from it.""" + """Fresh cache + sidecar → returns config and populates CORE.""" config = load_compiled_config(fresh_cache_files) assert config is not None @@ -92,37 +114,40 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" assert config["ota"][0]["password"] == "secret" - # CORE state derives from the cached config dict -- same source - # `read_config` uses, no separate sidecar schema. + # apply_to_core ran as part of the orchestration. assert CORE.name == "lite_test" - assert CORE.friendly_name == "Lite Test Device" - assert CORE.build_path == CORE.data_dir / "build/lite_test" + assert CORE.build_path == Path("/build/lite_test") assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" + assert "api" in CORE.loaded_integrations @pytest.mark.parametrize( "scenario", - ["missing_cache", "stale_cache", "corrupt_cache", "missing_esphome_block"], + ["missing_cache", "stale_cache", "corrupt_cache", "missing_sidecar"], ) def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None: """All non-happy cases return None so the caller falls back.""" yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path - cache_path = tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.yaml" + storage_dir = tmp_path / ".esphome" / "storage" + cache_path = storage_dir / "lite_test.yaml.validated.yaml" + sidecar_path = storage_dir / "lite_test.yaml.json" if scenario == "missing_cache": - pass # no cache + pass # no cache, no sidecar elif scenario == "stale_cache": + _write_storage(sidecar_path) _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=-60) elif scenario == "corrupt_cache": + _write_storage(sidecar_path) _set_cache_mtime( _write_cache(cache_path, "not: valid: yaml: ["), yaml_path, offset=5 ) - elif scenario == "missing_esphome_block": - # Parseable YAML but no esphome: block -- can't populate CORE. - _set_cache_mtime(_write_cache(cache_path, "logger:\n"), yaml_path, offset=5) + elif scenario == "missing_sidecar": + # Cache fresh + parseable, but no StorageJSON → can't populate CORE. + _set_cache_mtime(_write_cache(cache_path), yaml_path, offset=5) assert load_compiled_config(yaml_path) is None From a19e817d28f4873fdab42cc5505f46a64fb2fdd6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 17:29:29 -0500 Subject: [PATCH 11/16] [core] Shrink apply_to_core to what upload/logs actually read apply_to_core was over-populating: it restored friendly_name, loaded_integrations, and loaded_platforms even though every consumer of those three lives inside a component validator (esp32_camera, esp32, deep_sleep, zigbee, lvgl, zephyr_mcumgr), and the whole point of the fast path is to skip validation. Drop them. CORE.__init__ already leaves all three at safe defaults (None / empty set) for any incidental reader. What's left is exactly what upload/logs walk: - CORE.name (api.client.run_logs, firmware_bin path, mDNS) - CORE.build_path (firmware_bin / partition_table_bin / bootloader_bin) - CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] (module dispatch, .is_esp32 etc) - CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] (.is_arduino, firmware_bin branch) Method body shrinks from 9 statements to 4; setdefault + two conditional inserts collapse into one dict literal; the function-local import moves to module top. Drift surface drops from 7 paired fields to 4. The wizard-only-sidecar None case is gated once at the load_compiled_config boundary so apply_to_core no longer has to defend against it. --- esphome/compiled_config.py | 5 +++ esphome/storage_json.py | 42 +++++++++++------------- tests/unit_tests/test_compiled_config.py | 10 ++++-- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py index bb127a0de98..ac817bd9066 100644 --- a/esphome/compiled_config.py +++ b/esphome/compiled_config.py @@ -92,5 +92,10 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None: storage = StorageJSON.load(ext_storage_path(conf_path.name)) if storage is None: return None + # `apply_to_core` assumes the sidecar was written by `from_esphome_core` + # after a real compile, which always sets at least one of these. A + # wizard-only sidecar (no compile) can't drive upload/logs. + if not storage.core_platform and not storage.target_platform: + return None storage.apply_to_core() return config diff --git a/esphome/storage_json.py b/esphome/storage_json.py index a9a83f58962..0fd4a5cc0a8 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -8,7 +8,13 @@ import os from pathlib import Path from esphome import const -from esphome.const import CONF_DISABLED, CONF_MDNS +from esphome.const import ( + CONF_DISABLED, + CONF_MDNS, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, +) from esphome.core import CORE from esphome.helpers import write_file_if_changed from esphome.types import CoreType @@ -257,31 +263,23 @@ class StorageJSON: return None def apply_to_core(self) -> None: - """Populate ``CORE`` from this sidecar. + """Populate CORE with the metadata upload/logs read. - Used by the ``--from-storage-json`` fast path in - ``esphome upload`` / ``esphome logs``: those subcommands read - a handful of ``CORE`` attributes (``target_platform``, - ``build_path``, ``name``, ``loaded_integrations``) that the - normal flow populates during ``read_config``. Lifting them off - the sidecar lets us skip the validation pass entirely. + Inverse of :meth:`from_esphome_core`'s CORE→StorageJSON + projection. Keep paired -- a new CORE attribute the + ``upload`` / ``logs`` fast path needs has to be captured by + ``from_esphome_core`` too. Validators (``loaded_integrations``, + ``loaded_platforms``, ``friendly_name``) are deliberately not + restored: they're consumed by component validation, which the + fast path skips, and ``CORE.__init__`` already leaves them at + safe defaults. """ - from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM - CORE.name = self.name - CORE.friendly_name = self.friendly_name CORE.build_path = self.build_path - CORE.loaded_integrations = set(self.loaded_integrations) - CORE.loaded_platforms = set(self.loaded_platforms) - - core_platform = self.core_platform or ( - self.target_platform.lower() if self.target_platform else None - ) - CORE.data.setdefault(KEY_CORE, {}) - if core_platform is not None: - CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = core_platform - if self.framework is not None: - CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = self.framework + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: self.core_platform or self.target_platform.lower(), + KEY_TARGET_FRAMEWORK: self.framework, + } def __eq__(self, o) -> bool: return isinstance(o, StorageJSON) and self.as_dict() == o.as_dict() diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index d4736f38a35..5dc6df6944d 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -114,12 +114,18 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" assert config["ota"][0]["password"] == "secret" - # apply_to_core ran as part of the orchestration. + # apply_to_core populated exactly what upload/logs read off CORE. assert CORE.name == "lite_test" assert CORE.build_path == Path("/build/lite_test") assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" - assert "api" in CORE.loaded_integrations + + # The validator-only attributes are deliberately left at their + # CORE.__init__ defaults. The fast path skips validation, so + # nothing reads these. + assert CORE.loaded_integrations == set() + assert CORE.loaded_platforms == set() + assert CORE.friendly_name is None @pytest.mark.parametrize( From e7af2aaf9bb3b023e26bf368c17a210d9c734cc4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 17:39:43 -0500 Subject: [PATCH 12/16] [core] Address review: secrets perms, mtime freshness, substitution regression, tests - save_compiled_config: write the cache with mode 0600 (the YAML contains resolved secrets) and always rewrite so mtime advances even when content is unchanged. Previously write_file_if_changed could leave mtime pinned behind a whitespace YAML edit, locking the fast path to 'stale'. - __main__: skip the fast path when -s overrides are passed. The cache was written against the previous substitution set; reusing it would silently ignore the new values. - Add tests for save_compiled_config (write + swallow-errors), the wizard-only sidecar fallback, and the substitution-skip branch. - Trim docstrings/comments throughout. --- esphome/__main__.py | 13 ++-- esphome/compiled_config.py | 55 +++++------------ esphome/storage_json.py | 13 ++-- esphome/writer.py | 5 +- tests/unit_tests/test_compiled_config.py | 78 +++++++++++++++++++++--- 5 files changed, 96 insertions(+), 68 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index d68dcffaf22..d5a9b698331 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2419,15 +2419,12 @@ def run_esphome(argv): skip_external = args.command in ("logs", "clean") command_line_substitutions = dict(args.substitution) if args.substitution else {} - # Fast path for `upload` and `logs`: reuse the validated config the - # last compile cached on disk. Skips parse + schema validate + - # final-validate + external-component refresh, which is dead work - # for those subcommands -- the binary on disk is already known good - # and the caller already knows the device address. Falls back to a - # full `read_config()` when the cache is missing/stale/corrupt so a - # cold cache never produces a worse outcome than today. + # Fast path for upload/logs: reuse the validated-config cache the + # last compile wrote. Falls back to read_config when missing/stale. + # Skipped when -s overrides are passed, since the cache was written + # against the previous substitution set. config = None - if args.command in ("upload", "logs"): + if args.command in ("upload", "logs") and not command_line_substitutions: from esphome.compiled_config import load_compiled_config config = load_compiled_config(conf_path) diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py index ac817bd9066..92cbb7348a4 100644 --- a/esphome/compiled_config.py +++ b/esphome/compiled_config.py @@ -1,20 +1,9 @@ -"""Validated-config cache for the ``upload`` / ``logs`` fast path. +"""Validated-config cache for the upload/logs fast path. -After every successful ``esphome compile``, the writer dumps the -validated config to ``/storage/.validated.yaml``. -The next ``esphome upload`` / ``esphome logs`` for that YAML reuses -the cache instead of re-running the full ``read_config()`` pipeline -(parse + schema validate + final-validate + external-component -refresh), which is dead work once the binary on disk is known good. - -The cache uses YAML (via ``yaml_util``) rather than JSON so all -esphome-specific tags (``!lambda``, ``!include``, ``ID`` instances, -paths) round-trip cleanly, and so the small ``StorageJSON`` sidecar -isn't bloated by configs that can grow past a megabyte. Staleness -is gated by an mtime check against the source YAML; the loader -falls back to ``None`` (and the caller to ``read_config()``) whenever -the cache is missing, stale, unparseable, or the companion -``StorageJSON`` sidecar can't be loaded. +compile dumps the validated config to /storage/.validated.yaml; +the next upload/logs for that YAML reuses it instead of running the full +read_config pipeline. YAML round-trip (yaml_util.dump/load_yaml) keeps +!lambda/!include/IDs/paths intact; mtime gates staleness. """ from __future__ import annotations @@ -23,7 +12,7 @@ import logging from pathlib import Path from esphome.core import CORE -from esphome.helpers import write_file_if_changed +from esphome.helpers import write_file from esphome.storage_json import StorageJSON, ext_storage_path from esphome.types import ConfigType @@ -44,21 +33,16 @@ def _cache_is_fresh(cache_path: Path, source_path: Path) -> bool: def save_compiled_config(config: ConfigType) -> None: - """Dump the validated config to its sidecar YAML file. + """Write the validated-config cache. Always-write so mtime stays fresh. - Called from the writer after every successful compile so the next - ``esphome upload`` / ``esphome logs`` for this YAML can skip - validation via :func:`load_compiled_config`. Failures here are - non-fatal: the worst case is that the fast path falls back to a - full ``read_config`` next time. + Mode 0600 because show_secrets=True resolves !secret inline. + Failures are non-fatal: the fast path falls back to read_config. """ from esphome import yaml_util try: - # show_secrets=True keeps the cache self-contained; it lives - # in the same trust zone as the rest of .esphome/storage/. rendered = yaml_util.dump(config, show_secrets=True) - write_file_if_changed(compiled_config_path(CORE.config_filename), rendered) + write_file(compiled_config_path(CORE.config_filename), rendered, private=True) except Exception as err: # pylint: disable=broad-except _LOGGER.debug("Skipping compiled config cache write: %s", err) @@ -66,15 +50,9 @@ def save_compiled_config(config: ConfigType) -> None: def load_compiled_config(conf_path: Path) -> ConfigType | None: """Load the cached validated config and apply storage metadata to CORE. - Single entry point for the ``upload`` / ``logs`` fast path: loads - the cache, applies the ``StorageJSON`` sidecar's platform / build - metadata to ``CORE``, and returns the config dict. Returns - ``None`` (so the caller falls back to ``read_config``) when the - cache is missing, older than the source YAML, unparseable, or - when the sidecar can't be loaded. The mtime check catches the - common "user edited the YAML and forgot to recompile" case; - deeper drift (an edited ``!include`` whose parent YAML mtime - didn't change) is the user's responsibility. + Returns None (caller falls back to read_config) when the cache is + missing, older than the source YAML, unparseable, or the sidecar + is incomplete. """ cache_path = compiled_config_path(conf_path.name) if not _cache_is_fresh(cache_path, conf_path): @@ -83,8 +61,6 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None: from esphome import yaml_util try: - # clear_secrets=False keeps in-flight secret state intact; the - # cache is self-contained and resolves no !secret references. config = yaml_util.load_yaml(cache_path, clear_secrets=False) except Exception: # pylint: disable=broad-except return None @@ -92,9 +68,8 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None: storage = StorageJSON.load(ext_storage_path(conf_path.name)) if storage is None: return None - # `apply_to_core` assumes the sidecar was written by `from_esphome_core` - # after a real compile, which always sets at least one of these. A - # wizard-only sidecar (no compile) can't drive upload/logs. + # apply_to_core assumes a real compile wrote the sidecar; wizard-only + # sidecars leave both of these unset and can't drive upload/logs. if not storage.core_platform and not storage.target_platform: return None storage.apply_to_core() diff --git a/esphome/storage_json.py b/esphome/storage_json.py index 0fd4a5cc0a8..7d26b22f96c 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -265,14 +265,11 @@ class StorageJSON: def apply_to_core(self) -> None: """Populate CORE with the metadata upload/logs read. - Inverse of :meth:`from_esphome_core`'s CORE→StorageJSON - projection. Keep paired -- a new CORE attribute the - ``upload`` / ``logs`` fast path needs has to be captured by - ``from_esphome_core`` too. Validators (``loaded_integrations``, - ``loaded_platforms``, ``friendly_name``) are deliberately not - restored: they're consumed by component validation, which the - fast path skips, and ``CORE.__init__`` already leaves them at - safe defaults. + Inverse of :meth:`from_esphome_core`. Keep paired -- a new + attribute upload/logs needs has to be captured there too. + Validator-only fields (loaded_integrations/platforms, + friendly_name) are skipped; the fast path doesn't run + validation and CORE.__init__ defaults them. """ CORE.name = self.name CORE.build_path = self.build_path diff --git a/esphome/writer.py b/esphome/writer.py index 54eb254b1b8..cf04e4f8d2a 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -111,10 +111,7 @@ def update_storage_json() -> None: old = StorageJSON.load(path) new = StorageJSON.from_esphome_core(CORE, old) - # Refresh the validated-config cache. `esphome upload` and - # `esphome logs` look for it next time they run and skip the full - # validation pipeline when it's present and fresh (mtime >= YAML - # mtime). Cheap to write, big win to read. + # Refresh the cache upload/logs read on the next call. if CORE.config is not None: save_compiled_config(CORE.config) diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 5dc6df6944d..1fda7bf5bd7 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -10,7 +10,11 @@ from unittest.mock import patch import pytest from esphome.__main__ import run_esphome -from esphome.compiled_config import compiled_config_path, load_compiled_config +from esphome.compiled_config import ( + compiled_config_path, + load_compiled_config, + save_compiled_config, +) from esphome.const import ( CONF_API, CONF_ESPHOME, @@ -120,13 +124,6 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" - # The validator-only attributes are deliberately left at their - # CORE.__init__ defaults. The fast path skips validation, so - # nothing reads these. - assert CORE.loaded_integrations == set() - assert CORE.loaded_platforms == set() - assert CORE.friendly_name is None - @pytest.mark.parametrize( "scenario", @@ -200,6 +197,24 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache( mock_read.assert_called_once() +def test_run_esphome_upload_with_substitution_skips_cache( + fresh_cache_files: Path, +) -> None: + """`-s key value` forces a fresh validation -- the cache was written + against the prior substitution set, so reusing it would silently + ignore the override.""" + with ( + patch("esphome.__main__.read_config", return_value=None) as mock_read, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "-s", "var", "val", "upload", str(fresh_cache_files)]) + + mock_read.assert_called_once() + + def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None: """compile always re-validates -- it's what writes the cache.""" with ( @@ -212,3 +227,50 @@ def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None run_esphome(["esphome", "compile", str(fresh_cache_files)]) mock_read.assert_called_once() + + +def test_save_compiled_config_writes_cache(tmp_path: Path) -> None: + """`save_compiled_config` writes the dumped YAML next to the sidecar.""" + CORE.config_path = tmp_path / "lite_test.yaml" + save_compiled_config({"esphome": {"name": "lite_test"}, "logger": {}}) + + cache_path = compiled_config_path("lite_test.yaml") + assert cache_path.is_file() + body = cache_path.read_text() + assert "name: lite_test" in body + assert "logger:" in body + + +def test_save_compiled_config_swallows_dump_errors( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Failures during the dump are non-fatal -- a bad cache just means + the next fast path falls back to read_config().""" + CORE.config_path = tmp_path / "lite_test.yaml" + with patch("esphome.yaml_util.dump", side_effect=RuntimeError("boom")): + save_compiled_config({"esphome": {"name": "lite_test"}}) + assert not compiled_config_path("lite_test.yaml").exists() + + +def test_load_compiled_config_rejects_wizard_only_sidecar(tmp_path: Path) -> None: + """A wizard-only sidecar (no compile -- no core_platform / target_platform) + can't drive upload/logs, so the fast path falls back.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + storage_dir.mkdir(parents=True, exist_ok=True) + # StorageJSON with both core_platform and target_platform unset. + (storage_dir / "lite_test.yaml.json").write_text( + '{"storage_version": 1, "name": "lite_test", "friendly_name": null, ' + '"comment": null, "esphome_version": null, "src_version": 1, ' + '"address": null, "web_port": null, "esp_platform": null, ' + '"build_path": null, "firmware_bin_path": null, ' + '"loaded_integrations": [], "loaded_platforms": [], "no_mdns": false, ' + '"framework": null, "core_platform": null}' + ) + cache_path = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache_path, yaml_path, offset=5) + + assert load_compiled_config(yaml_path) is None From 1902519f3780e2bf61599a27ce836d967c1e3b7b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 17:48:17 -0500 Subject: [PATCH 13/16] [core] Fix Windows path separator in cache path test --- tests/unit_tests/test_compiled_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 1fda7bf5bd7..89542fad797 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -104,9 +104,9 @@ def fresh_cache_files(tmp_path: Path) -> Path: def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None: """The cache file shape is predictable from the YAML filename.""" - assert str(compiled_config_path("device.yaml")).endswith( - "storage/device.yaml.validated.yaml" - ) + path = compiled_config_path("device.yaml") + assert path.name == "device.yaml.validated.yaml" + assert path.parent.name == "storage" def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: From bcd07abb5fc957a2576deafa089cb12230f66919 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 17:48:44 -0500 Subject: [PATCH 14/16] [core] Fix flake8 D403 in test docstring --- tests/unit_tests/test_compiled_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 89542fad797..3db6965c1ff 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -216,7 +216,7 @@ def test_run_esphome_upload_with_substitution_skips_cache( def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None: - """compile always re-validates -- it's what writes the cache.""" + """The compile subcommand always re-validates -- it's what writes the cache.""" with ( patch("esphome.__main__.read_config", return_value=None) as mock_read, patch.dict( From c056bb317767cee18b09a5bb9db4736e1775346f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 17:49:03 -0500 Subject: [PATCH 15/16] [core] Type-annotate config local in dispatcher --- esphome/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index d5a9b698331..b492aa67c13 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2423,7 +2423,7 @@ def run_esphome(argv): # last compile wrote. Falls back to read_config when missing/stale. # Skipped when -s overrides are passed, since the cache was written # against the previous substitution set. - config = None + config: ConfigType | None = None if args.command in ("upload", "logs") and not command_line_substitutions: from esphome.compiled_config import load_compiled_config From b8c2d66c0eba0fce59eceb07a1fec15e1b20926f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 12 May 2026 17:51:49 -0500 Subject: [PATCH 16/16] [core] Assert on cache-hit log line so coverage is unambiguous --- tests/unit_tests/test_compiled_config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 3db6965c1ff..34e811b97bd 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -157,7 +157,9 @@ def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None: @pytest.mark.parametrize("command", ["upload", "logs"]) def test_run_esphome_upload_and_logs_use_cache_when_fresh( - command: str, fresh_cache_files: Path + command: str, + fresh_cache_files: Path, + caplog: pytest.LogCaptureFixture, ) -> None: """upload/logs skip read_config() when the cache is fresh.""" captured: dict = {} @@ -167,6 +169,7 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh( return 0 with ( + caplog.at_level("INFO", logger="esphome.__main__"), patch("esphome.__main__.read_config") as mock_read, patch.dict("esphome.__main__.POST_CONFIG_ACTIONS", {command: _stub}), ): @@ -175,6 +178,9 @@ def test_run_esphome_upload_and_logs_use_cache_when_fresh( mock_read.assert_not_called() assert captured["config"][CONF_ESPHOME][CONF_NAME] == "lite_test" assert captured["config"][CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" + # The success-branch log line is part of the patch; assert on it so + # branch coverage stays unambiguous in CI. + assert "Loaded validated config cache" in caplog.text @pytest.mark.parametrize("command", ["upload", "logs"])