Merge remote-tracking branch 'upstream/cli-from-storage-json' into integration

This commit is contained in:
J. Nick Koston
2026-05-12 17:54:11 -05:00
9 changed files with 635 additions and 8 deletions
@@ -0,0 +1,113 @@
name: Add Dashboard Deprecation Comment
on:
pull_request_target:
types: [opened, synchronize]
# All API calls (pulls.listFiles + issues.{list,create,update}Comment) are performed with
# the App token minted below, so the workflow's GITHUB_TOKEN does not need any scopes.
permissions: {}
jobs:
dashboard-deprecation-comment:
name: Dashboard deprecation comment
runs-on: ubuntu-latest
steps:
- name: Generate a token
id: generate-token
uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
with:
client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }}
private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }}
# pulls.listFiles + issues.{list,create,update}Comment on PRs. For PR resources
# the issues.*Comment APIs require the pull-requests scope, not issues.
permission-pull-requests: write
- name: Add dashboard deprecation comment
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.generate-token.outputs.token }}
script: |
const commentMarker = "<!-- This comment was generated automatically by the dashboard-deprecation-comment workflow. -->";
const commentBody = `Thanks for opening this PR!
Heads up: the legacy ESPHome dashboard (\`esphome/dashboard/\` and \`tests/dashboard/\`) is **deprecated** and is being replaced by [ESPHome Device Builder](https://github.com/esphome/device-builder). We are not adding new features to the legacy dashboard and it will eventually be removed from this repository.
What this means for your PR:
- **New features / enhancements**: please port the change to [esphome/device-builder](https://github.com/esphome/device-builder) instead. We are unlikely to review or merge new dashboard features here.
- **Bug fixes**: small fixes may still be considered, but please check first whether the same issue exists in Device Builder, where the fix will have a longer life.
- **Security issues**: please do not file a public PR. Report privately via [GitHub security advisories](https://github.com/esphome/esphome/security/advisories/new) so we can coordinate a fix.
We appreciate the contribution and apologize for the friction; flagging this early so your time isn't spent on a change that may not land.
---
(Added by the PR bot)
${commentMarker}`;
async function getDashboardChanges(github, owner, repo, prNumber) {
const changedFiles = await github.paginate(
github.rest.pulls.listFiles,
{
owner: owner,
repo: repo,
pull_number: prNumber,
per_page: 100,
}
);
return changedFiles.filter(file =>
file.filename.startsWith('esphome/dashboard/') ||
file.filename.startsWith('tests/dashboard/')
);
}
async function findBotComment(github, owner, repo, prNumber) {
const comments = await github.paginate(
github.rest.issues.listComments,
{
owner: owner,
repo: repo,
issue_number: prNumber,
per_page: 100,
}
);
return comments.find(comment =>
comment.body.includes(commentMarker) && comment.user.type === "Bot"
);
}
const prNumber = context.payload.pull_request.number;
const { owner, repo } = context.repo;
const dashboardChanges = await getDashboardChanges(github, owner, repo, prNumber);
const existingComment = await findBotComment(github, owner, repo, prNumber);
if (dashboardChanges.length === 0) {
// PR doesn't (or no longer) touches the legacy dashboard. If we previously
// commented (e.g. files were removed in a later push), leave the comment in
// place for history rather than thrash on edit/delete.
return;
}
if (existingComment) {
if (existingComment.body === commentBody) {
return;
}
await github.rest.issues.updateComment({
owner: owner,
repo: repo,
comment_id: existingComment.id,
body: commentBody,
});
} else {
await github.rest.issues.createComment({
owner: owner,
repo: repo,
issue_number: prNumber,
body: commentBody,
});
}
+27 -5
View File
@@ -738,7 +738,11 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
# If you change this format, update the regex in that script as well
_LOGGER.info("Compiling app... Build path: %s", CORE.build_path)
if CORE.using_toolchain_esp_idf:
module = importlib.import_module("esphome.components." + CORE.target_platform)
platform_run_compile = getattr(module, "run_compile", None)
if platform_run_compile is not None and platform_run_compile(args, config):
pass
elif CORE.using_toolchain_esp_idf:
from esphome.espidf import toolchain
rc = toolchain.run_compile(config, CORE.verbose)
@@ -2414,10 +2418,28 @@ def run_esphome(argv):
# Commands that don't need fresh external components: logs just connects
# to the device, and clean is about to delete the build directory.
skip_external = args.command in ("logs", "clean")
config = read_config(
dict(args.substitution) if args.substitution else {},
skip_external_update=skip_external,
)
command_line_substitutions = dict(args.substitution) if args.substitution else {}
# Fast path for upload/logs: reuse the validated-config cache the
# last compile wrote. Falls back to read_config when missing/stale.
# Skipped when -s overrides are passed, since the cache was written
# against the previous substitution set.
config: ConfigType | None = None
if args.command in ("upload", "logs") and not command_line_substitutions:
from esphome.compiled_config import load_compiled_config
config = load_compiled_config(conf_path)
if config is not None:
_LOGGER.info(
"Loaded validated config cache for %s, skipping validation.",
conf_path.name,
)
if config is None:
config = read_config(
command_line_substitutions,
skip_external_update=skip_external,
)
if config is None:
return 2
CORE.config = config
+76
View File
@@ -0,0 +1,76 @@
"""Validated-config cache for the upload/logs fast path.
compile dumps the validated config to <data_dir>/storage/<file>.validated.yaml;
the next upload/logs for that YAML reuses it instead of running the full
read_config pipeline. YAML round-trip (yaml_util.dump/load_yaml) keeps
!lambda/!include/IDs/paths intact; mtime gates staleness.
"""
from __future__ import annotations
import logging
from pathlib import Path
from esphome.core import CORE
from esphome.helpers import write_file
from esphome.storage_json import StorageJSON, ext_storage_path
from esphome.types import ConfigType
_LOGGER = logging.getLogger(__name__)
def compiled_config_path(config_filename: str) -> Path:
"""Path to the cached validated config alongside the storage sidecar."""
return CORE.data_dir / "storage" / f"{config_filename}.validated.yaml"
def _cache_is_fresh(cache_path: Path, source_path: Path) -> bool:
"""True iff the cache file exists and isn't older than the source."""
try:
return cache_path.stat().st_mtime >= source_path.stat().st_mtime
except OSError:
return False
def save_compiled_config(config: ConfigType) -> None:
"""Write the validated-config cache. Always-write so mtime stays fresh.
Mode 0600 because show_secrets=True resolves !secret inline.
Failures are non-fatal: the fast path falls back to read_config.
"""
from esphome import yaml_util
try:
rendered = yaml_util.dump(config, show_secrets=True)
write_file(compiled_config_path(CORE.config_filename), rendered, private=True)
except Exception as err: # pylint: disable=broad-except
_LOGGER.debug("Skipping compiled config cache write: %s", err)
def load_compiled_config(conf_path: Path) -> ConfigType | None:
"""Load the cached validated config and apply storage metadata to CORE.
Returns None (caller falls back to read_config) when the cache is
missing, older than the source YAML, unparseable, or the sidecar
is incomplete.
"""
cache_path = compiled_config_path(conf_path.name)
if not _cache_is_fresh(cache_path, conf_path):
return None
from esphome import yaml_util
try:
config = yaml_util.load_yaml(cache_path, clear_secrets=False)
except Exception: # pylint: disable=broad-except
return None
storage = StorageJSON.load(ext_storage_path(conf_path.name))
if storage is None:
return None
# apply_to_core assumes a real compile wrote the sidecar; wizard-only
# sidecars leave both of these unset and can't drive upload/logs.
if not storage.core_platform and not storage.target_platform:
return None
storage.apply_to_core()
return config
+23 -1
View File
@@ -8,7 +8,13 @@ import os
from pathlib import Path
from esphome import const
from esphome.const import CONF_DISABLED, CONF_MDNS
from esphome.const import (
CONF_DISABLED,
CONF_MDNS,
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
)
from esphome.core import CORE
from esphome.helpers import write_file_if_changed
from esphome.types import CoreType
@@ -256,6 +262,22 @@ class StorageJSON:
except Exception: # pylint: disable=broad-except
return None
def apply_to_core(self) -> None:
"""Populate CORE with the metadata upload/logs read.
Inverse of :meth:`from_esphome_core`. Keep paired -- a new
attribute upload/logs needs has to be captured there too.
Validator-only fields (loaded_integrations/platforms,
friendly_name) are skipped; the fast path doesn't run
validation and CORE.__init__ defaults them.
"""
CORE.name = self.name
CORE.build_path = self.build_path
CORE.data[KEY_CORE] = {
KEY_TARGET_PLATFORM: self.core_platform or self.target_platform.lower(),
KEY_TARGET_FRAMEWORK: self.framework,
}
def __eq__(self, o) -> bool:
return isinstance(o, StorageJSON) and self.as_dict() == o.as_dict()
+6
View File
@@ -7,6 +7,7 @@ import re
import time
from esphome import loader
from esphome.compiled_config import save_compiled_config
from esphome.config import iter_component_configs, iter_components
from esphome.const import (
HEADER_FILE_EXTENSIONS,
@@ -109,6 +110,11 @@ def update_storage_json() -> None:
path = storage_path()
old = StorageJSON.load(path)
new = StorageJSON.from_esphome_core(CORE, old)
# Refresh the cache upload/logs read on the next call.
if CORE.config is not None:
save_compiled_config(CORE.config)
if old == new:
return
+1 -1
View File
@@ -12,7 +12,7 @@ platformio==6.1.19
esptool==5.2.0
click==8.3.3
esphome-dashboard==20260425.0
aioesphomeapi==44.23.0
aioesphomeapi==44.24.2
zeroconf==0.148.0
puremagic==1.30
ruamel.yaml==0.19.1 # dashboard_import
+9 -1
View File
@@ -119,7 +119,15 @@ from esphome.util import Registry # noqa: E402
def sort_obj(obj):
if isinstance(obj, dict):
return {k: sort_obj(v) for k, v in sorted(obj.items(), key=lambda x: str(x[0]))}
is_enum = obj.get(S_TYPE) == "enum"
result = {}
for k, v in sorted(obj.items(), key=lambda x: str(x[0])):
if is_enum and k == "values" and isinstance(v, dict):
# Preserve source order of enum options
result[k] = {vk: sort_obj(vv) for vk, vv in v.items()}
else:
result[k] = sort_obj(v)
return result
if isinstance(obj, list):
return [sort_obj(item) for item in obj]
return obj
@@ -0,0 +1,98 @@
"""Unit tests for script/build_language_schema.py."""
from __future__ import annotations
import ast
from pathlib import Path
SCRIPT_PATH = (
Path(__file__).resolve().parent.parent.parent
/ "script"
/ "build_language_schema.py"
)
def _extract_sort_obj():
# build_language_schema.py runs argparse, loads every component, and
# calls build_schema() at import time, so a plain import isn't viable
# in a unit test. Pull just the pure helper out via AST instead.
tree = ast.parse(SCRIPT_PATH.read_text())
for node in tree.body:
if isinstance(node, ast.FunctionDef) and node.name == "sort_obj":
namespace: dict = {"S_TYPE": "type"}
module = ast.Module(body=[node], type_ignores=[])
exec(compile(module, str(SCRIPT_PATH), "exec"), namespace)
return namespace["sort_obj"]
raise AssertionError("sort_obj not found in build_language_schema.py")
sort_obj = _extract_sort_obj()
def test_sort_obj_sorts_dict_keys() -> None:
result = sort_obj({"b": 1, "a": 2, "c": 3})
assert list(result.keys()) == ["a", "b", "c"]
def test_sort_obj_sorts_nested_dicts() -> None:
result = sort_obj({"outer": {"z": 1, "a": 2}})
assert list(result["outer"].keys()) == ["a", "z"]
def test_sort_obj_preserves_enum_values_order() -> None:
config = {
"type": "enum",
"values": {
"2MB": None,
"4MB": None,
"8MB": None,
"16MB": None,
"32MB": None,
},
}
result = sort_obj(config)
assert list(result["values"].keys()) == ["2MB", "4MB", "8MB", "16MB", "32MB"]
def test_sort_obj_sorts_non_enum_values_key() -> None:
config = {"type": "schema", "values": {"z": 1, "a": 2}}
result = sort_obj(config)
assert list(result["values"].keys()) == ["a", "z"]
def test_sort_obj_sorts_other_keys_in_enum() -> None:
config = {
"type": "enum",
"default": "4MB",
"key": "Optional",
"values": {"2MB": None, "4MB": None},
}
result = sort_obj(config)
assert list(result.keys()) == ["default", "key", "type", "values"]
assert list(result["values"].keys()) == ["2MB", "4MB"]
def test_sort_obj_recurses_into_enum_value_entries() -> None:
config = {
"type": "enum",
"values": {
"esp32": {"name": "ESP32", "docs": "Original"},
"esp32-c3": {"name": "ESP32-C3", "docs": "RISC-V"},
},
}
result = sort_obj(config)
assert list(result["values"].keys()) == ["esp32", "esp32-c3"]
assert list(result["values"]["esp32"].keys()) == ["docs", "name"]
def test_sort_obj_handles_lists() -> None:
result = sort_obj([{"b": 1, "a": 2}, {"d": 3, "c": 4}])
assert list(result[0].keys()) == ["a", "b"]
assert list(result[1].keys()) == ["c", "d"]
def test_sort_obj_passes_through_scalars() -> None:
assert sort_obj("hello") == "hello"
assert sort_obj(42) == 42
assert sort_obj(None) is None
assert sort_obj(True) is True
+282
View File
@@ -0,0 +1,282 @@
"""Tests for the validated-config cache used by upload/logs."""
from __future__ import annotations
import json
import os
from pathlib import Path
from unittest.mock import patch
import pytest
from esphome.__main__ import run_esphome
from esphome.compiled_config import (
compiled_config_path,
load_compiled_config,
save_compiled_config,
)
from esphome.const import (
CONF_API,
CONF_ESPHOME,
CONF_NAME,
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
)
from esphome.core import CORE
_VALIDATED_CONFIG_YAML = """\
esphome:
name: lite_test
friendly_name: Lite Test Device
esp32:
board: nodemcu-32s
logger:
baud_rate: 115200
api:
port: 6053
encryption:
key: 6dGhpcyBpcyBhIHRlc3Q=
ota:
- platform: esphome
port: 3232
password: secret
wifi:
ssid: ssid
use_address: 192.168.1.42
"""
def _write_storage(storage_path: Path) -> None:
"""Write a vanilla StorageJSON sidecar for the cache tests."""
storage_path.parent.mkdir(parents=True, exist_ok=True)
data = {
"storage_version": 1,
"name": "lite_test",
"friendly_name": "Lite Test Device",
"comment": None,
"esphome_version": "2026.1.0",
"src_version": 1,
"address": "192.168.1.42",
"web_port": None,
"esp_platform": "ESP32",
"build_path": "/build/lite_test",
"firmware_bin_path": "/build/lite_test/firmware.bin",
"loaded_integrations": ["api", "logger", "ota", "wifi"],
"loaded_platforms": [],
"no_mdns": False,
"framework": "arduino",
"core_platform": "esp32",
}
storage_path.write_text(json.dumps(data))
def _write_cache(cache_path: Path, body: str = _VALIDATED_CONFIG_YAML) -> Path:
"""Write the cache file and return it."""
cache_path.parent.mkdir(parents=True, exist_ok=True)
cache_path.write_text(body)
return cache_path
def _set_cache_mtime(cache_path: Path, yaml_path: Path, *, offset: int) -> None:
"""Force the cache file's mtime relative to the source YAML.
Positive offset → cache is fresh. Negative → cache is stale.
"""
yaml_stat = yaml_path.stat()
os.utime(cache_path, (yaml_stat.st_atime, yaml_stat.st_mtime + offset))
@pytest.fixture
def fresh_cache_files(tmp_path: Path) -> Path:
"""YAML + StorageJSON + cache, all consistent and fresh."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
storage_dir = tmp_path / ".esphome" / "storage"
_write_storage(storage_dir / "lite_test.yaml.json")
cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
_set_cache_mtime(cache, yaml_path, offset=5)
return yaml_path
def test_compiled_config_path_lives_alongside_sidecar(setup_core: Path) -> None:
"""The cache file shape is predictable from the YAML filename."""
path = compiled_config_path("device.yaml")
assert path.name == "device.yaml.validated.yaml"
assert path.parent.name == "storage"
def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None:
"""Fresh cache + sidecar → returns config and populates CORE."""
config = load_compiled_config(fresh_cache_files)
assert config is not None
assert config[CONF_ESPHOME][CONF_NAME] == "lite_test"
assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q="
assert config["ota"][0]["password"] == "secret"
# apply_to_core populated exactly what upload/logs read off CORE.
assert CORE.name == "lite_test"
assert CORE.build_path == Path("/build/lite_test")
assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32"
assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino"
@pytest.mark.parametrize(
"scenario",
["missing_cache", "stale_cache", "corrupt_cache", "missing_sidecar"],
)
def test_load_compiled_config_falls_back(tmp_path: Path, scenario: str) -> None:
"""All non-happy cases return None so the caller falls back."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
storage_dir = tmp_path / ".esphome" / "storage"
cache_path = storage_dir / "lite_test.yaml.validated.yaml"
sidecar_path = storage_dir / "lite_test.yaml.json"
if scenario == "missing_cache":
pass # no cache, no sidecar
elif scenario == "stale_cache":
_write_storage(sidecar_path)
_set_cache_mtime(_write_cache(cache_path), yaml_path, offset=-60)
elif scenario == "corrupt_cache":
_write_storage(sidecar_path)
_set_cache_mtime(
_write_cache(cache_path, "not: valid: yaml: ["), yaml_path, offset=5
)
elif scenario == "missing_sidecar":
# Cache fresh + parseable, but no StorageJSON → can't populate CORE.
_set_cache_mtime(_write_cache(cache_path), yaml_path, offset=5)
assert load_compiled_config(yaml_path) is None
@pytest.mark.parametrize("command", ["upload", "logs"])
def test_run_esphome_upload_and_logs_use_cache_when_fresh(
command: str,
fresh_cache_files: Path,
caplog: pytest.LogCaptureFixture,
) -> None:
"""upload/logs skip read_config() when the cache is fresh."""
captured: dict = {}
def _stub(_args, config):
captured["config"] = config
return 0
with (
caplog.at_level("INFO", logger="esphome.__main__"),
patch("esphome.__main__.read_config") as mock_read,
patch.dict("esphome.__main__.POST_CONFIG_ACTIONS", {command: _stub}),
):
assert run_esphome(["esphome", command, str(fresh_cache_files)]) == 0
mock_read.assert_not_called()
assert captured["config"][CONF_ESPHOME][CONF_NAME] == "lite_test"
assert captured["config"][CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q="
# The success-branch log line is part of the patch; assert on it so
# branch coverage stays unambiguous in CI.
assert "Loaded validated config cache" in caplog.text
@pytest.mark.parametrize("command", ["upload", "logs"])
def test_run_esphome_upload_and_logs_fall_back_when_no_cache(
tmp_path: Path, command: str
) -> None:
"""Without a cache, the dispatcher falls back to read_config()."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
with (
patch("esphome.__main__.read_config", return_value=None) as mock_read,
patch.dict(
"esphome.__main__.POST_CONFIG_ACTIONS",
{command: lambda args, config: 0},
),
):
assert run_esphome(["esphome", command, str(yaml_path)]) == 2
mock_read.assert_called_once()
def test_run_esphome_upload_with_substitution_skips_cache(
fresh_cache_files: Path,
) -> None:
"""`-s key value` forces a fresh validation -- the cache was written
against the prior substitution set, so reusing it would silently
ignore the override."""
with (
patch("esphome.__main__.read_config", return_value=None) as mock_read,
patch.dict(
"esphome.__main__.POST_CONFIG_ACTIONS",
{"upload": lambda args, config: 0},
),
):
run_esphome(["esphome", "-s", "var", "val", "upload", str(fresh_cache_files)])
mock_read.assert_called_once()
def test_run_esphome_compile_does_not_use_cache(fresh_cache_files: Path) -> None:
"""The compile subcommand always re-validates -- it's what writes the cache."""
with (
patch("esphome.__main__.read_config", return_value=None) as mock_read,
patch.dict(
"esphome.__main__.POST_CONFIG_ACTIONS",
{"compile": lambda args, config: 0},
),
):
run_esphome(["esphome", "compile", str(fresh_cache_files)])
mock_read.assert_called_once()
def test_save_compiled_config_writes_cache(tmp_path: Path) -> None:
"""`save_compiled_config` writes the dumped YAML next to the sidecar."""
CORE.config_path = tmp_path / "lite_test.yaml"
save_compiled_config({"esphome": {"name": "lite_test"}, "logger": {}})
cache_path = compiled_config_path("lite_test.yaml")
assert cache_path.is_file()
body = cache_path.read_text()
assert "name: lite_test" in body
assert "logger:" in body
def test_save_compiled_config_swallows_dump_errors(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""Failures during the dump are non-fatal -- a bad cache just means
the next fast path falls back to read_config()."""
CORE.config_path = tmp_path / "lite_test.yaml"
with patch("esphome.yaml_util.dump", side_effect=RuntimeError("boom")):
save_compiled_config({"esphome": {"name": "lite_test"}})
assert not compiled_config_path("lite_test.yaml").exists()
def test_load_compiled_config_rejects_wizard_only_sidecar(tmp_path: Path) -> None:
"""A wizard-only sidecar (no compile -- no core_platform / target_platform)
can't drive upload/logs, so the fast path falls back."""
yaml_path = tmp_path / "lite_test.yaml"
yaml_path.write_text("esphome:\n name: lite_test\n")
CORE.config_path = yaml_path
storage_dir = tmp_path / ".esphome" / "storage"
storage_dir.mkdir(parents=True, exist_ok=True)
# StorageJSON with both core_platform and target_platform unset.
(storage_dir / "lite_test.yaml.json").write_text(
'{"storage_version": 1, "name": "lite_test", "friendly_name": null, '
'"comment": null, "esphome_version": null, "src_version": 1, '
'"address": null, "web_port": null, "esp_platform": null, '
'"build_path": null, "firmware_bin_path": null, '
'"loaded_integrations": [], "loaded_platforms": [], "no_mdns": false, '
'"framework": null, "core_platform": null}'
)
cache_path = _write_cache(storage_dir / "lite_test.yaml.validated.yaml")
_set_cache_mtime(cache_path, yaml_path, offset=5)
assert load_compiled_config(yaml_path) is None