mirror of
https://github.com/esphome/esphome.git
synced 2026-10-05 02:21:30 +00:00
[core] Add --from-storage-json flag to upload and logs
When a downstream caller (typically a dashboard) already has a known-good firmware binary on disk and only needs the CLI to ship bytes to a device or stream logs back, running the full ``read_config()`` pipeline for every subcommand is dead work. For the device-builder REMOTE install flow that runs compile, upload, and logs back-to-back this means three "Reading configuration ..." passes for one user action. Add an opt-in ``--from-storage-json`` flag to ``upload`` and ``logs`` that sources platform / build metadata from the StorageJSON sidecar produced by the last successful compile and re-parses just enough of the YAML head (substitutions + packages, no schema validation) to recover the ``api:`` / ``logger:`` / ``ota:`` / network blocks the subcommands consult. The flag is opt-in and falls back to a full ``read_config()`` pass when the sidecar is missing or older than the YAML, so a cold cache never produces a worse outcome than today. ``compile`` / ``run`` / ``clean`` / ``bundle`` continue to validate as before.
This commit is contained in:
J. Nick Koston
+53
-4
@@ -2117,6 +2117,16 @@ def parse_args(argv):
|
||||
help="Upload as bootloader (OTA).",
|
||||
action="store_true",
|
||||
)
|
||||
parser_upload.add_argument(
|
||||
"--from-storage-json",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Skip YAML schema validation by loading device metadata from "
|
||||
"the .esphome/storage/<file>.json sidecar produced by the last "
|
||||
"successful compile. Falls back to a full validation pass when "
|
||||
"the sidecar is missing or older than the YAML."
|
||||
),
|
||||
)
|
||||
|
||||
parser_logs = subparsers.add_parser(
|
||||
"logs",
|
||||
@@ -2144,6 +2154,16 @@ def parse_args(argv):
|
||||
action="store_true",
|
||||
help="Do not show entity state changes in log output.",
|
||||
)
|
||||
parser_logs.add_argument(
|
||||
"--from-storage-json",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Skip YAML schema validation by loading device metadata from "
|
||||
"the .esphome/storage/<file>.json sidecar produced by the last "
|
||||
"successful compile. Falls back to a full validation pass when "
|
||||
"the sidecar is missing or older than the YAML."
|
||||
),
|
||||
)
|
||||
|
||||
parser_discover = subparsers.add_parser(
|
||||
"discover",
|
||||
@@ -2417,10 +2437,39 @@ def run_esphome(argv):
|
||||
# Commands that don't need fresh external components: logs just connects
|
||||
# to the device, and clean is about to delete the build directory.
|
||||
skip_external = args.command in ("logs", "clean")
|
||||
config = read_config(
|
||||
dict(args.substitution) if args.substitution else {},
|
||||
skip_external_update=skip_external,
|
||||
)
|
||||
command_line_substitutions = dict(args.substitution) if args.substitution else {}
|
||||
|
||||
# Fast path for `upload --from-storage-json` and `logs --from-storage-json`:
|
||||
# the caller already has a binary on disk and only needs the CLI to ship
|
||||
# bytes to a device or stream logs back. Re-running the full
|
||||
# `read_config()` pipeline (parse + schema validate + final-validate +
|
||||
# external component refresh) for those two subcommands is dead work and
|
||||
# produces a wall of "Reading configuration ..." log lines for every
|
||||
# remote install. Source the platform / build metadata from the
|
||||
# StorageJSON sidecar instead; fall back to full validation if it's
|
||||
# missing or stale so a cold cache never produces a worse outcome.
|
||||
config = None
|
||||
if getattr(args, "from_storage_json", False) and args.command in (
|
||||
"upload",
|
||||
"logs",
|
||||
):
|
||||
from esphome.lite_config import load_lite_config_from_storage
|
||||
|
||||
config = load_lite_config_from_storage(conf_path, command_line_substitutions)
|
||||
if config is None:
|
||||
_LOGGER.warning(
|
||||
"StorageJSON sidecar missing or stale for %s; falling back to "
|
||||
"full config validation.",
|
||||
conf_path,
|
||||
)
|
||||
else:
|
||||
_LOGGER.info("Loaded device metadata from StorageJSON sidecar.")
|
||||
|
||||
if config is None:
|
||||
config = read_config(
|
||||
command_line_substitutions,
|
||||
skip_external_update=skip_external,
|
||||
)
|
||||
if config is None:
|
||||
return 2
|
||||
CORE.config = config
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Lite-config loader for ``upload`` / ``logs`` fast paths.
|
||||
|
||||
When the caller (typically a dashboard) already has a known-good
|
||||
firmware binary on disk and only needs the CLI to ship bytes to a
|
||||
device or stream logs back, running the full ``read_config()``
|
||||
pipeline is dead work. It re-parses every ``!include``, runs every
|
||||
component's schema validator, executes all final-validate hooks, and
|
||||
fetches/refreshes external components -- producing a fully validated
|
||||
``Config`` object that the upload / logs subcommands only consult for a
|
||||
handful of leaf keys.
|
||||
|
||||
This module loads the per-config ``StorageJSON`` sidecar that was
|
||||
written by the last successful ``compile`` run, populates ``CORE`` with
|
||||
the platform / build metadata from it, and re-parses just enough of the
|
||||
YAML head (substitutions + packages, no schema validation) to recover
|
||||
``api:`` / ``logger:`` / ``ota:`` / ``mqtt:`` / network blocks for the
|
||||
subcommand callers.
|
||||
|
||||
If the sidecar is missing or older than the YAML, this returns ``None``
|
||||
so the dispatcher can fall back to the full ``read_config()`` path.
|
||||
That makes the flag a pure optimisation: a cold cache never produces a
|
||||
worse outcome than today.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from esphome.const import (
|
||||
CONF_API,
|
||||
CONF_ESPHOME,
|
||||
CONF_ETHERNET,
|
||||
CONF_FRIENDLY_NAME,
|
||||
CONF_LOGGER,
|
||||
CONF_MQTT,
|
||||
CONF_NAME,
|
||||
CONF_OPENTHREAD,
|
||||
CONF_OTA,
|
||||
CONF_USE_ADDRESS,
|
||||
CONF_WEB_SERVER,
|
||||
CONF_WIFI,
|
||||
KEY_CORE,
|
||||
KEY_TARGET_FRAMEWORK,
|
||||
KEY_TARGET_PLATFORM,
|
||||
)
|
||||
from esphome.core import CORE
|
||||
from esphome.storage_json import StorageJSON, ext_storage_path
|
||||
from esphome.types import ConfigType
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Top-level keys we copy from the raw YAML into the lite config dict.
|
||||
# Anything outside this list is irrelevant to ``upload`` / ``logs``.
|
||||
_LITE_TOP_LEVEL_KEYS: tuple[str, ...] = (
|
||||
CONF_ESPHOME,
|
||||
CONF_API,
|
||||
CONF_LOGGER,
|
||||
CONF_OTA,
|
||||
CONF_MQTT,
|
||||
CONF_WIFI,
|
||||
CONF_ETHERNET,
|
||||
CONF_OPENTHREAD,
|
||||
CONF_WEB_SERVER,
|
||||
)
|
||||
|
||||
|
||||
def _parse_yaml_head(
|
||||
conf_path: Path, command_line_substitutions: dict[str, Any] | None
|
||||
) -> dict[str, Any] | None:
|
||||
"""Load and lightly process the YAML, without schema validation.
|
||||
|
||||
Resolves ``!include`` / ``!secret`` (via ``yaml_util``),
|
||||
substitutions, and ``packages:`` -- the three passes whose output
|
||||
the upload / logs subcommands need. Returns ``None`` on any error
|
||||
so the caller can fall back to the full ``read_config()`` path.
|
||||
"""
|
||||
from esphome import yaml_util
|
||||
from esphome.components.packages import resolve_packages
|
||||
from esphome.components.substitutions import do_substitution_pass
|
||||
|
||||
try:
|
||||
config = yaml_util.load_yaml(conf_path)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return None
|
||||
|
||||
try:
|
||||
config = do_substitution_pass(config, command_line_substitutions)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return None
|
||||
|
||||
try:
|
||||
config = resolve_packages(
|
||||
config, command_line_substitutions=command_line_substitutions
|
||||
)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return None
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def _build_lite_config(raw_config: dict[str, Any]) -> ConfigType:
|
||||
"""Project the raw YAML down to the keys upload / logs read."""
|
||||
return {key: raw_config[key] for key in _LITE_TOP_LEVEL_KEYS if key in raw_config}
|
||||
|
||||
|
||||
def _populate_core_from_storage(storage: StorageJSON) -> None:
|
||||
"""Populate ``CORE`` fields the subcommands read off the platform."""
|
||||
CORE.name = storage.name
|
||||
CORE.friendly_name = storage.friendly_name
|
||||
CORE.build_path = storage.build_path
|
||||
CORE.loaded_integrations = set(storage.loaded_integrations)
|
||||
CORE.loaded_platforms = set(storage.loaded_platforms)
|
||||
|
||||
core_platform = storage.core_platform or (
|
||||
storage.target_platform.lower() if storage.target_platform else None
|
||||
)
|
||||
CORE.data.setdefault(KEY_CORE, {})
|
||||
if core_platform is not None:
|
||||
CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = core_platform
|
||||
if storage.framework is not None:
|
||||
CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = storage.framework
|
||||
|
||||
|
||||
def _ensure_address_in_config(config: ConfigType, storage: StorageJSON) -> None:
|
||||
"""Make sure ``CORE.address`` resolves once the lite config is set.
|
||||
|
||||
``CORE.address`` reads ``use_address`` off the wifi / ethernet /
|
||||
openthread block; if the dashboard rewrote the YAML between the
|
||||
last compile and now those blocks may be missing from the lite
|
||||
parse. As a backstop, lift the address from the StorageJSON when
|
||||
we have one and the YAML doesn't supply one already.
|
||||
"""
|
||||
if storage.address is None:
|
||||
return
|
||||
for network_type in (CONF_WIFI, CONF_ETHERNET, CONF_OPENTHREAD):
|
||||
if network_type in config and CONF_USE_ADDRESS in config[network_type]:
|
||||
return
|
||||
# Fall back to a synthetic wifi block so ``CORE.address`` resolves.
|
||||
config.setdefault(CONF_WIFI, {})[CONF_USE_ADDRESS] = storage.address
|
||||
|
||||
|
||||
def load_lite_config_from_storage(
|
||||
conf_path: Path, command_line_substitutions: dict[str, Any] | None = None
|
||||
) -> ConfigType | None:
|
||||
"""Build a minimal config + populate CORE from the StorageJSON sidecar.
|
||||
|
||||
Returns the lite config dict on success, or ``None`` when the
|
||||
dispatcher should fall back to ``read_config()`` (sidecar missing,
|
||||
stale, unreadable, or required fields absent).
|
||||
"""
|
||||
storage_path = ext_storage_path(conf_path.name)
|
||||
try:
|
||||
yaml_stat = conf_path.stat()
|
||||
except OSError:
|
||||
return None
|
||||
try:
|
||||
storage_stat = storage_path.stat()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
# The sidecar is written by `compile`; if the YAML has been edited
|
||||
# since, the cached platform / loaded_integrations may no longer
|
||||
# describe what the binary on disk was built from. Fall back to a
|
||||
# full validation pass in that case.
|
||||
if storage_stat.st_mtime < yaml_stat.st_mtime:
|
||||
return None
|
||||
|
||||
storage = StorageJSON.load(storage_path)
|
||||
if storage is None:
|
||||
return None
|
||||
if not storage.target_platform and not storage.core_platform:
|
||||
# An incomplete sidecar (e.g. from a wizard run that never
|
||||
# compiled) can't drive the upload / logs subcommands.
|
||||
return None
|
||||
|
||||
raw_config = _parse_yaml_head(conf_path, command_line_substitutions)
|
||||
if raw_config is None:
|
||||
return None
|
||||
if CONF_ESPHOME not in raw_config or CONF_NAME not in raw_config[CONF_ESPHOME]:
|
||||
return None
|
||||
|
||||
config = _build_lite_config(raw_config)
|
||||
|
||||
# Backfill `esphome:` block fields from the sidecar so downstream
|
||||
# callers that read `config["esphome"]["name"]` work even when the
|
||||
# YAML uses substitutions that didn't survive the light parse.
|
||||
esphome_block = config.setdefault(CONF_ESPHOME, {})
|
||||
esphome_block.setdefault(CONF_NAME, storage.name)
|
||||
if storage.friendly_name is not None:
|
||||
esphome_block.setdefault(CONF_FRIENDLY_NAME, storage.friendly_name)
|
||||
|
||||
_populate_core_from_storage(storage)
|
||||
_ensure_address_in_config(config, storage)
|
||||
|
||||
return config
|
||||
Reference in New Issue
Block a user