From 5884ca4d3273ac47974ffc62efc28d7df14088cc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 7 Oct 2026 13:06:40 -1000 Subject: [PATCH] [core] Add YAML discovery, secret and zstd helpers Report unresolved includes and load errors from include discovery, share the !secret reference scan between bundle and other callers, let callers register extra secret values for a dump and see which names it emitted, swap secrets in the !extend, !remove and scalar !include representers, record the remote sources packages fetch, and move zstd_module out of the api wizard into esphome.helpers. --- esphome/bundle.py | 32 +++-- esphome/components/api/wizard.py | 16 +-- esphome/components/packages/__init__.py | 41 +++++- esphome/helpers.py | 10 ++ esphome/yaml_util.py | 127 ++++++++++++++++-- requirements.txt | 2 +- tests/component_tests/api/test_wizard.py | 23 +--- .../component_tests/packages/test_packages.py | 45 +++++++ tests/integration/conftest.py | 7 +- tests/unit_tests/conftest.py | 11 ++ tests/unit_tests/test_helpers.py | 17 +++ tests/unit_tests/test_yaml_util.py | 125 ++++++++++++++--- 12 files changed, 380 insertions(+), 76 deletions(-) diff --git a/esphome/bundle.py b/esphome/bundle.py index b633c5ca4f0..f499e0de1bc 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -102,25 +102,21 @@ _KNOWN_FILE_EXTENSIONS = frozenset( ) -# Matches !secret references in YAML text. An optional surrounding -# quote pair around the key is allowed and ignored: YAML treats -# ``!secret 'foo'`` and ``!secret foo`` as the same key. This is -# intentionally a simple regex scan rather than a YAML parse — it may -# match inside comments or multi-line strings, which is the conservative -# direction (include more secrets rather than fewer). -_SECRET_RE = re.compile(r"""!secret\s+['"]?([^\s'"]+)""") - - def _find_used_secret_keys(yaml_files: list[Path]) -> set[str]: """Scan YAML files for ``!secret `` references.""" keys: set[str] = set() for fpath in yaml_files: try: text = fpath.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): + except (OSError, UnicodeDecodeError) as err: + _LOGGER.warning( + "Could not scan %s for !secret references (%s); the bundled " + "secret set may be incomplete", + fpath, + err, + ) continue - for match in _SECRET_RE.finditer(text): - keys.add(match.group(1)) + keys |= yaml_util.find_secret_references(text) return keys @@ -422,6 +418,18 @@ class ConfigBundleCreator: must ship every candidate so the remote build can pick any one. """ discovered = yaml_util.discover_user_yaml_files(self._config_path) + if discovered.load_errors: + _LOGGER.warning( + "Bundle may be incomplete; could not load all configuration files: %s", + "; ".join(discovered.load_errors), + ) + if discovered.unresolved: + _LOGGER.warning( + "Bundle may be incomplete; %d !include path(s) use " + "substitutions and cannot be captured: %s", + len(discovered.unresolved), + ", ".join(discovered.unresolved), + ) self._secrets_paths.update(discovered.secrets) # A !secret inside a file this re-parse does not reach (for example # a git-fetched package the builder re-fetches) still resolves diff --git a/esphome/components/api/wizard.py b/esphome/components/api/wizard.py index 314b907e4d0..befed39972c 100644 --- a/esphome/components/api/wizard.py +++ b/esphome/components/api/wizard.py @@ -6,7 +6,6 @@ Home Assistant shows the wizard when the device is added. See api_wizard.h for t from collections.abc import Callable import importlib import json -from types import ModuleType from typing import Any from esphome import automation @@ -27,7 +26,12 @@ from esphome.const import ( ) from esphome.core import CORE, ID import esphome.final_validate as fv -from esphome.helpers import fnv1_hash, fnv1_hash_object_id, fnv1a_32bit_hash +from esphome.helpers import ( + fnv1_hash, + fnv1_hash_object_id, + fnv1a_32bit_hash, + zstd_module, +) from esphome.types import ConfigType API_DOMAIN = "api" @@ -351,14 +355,6 @@ automation.register_apply_condition( ) -def zstd_module() -> ModuleType: - """The zstd module: the standard library one from Python 3.14, otherwise the backport.""" - try: - return importlib.import_module("compression.zstd") - except ImportError: - return importlib.import_module("backports.zstd") - - def _entity_document(conf: ConfigType, config: fv.FinalValidateConfig) -> ConfigType: """An entity of the device that the page shows, keyed as ListEntitiesResponse keys it.""" _, declaration = _wizard_input_declaration(config, conf[CONF_ID]) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 703af8281d2..534f7622e8f 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -1,5 +1,6 @@ from collections import UserDict from collections.abc import Callable +from dataclasses import dataclass, field from functools import reduce import logging from pathlib import Path @@ -34,11 +35,45 @@ from esphome.const import ( CONF_VARS, __version__ as ESPHOME_VERSION, ) -from esphome.core import EsphomeError +from esphome.core import CORE, EsphomeError _LOGGER = logging.getLogger(__name__) DOMAIN = CONF_PACKAGES + + +@dataclass(frozen=True) +class RemotePackageSource: + """A remote source a package was fetched from while processing the config.""" + + url: str + ref: str | None + + +@dataclass +class PackagesData: + """Per-run package state, keyed under DOMAIN in CORE.data.""" + + remote_sources: list[RemotePackageSource] = field(default_factory=list) + + +def _get_data() -> PackagesData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = PackagesData() + return CORE.data[DOMAIN] + + +def get_remote_package_sources() -> list[RemotePackageSource]: + """Remote sources fetched while processing this config, in fetch order. + + Consumers (e.g. store_yaml) use this to tell which parts of the config + came from remote repositories rather than local files. + """ + if (data := CORE.data.get(DOMAIN)) is None: + return [] + return data.remote_sources + + # Guard against infinite include chains (e.g. A includes B includes A). MAX_INCLUDE_DEPTH = 20 @@ -192,6 +227,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: username=config.get(CONF_USERNAME), password=config.get(CONF_PASSWORD), ) + source = RemotePackageSource(config[CONF_URL], config.get(CONF_REF)) + remote_sources = _get_data().remote_sources + if source not in remote_sources: + remote_sources.append(source) files: list[dict[str, Any]] = [] # ``repo_root`` is the directory containing ``.git`` and must be passed diff --git a/esphome/helpers.py b/esphome/helpers.py index 5ec46be67e4..1f9b1407971 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations from collections.abc import Callable, Iterable, MutableMapping from contextlib import suppress +import importlib import ipaddress import logging import os @@ -10,6 +11,7 @@ import platform import re import stat import sys +from types import ModuleType from typing import TYPE_CHECKING, TextIO from esphome.const import __version__ as ESPHOME_VERSION @@ -827,3 +829,11 @@ def docs_url(path: str) -> str: path = path.removeprefix("/") return docs_format.format(path=path) + + +def zstd_module() -> ModuleType: + """The zstd module: the standard library one from Python 3.14, otherwise the backport.""" + try: + return importlib.import_module("compression.zstd") + except ImportError: + return importlib.import_module("backports.zstd") diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index e892664724c..a8d0ac845a1 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -11,7 +11,8 @@ import logging import math import os from pathlib import Path -from typing import Any +import re +from typing import Any, NamedTuple import uuid import yaml @@ -46,6 +47,10 @@ _LOGGER = logging.getLogger(__name__) SECRET_YAML = "secrets.yaml" _SECRET_CACHE = {} _SECRET_VALUES = {} +# Stack of collectors (one per active secret_values_registered context); the +# dumper records every emitted `!secret` name into the innermost one. YAML +# processing is single-threaded. +_EMITTED_SECRET_NAMES: list[set[str]] = [] # Not thread-safe — config processing is single-threaded today. _load_listeners: list[Callable[[Path], None]] = [] @@ -390,6 +395,7 @@ def _load_include_candidates( seen: set[int], expanded_paths: set[Path], keepalive: list[Any], + result: ForceLoadResult, ) -> None: """Load every filesystem candidate for an unresolved ``IncludeFile``.""" from voluptuous import Invalid @@ -402,6 +408,7 @@ def _load_include_candidates( include.file, include.parent_file, ) + result.unresolved.append(str(include.file)) return _LOGGER.debug( "Expanding !include %s (referenced from %s) to %d candidate file(s)", @@ -427,6 +434,7 @@ def _load_include_candidates( include.file, err, ) + result.errors.append(f"{candidate}: {err}") continue # The throwaway IncludeFile is this tree's only owner; keep the tree # alive so ids recorded in ``seen`` stay unique for the traversal. @@ -437,9 +445,36 @@ def _load_include_candidates( _seen=seen, _expanded_paths=expanded_paths, _keepalive=keepalive, + _result=result, ) +# Matches !secret references in YAML text. An optional surrounding +# quote pair around the key is allowed and ignored: YAML treats +# ``!secret 'foo'`` and ``!secret foo`` as the same key. This is +# intentionally a simple regex scan rather than a YAML parse — it may +# match inside comments or multi-line strings, which is the conservative +# direction (include more secrets rather than fewer). +_SECRET_REFERENCE_RE = re.compile(r"""!secret\s+['"]?([^\s'"]+)""") + + +def find_secret_references(text: str) -> set[str]: + """Return the ``!secret `` names referenced in a YAML document text.""" + return {match.group(1) for match in _SECRET_REFERENCE_RE.finditer(text)} + + +class ForceLoadResult(NamedTuple): + """Outcome of :func:`force_load_include_files`. + + ``unresolved`` lists ``!include`` path strings with substitution variables or + expressions that matched no candidate file; ``errors`` lists includes that failed + to load. Either being non-empty means the walk was incomplete. + """ + + unresolved: list[str] + errors: list[str] + + def force_load_include_files( obj: Any, *, @@ -447,7 +482,8 @@ def force_load_include_files( _seen: set[int] | None = None, _expanded_paths: set[Path] | None = None, _keepalive: list[Any] | None = None, -) -> None: + _result: ForceLoadResult | None = None, +) -> ForceLoadResult: """Recursively resolve any deferred ``IncludeFile`` instances in a YAML tree. Nested ``!include`` returns a deferred ``IncludeFile`` that is only resolved @@ -477,10 +513,12 @@ def force_load_include_files( # fresh tree look already seen. Discovery is a one-shot operation, # so holding the parsed trees costs nothing. _keepalive = [] + if _result is None: + _result = ForceLoadResult([], []) if isinstance(obj, IncludeFile): if id(obj) in _seen: - return + return _result _seen.add(id(obj)) if obj.has_unresolved_file(): _load_include_candidates( @@ -489,8 +527,9 @@ def force_load_include_files( seen=_seen, expanded_paths=_expanded_paths, keepalive=_keepalive, + result=_result, ) - return + return _result try: loaded = obj.load() except (EsphomeError, Invalid) as err: @@ -500,17 +539,19 @@ def force_load_include_files( obj.parent_file, err, ) - return + _result.errors.append(f"{obj.file}: {err}") + return _result force_load_include_files( loaded, warn_on_unresolved=warn_on_unresolved, _seen=_seen, _expanded_paths=_expanded_paths, _keepalive=_keepalive, + _result=_result, ) elif isinstance(obj, dict): if id(obj) in _seen: - return + return _result _seen.add(id(obj)) for value in obj.values(): force_load_include_files( @@ -519,10 +560,11 @@ def force_load_include_files( _seen=_seen, _expanded_paths=_expanded_paths, _keepalive=_keepalive, + _result=_result, ) elif isinstance(obj, (list, tuple)): if id(obj) in _seen: - return + return _result _seen.add(id(obj)) for item in obj: force_load_include_files( @@ -531,7 +573,9 @@ def force_load_include_files( _seen=_seen, _expanded_paths=_expanded_paths, _keepalive=_keepalive, + _result=_result, ) + return _result @dataclass(slots=True) @@ -542,11 +586,16 @@ class DiscoveredYamlFiles: were re-parsing the user's config; ``secrets`` is the subset whose *un-resolved* filename matched :data:`esphome.const.SECRETS_FILES` (so a ``secrets.yaml`` symlinked to a differently-named target is still - flagged as secrets). + flagged as secrets). ``unresolved`` lists ``!include`` path strings that + contain substitution variables and therefore could not be loaded, and + ``load_errors`` lists files that failed to parse or load — consumers + should treat ``files`` as incomplete when either is non-empty. """ files: list[Path] = field(default_factory=list) secrets: set[Path] = field(default_factory=set) + unresolved: list[str] = field(default_factory=list) + load_errors: list[str] = field(default_factory=list) def discover_user_yaml_files(config_path: Path) -> DiscoveredYamlFiles: @@ -576,9 +625,16 @@ def discover_user_yaml_files(config_path: Path) -> DiscoveredYamlFiles: try: try: data = load_yaml(config_path) - except EsphomeError: - return DiscoveredYamlFiles(list(loaded), secrets) - force_load_include_files(data, warn_on_unresolved=False) + except EsphomeError as err: + _LOGGER.warning( + "YAML discovery failed to parse %s: %s", config_path, err + ) + return DiscoveredYamlFiles( + list(loaded), secrets, load_errors=[f"{config_path}: {err}"] + ) + unresolved, load_errors = force_load_include_files( + data, warn_on_unresolved=False + ) finally: _load_listeners.remove(_capture_secret) @@ -589,7 +645,7 @@ def discover_user_yaml_files(config_path: Path) -> DiscoveredYamlFiles: if path not in seen: seen.add(path) unique.append(path) - return DiscoveredYamlFiles(unique, secrets) + return DiscoveredYamlFiles(unique, secrets, unresolved, load_errors) def _add_data_ref(fn): @@ -1119,6 +1175,38 @@ def _load_yaml_internal_with_type( loader.dispose() +def registered_secret_names() -> set[str]: + """Names of all ``!secret`` keys the loader has seen since the last clear.""" + return set(_SECRET_VALUES.values()) + + +@contextmanager +def secret_values_registered(values: dict[str, str]) -> Generator[set[str]]: + """Temporarily register value→name mappings so :func:`dump` renders those + scalars as ``!secret ``. + + Mappings already present in ``_SECRET_VALUES`` (values loaded through a + real ``!secret``) win over the supplied ones and are left untouched. + + Yields a set that collects the name of every ``!secret`` reference the + dumper emits while the context is active, so callers can tell exactly + which registered values were actually swapped. + """ + added = {v: n for v, n in values.items() if v not in _SECRET_VALUES} + _SECRET_VALUES.update(added) + emitted: set[str] = set() + _EMITTED_SECRET_NAMES.append(emitted) + try: + yield emitted + finally: + # Contexts unwind LIFO, so the innermost collector is always last; + # pop() removes by position where remove() would match the first + # *equal* set and could strip an outer context's collector. + _EMITTED_SECRET_NAMES.pop() + for value in added: + _SECRET_VALUES.pop(value, None) + + def dump( dict_, show_secrets=False, @@ -1333,7 +1421,10 @@ class ESPHomeDumper(yaml.SafeDumper): return node def represent_secret(self, value): - return self.represent_scalar(tag="!secret", value=_SECRET_VALUES[str(value)]) + name = _SECRET_VALUES[str(value)] + if _EMITTED_SECRET_NAMES: + _EMITTED_SECRET_NAMES[-1].add(name) + return self.represent_scalar(tag="!secret", value=name) def represent_stringify(self, value): if is_secret(value): @@ -1414,13 +1505,21 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_scalar(tag="!lambda", value=value.value, style="|") def represent_extend(self, value): + # Consult is_secret like the other scalar representers so a payload + # equal to a registered secret is never written out in cleartext. + if is_secret(value.value): + return self.represent_secret(value.value) return self.represent_scalar(tag="!extend", value=value.value) def represent_remove(self, value): + if is_secret(value.value): + return self.represent_secret(value.value) return self.represent_scalar(tag="!remove", value=value.value) def represent_include_file(self, value): if value.vars or value.condition is not None: + # The mapping values route through the regular representers, + # which already consult is_secret. mapping = {"file": value.file} if value.vars: mapping["vars"] = value.vars @@ -1429,6 +1528,8 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_mapping( tag="!include", mapping=mapping, flow_style=False ) + if is_secret(value.file): + return self.represent_secret(value.file) return self.represent_scalar(tag="!include", value=value.file) def represent_id(self, value): diff --git a/requirements.txt b/requirements.txt index 91a36ecbf70..fe3d3ab64e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import esphome-glyphsets==0.2.0 -backports.zstd==1.7.0; python_version < "3.14" # api wizard; compression.zstd is in the standard library from 3.14 +backports.zstd==1.7.0; python_version < "3.14" # esphome.helpers.zstd_module; compression.zstd is in the standard library from 3.14 pillow==12.3.0 resvg-py==0.5.0 freetype-py==2.5.1 diff --git a/tests/component_tests/api/test_wizard.py b/tests/component_tests/api/test_wizard.py index 276f42d624e..112092482b2 100644 --- a/tests/component_tests/api/test_wizard.py +++ b/tests/component_tests/api/test_wizard.py @@ -14,7 +14,7 @@ from esphome.components.api import wizard from esphome.components.homeassistant.switch import SUPPORTED_DOMAINS as SWITCH_DOMAINS from esphome.config import load_config from esphome.core import CORE -from esphome.helpers import fnv1_hash, fnv1a_32bit_hash +from esphome.helpers import fnv1_hash, fnv1a_32bit_hash, zstd_module from tests.component_tests.helpers import get_define_value ESP32_HEADER = """ @@ -613,7 +613,7 @@ def test_text_select_and_button_default_to_their_domains( tmp_path: Path, generate_main: Callable[[str | Path], str] ) -> None: main_cpp = generate_main(write_input_config(tmp_path, ESP32_HEADER)) - document = json.loads(wizard.zstd_module().decompress(blob_in(main_cpp))) + document = json.loads(zstd_module().decompress(blob_in(main_cpp))) filters = { entry["key"]: entry.get("entity_filters") for page in document["pages"] @@ -733,7 +733,7 @@ def test_the_blob_is_the_exact_json_document( ) blob = blob_in(main_cpp) - text = wizard.zstd_module().decompress(blob).decode("utf-8") + text = zstd_module().decompress(blob).decode("utf-8") expected = { "version": 1, "pages": [ @@ -788,7 +788,7 @@ def test_the_blob_is_deterministic_and_the_same_on_every_platform( ) # Compressing the same document again gives the same bytes document = wizard.wizard_document(CORE.config["api"]["wizard"], CORE.config) - again = wizard.zstd_module().compress( + again = zstd_module().compress( json.dumps( document, separators=(",", ":"), sort_keys=True, ensure_ascii=False ).encode("utf-8"), @@ -859,21 +859,6 @@ def test_an_entity_without_a_name_cannot_be_in_the_wizard(tmp_path: Path) -> Non assert any("has no name of its own" in error for error in errors), errors -def test_zstd_falls_back_to_the_backport(monkeypatch: pytest.MonkeyPatch) -> None: - """Before Python 3.14 the standard library has no zstd, so the backport is used.""" - backport = object() - - def import_module(name: str) -> object: - if name == "compression.zstd": - raise ImportError(name) - assert name == "backports.zstd" - return backport - - monkeypatch.setattr(wizard.importlib, "import_module", import_module) - - assert wizard.zstd_module() is backport - - def test_a_wizard_too_big_for_one_message_is_rejected(tmp_path: Path) -> None: # Random text does not compress, so this needs more than the limit even compressed rng = random.Random(1) diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 418cab5ea15..a662a291ed8 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -10,9 +10,11 @@ import pytest from esphome import bundle from esphome.components.packages import ( CONFIG_SCHEMA, + RemotePackageSource, _substitute_package_definition, _walk_packages, do_packages_pass, + get_remote_package_sources, merge_packages, resolve_packages, ) @@ -614,6 +616,49 @@ def test_package_remove_by_missing_id() -> None: assert actual == expected +def test_get_remote_package_sources_empty_without_remote_packages() -> None: + """A config without remote packages records no sources.""" + assert get_remote_package_sources() == [] + + +@patch("esphome.yaml_util.load_yaml") +@patch("pathlib.Path.is_file") +@patch("esphome.git.clone_or_update") +def test_get_remote_package_sources_records_each_fetch_once( + mock_clone_or_update, mock_is_file, mock_load_yaml +) -> None: + """Every fetched remote source is recorded exactly once.""" + mock_clone_or_update.return_value = (Path("/tmp/noexists"), MagicMock()) + mock_is_file.return_value = True + mock_load_yaml.side_effect = lambda *args, **kwargs: OrderedDict() + + config = { + CONF_PACKAGES: { + "first": { + CONF_URL: "https://github.com/esphome/repo-a", + CONF_REF: "main", + CONF_FILES: [TEST_YAML_FILENAME], + }, + "again": { + CONF_URL: "https://github.com/esphome/repo-a", + CONF_REF: "main", + CONF_FILES: ["other.yaml"], + }, + "second": { + CONF_URL: "https://github.com/esphome/repo-b", + CONF_FILES: [TEST_YAML_FILENAME], + }, + } + } + packages_pass(config) + + sources = get_remote_package_sources() + assert sorted(sources, key=lambda source: source.url) == [ + RemotePackageSource("https://github.com/esphome/repo-a", "main"), + RemotePackageSource("https://github.com/esphome/repo-b", None), + ] + + @patch("esphome.yaml_util.load_yaml") @patch("pathlib.Path.is_file") @patch("esphome.git.clone_or_update") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ce2e7f5bbc0..5090ad20036 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -212,10 +212,9 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s loop = asyncio.get_running_loop() content = await loop.run_in_executor(None, read_file, fixture_path) - # Replace the port in the config if it contains api section - if "api:" in content: - # Add port configuration after api: - content = content.replace("api:", f"api:\n port: {unused_tcp_port}") + # Replace the port in the config if it contains an api section. Anchored to + # the start of a line so keys that merely end in "api:" are left alone. + content = re.sub(r"(?m)^api:", f"api:\n port: {unused_tcp_port}", content) # Add debug build flags for integration tests to enable assertions if "esphome:" in content and "build_flags:" not in content: diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index ff5564e8150..aba47c9237e 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -17,6 +17,7 @@ from unittest.mock import Mock, patch import pytest +from esphome import yaml_util from esphome.core import CORE here = Path(__file__).parent @@ -33,6 +34,16 @@ def reset_core(): CORE.reset() +@pytest.fixture(autouse=True) +def clear_yaml_secrets(): + """Isolate the yaml_util secrets registry between tests.""" + yaml_util._SECRET_VALUES.clear() + yaml_util._SECRET_CACHE.clear() + yield + yaml_util._SECRET_VALUES.clear() + yaml_util._SECRET_CACHE.clear() + + @pytest.fixture def fixture_path() -> Path: """ diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index 8dae1e87b20..764f160cbc0 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -1285,3 +1285,20 @@ def test_get_usable_cpu_count_sources() -> None: mock_os_unknown = types.SimpleNamespace(cpu_count=lambda: None) with patch("esphome.helpers.os", mock_os_unknown): assert helpers.get_usable_cpu_count() == 1 + + +def test_zstd_module_falls_back_to_the_backport( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Before Python 3.14 the standard library has no zstd, so the backport is used.""" + backport = object() + + def import_module(name: str) -> object: + if name == "compression.zstd": + raise ImportError(name) + assert name == "backports.zstd" + return backport + + monkeypatch.setattr(helpers.importlib, "import_module", import_module) + + assert helpers.zstd_module() is backport diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index e1c3080e291..7feb4f872f5 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -25,16 +25,6 @@ from esphome.yaml_util import ( ) -@pytest.fixture(autouse=True) -def clear_secrets_cache() -> None: - """Clear the secrets cache before each test.""" - yaml_util._SECRET_VALUES.clear() - yaml_util._SECRET_CACHE.clear() - yield - yaml_util._SECRET_VALUES.clear() - yaml_util._SECRET_CACHE.clear() - - @pytest.fixture(autouse=True) def clear_core_frontmatter() -> None: """Reset CORE.frontmatter between tests.""" @@ -1289,22 +1279,38 @@ def test_force_load_include_files_unresolved_log_level( assert matching == [expect_level] +def test_force_load_include_files_returns_unresolved_paths( + patch_include_file: None, +) -> None: + """Includes with substitution-templated paths are reported back to the + caller; resolvable ones are not.""" + templated = _StubInclude("${var}.yaml", unresolved=True) + plain = _StubInclude("ok.yaml") + result = force_load_include_files({"a": templated, "b": plain}) + assert result.unresolved == [str(templated.file)] + assert result.errors == [] + assert plain.load_calls == 1 + + def test_force_load_include_files_warns_on_load_failure( patch_include_file: None, caplog: pytest.LogCaptureFixture, ) -> None: - """An `EsphomeError` raised by `load()` is caught and logged, not propagated.""" + """An `EsphomeError` raised by `load()` is caught, logged, and reported to + the caller — not propagated.""" stub = _StubInclude("missing.yaml", raise_on_load=EsphomeError("boom")) with caplog.at_level("WARNING", logger="esphome.yaml_util"): - force_load_include_files({"k": stub}) + result = force_load_include_files({"k": stub}) assert any( "Failed to load !include" in r.message and "missing.yaml" in r.message for r in caplog.records ) + assert result.errors == [f"{stub.file}: boom"] + assert result.unresolved == [] def test_discovered_yaml_files_holds_files_and_secrets() -> None: - """`DiscoveredYamlFiles` is a small data carrier; both fields are mandatory.""" + """`DiscoveredYamlFiles` is a small data carrier.""" files = [Path("/tmp/a.yaml")] secrets = {Path("/tmp/a.yaml")} discovered = DiscoveredYamlFiles(files, secrets) @@ -1367,11 +1373,28 @@ def test_discover_user_yaml_files_flags_secrets_symlink(tmp_path: Path) -> None: assert target.resolve() in discovered.secrets -def test_discover_user_yaml_files_swallows_parse_errors(tmp_path: Path) -> None: - """A YAML parse failure returns whatever was tracked so far without raising.""" +def test_discover_user_yaml_files_reports_parse_errors( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A YAML parse failure is logged and surfaced in `.load_errors` (not + raised), so consumers can tell the file set is incomplete.""" entry = _write(tmp_path, "entry.yaml", "esphome: [unterminated\n") - discovered = discover_user_yaml_files(entry) + with caplog.at_level("WARNING", logger="esphome.yaml_util"): + discovered = discover_user_yaml_files(entry) assert isinstance(discovered, DiscoveredYamlFiles) + assert len(discovered.load_errors) == 1 + assert "entry.yaml" in discovered.load_errors[0] + assert any("discovery failed to parse" in r.message for r in caplog.records) + + +def test_discover_user_yaml_files_reports_unresolved_includes( + tmp_path: Path, +) -> None: + """A substitution-templated `!include` path is surfaced in `.unresolved`.""" + entry = _write_entry_including(tmp_path, "${board}.yaml") + discovered = discover_user_yaml_files(entry) + assert len(discovered.unresolved) == 1 + assert "${board}.yaml" in discovered.unresolved[0] def test_discover_user_yaml_files_deduplicates(tmp_path: Path) -> None: @@ -1941,6 +1964,49 @@ def test_dump__redaction_flag_does_not_leak_between_calls() -> None: assert "\\033[8m" in redacted_again +def test_secret_values_registered_swaps_scalars_in_dump() -> None: + """Registered value→name mappings make dump() emit `!secret ` for + matching scalars, and are removed again on exit.""" + with yaml_util.secret_values_registered({"hunter2": "wifi_password"}): + out = yaml_util.dump({"password": make_data_base("hunter2")}) + assert "password: !secret 'wifi_password'" in out + assert "hunter2" not in out + out_after = yaml_util.dump({"password": make_data_base("hunter2")}) + assert "hunter2" in out_after + assert "!secret" not in out_after + assert yaml_util.is_secret("hunter2") is None + + +def test_secret_values_registered_collects_emitted_names() -> None: + """The context yields a set recording exactly the `!secret` names the + dumper emitted, including real secrets, and not names never swapped.""" + yaml_util._SECRET_VALUES["real_value"] = "real_name" + with yaml_util.secret_values_registered({"hunter2": "wifi_password"}) as emitted: + yaml_util.dump( + { + "password": make_data_base("hunter2"), + "key": make_data_base("real_value"), + "plain": make_data_base("nothing"), + } + ) + assert emitted == {"wifi_password", "real_name"} + + +def test_secret_values_registered_does_not_clobber_real_secrets() -> None: + """A value already mapped by a real `!secret` keeps its original name.""" + yaml_util._SECRET_VALUES["hunter2"] = "original_name" + with yaml_util.secret_values_registered({"hunter2": "generated_name"}): + out = yaml_util.dump({"password": make_data_base("hunter2")}) + assert "!secret 'original_name'" in out + # The pre-existing mapping survives the context exit. + assert yaml_util.is_secret("hunter2") == "original_name" + + +def test_registered_secret_names() -> None: + yaml_util._SECRET_VALUES["value_a"] = "name_a" + assert "name_a" in yaml_util.registered_secret_names() + + @pytest.fixture(autouse=True) def clear_dropped_merge_keys() -> None: """Reset the dropped-merge-key queue between tests.""" @@ -1986,6 +2052,33 @@ def test_merge_include_no_overlap_records_nothing(tmp_path: Path) -> None: assert yaml_util.take_dropped_merge_keys() == [] +def test_wrapper_representers_consult_is_secret() -> None: + """!extend / !remove payloads and scalar !include paths equal to a + registered secret are swapped, never written in cleartext.""" + from esphome.config_helpers import Extend, Remove + + with yaml_util.secret_values_registered({"hunter2": "the_secret"}): + out = yaml_util.dump( + { + "a": Extend("hunter2"), + "b": Remove("hunter2"), + "c": Extend("plain_id"), + } + ) + assert out.count("!secret 'the_secret'") == 2 + assert "hunter2" not in out + assert "!extend 'plain_id'" in out + + +def test_scalar_include_path_equal_to_secret_is_swapped() -> None: + """A scalar !include whose path equals a registered secret is swapped, + matching the other wrapper representers.""" + include = yaml_util.IncludeFile(Path("/fake/main.yaml"), "hunter2", lambda _: {}) + with yaml_util.secret_values_registered({"hunter2": "the_secret"}): + out = yaml_util.dump({"key": include}) + assert out == "key: !secret 'the_secret'\n" + + # --------------------------------------------------------------------------- # track_document_range=False (validated-config-cache fast path) # ---------------------------------------------------------------------------