diff --git a/esphome/bundle.py b/esphome/bundle.py index b633c5ca4f0..f499e0de1bc 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -102,25 +102,21 @@ _KNOWN_FILE_EXTENSIONS = frozenset( ) -# Matches !secret references in YAML text. An optional surrounding -# quote pair around the key is allowed and ignored: YAML treats -# ``!secret 'foo'`` and ``!secret foo`` as the same key. This is -# intentionally a simple regex scan rather than a YAML parse — it may -# match inside comments or multi-line strings, which is the conservative -# direction (include more secrets rather than fewer). -_SECRET_RE = re.compile(r"""!secret\s+['"]?([^\s'"]+)""") - - def _find_used_secret_keys(yaml_files: list[Path]) -> set[str]: """Scan YAML files for ``!secret `` references.""" keys: set[str] = set() for fpath in yaml_files: try: text = fpath.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): + except (OSError, UnicodeDecodeError) as err: + _LOGGER.warning( + "Could not scan %s for !secret references (%s); the bundled " + "secret set may be incomplete", + fpath, + err, + ) continue - for match in _SECRET_RE.finditer(text): - keys.add(match.group(1)) + keys |= yaml_util.find_secret_references(text) return keys @@ -422,6 +418,18 @@ class ConfigBundleCreator: must ship every candidate so the remote build can pick any one. """ discovered = yaml_util.discover_user_yaml_files(self._config_path) + if discovered.load_errors: + _LOGGER.warning( + "Bundle may be incomplete; could not load all configuration files: %s", + "; ".join(discovered.load_errors), + ) + if discovered.unresolved: + _LOGGER.warning( + "Bundle may be incomplete; %d !include path(s) use " + "substitutions and cannot be captured: %s", + len(discovered.unresolved), + ", ".join(discovered.unresolved), + ) self._secrets_paths.update(discovered.secrets) # A !secret inside a file this re-parse does not reach (for example # a git-fetched package the builder re-fetches) still resolves diff --git a/esphome/components/api/wizard.py b/esphome/components/api/wizard.py index 314b907e4d0..befed39972c 100644 --- a/esphome/components/api/wizard.py +++ b/esphome/components/api/wizard.py @@ -6,7 +6,6 @@ Home Assistant shows the wizard when the device is added. See api_wizard.h for t from collections.abc import Callable import importlib import json -from types import ModuleType from typing import Any from esphome import automation @@ -27,7 +26,12 @@ from esphome.const import ( ) from esphome.core import CORE, ID import esphome.final_validate as fv -from esphome.helpers import fnv1_hash, fnv1_hash_object_id, fnv1a_32bit_hash +from esphome.helpers import ( + fnv1_hash, + fnv1_hash_object_id, + fnv1a_32bit_hash, + zstd_module, +) from esphome.types import ConfigType API_DOMAIN = "api" @@ -351,14 +355,6 @@ automation.register_apply_condition( ) -def zstd_module() -> ModuleType: - """The zstd module: the standard library one from Python 3.14, otherwise the backport.""" - try: - return importlib.import_module("compression.zstd") - except ImportError: - return importlib.import_module("backports.zstd") - - def _entity_document(conf: ConfigType, config: fv.FinalValidateConfig) -> ConfigType: """An entity of the device that the page shows, keyed as ListEntitiesResponse keys it.""" _, declaration = _wizard_input_declaration(config, conf[CONF_ID]) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 703af8281d2..534f7622e8f 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -1,5 +1,6 @@ from collections import UserDict from collections.abc import Callable +from dataclasses import dataclass, field from functools import reduce import logging from pathlib import Path @@ -34,11 +35,45 @@ from esphome.const import ( CONF_VARS, __version__ as ESPHOME_VERSION, ) -from esphome.core import EsphomeError +from esphome.core import CORE, EsphomeError _LOGGER = logging.getLogger(__name__) DOMAIN = CONF_PACKAGES + + +@dataclass(frozen=True) +class RemotePackageSource: + """A remote source a package was fetched from while processing the config.""" + + url: str + ref: str | None + + +@dataclass +class PackagesData: + """Per-run package state, keyed under DOMAIN in CORE.data.""" + + remote_sources: list[RemotePackageSource] = field(default_factory=list) + + +def _get_data() -> PackagesData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = PackagesData() + return CORE.data[DOMAIN] + + +def get_remote_package_sources() -> list[RemotePackageSource]: + """Remote sources fetched while processing this config, in fetch order. + + Consumers (e.g. store_yaml) use this to tell which parts of the config + came from remote repositories rather than local files. + """ + if (data := CORE.data.get(DOMAIN)) is None: + return [] + return data.remote_sources + + # Guard against infinite include chains (e.g. A includes B includes A). MAX_INCLUDE_DEPTH = 20 @@ -192,6 +227,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: username=config.get(CONF_USERNAME), password=config.get(CONF_PASSWORD), ) + source = RemotePackageSource(config[CONF_URL], config.get(CONF_REF)) + remote_sources = _get_data().remote_sources + if source not in remote_sources: + remote_sources.append(source) files: list[dict[str, Any]] = [] # ``repo_root`` is the directory containing ``.git`` and must be passed diff --git a/esphome/helpers.py b/esphome/helpers.py index 5ec46be67e4..1f9b1407971 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations from collections.abc import Callable, Iterable, MutableMapping from contextlib import suppress +import importlib import ipaddress import logging import os @@ -10,6 +11,7 @@ import platform import re import stat import sys +from types import ModuleType from typing import TYPE_CHECKING, TextIO from esphome.const import __version__ as ESPHOME_VERSION @@ -827,3 +829,11 @@ def docs_url(path: str) -> str: path = path.removeprefix("/") return docs_format.format(path=path) + + +def zstd_module() -> ModuleType: + """The zstd module: the standard library one from Python 3.14, otherwise the backport.""" + try: + return importlib.import_module("compression.zstd") + except ImportError: + return importlib.import_module("backports.zstd") diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index e892664724c..a8d0ac845a1 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -11,7 +11,8 @@ import logging import math import os from pathlib import Path -from typing import Any +import re +from typing import Any, NamedTuple import uuid import yaml @@ -46,6 +47,10 @@ _LOGGER = logging.getLogger(__name__) SECRET_YAML = "secrets.yaml" _SECRET_CACHE = {} _SECRET_VALUES = {} +# Stack of collectors (one per active secret_values_registered context); the +# dumper records every emitted `!secret` name into the innermost one. YAML +# processing is single-threaded. +_EMITTED_SECRET_NAMES: list[set[str]] = [] # Not thread-safe — config processing is single-threaded today. _load_listeners: list[Callable[[Path], None]] = [] @@ -390,6 +395,7 @@ def _load_include_candidates( seen: set[int], expanded_paths: set[Path], keepalive: list[Any], + result: ForceLoadResult, ) -> None: """Load every filesystem candidate for an unresolved ``IncludeFile``.""" from voluptuous import Invalid @@ -402,6 +408,7 @@ def _load_include_candidates( include.file, include.parent_file, ) + result.unresolved.append(str(include.file)) return _LOGGER.debug( "Expanding !include %s (referenced from %s) to %d candidate file(s)", @@ -427,6 +434,7 @@ def _load_include_candidates( include.file, err, ) + result.errors.append(f"{candidate}: {err}") continue # The throwaway IncludeFile is this tree's only owner; keep the tree # alive so ids recorded in ``seen`` stay unique for the traversal. @@ -437,9 +445,36 @@ def _load_include_candidates( _seen=seen, _expanded_paths=expanded_paths, _keepalive=keepalive, + _result=result, ) +# Matches !secret references in YAML text. An optional surrounding +# quote pair around the key is allowed and ignored: YAML treats +# ``!secret 'foo'`` and ``!secret foo`` as the same key. This is +# intentionally a simple regex scan rather than a YAML parse — it may +# match inside comments or multi-line strings, which is the conservative +# direction (include more secrets rather than fewer). +_SECRET_REFERENCE_RE = re.compile(r"""!secret\s+['"]?([^\s'"]+)""") + + +def find_secret_references(text: str) -> set[str]: + """Return the ``!secret `` names referenced in a YAML document text.""" + return {match.group(1) for match in _SECRET_REFERENCE_RE.finditer(text)} + + +class ForceLoadResult(NamedTuple): + """Outcome of :func:`force_load_include_files`. + + ``unresolved`` lists ``!include`` path strings with substitution variables or + expressions that matched no candidate file; ``errors`` lists includes that failed + to load. Either being non-empty means the walk was incomplete. + """ + + unresolved: list[str] + errors: list[str] + + def force_load_include_files( obj: Any, *, @@ -447,7 +482,8 @@ def force_load_include_files( _seen: set[int] | None = None, _expanded_paths: set[Path] | None = None, _keepalive: list[Any] | None = None, -) -> None: + _result: ForceLoadResult | None = None, +) -> ForceLoadResult: """Recursively resolve any deferred ``IncludeFile`` instances in a YAML tree. Nested ``!include`` returns a deferred ``IncludeFile`` that is only resolved @@ -477,10 +513,12 @@ def force_load_include_files( # fresh tree look already seen. Discovery is a one-shot operation, # so holding the parsed trees costs nothing. _keepalive = [] + if _result is None: + _result = ForceLoadResult([], []) if isinstance(obj, IncludeFile): if id(obj) in _seen: - return + return _result _seen.add(id(obj)) if obj.has_unresolved_file(): _load_include_candidates( @@ -489,8 +527,9 @@ def force_load_include_files( seen=_seen, expanded_paths=_expanded_paths, keepalive=_keepalive, + result=_result, ) - return + return _result try: loaded = obj.load() except (EsphomeError, Invalid) as err: @@ -500,17 +539,19 @@ def force_load_include_files( obj.parent_file, err, ) - return + _result.errors.append(f"{obj.file}: {err}") + return _result force_load_include_files( loaded, warn_on_unresolved=warn_on_unresolved, _seen=_seen, _expanded_paths=_expanded_paths, _keepalive=_keepalive, + _result=_result, ) elif isinstance(obj, dict): if id(obj) in _seen: - return + return _result _seen.add(id(obj)) for value in obj.values(): force_load_include_files( @@ -519,10 +560,11 @@ def force_load_include_files( _seen=_seen, _expanded_paths=_expanded_paths, _keepalive=_keepalive, + _result=_result, ) elif isinstance(obj, (list, tuple)): if id(obj) in _seen: - return + return _result _seen.add(id(obj)) for item in obj: force_load_include_files( @@ -531,7 +573,9 @@ def force_load_include_files( _seen=_seen, _expanded_paths=_expanded_paths, _keepalive=_keepalive, + _result=_result, ) + return _result @dataclass(slots=True) @@ -542,11 +586,16 @@ class DiscoveredYamlFiles: were re-parsing the user's config; ``secrets`` is the subset whose *un-resolved* filename matched :data:`esphome.const.SECRETS_FILES` (so a ``secrets.yaml`` symlinked to a differently-named target is still - flagged as secrets). + flagged as secrets). ``unresolved`` lists ``!include`` path strings that + contain substitution variables and therefore could not be loaded, and + ``load_errors`` lists files that failed to parse or load — consumers + should treat ``files`` as incomplete when either is non-empty. """ files: list[Path] = field(default_factory=list) secrets: set[Path] = field(default_factory=set) + unresolved: list[str] = field(default_factory=list) + load_errors: list[str] = field(default_factory=list) def discover_user_yaml_files(config_path: Path) -> DiscoveredYamlFiles: @@ -576,9 +625,16 @@ def discover_user_yaml_files(config_path: Path) -> DiscoveredYamlFiles: try: try: data = load_yaml(config_path) - except EsphomeError: - return DiscoveredYamlFiles(list(loaded), secrets) - force_load_include_files(data, warn_on_unresolved=False) + except EsphomeError as err: + _LOGGER.warning( + "YAML discovery failed to parse %s: %s", config_path, err + ) + return DiscoveredYamlFiles( + list(loaded), secrets, load_errors=[f"{config_path}: {err}"] + ) + unresolved, load_errors = force_load_include_files( + data, warn_on_unresolved=False + ) finally: _load_listeners.remove(_capture_secret) @@ -589,7 +645,7 @@ def discover_user_yaml_files(config_path: Path) -> DiscoveredYamlFiles: if path not in seen: seen.add(path) unique.append(path) - return DiscoveredYamlFiles(unique, secrets) + return DiscoveredYamlFiles(unique, secrets, unresolved, load_errors) def _add_data_ref(fn): @@ -1119,6 +1175,38 @@ def _load_yaml_internal_with_type( loader.dispose() +def registered_secret_names() -> set[str]: + """Names of all ``!secret`` keys the loader has seen since the last clear.""" + return set(_SECRET_VALUES.values()) + + +@contextmanager +def secret_values_registered(values: dict[str, str]) -> Generator[set[str]]: + """Temporarily register value→name mappings so :func:`dump` renders those + scalars as ``!secret ``. + + Mappings already present in ``_SECRET_VALUES`` (values loaded through a + real ``!secret``) win over the supplied ones and are left untouched. + + Yields a set that collects the name of every ``!secret`` reference the + dumper emits while the context is active, so callers can tell exactly + which registered values were actually swapped. + """ + added = {v: n for v, n in values.items() if v not in _SECRET_VALUES} + _SECRET_VALUES.update(added) + emitted: set[str] = set() + _EMITTED_SECRET_NAMES.append(emitted) + try: + yield emitted + finally: + # Contexts unwind LIFO, so the innermost collector is always last; + # pop() removes by position where remove() would match the first + # *equal* set and could strip an outer context's collector. + _EMITTED_SECRET_NAMES.pop() + for value in added: + _SECRET_VALUES.pop(value, None) + + def dump( dict_, show_secrets=False, @@ -1333,7 +1421,10 @@ class ESPHomeDumper(yaml.SafeDumper): return node def represent_secret(self, value): - return self.represent_scalar(tag="!secret", value=_SECRET_VALUES[str(value)]) + name = _SECRET_VALUES[str(value)] + if _EMITTED_SECRET_NAMES: + _EMITTED_SECRET_NAMES[-1].add(name) + return self.represent_scalar(tag="!secret", value=name) def represent_stringify(self, value): if is_secret(value): @@ -1414,13 +1505,21 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_scalar(tag="!lambda", value=value.value, style="|") def represent_extend(self, value): + # Consult is_secret like the other scalar representers so a payload + # equal to a registered secret is never written out in cleartext. + if is_secret(value.value): + return self.represent_secret(value.value) return self.represent_scalar(tag="!extend", value=value.value) def represent_remove(self, value): + if is_secret(value.value): + return self.represent_secret(value.value) return self.represent_scalar(tag="!remove", value=value.value) def represent_include_file(self, value): if value.vars or value.condition is not None: + # The mapping values route through the regular representers, + # which already consult is_secret. mapping = {"file": value.file} if value.vars: mapping["vars"] = value.vars @@ -1429,6 +1528,8 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_mapping( tag="!include", mapping=mapping, flow_style=False ) + if is_secret(value.file): + return self.represent_secret(value.file) return self.represent_scalar(tag="!include", value=value.file) def represent_id(self, value): diff --git a/requirements.txt b/requirements.txt index 91a36ecbf70..fe3d3ab64e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import esphome-glyphsets==0.2.0 -backports.zstd==1.7.0; python_version < "3.14" # api wizard; compression.zstd is in the standard library from 3.14 +backports.zstd==1.7.0; python_version < "3.14" # esphome.helpers.zstd_module; compression.zstd is in the standard library from 3.14 pillow==12.3.0 resvg-py==0.5.0 freetype-py==2.5.1 diff --git a/tests/component_tests/api/test_wizard.py b/tests/component_tests/api/test_wizard.py index 276f42d624e..112092482b2 100644 --- a/tests/component_tests/api/test_wizard.py +++ b/tests/component_tests/api/test_wizard.py @@ -14,7 +14,7 @@ from esphome.components.api import wizard from esphome.components.homeassistant.switch import SUPPORTED_DOMAINS as SWITCH_DOMAINS from esphome.config import load_config from esphome.core import CORE -from esphome.helpers import fnv1_hash, fnv1a_32bit_hash +from esphome.helpers import fnv1_hash, fnv1a_32bit_hash, zstd_module from tests.component_tests.helpers import get_define_value ESP32_HEADER = """ @@ -613,7 +613,7 @@ def test_text_select_and_button_default_to_their_domains( tmp_path: Path, generate_main: Callable[[str | Path], str] ) -> None: main_cpp = generate_main(write_input_config(tmp_path, ESP32_HEADER)) - document = json.loads(wizard.zstd_module().decompress(blob_in(main_cpp))) + document = json.loads(zstd_module().decompress(blob_in(main_cpp))) filters = { entry["key"]: entry.get("entity_filters") for page in document["pages"] @@ -733,7 +733,7 @@ def test_the_blob_is_the_exact_json_document( ) blob = blob_in(main_cpp) - text = wizard.zstd_module().decompress(blob).decode("utf-8") + text = zstd_module().decompress(blob).decode("utf-8") expected = { "version": 1, "pages": [ @@ -788,7 +788,7 @@ def test_the_blob_is_deterministic_and_the_same_on_every_platform( ) # Compressing the same document again gives the same bytes document = wizard.wizard_document(CORE.config["api"]["wizard"], CORE.config) - again = wizard.zstd_module().compress( + again = zstd_module().compress( json.dumps( document, separators=(",", ":"), sort_keys=True, ensure_ascii=False ).encode("utf-8"), @@ -859,21 +859,6 @@ def test_an_entity_without_a_name_cannot_be_in_the_wizard(tmp_path: Path) -> Non assert any("has no name of its own" in error for error in errors), errors -def test_zstd_falls_back_to_the_backport(monkeypatch: pytest.MonkeyPatch) -> None: - """Before Python 3.14 the standard library has no zstd, so the backport is used.""" - backport = object() - - def import_module(name: str) -> object: - if name == "compression.zstd": - raise ImportError(name) - assert name == "backports.zstd" - return backport - - monkeypatch.setattr(wizard.importlib, "import_module", import_module) - - assert wizard.zstd_module() is backport - - def test_a_wizard_too_big_for_one_message_is_rejected(tmp_path: Path) -> None: # Random text does not compress, so this needs more than the limit even compressed rng = random.Random(1) diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 418cab5ea15..a662a291ed8 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -10,9 +10,11 @@ import pytest from esphome import bundle from esphome.components.packages import ( CONFIG_SCHEMA, + RemotePackageSource, _substitute_package_definition, _walk_packages, do_packages_pass, + get_remote_package_sources, merge_packages, resolve_packages, ) @@ -614,6 +616,49 @@ def test_package_remove_by_missing_id() -> None: assert actual == expected +def test_get_remote_package_sources_empty_without_remote_packages() -> None: + """A config without remote packages records no sources.""" + assert get_remote_package_sources() == [] + + +@patch("esphome.yaml_util.load_yaml") +@patch("pathlib.Path.is_file") +@patch("esphome.git.clone_or_update") +def test_get_remote_package_sources_records_each_fetch_once( + mock_clone_or_update, mock_is_file, mock_load_yaml +) -> None: + """Every fetched remote source is recorded exactly once.""" + mock_clone_or_update.return_value = (Path("/tmp/noexists"), MagicMock()) + mock_is_file.return_value = True + mock_load_yaml.side_effect = lambda *args, **kwargs: OrderedDict() + + config = { + CONF_PACKAGES: { + "first": { + CONF_URL: "https://github.com/esphome/repo-a", + CONF_REF: "main", + CONF_FILES: [TEST_YAML_FILENAME], + }, + "again": { + CONF_URL: "https://github.com/esphome/repo-a", + CONF_REF: "main", + CONF_FILES: ["other.yaml"], + }, + "second": { + CONF_URL: "https://github.com/esphome/repo-b", + CONF_FILES: [TEST_YAML_FILENAME], + }, + } + } + packages_pass(config) + + sources = get_remote_package_sources() + assert sorted(sources, key=lambda source: source.url) == [ + RemotePackageSource("https://github.com/esphome/repo-a", "main"), + RemotePackageSource("https://github.com/esphome/repo-b", None), + ] + + @patch("esphome.yaml_util.load_yaml") @patch("pathlib.Path.is_file") @patch("esphome.git.clone_or_update") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ce2e7f5bbc0..5090ad20036 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -212,10 +212,9 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s loop = asyncio.get_running_loop() content = await loop.run_in_executor(None, read_file, fixture_path) - # Replace the port in the config if it contains api section - if "api:" in content: - # Add port configuration after api: - content = content.replace("api:", f"api:\n port: {unused_tcp_port}") + # Replace the port in the config if it contains an api section. Anchored to + # the start of a line so keys that merely end in "api:" are left alone. + content = re.sub(r"(?m)^api:", f"api:\n port: {unused_tcp_port}", content) # Add debug build flags for integration tests to enable assertions if "esphome:" in content and "build_flags:" not in content: diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index ff5564e8150..aba47c9237e 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -17,6 +17,7 @@ from unittest.mock import Mock, patch import pytest +from esphome import yaml_util from esphome.core import CORE here = Path(__file__).parent @@ -33,6 +34,16 @@ def reset_core(): CORE.reset() +@pytest.fixture(autouse=True) +def clear_yaml_secrets(): + """Isolate the yaml_util secrets registry between tests.""" + yaml_util._SECRET_VALUES.clear() + yaml_util._SECRET_CACHE.clear() + yield + yaml_util._SECRET_VALUES.clear() + yaml_util._SECRET_CACHE.clear() + + @pytest.fixture def fixture_path() -> Path: """ diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index 8dae1e87b20..764f160cbc0 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -1285,3 +1285,20 @@ def test_get_usable_cpu_count_sources() -> None: mock_os_unknown = types.SimpleNamespace(cpu_count=lambda: None) with patch("esphome.helpers.os", mock_os_unknown): assert helpers.get_usable_cpu_count() == 1 + + +def test_zstd_module_falls_back_to_the_backport( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Before Python 3.14 the standard library has no zstd, so the backport is used.""" + backport = object() + + def import_module(name: str) -> object: + if name == "compression.zstd": + raise ImportError(name) + assert name == "backports.zstd" + return backport + + monkeypatch.setattr(helpers.importlib, "import_module", import_module) + + assert helpers.zstd_module() is backport diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index e1c3080e291..7feb4f872f5 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -25,16 +25,6 @@ from esphome.yaml_util import ( ) -@pytest.fixture(autouse=True) -def clear_secrets_cache() -> None: - """Clear the secrets cache before each test.""" - yaml_util._SECRET_VALUES.clear() - yaml_util._SECRET_CACHE.clear() - yield - yaml_util._SECRET_VALUES.clear() - yaml_util._SECRET_CACHE.clear() - - @pytest.fixture(autouse=True) def clear_core_frontmatter() -> None: """Reset CORE.frontmatter between tests.""" @@ -1289,22 +1279,38 @@ def test_force_load_include_files_unresolved_log_level( assert matching == [expect_level] +def test_force_load_include_files_returns_unresolved_paths( + patch_include_file: None, +) -> None: + """Includes with substitution-templated paths are reported back to the + caller; resolvable ones are not.""" + templated = _StubInclude("${var}.yaml", unresolved=True) + plain = _StubInclude("ok.yaml") + result = force_load_include_files({"a": templated, "b": plain}) + assert result.unresolved == [str(templated.file)] + assert result.errors == [] + assert plain.load_calls == 1 + + def test_force_load_include_files_warns_on_load_failure( patch_include_file: None, caplog: pytest.LogCaptureFixture, ) -> None: - """An `EsphomeError` raised by `load()` is caught and logged, not propagated.""" + """An `EsphomeError` raised by `load()` is caught, logged, and reported to + the caller — not propagated.""" stub = _StubInclude("missing.yaml", raise_on_load=EsphomeError("boom")) with caplog.at_level("WARNING", logger="esphome.yaml_util"): - force_load_include_files({"k": stub}) + result = force_load_include_files({"k": stub}) assert any( "Failed to load !include" in r.message and "missing.yaml" in r.message for r in caplog.records ) + assert result.errors == [f"{stub.file}: boom"] + assert result.unresolved == [] def test_discovered_yaml_files_holds_files_and_secrets() -> None: - """`DiscoveredYamlFiles` is a small data carrier; both fields are mandatory.""" + """`DiscoveredYamlFiles` is a small data carrier.""" files = [Path("/tmp/a.yaml")] secrets = {Path("/tmp/a.yaml")} discovered = DiscoveredYamlFiles(files, secrets) @@ -1367,11 +1373,28 @@ def test_discover_user_yaml_files_flags_secrets_symlink(tmp_path: Path) -> None: assert target.resolve() in discovered.secrets -def test_discover_user_yaml_files_swallows_parse_errors(tmp_path: Path) -> None: - """A YAML parse failure returns whatever was tracked so far without raising.""" +def test_discover_user_yaml_files_reports_parse_errors( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A YAML parse failure is logged and surfaced in `.load_errors` (not + raised), so consumers can tell the file set is incomplete.""" entry = _write(tmp_path, "entry.yaml", "esphome: [unterminated\n") - discovered = discover_user_yaml_files(entry) + with caplog.at_level("WARNING", logger="esphome.yaml_util"): + discovered = discover_user_yaml_files(entry) assert isinstance(discovered, DiscoveredYamlFiles) + assert len(discovered.load_errors) == 1 + assert "entry.yaml" in discovered.load_errors[0] + assert any("discovery failed to parse" in r.message for r in caplog.records) + + +def test_discover_user_yaml_files_reports_unresolved_includes( + tmp_path: Path, +) -> None: + """A substitution-templated `!include` path is surfaced in `.unresolved`.""" + entry = _write_entry_including(tmp_path, "${board}.yaml") + discovered = discover_user_yaml_files(entry) + assert len(discovered.unresolved) == 1 + assert "${board}.yaml" in discovered.unresolved[0] def test_discover_user_yaml_files_deduplicates(tmp_path: Path) -> None: @@ -1941,6 +1964,49 @@ def test_dump__redaction_flag_does_not_leak_between_calls() -> None: assert "\\033[8m" in redacted_again +def test_secret_values_registered_swaps_scalars_in_dump() -> None: + """Registered value→name mappings make dump() emit `!secret ` for + matching scalars, and are removed again on exit.""" + with yaml_util.secret_values_registered({"hunter2": "wifi_password"}): + out = yaml_util.dump({"password": make_data_base("hunter2")}) + assert "password: !secret 'wifi_password'" in out + assert "hunter2" not in out + out_after = yaml_util.dump({"password": make_data_base("hunter2")}) + assert "hunter2" in out_after + assert "!secret" not in out_after + assert yaml_util.is_secret("hunter2") is None + + +def test_secret_values_registered_collects_emitted_names() -> None: + """The context yields a set recording exactly the `!secret` names the + dumper emitted, including real secrets, and not names never swapped.""" + yaml_util._SECRET_VALUES["real_value"] = "real_name" + with yaml_util.secret_values_registered({"hunter2": "wifi_password"}) as emitted: + yaml_util.dump( + { + "password": make_data_base("hunter2"), + "key": make_data_base("real_value"), + "plain": make_data_base("nothing"), + } + ) + assert emitted == {"wifi_password", "real_name"} + + +def test_secret_values_registered_does_not_clobber_real_secrets() -> None: + """A value already mapped by a real `!secret` keeps its original name.""" + yaml_util._SECRET_VALUES["hunter2"] = "original_name" + with yaml_util.secret_values_registered({"hunter2": "generated_name"}): + out = yaml_util.dump({"password": make_data_base("hunter2")}) + assert "!secret 'original_name'" in out + # The pre-existing mapping survives the context exit. + assert yaml_util.is_secret("hunter2") == "original_name" + + +def test_registered_secret_names() -> None: + yaml_util._SECRET_VALUES["value_a"] = "name_a" + assert "name_a" in yaml_util.registered_secret_names() + + @pytest.fixture(autouse=True) def clear_dropped_merge_keys() -> None: """Reset the dropped-merge-key queue between tests.""" @@ -1986,6 +2052,33 @@ def test_merge_include_no_overlap_records_nothing(tmp_path: Path) -> None: assert yaml_util.take_dropped_merge_keys() == [] +def test_wrapper_representers_consult_is_secret() -> None: + """!extend / !remove payloads and scalar !include paths equal to a + registered secret are swapped, never written in cleartext.""" + from esphome.config_helpers import Extend, Remove + + with yaml_util.secret_values_registered({"hunter2": "the_secret"}): + out = yaml_util.dump( + { + "a": Extend("hunter2"), + "b": Remove("hunter2"), + "c": Extend("plain_id"), + } + ) + assert out.count("!secret 'the_secret'") == 2 + assert "hunter2" not in out + assert "!extend 'plain_id'" in out + + +def test_scalar_include_path_equal_to_secret_is_swapped() -> None: + """A scalar !include whose path equals a registered secret is swapped, + matching the other wrapper representers.""" + include = yaml_util.IncludeFile(Path("/fake/main.yaml"), "hunter2", lambda _: {}) + with yaml_util.secret_values_registered({"hunter2": "the_secret"}): + out = yaml_util.dump({"key": include}) + assert out == "key: !secret 'the_secret'\n" + + # --------------------------------------------------------------------------- # track_document_range=False (validated-config-cache fast path) # ---------------------------------------------------------------------------