[substitutions][core] Expand templated !include paths to on-disk candidates during bundle discovery (#17647)

This commit is contained in:
J. Nick Koston
2026-08-03 20:48:11 -05:00
committed by GitHub
parent 25d5985775
commit 4f67932e0d
6 changed files with 648 additions and 23 deletions
+58 -1
View File
@@ -1248,7 +1248,8 @@ def test_discover_files_deeply_nested_include(tmp_path: Path) -> None:
def test_discover_files_nested_include_unresolved_substitution(
tmp_path: Path,
) -> None:
"""!include with substitution vars in path cannot be resolved; skipped gracefully."""
"""!include with substitution vars in path but no candidate files on disk
(the glob's only match is the config itself) is skipped gracefully."""
config_dir = _setup_config_dir(tmp_path)
(config_dir / "test.yaml").write_text(
"esphome:\n name: test\nwifi: !include ${platform}.yaml\n"
@@ -1262,6 +1263,62 @@ def test_discover_files_nested_include_unresolved_substitution(
assert "test.yaml" in paths
def test_discover_files_bundles_all_include_candidates(tmp_path: Path) -> None:
"""The issue-17650 layout: templated package includes chain through a glob
candidate into a Jinja conditional whose ``../`` branch is bundled."""
config_dir = _setup_config_dir(
tmp_path,
files={
"includes/esp-basics.yaml": (
"packages:\n"
" - !include boards/${board}.yaml\n"
" - !include keys/${system_name}.yaml\n"
),
"includes/boards/wemos-d1-mini.yaml": (
'packages:\n - !include ${ "NO BT.yaml" if bt else "../empty.yaml" }\n'
),
"includes/keys/device-a.yaml": "api:\n",
"includes/keys/device-b.yaml": "api:\n",
"includes/empty.yaml": "{}\n",
},
)
(config_dir / "test.yaml").write_text(
"esphome:\n name: test\npackages:\n - !include includes/esp-basics.yaml\n"
)
creator = ConfigBundleCreator({})
files = creator.discover_files()
paths = [f.path for f in files]
assert "includes/esp-basics.yaml" in paths
assert "includes/boards/wemos-d1-mini.yaml" in paths
assert "includes/keys/device-a.yaml" in paths
assert "includes/keys/device-b.yaml" in paths
assert "includes/empty.yaml" in paths
def test_discover_files_candidate_outside_config_dir_skipped(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A candidate branch resolving above the config dir is not bundled."""
config_dir = _setup_config_dir(tmp_path)
(tmp_path / "outside.yaml").write_text("api:\n")
(config_dir / "test.yaml").write_text(
"esphome:\n name: test\n"
'wifi: !include ${ "a.yaml" if x else "../outside.yaml" }\n'
)
creator = ConfigBundleCreator({})
files = creator.discover_files()
paths = [f.path for f in files]
assert not any("outside" in p for p in paths)
assert any(
"outside config directory" in r.message and "outside.yaml" in r.message
for r in caplog.records
)
def test_discover_files_nested_include_load_failure(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
+133
View File
@@ -1,3 +1,5 @@
from collections import ChainMap
from fnmatch import fnmatchcase
import logging
from pathlib import Path
from typing import Any
@@ -961,3 +963,134 @@ def test_remote_package_scalar_yaml_raises_helpful_error(
msg = str(exc_info.value)
assert "mapping at the top level" in msg
assert "file1.yaml" in msg
@pytest.mark.parametrize(
("value", "expected"),
[
pytest.param("wifi.yaml", ["wifi.yaml"], id="literal_passthrough"),
pytest.param(
"keys/${system_name}.yaml", ["keys/*.yaml"], id="embedded_substitution"
),
pytest.param(
"network/${eth_model}/config.yaml",
["network/*/config.yaml"],
id="directory_substitution",
),
pytest.param(
"device-$platform.yaml", ["device-*.yaml"], id="unbraced_substitution"
),
pytest.param("${a}${b}.yaml", ["*.yaml"], id="adjacent_wildcards_collapse"),
pytest.param(
'${ "a.yaml" if x else "../empty.yaml" }',
["a.yaml", "../empty.yaml"],
id="conditional_literals",
),
pytest.param(
'pre-${ "a" if c else "b" }.yaml',
["pre-a.yaml", "pre-b.yaml"],
id="conditional_spliced",
),
pytest.param(
'${ "x.yaml" if a else ("y.yaml" if b else "z.yaml") }',
["x.yaml", "y.yaml", "z.yaml"],
id="nested_conditional",
),
pytest.param(
'${ "same.yaml" if x else "same.yaml" }',
["same.yaml"],
id="duplicate_literals_dedupe",
),
pytest.param('${ "a.yaml" if x }', ["a.yaml"], id="conditional_no_else"),
pytest.param(
'${ "NO BLUETOOTH SUPPORT ON ESP8266.yaml"'
' if enable_bluetooth_proxy else "../empty.yaml" }',
["NO BLUETOOTH SUPPORT ON ESP8266.yaml", "../empty.yaml"],
id="issue_17650_verbatim",
),
pytest.param(
'${ "" if x else "b.yaml" }', ["b.yaml"], id="empty_literal_dropped"
),
pytest.param(
"keys\\${system_name}.yaml",
["keys\\*.yaml"],
id="backslash_separator",
),
pytest.param(
'${ "it\'s.yaml" if x else "b.yaml" }',
["it's.yaml", "b.yaml"],
id="apostrophe_in_literal",
),
pytest.param(
'${ "a-${x}.yaml" if c else "b.yaml" }',
["a-*.yaml", "b.yaml"],
id="substitution_inside_literal",
),
pytest.param("sensor [${x}].yaml", ["sensor [[]*].yaml"], id="bracket_escaped"),
pytest.param(
"config?${x}.yaml", ["config[?]*.yaml"], id="question_mark_escaped"
),
pytest.param(
"../${x}/config.yaml", ["../*/config.yaml"], id="ascending_directory"
),
pytest.param("${file}", [], id="bare_variable_dropped"),
pytest.param("../${file}", [], id="ascending_bare_variable_dropped"),
pytest.param(
'${ name ~ ".yaml" }', [".yaml"], id="dynamic_concat_extracts_literal"
),
pytest.param("${ if }", [], id="no_literal_expression_dropped"),
pytest.param(
"<% if x %>a.yaml<% endif %>", ["*a.yaml*"], id="block_statement_globs"
),
],
)
def test_include_candidate_patterns(value: str, expected: list[str]) -> None:
"""Templated include paths expand to glob patterns and branch literals."""
assert substitutions.include_candidate_patterns(value) == expected
@pytest.mark.parametrize(
("template", "variables"),
[
pytest.param(
"keys/${system_name}.yaml", {"system_name": "esp-buero"}, id="embedded"
),
pytest.param("device-$platform.yaml", {"platform": "esp32"}, id="unbraced"),
pytest.param(
"network/${eth_model}/config.yaml", {"eth_model": "eth01"}, id="directory"
),
pytest.param(
'${ "NO BT.yaml" if bt else "../empty.yaml" }',
{"bt": True},
id="conditional_true",
),
pytest.param(
'${ "NO BT.yaml" if bt else "../empty.yaml" }',
{"bt": False},
id="conditional_false",
),
pytest.param('pre-${ "a" if c else "b" }.yaml', {"c": True}, id="spliced"),
pytest.param("${a}${b}.yaml", {"a": "x", "b": "y"}, id="adjacent"),
pytest.param("sensor [${x}].yaml", {"x": "a"}, id="bracket"),
],
)
def test_include_candidate_patterns_cover_real_expansion(
template: str, variables: dict[str, Any]
) -> None:
"""
Lockstep pin against the real substitution machinery.
include_candidate_patterns mirrors _expand_substitutions without
variable values (the evaluator returns the one selected branch, so it
cannot enumerate candidates itself); this asserts every filename the
real pass resolves is covered by a candidate pattern, so a change to
reference syntax or expansion order breaks here instead of silently
dropping files from bundles.
"""
resolved = str(
substitutions._expand_substitutions(
template, [], ChainMap(variables), True, None
)
)
patterns = substitutions.include_candidate_patterns(template)
assert any(fnmatchcase(resolved, p) or resolved == p for p in patterns)
+244 -1
View File
@@ -1003,8 +1003,10 @@ class _StubInclude:
load_result: object = None,
raise_on_load: EsphomeError | None = None,
) -> None:
# Default parent lives in a nonexistent directory so unresolved
# stubs never glob real files during candidate expansion.
self.file = Path(file)
self.parent_file = parent_file or Path("/tmp/parent.yaml")
self.parent_file = parent_file or Path("/nonexistent/parent.yaml")
self._unresolved = unresolved
self._load_result = load_result if load_result is not None else {}
self._raise = raise_on_load
@@ -1182,6 +1184,247 @@ def test_discover_user_yaml_files_deduplicates(tmp_path: Path) -> None:
assert discovered.files.count(wifi_resolved) == 1
def test_discover_user_yaml_files_expands_directory_substitution(
tmp_path: Path,
) -> None:
"""A substitution spanning a directory segment globs across directories."""
_write(tmp_path, "network/eth01/config.yaml", "ethernet:\n")
_write(tmp_path, "network/eth02/config.yaml", "ethernet:\n")
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "network/${eth_model}/config.yaml")
)
resolved = set(discovered.files)
assert (tmp_path / "network/eth01/config.yaml").resolve() in resolved
assert (tmp_path / "network/eth02/config.yaml").resolve() in resolved
def test_discover_user_yaml_files_loads_both_branches_of_issue_conditional(
tmp_path: Path,
) -> None:
"""Both branch files of the issue-17650 conditional load when present,
including the filename with spaces."""
_write(tmp_path, "empty.yaml", "{}\n")
_write(tmp_path, "boards/NO BLUETOOTH SUPPORT ON ESP8266.yaml", "api:\n")
_write(
tmp_path,
"boards/esp8266.yaml",
"packages:\n"
' - !include ${ "NO BLUETOOTH SUPPORT ON ESP8266.yaml"'
' if enable_bluetooth_proxy else "../empty.yaml" }\n',
)
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "boards/esp8266.yaml")
)
resolved = set(discovered.files)
assert (tmp_path / "boards/NO BLUETOOTH SUPPORT ON ESP8266.yaml").resolve() in (
resolved
)
assert (tmp_path / "empty.yaml").resolve() in resolved
def test_discover_user_yaml_files_glob_matches_bracket_filenames(
tmp_path: Path,
) -> None:
"""Glob metacharacters in the literal filename text stay literal."""
_write(tmp_path, "sensor [a].yaml", "api:\n")
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "sensor [${x}].yaml")
)
assert "sensor [a].yaml" in {p.name for p in discovered.files}
def test_discover_user_yaml_files_ascending_glob(tmp_path: Path) -> None:
"""A templated include reaching into a sibling directory via ``..`` globs."""
_write(tmp_path, "shared/common.yaml", "api:\n")
_write(tmp_path, "nodes/dev.yaml", "p: !include ../shared/${x}.yaml\n")
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "nodes/dev.yaml")
)
assert (tmp_path / "shared/common.yaml").resolve() in discovered.files
def test_discover_user_yaml_files_mapping_include_with_vars(tmp_path: Path) -> None:
"""The mapping !include form (file + vars) expands a templated filename."""
_write(tmp_path, "keys/a.yaml", "pin: ${num}\n")
entry = _write(
tmp_path,
"entry.yaml",
"wifi: !include\n file: keys/${n}.yaml\n vars:\n num: 4\n",
)
discovered = discover_user_yaml_files(entry)
assert (tmp_path / "keys/a.yaml").resolve() in discovered.files
def test_discover_user_yaml_files_absolute_templated_include_skipped(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""An absolute templated include is skipped gracefully instead of crashing."""
shared = tmp_path / "shared"
_write(tmp_path, "shared/common.yaml", "api:\n")
with caplog.at_level("DEBUG", logger="esphome.yaml_util"):
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, f"{shared}/${{x}}.yaml")
)
assert (shared / "common.yaml").resolve() not in discovered.files
assert any("Cannot glob include pattern" in r.message for r in caplog.records)
def test_discover_user_yaml_files_glob_skips_dollar_named_files(
tmp_path: Path,
) -> None:
"""An on-disk filename containing ``$`` can't load; the glob skips it."""
_write(tmp_path, "keys/a.yaml", "api:\n")
_write(tmp_path, "keys/b$roken.yaml", "api:\n")
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "keys/${n}.yaml")
)
names = {p.name for p in discovered.files}
assert "a.yaml" in names
assert "b$roken.yaml" not in names
def test_discover_user_yaml_files_glob_error_skips_include(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A filesystem error during candidate globbing warns and skips the include."""
entry = _write_entry_including(tmp_path, "keys/${n}.yaml")
with (
patch.object(Path, "glob", side_effect=OSError("boom")),
caplog.at_level("DEBUG", logger="esphome.yaml_util"),
):
discovered = discover_user_yaml_files(entry)
assert [p.name for p in discovered.files] == ["entry.yaml"]
matching = [
r.levelname
for r in caplog.records
if "I/O error globbing include pattern" in r.message
]
assert matching == ["WARNING"]
def test_force_load_candidate_failure_warns_by_default(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A broken candidate logs at WARNING outside the discovery re-parse."""
_write(tmp_path, "keys/bad.yaml", "esphome: [unterminated\n")
entry = _write_entry_including(tmp_path, "keys/${n}.yaml")
with caplog.at_level("DEBUG", logger="esphome.yaml_util"):
force_load_include_files(yaml_util.load_yaml(entry))
matching = [
r.levelname for r in caplog.records if "Failed to load candidate" in r.message
]
assert matching == ["WARNING"]
def test_discover_user_yaml_files_glob_skips_hidden_files(tmp_path: Path) -> None:
"""Candidate globs exclude hidden files, matching ``!include_dir_*``."""
_write(tmp_path, "keys/device-a.yaml", "api:\n")
_write(tmp_path, "keys/.hidden.yaml", "api:\n")
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "keys/${name}.yaml")
)
names = {p.name for p in discovered.files}
assert "device-a.yaml" in names
assert ".hidden.yaml" not in names
def test_discover_user_yaml_files_bare_expression_not_expanded(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A fully dynamic filename never globs the whole directory."""
_write(tmp_path, "sibling.yaml", "api:\n")
with caplog.at_level("DEBUG", logger="esphome.yaml_util"):
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "${file}")
)
assert (tmp_path / "sibling.yaml").resolve() not in discovered.files
assert any(
"Cannot resolve !include" in r.message and r.levelname == "DEBUG"
for r in caplog.records
)
def test_discover_user_yaml_files_self_glob_match_skipped(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A glob whose only match is the including file itself claims nothing."""
entry = _write_entry_including(tmp_path, "${platform}.yaml")
with caplog.at_level("DEBUG", logger="esphome.yaml_util"):
discovered = discover_user_yaml_files(entry)
assert [p.name for p in discovered.files] == ["entry.yaml"]
assert any("Cannot resolve !include" in r.message for r in caplog.records)
def test_discover_user_yaml_files_candidate_cycle_terminates(tmp_path: Path) -> None:
"""Mutually glob-matching includes expand finitely and capture both files."""
_write(tmp_path, "sub/a.yaml", "p: !include ${x}.yaml\n")
_write(tmp_path, "sub/b.yaml", "p: !include ${y}.yaml\n")
entry = _write(tmp_path, "entry.yaml", "wifi: !include sub/a.yaml\n")
discovered = discover_user_yaml_files(entry)
names = {p.name for p in discovered.files}
assert names == {"entry.yaml", "a.yaml", "b.yaml"}
def test_discover_user_yaml_files_many_candidates_keep_nested_includes(
tmp_path: Path,
) -> None:
"""Every candidate's nested includes are discovered.
Regression test: the id()-based cycle guard is only safe while every
traversed tree stays alive. Candidate trees used to be freed between
loop iterations, so CPython recycled their addresses and later
candidates' fresh trees were skipped as already seen, silently dropping
their nested includes. Needs several candidates to manifest; two were
not enough to trigger the reuse."""
count = 12
for i in range(count):
_write(
tmp_path, f"keys/k{i}.yaml", f"sensor{i}: !include ../nested/n{i}.yaml\n"
)
_write(tmp_path, f"nested/n{i}.yaml", f"api{i}: true\n")
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "keys/${x}.yaml")
)
names = {p.name for p in discovered.files}
expected = {f"n{i}.yaml" for i in range(count)}
expected |= {f"k{i}.yaml" for i in range(count)}
expected.add("entry.yaml")
assert names == expected
def test_discover_user_yaml_files_bad_candidate_still_tracked(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A matched candidate that fails to parse warns even during discovery,
stays tracked (the load listener fires before parsing), and doesn't block
other candidates."""
_write(tmp_path, "keys/good.yaml", "api:\n")
_write(tmp_path, "keys/bad.yaml", "esphome: [unterminated\n")
with caplog.at_level("DEBUG", logger="esphome.yaml_util"):
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "keys/${name}.yaml")
)
resolved = set(discovered.files)
assert (tmp_path / "keys/good.yaml").resolve() in resolved
assert (tmp_path / "keys/bad.yaml").resolve() in resolved
matching = [
r.levelname for r in caplog.records if "Failed to load candidate" in r.message
]
assert matching == ["WARNING"]
def test_discover_user_yaml_files_tolerates_templated_top_level_include(
tmp_path: Path,
) -> None:
"""A literal include whose entire content is a templated ``!include`` is
tracked and skipped instead of aborting discovery."""
_write(tmp_path, "wrapper.yaml", "!include ${x}_settings.yaml\n")
discovered = discover_user_yaml_files(
_write_entry_including(tmp_path, "wrapper.yaml")
)
assert (tmp_path / "wrapper.yaml").resolve() in discovered.files
def test_track_yaml_loads_records_resolved_paths(tmp_path: Path) -> None:
"""`track_yaml_loads` is the building block — sanity-check it resolves
symlinks so callers can dedupe by identity."""