Merge branch 'dev' into decouple_scheduler_loop_cadence

tests/component_tests/packages/test_packages.py

@@ -2,18 +2,20 @@
 
 import logging
 from pathlib import Path
+import re
 from unittest.mock import MagicMock, patch
 
 import pytest
 
 from esphome.components.packages import (
     CONFIG_SCHEMA,
+    _substitute_package_definition,
     _walk_packages,
     do_packages_pass,
     is_package_definition,
     merge_packages,
 )
-from esphome.components.substitutions import do_substitution_pass
+from esphome.components.substitutions import ContextVars, do_substitution_pass
 import esphome.config as config_module
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, Remove
@@ -44,7 +46,7 @@ from esphome.const import (
 )
 from esphome.core import CORE
 from esphome.util import OrderedDict
-from esphome.yaml_util import IncludeFile, add_context
+from esphome.yaml_util import IncludeFile, add_context, load_yaml
 
 # Test strings
 TEST_DEVICE_NAME = "test_device_name"
@@ -1399,3 +1401,85 @@ def test_raw_config_contains_merged_esphome_from_package(tmp_path) -> None:
         "CORE.raw_config should contain esphome section after package merge"
     )
     assert CORE.raw_config[CONF_ESPHOME][CONF_NAME] == TEST_DEVICE_NAME
+
+
+# ---------------------------------------------------------------------------
+# _substitute_package_definition
+# ---------------------------------------------------------------------------
+
+
+def test_substitute_package_definition_local_dict_returned_unchanged() -> None:
+    """A plain local config dict is not substituted and is returned as-is."""
+    pkg = {CONF_WIFI: {CONF_SSID: "test"}}
+    result = _substitute_package_definition(pkg, ContextVars())
+    assert result is pkg
+
+
+def test_substitute_package_definition_string_resolved_with_context() -> None:
+    """A string package definition has its variables substituted."""
+    ctx = ContextVars({"variant": "esp32"})
+    result = _substitute_package_definition("device-${variant}.yaml", ctx)
+    assert result == "device-esp32.yaml"
+
+
+def test_substitute_package_definition_undefined_in_string() -> None:
+    """An undefined variable in a package URL string raises cv.Invalid."""
+    with pytest.raises(cv.Invalid, match="Undefined variable in package definition"):
+        _substitute_package_definition(
+            "github://org/repo/${undefined_var}/pkg.yaml", ContextVars()
+        )
+
+
+def test_substitute_package_definition_undefined_in_remote_dict_field() -> None:
+    """An undefined variable inside a remote-dict field names the offending field."""
+    with pytest.raises(cv.Invalid) as exc_info:
+        _substitute_package_definition(
+            {CONF_URL: "github://${typo}/repo"}, ContextVars()
+        )
+    err = str(exc_info.value)
+    assert "'typo' is undefined" in err
+    assert CONF_URL in err
+
+
+def test_substitute_package_definition_undefined_in_remote_dict_non_first_field() -> (
+    None
+):
+    """The field path joins correctly for non-first dict fields (e.g. ``ref``)."""
+    with pytest.raises(cv.Invalid) as exc_info:
+        _substitute_package_definition(
+            {
+                CONF_URL: "github://org/repo",
+                CONF_REF: "branch-${branch_typo}",
+            },
+            ContextVars(),
+        )
+    err = str(exc_info.value)
+    assert "'branch_typo' is undefined" in err
+    assert CONF_REF in err
+
+
+def test_substitute_package_definition_includes_source_location(tmp_path: Path) -> None:
+    """A package loaded from YAML surfaces file/line/col in the cv.Invalid message.
+
+    Line/column are rendered 1-based (matching config.line_info() and editor
+    line numbering) and point at the offending scalar, not the enclosing dict.
+    """
+    yaml_file = tmp_path / "main.yaml"
+    yaml_file.write_text(
+        "packages:\n  broken: github://org/repo/${undefined_var}/pkg.yaml\n"
+    )
+    config = load_yaml(yaml_file)
+    package_config = config[CONF_PACKAGES]["broken"]
+
+    with pytest.raises(cv.Invalid) as exc_info:
+        _substitute_package_definition(package_config, ContextVars())
+
+    err = str(exc_info.value)
+    assert "main.yaml" in err
+    # The offending value lives on line 2 (1-based). Column depends on the YAML
+    # loader, so we only pin line and check that a 1-based column is present.
+    match = re.search(r"main\.yaml (\d+):(\d+)", err)
+    assert match, err
+    line, col = int(match.group(1)), int(match.group(2))
+    assert line == 2, f"expected 1-based line 2, got {line} (err={err!r})"
+    assert col >= 1, f"expected 1-based column ≥ 1, got {col} (err={err!r})"

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml

@@ -0,0 +1,5 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+wifi:
+  ssid: main_ssid
+  password: $wifi_password

tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml

@@ -0,0 +1 @@
+wifi_password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml

@@ -0,0 +1,9 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+packages:
+  wifi_pkg:
+    wifi:
+      password: $wifi_password
+
+wifi:
+  ssid: main_ssid

tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml

@@ -0,0 +1,6 @@
+substitutions:
+  subs_file: 15-substitutions_inc
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml

@@ -0,0 +1,8 @@
+command_line_substitutions:
+  subs_file: 15-substitutions_inc
+
+substitutions: !include ${subs_file}.yaml
+
+wifi:
+  ssid: main_ssid
+  password: $wifi_password

tests/unit_tests/test_substitutions.py

@@ -14,6 +14,7 @@ from esphome.components.packages import (
     do_packages_pass,
     merge_packages,
 )
+from esphome.components.substitutions.jinja import UndefinedError
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, merge_config
 import esphome.config_validation as cv
@@ -675,6 +676,90 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None:
         substitutions.do_substitution_pass(config)
 
 
+def test_raise_first_undefined_logs_extras_at_debug(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Only the first undefined error is raised; extras are logged at debug."""
+    errors: substitutions.ErrList = [
+        (UndefinedError("'a' is undefined"), ["url"], None),
+        (UndefinedError("'b' is undefined"), ["ref"], None),
+        (UndefinedError("'c' is undefined"), ["path"], None),
+    ]
+
+    with (
+        caplog.at_level(logging.DEBUG, logger="esphome.components.substitutions"),
+        pytest.raises(cv.Invalid) as exc_info,
+    ):
+        substitutions.raise_first_undefined(errors, None, "package definition")
+
+    # First error is surfaced as the cv.Invalid message.
+    raised = str(exc_info.value)
+    assert "'a' is undefined" in raised
+    assert "'b' is undefined" not in raised
+    assert "'c' is undefined" not in raised
+
+    # Remaining errors are captured via debug logging for troubleshooting.
+    assert "Additional undefined variables in package definition" in caplog.text
+    assert "'b' is undefined at 'ref'" in caplog.text
+    assert "'c' is undefined at 'path'" in caplog.text
+
+
+def test_raise_first_undefined_noop_on_empty() -> None:
+    """An empty errors list is a no-op — no exception, no log."""
+    substitutions.raise_first_undefined([], None, "package definition")
+
+
+def test_do_substitution_pass_included_substitutions_must_be_mapping(
+    tmp_path: Path,
+) -> None:
+    """`substitutions: !include list.yaml` where the file holds a list raises cv.Invalid.
+
+    Locks in the shape check that runs after the deferred IncludeFile has been
+    resolved.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        return ["not", "a", "mapping"]
+
+    include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
+    config = OrderedDict({CONF_SUBSTITUTIONS: include})
+
+    with pytest.raises(
+        cv.Invalid, match="Substitutions must be a key to value mapping"
+    ):
+        substitutions.do_substitution_pass(config)
+
+
+def test_do_packages_pass_included_substitutions_must_be_mapping(
+    tmp_path: Path,
+) -> None:
+    """`substitutions: !include list.yaml` alongside `packages:` raises cv.Invalid.
+
+    Without the shape check, ``UserDict(...)`` would surface a low-level
+    ``TypeError``; the explicit ``cv.Invalid`` points at the substitutions path.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        return ["not", "a", "mapping"]
+
+    include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: include,
+            "packages": {"noop": {"wifi": {"ssid": "main"}}},
+        }
+    )
+
+    with pytest.raises(
+        cv.Invalid, match="Substitutions must be a key to value mapping"
+    ):
+        do_packages_pass(config)
+
+
 def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> None:
     """An undefined substitution in a package include filename raises cv.Invalid.