Merge branch 'dev' into app-loop-optimize-speed

tests/unit_tests/fixtures/bundle/bundle_test.yaml

@@ -11,9 +11,9 @@ esp32:
 logger:
   <<: !include common/base.yaml
 
-wifi:
-  ssid: !secret wifi_ssid
-  password: !secret wifi_password
+# Plain nested !include — deferred as an IncludeFile until the substitution
+# pass. The bundle must force-resolve it to pick up common/wifi.yaml.
+wifi: !include common/wifi.yaml
 
 api:
 

tests/unit_tests/fixtures/bundle/common/wifi.yaml

@@ -0,0 +1,2 @@
+ssid: !secret wifi_ssid
+password: !secret wifi_password

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml

@@ -0,0 +1,5 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+wifi:
+  ssid: main_ssid
+  password: $wifi_password

tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml

@@ -0,0 +1 @@
+wifi_password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml

@@ -0,0 +1,9 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+packages:
+  wifi_pkg:
+    wifi:
+      password: $wifi_password
+
+wifi:
+  ssid: main_ssid

tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml

@@ -0,0 +1,6 @@
+substitutions:
+  subs_file: 15-substitutions_inc
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml

@@ -0,0 +1,8 @@
+command_line_substitutions:
+  subs_file: 15-substitutions_inc
+
+substitutions: !include ${subs_file}.yaml
+
+wifi:
+  ssid: main_ssid
+  password: $wifi_password

tests/unit_tests/test_bundle.py

@@ -5,8 +5,10 @@ from __future__ import annotations
 import io
 import json
 from pathlib import Path
+import shutil
 import tarfile
 from typing import Any
+from unittest.mock import patch
 
 import pytest
 
@@ -20,6 +22,7 @@ from esphome.bundle import (
     _add_bytes_to_tar,
     _default_target_dir,
     _find_used_secret_keys,
+    _force_load_include_files,
     extract_bundle,
     is_bundle_path,
     prepare_bundle_for_compile,
@@ -485,7 +488,7 @@ def test_read_bundle_manifest_minimal(tmp_path: Path) -> None:
 
     result = read_bundle_manifest(bundle_path)
     assert result.esphome_version == "unknown"
-    assert result.files == []
+    assert not result.files
     assert result.has_secrets is False
 
 
@@ -862,6 +865,117 @@ def test_discover_files_skips_missing_directory(tmp_path: Path) -> None:
     assert len(files) == 1
 
 
+def test_discover_files_nested_include(tmp_path: Path) -> None:
+    """Nested !include files (e.g. wifi: !include wifi.yaml) are bundled."""
+    config_dir = _setup_config_dir(tmp_path)
+    (config_dir / "test.yaml").write_text(
+        "esphome:\n  name: test\nwifi: !include wifi.yaml\n"
+    )
+    (config_dir / "wifi.yaml").write_text('ssid: "a"\npassword: "b"\n')
+
+    creator = ConfigBundleCreator({})
+    files = creator.discover_files()
+
+    paths = [f.path for f in files]
+    assert "test.yaml" in paths
+    assert "wifi.yaml" in paths
+
+
+def test_discover_files_deeply_nested_include(tmp_path: Path) -> None:
+    """Chains of !include (a includes b includes c) are fully resolved."""
+    config_dir = _setup_config_dir(tmp_path)
+    (config_dir / "test.yaml").write_text(
+        "esphome:\n  name: test\nwifi: !include level1.yaml\n"
+    )
+    (config_dir / "level1.yaml").write_text("nested: !include level2.yaml\n")
+    (config_dir / "level2.yaml").write_text('value: "leaf"\n')
+
+    creator = ConfigBundleCreator({})
+    files = creator.discover_files()
+
+    paths = [f.path for f in files]
+    assert "level1.yaml" in paths
+    assert "level2.yaml" in paths
+
+
+def test_discover_files_nested_include_unresolved_substitution(
+    tmp_path: Path,
+) -> None:
+    """!include with substitution vars in path cannot be resolved; skipped gracefully."""
+    config_dir = _setup_config_dir(tmp_path)
+    (config_dir / "test.yaml").write_text(
+        "esphome:\n  name: test\nwifi: !include ${platform}.yaml\n"
+    )
+
+    creator = ConfigBundleCreator({})
+    # Should not raise
+    files = creator.discover_files()
+
+    paths = [f.path for f in files]
+    assert "test.yaml" in paths
+
+
+def test_discover_files_nested_include_load_failure(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """A nested !include pointing at a missing file is logged and skipped."""
+    config_dir = _setup_config_dir(tmp_path)
+    (config_dir / "test.yaml").write_text(
+        "esphome:\n  name: test\nwifi: !include missing.yaml\n"
+    )
+
+    creator = ConfigBundleCreator({})
+    files = creator.discover_files()
+
+    paths = [f.path for f in files]
+    assert "test.yaml" in paths
+    assert any(
+        "failed to load !include" in r.message and "missing.yaml" in r.message
+        for r in caplog.records
+    )
+
+
+def test_force_load_skips_duplicate_include_file() -> None:
+    """The same IncludeFile referenced twice is only loaded once."""
+
+    class _StubInclude:
+        """Mimics yaml_util.IncludeFile minimally for _force_load testing."""
+
+        def __init__(self) -> None:
+            self.file = Path("dup.yaml")
+            self.parent_file = Path("root.yaml")
+            self.load_calls = 0
+
+        def has_unresolved_expressions(self) -> bool:
+            return False
+
+        def load(self) -> dict[str, Any]:
+            self.load_calls += 1
+            return {}
+
+    stub = _StubInclude()
+    # Same instance appears twice — second visit must hit the _seen guard.
+    tree = {"a": stub, "b": [stub]}
+
+    with patch("esphome.bundle.yaml_util.IncludeFile", _StubInclude):
+        _force_load_include_files(tree)
+
+    assert stub.load_calls == 1
+
+
+def test_force_load_handles_cyclic_containers() -> None:
+    """Cyclic dict/list references don't cause infinite recursion."""
+    cyclic_dict: dict[str, Any] = {}
+    cyclic_dict["self"] = cyclic_dict
+
+    cyclic_list: list[Any] = []
+    cyclic_list.append(cyclic_list)
+
+    # Should return without recursing forever
+    _force_load_include_files(cyclic_dict)
+    _force_load_include_files(cyclic_list)
+
+
 def test_discover_files_yaml_reload_failure(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
@@ -1008,6 +1122,40 @@ def test_discover_files_walk_tuple_values(tmp_path: Path) -> None:
     assert "a.pem" in paths
 
 
+# ---------------------------------------------------------------------------
+# ConfigBundleCreator - fixture-based end-to-end
+# ---------------------------------------------------------------------------
+
+
+def test_discover_files_fixture_config(fixture_path: Path, tmp_path: Path) -> None:
+    """Use the real ``fixtures/bundle/`` tree as an end-to-end reproducer.
+
+    The fixture config uses ``wifi: !include common/wifi.yaml`` — a plain
+    nested !include that is returned as a deferred ``IncludeFile`` and only
+    resolved during the substitution pass. Before this fix, bundle discovery
+    never ran substitutions, so ``common/wifi.yaml`` was silently missing
+    from the bundle.
+    """
+    # Copy the fixture tree into a tmp dir so the test doesn't rely on the
+    # source repo being writable and so we can set CORE.config_path freely.
+    src = fixture_path / "bundle"
+    dst = tmp_path / "bundle"
+    shutil.copytree(src, dst)
+
+    CORE.config_path = dst / "bundle_test.yaml"
+
+    creator = ConfigBundleCreator({})
+    files = creator.discover_files()
+    paths = {f.path for f in files}
+
+    # Root and top-level !secret-referenced files
+    assert "bundle_test.yaml" in paths
+    assert "secrets.yaml" in paths
+    # The nested !include — this is what regressed when IncludeFile became
+    # deferred (PR #12213).
+    assert "common/wifi.yaml" in paths
+
+
 # ---------------------------------------------------------------------------
 # ConfigBundleCreator - create_bundle
 # ---------------------------------------------------------------------------

tests/unit_tests/test_config_validation.py

@@ -24,6 +24,7 @@ from esphome.const import (
     PLATFORM_LN882X,
     PLATFORM_RP2040,
     PLATFORM_RTL87XX,
+    SCHEDULER_DONT_RUN,
 )
 from esphome.core import CORE, HexInt, Lambda
 
@@ -765,3 +766,30 @@ def test_percentage_validators__raw_number_above_one_without_percent_sign(
         config_validation.unbounded_percentage(value)
     with pytest.raises(Invalid, match="percent sign"):
         config_validation.unbounded_possibly_negative_percentage(value)
+
+
+def test_update_interval__coerces_zero_to_one_ms(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """update_interval: 0ms must be coerced to 1ms (not rejected) because a
+    literal 0ms schedule causes Scheduler::call() to spin. Coercion keeps
+    existing configs compiling on upgrade while emitting a user-facing
+    warning that directs them to set a non-zero value."""
+    with caplog.at_level("WARNING"):
+        result = config_validation.update_interval("0ms")
+    assert result.total_milliseconds == 1
+    assert "update_interval of 0ms is not supported" in caplog.text
+    assert "1ms" in caplog.text
+
+
+def test_update_interval__preserves_nonzero_values() -> None:
+    """Non-zero update_interval values must pass through unchanged."""
+    assert config_validation.update_interval("1ms").total_milliseconds == 1
+    assert config_validation.update_interval("50ms").total_milliseconds == 50
+    assert config_validation.update_interval("60s").total_milliseconds == 60000
+
+
+def test_update_interval__never_passes_through() -> None:
+    """update_interval: never must still map to SCHEDULER_DONT_RUN."""
+    result = config_validation.update_interval("never")
+    assert result.total_milliseconds == SCHEDULER_DONT_RUN

tests/unit_tests/test_substitutions.py

@@ -14,6 +14,7 @@ from esphome.components.packages import (
     do_packages_pass,
     merge_packages,
 )
+from esphome.components.substitutions.jinja import UndefinedError
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, merge_config
 import esphome.config_validation as cv
@@ -675,6 +676,90 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None:
         substitutions.do_substitution_pass(config)
 
 
+def test_raise_first_undefined_logs_extras_at_debug(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Only the first undefined error is raised; extras are logged at debug."""
+    errors: substitutions.ErrList = [
+        (UndefinedError("'a' is undefined"), ["url"], None),
+        (UndefinedError("'b' is undefined"), ["ref"], None),
+        (UndefinedError("'c' is undefined"), ["path"], None),
+    ]
+
+    with (
+        caplog.at_level(logging.DEBUG, logger="esphome.components.substitutions"),
+        pytest.raises(cv.Invalid) as exc_info,
+    ):
+        substitutions.raise_first_undefined(errors, None, "package definition")
+
+    # First error is surfaced as the cv.Invalid message.
+    raised = str(exc_info.value)
+    assert "'a' is undefined" in raised
+    assert "'b' is undefined" not in raised
+    assert "'c' is undefined" not in raised
+
+    # Remaining errors are captured via debug logging for troubleshooting.
+    assert "Additional undefined variables in package definition" in caplog.text
+    assert "'b' is undefined at 'ref'" in caplog.text
+    assert "'c' is undefined at 'path'" in caplog.text
+
+
+def test_raise_first_undefined_noop_on_empty() -> None:
+    """An empty errors list is a no-op — no exception, no log."""
+    substitutions.raise_first_undefined([], None, "package definition")
+
+
+def test_do_substitution_pass_included_substitutions_must_be_mapping(
+    tmp_path: Path,
+) -> None:
+    """`substitutions: !include list.yaml` where the file holds a list raises cv.Invalid.
+
+    Locks in the shape check that runs after the deferred IncludeFile has been
+    resolved.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        return ["not", "a", "mapping"]
+
+    include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
+    config = OrderedDict({CONF_SUBSTITUTIONS: include})
+
+    with pytest.raises(
+        cv.Invalid, match="Substitutions must be a key to value mapping"
+    ):
+        substitutions.do_substitution_pass(config)
+
+
+def test_do_packages_pass_included_substitutions_must_be_mapping(
+    tmp_path: Path,
+) -> None:
+    """`substitutions: !include list.yaml` alongside `packages:` raises cv.Invalid.
+
+    Without the shape check, ``UserDict(...)`` would surface a low-level
+    ``TypeError``; the explicit ``cv.Invalid`` points at the substitutions path.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        return ["not", "a", "mapping"]
+
+    include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: include,
+            "packages": {"noop": {"wifi": {"ssid": "main"}}},
+        }
+    )
+
+    with pytest.raises(
+        cv.Invalid, match="Substitutions must be a key to value mapping"
+    ):
+        do_packages_pass(config)
+
+
 def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> None:
     """An undefined substitution in a package include filename raises cv.Invalid.