Merge remote-tracking branch 'upstream/templatable-value-specialize' into integration

tests/components/emontx/test.esp32-idf.yaml

@@ -1,4 +1,4 @@
 packages:
-  uart: !include ../../test_build_components/common/uart_115200/esp32-idf.yaml
+  uart_115200: !include ../../test_build_components/common/uart_115200/esp32-idf.yaml
 
 <<: !include common.yaml

tests/components/emontx/test.esp8266-ard.yaml

@@ -1,4 +1,4 @@
 packages:
-  uart: !include ../../test_build_components/common/uart_115200/esp8266-ard.yaml
+  uart_115200: !include ../../test_build_components/common/uart_115200/esp8266-ard.yaml
 
 <<: !include common.yaml

tests/components/emontx/test.rp2040-ard.yaml

@@ -1,4 +1,4 @@
 packages:
-  uart: !include ../../test_build_components/common/uart_115200/rp2040-ard.yaml
+  uart_115200: !include ../../test_build_components/common/uart_115200/rp2040-ard.yaml
 
 <<: !include common.yaml

tests/components/epaper_spi/test.esp32-s3-idf.yaml

@@ -78,6 +78,7 @@ display:
     model: seeed-reterminal-e1002
   - platform: epaper_spi
     model: seeed-ee04-mono-4.26
+    full_update_every: 10
     # Override pins to avoid conflict with other display configs
     busy_pin: 43
     dc_pin: 42

tests/components/ethernet/common-w6100-rp2040.yaml

@@ -0,0 +1,18 @@
+ethernet:
+  type: W6100
+  clk_pin: 18
+  mosi_pin: 19
+  miso_pin: 16
+  cs_pin: 17
+  interrupt_pin: 21
+  reset_pin: 20
+  manual_ip:
+    static_ip: 192.168.178.56
+    gateway: 192.168.178.1
+    subnet: 255.255.255.0
+  domain: .local
+  mac_address: "02:AA:BB:CC:DD:01"
+  on_connect:
+    - logger.log: "Ethernet connected!"
+  on_disconnect:
+    - logger.log: "Ethernet disconnected!"

tests/components/ethernet/common-w6300-rp2040.yaml

@@ -0,0 +1,18 @@
+ethernet:
+  type: W6300
+  clk_pin: 18
+  mosi_pin: 19
+  miso_pin: 16
+  cs_pin: 17
+  interrupt_pin: 21
+  reset_pin: 20
+  manual_ip:
+    static_ip: 192.168.178.56
+    gateway: 192.168.178.1
+    subnet: 255.255.255.0
+  domain: .local
+  mac_address: "02:AA:BB:CC:DD:01"
+  on_connect:
+    - logger.log: "Ethernet connected!"
+  on_disconnect:
+    - logger.log: "Ethernet disconnected!"

tests/components/ethernet/test-w6100.rp2040-ard.yaml

@@ -0,0 +1 @@
+<<: !include common-w6100-rp2040.yaml

tests/components/ethernet/test-w6300.rp2040-ard.yaml

@@ -0,0 +1 @@
+<<: !include common-w6300-rp2040.yaml

tests/components/lvgl/test.esp32-idf.yaml

@@ -21,7 +21,7 @@ binary_sensor:
       ignore_strapping_warning: true
 
 display:
-  - platform: ili9xxx
+  - platform: mipi_spi
     spi_id: spi_bus
     model: st7789v
     id: second_display
@@ -41,7 +41,7 @@ display:
     invert_colors: false
     update_interval: never
 
-  - platform: ili9xxx
+  - platform: mipi_spi
     spi_id: spi_bus
     model: st7789v
     id: tft_display

tests/components/lvgl/test.esp32-p4-idf.yaml

@@ -0,0 +1,12 @@
+display:
+  - platform: mipi_dsi
+    model: WAVESHARE-ESP32-P4-WIFI6-TOUCH-LCD-3.4C
+lvgl:
+  byte_order: little_endian
+  rotation: 90
+
+psram:
+
+esp_ldo:
+  - channel: 3
+    voltage: 2.5V

tests/unit_tests/fixtures/substitutions/11-include_path.approved.yaml

@@ -0,0 +1,15 @@
+values:
+  - var1: 4
+  - a: 5
+  - b: 6
+  - c: The value of C is 7
+  - This value comes from inc2.yaml. x is 3, y is 4
+  - From main config, x is 3, y is 2
+  - $a $b $c are out of scope here
+  - keys_in_inc3:
+      x: 3
+      y: 2
+substitutions:
+  x: 3
+  y: 2
+  include_file: inc1

tests/unit_tests/fixtures/substitutions/11-include_path.input.yaml

@@ -0,0 +1,21 @@
+substitutions:
+  include_file: inc1
+  x: 3  # override x from inc2.yaml
+
+packages:
+  my_package: !include
+    file: ${include_file + ".yaml"}  # includes inc1.yaml
+    vars:
+      var1: 4
+      a: ${x+2}
+      b: ${a+1}
+      c: 7
+  other_package: !include
+    file: inc${1+1}.yaml  # includes inc2.yaml
+    vars:
+      y: 4
+
+values:
+  - From main config, x is $x, y is $y
+  - $a $b $c are out of scope here
+  - !include ${"inc" + "3.yaml"}  # includes inc3.yaml here (not a package)

tests/unit_tests/fixtures/substitutions/12-yaml-merge.approved.yaml

@@ -0,0 +1,9 @@
+substitutions:
+  x: 7
+test_list:
+  - content:
+      before: Content before
+      after: Content after
+      keys_in_inc3:
+        x: 7
+        y: 8

tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml

@@ -0,0 +1,10 @@
+substitutions:
+  x: 7
+test_list:
+  - content:
+      before: Content before
+      <<: !include
+        file: inc3.yaml
+        vars:
+          y: 8
+      after: Content after

tests/unit_tests/fixtures/substitutions/inc2.yaml

@@ -0,0 +1,6 @@
+substitutions:
+  x: 1
+  y: 2
+
+values:
+  - This value comes from inc2.yaml. x is $x, y is $y

tests/unit_tests/fixtures/substitutions/inc3.yaml

@@ -0,0 +1,3 @@
+keys_in_inc3:
+  x: ${x}
+  y: ${y}

tests/unit_tests/test_cpp_generator.py

@@ -684,6 +684,15 @@ async def test_templatable__with_to_exp_callable() -> None:
     assert result == 84
 
 
+@pytest.mark.asyncio
+async def test_templatable__with_to_exp_callable_and_output_type() -> None:
+    """When to_exp is provided with non-string output_type, result is lambda-wrapped."""
+    result = await cg.templatable(42, [], ct.int_, to_exp=lambda x: x * 2)
+
+    assert isinstance(result, cg.LambdaExpression)
+    assert result.capture == ""
+
+
 @pytest.mark.asyncio
 async def test_templatable__with_to_exp_dict() -> None:
     """When to_exp is a dict, value is looked up."""

tests/unit_tests/test_substitutions.py

@@ -8,12 +8,17 @@ import pytest
 
 from esphome import config as config_module, yaml_util
 from esphome.components import substitutions
-from esphome.components.packages import do_packages_pass, merge_packages
+from esphome.components.packages import (
+    MAX_INCLUDE_DEPTH,
+    _PackageProcessor,
+    do_packages_pass,
+    merge_packages,
+)
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, merge_config
 import esphome.config_validation as cv
 from esphome.const import CONF_SUBSTITUTIONS
-from esphome.core import CORE, Lambda
+from esphome.core import CORE, EsphomeError, Lambda
 from esphome.util import OrderedDict
 
 _LOGGER = logging.getLogger(__name__)
@@ -630,3 +635,62 @@ def test_do_substitution_pass_substitutions_must_be_mapping_from_config() -> Non
         cv.Invalid, match="Substitutions must be a key to value mapping"
     ):
         substitutions.do_substitution_pass(config)
+
+
+# ── IncludeFile / package loading tests ────────────────────────────────────
+
+
+def test_resolve_package_max_depth_exceeded(tmp_path: Path) -> None:
+    """A yaml_loader that always returns another IncludeFile triggers the depth guard."""
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    # Each call to the loader returns a fresh IncludeFile pointing at itself,
+    # so PACKAGE_SCHEMA always sees an IncludeFile and never a dict.
+    def always_returns_include(path: Path) -> yaml_util.IncludeFile:
+        return yaml_util.IncludeFile(parent, path.name, None, always_returns_include)
+
+    package_config = yaml_util.IncludeFile(
+        parent, "test.yaml", None, always_returns_include
+    )
+    processor = _PackageProcessor({}, None, False)
+    with pytest.raises(
+        cv.Invalid,
+        match=f"Maximum include nesting depth \\({MAX_INCLUDE_DEPTH}\\) exceeded",
+    ):
+        processor.resolve_package(package_config, substitutions.ContextVars())
+
+
+def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None:
+    """!include with an undefined substitution variable raises cv.Invalid.
+
+    The error message must reference the unresolved filename template so the
+    user knows which include failed, rather than seeing a bare file-not-found.
+    """
+    main_file = tmp_path / "main.yaml"
+    main_file.write_text("result: !include ${undefined_var}.yaml\n")
+
+    config = yaml_util.load_yaml(main_file)
+    with pytest.raises(cv.Invalid, match=r"\$\{undefined_var\}"):
+        substitutions.do_substitution_pass(config)
+
+
+def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> None:
+    """An undefined substitution in a package include filename raises cv.Invalid.
+
+    Previously this would raise an unhandled UndefinedError. With
+    strict_undefined=False, the unresolved filename passes through to
+    file loading which produces a clean cv.Invalid error.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        raise EsphomeError(f"Error reading file {path}: No such file")
+
+    package_config = yaml_util.IncludeFile(
+        parent, "${undefined_var}.yaml", None, loader
+    )
+    processor = _PackageProcessor({}, None, False)
+    with pytest.raises(cv.Invalid, match="unresolved substitutions"):
+        processor.resolve_package(package_config, substitutions.ContextVars())

tests/unit_tests/test_yaml_util.py

@@ -1,3 +1,4 @@
+import io
 from pathlib import Path
 import shutil
 from unittest.mock import patch
@@ -7,6 +8,7 @@ import pytest
 from esphome import core, yaml_util
 from esphome.components import substitutions
 from esphome.config_helpers import Extend, Remove
+import esphome.config_validation as cv
 from esphome.core import EsphomeError
 from esphome.util import OrderedDict
 
@@ -74,7 +76,9 @@ def test_parsing_with_custom_loader(fixture_path):
         loader_calls.append(fname)
 
     with yaml_file.open(encoding="utf-8") as f_handle:
-        yaml_util.parse_yaml(yaml_file, f_handle, custom_loader)
+        config = yaml_util.parse_yaml(yaml_file, f_handle, custom_loader)
+        # substitute config to expand includes:
+        substitutions.substitute(config, [], substitutions.ContextVars(), False)
 
     assert len(loader_calls) == 3
     assert loader_calls[0].parts[-2:] == ("includes", "included.yaml")
@@ -348,7 +352,9 @@ def test_track_yaml_loads_records_includes(tmp_path: Path) -> None:
     main.write_text("child: !include included.yaml\n")
 
     with yaml_util.track_yaml_loads() as loaded:
-        yaml_util.load_yaml(main)
+        result = yaml_util.load_yaml(main)
+        # !include is deferred; resolve it to trigger the nested load
+        result["child"].load()
 
     resolved = [p.name for p in loaded]
     assert "main.yaml" in resolved
@@ -500,3 +506,197 @@ def test_represent_extend() -> None:
 def test_represent_remove() -> None:
     """Test that Remove objects are dumped as plain !remove scalars."""
     assert yaml_util.dump({"key": Remove("my_id")}) == "key: !remove 'my_id'\n"
+
+
+def test_represent_include_file() -> None:
+    """Test that IncludeFile objects are dumped as !include scalars."""
+    include = yaml_util.IncludeFile(
+        Path("/fake/main.yaml"), "path/to/file.yaml", None, lambda _: {}
+    )
+    assert yaml_util.dump({"key": include}) == "key: !include 'path/to/file.yaml'\n"
+
+
+def test_represent_include_file_with_vars() -> None:
+    """Test that IncludeFile with vars is dumped as !include mapping form."""
+    include = yaml_util.IncludeFile(
+        Path("/fake/main.yaml"),
+        "path/to/file.yaml",
+        {"key": "value"},
+        lambda _: {},
+    )
+    result = yaml_util.dump({"key": include})
+    assert "!include" in result
+    assert "file: path/to/file.yaml" in result
+    assert "key: value" in result
+
+
+def test_represent_include_file_with_data_base_mixin() -> None:
+    """Test that IncludeFile wrapped with ESPHomeDataBase mixin is also dumped correctly.
+
+    The YAML loader wraps IncludeFile via add_class_to_obj, creating a dynamic
+    subclass. add_multi_representer must match this subclass through the MRO.
+    """
+    include = yaml_util.IncludeFile(
+        Path("/fake/main.yaml"), "common/spi.yaml", None, lambda _: {}
+    )
+    wrapped = yaml_util.make_data_base(include)
+    assert isinstance(wrapped, yaml_util.ESPHomeDataBase)
+    assert yaml_util.dump({"pkg": wrapped}) == "pkg: !include 'common/spi.yaml'\n"
+
+
+# ── IncludeFile unit tests ──────────────────────────────────────────────────
+
+
+def test_include_file_repr(tmp_path: Path) -> None:
+    """repr() includes the filename so it appears usefully in error messages."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, "some/nested.yaml", None, lambda _: {})
+    assert repr(include) == "IncludeFile(some/nested.yaml)"
+
+
+def test_include_file_load_caches_result(tmp_path: Path) -> None:
+    """load() invokes the yaml_loader only once; subsequent calls return the cached object."""
+    parent = tmp_path / "main.yaml"
+    content = {"key": "value"}
+    call_count = 0
+
+    def counting_loader(_):
+        nonlocal call_count
+        call_count += 1
+        return content
+
+    include = yaml_util.IncludeFile(parent, "child.yaml", None, counting_loader)
+    first = include.load()
+    second = include.load()
+
+    assert call_count == 1
+    assert first is second
+
+
+def test_include_file_load_caches_none_result(tmp_path: Path) -> None:
+    """load() caches None content (empty YAML files) and does not re-invoke the loader."""
+    parent = tmp_path / "main.yaml"
+    call_count = 0
+
+    def counting_loader(_):
+        nonlocal call_count
+        call_count += 1
+
+    include = yaml_util.IncludeFile(parent, "empty.yaml", None, counting_loader)
+    first = include.load()
+    second = include.load()
+
+    assert call_count == 1
+    assert first is None
+    assert second is None
+
+
+def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> None:
+    """load() raises if the filename contains unresolved substitutions or expressions."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", None, lambda _: {})
+    with pytest.raises(cv.Invalid, match="unresolved"):
+        include.load()
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected"),
+    [
+        ("device-${platform}.yaml", True),
+        ("$platform.yaml", True),
+        ("${a + b}.yaml", True),  # Jinja expression
+        ("device.yaml", False),
+        ("path/to/device.yaml", False),
+        ("my$file.yaml", True),  # $file is a valid substitution
+        ("price-100$.yaml", False),  # $ at end, not followed by valid substitution
+    ],
+)
+def test_include_file_has_unresolved_expressions(
+    tmp_path: Path, filename: str, expected: bool
+) -> None:
+    """has_unresolved_expressions() detects substitution patterns in the filename."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, filename, None, lambda _: {})
+    assert include.has_unresolved_expressions() == expected
+
+
+def test_include_in_list_context() -> None:
+    """!include of a file returning a list is handled correctly,
+    including when that list itself contains a nested IncludeFile."""
+    parent = Path("/fake/main.yaml")
+
+    # The nested IncludeFile resolves to a plain string value
+    inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: "gamma")
+
+    # The outer IncludeFile returns a list whose last element is itself an IncludeFile,
+    # exercising the substitution pass's ability to recurse into loaded content.
+    outer = yaml_util.IncludeFile(
+        parent, "items.yaml", None, lambda _: ["alpha", "beta", inner]
+    )
+
+    config = OrderedDict({"values": outer})
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["values"] == ["alpha", "beta", "gamma"]
+
+
+def test_include_plain_filename_loads_after_deferred_refactor() -> None:
+    """!include with a plain filename (no $ expressions) still loads correctly.
+
+    Regression guard: the deferred-loading refactor must not break the simple case.
+    """
+    parent = Path("/fake/main.yaml")
+    include = yaml_util.IncludeFile(
+        parent, "child.yaml", None, lambda _: {"answer": 42}
+    )
+
+    config = OrderedDict({"result": include})
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["result"]["answer"] == 42
+
+
+def test_yaml_merge_include_with_filename_substitution_raises() -> None:
+    """<<: !include ${expr} raises a clear error — substitutions in merge-key filenames
+    are not yet supported, and the error message must say so."""
+    yaml_text = "base:\n  existing: value\n  <<: !include ${filename}.yaml\n"
+    with pytest.raises(EsphomeError, match="not supported yet"):
+        yaml_util.parse_yaml(
+            Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
+        )
+
+
+def test_yaml_merge_list_include_with_filename_substitution_raises() -> None:
+    """Substitutions in include filenames within merge-key lists raise a clear error."""
+    yaml_text = "base:\n  existing: value\n  <<:\n    - !include ${filename}.yaml\n"
+    with pytest.raises(EsphomeError, match="not supported yet"):
+        yaml_util.parse_yaml(
+            Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
+        )
+
+
+def test_yaml_merge_chain_include_resolves() -> None:
+    """Chained includes in merge keys resolve through multiple IncludeFile layers."""
+    parent = Path("/fake/main.yaml")
+
+    inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: {"x": 1})
+    outer = yaml_util.IncludeFile(parent, "outer.yaml", None, lambda _: inner)
+
+    yaml_text = "base:\n  existing: value\n  <<: !include outer.yaml\n"
+    config = yaml_util.parse_yaml(parent, io.StringIO(yaml_text), lambda _: outer)
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["base"]["x"] == 1
+    assert config["base"]["existing"] == "value"
+
+
+def test_yaml_merge_chain_include_depth_exceeded() -> None:
+    """Chain includes in merge keys exceeding depth limit raise a clear error."""
+    parent = Path("/fake/main.yaml")
+
+    def self_referencing_loader(path: Path) -> yaml_util.IncludeFile:
+        return yaml_util.IncludeFile(parent, path.name, None, self_referencing_loader)
+
+    yaml_text = "base:\n  <<: !include loop.yaml\n"
+    with pytest.raises(EsphomeError, match="Maximum include chain depth"):
+        yaml_util.parse_yaml(parent, io.StringIO(yaml_text), self_referencing_loader)