[substitutions] [packages] Fix substitutions: !include file.yaml regression

Resolve a deferred IncludeFile before validating the substitutions shape in
do_substitution_pass, and before wrapping it in UserDict in do_packages_pass.

Fixes esphome/esphome#15848

esphome/components/packages/__init__.py

@@ -516,7 +516,14 @@ def do_packages_pass(
     if CONF_PACKAGES not in config:
         return config
 
-    substitutions = UserDict(config.pop(CONF_SUBSTITUTIONS, {}))
+    raw_substitutions = config.pop(CONF_SUBSTITUTIONS, {})
+    if isinstance(raw_substitutions, yaml_util.IncludeFile):
+        # Resolve `substitutions: !include file.yaml` before feeding into UserDict.
+        with cv.prepend_path(CONF_SUBSTITUTIONS):
+            raw_substitutions, _ = resolve_include(
+                raw_substitutions, [], ContextVars(), strict_undefined=False
+            )
+    substitutions = UserDict(raw_substitutions)
     processor = _PackageProcessor(
         substitutions, command_line_substitutions, skip_update
     )

esphome/components/substitutions/__init__.py

@@ -429,6 +429,11 @@ def do_substitution_pass(
     # Use merge_dicts_ordered to preserve OrderedDict type for move_to_end()
     substitutions = config.pop(CONF_SUBSTITUTIONS, {})
     with cv.prepend_path(CONF_SUBSTITUTIONS):
+        if isinstance(substitutions, IncludeFile):
+            # Resolve `substitutions: !include file.yaml` before validating the shape.
+            substitutions, _ = resolve_include(
+                substitutions, [], ContextVars(), strict_undefined=False
+            )
         if not isinstance(substitutions, dict):
             raise cv.Invalid(
                 f"Substitutions must be a key to value mapping, got {type(substitutions)}"

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml

@@ -0,0 +1,5 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+wifi:
+  ssid: main_ssid
+  password: $wifi_password

tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml

@@ -0,0 +1 @@
+wifi_password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml

@@ -0,0 +1,9 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+packages:
+  wifi_pkg:
+    wifi:
+      password: $wifi_password
+
+wifi:
+  ssid: main_ssid