[ci] Add validate.*.yaml for config-only component tests (#16384)

script/test_build_components.py

@@ -39,7 +39,11 @@ from script.analyze_component_buses import (
     merge_compatible_bus_groups,
     uses_local_file_references,
 )
-from script.helpers import get_component_test_files, split_conflicting_groups
+from script.helpers import (
+    get_component_test_files,
+    is_validate_only_file,
+    split_conflicting_groups,
+)
 from script.merge_component_configs import merge_component_configs
 
 
@@ -83,7 +87,10 @@ def show_disk_space_if_ci(esphome_command: str) -> None:
 
 
 def find_component_tests(
-    components_dir: Path, component_pattern: str = "*", base_only: bool = False
+    components_dir: Path,
+    component_pattern: str = "*",
+    base_only: bool = False,
+    include_validate: bool = False,
 ) -> dict[str, list[Path]]:
     """Find all component test files.
 
@@ -91,6 +98,8 @@ def find_component_tests(
         components_dir: Path to tests/components directory
         component_pattern: Glob pattern for component names
         base_only: If True, only find base test files (test.*.yaml), not variant files (test-*.yaml)
+        include_validate: If True, also include config-only files (validate.*.yaml).
+            These are run with `esphome config` only and never compiled.
 
     Returns:
         Dictionary mapping component name to list of test files
@@ -102,7 +111,11 @@ def find_component_tests(
             continue
 
         # Get test files using helper function
-        test_files = get_component_test_files(comp_dir.name, all_variants=not base_only)
+        test_files = get_component_test_files(
+            comp_dir.name,
+            all_variants=not base_only,
+            include_validate=include_validate,
+        )
         if test_files:
             component_tests[comp_dir.name] = test_files
 
@@ -836,12 +849,25 @@ def run_grouped_component_tests(
     # With grouping:
     # - 1 build per group (regardless of how many components)
     # - Individual components still need all their platform builds
+    # - Validate files of grouped components still run individually
+    #   (they're config-only and bypass the grouped compile, see
+    #   run_individual_component_test), so each adds one more invocation.
     individual_test_file_count = sum(
         len(all_tests[comp]) for comp in individual_tests if comp in all_tests
     )
 
+    grouped_component_set = {c for _, _, comps in groups_to_test for c in comps}
+    grouped_validate_file_count = sum(
+        1
+        for comp in grouped_component_set
+        for test_file in all_tests.get(comp, [])
+        if is_validate_only_file(test_file)
+    )
+
     total_grouped_components = sum(len(comps) for _, _, comps in groups_to_test)
-    total_builds_with_grouping = len(groups_to_test) + individual_test_file_count
+    total_builds_with_grouping = (
+        len(groups_to_test) + individual_test_file_count + grouped_validate_file_count
+    )
     builds_saved = total_test_files - total_builds_with_grouping
 
     print(f"\n{'=' * 80}")
@@ -854,6 +880,10 @@ def run_grouped_component_tests(
     print(
         f"  • {individual_test_file_count} individual builds ({len(individual_tests)} components)"
     )
+    if grouped_validate_file_count:
+        print(
+            f"  • {grouped_validate_file_count} validate-only invocations for grouped components"
+        )
     if total_test_files > 0:
         reduction_pct = (builds_saved / total_test_files) * 100
         print(f"  • Saves {builds_saved} builds ({reduction_pct:.1f}% reduction)")
@@ -937,8 +967,13 @@ def run_individual_component_test(
         tested_components: Set of already tested components
         test_results: List to append test results
     """
-    # Skip if already tested in a group
-    if (component, platform_with_version) in tested_components:
+    # Validate files (validate.*.yaml) are config-only and never participate
+    # in compile-time bus grouping, so always run them individually even when
+    # the (component, platform) pair was covered by a group test.
+    if (
+        not is_validate_only_file(test_file)
+        and (component, platform_with_version) in tested_components
+    ):
         return
 
     test_result = run_esphome_test(
@@ -992,13 +1027,23 @@ def test_components(
     # Get platform base files
     platform_bases = get_platform_base_files(build_components_dir)
 
+    # Validate files (validate.*.yaml) are config-only -- they exercise
+    # schema/validation paths but are never compiled. Include them when running
+    # `config` or `clean`; exclude them under `compile` so they never reach a
+    # toolchain build.
+    include_validate = esphome_command != "compile"
+
     # Find all component tests
     all_tests = {}
     for pattern in component_patterns:
         # Skip empty patterns (happens when components list is empty string)
         if not pattern:
             continue
-        all_tests.update(find_component_tests(tests_dir, pattern, base_only))
+        all_tests.update(
+            find_component_tests(
+                tests_dir, pattern, base_only, include_validate=include_validate
+            )
+        )
 
     # If no components found, build a reference configuration for baseline comparison
     # Create a synthetic "empty" component test that will build just the base config