[ci] Add validate.*.yaml for config-only component tests (#16384)

script/analyze_component_buses.py

@@ -34,7 +34,7 @@ from typing import Any
 # Add esphome to path
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
-from helpers import BASE_BUS_COMPONENTS
+from helpers import BASE_BUS_COMPONENTS, is_validate_only_file
 
 from esphome import yaml_util
 from esphome.config_helpers import Extend, Remove
@@ -283,6 +283,13 @@ def analyze_component(component_dir: Path) -> tuple[dict[str, list[str]], bool,
 
     # Analyze all YAML files in the component directory
     for yaml_file in component_dir.glob("*.yaml"):
+        # validate.*.yaml files are config-only -- they don't compile, so
+        # their contents must not influence compile-time grouping decisions
+        # (e.g. a !extend used only to exercise schema validation must not
+        # disqualify the whole component from being grouped).
+        if is_validate_only_file(yaml_file):
+            continue
+
         analysis = analyze_yaml_file(yaml_file)
 
         # Track if any file uses extend/remove

script/ci-custom.py

@@ -1068,7 +1068,12 @@ PACKAGE_BUS_RE = re.compile(
 )
 
 
-@lint_content_check(include=["tests/components/*/test.*.yaml"])
+@lint_content_check(
+    include=[
+        "tests/components/*/test.*.yaml",
+        "tests/components/*/validate.*.yaml",
+    ]
+)
 def lint_test_package_key_matches_bus(fname, content):
     """Ensure package keys match the common bus directory name.
 

script/determine-jobs.py

@@ -29,6 +29,8 @@ The CI workflow uses this information to:
 - Skip or run downstream esphome/device-builder tests against the PR's Python code
 - Determine which components to test individually
 - Decide how to split component tests (if there are many)
+- Identify directly-changed components whose only edits are validate.*.yaml files,
+  so CI can skip the compile stage for them and run config validation only
 - Run memory impact analysis whenever there are changed components (merged config), and also for core-only changes
 
 Usage:
@@ -68,6 +70,7 @@ from helpers import (
     get_integration_test_files_for_components,
     get_target_branch,
     git_ls_files,
+    is_validate_only_file,
     parse_test_filename,
     root_path,
 )
@@ -600,14 +603,41 @@ def _component_has_tests(component: str) -> bool:
     """Check if a component has test files.
 
     Cached to avoid repeated filesystem operations for the same component.
+    Validate files (validate.*.yaml) count -- they exercise schema validation
+    in CI even though they are never compiled.
 
     Args:
         component: Component name to check
 
     Returns:
-        True if the component has test YAML files
+        True if the component has test or validate YAML files
     """
-    return bool(get_component_test_files(component, all_variants=True))
+    return bool(
+        get_component_test_files(component, all_variants=True, include_validate=True)
+    )
+
+
+def _component_change_is_validate_only(component: str, changed: list[str]) -> bool:
+    """Return True if every changed file for this component is a validate.*.yaml.
+
+    Used to decide whether a directly-changed component can skip the compile
+    stage in CI. A component qualifies when:
+    - at least one file under ``tests/components/<component>/`` changed, AND
+    - no source file under ``esphome/components/<component>/`` changed, AND
+    - every changed test file is a ``validate.*.yaml`` or
+      ``validate-*.yaml`` (i.e. no regular ``test.*.yaml`` was touched).
+    """
+    test_prefix = f"tests/components/{component}/"
+    src_prefix = f"esphome/components/{component}/"
+    test_changes: list[Path] = []
+    for path in changed:
+        if path.startswith(src_prefix):
+            return False
+        if path.startswith(test_prefix):
+            test_changes.append(Path(path))
+    if not test_changes:
+        return False
+    return all(is_validate_only_file(p) for p in test_changes)
 
 
 def _select_platform_by_preference(
@@ -977,6 +1007,17 @@ def main() -> None:
         if component not in directly_changed_components
     ]
 
+    # Components whose only changes are validate.*.yaml files can skip the
+    # compile stage in CI -- their source and test fixtures didn't move, so
+    # rebuilding firmware adds no signal. Only directly-changed components
+    # qualify: a component pulled in transitively (because a dependency
+    # changed) still needs the compile to catch regressions.
+    validate_only_components = sorted(
+        component
+        for component in directly_changed_with_tests
+        if _component_change_is_validate_only(component, changed)
+    )
+
     # Detect components for memory impact analysis (merged config)
     memory_impact = detect_memory_impact_config(args.branch)
 
@@ -1073,6 +1114,7 @@ def main() -> None:
         "cpp_unit_tests_run_all": cpp_run_all,
         "cpp_unit_tests_components": cpp_components,
         "component_test_batches": component_test_batches,
+        "validate_only_components": validate_only_components,
         "benchmarks": run_benchmarks,
     }
 

script/helpers.py

@@ -117,7 +117,7 @@ def get_component_from_path(file_path: str) -> str | None:
 
 
 def get_component_test_files(
-    component: str, *, all_variants: bool = False
+    component: str, *, all_variants: bool = False, include_validate: bool = False
 ) -> list[Path]:
     """Get test files for a component.
 
@@ -126,6 +126,10 @@ def get_component_test_files(
         all_variants: If True, returns all test files including variants (test-*.yaml).
                      If False, returns only base test files (test.*.yaml).
                      Default is False.
+        include_validate: If True, also returns config-only files (validate.*.yaml,
+                     and validate-*.yaml when all_variants is True). These files
+                     are validated with `esphome config` but never compiled.
+                     Default is False.
 
     Returns:
         List of test file paths for the component, or empty list if none exist
@@ -136,9 +140,27 @@ def get_component_test_files(
 
     if all_variants:
         # Match both test.*.yaml and test-*.yaml patterns
-        return list(tests_dir.glob("test[.-]*.yaml"))
+        files = list(tests_dir.glob("test[.-]*.yaml"))
+        if include_validate:
+            files.extend(tests_dir.glob("validate[.-]*.yaml"))
+        return files
     # Match only test.*.yaml (base tests)
-    return list(tests_dir.glob("test.*.yaml"))
+    files = list(tests_dir.glob("test.*.yaml"))
+    if include_validate:
+        files.extend(tests_dir.glob("validate.*.yaml"))
+    return files
+
+
+def is_validate_only_file(test_file: Path) -> bool:
+    """Return True if the given path is a config-only validate file.
+
+    Validate files follow the same grammar as test files but with a
+    ``validate`` prefix instead of ``test``: ``validate.<platform>.yaml``
+    or ``validate-<variant>.<platform>.yaml``. They are exercised with
+    ``esphome config`` only and skipped during compile.
+    """
+    name = test_file.name
+    return name.startswith("validate.") or name.startswith("validate-")
 
 
 @dataclass(frozen=True)

script/split_components_for_ci.py

@@ -44,14 +44,21 @@ ALL_PLATFORMS = "all"
 def has_test_files(component_name: str, tests_dir: Path) -> bool:
     """Check if a component has test files.
 
+    Validate files (validate.*.yaml) count -- a component with only config-only
+    test files still needs a CI runner for schema validation.
+
     Args:
         component_name: Name of the component
         tests_dir: Path to tests/components directory (unused, kept for compatibility)
 
     Returns:
-        True if the component has test.*.yaml or test-*.yaml files
+        True if the component has test.*.yaml, test-*.yaml, or validate.*.yaml files
     """
-    return bool(get_component_test_files(component_name, all_variants=True))
+    return bool(
+        get_component_test_files(
+            component_name, all_variants=True, include_validate=True
+        )
+    )
 
 
 def create_intelligent_batches(

script/test_build_components.py

@@ -39,7 +39,11 @@ from script.analyze_component_buses import (
     merge_compatible_bus_groups,
     uses_local_file_references,
 )
-from script.helpers import get_component_test_files, split_conflicting_groups
+from script.helpers import (
+    get_component_test_files,
+    is_validate_only_file,
+    split_conflicting_groups,
+)
 from script.merge_component_configs import merge_component_configs
 
 
@@ -83,7 +87,10 @@ def show_disk_space_if_ci(esphome_command: str) -> None:
 
 
 def find_component_tests(
-    components_dir: Path, component_pattern: str = "*", base_only: bool = False
+    components_dir: Path,
+    component_pattern: str = "*",
+    base_only: bool = False,
+    include_validate: bool = False,
 ) -> dict[str, list[Path]]:
     """Find all component test files.
 
@@ -91,6 +98,8 @@ def find_component_tests(
         components_dir: Path to tests/components directory
         component_pattern: Glob pattern for component names
         base_only: If True, only find base test files (test.*.yaml), not variant files (test-*.yaml)
+        include_validate: If True, also include config-only files (validate.*.yaml).
+            These are run with `esphome config` only and never compiled.
 
     Returns:
         Dictionary mapping component name to list of test files
@@ -102,7 +111,11 @@ def find_component_tests(
             continue
 
         # Get test files using helper function
-        test_files = get_component_test_files(comp_dir.name, all_variants=not base_only)
+        test_files = get_component_test_files(
+            comp_dir.name,
+            all_variants=not base_only,
+            include_validate=include_validate,
+        )
         if test_files:
             component_tests[comp_dir.name] = test_files
 
@@ -836,12 +849,25 @@ def run_grouped_component_tests(
     # With grouping:
     # - 1 build per group (regardless of how many components)
     # - Individual components still need all their platform builds
+    # - Validate files of grouped components still run individually
+    #   (they're config-only and bypass the grouped compile, see
+    #   run_individual_component_test), so each adds one more invocation.
     individual_test_file_count = sum(
         len(all_tests[comp]) for comp in individual_tests if comp in all_tests
     )
 
+    grouped_component_set = {c for _, _, comps in groups_to_test for c in comps}
+    grouped_validate_file_count = sum(
+        1
+        for comp in grouped_component_set
+        for test_file in all_tests.get(comp, [])
+        if is_validate_only_file(test_file)
+    )
+
     total_grouped_components = sum(len(comps) for _, _, comps in groups_to_test)
-    total_builds_with_grouping = len(groups_to_test) + individual_test_file_count
+    total_builds_with_grouping = (
+        len(groups_to_test) + individual_test_file_count + grouped_validate_file_count
+    )
     builds_saved = total_test_files - total_builds_with_grouping
 
     print(f"\n{'=' * 80}")
@@ -854,6 +880,10 @@ def run_grouped_component_tests(
     print(
         f"  • {individual_test_file_count} individual builds ({len(individual_tests)} components)"
     )
+    if grouped_validate_file_count:
+        print(
+            f"  • {grouped_validate_file_count} validate-only invocations for grouped components"
+        )
     if total_test_files > 0:
         reduction_pct = (builds_saved / total_test_files) * 100
         print(f"  • Saves {builds_saved} builds ({reduction_pct:.1f}% reduction)")
@@ -937,8 +967,13 @@ def run_individual_component_test(
         tested_components: Set of already tested components
         test_results: List to append test results
     """
-    # Skip if already tested in a group
-    if (component, platform_with_version) in tested_components:
+    # Validate files (validate.*.yaml) are config-only and never participate
+    # in compile-time bus grouping, so always run them individually even when
+    # the (component, platform) pair was covered by a group test.
+    if (
+        not is_validate_only_file(test_file)
+        and (component, platform_with_version) in tested_components
+    ):
         return
 
     test_result = run_esphome_test(
@@ -992,13 +1027,23 @@ def test_components(
     # Get platform base files
     platform_bases = get_platform_base_files(build_components_dir)
 
+    # Validate files (validate.*.yaml) are config-only -- they exercise
+    # schema/validation paths but are never compiled. Include them when running
+    # `config` or `clean`; exclude them under `compile` so they never reach a
+    # toolchain build.
+    include_validate = esphome_command != "compile"
+
     # Find all component tests
     all_tests = {}
     for pattern in component_patterns:
         # Skip empty patterns (happens when components list is empty string)
         if not pattern:
             continue
-        all_tests.update(find_component_tests(tests_dir, pattern, base_only))
+        all_tests.update(
+            find_component_tests(
+                tests_dir, pattern, base_only, include_validate=include_validate
+            )
+        )
 
     # If no components found, build a reference configuration for baseline comparison
     # Create a synthetic "empty" component test that will build just the base config