[ci] Run benchmarks when a component the Python import benchmark imports changes (#20763)

script/determine-jobs.py

@@ -49,6 +49,7 @@ Options:
 from __future__ import annotations
 
 import argparse
+import ast
 from collections import Counter
 from collections.abc import Callable
 from enum import StrEnum
@@ -58,6 +59,7 @@ import math
 import os
 from pathlib import Path
 import statistics
+import subprocess
 import sys
 from typing import Any
 
@@ -69,6 +71,7 @@ from clang_tidy_hash import (
 )
 from helpers import (
     CPP_FILE_EXTENSIONS,
+    ESPHOME_COMPONENTS_PATH,
     ESPHOME_TESTS_COMPONENTS_PATH,
     INTEGRATION_TESTS_PATH,
     PYTHON_FILE_EXTENSIONS,
@@ -805,6 +808,9 @@ def determine_cpp_unit_tests(
 # Paths within tests/benchmarks/ that contain component benchmark files
 BENCHMARKS_COMPONENTS_PATH = "tests/benchmarks/components"
 
+# The Python benchmark that imports components; its COMPONENTS tuple says which ones
+PYTHON_IMPORT_BENCHMARK_FILE = "tests/benchmarks/python/test_component_import_bench.py"
+
 # Files that, when changed, should trigger benchmark runs
 BENCHMARK_INFRASTRUCTURE_FILES = frozenset(
     {
@@ -815,6 +821,53 @@ BENCHMARK_INFRASTRUCTURE_FILES = frozenset(
 )
 
 
+def _benchmark_components_tuple(path: Path) -> frozenset[str]:
+    """Return the benchmark file's ``COMPONENTS`` tuple.
+
+    Raises when the file has none, so renaming it fails CI instead of quietly
+    turning the trigger off.
+    """
+    for node in ast.parse(path.read_text(encoding="utf-8")).body:
+        if isinstance(node, ast.Assign) and any(
+            isinstance(t, ast.Name) and t.id == "COMPONENTS" for t in node.targets
+        ):
+            return frozenset(ast.literal_eval(node.value))
+    raise RuntimeError(f"{PYTHON_IMPORT_BENCHMARK_FILE} has no COMPONENTS tuple")
+
+
+# Imports the given components and prints every component whose module got loaded
+_IMPORT_PROBE = (
+    "import importlib, sys\n"
+    "for c in sys.argv[1:]: importlib.import_module('esphome.components.' + c)\n"
+    "print(*{n.split('.')[2] for n in sys.modules if n.startswith('esphome.components.')})"
+)
+
+
+def python_import_benchmarked_components() -> frozenset[str] | None:
+    """Return the components loaded by importing the benchmark's ``COMPONENTS``.
+
+    Imported in a fresh interpreter from the checkout, so the set is exactly what
+    the benchmark times. None when the import fails.
+    """
+    path = Path(root_path) / PYTHON_IMPORT_BENCHMARK_FILE
+    if not path.is_file():
+        return frozenset()
+    result = subprocess.run(
+        [sys.executable, "-c", _IMPORT_PROBE, *_benchmark_components_tuple(path)],
+        cwd=root_path,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        print(
+            f"Benchmark import probe failed, running benchmarks:\n{result.stderr}",
+            file=sys.stderr,
+        )
+        return None
+    return frozenset(result.stdout.split())
+
+
 def should_run_benchmarks(branch: str | None = None) -> bool:
     """Determine if benchmarks (C++ and Python) should run based on changed files.
 
@@ -831,6 +884,8 @@ def should_run_benchmarks(branch: str | None = None) -> bool:
     4. A directly changed component has benchmark files (no dependency expansion)
     5. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py,
        script/build_helpers.py, script/setup_codspeed_lib.py)
+    6. A Python file changed in a component that importing the Python import
+       benchmark's ``COMPONENTS`` loads, even if it has no C++ benchmark
 
     Unlike unit tests, benchmarks do NOT expand to dependent components.
     Changing ``sensor`` does not trigger ``api`` benchmarks just because
@@ -862,6 +917,19 @@ def should_run_benchmarks(branch: str | None = None) -> bool:
     ):
         return True
 
+    # The Python import benchmark times importing its components and what they pull
+    # in, so their Python sources move its numbers even without a C++ benchmark.
+    # A failed import runs the benchmarks rather than risk skipping them.
+    changed_python = {
+        get_component_from_path(f)
+        for f in files
+        if f.startswith(ESPHOME_COMPONENTS_PATH) and f.endswith(PYTHON_FILE_EXTENSIONS)
+    }
+    if changed_python:
+        loaded = python_import_benchmarked_components()
+        if loaded is None or changed_python & loaded:
+            return True
+
     # Check if any directly changed component has benchmarks
     benchmarks_dir = Path(root_path) / BENCHMARKS_COMPONENTS_PATH
     if not benchmarks_dir.is_dir():

tests/script/test_determine_jobs.py

@@ -4,6 +4,7 @@ from collections.abc import Generator
 import importlib.util
 import json
 from pathlib import Path
+import subprocess
 import sys
 from unittest.mock import Mock, patch
 
@@ -2784,6 +2785,87 @@ def test_should_run_benchmarks_core_benchmark_file_change() -> None:
         assert determine_jobs.should_run_benchmarks() is True
 
 
+def test_python_import_benchmarked_components_imports_real_components() -> None:
+    """The set is what importing the real benchmark's components loads."""
+    loaded = determine_jobs.python_import_benchmarked_components()
+    # mqtt is not listed in COMPONENTS; sensor imports it at module scope
+    assert {"binary_sensor", "sensor", "logger", "mqtt"} <= loaded
+
+
+def test_should_run_benchmarks_python_import_benchmarked_component() -> None:
+    """A Python change in a component the import benchmark loads runs benchmarks.
+
+    logger has no C++ benchmark, so before this a change to its __init__.py
+    skipped the Python import benchmark that times it.
+    """
+    for changed, expected in [
+        ("esphome/components/logger/__init__.py", True),
+        ("esphome/components/logger/select/__init__.py", True),
+        ("esphome/components/mqtt/__init__.py", True),
+        ("esphome/components/logger/logger.cpp", False),
+        ("tests/components/logger/common.yaml", False),
+        ("esphome/components/wifi/__init__.py", False),
+    ]:
+        with (
+            patch.object(determine_jobs, "changed_files", return_value=[changed]),
+            patch.object(
+                determine_jobs,
+                "python_import_benchmarked_components",
+                return_value=frozenset({"logger", "mqtt"}),
+            ),
+        ):
+            assert determine_jobs.should_run_benchmarks() is expected, changed
+
+
+def test_should_run_benchmarks_when_import_probe_fails() -> None:
+    """A component import that fails runs the benchmarks instead of skipping them."""
+    with (
+        patch.object(
+            determine_jobs,
+            "changed_files",
+            return_value=["esphome/components/wifi/__init__.py"],
+        ),
+        patch.object(
+            determine_jobs, "python_import_benchmarked_components", return_value=None
+        ),
+    ):
+        assert determine_jobs.should_run_benchmarks() is True
+
+
+def test_python_import_benchmarked_components_logs_probe_failure(
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """A failed probe returns None and prints why on stderr, never stdout."""
+    failed = subprocess.CompletedProcess(
+        args=[], returncode=1, stdout="", stderr="ModuleNotFoundError: boom"
+    )
+    with patch.object(determine_jobs.subprocess, "run", return_value=failed):
+        assert determine_jobs.python_import_benchmarked_components() is None
+    captured = capsys.readouterr()
+    assert "ModuleNotFoundError: boom" in captured.err
+    assert captured.out == ""
+
+
+def test_python_import_benchmarked_components_missing_file(tmp_path: Path) -> None:
+    """Without the benchmark file no component is treated as benchmarked."""
+    with patch.object(determine_jobs, "root_path", str(tmp_path)):
+        assert determine_jobs.python_import_benchmarked_components() == frozenset()
+
+
+def test_python_import_benchmarked_components_requires_components(
+    tmp_path: Path,
+) -> None:
+    """A benchmark file without COMPONENTS fails loudly instead of disabling the rule."""
+    bench = tmp_path / determine_jobs.PYTHON_IMPORT_BENCHMARK_FILE
+    bench.parent.mkdir(parents=True)
+    bench.write_text('NAMES = ("logger",)\n')
+    with (
+        patch.object(determine_jobs, "root_path", str(tmp_path)),
+        pytest.raises(RuntimeError, match="COMPONENTS"),
+    ):
+        determine_jobs.python_import_benchmarked_components()
+
+
 def test_should_run_benchmarks_benchmarked_component_change(tmp_path: Path) -> None:
     """Test benchmarks trigger when a benchmarked component changes."""
     # Create a fake benchmarks directory with an 'api' component