From a4278b8decaa087f593f4a50418e8b2167e2df31 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 14 Oct 2026 10:35:33 -1000 Subject: [PATCH] [ci] Run benchmarks when a component the Python import benchmark imports changes (#20763) --- script/determine-jobs.py | 68 ++++++++++++++++++++++++ tests/script/test_determine_jobs.py | 82 +++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 9d146c94a0b..e1483446d4d 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -49,6 +49,7 @@ Options: from __future__ import annotations import argparse +import ast from collections import Counter from collections.abc import Callable from enum import StrEnum @@ -58,6 +59,7 @@ import math import os from pathlib import Path import statistics +import subprocess import sys from typing import Any @@ -69,6 +71,7 @@ from clang_tidy_hash import ( ) from helpers import ( CPP_FILE_EXTENSIONS, + ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH, INTEGRATION_TESTS_PATH, PYTHON_FILE_EXTENSIONS, @@ -805,6 +808,9 @@ def determine_cpp_unit_tests( # Paths within tests/benchmarks/ that contain component benchmark files BENCHMARKS_COMPONENTS_PATH = "tests/benchmarks/components" +# The Python benchmark that imports components; its COMPONENTS tuple says which ones +PYTHON_IMPORT_BENCHMARK_FILE = "tests/benchmarks/python/test_component_import_bench.py" + # Files that, when changed, should trigger benchmark runs BENCHMARK_INFRASTRUCTURE_FILES = frozenset( { @@ -815,6 +821,53 @@ BENCHMARK_INFRASTRUCTURE_FILES = frozenset( ) +def _benchmark_components_tuple(path: Path) -> frozenset[str]: + """Return the benchmark file's ``COMPONENTS`` tuple. + + Raises when the file has none, so renaming it fails CI instead of quietly + turning the trigger off. + """ + for node in ast.parse(path.read_text(encoding="utf-8")).body: + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id == "COMPONENTS" for t in node.targets + ): + return frozenset(ast.literal_eval(node.value)) + raise RuntimeError(f"{PYTHON_IMPORT_BENCHMARK_FILE} has no COMPONENTS tuple") + + +# Imports the given components and prints every component whose module got loaded +_IMPORT_PROBE = ( + "import importlib, sys\n" + "for c in sys.argv[1:]: importlib.import_module('esphome.components.' + c)\n" + "print(*{n.split('.')[2] for n in sys.modules if n.startswith('esphome.components.')})" +) + + +def python_import_benchmarked_components() -> frozenset[str] | None: + """Return the components loaded by importing the benchmark's ``COMPONENTS``. + + Imported in a fresh interpreter from the checkout, so the set is exactly what + the benchmark times. None when the import fails. + """ + path = Path(root_path) / PYTHON_IMPORT_BENCHMARK_FILE + if not path.is_file(): + return frozenset() + result = subprocess.run( + [sys.executable, "-c", _IMPORT_PROBE, *_benchmark_components_tuple(path)], + cwd=root_path, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + print( + f"Benchmark import probe failed, running benchmarks:\n{result.stderr}", + file=sys.stderr, + ) + return None + return frozenset(result.stdout.split()) + + def should_run_benchmarks(branch: str | None = None) -> bool: """Determine if benchmarks (C++ and Python) should run based on changed files. @@ -831,6 +884,8 @@ def should_run_benchmarks(branch: str | None = None) -> bool: 4. A directly changed component has benchmark files (no dependency expansion) 5. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py, script/build_helpers.py, script/setup_codspeed_lib.py) + 6. A Python file changed in a component that importing the Python import + benchmark's ``COMPONENTS`` loads, even if it has no C++ benchmark Unlike unit tests, benchmarks do NOT expand to dependent components. Changing ``sensor`` does not trigger ``api`` benchmarks just because @@ -862,6 +917,19 @@ def should_run_benchmarks(branch: str | None = None) -> bool: ): return True + # The Python import benchmark times importing its components and what they pull + # in, so their Python sources move its numbers even without a C++ benchmark. + # A failed import runs the benchmarks rather than risk skipping them. + changed_python = { + get_component_from_path(f) + for f in files + if f.startswith(ESPHOME_COMPONENTS_PATH) and f.endswith(PYTHON_FILE_EXTENSIONS) + } + if changed_python: + loaded = python_import_benchmarked_components() + if loaded is None or changed_python & loaded: + return True + # Check if any directly changed component has benchmarks benchmarks_dir = Path(root_path) / BENCHMARKS_COMPONENTS_PATH if not benchmarks_dir.is_dir(): diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index e63e0849deb..6b63712f2a4 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -4,6 +4,7 @@ from collections.abc import Generator import importlib.util import json from pathlib import Path +import subprocess import sys from unittest.mock import Mock, patch @@ -2784,6 +2785,87 @@ def test_should_run_benchmarks_core_benchmark_file_change() -> None: assert determine_jobs.should_run_benchmarks() is True +def test_python_import_benchmarked_components_imports_real_components() -> None: + """The set is what importing the real benchmark's components loads.""" + loaded = determine_jobs.python_import_benchmarked_components() + # mqtt is not listed in COMPONENTS; sensor imports it at module scope + assert {"binary_sensor", "sensor", "logger", "mqtt"} <= loaded + + +def test_should_run_benchmarks_python_import_benchmarked_component() -> None: + """A Python change in a component the import benchmark loads runs benchmarks. + + logger has no C++ benchmark, so before this a change to its __init__.py + skipped the Python import benchmark that times it. + """ + for changed, expected in [ + ("esphome/components/logger/__init__.py", True), + ("esphome/components/logger/select/__init__.py", True), + ("esphome/components/mqtt/__init__.py", True), + ("esphome/components/logger/logger.cpp", False), + ("tests/components/logger/common.yaml", False), + ("esphome/components/wifi/__init__.py", False), + ]: + with ( + patch.object(determine_jobs, "changed_files", return_value=[changed]), + patch.object( + determine_jobs, + "python_import_benchmarked_components", + return_value=frozenset({"logger", "mqtt"}), + ), + ): + assert determine_jobs.should_run_benchmarks() is expected, changed + + +def test_should_run_benchmarks_when_import_probe_fails() -> None: + """A component import that fails runs the benchmarks instead of skipping them.""" + with ( + patch.object( + determine_jobs, + "changed_files", + return_value=["esphome/components/wifi/__init__.py"], + ), + patch.object( + determine_jobs, "python_import_benchmarked_components", return_value=None + ), + ): + assert determine_jobs.should_run_benchmarks() is True + + +def test_python_import_benchmarked_components_logs_probe_failure( + capsys: pytest.CaptureFixture[str], +) -> None: + """A failed probe returns None and prints why on stderr, never stdout.""" + failed = subprocess.CompletedProcess( + args=[], returncode=1, stdout="", stderr="ModuleNotFoundError: boom" + ) + with patch.object(determine_jobs.subprocess, "run", return_value=failed): + assert determine_jobs.python_import_benchmarked_components() is None + captured = capsys.readouterr() + assert "ModuleNotFoundError: boom" in captured.err + assert captured.out == "" + + +def test_python_import_benchmarked_components_missing_file(tmp_path: Path) -> None: + """Without the benchmark file no component is treated as benchmarked.""" + with patch.object(determine_jobs, "root_path", str(tmp_path)): + assert determine_jobs.python_import_benchmarked_components() == frozenset() + + +def test_python_import_benchmarked_components_requires_components( + tmp_path: Path, +) -> None: + """A benchmark file without COMPONENTS fails loudly instead of disabling the rule.""" + bench = tmp_path / determine_jobs.PYTHON_IMPORT_BENCHMARK_FILE + bench.parent.mkdir(parents=True) + bench.write_text('NAMES = ("logger",)\n') + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + pytest.raises(RuntimeError, match="COMPONENTS"), + ): + determine_jobs.python_import_benchmarked_components() + + def test_should_run_benchmarks_benchmarked_component_change(tmp_path: Path) -> None: """Test benchmarks trigger when a benchmarked component changes.""" # Create a fake benchmarks directory with an 'api' component