Move benchmarks into ci.yml with determine-jobs integration

- Add should_run_benchmarks() to determine-jobs.py that checks if
  directly changed components have benchmark files (no dependency
  expansion - changing sensor won't trigger api benchmarks)
- Move benchmark job from separate workflow into ci.yml
- Pin CodSpeed action to full commit SHA
- Delete separate ci-benchmarks.yml
This commit is contained in:
J. Nick Koston
2026-03-16 20:44:37 -10:00
parent 38ff332fec
commit d9cab03c09
3 changed files with 81 additions and 87 deletions
-87
View File
@@ -1,87 +0,0 @@
---
name: CodSpeed Benchmarks
on:
push:
branches: [dev]
pull_request:
paths:
- "esphome/components/api/**"
- "esphome/core/**"
- "tests/benchmarks/**"
- "script/cpp_benchmark.py"
- "script/test_helpers.py"
- ".github/workflows/ci-benchmarks.yml"
merge_group:
permissions:
contents: read
concurrency:
# yamllint disable-line rule:line-length
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
DEFAULT_PYTHON: "3.11"
jobs:
common:
name: Create common environment
runs-on: ubuntu-24.04
outputs:
cache-key: ${{ steps.cache-key.outputs.key }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Generate cache-key
id: cache-key
run: echo key="${{ hashFiles('requirements.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ steps.cache-key.outputs.key }}
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
python -m venv venv
. venv/bin/activate
python --version
pip install -r requirements.txt -r requirements_test.txt
pip install -e .
benchmarks:
name: Run CodSpeed Benchmarks
runs-on: ubuntu-24.04
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Restore Python
uses: ./.github/actions/restore-python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
cache-key: ${{ needs.common.outputs.cache-key }}
- name: Build benchmarks
id: build
run: |
. venv/bin/activate
BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1)
echo "binary=$BINARY" >> $GITHUB_OUTPUT
- name: Run CodSpeed benchmarks
uses: CodSpeedHQ/action@v4
with:
run: ${{ steps.build.outputs.binary }}
token: ${{ secrets.CODSPEED_TOKEN }}
+32
View File
@@ -185,6 +185,7 @@ jobs:
cpp-unit-tests-run-all: ${{ steps.determine.outputs.cpp-unit-tests-run-all }}
cpp-unit-tests-components: ${{ steps.determine.outputs.cpp-unit-tests-components }}
component-test-batches: ${{ steps.determine.outputs.component-test-batches }}
benchmarks: ${{ steps.determine.outputs.benchmarks }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -227,6 +228,7 @@ jobs:
echo "cpp-unit-tests-run-all=$(echo "$output" | jq -r '.cpp_unit_tests_run_all')" >> $GITHUB_OUTPUT
echo "cpp-unit-tests-components=$(echo "$output" | jq -c '.cpp_unit_tests_components')" >> $GITHUB_OUTPUT
echo "component-test-batches=$(echo "$output" | jq -c '.component_test_batches')" >> $GITHUB_OUTPUT
echo "benchmarks=$(echo "$output" | jq -r '.benchmarks')" >> $GITHUB_OUTPUT
- name: Save components graph cache
if: github.ref == 'refs/heads/dev'
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
@@ -308,6 +310,36 @@ jobs:
script/cpp_unit_test.py $ARGS
fi
benchmarks:
name: Run CodSpeed benchmarks
runs-on: ubuntu-24.04
needs:
- common
- determine-jobs
if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.benchmarks == 'True'
steps:
- name: Check out code from GitHub
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Restore Python
uses: ./.github/actions/restore-python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
cache-key: ${{ needs.common.outputs.cache-key }}
- name: Build benchmarks
id: build
run: |
. venv/bin/activate
BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1)
echo "binary=$BINARY" >> $GITHUB_OUTPUT
- name: Run CodSpeed benchmarks
uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4
with:
run: ${{ steps.build.outputs.binary }}
token: ${{ secrets.CODSPEED_TOKEN }}
clang-tidy-single:
name: ${{ matrix.name }}
runs-on: ubuntu-24.04
+49
View File
@@ -381,6 +381,51 @@ def determine_cpp_unit_tests(
return (False, get_cpp_changed_components(cpp_files))
def should_run_benchmarks(branch: str | None = None) -> bool:
"""Determine if C++ benchmarks should run based on changed files.
Benchmarks run when any of the following conditions are met:
1. Core C++ files changed (esphome/core/*)
2. A directly changed component has benchmark files (no dependency expansion)
3. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py,
script/test_helpers.py)
Unlike unit tests, benchmarks do NOT expand to dependent components.
Changing ``sensor`` does not trigger ``api`` benchmarks just because
api depends on sensor.
Args:
branch: Branch to compare against. If None, uses default.
Returns:
True if benchmarks should run, False otherwise.
"""
files = changed_files(branch)
if core_changed(files):
return True
# Check if benchmark infrastructure changed
if any(
f.startswith("tests/benchmarks/")
or f in ("script/cpp_benchmark.py", "script/test_helpers.py")
for f in files
):
return True
# Check if any directly changed component has benchmarks
benchmarks_dir = Path(root_path) / "tests" / "benchmarks" / "components"
if not benchmarks_dir.is_dir():
return False
benchmarked_components = {
d.name
for d in benchmarks_dir.iterdir()
if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h")))
}
# Only direct changes — no dependency expansion
return any(get_component_from_path(f) in benchmarked_components for f in files)
def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) -> bool:
"""Check if a changed file ends with any of the specified extensions."""
return any(file.endswith(extensions) for file in changed_files(branch))
@@ -804,6 +849,9 @@ def main() -> None:
# Determine which C++ unit tests to run
cpp_run_all, cpp_components = determine_cpp_unit_tests(args.branch)
# Determine if benchmarks should run
run_benchmarks = should_run_benchmarks(args.branch)
# Split components into batches for CI testing
# This intelligently groups components with similar bus configurations
component_test_batches: list[str]
@@ -856,6 +904,7 @@ def main() -> None:
"cpp_unit_tests_run_all": cpp_run_all,
"cpp_unit_tests_components": cpp_components,
"component_test_batches": component_test_batches,
"benchmarks": run_benchmarks,
}
# Output as JSON