diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml deleted file mode 100644 index 416e65bfc2..0000000000 --- a/.github/workflows/ci-benchmarks.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- -name: CodSpeed Benchmarks - -on: - push: - branches: [dev] - pull_request: - paths: - - "esphome/components/api/**" - - "esphome/core/**" - - "tests/benchmarks/**" - - "script/cpp_benchmark.py" - - "script/test_helpers.py" - - ".github/workflows/ci-benchmarks.yml" - merge_group: - -permissions: - contents: read - -concurrency: - # yamllint disable-line rule:line-length - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - DEFAULT_PYTHON: "3.11" - -jobs: - common: - name: Create common environment - runs-on: ubuntu-24.04 - outputs: - cache-key: ${{ steps.cache-key.outputs.key }} - steps: - - name: Check out code from GitHub - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Generate cache-key - id: cache-key - run: echo key="${{ hashFiles('requirements.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - id: python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Restore Python virtual environment - id: cache-venv - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: venv - # yamllint disable-line rule:line-length - key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ steps.cache-key.outputs.key }} - - name: Create Python virtual environment - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - python -m venv venv - . venv/bin/activate - python --version - pip install -r requirements.txt -r requirements_test.txt - pip install -e . - - benchmarks: - name: Run CodSpeed Benchmarks - runs-on: ubuntu-24.04 - needs: - - common - steps: - - name: Check out code from GitHub - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Restore Python - uses: ./.github/actions/restore-python - with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache-key: ${{ needs.common.outputs.cache-key }} - - - name: Build benchmarks - id: build - run: | - . venv/bin/activate - BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1) - echo "binary=$BINARY" >> $GITHUB_OUTPUT - - - name: Run CodSpeed benchmarks - uses: CodSpeedHQ/action@v4 - with: - run: ${{ steps.build.outputs.binary }} - token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7710589c5..72c3762189 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,6 +185,7 @@ jobs: cpp-unit-tests-run-all: ${{ steps.determine.outputs.cpp-unit-tests-run-all }} cpp-unit-tests-components: ${{ steps.determine.outputs.cpp-unit-tests-components }} component-test-batches: ${{ steps.determine.outputs.component-test-batches }} + benchmarks: ${{ steps.determine.outputs.benchmarks }} steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -227,6 +228,7 @@ jobs: echo "cpp-unit-tests-run-all=$(echo "$output" | jq -r '.cpp_unit_tests_run_all')" >> $GITHUB_OUTPUT echo "cpp-unit-tests-components=$(echo "$output" | jq -c '.cpp_unit_tests_components')" >> $GITHUB_OUTPUT echo "component-test-batches=$(echo "$output" | jq -c '.component_test_batches')" >> $GITHUB_OUTPUT + echo "benchmarks=$(echo "$output" | jq -r '.benchmarks')" >> $GITHUB_OUTPUT - name: Save components graph cache if: github.ref == 'refs/heads/dev' uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 @@ -308,6 +310,36 @@ jobs: script/cpp_unit_test.py $ARGS fi + benchmarks: + name: Run CodSpeed benchmarks + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.benchmarks == 'True' + steps: + - name: Check out code from GitHub + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + + - name: Build benchmarks + id: build + run: | + . venv/bin/activate + BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1) + echo "binary=$BINARY" >> $GITHUB_OUTPUT + + - name: Run CodSpeed benchmarks + uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4 + with: + run: ${{ steps.build.outputs.binary }} + token: ${{ secrets.CODSPEED_TOKEN }} + clang-tidy-single: name: ${{ matrix.name }} runs-on: ubuntu-24.04 diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 6808a3cf6c..e9fcd8925f 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -381,6 +381,51 @@ def determine_cpp_unit_tests( return (False, get_cpp_changed_components(cpp_files)) +def should_run_benchmarks(branch: str | None = None) -> bool: + """Determine if C++ benchmarks should run based on changed files. + + Benchmarks run when any of the following conditions are met: + + 1. Core C++ files changed (esphome/core/*) + 2. A directly changed component has benchmark files (no dependency expansion) + 3. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py, + script/test_helpers.py) + + Unlike unit tests, benchmarks do NOT expand to dependent components. + Changing ``sensor`` does not trigger ``api`` benchmarks just because + api depends on sensor. + + Args: + branch: Branch to compare against. If None, uses default. + + Returns: + True if benchmarks should run, False otherwise. + """ + files = changed_files(branch) + if core_changed(files): + return True + + # Check if benchmark infrastructure changed + if any( + f.startswith("tests/benchmarks/") + or f in ("script/cpp_benchmark.py", "script/test_helpers.py") + for f in files + ): + return True + + # Check if any directly changed component has benchmarks + benchmarks_dir = Path(root_path) / "tests" / "benchmarks" / "components" + if not benchmarks_dir.is_dir(): + return False + benchmarked_components = { + d.name + for d in benchmarks_dir.iterdir() + if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h"))) + } + # Only direct changes — no dependency expansion + return any(get_component_from_path(f) in benchmarked_components for f in files) + + def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) -> bool: """Check if a changed file ends with any of the specified extensions.""" return any(file.endswith(extensions) for file in changed_files(branch)) @@ -804,6 +849,9 @@ def main() -> None: # Determine which C++ unit tests to run cpp_run_all, cpp_components = determine_cpp_unit_tests(args.branch) + # Determine if benchmarks should run + run_benchmarks = should_run_benchmarks(args.branch) + # Split components into batches for CI testing # This intelligently groups components with similar bus configurations component_test_batches: list[str] @@ -856,6 +904,7 @@ def main() -> None: "cpp_unit_tests_run_all": cpp_run_all, "cpp_unit_tests_components": cpp_components, "component_test_batches": component_test_batches, + "benchmarks": run_benchmarks, } # Output as JSON