From 5cd2f58582cce64f7a1081e927602446ae2ee662 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:01:31 -1000 Subject: [PATCH] Use CodSpeed's codspeed-cpp fork for simulation mode - Clone CodSpeed's codspeed-cpp repo (pinned to SHA) in CI - Create PlatformIO-compatible library.json combining google_benchmark and codspeed core sources for proper instrumentation - Pass library path via BENCHMARK_LIB env var to cpp_benchmark.py - Use simulation mode for reproducible CPU instruction counting - Locally, vanilla google/benchmark is used (no CodSpeed instrumentation) --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++++++- script/cpp_benchmark.py | 5 ++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6b9f3580d..38fb0c832e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -327,11 +327,39 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + - name: Set up CodSpeed benchmark library + # CodSpeed requires their google_benchmark fork for instrumentation. + # Clone and create a PlatformIO-compatible library layout. + run: | + CODSPEED_SHA=d6b4111428ae1f1667fec9bff009522378d5d347 + git clone https://github.com/CodSpeedHQ/codspeed-cpp.git /tmp/codspeed-cpp + git -C /tmp/codspeed-cpp checkout "$CODSPEED_SHA" + # Create library.json combining google_benchmark + codspeed core + cat > /tmp/codspeed-cpp/google_benchmark/library.json << 'LIBJSON' + { + "name": "benchmark", + "version": "0.0.0", + "build": { + "flags": [ + "-I/tmp/codspeed-cpp/core/include", + "-I/tmp/codspeed-cpp/core/instrument-hooks", + "-DHAVE_STD_REGEX", + "-DHAVE_STEADY_CLOCK", + "-DBENCHMARK_STATIC_DEFINE" + ], + "srcFilter": ["+", "+"], + "includeDir": "include" + } + } + LIBJSON + - name: Build benchmarks id: build run: | . venv/bin/activate - BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1) + BENCHMARK_LIB="symlink:///tmp/codspeed-cpp/google_benchmark" \ + script/cpp_benchmark.py --all --build-only 2>&1 | tee /tmp/bench-build.log + BINARY=$(tail -1 /tmp/bench-build.log) echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 31a6896f47..06377ae51f 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -2,6 +2,7 @@ """Build and run C++ benchmarks for ESPHome components using Google Benchmark.""" import argparse +import os from pathlib import Path import sys @@ -29,13 +30,15 @@ PLATFORMIO_OPTIONS = { def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int: + # Allow CI to override the benchmark library (e.g. with CodSpeed's fork) + benchmark_lib = os.environ.get("BENCHMARK_LIB", PLATFORMIO_GOOGLE_BENCHMARK_LIB) return build_and_run( selected_components=selected_components, tests_dir=BENCHMARKS_DIR, codegen_components=BENCHMARK_CODEGEN_COMPONENTS, config_prefix="cppbench", friendly_name="CPP Benchmarks", - libraries=PLATFORMIO_GOOGLE_BENCHMARK_LIB, + libraries=benchmark_lib, platformio_options=PLATFORMIO_OPTIONS, main_entry="main.cpp", label="benchmarks",