From dd52c9129005a5313d70ae175827936e5edde469 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 23:32:49 -1000 Subject: [PATCH] Add inner iteration loops to amortize CodSpeed instrumentation overhead Sub-microsecond benchmarks are dominated by the ~60ns per-iteration valgrind start/stop cost in CodSpeed simulation mode. Add kInnerIterations (1000) inner loops to all fast benchmarks so the actual work dominates. Move DoNotOptimize calls outside inner loops to prevent artificial overhead. Also address review feedback: - Use tokenless CodSpeed (public repo, no CODSPEED_TOKEN needed) - Fix warning message to show component-specific path - Fix stray ". :" in error message - Verify pinned SHA on re-runs to prevent stale checkouts --- .github/workflows/ci.yml | 1 - script/setup_codspeed_lib.py | 15 +++ script/test_helpers.py | 4 +- .../components/api/bench_proto_decode.cpp | 20 +++- .../components/api/bench_proto_encode.cpp | 98 ++++++++++++++----- .../components/api/bench_proto_varint.cpp | 63 +++++++++--- tests/benchmarks/core/bench_helpers.cpp | 19 +++- tests/benchmarks/core/bench_scheduler.cpp | 21 +++- 8 files changed, 187 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f64c6ffbb4f..4a5ac0e29ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,7 +340,6 @@ jobs: uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4 with: run: ${{ steps.build.outputs.binary }} - token: ${{ secrets.CODSPEED_TOKEN }} mode: simulation clang-tidy-single: diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index 214714d6986..959c89d05b6 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -171,6 +171,21 @@ def setup_codspeed_lib(output_dir: Path) -> None: """ if not (output_dir / ".git").exists(): _clone_repo(output_dir) + else: + # Verify the existing checkout matches the pinned SHA + result = subprocess.run( + ["git", "-C", str(output_dir), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or result.stdout.strip() != CODSPEED_CPP_SHA: + print( + f"Stale codspeed-cpp checkout, re-cloning at {CODSPEED_CPP_SHA}", + file=sys.stderr, + ) + shutil.rmtree(output_dir) + _clone_repo(output_dir) benchmark_dir = output_dir / GOOGLE_BENCHMARK_SUBDIR lib_src = benchmark_dir / "src" diff --git a/script/test_helpers.py b/script/test_helpers.py index 36a3e4b8592..db1c981b825 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -81,7 +81,7 @@ def filter_components_with_files(components: list[str], tests_dir: Path) -> list filtered_components.append(component) else: print( - f"WARNING: No files found for component '{component}' in {tests_dir}, skipping.", + f"WARNING: No files found for component '{component}' in {test_dir}, skipping.", file=sys.stderr, ) return filtered_components @@ -284,7 +284,7 @@ def compile_and_get_binary( print(f"Error compiling {label} for {', '.join(components)}") return exit_code, None except Exception as e: - print(f"Error compiling {label} for {', '.join(components)}. Check path. : {e}") + print(f"Error compiling {label} for {', '.join(components)}: {e}") return EXIT_COMPILE_ERROR, None # After a successful compilation, locate the executable: diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp index c9313c2fca0..2208a616921 100644 --- a/tests/benchmarks/components/api/bench_proto_decode.cpp +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -5,6 +5,11 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- HelloRequest decode (string + varint fields) --- static void Decode_HelloRequest(benchmark::State &state) { @@ -21,9 +26,12 @@ static void Decode_HelloRequest(benchmark::State &state) { for (auto _ : state) { HelloRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.api_version_major); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_HelloRequest); @@ -39,9 +47,12 @@ static void Decode_SwitchCommandRequest(benchmark::State &state) { for (auto _ : state) { SwitchCommandRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.state); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_SwitchCommandRequest); @@ -110,9 +121,12 @@ static void Decode_LightCommandRequest(benchmark::State &state) { for (auto _ : state) { LightCommandRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.brightness); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_LightCommandRequest); diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index 2cd050f8209..d58a2b01d20 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -5,6 +5,11 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- SensorStateResponse (highest frequency message) --- static void Encode_SensorStateResponse(benchmark::State &state) { @@ -17,10 +22,13 @@ static void Encode_SensorStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_SensorStateResponse); @@ -31,8 +39,13 @@ static void CalculateSize_SensorStateResponse(benchmark::State &state) { msg.missing_state = false; for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_SensorStateResponse); @@ -45,12 +58,15 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { msg.missing_state = false; for (auto _ : state) { - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_SensorStateResponse); @@ -84,10 +100,13 @@ static void Encode_BinarySensorStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_BinarySensorStateResponse); @@ -104,10 +123,13 @@ static void Encode_HelloResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_HelloResponse); @@ -133,10 +155,13 @@ static void Encode_LightStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_LightStateResponse); @@ -157,8 +182,13 @@ static void CalculateSize_LightStateResponse(benchmark::State &state) { msg.effect = StringRef::from_lit("rainbow"); for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_LightStateResponse); @@ -193,8 +223,13 @@ static void CalculateSize_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_DeviceInfoResponse); @@ -205,10 +240,13 @@ static void Encode_DeviceInfoResponse(benchmark::State &state) { buffer.resize(total_size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_DeviceInfoResponse); @@ -218,12 +256,15 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { APIBuffer buffer; for (auto _ : state) { - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse); @@ -232,13 +273,16 @@ static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - APIBuffer buffer; - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); - benchmark::DoNotOptimize(buffer.data()); + for (int i = 0; i < kInnerIterations; i++) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse_Fresh); diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp index d33dd4b5acc..440769ea673 100644 --- a/tests/benchmarks/components/api/bench_proto_varint.cpp +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -5,66 +5,84 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- ProtoVarInt::parse() benchmarks --- static void ProtoVarInt_Parse_SingleByte(benchmark::State &state) { - // Single-byte varint (0-127) — the most common case (fast path) uint8_t buf[] = {0x42}; // value = 66 for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_SingleByte); static void ProtoVarInt_Parse_TwoByte(benchmark::State &state) { - // Two-byte varint (128-16383) uint8_t buf[] = {0x80, 0x01}; // value = 128 for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_TwoByte); static void ProtoVarInt_Parse_FiveByte(benchmark::State &state) { - // Five-byte varint (max uint32 = 4294967295) uint8_t buf[] = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F}; for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_FiveByte); // --- Varint encoding benchmarks --- static void Encode_Varint_Small(benchmark::State &state) { - // Value < 128 — single byte fast path APIBuffer buffer; buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(42); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(42); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_Small); static void Encode_Varint_Large(benchmark::State &state) { - // Value > 128 — multi-byte slow path APIBuffer buffer; buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(300); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(300); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_Large); @@ -73,10 +91,13 @@ static void Encode_Varint_MaxUint32(benchmark::State &state) { buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(0xFFFFFFFF); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(0xFFFFFFFF); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_MaxUint32); @@ -84,15 +105,25 @@ BENCHMARK(Encode_Varint_MaxUint32); static void ProtoSize_Varint_Small(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(ProtoSize::varint(42)); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += ProtoSize::varint(42); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoSize_Varint_Small); static void ProtoSize_Varint_Large(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(ProtoSize::varint(0xFFFFFFFF)); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += ProtoSize::varint(0xFFFFFFFF); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoSize_Varint_Large); diff --git a/tests/benchmarks/core/bench_helpers.cpp b/tests/benchmarks/core/bench_helpers.cpp index d337a27f61b..5610d833e3f 100644 --- a/tests/benchmarks/core/bench_helpers.cpp +++ b/tests/benchmarks/core/bench_helpers.cpp @@ -4,13 +4,23 @@ namespace esphome::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- random_float() --- // Ported from ol.yaml:148 "Random Float Benchmark" static void RandomFloat(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(random_float()); + float result = 0.0f; + for (int i = 0; i < kInnerIterations; i++) { + result += random_float(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(RandomFloat); @@ -18,8 +28,13 @@ BENCHMARK(RandomFloat); static void RandomUint32(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(random_uint32()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += random_uint32(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(RandomUint32); diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 4ce24abf949..8382a4b228c 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -5,6 +5,11 @@ namespace esphome::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- Scheduler fast path: no work to do --- static void Scheduler_Call_NoWork(benchmark::State &state) { @@ -12,9 +17,12 @@ static void Scheduler_Call_NoWork(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - scheduler.call(now); + for (int i = 0; i < kInnerIterations; i++) { + scheduler.call(now); + } benchmark::DoNotOptimize(now); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_Call_NoWork); @@ -33,9 +41,12 @@ static void Scheduler_Call_TimersNotDue(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - scheduler.call(now); + for (int i = 0; i < kInnerIterations; i++) { + scheduler.call(now); + } benchmark::DoNotOptimize(now); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_Call_TimersNotDue); @@ -54,9 +65,13 @@ static void Scheduler_NextScheduleIn(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - auto result = scheduler.next_schedule_in(now); + optional result; + for (int i = 0; i < kInnerIterations; i++) { + result = scheduler.next_schedule_in(now); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_NextScheduleIn);