diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f64c6ffbb4f..4a5ac0e29ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,7 +340,6 @@ jobs: uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4 with: run: ${{ steps.build.outputs.binary }} - token: ${{ secrets.CODSPEED_TOKEN }} mode: simulation clang-tidy-single: diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index 214714d6986..959c89d05b6 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -171,6 +171,21 @@ def setup_codspeed_lib(output_dir: Path) -> None: """ if not (output_dir / ".git").exists(): _clone_repo(output_dir) + else: + # Verify the existing checkout matches the pinned SHA + result = subprocess.run( + ["git", "-C", str(output_dir), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or result.stdout.strip() != CODSPEED_CPP_SHA: + print( + f"Stale codspeed-cpp checkout, re-cloning at {CODSPEED_CPP_SHA}", + file=sys.stderr, + ) + shutil.rmtree(output_dir) + _clone_repo(output_dir) benchmark_dir = output_dir / GOOGLE_BENCHMARK_SUBDIR lib_src = benchmark_dir / "src" diff --git a/script/test_helpers.py b/script/test_helpers.py index 36a3e4b8592..db1c981b825 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -81,7 +81,7 @@ def filter_components_with_files(components: list[str], tests_dir: Path) -> list filtered_components.append(component) else: print( - f"WARNING: No files found for component '{component}' in {tests_dir}, skipping.", + f"WARNING: No files found for component '{component}' in {test_dir}, skipping.", file=sys.stderr, ) return filtered_components @@ -284,7 +284,7 @@ def compile_and_get_binary( print(f"Error compiling {label} for {', '.join(components)}") return exit_code, None except Exception as e: - print(f"Error compiling {label} for {', '.join(components)}. Check path. : {e}") + print(f"Error compiling {label} for {', '.join(components)}: {e}") return EXIT_COMPILE_ERROR, None # After a successful compilation, locate the executable: diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp index c9313c2fca0..2208a616921 100644 --- a/tests/benchmarks/components/api/bench_proto_decode.cpp +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -5,6 +5,11 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- HelloRequest decode (string + varint fields) --- static void Decode_HelloRequest(benchmark::State &state) { @@ -21,9 +26,12 @@ static void Decode_HelloRequest(benchmark::State &state) { for (auto _ : state) { HelloRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.api_version_major); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_HelloRequest); @@ -39,9 +47,12 @@ static void Decode_SwitchCommandRequest(benchmark::State &state) { for (auto _ : state) { SwitchCommandRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.state); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_SwitchCommandRequest); @@ -110,9 +121,12 @@ static void Decode_LightCommandRequest(benchmark::State &state) { for (auto _ : state) { LightCommandRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.brightness); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_LightCommandRequest); diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index 2cd050f8209..d58a2b01d20 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -5,6 +5,11 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- SensorStateResponse (highest frequency message) --- static void Encode_SensorStateResponse(benchmark::State &state) { @@ -17,10 +22,13 @@ static void Encode_SensorStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_SensorStateResponse); @@ -31,8 +39,13 @@ static void CalculateSize_SensorStateResponse(benchmark::State &state) { msg.missing_state = false; for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_SensorStateResponse); @@ -45,12 +58,15 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { msg.missing_state = false; for (auto _ : state) { - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_SensorStateResponse); @@ -84,10 +100,13 @@ static void Encode_BinarySensorStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_BinarySensorStateResponse); @@ -104,10 +123,13 @@ static void Encode_HelloResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_HelloResponse); @@ -133,10 +155,13 @@ static void Encode_LightStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_LightStateResponse); @@ -157,8 +182,13 @@ static void CalculateSize_LightStateResponse(benchmark::State &state) { msg.effect = StringRef::from_lit("rainbow"); for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_LightStateResponse); @@ -193,8 +223,13 @@ static void CalculateSize_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_DeviceInfoResponse); @@ -205,10 +240,13 @@ static void Encode_DeviceInfoResponse(benchmark::State &state) { buffer.resize(total_size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_DeviceInfoResponse); @@ -218,12 +256,15 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { APIBuffer buffer; for (auto _ : state) { - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse); @@ -232,13 +273,16 @@ static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - APIBuffer buffer; - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); - benchmark::DoNotOptimize(buffer.data()); + for (int i = 0; i < kInnerIterations; i++) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse_Fresh); diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp index d33dd4b5acc..440769ea673 100644 --- a/tests/benchmarks/components/api/bench_proto_varint.cpp +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -5,66 +5,84 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- ProtoVarInt::parse() benchmarks --- static void ProtoVarInt_Parse_SingleByte(benchmark::State &state) { - // Single-byte varint (0-127) — the most common case (fast path) uint8_t buf[] = {0x42}; // value = 66 for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_SingleByte); static void ProtoVarInt_Parse_TwoByte(benchmark::State &state) { - // Two-byte varint (128-16383) uint8_t buf[] = {0x80, 0x01}; // value = 128 for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_TwoByte); static void ProtoVarInt_Parse_FiveByte(benchmark::State &state) { - // Five-byte varint (max uint32 = 4294967295) uint8_t buf[] = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F}; for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_FiveByte); // --- Varint encoding benchmarks --- static void Encode_Varint_Small(benchmark::State &state) { - // Value < 128 — single byte fast path APIBuffer buffer; buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(42); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(42); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_Small); static void Encode_Varint_Large(benchmark::State &state) { - // Value > 128 — multi-byte slow path APIBuffer buffer; buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(300); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(300); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_Large); @@ -73,10 +91,13 @@ static void Encode_Varint_MaxUint32(benchmark::State &state) { buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(0xFFFFFFFF); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(0xFFFFFFFF); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_MaxUint32); @@ -84,15 +105,25 @@ BENCHMARK(Encode_Varint_MaxUint32); static void ProtoSize_Varint_Small(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(ProtoSize::varint(42)); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += ProtoSize::varint(42); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoSize_Varint_Small); static void ProtoSize_Varint_Large(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(ProtoSize::varint(0xFFFFFFFF)); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += ProtoSize::varint(0xFFFFFFFF); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoSize_Varint_Large); diff --git a/tests/benchmarks/core/bench_helpers.cpp b/tests/benchmarks/core/bench_helpers.cpp index d337a27f61b..5610d833e3f 100644 --- a/tests/benchmarks/core/bench_helpers.cpp +++ b/tests/benchmarks/core/bench_helpers.cpp @@ -4,13 +4,23 @@ namespace esphome::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- random_float() --- // Ported from ol.yaml:148 "Random Float Benchmark" static void RandomFloat(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(random_float()); + float result = 0.0f; + for (int i = 0; i < kInnerIterations; i++) { + result += random_float(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(RandomFloat); @@ -18,8 +28,13 @@ BENCHMARK(RandomFloat); static void RandomUint32(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(random_uint32()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += random_uint32(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(RandomUint32); diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 4ce24abf949..8382a4b228c 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -5,6 +5,11 @@ namespace esphome::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- Scheduler fast path: no work to do --- static void Scheduler_Call_NoWork(benchmark::State &state) { @@ -12,9 +17,12 @@ static void Scheduler_Call_NoWork(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - scheduler.call(now); + for (int i = 0; i < kInnerIterations; i++) { + scheduler.call(now); + } benchmark::DoNotOptimize(now); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_Call_NoWork); @@ -33,9 +41,12 @@ static void Scheduler_Call_TimersNotDue(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - scheduler.call(now); + for (int i = 0; i < kInnerIterations; i++) { + scheduler.call(now); + } benchmark::DoNotOptimize(now); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_Call_TimersNotDue); @@ -54,9 +65,13 @@ static void Scheduler_NextScheduleIn(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - auto result = scheduler.next_schedule_in(now); + optional result; + for (int i = 0; i < kInnerIterations; i++) { + result = scheduler.next_schedule_in(now); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_NextScheduleIn);