Add inner iteration loops to amortize CodSpeed instrumentation overhead

Sub-microsecond benchmarks are dominated by the ~60ns per-iteration
valgrind start/stop cost in CodSpeed simulation mode. Add kInnerIterations
(1000) inner loops to all fast benchmarks so the actual work dominates.
Move DoNotOptimize calls outside inner loops to prevent artificial overhead.

Also address review feedback:
- Use tokenless CodSpeed (public repo, no CODSPEED_TOKEN needed)
- Fix warning message to show component-specific path
- Fix stray ". :" in error message
- Verify pinned SHA on re-runs to prevent stale checkouts

.github/workflows/ci.yml

@@ -340,7 +340,6 @@ jobs:
         uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4
         with:
           run: ${{ steps.build.outputs.binary }}
-          token: ${{ secrets.CODSPEED_TOKEN }}
           mode: simulation
 
   clang-tidy-single:

script/setup_codspeed_lib.py

@@ -171,6 +171,21 @@ def setup_codspeed_lib(output_dir: Path) -> None:
     """
     if not (output_dir / ".git").exists():
         _clone_repo(output_dir)
+    else:
+        # Verify the existing checkout matches the pinned SHA
+        result = subprocess.run(
+            ["git", "-C", str(output_dir), "rev-parse", "HEAD"],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.returncode != 0 or result.stdout.strip() != CODSPEED_CPP_SHA:
+            print(
+                f"Stale codspeed-cpp checkout, re-cloning at {CODSPEED_CPP_SHA}",
+                file=sys.stderr,
+            )
+            shutil.rmtree(output_dir)
+            _clone_repo(output_dir)
 
     benchmark_dir = output_dir / GOOGLE_BENCHMARK_SUBDIR
     lib_src = benchmark_dir / "src"

script/test_helpers.py

@@ -81,7 +81,7 @@ def filter_components_with_files(components: list[str], tests_dir: Path) -> list
             filtered_components.append(component)
         else:
             print(
-                f"WARNING: No files found for component '{component}' in {tests_dir}, skipping.",
+                f"WARNING: No files found for component '{component}' in {test_dir}, skipping.",
                 file=sys.stderr,
             )
     return filtered_components
@@ -284,7 +284,7 @@ def compile_and_get_binary(
             print(f"Error compiling {label} for {', '.join(components)}")
             return exit_code, None
     except Exception as e:
-        print(f"Error compiling {label} for {', '.join(components)}. Check path. : {e}")
+        print(f"Error compiling {label} for {', '.join(components)}: {e}")
         return EXIT_COMPILE_ERROR, None
 
     # After a successful compilation, locate the executable:

tests/benchmarks/components/api/bench_proto_decode.cpp

@@ -5,6 +5,11 @@
 
 namespace esphome::api::benchmarks {
 
+// Inner iteration count to amortize CodSpeed instrumentation overhead.
+// Without this, the ~60ns per-iteration valgrind start/stop cost dominates
+// sub-microsecond benchmarks.
+static constexpr int kInnerIterations = 1000;
+
 // --- HelloRequest decode (string + varint fields) ---
 
 static void Decode_HelloRequest(benchmark::State &state) {
@@ -21,9 +26,12 @@ static void Decode_HelloRequest(benchmark::State &state) {
 
   for (auto _ : state) {
     HelloRequest msg;
-    msg.decode(encoded, sizeof(encoded));
+    for (int i = 0; i < kInnerIterations; i++) {
+      msg.decode(encoded, sizeof(encoded));
+    }
     benchmark::DoNotOptimize(msg.api_version_major);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Decode_HelloRequest);
 
@@ -39,9 +47,12 @@ static void Decode_SwitchCommandRequest(benchmark::State &state) {
 
   for (auto _ : state) {
     SwitchCommandRequest msg;
-    msg.decode(encoded, sizeof(encoded));
+    for (int i = 0; i < kInnerIterations; i++) {
+      msg.decode(encoded, sizeof(encoded));
+    }
     benchmark::DoNotOptimize(msg.state);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Decode_SwitchCommandRequest);
 
@@ -110,9 +121,12 @@ static void Decode_LightCommandRequest(benchmark::State &state) {
 
   for (auto _ : state) {
     LightCommandRequest msg;
-    msg.decode(encoded, sizeof(encoded));
+    for (int i = 0; i < kInnerIterations; i++) {
+      msg.decode(encoded, sizeof(encoded));
+    }
     benchmark::DoNotOptimize(msg.brightness);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Decode_LightCommandRequest);
 

tests/benchmarks/components/api/bench_proto_encode.cpp

@@ -5,6 +5,11 @@
 
 namespace esphome::api::benchmarks {
 
+// Inner iteration count to amortize CodSpeed instrumentation overhead.
+// Without this, the ~60ns per-iteration valgrind start/stop cost dominates
+// sub-microsecond benchmarks.
+static constexpr int kInnerIterations = 1000;
+
 // --- SensorStateResponse (highest frequency message) ---
 
 static void Encode_SensorStateResponse(benchmark::State &state) {
@@ -17,10 +22,13 @@ static void Encode_SensorStateResponse(benchmark::State &state) {
   buffer.resize(size);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_SensorStateResponse);
 
@@ -31,8 +39,13 @@ static void CalculateSize_SensorStateResponse(benchmark::State &state) {
   msg.missing_state = false;
 
   for (auto _ : state) {
-    benchmark::DoNotOptimize(msg.calculate_size());
+    uint32_t result = 0;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result += msg.calculate_size();
+    }
+    benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(CalculateSize_SensorStateResponse);
 
@@ -45,12 +58,15 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) {
   msg.missing_state = false;
 
   for (auto _ : state) {
-    uint32_t size = msg.calculate_size();
-    buffer.resize(size);
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
+    for (int i = 0; i < kInnerIterations; i++) {
+      uint32_t size = msg.calculate_size();
+      buffer.resize(size);
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(CalcAndEncode_SensorStateResponse);
 
@@ -84,10 +100,13 @@ static void Encode_BinarySensorStateResponse(benchmark::State &state) {
   buffer.resize(size);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_BinarySensorStateResponse);
 
@@ -104,10 +123,13 @@ static void Encode_HelloResponse(benchmark::State &state) {
   buffer.resize(size);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_HelloResponse);
 
@@ -133,10 +155,13 @@ static void Encode_LightStateResponse(benchmark::State &state) {
   buffer.resize(size);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_LightStateResponse);
 
@@ -157,8 +182,13 @@ static void CalculateSize_LightStateResponse(benchmark::State &state) {
   msg.effect = StringRef::from_lit("rainbow");
 
   for (auto _ : state) {
-    benchmark::DoNotOptimize(msg.calculate_size());
+    uint32_t result = 0;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result += msg.calculate_size();
+    }
+    benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(CalculateSize_LightStateResponse);
 
@@ -193,8 +223,13 @@ static void CalculateSize_DeviceInfoResponse(benchmark::State &state) {
   auto msg = make_device_info_response();
 
   for (auto _ : state) {
-    benchmark::DoNotOptimize(msg.calculate_size());
+    uint32_t result = 0;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result += msg.calculate_size();
+    }
+    benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(CalculateSize_DeviceInfoResponse);
 
@@ -205,10 +240,13 @@ static void Encode_DeviceInfoResponse(benchmark::State &state) {
   buffer.resize(total_size);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_DeviceInfoResponse);
 
@@ -218,12 +256,15 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) {
   APIBuffer buffer;
 
   for (auto _ : state) {
-    uint32_t size = msg.calculate_size();
-    buffer.resize(size);
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
+    for (int i = 0; i < kInnerIterations; i++) {
+      uint32_t size = msg.calculate_size();
+      buffer.resize(size);
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(CalcAndEncode_DeviceInfoResponse);
 
@@ -232,13 +273,16 @@ static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) {
   auto msg = make_device_info_response();
 
   for (auto _ : state) {
-    APIBuffer buffer;
-    uint32_t size = msg.calculate_size();
-    buffer.resize(size);
-    ProtoWriteBuffer writer(&buffer, 0);
-    msg.encode(writer);
-    benchmark::DoNotOptimize(buffer.data());
+    for (int i = 0; i < kInnerIterations; i++) {
+      APIBuffer buffer;
+      uint32_t size = msg.calculate_size();
+      buffer.resize(size);
+      ProtoWriteBuffer writer(&buffer, 0);
+      msg.encode(writer);
+      benchmark::DoNotOptimize(buffer.data());
+    }
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(CalcAndEncode_DeviceInfoResponse_Fresh);
 

tests/benchmarks/components/api/bench_proto_varint.cpp

@@ -5,66 +5,84 @@
 
 namespace esphome::api::benchmarks {
 
+// Inner iteration count to amortize CodSpeed instrumentation overhead.
+// Without this, the ~60ns per-iteration valgrind start/stop cost dominates
+// sub-microsecond benchmarks.
+static constexpr int kInnerIterations = 1000;
+
 // --- ProtoVarInt::parse() benchmarks ---
 
 static void ProtoVarInt_Parse_SingleByte(benchmark::State &state) {
-  // Single-byte varint (0-127) — the most common case (fast path)
   uint8_t buf[] = {0x42};  // value = 66
 
   for (auto _ : state) {
-    auto result = ProtoVarInt::parse(buf, sizeof(buf));
+    ProtoVarIntResult result{};
+    for (int i = 0; i < kInnerIterations; i++) {
+      result = ProtoVarInt::parse(buf, sizeof(buf));
+    }
     benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(ProtoVarInt_Parse_SingleByte);
 
 static void ProtoVarInt_Parse_TwoByte(benchmark::State &state) {
-  // Two-byte varint (128-16383)
   uint8_t buf[] = {0x80, 0x01};  // value = 128
 
   for (auto _ : state) {
-    auto result = ProtoVarInt::parse(buf, sizeof(buf));
+    ProtoVarIntResult result{};
+    for (int i = 0; i < kInnerIterations; i++) {
+      result = ProtoVarInt::parse(buf, sizeof(buf));
+    }
     benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(ProtoVarInt_Parse_TwoByte);
 
 static void ProtoVarInt_Parse_FiveByte(benchmark::State &state) {
-  // Five-byte varint (max uint32 = 4294967295)
   uint8_t buf[] = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F};
 
   for (auto _ : state) {
-    auto result = ProtoVarInt::parse(buf, sizeof(buf));
+    ProtoVarIntResult result{};
+    for (int i = 0; i < kInnerIterations; i++) {
+      result = ProtoVarInt::parse(buf, sizeof(buf));
+    }
     benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(ProtoVarInt_Parse_FiveByte);
 
 // --- Varint encoding benchmarks ---
 
 static void Encode_Varint_Small(benchmark::State &state) {
-  // Value < 128 — single byte fast path
   APIBuffer buffer;
   buffer.resize(16);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    writer.encode_varint_raw(42);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      writer.encode_varint_raw(42);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_Varint_Small);
 
 static void Encode_Varint_Large(benchmark::State &state) {
-  // Value > 128 — multi-byte slow path
   APIBuffer buffer;
   buffer.resize(16);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    writer.encode_varint_raw(300);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      writer.encode_varint_raw(300);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_Varint_Large);
 
@@ -73,10 +91,13 @@ static void Encode_Varint_MaxUint32(benchmark::State &state) {
   buffer.resize(16);
 
   for (auto _ : state) {
-    ProtoWriteBuffer writer(&buffer, 0);
-    writer.encode_varint_raw(0xFFFFFFFF);
+    for (int i = 0; i < kInnerIterations; i++) {
+      ProtoWriteBuffer writer(&buffer, 0);
+      writer.encode_varint_raw(0xFFFFFFFF);
+    }
     benchmark::DoNotOptimize(buffer.data());
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Encode_Varint_MaxUint32);
 
@@ -84,15 +105,25 @@ BENCHMARK(Encode_Varint_MaxUint32);
 
 static void ProtoSize_Varint_Small(benchmark::State &state) {
   for (auto _ : state) {
-    benchmark::DoNotOptimize(ProtoSize::varint(42));
+    uint32_t result = 0;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result += ProtoSize::varint(42);
+    }
+    benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(ProtoSize_Varint_Small);
 
 static void ProtoSize_Varint_Large(benchmark::State &state) {
   for (auto _ : state) {
-    benchmark::DoNotOptimize(ProtoSize::varint(0xFFFFFFFF));
+    uint32_t result = 0;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result += ProtoSize::varint(0xFFFFFFFF);
+    }
+    benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(ProtoSize_Varint_Large);
 

tests/benchmarks/core/bench_helpers.cpp

@@ -4,13 +4,23 @@
 
 namespace esphome::benchmarks {
 
+// Inner iteration count to amortize CodSpeed instrumentation overhead.
+// Without this, the ~60ns per-iteration valgrind start/stop cost dominates
+// sub-microsecond benchmarks.
+static constexpr int kInnerIterations = 1000;
+
 // --- random_float() ---
 // Ported from ol.yaml:148 "Random Float Benchmark"
 
 static void RandomFloat(benchmark::State &state) {
   for (auto _ : state) {
-    benchmark::DoNotOptimize(random_float());
+    float result = 0.0f;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result += random_float();
+    }
+    benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(RandomFloat);
 
@@ -18,8 +28,13 @@ BENCHMARK(RandomFloat);
 
 static void RandomUint32(benchmark::State &state) {
   for (auto _ : state) {
-    benchmark::DoNotOptimize(random_uint32());
+    uint32_t result = 0;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result += random_uint32();
+    }
+    benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(RandomUint32);
 

tests/benchmarks/core/bench_scheduler.cpp

@@ -5,6 +5,11 @@
 
 namespace esphome::benchmarks {
 
+// Inner iteration count to amortize CodSpeed instrumentation overhead.
+// Without this, the ~60ns per-iteration valgrind start/stop cost dominates
+// sub-microsecond benchmarks.
+static constexpr int kInnerIterations = 1000;
+
 // --- Scheduler fast path: no work to do ---
 
 static void Scheduler_Call_NoWork(benchmark::State &state) {
@@ -12,9 +17,12 @@ static void Scheduler_Call_NoWork(benchmark::State &state) {
   uint32_t now = millis();
 
   for (auto _ : state) {
-    scheduler.call(now);
+    for (int i = 0; i < kInnerIterations; i++) {
+      scheduler.call(now);
+    }
     benchmark::DoNotOptimize(now);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Scheduler_Call_NoWork);
 
@@ -33,9 +41,12 @@ static void Scheduler_Call_TimersNotDue(benchmark::State &state) {
   uint32_t now = millis();
 
   for (auto _ : state) {
-    scheduler.call(now);
+    for (int i = 0; i < kInnerIterations; i++) {
+      scheduler.call(now);
+    }
     benchmark::DoNotOptimize(now);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Scheduler_Call_TimersNotDue);
 
@@ -54,9 +65,13 @@ static void Scheduler_NextScheduleIn(benchmark::State &state) {
   uint32_t now = millis();
 
   for (auto _ : state) {
-    auto result = scheduler.next_schedule_in(now);
+    optional<uint32_t> result;
+    for (int i = 0; i < kInnerIterations; i++) {
+      result = scheduler.next_schedule_in(now);
+    }
     benchmark::DoNotOptimize(result);
   }
+  state.SetItemsProcessed(state.iterations() * kInnerIterations);
 }
 BENCHMARK(Scheduler_NextScheduleIn);