[core] Add CodSpeed C++ benchmarks for protobuf, main loop, and helpers (#14878)

script/clang-tidy

@@ -231,6 +231,9 @@ def main():
 
     cwd = os.getcwd()
     files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])]
+    # Exclude benchmark files — they require google benchmark headers not
+    # available in the ESP32 toolchain and use different naming conventions.
+    files = [f for f in files if not f.startswith("tests/benchmarks/")]
 
     # Print initial file count if it's large
     if len(files) > 50:

script/cpp_benchmark.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Build and run C++ benchmarks for ESPHome components using Google Benchmark."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+
+from helpers import root_path
+from test_helpers import (
+    BASE_CODEGEN_COMPONENTS,
+    PLATFORMIO_GOOGLE_BENCHMARK_LIB,
+    USE_TIME_TIMEZONE_FLAG,
+    build_and_run,
+)
+
+# Path to /tests/benchmarks/components
+BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "components"
+
+# Path to /tests/benchmarks/core (always included, not a component)
+CORE_BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "core"
+
+# Additional codegen components beyond the base set.
+# json is needed because its to_code adds the ArduinoJson library
+# (auto-loaded by api, but cpp_testing suppresses to_code unless listed).
+BENCHMARK_CODEGEN_COMPONENTS = BASE_CODEGEN_COMPONENTS | {"json"}
+
+PLATFORMIO_OPTIONS = {
+    "build_unflags": [
+        "-Os",  # remove default size-opt
+    ],
+    "build_flags": [
+        "-O2",  # optimize for speed (CodSpeed recommends RelWithDebInfo)
+        "-g",  # debug symbols for profiling
+        USE_TIME_TIMEZONE_FLAG,
+        "-DUSE_BENCHMARK",  # disable WarnIfComponentBlockingGuard in finish()
+    ],
+    # Use deep+ LDF mode to ensure PlatformIO detects the benchmark
+    # library dependency from nested includes.
+    "lib_ldf_mode": "deep+",
+}
+
+
+def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int:
+    # Allow CI to override the benchmark library (e.g. with CodSpeed's fork).
+    # BENCHMARK_LIB_CONFIG is a JSON string from setup_codspeed_lib.py
+    # containing {"lib_path": "/path/to/google_benchmark"}.
+    lib_config_json = os.environ.get("BENCHMARK_LIB_CONFIG")
+
+    pio_options = PLATFORMIO_OPTIONS
+    if lib_config_json:
+        lib_config = json.loads(lib_config_json)
+        benchmark_lib = f"benchmark=symlink://{lib_config['lib_path']}"
+        # These defines must be global (not just in library.json) because
+        # benchmark.h uses #ifdef CODSPEED_ENABLED to switch benchmark
+        # registration to CodSpeed-instrumented variants, and
+        # CODSPEED_ROOT_DIR is used to display relative file paths in reports.
+        project_root = Path(__file__).resolve().parent.parent
+        codspeed_flags = [
+            "-DNDEBUG",
+            "-DCODSPEED_ENABLED",
+            "-DCODSPEED_ANALYSIS",
+            f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"',
+        ]
+        pio_options = {
+            **PLATFORMIO_OPTIONS,
+            "build_flags": PLATFORMIO_OPTIONS["build_flags"] + codspeed_flags,
+        }
+    else:
+        benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB
+
+    return build_and_run(
+        selected_components=selected_components,
+        tests_dir=BENCHMARKS_DIR,
+        codegen_components=BENCHMARK_CODEGEN_COMPONENTS,
+        config_prefix="cppbench",
+        friendly_name="CPP Benchmarks",
+        libraries=benchmark_lib,
+        platformio_options=pio_options,
+        main_entry="main.cpp",
+        label="benchmarks",
+        build_only=build_only,
+        extra_include_dirs=[CORE_BENCHMARKS_DIR],
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Build and run C++ benchmarks for ESPHome components."
+    )
+    parser.add_argument(
+        "components",
+        nargs="*",
+        help="List of components to benchmark (must have files in tests/benchmarks/components/).",
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Benchmark all components with benchmark files.",
+    )
+    parser.add_argument(
+        "--build-only",
+        action="store_true",
+        help="Only build, print binary path without running.",
+    )
+
+    args = parser.parse_args()
+
+    if args.all:
+        # Find all component directories that have .cpp files
+        components: list[str] = (
+            sorted(
+                d.name
+                for d in BENCHMARKS_DIR.iterdir()
+                if d.is_dir()
+                and d.name != "__pycache__"
+                and (any(d.glob("*.cpp")) or any(d.glob("*.h")))
+            )
+            if BENCHMARKS_DIR.is_dir()
+            else []
+        )
+    else:
+        components: list[str] = args.components
+
+    sys.exit(run_benchmarks(components, build_only=args.build_only))
+
+
+if __name__ == "__main__":
+    main()

script/determine-jobs.py

@@ -381,6 +381,63 @@ def determine_cpp_unit_tests(
     return (False, get_cpp_changed_components(cpp_files))
 
 
+# Paths within tests/benchmarks/ that contain component benchmark files
+BENCHMARKS_COMPONENTS_PATH = "tests/benchmarks/components"
+
+# Files that, when changed, should trigger benchmark runs
+BENCHMARK_INFRASTRUCTURE_FILES = frozenset(
+    {
+        "script/cpp_benchmark.py",
+        "script/test_helpers.py",
+        "script/setup_codspeed_lib.py",
+    }
+)
+
+
+def should_run_benchmarks(branch: str | None = None) -> bool:
+    """Determine if C++ benchmarks should run based on changed files.
+
+    Benchmarks run when any of the following conditions are met:
+
+    1. Core C++ files changed (esphome/core/*)
+    2. A directly changed component has benchmark files (no dependency expansion)
+    3. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py,
+       script/test_helpers.py, script/setup_codspeed_lib.py)
+
+    Unlike unit tests, benchmarks do NOT expand to dependent components.
+    Changing ``sensor`` does not trigger ``api`` benchmarks just because
+    api depends on sensor.
+
+    Args:
+        branch: Branch to compare against. If None, uses default.
+
+    Returns:
+        True if benchmarks should run, False otherwise.
+    """
+    files = changed_files(branch)
+    if core_changed(files):
+        return True
+
+    # Check if benchmark infrastructure changed
+    if any(
+        f.startswith("tests/benchmarks/") or f in BENCHMARK_INFRASTRUCTURE_FILES
+        for f in files
+    ):
+        return True
+
+    # Check if any directly changed component has benchmarks
+    benchmarks_dir = Path(root_path) / BENCHMARKS_COMPONENTS_PATH
+    if not benchmarks_dir.is_dir():
+        return False
+    benchmarked_components = {
+        d.name
+        for d in benchmarks_dir.iterdir()
+        if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h")))
+    }
+    # Only direct changes — no dependency expansion
+    return any(get_component_from_path(f) in benchmarked_components for f in files)
+
+
 def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) -> bool:
     """Check if a changed file ends with any of the specified extensions."""
     return any(file.endswith(extensions) for file in changed_files(branch))
@@ -804,6 +861,9 @@ def main() -> None:
     # Determine which C++ unit tests to run
     cpp_run_all, cpp_components = determine_cpp_unit_tests(args.branch)
 
+    # Determine if benchmarks should run
+    run_benchmarks = should_run_benchmarks(args.branch)
+
     # Split components into batches for CI testing
     # This intelligently groups components with similar bus configurations
     component_test_batches: list[str]
@@ -856,6 +916,7 @@ def main() -> None:
         "cpp_unit_tests_run_all": cpp_run_all,
         "cpp_unit_tests_components": cpp_components,
         "component_test_batches": component_test_batches,
+        "benchmarks": run_benchmarks,
     }
 
     # Output as JSON

script/setup_codspeed_lib.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""Set up CodSpeed's google_benchmark fork as a PlatformIO library.
+
+CodSpeed requires their codspeed-cpp fork for CPU simulation instrumentation.
+This script clones the repo and assembles a flat PlatformIO-compatible library
+by combining google_benchmark sources, codspeed core, and instrument-hooks.
+
+PlatformIO quirks addressed:
+  - .cc files renamed to .cpp (PlatformIO ignores .cc)
+  - All sources merged into one src/ dir (PlatformIO can't compile from
+    multiple source directories in a single library)
+  - library.json created with required CodSpeed preprocessor defines
+
+Usage:
+    python script/setup_codspeed_lib.py [--output-dir DIR]
+
+Prints JSON to stdout with lib_path for cpp_benchmark.py.
+Git output goes to stderr.
+
+See https://codspeed.io/docs/benchmarks/cpp#custom-build-systems
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+# Pin to a specific release for reproducibility
+CODSPEED_CPP_REPO = "https://github.com/CodSpeedHQ/codspeed-cpp.git"
+CODSPEED_CPP_SHA = "e633aca00da3d0ad14e7bf424d9cb47165a29028"  # v2.1.0
+
+DEFAULT_OUTPUT_DIR = "/tmp/codspeed-cpp"
+
+# Well-known paths within the codspeed-cpp repository
+GOOGLE_BENCHMARK_SUBDIR = "google_benchmark"
+CORE_SUBDIR = "core"
+INSTRUMENT_HOOKS_SUBDIR = Path(CORE_SUBDIR) / "instrument-hooks"
+INSTRUMENT_HOOKS_INCLUDES = INSTRUMENT_HOOKS_SUBDIR / "includes"
+INSTRUMENT_HOOKS_DIST = INSTRUMENT_HOOKS_SUBDIR / "dist" / "core.c"
+CORE_CMAKE = Path(CORE_SUBDIR) / "CMakeLists.txt"
+
+
+def _git(args: list[str], **kwargs: object) -> None:
+    """Run a git command, sending output to stderr."""
+    subprocess.run(
+        ["git", *args],
+        check=True,
+        stdout=kwargs.pop("stdout", sys.stderr),
+        stderr=kwargs.pop("stderr", sys.stderr),
+        **kwargs,
+    )
+
+
+def _clone_repo(output_dir: Path) -> None:
+    """Shallow-clone codspeed-cpp at the pinned SHA with submodules."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    _git(["init", str(output_dir)])
+    _git(["-C", str(output_dir), "remote", "add", "origin", CODSPEED_CPP_REPO])
+    _git(["-C", str(output_dir), "fetch", "--depth", "1", "origin", CODSPEED_CPP_SHA])
+    _git(
+        ["-C", str(output_dir), "checkout", "FETCH_HEAD"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    _git(
+        [
+            "-C",
+            str(output_dir),
+            "submodule",
+            "update",
+            "--init",
+            "--recursive",
+            "--depth",
+            "1",
+        ]
+    )
+
+
+def _read_codspeed_version(cmake_path: Path) -> str:
+    """Extract CODSPEED_VERSION from core/CMakeLists.txt."""
+    if not cmake_path.exists():
+        return "0.0.0"
+    for line in cmake_path.read_text().splitlines():
+        if line.startswith("set(CODSPEED_VERSION"):
+            return line.split()[1].rstrip(")")
+    return "0.0.0"
+
+
+def _rename_cc_to_cpp(src_dir: Path) -> None:
+    """Rename .cc files to .cpp so PlatformIO compiles them."""
+    for cc_file in src_dir.glob("*.cc"):
+        cpp_file = cc_file.with_suffix(".cpp")
+        if not cpp_file.exists():
+            cc_file.rename(cpp_file)
+
+
+def _copy_if_missing(src: Path, dest: Path) -> None:
+    """Copy a file only if the destination doesn't already exist."""
+    if not dest.exists():
+        shutil.copy2(src, dest)
+
+
+def _merge_codspeed_core_into_lib(core_src: Path, lib_src: Path) -> None:
+    """Copy codspeed core sources into the benchmark library src/.
+
+    .cpp files get a ``codspeed_`` prefix to avoid name collisions with
+    google_benchmark's own sources.  .h files keep their original names
+    since they're referenced by ``#include "walltime.h"`` etc.
+    """
+    for src_file in core_src.iterdir():
+        if src_file.suffix == ".cpp":
+            _copy_if_missing(src_file, lib_src / f"codspeed_{src_file.name}")
+        elif src_file.suffix == ".h":
+            _copy_if_missing(src_file, lib_src / src_file.name)
+
+
+def _write_library_json(
+    benchmark_dir: Path,
+    core_include: Path,
+    hooks_include: Path,
+    version: str,
+    project_root: Path,
+) -> None:
+    """Write a PlatformIO library.json with CodSpeed build flags."""
+    library_json = {
+        "name": "benchmark",
+        "version": "0.0.0",
+        "build": {
+            "flags": [
+                f"-I{core_include}",
+                f"-I{hooks_include}",
+                # google benchmark build flags
+                # -O2 is critical: without it, instrument_hooks_start_benchmark_inline
+                # doesn't get inlined and shows up as overhead in profiles
+                "-O2",
+                "-DNDEBUG",
+                "-DHAVE_STD_REGEX",
+                "-DHAVE_STEADY_CLOCK",
+                "-DBENCHMARK_STATIC_DEFINE",
+                # CodSpeed instrumentation flags
+                # https://codspeed.io/docs/benchmarks/cpp#custom-build-systems
+                "-DCODSPEED_ENABLED",
+                "-DCODSPEED_ANALYSIS",
+                f'-DCODSPEED_VERSION=\\"{version}\\"',
+                f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"',
+                '-DCODSPEED_MODE_DISPLAY=\\"simulation\\"',
+            ],
+            "includeDir": "include",
+        },
+    }
+    (benchmark_dir / "library.json").write_text(
+        json.dumps(library_json, indent=2) + "\n"
+    )
+
+
+def setup_codspeed_lib(output_dir: Path) -> None:
+    """Clone codspeed-cpp and assemble a flat PlatformIO library.
+
+    The resulting library at ``output_dir/google_benchmark/`` contains:
+      - google_benchmark sources (.cc renamed to .cpp)
+      - codspeed core sources (prefixed ``codspeed_``)
+      - instrument-hooks C source (as ``instrument_hooks.c``)
+      - library.json with all required CodSpeed defines
+
+    Args:
+        output_dir: Directory to clone the repository into
+    """
+    if not (output_dir / ".git").exists():
+        _clone_repo(output_dir)
+    else:
+        # Verify the existing checkout matches the pinned SHA
+        result = subprocess.run(
+            ["git", "-C", str(output_dir), "rev-parse", "HEAD"],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.returncode != 0 or result.stdout.strip() != CODSPEED_CPP_SHA:
+            print(
+                f"Stale codspeed-cpp checkout, re-cloning at {CODSPEED_CPP_SHA}",
+                file=sys.stderr,
+            )
+            shutil.rmtree(output_dir)
+            _clone_repo(output_dir)
+
+    benchmark_dir = output_dir / GOOGLE_BENCHMARK_SUBDIR
+    lib_src = benchmark_dir / "src"
+    core_dir = output_dir / CORE_SUBDIR
+    core_include = core_dir / "include"
+    hooks_include = output_dir / INSTRUMENT_HOOKS_INCLUDES
+    hooks_dist_c = output_dir / INSTRUMENT_HOOKS_DIST
+    project_root = Path(__file__).resolve().parent.parent
+
+    # 1. Rename .cc → .cpp (PlatformIO doesn't compile .cc)
+    _rename_cc_to_cpp(lib_src)
+
+    # 2. Merge codspeed core sources into the library
+    _merge_codspeed_core_into_lib(core_dir / "src", lib_src)
+
+    # 3. Copy instrument-hooks C source (provides instrument_hooks_* symbols)
+    if hooks_dist_c.exists():
+        _copy_if_missing(hooks_dist_c, lib_src / "instrument_hooks.c")
+
+    # 4. Write library.json
+    version = _read_codspeed_version(output_dir / CORE_CMAKE)
+    _write_library_json(
+        benchmark_dir, core_include, hooks_include, version, project_root
+    )
+
+    # Output JSON config for cpp_benchmark.py
+    print(json.dumps({"lib_path": str(benchmark_dir)}))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=Path(DEFAULT_OUTPUT_DIR),
+        help=f"Directory to clone codspeed-cpp into (default: {DEFAULT_OUTPUT_DIR})",
+    )
+    args = parser.parse_args()
+    setup_codspeed_lib(args.output_dir)
+
+
+if __name__ == "__main__":
+    main()