From 38ff332fec636006153c0ee64717631ed2dee4a7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 19:25:46 -1000 Subject: [PATCH 01/61] [core] Add CodSpeed C++ benchmarks for protobuf and main loop Add automated benchmarks using Google Benchmark to prevent performance regressions in the API protobuf encoding/decoding and core loop paths. Benchmarks cover: - Protobuf encode: SensorState, BinarySensorState, HelloResponse, LightState, DeviceInfoResponse (20 nested devices + 20 areas) - Protobuf decode: HelloRequest, SwitchCommand, LightCommand - Protobuf calculate_size and full calc+encode send path - Varint parse/encode/size for various value ranges - Scheduler call/next_schedule_in with idle and active timers - Application loop component dispatch and blocking guard overhead Infrastructure: - Extract shared build logic from cpp_unit_test.py into test_helpers.py - Add cpp_benchmark.py mirroring the unit test build pattern - Support benchmark.yaml per component dir for declaring dependencies - Add CodSpeed CI workflow triggered on api/core changes - Fix ProtoMessage protected destructor on host platform --- .github/workflows/ci-benchmarks.yml | 87 ++++ esphome/components/api/proto.h | 4 + script/cpp_benchmark.py | 85 ++++ script/cpp_unit_test.py | 245 ++---------- script/test_helpers.py | 375 ++++++++++++++++++ tests/benchmarks/components/.gitignore | 5 + .../components/api/bench_application_loop.cpp | 50 +++ .../components/api/bench_proto_decode.cpp | 119 ++++++ .../components/api/bench_proto_encode.cpp | 210 ++++++++++ .../components/api/bench_proto_varint.cpp | 99 +++++ .../components/api/bench_scheduler.cpp | 63 +++ .../benchmarks/components/api/benchmark.yaml | 114 ++++++ tests/benchmarks/components/main.cpp | 38 ++ 13 files changed, 1280 insertions(+), 214 deletions(-) create mode 100644 .github/workflows/ci-benchmarks.yml create mode 100755 script/cpp_benchmark.py create mode 100644 script/test_helpers.py create mode 100644 tests/benchmarks/components/.gitignore create mode 100644 tests/benchmarks/components/api/bench_application_loop.cpp create mode 100644 tests/benchmarks/components/api/bench_proto_decode.cpp create mode 100644 tests/benchmarks/components/api/bench_proto_encode.cpp create mode 100644 tests/benchmarks/components/api/bench_proto_varint.cpp create mode 100644 tests/benchmarks/components/api/bench_scheduler.cpp create mode 100644 tests/benchmarks/components/api/benchmark.yaml create mode 100644 tests/benchmarks/components/main.cpp diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml new file mode 100644 index 0000000000..416e65bfc2 --- /dev/null +++ b/.github/workflows/ci-benchmarks.yml @@ -0,0 +1,87 @@ +--- +name: CodSpeed Benchmarks + +on: + push: + branches: [dev] + pull_request: + paths: + - "esphome/components/api/**" + - "esphome/core/**" + - "tests/benchmarks/**" + - "script/cpp_benchmark.py" + - "script/test_helpers.py" + - ".github/workflows/ci-benchmarks.yml" + merge_group: + +permissions: + contents: read + +concurrency: + # yamllint disable-line rule:line-length + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + DEFAULT_PYTHON: "3.11" + +jobs: + common: + name: Create common environment + runs-on: ubuntu-24.04 + outputs: + cache-key: ${{ steps.cache-key.outputs.key }} + steps: + - name: Check out code from GitHub + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Generate cache-key + id: cache-key + run: echo key="${{ hashFiles('requirements.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + id: python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: venv + # yamllint disable-line rule:line-length + key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ steps.cache-key.outputs.key }} + - name: Create Python virtual environment + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + python -m venv venv + . venv/bin/activate + python --version + pip install -r requirements.txt -r requirements_test.txt + pip install -e . + + benchmarks: + name: Run CodSpeed Benchmarks + runs-on: ubuntu-24.04 + needs: + - common + steps: + - name: Check out code from GitHub + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + + - name: Build benchmarks + id: build + run: | + . venv/bin/activate + BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1) + echo "binary=$BINARY" >> $GITHUB_OUTPUT + + - name: Run CodSpeed benchmarks + uses: CodSpeedHQ/action@v4 + with: + run: ${{ steps.build.outputs.binary }} + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 6752dfb9cd..d6e993d3a5 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -442,8 +442,12 @@ class ProtoMessage { virtual const char *message_name() const { return "unknown"; } #endif +#ifndef USE_HOST protected: +#endif // Non-virtual destructor is protected to prevent polymorphic deletion. + // On host platform, made public to allow value-initialization of std::array + // members (e.g. DeviceInfoResponse::devices) without clang errors. ~ProtoMessage() = default; }; diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py new file mode 100755 index 0000000000..f630aac3d4 --- /dev/null +++ b/script/cpp_benchmark.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Build and run C++ benchmarks for ESPHome components using Google Benchmark.""" + +import argparse +from pathlib import Path +import sys + +from helpers import root_path +from test_helpers import PLATFORMIO_GOOGLE_BENCHMARK_LIB, build_and_run + +# Path to /tests/benchmarks/components +BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "components" + +# Components whose to_code should run during benchmark builds. +# core/host/logger are infrastructure. json is needed because its +# to_code adds the ArduinoJson library (it's auto-loaded by api but +# cpp_testing suppresses to_code for components not in this set). +BENCHMARK_CODEGEN_COMPONENTS = {"core", "host", "logger", "json"} + +PLATFORMIO_OPTIONS = { + "build_flags": [ + "-DUSE_TIME_TIMEZONE", # enable timezone code paths + "-g", # debug symbols for profiling + ], +} + + +def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int: + return build_and_run( + selected_components=selected_components, + tests_dir=BENCHMARKS_DIR, + codegen_components=BENCHMARK_CODEGEN_COMPONENTS, + config_prefix="cppbench", + friendly_name="CPP Benchmarks", + libraries=PLATFORMIO_GOOGLE_BENCHMARK_LIB, + platformio_options=PLATFORMIO_OPTIONS, + main_entry="main.cpp", + label="benchmarks", + build_only=build_only, + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Build and run C++ benchmarks for ESPHome components." + ) + parser.add_argument( + "components", + nargs="*", + help="List of components to benchmark (must have files in tests/benchmarks/components/).", + ) + parser.add_argument( + "--all", + action="store_true", + help="Benchmark all components with benchmark files.", + ) + parser.add_argument( + "--build-only", + action="store_true", + help="Only build, print binary path without running.", + ) + + args = parser.parse_args() + + if args.all: + # Find all component directories that have .cpp files + components: list[str] = ( + sorted( + d.name + for d in BENCHMARKS_DIR.iterdir() + if d.is_dir() + and d.name != "__pycache__" + and (any(d.glob("*.cpp")) or any(d.glob("*.h"))) + ) + if BENCHMARKS_DIR.is_dir() + else [] + ) + else: + components: list[str] = args.components + + sys.exit(run_benchmarks(components, build_only=args.build_only)) + + +if __name__ == "__main__": + main() diff --git a/script/cpp_unit_test.py b/script/cpp_unit_test.py index c6cfd8270f..c0a6d9647d 100755 --- a/script/cpp_unit_test.py +++ b/script/cpp_unit_test.py @@ -1,22 +1,10 @@ #!/usr/bin/env python3 import argparse -import hashlib -import os from pathlib import Path -import subprocess import sys -from helpers import get_all_components, get_all_dependencies, root_path - -from esphome.__main__ import command_compile, parse_args -from esphome.config import validate_config -from esphome.const import CONF_PLATFORM -from esphome.core import CORE -from esphome.loader import get_component -from esphome.platformio_api import get_idedata - -# This must coincide with the version in /platformio.ini -PLATFORMIO_GOOGLE_TEST_LIB = "google/googletest@^1.15.2" +from helpers import get_all_components, root_path +from test_helpers import PLATFORMIO_GOOGLE_TEST_LIB, build_and_run # Path to /tests/components COMPONENTS_TESTS_DIR: Path = Path(root_path) / "tests" / "components" @@ -28,211 +16,40 @@ COMPONENTS_TESTS_DIR: Path = Path(root_path) / "tests" / "components" # which registers under package name "core" not "esphome". CPP_TESTING_CODEGEN_COMPONENTS = {"core", "host", "logger"} - -def hash_components(components: list[str]) -> str: - key = ",".join(components) - return hashlib.sha256(key.encode()).hexdigest()[:16] - - -def filter_components_without_tests(components: list[str]) -> list[str]: - """Filter out components that do not have a corresponding test file. - - This is done by checking if the component's directory contains at - least a .cpp or .h file. - """ - filtered_components: list[str] = [] - for component in components: - test_dir = COMPONENTS_TESTS_DIR / component - if test_dir.is_dir() and ( - any(test_dir.glob("*.cpp")) or any(test_dir.glob("*.h")) - ): - filtered_components.append(component) - else: - print( - f"WARNING: No tests found for component '{component}', skipping.", - file=sys.stderr, - ) - return filtered_components - - -def create_test_config(config_name: str, includes: list[str]) -> dict: - """Create ESPHome test configuration for C++ unit tests. - - Args: - config_name: Unique name for this test configuration - includes: List of include folders for the test build - - Returns: - Configuration dict for ESPHome - """ - return { - "esphome": { - "name": config_name, - "friendly_name": "CPP Unit Tests", - "libraries": PLATFORMIO_GOOGLE_TEST_LIB, - "platformio_options": { - "build_type": "debug", - "build_unflags": [ - "-Os", # remove size-opt flag - ], - "build_flags": [ - "-Og", # optimize for debug - "-DUSE_TIME_TIMEZONE", # enable timezone code paths for testing - "-DESPHOME_DEBUG", # enable debug assertions - # Enable the address and undefined behavior sanitizers - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-omit-frame-pointer", - ], - "debug_build_flags": [ # only for debug builds - "-g3", # max debug info - "-ggdb3", - ], - }, - "includes": includes, - }, - "host": {}, - "logger": {"level": "DEBUG"}, - } - - -def get_platform_components(components: list[str]) -> list[str]: - """Discover platform sub-components referenced by test directory structure. - - For each component being tested, any sub-directory named after a platform - domain (e.g. ``sensor``, ``binary_sensor``) is treated as a request to - include that ``.`` platform in the build. The sub- - directory must name a valid platform domain; anything else raises an error - so that typos are caught early. - - Returns: - List of ``"domain.component"`` strings, one per discovered sub-directory. - """ - platform_components: list[str] = [] - for component in components: - test_dir = COMPONENTS_TESTS_DIR / component - if not test_dir.is_dir(): - continue - # Each sub-directory name is expected to be a platform domain - # (e.g. tests/components/bthome/sensor/ → sensor.bthome). - for domain_dir in test_dir.iterdir(): - if not domain_dir.is_dir(): - continue - domain = domain_dir.name - domain_module = get_component(domain) - if domain_module is None or not domain_module.is_platform_component: - raise ValueError( - f"Component tests for '{component}' reference non-existing or invalid domain '{domain}'" - f" in its directory structure. See ({COMPONENTS_TESTS_DIR / component / domain})." - ) - platform_components.append(f"{domain}.{component}") - return platform_components - - -# Exit codes for run_tests -EXIT_OK = 0 -EXIT_SKIPPED = 1 -EXIT_COMPILE_ERROR = 2 -EXIT_CONFIG_ERROR = 3 -EXIT_NO_EXECUTABLE = 4 +PLATFORMIO_OPTIONS = { + "build_type": "debug", + "build_unflags": [ + "-Os", # remove size-opt flag + ], + "build_flags": [ + "-Og", # optimize for debug + "-DUSE_TIME_TIMEZONE", # enable timezone code paths for testing + "-DESPHOME_DEBUG", # enable debug assertions + # Enable the address and undefined behavior sanitizers + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-omit-frame-pointer", + ], + "debug_build_flags": [ # only for debug builds + "-g3", # max debug info + "-ggdb3", + ], +} def run_tests(selected_components: list[str]) -> int: - # Skip tests on Windows - if os.name == "nt": - print("Skipping esphome tests on Windows", file=sys.stderr) - return EXIT_SKIPPED - - # Remove components that do not have tests - components = filter_components_without_tests(selected_components) - - if len(components) == 0: - print( - "No components specified or no tests found for the specified components.", - file=sys.stderr, - ) - return EXIT_OK - - components = sorted(components) - - # Build a list of include folders relative to COMPONENTS_TESTS_DIR. These folders will - # be added along with their subfolders. - # "main.cpp" is a special entry that points to /tests/components/main.cpp, - # which provides a custom test runner entry-point replacing the default one. - # Each remaining entry is a component folder whose *.cpp files are compiled. - includes: list[str] = ["main.cpp"] + components - - # Obtain a list of platform components to be tested: - try: - platform_components = get_platform_components(components) - except ValueError as e: - print(f"Error obtaining platform components: {e}") - return EXIT_CONFIG_ERROR - - components = sorted(components + platform_components) - - # Create a unique name for this config based on the actual components being tested - # to maximize cache during testing - config_name: str = "cpptests-" + hash_components(components) - - # Obtain possible dependencies for the requested components. - # Always include 'time' because USE_TIME_TIMEZONE is defined as a build flag, - # which causes core/time.h to include components/time/posix_tz.h. - components_with_dependencies: list[str] = sorted( - get_all_dependencies(set(components) | {"time"}, cpp_testing=True) + return build_and_run( + selected_components=selected_components, + tests_dir=COMPONENTS_TESTS_DIR, + codegen_components=CPP_TESTING_CODEGEN_COMPONENTS, + config_prefix="cpptests", + friendly_name="CPP Unit Tests", + libraries=PLATFORMIO_GOOGLE_TEST_LIB, + platformio_options=PLATFORMIO_OPTIONS, + main_entry="main.cpp", + label="unit tests", ) - config = create_test_config(config_name, includes) - - CORE.config_path = COMPONENTS_TESTS_DIR / "dummy.yaml" - CORE.dashboard = None - CORE.cpp_testing = True - CORE.cpp_testing_codegen = CPP_TESTING_CODEGEN_COMPONENTS - - # Validate config will expand the above with defaults: - config = validate_config(config, {}) - - # Add all components and dependencies to the base configuration after validation, so their files - # are added to the build. - for component_name in components_with_dependencies: - if "." in component_name: - # Format is always "domain.component" (exactly one dot), - # as produced by get_platform_components(). - domain, component = component_name.split(".", maxsplit=1) - domain_list = config.setdefault(domain, []) - CORE.testing_ensure_platform_registered(domain) - domain_list.append({CONF_PLATFORM: component}) - else: - config.setdefault(component_name, []) - - dependencies = set(components_with_dependencies) - set(components) - deps_str = ", ".join(dependencies) if dependencies else "None" - print(f"Testing components: {', '.join(components)}. Dependencies: {deps_str}") - CORE.config = config - args = parse_args(["program", "compile", str(CORE.config_path)]) - try: - exit_code: int = command_compile(args, config) - - if exit_code != 0: - print(f"Error compiling unit tests for {', '.join(components)}") - return exit_code - except Exception as e: - print( - f"Error compiling unit tests for {', '.join(components)}. Check path. : {e}" - ) - return EXIT_COMPILE_ERROR - - # After a successful compilation, locate the executable and run it: - idedata = get_idedata(config) - if idedata is None: - print("Cannot find executable") - return EXIT_NO_EXECUTABLE - - program_path: str = idedata.raw["prog_path"] - run_cmd: list[str] = [program_path] - run_proc = subprocess.run(run_cmd, check=False) - return run_proc.returncode - def main() -> None: parser = argparse.ArgumentParser( diff --git a/script/test_helpers.py b/script/test_helpers.py new file mode 100644 index 0000000000..1941168daf --- /dev/null +++ b/script/test_helpers.py @@ -0,0 +1,375 @@ +"""Shared helpers for C++ unit test and benchmark build scripts.""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import subprocess +import sys + +from helpers import get_all_dependencies +import yaml + +from esphome.__main__ import command_compile, parse_args +from esphome.config import validate_config +from esphome.const import CONF_PLATFORM +from esphome.core import CORE +from esphome.loader import get_component +from esphome.platformio_api import get_idedata + +# This must coincide with the version in /platformio.ini +PLATFORMIO_GOOGLE_TEST_LIB = "google/googletest@^1.15.2" + +# Google Benchmark library for PlatformIO +# Format: name=repository_url (see esphome/core/config.py library parsing) +PLATFORMIO_GOOGLE_BENCHMARK_LIB = ( + "benchmark=https://github.com/google/benchmark.git#v1.9.1" +) + +# Key names for the base config sections +ESPHOME_KEY = "esphome" +HOST_KEY = "host" +LOGGER_KEY = "logger" + +# Base config keys that are always present and must not be fully overridden +# by component benchmark.yaml files. esphome: allows sub-key merging. +BASE_CONFIG_KEYS = frozenset({ESPHOME_KEY, HOST_KEY, LOGGER_KEY}) + +# Exit codes +EXIT_OK = 0 +EXIT_SKIPPED = 1 +EXIT_COMPILE_ERROR = 2 +EXIT_CONFIG_ERROR = 3 +EXIT_NO_EXECUTABLE = 4 + +# Name of the per-component YAML config file in benchmark directories +BENCHMARK_YAML_FILENAME = "benchmark.yaml" + + +def hash_components(components: list[str]) -> str: + """Create a short hash of component names for unique config naming.""" + key = ",".join(components) + return hashlib.sha256(key.encode()).hexdigest()[:16] + + +def filter_components_with_files(components: list[str], tests_dir: Path) -> list[str]: + """Filter out components that do not have .cpp or .h files in the tests dir. + + Args: + components: List of component names to check + tests_dir: Base directory containing component test/benchmark folders + + Returns: + Filtered list of components that have test files + """ + filtered_components: list[str] = [] + for component in components: + test_dir = tests_dir / component + if test_dir.is_dir() and ( + any(test_dir.glob("*.cpp")) or any(test_dir.glob("*.h")) + ): + filtered_components.append(component) + else: + print( + f"WARNING: No files found for component '{component}' in {tests_dir}, skipping.", + file=sys.stderr, + ) + return filtered_components + + +def get_platform_components(components: list[str], tests_dir: Path) -> list[str]: + """Discover platform sub-components referenced by test directory structure. + + For each component, any sub-directory named after a platform domain + (e.g. ``sensor``, ``binary_sensor``) is treated as a request to include + that ``.`` platform in the build. + + Args: + components: List of component names to scan + tests_dir: Base directory containing component test/benchmark folders + + Returns: + List of ``"domain.component"`` strings + """ + platform_components: list[str] = [] + for component in components: + test_dir = tests_dir / component + if not test_dir.is_dir(): + continue + for domain_dir in test_dir.iterdir(): + if not domain_dir.is_dir(): + continue + domain = domain_dir.name + domain_module = get_component(domain) + if domain_module is None or not domain_module.is_platform_component: + raise ValueError( + f"Component '{component}' references non-existing or invalid domain '{domain}'" + f" in its directory structure. See ({tests_dir / component / domain})." + ) + platform_components.append(f"{domain}.{component}") + return platform_components + + +def load_component_yaml_configs(components: list[str], tests_dir: Path) -> dict: + """Load and merge benchmark.yaml files from component directories. + + Each component directory may contain a ``benchmark.yaml`` file that + declares additional ESPHome components needed for the build (e.g. + ``api:``, ``sensor:``). These get merged into the base config before + validation so that dependencies are properly resolved with defaults. + + The ``esphome:`` key is special: its sub-keys are merged into the + existing esphome config (e.g. to add ``areas:`` or ``devices:``). + Other base config keys (``host:``, ``logger:``) are not overridable. + + Args: + components: List of component directory names + tests_dir: Base directory containing component folders + + Returns: + Merged dict of component configs to add to the base config + """ + merged: dict = {} + for component in components: + yaml_path = tests_dir / component / BENCHMARK_YAML_FILENAME + if not yaml_path.is_file(): + continue + with open(yaml_path) as f: + component_config = yaml.safe_load(f) + if component_config and isinstance(component_config, dict): + for key, value in component_config.items(): + if key in BASE_CONFIG_KEYS - {ESPHOME_KEY}: + # host: and logger: are not overridable + continue + if key == ESPHOME_KEY and isinstance(value, dict): + # Merge esphome sub-keys rather than replacing + esphome_extra = merged.setdefault(ESPHOME_KEY, {}) + for sub_key, sub_value in value.items(): + esphome_extra.setdefault(sub_key, sub_value) + continue + merged.setdefault(key, value) + return merged + + +def create_host_config( + config_name: str, + friendly_name: str, + libraries: str | list[str], + includes: list[str], + platformio_options: dict, +) -> dict: + """Create an ESPHome host configuration for C++ builds. + + Args: + config_name: Unique name for this configuration + friendly_name: Human-readable name + libraries: PlatformIO library specification(s) + includes: List of include folders for the build + platformio_options: Dict of platformio_options to set + + Returns: + Configuration dict for ESPHome + """ + return { + ESPHOME_KEY: { + "name": config_name, + "friendly_name": friendly_name, + "libraries": libraries, + "platformio_options": platformio_options, + "includes": includes, + }, + HOST_KEY: {}, + LOGGER_KEY: {"level": "DEBUG"}, + } + + +def compile_and_get_binary( + config: dict, + components: list[str], + codegen_components: set[str], + tests_dir: Path, + label: str = "build", +) -> tuple[int, str | None]: + """Compile an ESPHome configuration and return the binary path. + + Args: + config: ESPHome configuration dict (already created via create_host_config) + components: List of components to include in the build + codegen_components: Set of component names whose to_code should run + tests_dir: Base directory for test files (used as config_path base) + label: Label for log messages (e.g. "unit tests", "benchmarks") + + Returns: + Tuple of (exit_code, program_path_or_none) + """ + # Load any benchmark.yaml files from component directories and merge + # them into the config BEFORE dependency resolution and validation. + # This allows each benchmark/test dir to declare which ESPHome components + # it needs (e.g. api:) so they get proper config defaults. + extra_config = load_component_yaml_configs(components, tests_dir) + for key, value in extra_config.items(): + if key == ESPHOME_KEY and isinstance(value, dict): + # Merge esphome sub-keys into existing esphome config. + # For list values (e.g. libraries), extend rather than replace. + for sub_key, sub_value in value.items(): + existing = config[ESPHOME_KEY].get(sub_key) + if existing is not None and isinstance(sub_value, list): + # Ensure existing is a list, then extend + if not isinstance(existing, list): + config[ESPHOME_KEY][sub_key] = [existing] + config[ESPHOME_KEY][sub_key].extend(sub_value) + else: + config[ESPHOME_KEY].setdefault(sub_key, sub_value) + else: + config.setdefault(key, value) + + # Obtain possible dependencies BEFORE validate_config, because + # get_all_dependencies calls CORE.reset() which clears build_path. + # Always include 'time' because USE_TIME_TIMEZONE is defined as a build flag, + # which causes core/time.h to include components/time/posix_tz.h. + components_with_dependencies: list[str] = sorted( + get_all_dependencies(set(components) | {"time"}, cpp_testing=True) + ) + + CORE.config_path = tests_dir / "dummy.yaml" + CORE.dashboard = None + CORE.cpp_testing = True + CORE.cpp_testing_codegen = codegen_components + + # Validate config will expand the above with defaults: + config = validate_config(config, {}) + + # Add remaining components and dependencies to the configuration after + # validation, so their source files are included in the build. + for component_name in components_with_dependencies: + if "." in component_name: + domain, component = component_name.split(".", maxsplit=1) + domain_list = config.setdefault(domain, []) + CORE.testing_ensure_platform_registered(domain) + domain_list.append({CONF_PLATFORM: component}) + else: + config.setdefault(component_name, []) + + # Register platforms from the extra config (benchmark.yaml) so + # USE_SENSOR, USE_LIGHT, etc. defines are emitted without needing + # real entity instances. + for key in extra_config: + if key == ESPHOME_KEY: + continue + comp = get_component(key) + if comp is not None and comp.is_platform_component: + CORE.testing_ensure_platform_registered(key) + + dependencies = set(components_with_dependencies) - set(components) + deps_str = ", ".join(dependencies) if dependencies else "None" + print(f"Building {label}: {', '.join(components)}. Dependencies: {deps_str}") + CORE.config = config + args = parse_args(["program", "compile", str(CORE.config_path)]) + try: + exit_code: int = command_compile(args, config) + + if exit_code != 0: + print(f"Error compiling {label} for {', '.join(components)}") + return exit_code, None + except Exception as e: + print(f"Error compiling {label} for {', '.join(components)}. Check path. : {e}") + return EXIT_COMPILE_ERROR, None + + # After a successful compilation, locate the executable: + idedata = get_idedata(config) + if idedata is None: + print("Cannot find executable") + return EXIT_NO_EXECUTABLE, None + + program_path: str = idedata.raw["prog_path"] + return EXIT_OK, program_path + + +def build_and_run( + selected_components: list[str], + tests_dir: Path, + codegen_components: set[str], + config_prefix: str, + friendly_name: str, + libraries: str | list[str], + platformio_options: dict, + main_entry: str, + label: str = "build", + build_only: bool = False, + extra_run_args: list[str] | None = None, +) -> int: + """Build and optionally run a C++ test/benchmark binary. + + This is the main orchestration function shared between unit tests + and benchmarks. + + Args: + selected_components: Components to include (directory names in tests_dir) + tests_dir: Directory containing test/benchmark files + codegen_components: Components whose to_code should run + config_prefix: Prefix for the config name (e.g. "cpptests", "cppbench") + friendly_name: Human-readable name for the config + libraries: PlatformIO library specification(s) + platformio_options: PlatformIO options dict + main_entry: Name of the main entry file (e.g. "main.cpp") + label: Label for log messages + build_only: If True, print binary path and return without running + extra_run_args: Extra arguments to pass to the binary + + Returns: + Exit code + """ + # Skip on Windows + if os.name == "nt": + print(f"Skipping {label} on Windows", file=sys.stderr) + return EXIT_SKIPPED + + # Remove components that do not have files + components = filter_components_with_files(selected_components, tests_dir) + + if len(components) == 0: + print( + f"No components specified or no files found for {label}.", + file=sys.stderr, + ) + return EXIT_OK + + components = sorted(components) + + # Build include list: main entry point + component folders + includes: list[str] = [main_entry] + components + + # Discover platform sub-components + try: + platform_components = get_platform_components(components, tests_dir) + except ValueError as e: + print(f"Error obtaining platform components: {e}") + return EXIT_CONFIG_ERROR + + components = sorted(components + platform_components) + + # Create unique config name + config_name: str = f"{config_prefix}-" + hash_components(components) + + config = create_host_config( + config_name, friendly_name, libraries, includes, platformio_options + ) + + exit_code, program_path = compile_and_get_binary( + config, components, codegen_components, tests_dir, label + ) + + if exit_code != EXIT_OK or program_path is None: + return exit_code + + if build_only: + print(program_path) + return EXIT_OK + + # Run the binary + run_cmd: list[str] = [program_path] + if extra_run_args: + run_cmd.extend(extra_run_args) + run_proc = subprocess.run(run_cmd, check=False) + return run_proc.returncode diff --git a/tests/benchmarks/components/.gitignore b/tests/benchmarks/components/.gitignore new file mode 100644 index 0000000000..d8b4157aef --- /dev/null +++ b/tests/benchmarks/components/.gitignore @@ -0,0 +1,5 @@ +# Gitignore settings for ESPHome +# This is an example and may include too much for your use-case. +# You can modify this file to suit your needs. +/.esphome/ +/secrets.yaml diff --git a/tests/benchmarks/components/api/bench_application_loop.cpp b/tests/benchmarks/components/api/bench_application_loop.cpp new file mode 100644 index 0000000000..2489d4e9ac --- /dev/null +++ b/tests/benchmarks/components/api/bench_application_loop.cpp @@ -0,0 +1,50 @@ +#include + +#include "esphome/core/application.h" +#include "esphome/core/component.h" +#include "esphome/core/hal.h" + +namespace esphome::benchmarks { + +// A minimal component with an empty loop for benchmarking dispatch overhead +class NoopComponent : public Component { + public: + void loop() override {} + float get_setup_priority() const override { return 0.0f; } + // Expose protected method for benchmarking + void set_loop_state() { this->set_component_state_(COMPONENT_STATE_LOOP); } +}; + +// --- WarnIfComponentBlockingGuard overhead --- + +static void BM_WarnIfComponentBlockingGuard(benchmark::State &state) { + NoopComponent component; + uint32_t now = millis(); + + for (auto _ : state) { + WarnIfComponentBlockingGuard guard{&component, now}; + now = guard.finish(); + benchmark::DoNotOptimize(now); + } +} +BENCHMARK(BM_WarnIfComponentBlockingGuard); + +// --- Component virtual dispatch overhead --- + +static void BM_ComponentDispatch(benchmark::State &state) { + NoopComponent component; + component.set_loop_state(); + uint32_t now = millis(); + + for (auto _ : state) { + { + WarnIfComponentBlockingGuard guard{&component, now}; + component.loop(); + now = guard.finish(); + } + benchmark::DoNotOptimize(now); + } +} +BENCHMARK(BM_ComponentDispatch); + +} // namespace esphome::benchmarks diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp new file mode 100644 index 0000000000..8b7215497d --- /dev/null +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -0,0 +1,119 @@ +#include + +#include "esphome/components/api/api_pb2.h" +#include "esphome/components/api/api_buffer.h" + +namespace esphome::api::benchmarks { + +// --- HelloRequest decode (string + varint fields) --- + +static void BM_Decode_HelloRequest(benchmark::State &state) { + // Manually encoded HelloRequest: + // field 1 (string): "aioesphomeapi" + // field 2 (varint): 1 (api_version_major) + // field 3 (varint): 10 (api_version_minor) + uint8_t encoded[] = { + 0x0A, 0x0D, // field 1, length 13 + 'a', 'i', 'o', 'e', 's', 'p', 'h', 'o', 'm', 'e', 'a', 'p', 'i', // "aioesphomeapi" + 0x10, 0x01, // field 2, value 1 + 0x18, 0x0A, // field 3, value 10 + }; + + for (auto _ : state) { + HelloRequest msg; + msg.decode(encoded, sizeof(encoded)); + benchmark::DoNotOptimize(msg.api_version_major); + } +} +BENCHMARK(BM_Decode_HelloRequest); + +// --- SwitchCommandRequest decode (simple command) --- + +static void BM_Decode_SwitchCommandRequest(benchmark::State &state) { + // field 1 (fixed32): key = 0x12345678 + // field 2 (varint): state = true + uint8_t encoded[] = { + 0x0D, 0x78, 0x56, 0x34, 0x12, // field 1, fixed32 + 0x10, 0x01, // field 2, varint true + }; + + for (auto _ : state) { + SwitchCommandRequest msg; + msg.decode(encoded, sizeof(encoded)); + benchmark::DoNotOptimize(msg.state); + } +} +BENCHMARK(BM_Decode_SwitchCommandRequest); + +// --- LightCommandRequest decode (complex command with many fields) --- + +static void BM_Decode_LightCommandRequest(benchmark::State &state) { + uint8_t encoded[] = { + // field 1: key (fixed32) = 0x11223344 + 0x0D, + 0x44, + 0x33, + 0x22, + 0x11, + // field 2: has_state (varint) = true + 0x10, + 0x01, + // field 3: state (varint) = true + 0x18, + 0x01, + // field 4: has_brightness (varint) = true + 0x20, + 0x01, + // field 5: brightness (fixed32/float) = 0.8 + 0x2D, + 0xCD, + 0xCC, + 0x4C, + 0x3F, + // field 9: has_rgb (varint) = true + 0x48, + 0x01, + // field 10: red (fixed32/float) = 1.0 + 0x55, + 0x00, + 0x00, + 0x80, + 0x3F, + // field 11: green (fixed32/float) = 0.5 + 0x5D, + 0x00, + 0x00, + 0x00, + 0x3F, + // field 12: blue (fixed32/float) = 0.2 + 0x65, + 0xCD, + 0xCC, + 0x4C, + 0x3E, + // field 20: has_effect (varint) = true + 0xA0, + 0x01, + 0x01, + // field 21: effect (string) = "rainbow" + 0xAA, + 0x01, + 0x07, + 'r', + 'a', + 'i', + 'n', + 'b', + 'o', + 'w', + }; + + for (auto _ : state) { + LightCommandRequest msg; + msg.decode(encoded, sizeof(encoded)); + benchmark::DoNotOptimize(msg.brightness); + } +} +BENCHMARK(BM_Decode_LightCommandRequest); + +} // namespace esphome::api::benchmarks diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp new file mode 100644 index 0000000000..b5a89864e7 --- /dev/null +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -0,0 +1,210 @@ +#include + +#include "esphome/components/api/api_pb2.h" +#include "esphome/components/api/api_buffer.h" + +namespace esphome::api::benchmarks { + +// --- SensorStateResponse (highest frequency message) --- + +static void BM_Encode_SensorStateResponse(benchmark::State &state) { + APIBuffer buffer; + SensorStateResponse msg; + msg.key = 0x12345678; + msg.state = 23.5f; + msg.missing_state = false; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_SensorStateResponse); + +static void BM_CalculateSize_SensorStateResponse(benchmark::State &state) { + SensorStateResponse msg; + msg.key = 0x12345678; + msg.state = 23.5f; + msg.missing_state = false; + + for (auto _ : state) { + benchmark::DoNotOptimize(msg.calculate_size()); + } +} +BENCHMARK(BM_CalculateSize_SensorStateResponse); + +static void BM_CalcAndEncode_SensorStateResponse(benchmark::State &state) { + APIBuffer buffer; + SensorStateResponse msg; + msg.key = 0x12345678; + msg.state = 23.5f; + msg.missing_state = false; + + for (auto _ : state) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_CalcAndEncode_SensorStateResponse); + +// --- BinarySensorStateResponse --- + +static void BM_Encode_BinarySensorStateResponse(benchmark::State &state) { + APIBuffer buffer; + BinarySensorStateResponse msg; + msg.key = 0xAABBCCDD; + msg.state = true; + msg.missing_state = false; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_BinarySensorStateResponse); + +// --- HelloResponse (string fields) --- + +static void BM_Encode_HelloResponse(benchmark::State &state) { + APIBuffer buffer; + HelloResponse msg; + msg.api_version_major = 1; + msg.api_version_minor = 10; + msg.server_info = StringRef::from_lit("esphome v2026.3.0"); + msg.name = StringRef::from_lit("living-room-sensor"); + uint32_t size = msg.calculate_size(); + buffer.resize(size); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_HelloResponse); + +// --- LightStateResponse (complex multi-field message) --- + +static void BM_Encode_LightStateResponse(benchmark::State &state) { + APIBuffer buffer; + LightStateResponse msg; + msg.key = 0x11223344; + msg.state = true; + msg.brightness = 0.8f; + msg.color_mode = enums::COLOR_MODE_RGB_WHITE; + msg.color_brightness = 1.0f; + msg.red = 1.0f; + msg.green = 0.5f; + msg.blue = 0.2f; + msg.white = 0.0f; + msg.color_temperature = 4000.0f; + msg.cold_white = 0.0f; + msg.warm_white = 0.0f; + msg.effect = StringRef::from_lit("rainbow"); + uint32_t size = msg.calculate_size(); + buffer.resize(size); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_LightStateResponse); + +static void BM_CalculateSize_LightStateResponse(benchmark::State &state) { + LightStateResponse msg; + msg.key = 0x11223344; + msg.state = true; + msg.brightness = 0.8f; + msg.color_mode = enums::COLOR_MODE_RGB_WHITE; + msg.color_brightness = 1.0f; + msg.red = 1.0f; + msg.green = 0.5f; + msg.blue = 0.2f; + msg.white = 0.0f; + msg.color_temperature = 4000.0f; + msg.cold_white = 0.0f; + msg.warm_white = 0.0f; + msg.effect = StringRef::from_lit("rainbow"); + + for (auto _ : state) { + benchmark::DoNotOptimize(msg.calculate_size()); + } +} +BENCHMARK(BM_CalculateSize_LightStateResponse); + +// --- DeviceInfoResponse (nested submessages: 20 devices + 20 areas) --- + +static DeviceInfoResponse make_device_info_response() { + DeviceInfoResponse msg; + msg.name = StringRef::from_lit("living-room-sensor"); + msg.mac_address = StringRef::from_lit("AA:BB:CC:DD:EE:FF"); + msg.esphome_version = StringRef::from_lit("2026.3.0"); + msg.compilation_time = StringRef::from_lit("Mar 16 2026, 12:00:00"); + msg.model = StringRef::from_lit("esp32-poe-iso"); + msg.manufacturer = StringRef::from_lit("Olimex"); + msg.friendly_name = StringRef::from_lit("Living Room Sensor"); +#ifdef USE_DEVICES + for (uint32_t i = 0; i < ESPHOME_DEVICE_COUNT && i < 20; i++) { + msg.devices[i].device_id = i + 1; + msg.devices[i].name = StringRef::from_lit("device"); + msg.devices[i].area_id = (i % 20) + 1; + } +#endif +#ifdef USE_AREAS + for (uint32_t i = 0; i < ESPHOME_AREA_COUNT && i < 20; i++) { + msg.areas[i].area_id = i + 1; + msg.areas[i].name = StringRef::from_lit("area"); + } +#endif + return msg; +} + +static void BM_CalculateSize_DeviceInfoResponse(benchmark::State &state) { + auto msg = make_device_info_response(); + + for (auto _ : state) { + benchmark::DoNotOptimize(msg.calculate_size()); + } +} +BENCHMARK(BM_CalculateSize_DeviceInfoResponse); + +static void BM_Encode_DeviceInfoResponse(benchmark::State &state) { + auto msg = make_device_info_response(); + APIBuffer buffer; + uint32_t total_size = msg.calculate_size(); + buffer.resize(total_size); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_DeviceInfoResponse); + +static void BM_CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { + auto msg = make_device_info_response(); + APIBuffer buffer; + + for (auto _ : state) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_CalcAndEncode_DeviceInfoResponse); + +} // namespace esphome::api::benchmarks diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp new file mode 100644 index 0000000000..dbe03cafb5 --- /dev/null +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -0,0 +1,99 @@ +#include + +#include "esphome/components/api/proto.h" +#include "esphome/components/api/api_buffer.h" + +namespace esphome::api::benchmarks { + +// --- ProtoVarInt::parse() benchmarks --- + +static void BM_ProtoVarInt_Parse_SingleByte(benchmark::State &state) { + // Single-byte varint (0-127) — the most common case (fast path) + uint8_t buf[] = {0x42}; // value = 66 + + for (auto _ : state) { + auto result = ProtoVarInt::parse(buf, sizeof(buf)); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK(BM_ProtoVarInt_Parse_SingleByte); + +static void BM_ProtoVarInt_Parse_TwoByte(benchmark::State &state) { + // Two-byte varint (128-16383) + uint8_t buf[] = {0x80, 0x01}; // value = 128 + + for (auto _ : state) { + auto result = ProtoVarInt::parse(buf, sizeof(buf)); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK(BM_ProtoVarInt_Parse_TwoByte); + +static void BM_ProtoVarInt_Parse_FiveByte(benchmark::State &state) { + // Five-byte varint (max uint32 = 4294967295) + uint8_t buf[] = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F}; + + for (auto _ : state) { + auto result = ProtoVarInt::parse(buf, sizeof(buf)); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK(BM_ProtoVarInt_Parse_FiveByte); + +// --- Varint encoding benchmarks --- + +static void BM_Encode_Varint_Small(benchmark::State &state) { + // Value < 128 — single byte fast path + APIBuffer buffer; + buffer.resize(16); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(42); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_Varint_Small); + +static void BM_Encode_Varint_Large(benchmark::State &state) { + // Value > 128 — multi-byte slow path + APIBuffer buffer; + buffer.resize(16); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(300); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_Varint_Large); + +static void BM_Encode_Varint_MaxUint32(benchmark::State &state) { + APIBuffer buffer; + buffer.resize(16); + + for (auto _ : state) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(0xFFFFFFFF); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(BM_Encode_Varint_MaxUint32); + +// --- ProtoSize::varint() benchmarks --- + +static void BM_ProtoSize_Varint_Small(benchmark::State &state) { + for (auto _ : state) { + benchmark::DoNotOptimize(ProtoSize::varint(42)); + } +} +BENCHMARK(BM_ProtoSize_Varint_Small); + +static void BM_ProtoSize_Varint_Large(benchmark::State &state) { + for (auto _ : state) { + benchmark::DoNotOptimize(ProtoSize::varint(0xFFFFFFFF)); + } +} +BENCHMARK(BM_ProtoSize_Varint_Large); + +} // namespace esphome::api::benchmarks diff --git a/tests/benchmarks/components/api/bench_scheduler.cpp b/tests/benchmarks/components/api/bench_scheduler.cpp new file mode 100644 index 0000000000..0bd343d94b --- /dev/null +++ b/tests/benchmarks/components/api/bench_scheduler.cpp @@ -0,0 +1,63 @@ +#include + +#include "esphome/core/scheduler.h" +#include "esphome/core/hal.h" + +namespace esphome::benchmarks { + +// --- Scheduler fast path: no work to do --- + +static void BM_Scheduler_Call_NoWork(benchmark::State &state) { + Scheduler scheduler; + uint32_t now = millis(); + + for (auto _ : state) { + scheduler.call(now); + benchmark::DoNotOptimize(now); + } +} +BENCHMARK(BM_Scheduler_Call_NoWork); + +// --- Scheduler with timers: call() when timers exist but aren't due --- + +static void BM_Scheduler_Call_TimersNotDue(benchmark::State &state) { + Scheduler scheduler; + Component dummy_component; + + // Add some timeouts far in the future + for (int i = 0; i < 10; i++) { + scheduler.set_timeout(&dummy_component, static_cast(i), 1000000, []() {}); + } + scheduler.process_to_add(); + + uint32_t now = millis(); + + for (auto _ : state) { + scheduler.call(now); + benchmark::DoNotOptimize(now); + } +} +BENCHMARK(BM_Scheduler_Call_TimersNotDue); + +// --- Scheduler: next_schedule_in() calculation --- + +static void BM_Scheduler_NextScheduleIn(benchmark::State &state) { + Scheduler scheduler; + Component dummy_component; + + // Add some timeouts + for (int i = 0; i < 10; i++) { + scheduler.set_timeout(&dummy_component, static_cast(i), 1000 * (i + 1), []() {}); + } + scheduler.process_to_add(); + + uint32_t now = millis(); + + for (auto _ : state) { + auto result = scheduler.next_schedule_in(now); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK(BM_Scheduler_NextScheduleIn); + +} // namespace esphome::benchmarks diff --git a/tests/benchmarks/components/api/benchmark.yaml b/tests/benchmarks/components/api/benchmark.yaml new file mode 100644 index 0000000000..bfc24d7440 --- /dev/null +++ b/tests/benchmarks/components/api/benchmark.yaml @@ -0,0 +1,114 @@ +# Components needed for API protobuf benchmarks. +# Merged into the base config before validation so all +# dependencies get proper defaults. +# +# esphome: sub-keys are merged into the base config. +esphome: + areas: + - id: area_1 + name: "Area 1" + - id: area_2 + name: "Area 2" + - id: area_3 + name: "Area 3" + - id: area_4 + name: "Area 4" + - id: area_5 + name: "Area 5" + - id: area_6 + name: "Area 6" + - id: area_7 + name: "Area 7" + - id: area_8 + name: "Area 8" + - id: area_9 + name: "Area 9" + - id: area_10 + name: "Area 10" + - id: area_11 + name: "Area 11" + - id: area_12 + name: "Area 12" + - id: area_13 + name: "Area 13" + - id: area_14 + name: "Area 14" + - id: area_15 + name: "Area 15" + - id: area_16 + name: "Area 16" + - id: area_17 + name: "Area 17" + - id: area_18 + name: "Area 18" + - id: area_19 + name: "Area 19" + - id: area_20 + name: "Area 20" + devices: + - id: device_1 + name: "Device 1" + area_id: area_1 + - id: device_2 + name: "Device 2" + area_id: area_2 + - id: device_3 + name: "Device 3" + area_id: area_3 + - id: device_4 + name: "Device 4" + area_id: area_4 + - id: device_5 + name: "Device 5" + area_id: area_5 + - id: device_6 + name: "Device 6" + area_id: area_6 + - id: device_7 + name: "Device 7" + area_id: area_7 + - id: device_8 + name: "Device 8" + area_id: area_8 + - id: device_9 + name: "Device 9" + area_id: area_9 + - id: device_10 + name: "Device 10" + area_id: area_10 + - id: device_11 + name: "Device 11" + area_id: area_11 + - id: device_12 + name: "Device 12" + area_id: area_12 + - id: device_13 + name: "Device 13" + area_id: area_13 + - id: device_14 + name: "Device 14" + area_id: area_14 + - id: device_15 + name: "Device 15" + area_id: area_15 + - id: device_16 + name: "Device 16" + area_id: area_16 + - id: device_17 + name: "Device 17" + area_id: area_17 + - id: device_18 + name: "Device 18" + area_id: area_18 + - id: device_19 + name: "Device 19" + area_id: area_19 + - id: device_20 + name: "Device 20" + area_id: area_20 + +api: +sensor: +binary_sensor: +light: +switch: diff --git a/tests/benchmarks/components/main.cpp b/tests/benchmarks/components/main.cpp new file mode 100644 index 0000000000..20b84c087a --- /dev/null +++ b/tests/benchmarks/components/main.cpp @@ -0,0 +1,38 @@ +#include + +#include "esphome/components/logger/logger.h" + +/* +This special main.cpp provides the entry point for Google Benchmark. +It replaces the default ESPHome main with a benchmark runner. +See the codspeed_plan.md for more information. +*/ + +// Auto generated code by esphome +// ========== AUTO GENERATED INCLUDE BLOCK BEGIN =========== +// ========== AUTO GENERATED INCLUDE BLOCK END ===========" + +void original_setup() { + // This function won't be run. + + // ========== AUTO GENERATED CODE BEGIN =========== + // =========== AUTO GENERATED CODE END ============ +} + +void setup() { + // Log functions call global_logger->log_vprintf_() without a null check, + // so we must set up a Logger before any test that triggers logging. + static esphome::logger::Logger test_logger(0); + test_logger.set_log_level(ESPHOME_LOG_LEVEL); + test_logger.pre_setup(); + + int argc = 1; + char arg0[] = "benchmark"; + char *argv[] = {arg0, nullptr}; + ::benchmark::Initialize(&argc, argv); + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + exit(0); +} + +void loop() {} From d9cab03c0946a902eaacae3836fbd5477e7455aa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 20:44:37 -1000 Subject: [PATCH 02/61] Move benchmarks into ci.yml with determine-jobs integration - Add should_run_benchmarks() to determine-jobs.py that checks if directly changed components have benchmark files (no dependency expansion - changing sensor won't trigger api benchmarks) - Move benchmark job from separate workflow into ci.yml - Pin CodSpeed action to full commit SHA - Delete separate ci-benchmarks.yml --- .github/workflows/ci-benchmarks.yml | 87 ----------------------------- .github/workflows/ci.yml | 32 +++++++++++ script/determine-jobs.py | 49 ++++++++++++++++ 3 files changed, 81 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/ci-benchmarks.yml diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml deleted file mode 100644 index 416e65bfc2..0000000000 --- a/.github/workflows/ci-benchmarks.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- -name: CodSpeed Benchmarks - -on: - push: - branches: [dev] - pull_request: - paths: - - "esphome/components/api/**" - - "esphome/core/**" - - "tests/benchmarks/**" - - "script/cpp_benchmark.py" - - "script/test_helpers.py" - - ".github/workflows/ci-benchmarks.yml" - merge_group: - -permissions: - contents: read - -concurrency: - # yamllint disable-line rule:line-length - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - DEFAULT_PYTHON: "3.11" - -jobs: - common: - name: Create common environment - runs-on: ubuntu-24.04 - outputs: - cache-key: ${{ steps.cache-key.outputs.key }} - steps: - - name: Check out code from GitHub - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Generate cache-key - id: cache-key - run: echo key="${{ hashFiles('requirements.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - id: python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Restore Python virtual environment - id: cache-venv - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: venv - # yamllint disable-line rule:line-length - key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ steps.cache-key.outputs.key }} - - name: Create Python virtual environment - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - python -m venv venv - . venv/bin/activate - python --version - pip install -r requirements.txt -r requirements_test.txt - pip install -e . - - benchmarks: - name: Run CodSpeed Benchmarks - runs-on: ubuntu-24.04 - needs: - - common - steps: - - name: Check out code from GitHub - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Restore Python - uses: ./.github/actions/restore-python - with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache-key: ${{ needs.common.outputs.cache-key }} - - - name: Build benchmarks - id: build - run: | - . venv/bin/activate - BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1) - echo "binary=$BINARY" >> $GITHUB_OUTPUT - - - name: Run CodSpeed benchmarks - uses: CodSpeedHQ/action@v4 - with: - run: ${{ steps.build.outputs.binary }} - token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7710589c5..72c3762189 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,6 +185,7 @@ jobs: cpp-unit-tests-run-all: ${{ steps.determine.outputs.cpp-unit-tests-run-all }} cpp-unit-tests-components: ${{ steps.determine.outputs.cpp-unit-tests-components }} component-test-batches: ${{ steps.determine.outputs.component-test-batches }} + benchmarks: ${{ steps.determine.outputs.benchmarks }} steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -227,6 +228,7 @@ jobs: echo "cpp-unit-tests-run-all=$(echo "$output" | jq -r '.cpp_unit_tests_run_all')" >> $GITHUB_OUTPUT echo "cpp-unit-tests-components=$(echo "$output" | jq -c '.cpp_unit_tests_components')" >> $GITHUB_OUTPUT echo "component-test-batches=$(echo "$output" | jq -c '.component_test_batches')" >> $GITHUB_OUTPUT + echo "benchmarks=$(echo "$output" | jq -r '.benchmarks')" >> $GITHUB_OUTPUT - name: Save components graph cache if: github.ref == 'refs/heads/dev' uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 @@ -308,6 +310,36 @@ jobs: script/cpp_unit_test.py $ARGS fi + benchmarks: + name: Run CodSpeed benchmarks + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.benchmarks == 'True' + steps: + - name: Check out code from GitHub + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + + - name: Build benchmarks + id: build + run: | + . venv/bin/activate + BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1) + echo "binary=$BINARY" >> $GITHUB_OUTPUT + + - name: Run CodSpeed benchmarks + uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4 + with: + run: ${{ steps.build.outputs.binary }} + token: ${{ secrets.CODSPEED_TOKEN }} + clang-tidy-single: name: ${{ matrix.name }} runs-on: ubuntu-24.04 diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 6808a3cf6c..e9fcd8925f 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -381,6 +381,51 @@ def determine_cpp_unit_tests( return (False, get_cpp_changed_components(cpp_files)) +def should_run_benchmarks(branch: str | None = None) -> bool: + """Determine if C++ benchmarks should run based on changed files. + + Benchmarks run when any of the following conditions are met: + + 1. Core C++ files changed (esphome/core/*) + 2. A directly changed component has benchmark files (no dependency expansion) + 3. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py, + script/test_helpers.py) + + Unlike unit tests, benchmarks do NOT expand to dependent components. + Changing ``sensor`` does not trigger ``api`` benchmarks just because + api depends on sensor. + + Args: + branch: Branch to compare against. If None, uses default. + + Returns: + True if benchmarks should run, False otherwise. + """ + files = changed_files(branch) + if core_changed(files): + return True + + # Check if benchmark infrastructure changed + if any( + f.startswith("tests/benchmarks/") + or f in ("script/cpp_benchmark.py", "script/test_helpers.py") + for f in files + ): + return True + + # Check if any directly changed component has benchmarks + benchmarks_dir = Path(root_path) / "tests" / "benchmarks" / "components" + if not benchmarks_dir.is_dir(): + return False + benchmarked_components = { + d.name + for d in benchmarks_dir.iterdir() + if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h"))) + } + # Only direct changes — no dependency expansion + return any(get_component_from_path(f) in benchmarked_components for f in files) + + def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) -> bool: """Check if a changed file ends with any of the specified extensions.""" return any(file.endswith(extensions) for file in changed_files(branch)) @@ -804,6 +849,9 @@ def main() -> None: # Determine which C++ unit tests to run cpp_run_all, cpp_components = determine_cpp_unit_tests(args.branch) + # Determine if benchmarks should run + run_benchmarks = should_run_benchmarks(args.branch) + # Split components into batches for CI testing # This intelligently groups components with similar bus configurations component_test_batches: list[str] @@ -856,6 +904,7 @@ def main() -> None: "cpp_unit_tests_run_all": cpp_run_all, "cpp_unit_tests_components": cpp_components, "component_test_batches": component_test_batches, + "benchmarks": run_benchmarks, } # Output as JSON From 9c148da76adb8189605ab3860782563b5c8ee3f8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 20:49:48 -1000 Subject: [PATCH 03/61] Add core benchmarks dir, fix core pseudo-component, use simulation mode - Move scheduler/loop/helpers benchmarks to tests/benchmarks/components/core/ - Add random_float and random_uint32 benchmarks (from ol.yaml) - Fix core pseudo-component crash: skip components where get_component() returns None when adding dependencies to config - Use CodSpeed simulation mode (CPU instruction counting) for reproducible CI results instead of walltime --- .github/workflows/ci.yml | 1 + script/test_helpers.py | 4 ++- .../{api => core}/bench_application_loop.cpp | 0 .../components/core/bench_helpers.cpp | 26 +++++++++++++++++++ .../{api => core}/bench_scheduler.cpp | 0 5 files changed, 30 insertions(+), 1 deletion(-) rename tests/benchmarks/components/{api => core}/bench_application_loop.cpp (100%) create mode 100644 tests/benchmarks/components/core/bench_helpers.cpp rename tests/benchmarks/components/{api => core}/bench_scheduler.cpp (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72c3762189..a6b9f3580d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -339,6 +339,7 @@ jobs: with: run: ${{ steps.build.outputs.binary }} token: ${{ secrets.CODSPEED_TOKEN }} + mode: simulation clang-tidy-single: name: ${{ matrix.name }} diff --git a/script/test_helpers.py b/script/test_helpers.py index 1941168daf..3a7e23c0a7 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -248,7 +248,9 @@ def compile_and_get_binary( domain_list = config.setdefault(domain, []) CORE.testing_ensure_platform_registered(domain) domain_list.append({CONF_PLATFORM: component}) - else: + # Skip "core" — it's a pseudo-component handled by the build + # system, not a real loadable component (get_component returns None) + elif get_component(component_name) is not None: config.setdefault(component_name, []) # Register platforms from the extra config (benchmark.yaml) so diff --git a/tests/benchmarks/components/api/bench_application_loop.cpp b/tests/benchmarks/components/core/bench_application_loop.cpp similarity index 100% rename from tests/benchmarks/components/api/bench_application_loop.cpp rename to tests/benchmarks/components/core/bench_application_loop.cpp diff --git a/tests/benchmarks/components/core/bench_helpers.cpp b/tests/benchmarks/components/core/bench_helpers.cpp new file mode 100644 index 0000000000..15e0bc5ab1 --- /dev/null +++ b/tests/benchmarks/components/core/bench_helpers.cpp @@ -0,0 +1,26 @@ +#include + +#include "esphome/core/helpers.h" + +namespace esphome::benchmarks { + +// --- random_float() --- +// Ported from ol.yaml:148 "Random Float Benchmark" + +static void BM_RandomFloat(benchmark::State &state) { + for (auto _ : state) { + benchmark::DoNotOptimize(random_float()); + } +} +BENCHMARK(BM_RandomFloat); + +// --- random_uint32() --- + +static void BM_RandomUint32(benchmark::State &state) { + for (auto _ : state) { + benchmark::DoNotOptimize(random_uint32()); + } +} +BENCHMARK(BM_RandomUint32); + +} // namespace esphome::benchmarks diff --git a/tests/benchmarks/components/api/bench_scheduler.cpp b/tests/benchmarks/components/core/bench_scheduler.cpp similarity index 100% rename from tests/benchmarks/components/api/bench_scheduler.cpp rename to tests/benchmarks/components/core/bench_scheduler.cpp From a6219434b97af42b310c39f4c60dd5e76d81c8b1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 20:52:53 -1000 Subject: [PATCH 04/61] Move core benchmarks to tests/benchmarks/core/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Core is not a component — its benchmarks belong in tests/benchmarks/core/ not tests/benchmarks/components/core/. Add extra_include_dirs parameter to build_and_run to support non-component benchmark directories. --- script/cpp_benchmark.py | 4 ++++ script/test_helpers.py | 15 ++++++++++++++- .../core/bench_application_loop.cpp | 0 .../{components => }/core/bench_helpers.cpp | 0 .../{components => }/core/bench_scheduler.cpp | 0 5 files changed, 18 insertions(+), 1 deletion(-) rename tests/benchmarks/{components => }/core/bench_application_loop.cpp (100%) rename tests/benchmarks/{components => }/core/bench_helpers.cpp (100%) rename tests/benchmarks/{components => }/core/bench_scheduler.cpp (100%) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index f630aac3d4..31a6896f47 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -11,6 +11,9 @@ from test_helpers import PLATFORMIO_GOOGLE_BENCHMARK_LIB, build_and_run # Path to /tests/benchmarks/components BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "components" +# Path to /tests/benchmarks/core (always included, not a component) +CORE_BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "core" + # Components whose to_code should run during benchmark builds. # core/host/logger are infrastructure. json is needed because its # to_code adds the ArduinoJson library (it's auto-loaded by api but @@ -37,6 +40,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> main_entry="main.cpp", label="benchmarks", build_only=build_only, + extra_include_dirs=[CORE_BENCHMARKS_DIR], ) diff --git a/script/test_helpers.py b/script/test_helpers.py index 3a7e23c0a7..fa79210172 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -300,6 +300,7 @@ def build_and_run( label: str = "build", build_only: bool = False, extra_run_args: list[str] | None = None, + extra_include_dirs: list[Path] | None = None, ) -> int: """Build and optionally run a C++ test/benchmark binary. @@ -318,6 +319,8 @@ def build_and_run( label: Label for log messages build_only: If True, print binary path and return without running extra_run_args: Extra arguments to pass to the binary + extra_include_dirs: Additional directories (relative to tests_dir) + whose .cpp files should be compiled Returns: Exit code @@ -339,8 +342,18 @@ def build_and_run( components = sorted(components) - # Build include list: main entry point + component folders + # Build include list: main entry point + component folders + extra dirs includes: list[str] = [main_entry] + components + if extra_include_dirs: + for d in extra_include_dirs: + if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h"))): + # Use path relative to tests_dir for PlatformIO includes + try: + rel = d.relative_to(tests_dir) + includes.append(str(rel)) + except ValueError: + # Not relative to tests_dir, use absolute + includes.append(str(d)) # Discover platform sub-components try: diff --git a/tests/benchmarks/components/core/bench_application_loop.cpp b/tests/benchmarks/core/bench_application_loop.cpp similarity index 100% rename from tests/benchmarks/components/core/bench_application_loop.cpp rename to tests/benchmarks/core/bench_application_loop.cpp diff --git a/tests/benchmarks/components/core/bench_helpers.cpp b/tests/benchmarks/core/bench_helpers.cpp similarity index 100% rename from tests/benchmarks/components/core/bench_helpers.cpp rename to tests/benchmarks/core/bench_helpers.cpp diff --git a/tests/benchmarks/components/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp similarity index 100% rename from tests/benchmarks/components/core/bench_scheduler.cpp rename to tests/benchmarks/core/bench_scheduler.cpp From 5cd2f58582cce64f7a1081e927602446ae2ee662 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:01:31 -1000 Subject: [PATCH 05/61] Use CodSpeed's codspeed-cpp fork for simulation mode - Clone CodSpeed's codspeed-cpp repo (pinned to SHA) in CI - Create PlatformIO-compatible library.json combining google_benchmark and codspeed core sources for proper instrumentation - Pass library path via BENCHMARK_LIB env var to cpp_benchmark.py - Use simulation mode for reproducible CPU instruction counting - Locally, vanilla google/benchmark is used (no CodSpeed instrumentation) --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++++++- script/cpp_benchmark.py | 5 ++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6b9f3580d..38fb0c832e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -327,11 +327,39 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + - name: Set up CodSpeed benchmark library + # CodSpeed requires their google_benchmark fork for instrumentation. + # Clone and create a PlatformIO-compatible library layout. + run: | + CODSPEED_SHA=d6b4111428ae1f1667fec9bff009522378d5d347 + git clone https://github.com/CodSpeedHQ/codspeed-cpp.git /tmp/codspeed-cpp + git -C /tmp/codspeed-cpp checkout "$CODSPEED_SHA" + # Create library.json combining google_benchmark + codspeed core + cat > /tmp/codspeed-cpp/google_benchmark/library.json << 'LIBJSON' + { + "name": "benchmark", + "version": "0.0.0", + "build": { + "flags": [ + "-I/tmp/codspeed-cpp/core/include", + "-I/tmp/codspeed-cpp/core/instrument-hooks", + "-DHAVE_STD_REGEX", + "-DHAVE_STEADY_CLOCK", + "-DBENCHMARK_STATIC_DEFINE" + ], + "srcFilter": ["+", "+"], + "includeDir": "include" + } + } + LIBJSON + - name: Build benchmarks id: build run: | . venv/bin/activate - BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1) + BENCHMARK_LIB="symlink:///tmp/codspeed-cpp/google_benchmark" \ + script/cpp_benchmark.py --all --build-only 2>&1 | tee /tmp/bench-build.log + BINARY=$(tail -1 /tmp/bench-build.log) echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 31a6896f47..06377ae51f 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -2,6 +2,7 @@ """Build and run C++ benchmarks for ESPHome components using Google Benchmark.""" import argparse +import os from pathlib import Path import sys @@ -29,13 +30,15 @@ PLATFORMIO_OPTIONS = { def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int: + # Allow CI to override the benchmark library (e.g. with CodSpeed's fork) + benchmark_lib = os.environ.get("BENCHMARK_LIB", PLATFORMIO_GOOGLE_BENCHMARK_LIB) return build_and_run( selected_components=selected_components, tests_dir=BENCHMARKS_DIR, codegen_components=BENCHMARK_CODEGEN_COMPONENTS, config_prefix="cppbench", friendly_name="CPP Benchmarks", - libraries=PLATFORMIO_GOOGLE_BENCHMARK_LIB, + libraries=benchmark_lib, platformio_options=PLATFORMIO_OPTIONS, main_entry="main.cpp", label="benchmarks", From ff39fcbb94467e6feb5f5606e19e413c3b6a7ff3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:02:39 -1000 Subject: [PATCH 06/61] Move CodSpeed library setup to Python script Extract inline shell from CI workflow into script/setup_codspeed_lib.py. Pins codspeed-cpp to a specific commit SHA for reproducibility. --- .github/workflows/ci.yml | 29 +---------- script/setup_codspeed_lib.py | 93 ++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 27 deletions(-) create mode 100755 script/setup_codspeed_lib.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38fb0c832e..3eb4c8d3f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -327,37 +327,12 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - - name: Set up CodSpeed benchmark library - # CodSpeed requires their google_benchmark fork for instrumentation. - # Clone and create a PlatformIO-compatible library layout. - run: | - CODSPEED_SHA=d6b4111428ae1f1667fec9bff009522378d5d347 - git clone https://github.com/CodSpeedHQ/codspeed-cpp.git /tmp/codspeed-cpp - git -C /tmp/codspeed-cpp checkout "$CODSPEED_SHA" - # Create library.json combining google_benchmark + codspeed core - cat > /tmp/codspeed-cpp/google_benchmark/library.json << 'LIBJSON' - { - "name": "benchmark", - "version": "0.0.0", - "build": { - "flags": [ - "-I/tmp/codspeed-cpp/core/include", - "-I/tmp/codspeed-cpp/core/instrument-hooks", - "-DHAVE_STD_REGEX", - "-DHAVE_STEADY_CLOCK", - "-DBENCHMARK_STATIC_DEFINE" - ], - "srcFilter": ["+", "+"], - "includeDir": "include" - } - } - LIBJSON - - name: Build benchmarks id: build run: | . venv/bin/activate - BENCHMARK_LIB="symlink:///tmp/codspeed-cpp/google_benchmark" \ + BENCHMARK_LIB=$(python script/setup_codspeed_lib.py) + BENCHMARK_LIB="$BENCHMARK_LIB" \ script/cpp_benchmark.py --all --build-only 2>&1 | tee /tmp/bench-build.log BINARY=$(tail -1 /tmp/bench-build.log) echo "binary=$BINARY" >> $GITHUB_OUTPUT diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py new file mode 100755 index 0000000000..611abb4525 --- /dev/null +++ b/script/setup_codspeed_lib.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Set up CodSpeed's google_benchmark fork as a PlatformIO library. + +CodSpeed requires their codspeed-cpp fork for CPU simulation instrumentation. +This script clones the repo and creates a PlatformIO-compatible library.json +that combines the google_benchmark and codspeed core sources. + +Usage: + python script/setup_codspeed_lib.py [--output-dir DIR] + +Prints the PlatformIO library path (symlink:// URL) to stdout. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess + +# Pin to a specific commit for reproducibility +CODSPEED_CPP_REPO = "https://github.com/CodSpeedHQ/codspeed-cpp.git" +CODSPEED_CPP_SHA = "d6b4111428ae1f1667fec9bff009522378d5d347" + +DEFAULT_OUTPUT_DIR = "/tmp/codspeed-cpp" + + +def setup_codspeed_lib(output_dir: Path) -> str: + """Clone codspeed-cpp and create PlatformIO library layout. + + Args: + output_dir: Directory to clone into + + Returns: + PlatformIO library path (symlink:// URL) + """ + if not (output_dir / ".git").exists(): + subprocess.run( + ["git", "clone", CODSPEED_CPP_REPO, str(output_dir)], + check=True, + ) + subprocess.run( + ["git", "-C", str(output_dir), "checkout", CODSPEED_CPP_SHA], + check=True, + capture_output=True, + ) + + benchmark_dir = output_dir / "google_benchmark" + core_dir = output_dir / "core" + + # Create library.json combining google_benchmark + codspeed core + library_json = { + "name": "benchmark", + "version": "0.0.0", + "build": { + "flags": [ + f"-I{core_dir / 'include'}", + f"-I{core_dir / 'instrument-hooks'}", + "-DHAVE_STD_REGEX", + "-DHAVE_STEADY_CLOCK", + "-DBENCHMARK_STATIC_DEFINE", + ], + "srcFilter": [ + "+", + f"+<{core_dir / 'src' / '*.cpp'}>", + ], + "includeDir": "include", + }, + } + + (benchmark_dir / "library.json").write_text( + json.dumps(library_json, indent=2) + "\n" + ) + + return f"symlink://{benchmark_dir}" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(DEFAULT_OUTPUT_DIR), + help=f"Directory to clone codspeed-cpp into (default: {DEFAULT_OUTPUT_DIR})", + ) + args = parser.parse_args() + + lib_path = setup_codspeed_lib(args.output_dir) + print(lib_path) + + +if __name__ == "__main__": + main() From f13513239d82f5362f059c0a0eae6ceb5fe42078 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:21:44 -1000 Subject: [PATCH 07/61] Fix CodSpeed instrumentation with PlatformIO - Use CodSpeed's codspeed-cpp fork with proper instrumentation for simulation mode benchmark detection - setup_codspeed_lib.py creates a flat PlatformIO-compatible library by combining google_benchmark sources, codspeed core, and instrument-hooks into a single library directory - Renames .cc to .cpp (PlatformIO doesn't compile .cc by default) - Adds all required defines: CODSPEED_ENABLED, CODSPEED_SIMULATION, CODSPEED_VERSION, CODSPEED_ROOT_DIR, CODSPEED_MODE_DISPLAY - Output JSON config consumed by cpp_benchmark.py via env var --- .github/workflows/ci.yml | 5 +- script/cpp_benchmark.py | 18 +++++- script/setup_codspeed_lib.py | 105 +++++++++++++++++++++++++++-------- 3 files changed, 101 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3eb4c8d3f1..554a49bd13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -331,9 +331,8 @@ jobs: id: build run: | . venv/bin/activate - BENCHMARK_LIB=$(python script/setup_codspeed_lib.py) - BENCHMARK_LIB="$BENCHMARK_LIB" \ - script/cpp_benchmark.py --all --build-only 2>&1 | tee /tmp/bench-build.log + export BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) + script/cpp_benchmark.py --all --build-only 2>&1 | tee /tmp/bench-build.log BINARY=$(tail -1 /tmp/bench-build.log) echo "binary=$BINARY" >> $GITHUB_OUTPUT diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 06377ae51f..36980fff88 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -26,12 +26,26 @@ PLATFORMIO_OPTIONS = { "-DUSE_TIME_TIMEZONE", # enable timezone code paths "-g", # debug symbols for profiling ], + # Use deep+ LDF mode to ensure PlatformIO detects the benchmark + # library dependency from nested includes. + "lib_ldf_mode": "deep+", } def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int: - # Allow CI to override the benchmark library (e.g. with CodSpeed's fork) - benchmark_lib = os.environ.get("BENCHMARK_LIB", PLATFORMIO_GOOGLE_BENCHMARK_LIB) + # Allow CI to override the benchmark library (e.g. with CodSpeed's fork). + # BENCHMARK_LIB_CONFIG is a JSON string from setup_codspeed_lib.py + # containing {"lib_path": "/path/to/google_benchmark"}. + lib_config_json = os.environ.get("BENCHMARK_LIB_CONFIG") + + if lib_config_json: + import json + + lib_config = json.loads(lib_config_json) + benchmark_lib = f"benchmark=symlink://{lib_config['lib_path']}" + else: + benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB + return build_and_run( selected_components=selected_components, tests_dir=BENCHMARKS_DIR, diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index 611abb4525..322a8069a6 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -2,13 +2,14 @@ """Set up CodSpeed's google_benchmark fork as a PlatformIO library. CodSpeed requires their codspeed-cpp fork for CPU simulation instrumentation. -This script clones the repo and creates a PlatformIO-compatible library.json -that combines the google_benchmark and codspeed core sources. +This script clones the repo and creates a flat PlatformIO-compatible library +by combining google_benchmark sources and codspeed core sources. Usage: python script/setup_codspeed_lib.py [--output-dir DIR] -Prints the PlatformIO library path (symlink:// URL) to stdout. +Prints JSON to stdout with lib_path and build_flags for cpp_benchmark.py. +Git output goes to stderr. """ from __future__ import annotations @@ -16,7 +17,9 @@ from __future__ import annotations import argparse import json from pathlib import Path +import shutil import subprocess +import sys # Pin to a specific commit for reproducibility CODSPEED_CPP_REPO = "https://github.com/CodSpeedHQ/codspeed-cpp.git" @@ -25,44 +28,99 @@ CODSPEED_CPP_SHA = "d6b4111428ae1f1667fec9bff009522378d5d347" DEFAULT_OUTPUT_DIR = "/tmp/codspeed-cpp" -def setup_codspeed_lib(output_dir: Path) -> str: - """Clone codspeed-cpp and create PlatformIO library layout. +def setup_codspeed_lib(output_dir: Path) -> None: + """Clone codspeed-cpp and create a flat PlatformIO library. Args: output_dir: Directory to clone into - - Returns: - PlatformIO library path (symlink:// URL) """ if not (output_dir / ".git").exists(): subprocess.run( ["git", "clone", CODSPEED_CPP_REPO, str(output_dir)], check=True, + stdout=sys.stderr, + stderr=sys.stderr, + ) + # Checkout pinned SHA and init submodules in one pass + subprocess.run( + ["git", "-C", str(output_dir), "checkout", CODSPEED_CPP_SHA], + check=True, + capture_output=True, + ) + subprocess.run( + [ + "git", + "-C", + str(output_dir), + "submodule", + "update", + "--init", + "--recursive", + ], + check=True, + stdout=sys.stderr, + stderr=sys.stderr, ) - subprocess.run( - ["git", "-C", str(output_dir), "checkout", CODSPEED_CPP_SHA], - check=True, - capture_output=True, - ) benchmark_dir = output_dir / "google_benchmark" core_dir = output_dir / "core" + instrument_hooks_dir = core_dir / "instrument-hooks" / "includes" - # Create library.json combining google_benchmark + codspeed core + # Read version from core/CMakeLists.txt (needed by walltime.cpp) + version = "0.0.0" + cmake_file = core_dir / "CMakeLists.txt" + if cmake_file.exists(): + for line in cmake_file.read_text().splitlines(): + if line.startswith("set(CODSPEED_VERSION"): + version = line.split()[1].rstrip(")") + break + + # PlatformIO doesn't compile .cc files — rename to .cpp + for cc_file in (benchmark_dir / "src").glob("*.cc"): + cpp_file = cc_file.with_suffix(".cpp") + if not cpp_file.exists(): + cc_file.rename(cpp_file) + + # Copy codspeed core sources and headers into google_benchmark/src/ + # so PlatformIO compiles everything as one library. + # .cpp files get a codspeed_ prefix to avoid name collisions. + # .h files keep their original names since they're referenced by includes. + for src_file in (core_dir / "src").glob("*"): + if src_file.suffix == ".cpp": + dest = benchmark_dir / "src" / f"codspeed_{src_file.name}" + elif src_file.suffix == ".h": + dest = benchmark_dir / "src" / src_file.name + else: + continue + if not dest.exists(): + shutil.copy2(src_file, dest) + + # Copy instrument-hooks C source (provides instrument_hooks_* symbols) + hooks_c = instrument_hooks_dir.parent / "dist" / "core.c" + if hooks_c.exists(): + dest = benchmark_dir / "src" / "instrument_hooks.c" + if not dest.exists(): + shutil.copy2(hooks_c, dest) + + # Resolve the ESPHome project root for CODSPEED_ROOT_DIR + project_root = Path(__file__).resolve().parent.parent + + # Create library.json library_json = { "name": "benchmark", "version": "0.0.0", "build": { "flags": [ f"-I{core_dir / 'include'}", - f"-I{core_dir / 'instrument-hooks'}", + f"-I{instrument_hooks_dir}", "-DHAVE_STD_REGEX", "-DHAVE_STEADY_CLOCK", "-DBENCHMARK_STATIC_DEFINE", - ], - "srcFilter": [ - "+", - f"+<{core_dir / 'src' / '*.cpp'}>", + "-DCODSPEED_ENABLED", + "-DCODSPEED_SIMULATION", + f'-DCODSPEED_VERSION=\\"{version}\\"', + f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', + '-DCODSPEED_MODE_DISPLAY=\\"simulation\\"', ], "includeDir": "include", }, @@ -72,7 +130,11 @@ def setup_codspeed_lib(output_dir: Path) -> str: json.dumps(library_json, indent=2) + "\n" ) - return f"symlink://{benchmark_dir}" + # Output JSON config for cpp_benchmark.py + result = { + "lib_path": str(benchmark_dir), + } + print(json.dumps(result)) def main() -> None: @@ -85,8 +147,7 @@ def main() -> None: ) args = parser.parse_args() - lib_path = setup_codspeed_lib(args.output_dir) - print(lib_path) + setup_codspeed_lib(args.output_dir) if __name__ == "__main__": From 0fc3fc2776af4ce7dc11da4e1177677b3b4e37cb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:25:25 -1000 Subject: [PATCH 08/61] Clean up shared constants and setup script - Move BASE_CODEGEN_COMPONENTS and USE_TIME_TIMEZONE_FLAG to test_helpers.py - Use shared constants in both cpp_unit_test.py and cpp_benchmark.py - Move json import to top level in cpp_benchmark.py - Refactor setup_codspeed_lib.py into focused helper functions - Combine clone + submodule init, use --shallow-submodules --- script/cpp_benchmark.py | 21 ++-- script/cpp_unit_test.py | 18 ++- script/setup_codspeed_lib.py | 212 ++++++++++++++++++++++------------- script/test_helpers.py | 9 ++ 4 files changed, 164 insertions(+), 96 deletions(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 36980fff88..3dc6e279b9 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -2,12 +2,18 @@ """Build and run C++ benchmarks for ESPHome components using Google Benchmark.""" import argparse +import json import os from pathlib import Path import sys from helpers import root_path -from test_helpers import PLATFORMIO_GOOGLE_BENCHMARK_LIB, build_and_run +from test_helpers import ( + BASE_CODEGEN_COMPONENTS, + PLATFORMIO_GOOGLE_BENCHMARK_LIB, + USE_TIME_TIMEZONE_FLAG, + build_and_run, +) # Path to /tests/benchmarks/components BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "components" @@ -15,15 +21,14 @@ BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "components" # Path to /tests/benchmarks/core (always included, not a component) CORE_BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "core" -# Components whose to_code should run during benchmark builds. -# core/host/logger are infrastructure. json is needed because its -# to_code adds the ArduinoJson library (it's auto-loaded by api but -# cpp_testing suppresses to_code for components not in this set). -BENCHMARK_CODEGEN_COMPONENTS = {"core", "host", "logger", "json"} +# Additional codegen components beyond the base set. +# json is needed because its to_code adds the ArduinoJson library +# (auto-loaded by api, but cpp_testing suppresses to_code unless listed). +BENCHMARK_CODEGEN_COMPONENTS = BASE_CODEGEN_COMPONENTS | {"json"} PLATFORMIO_OPTIONS = { "build_flags": [ - "-DUSE_TIME_TIMEZONE", # enable timezone code paths + USE_TIME_TIMEZONE_FLAG, "-g", # debug symbols for profiling ], # Use deep+ LDF mode to ensure PlatformIO detects the benchmark @@ -39,8 +44,6 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> lib_config_json = os.environ.get("BENCHMARK_LIB_CONFIG") if lib_config_json: - import json - lib_config = json.loads(lib_config_json) benchmark_lib = f"benchmark=symlink://{lib_config['lib_path']}" else: diff --git a/script/cpp_unit_test.py b/script/cpp_unit_test.py index c0a6d9647d..81c56b82da 100755 --- a/script/cpp_unit_test.py +++ b/script/cpp_unit_test.py @@ -4,18 +4,16 @@ from pathlib import Path import sys from helpers import get_all_components, root_path -from test_helpers import PLATFORMIO_GOOGLE_TEST_LIB, build_and_run +from test_helpers import ( + BASE_CODEGEN_COMPONENTS, + PLATFORMIO_GOOGLE_TEST_LIB, + USE_TIME_TIMEZONE_FLAG, + build_and_run, +) # Path to /tests/components COMPONENTS_TESTS_DIR: Path = Path(root_path) / "tests" / "components" -# Components whose to_code should run during C++ test builds. -# Most components don't need code generation for tests; only these -# essential ones (platform setup, logging, core config) are needed. -# Note: "core" is the esphome core config module (esphome/core/config.py), -# which registers under package name "core" not "esphome". -CPP_TESTING_CODEGEN_COMPONENTS = {"core", "host", "logger"} - PLATFORMIO_OPTIONS = { "build_type": "debug", "build_unflags": [ @@ -23,7 +21,7 @@ PLATFORMIO_OPTIONS = { ], "build_flags": [ "-Og", # optimize for debug - "-DUSE_TIME_TIMEZONE", # enable timezone code paths for testing + USE_TIME_TIMEZONE_FLAG, "-DESPHOME_DEBUG", # enable debug assertions # Enable the address and undefined behavior sanitizers "-fsanitize=address", @@ -41,7 +39,7 @@ def run_tests(selected_components: list[str]) -> int: return build_and_run( selected_components=selected_components, tests_dir=COMPONENTS_TESTS_DIR, - codegen_components=CPP_TESTING_CODEGEN_COMPONENTS, + codegen_components=BASE_CODEGEN_COMPONENTS, config_prefix="cpptests", friendly_name="CPP Unit Tests", libraries=PLATFORMIO_GOOGLE_TEST_LIB, diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index 322a8069a6..b827610102 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -2,14 +2,22 @@ """Set up CodSpeed's google_benchmark fork as a PlatformIO library. CodSpeed requires their codspeed-cpp fork for CPU simulation instrumentation. -This script clones the repo and creates a flat PlatformIO-compatible library -by combining google_benchmark sources and codspeed core sources. +This script clones the repo and assembles a flat PlatformIO-compatible library +by combining google_benchmark sources, codspeed core, and instrument-hooks. + +PlatformIO quirks addressed: + - .cc files renamed to .cpp (PlatformIO ignores .cc) + - All sources merged into one src/ dir (PlatformIO can't compile from + multiple source directories in a single library) + - library.json created with required CodSpeed preprocessor defines Usage: python script/setup_codspeed_lib.py [--output-dir DIR] -Prints JSON to stdout with lib_path and build_flags for cpp_benchmark.py. +Prints JSON to stdout with lib_path for cpp_benchmark.py. Git output goes to stderr. + +See https://codspeed.io/docs/benchmarks/cpp#custom-build-systems """ from __future__ import annotations @@ -27,95 +35,110 @@ CODSPEED_CPP_SHA = "d6b4111428ae1f1667fec9bff009522378d5d347" DEFAULT_OUTPUT_DIR = "/tmp/codspeed-cpp" +# Well-known paths within the codspeed-cpp repository +GOOGLE_BENCHMARK_SUBDIR = "google_benchmark" +CORE_SUBDIR = "core" +INSTRUMENT_HOOKS_SUBDIR = Path(CORE_SUBDIR) / "instrument-hooks" +INSTRUMENT_HOOKS_INCLUDES = INSTRUMENT_HOOKS_SUBDIR / "includes" +INSTRUMENT_HOOKS_DIST = INSTRUMENT_HOOKS_SUBDIR / "dist" / "core.c" +CORE_CMAKE = Path(CORE_SUBDIR) / "CMakeLists.txt" -def setup_codspeed_lib(output_dir: Path) -> None: - """Clone codspeed-cpp and create a flat PlatformIO library. - Args: - output_dir: Directory to clone into - """ - if not (output_dir / ".git").exists(): - subprocess.run( - ["git", "clone", CODSPEED_CPP_REPO, str(output_dir)], - check=True, - stdout=sys.stderr, - stderr=sys.stderr, - ) - # Checkout pinned SHA and init submodules in one pass - subprocess.run( - ["git", "-C", str(output_dir), "checkout", CODSPEED_CPP_SHA], - check=True, - capture_output=True, - ) - subprocess.run( - [ - "git", - "-C", - str(output_dir), - "submodule", - "update", - "--init", - "--recursive", - ], - check=True, - stdout=sys.stderr, - stderr=sys.stderr, - ) +def _git(args: list[str], **kwargs: object) -> None: + """Run a git command, sending output to stderr.""" + subprocess.run( + ["git", *args], + check=True, + stdout=kwargs.pop("stdout", sys.stderr), + stderr=kwargs.pop("stderr", sys.stderr), + **kwargs, + ) - benchmark_dir = output_dir / "google_benchmark" - core_dir = output_dir / "core" - instrument_hooks_dir = core_dir / "instrument-hooks" / "includes" - # Read version from core/CMakeLists.txt (needed by walltime.cpp) - version = "0.0.0" - cmake_file = core_dir / "CMakeLists.txt" - if cmake_file.exists(): - for line in cmake_file.read_text().splitlines(): - if line.startswith("set(CODSPEED_VERSION"): - version = line.split()[1].rstrip(")") - break +def _clone_repo(output_dir: Path) -> None: + """Clone codspeed-cpp at the pinned SHA with submodules.""" + _git( + [ + "clone", + "--recurse-submodules", + "--shallow-submodules", + CODSPEED_CPP_REPO, + str(output_dir), + ] + ) + _git( + [ + "-C", + str(output_dir), + "-c", + "advice.detachedHead=false", + "checkout", + CODSPEED_CPP_SHA, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) - # PlatformIO doesn't compile .cc files — rename to .cpp - for cc_file in (benchmark_dir / "src").glob("*.cc"): + +def _read_codspeed_version(cmake_path: Path) -> str: + """Extract CODSPEED_VERSION from core/CMakeLists.txt.""" + if not cmake_path.exists(): + return "0.0.0" + for line in cmake_path.read_text().splitlines(): + if line.startswith("set(CODSPEED_VERSION"): + return line.split()[1].rstrip(")") + return "0.0.0" + + +def _rename_cc_to_cpp(src_dir: Path) -> None: + """Rename .cc files to .cpp so PlatformIO compiles them.""" + for cc_file in src_dir.glob("*.cc"): cpp_file = cc_file.with_suffix(".cpp") if not cpp_file.exists(): cc_file.rename(cpp_file) - # Copy codspeed core sources and headers into google_benchmark/src/ - # so PlatformIO compiles everything as one library. - # .cpp files get a codspeed_ prefix to avoid name collisions. - # .h files keep their original names since they're referenced by includes. - for src_file in (core_dir / "src").glob("*"): + +def _copy_if_missing(src: Path, dest: Path) -> None: + """Copy a file only if the destination doesn't already exist.""" + if not dest.exists(): + shutil.copy2(src, dest) + + +def _merge_codspeed_core_into_lib(core_src: Path, lib_src: Path) -> None: + """Copy codspeed core sources into the benchmark library src/. + + .cpp files get a ``codspeed_`` prefix to avoid name collisions with + google_benchmark's own sources. .h files keep their original names + since they're referenced by ``#include "walltime.h"`` etc. + """ + for src_file in core_src.iterdir(): if src_file.suffix == ".cpp": - dest = benchmark_dir / "src" / f"codspeed_{src_file.name}" + _copy_if_missing(src_file, lib_src / f"codspeed_{src_file.name}") elif src_file.suffix == ".h": - dest = benchmark_dir / "src" / src_file.name - else: - continue - if not dest.exists(): - shutil.copy2(src_file, dest) + _copy_if_missing(src_file, lib_src / src_file.name) - # Copy instrument-hooks C source (provides instrument_hooks_* symbols) - hooks_c = instrument_hooks_dir.parent / "dist" / "core.c" - if hooks_c.exists(): - dest = benchmark_dir / "src" / "instrument_hooks.c" - if not dest.exists(): - shutil.copy2(hooks_c, dest) - # Resolve the ESPHome project root for CODSPEED_ROOT_DIR - project_root = Path(__file__).resolve().parent.parent - - # Create library.json +def _write_library_json( + benchmark_dir: Path, + core_include: Path, + hooks_include: Path, + version: str, + project_root: Path, +) -> None: + """Write a PlatformIO library.json with CodSpeed build flags.""" library_json = { "name": "benchmark", "version": "0.0.0", "build": { "flags": [ - f"-I{core_dir / 'include'}", - f"-I{instrument_hooks_dir}", + f"-I{core_include}", + f"-I{hooks_include}", + # google benchmark build flags "-DHAVE_STD_REGEX", "-DHAVE_STEADY_CLOCK", "-DBENCHMARK_STATIC_DEFINE", + # CodSpeed instrumentation flags + # https://codspeed.io/docs/benchmarks/cpp#custom-build-systems "-DCODSPEED_ENABLED", "-DCODSPEED_SIMULATION", f'-DCODSPEED_VERSION=\\"{version}\\"', @@ -125,16 +148,52 @@ def setup_codspeed_lib(output_dir: Path) -> None: "includeDir": "include", }, } - (benchmark_dir / "library.json").write_text( json.dumps(library_json, indent=2) + "\n" ) + +def setup_codspeed_lib(output_dir: Path) -> None: + """Clone codspeed-cpp and assemble a flat PlatformIO library. + + The resulting library at ``output_dir/google_benchmark/`` contains: + - google_benchmark sources (.cc renamed to .cpp) + - codspeed core sources (prefixed ``codspeed_``) + - instrument-hooks C source (as ``instrument_hooks.c``) + - library.json with all required CodSpeed defines + + Args: + output_dir: Directory to clone the repository into + """ + if not (output_dir / ".git").exists(): + _clone_repo(output_dir) + + benchmark_dir = output_dir / GOOGLE_BENCHMARK_SUBDIR + lib_src = benchmark_dir / "src" + core_dir = output_dir / CORE_SUBDIR + core_include = core_dir / "include" + hooks_include = output_dir / INSTRUMENT_HOOKS_INCLUDES + hooks_dist_c = output_dir / INSTRUMENT_HOOKS_DIST + project_root = Path(__file__).resolve().parent.parent + + # 1. Rename .cc → .cpp (PlatformIO doesn't compile .cc) + _rename_cc_to_cpp(lib_src) + + # 2. Merge codspeed core sources into the library + _merge_codspeed_core_into_lib(core_dir / "src", lib_src) + + # 3. Copy instrument-hooks C source (provides instrument_hooks_* symbols) + if hooks_dist_c.exists(): + _copy_if_missing(hooks_dist_c, lib_src / "instrument_hooks.c") + + # 4. Write library.json + version = _read_codspeed_version(output_dir / CORE_CMAKE) + _write_library_json( + benchmark_dir, core_include, hooks_include, version, project_root + ) + # Output JSON config for cpp_benchmark.py - result = { - "lib_path": str(benchmark_dir), - } - print(json.dumps(result)) + print(json.dumps({"lib_path": str(benchmark_dir)})) def main() -> None: @@ -146,7 +205,6 @@ def main() -> None: help=f"Directory to clone codspeed-cpp into (default: {DEFAULT_OUTPUT_DIR})", ) args = parser.parse_args() - setup_codspeed_lib(args.output_dir) diff --git a/script/test_helpers.py b/script/test_helpers.py index fa79210172..5ca2e4e408 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -36,6 +36,15 @@ LOGGER_KEY = "logger" # by component benchmark.yaml files. esphome: allows sub-key merging. BASE_CONFIG_KEYS = frozenset({ESPHOME_KEY, HOST_KEY, LOGGER_KEY}) +# Shared build flag — enables timezone code paths for testing/benchmarking. +USE_TIME_TIMEZONE_FLAG = "-DUSE_TIME_TIMEZONE" + +# Components whose to_code should always run during C++ test/benchmark builds. +# These are the minimal infrastructure components needed for host compilation. +# Note: "core" is the esphome core config module (esphome/core/config.py), +# which registers under package name "core" not "esphome". +BASE_CODEGEN_COMPONENTS = {"core", "host", "logger"} + # Exit codes EXIT_OK = 0 EXIT_SKIPPED = 1 From e7f16131598ff99557c72aefb3399aff7c22b3a6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:26:22 -1000 Subject: [PATCH 09/61] Use shallow clone (depth 1) for codspeed-cpp --- script/setup_codspeed_lib.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index b827610102..b590587268 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -56,27 +56,27 @@ def _git(args: list[str], **kwargs: object) -> None: def _clone_repo(output_dir: Path) -> None: - """Clone codspeed-cpp at the pinned SHA with submodules.""" + """Shallow-clone codspeed-cpp at the pinned SHA with submodules.""" + output_dir.mkdir(parents=True, exist_ok=True) + _git(["init", str(output_dir)]) + _git(["-C", str(output_dir), "remote", "add", "origin", CODSPEED_CPP_REPO]) + _git(["-C", str(output_dir), "fetch", "--depth", "1", "origin", CODSPEED_CPP_SHA]) _git( - [ - "clone", - "--recurse-submodules", - "--shallow-submodules", - CODSPEED_CPP_REPO, - str(output_dir), - ] + ["-C", str(output_dir), "checkout", "FETCH_HEAD"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) _git( [ "-C", str(output_dir), - "-c", - "advice.detachedHead=false", - "checkout", - CODSPEED_CPP_SHA, - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + "submodule", + "update", + "--init", + "--recursive", + "--depth", + "1", + ] ) From 637180a628eab91ab7cb1b7f68549c09d31c31cf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:30:00 -1000 Subject: [PATCH 10/61] Fix review issues: boolean comparison, missing trigger, constants - Fix benchmarks CI condition: 'true' not 'True' (jq outputs lowercase) - Add script/setup_codspeed_lib.py to benchmark infrastructure triggers - Extract BENCHMARK_INFRASTRUCTURE_FILES and BENCHMARKS_COMPONENTS_PATH as top-level constants in determine-jobs.py - Fix extra_include_dirs docstring --- .github/workflows/ci.yml | 2 +- script/determine-jobs.py | 20 ++++++++++++++++---- script/test_helpers.py | 4 ++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 554a49bd13..6f20a4a7d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -316,7 +316,7 @@ jobs: needs: - common - determine-jobs - if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.benchmarks == 'True' + if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.benchmarks == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/script/determine-jobs.py b/script/determine-jobs.py index e9fcd8925f..ad08f8dce5 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -381,6 +381,19 @@ def determine_cpp_unit_tests( return (False, get_cpp_changed_components(cpp_files)) +# Paths within tests/benchmarks/ that contain component benchmark files +BENCHMARKS_COMPONENTS_PATH = "tests/benchmarks/components" + +# Files that, when changed, should trigger benchmark runs +BENCHMARK_INFRASTRUCTURE_FILES = frozenset( + { + "script/cpp_benchmark.py", + "script/test_helpers.py", + "script/setup_codspeed_lib.py", + } +) + + def should_run_benchmarks(branch: str | None = None) -> bool: """Determine if C++ benchmarks should run based on changed files. @@ -389,7 +402,7 @@ def should_run_benchmarks(branch: str | None = None) -> bool: 1. Core C++ files changed (esphome/core/*) 2. A directly changed component has benchmark files (no dependency expansion) 3. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py, - script/test_helpers.py) + script/test_helpers.py, script/setup_codspeed_lib.py) Unlike unit tests, benchmarks do NOT expand to dependent components. Changing ``sensor`` does not trigger ``api`` benchmarks just because @@ -407,14 +420,13 @@ def should_run_benchmarks(branch: str | None = None) -> bool: # Check if benchmark infrastructure changed if any( - f.startswith("tests/benchmarks/") - or f in ("script/cpp_benchmark.py", "script/test_helpers.py") + f.startswith("tests/benchmarks/") or f in BENCHMARK_INFRASTRUCTURE_FILES for f in files ): return True # Check if any directly changed component has benchmarks - benchmarks_dir = Path(root_path) / "tests" / "benchmarks" / "components" + benchmarks_dir = Path(root_path) / BENCHMARKS_COMPONENTS_PATH if not benchmarks_dir.is_dir(): return False benchmarked_components = { diff --git a/script/test_helpers.py b/script/test_helpers.py index 5ca2e4e408..25ff062e02 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -328,8 +328,8 @@ def build_and_run( label: Label for log messages build_only: If True, print binary path and return without running extra_run_args: Extra arguments to pass to the binary - extra_include_dirs: Additional directories (relative to tests_dir) - whose .cpp files should be compiled + extra_include_dirs: Additional directories whose .cpp files + should be compiled (resolved relative to tests_dir if possible) Returns: Exit code From 7c95a83c6c9193abaa9fc81fc7eda08d02542241 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:31:31 -1000 Subject: [PATCH 11/61] Remove stale codspeed_plan.md reference from main.cpp --- tests/benchmarks/components/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmarks/components/main.cpp b/tests/benchmarks/components/main.cpp index 20b84c087a..bae733cbc6 100644 --- a/tests/benchmarks/components/main.cpp +++ b/tests/benchmarks/components/main.cpp @@ -5,7 +5,7 @@ /* This special main.cpp provides the entry point for Google Benchmark. It replaces the default ESPHome main with a benchmark runner. -See the codspeed_plan.md for more information. + */ // Auto generated code by esphome From b2e78d3753b7da3be99dc4f4f88b601f45faba06 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:36:25 -1000 Subject: [PATCH 12/61] Define CODSPEED_ENABLED globally for benchmark source files benchmark.h uses #ifdef CODSPEED_ENABLED to switch benchmark registration to CodSpeed-instrumented variants. This define was only in library.json (applied to library compilation) but not to the benchmark .cpp files that #include . Without it, CodSpeed reports "No benchmarks found". --- script/cpp_benchmark.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 3dc6e279b9..ce9a0335fa 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -43,9 +43,18 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> # containing {"lib_path": "/path/to/google_benchmark"}. lib_config_json = os.environ.get("BENCHMARK_LIB_CONFIG") + pio_options = PLATFORMIO_OPTIONS if lib_config_json: lib_config = json.loads(lib_config_json) benchmark_lib = f"benchmark=symlink://{lib_config['lib_path']}" + # CODSPEED_ENABLED must be defined globally (not just in library build + # flags) because benchmark.h uses #ifdef CODSPEED_ENABLED to switch + # benchmark registration to CodSpeed-instrumented variants. + pio_options = { + **PLATFORMIO_OPTIONS, + "build_flags": PLATFORMIO_OPTIONS["build_flags"] + + ["-DCODSPEED_ENABLED", "-DCODSPEED_SIMULATION"], + } else: benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB @@ -56,7 +65,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> config_prefix="cppbench", friendly_name="CPP Benchmarks", libraries=benchmark_lib, - platformio_options=PLATFORMIO_OPTIONS, + platformio_options=pio_options, main_entry="main.cpp", label="benchmarks", build_only=build_only, From cf672c1dc7ea341756e675650d7cac6364b43aa1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:37:18 -1000 Subject: [PATCH 13/61] Add CODSPEED_ROOT_DIR to global build flags for relative paths in reports --- script/cpp_benchmark.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index ce9a0335fa..aceafaa04b 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -47,13 +47,19 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> if lib_config_json: lib_config = json.loads(lib_config_json) benchmark_lib = f"benchmark=symlink://{lib_config['lib_path']}" - # CODSPEED_ENABLED must be defined globally (not just in library build - # flags) because benchmark.h uses #ifdef CODSPEED_ENABLED to switch - # benchmark registration to CodSpeed-instrumented variants. + # These defines must be global (not just in library.json) because + # benchmark.h uses #ifdef CODSPEED_ENABLED to switch benchmark + # registration to CodSpeed-instrumented variants, and + # CODSPEED_ROOT_DIR is used to display relative file paths in reports. + project_root = Path(__file__).resolve().parent.parent pio_options = { **PLATFORMIO_OPTIONS, "build_flags": PLATFORMIO_OPTIONS["build_flags"] - + ["-DCODSPEED_ENABLED", "-DCODSPEED_SIMULATION"], + + [ + "-DCODSPEED_ENABLED", + "-DCODSPEED_SIMULATION", + f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', + ], } else: benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB From 374ca70f5b52898146ea337e3b305418435ebd33 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:43:47 -1000 Subject: [PATCH 14/61] Add debug logging for CodSpeed build flags --- script/cpp_benchmark.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index aceafaa04b..d349c41a8d 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -52,15 +52,18 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> # registration to CodSpeed-instrumented variants, and # CODSPEED_ROOT_DIR is used to display relative file paths in reports. project_root = Path(__file__).resolve().parent.parent + codspeed_flags = [ + "-DCODSPEED_ENABLED", + "-DCODSPEED_SIMULATION", + f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', + ] pio_options = { **PLATFORMIO_OPTIONS, - "build_flags": PLATFORMIO_OPTIONS["build_flags"] - + [ - "-DCODSPEED_ENABLED", - "-DCODSPEED_SIMULATION", - f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', - ], + "build_flags": PLATFORMIO_OPTIONS["build_flags"] + codspeed_flags, } + print(f"CodSpeed library: {lib_config['lib_path']}", file=sys.stderr) + print(f"CodSpeed build flags: {codspeed_flags}", file=sys.stderr) + print(f"PlatformIO options: {pio_options}", file=sys.stderr) else: benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB From baafecd0b292d022e3d6d6778669e8d15e7176e0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:50:04 -1000 Subject: [PATCH 15/61] Enable CodSpeed debug logging to diagnose 0 benchmarks issue --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f20a4a7d2..42817bf2bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -338,6 +338,8 @@ jobs: - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4 + env: + CODSPEED_LOG: debug with: run: ${{ steps.build.outputs.binary }} token: ${{ secrets.CODSPEED_TOKEN }} From 63201d44372eccff025d267ee13ec139d4d52495 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:55:53 -1000 Subject: [PATCH 16/61] Fix CodSpeed: use CODSPEED_ANALYSIS not CODSPEED_SIMULATION CodSpeed's CMake uses CODSPEED_ANALYSIS (not CODSPEED_SIMULATION) for simulation mode. This define gates all the actual benchmark measurement hooks in benchmark_runner.cpp and benchmark.h. Without it, benchmarks run normally but CodSpeed doesn't detect them as instrumented. --- script/cpp_benchmark.py | 2 +- script/setup_codspeed_lib.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index d349c41a8d..e26668a055 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -54,7 +54,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> project_root = Path(__file__).resolve().parent.parent codspeed_flags = [ "-DCODSPEED_ENABLED", - "-DCODSPEED_SIMULATION", + "-DCODSPEED_ANALYSIS", f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', ] pio_options = { diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index b590587268..dbe703ff63 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -140,7 +140,7 @@ def _write_library_json( # CodSpeed instrumentation flags # https://codspeed.io/docs/benchmarks/cpp#custom-build-systems "-DCODSPEED_ENABLED", - "-DCODSPEED_SIMULATION", + "-DCODSPEED_ANALYSIS", f'-DCODSPEED_VERSION=\\"{version}\\"', f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', '-DCODSPEED_MODE_DISPLAY=\\"simulation\\"', From f7d4b437d0d993af0ac3bc2b76a0f2a0b1c3eb52 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:58:52 -1000 Subject: [PATCH 17/61] Pin codspeed-cpp to v2.1.0 release tag --- script/setup_codspeed_lib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index dbe703ff63..b34eed4816 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -29,9 +29,9 @@ import shutil import subprocess import sys -# Pin to a specific commit for reproducibility +# Pin to a specific release for reproducibility CODSPEED_CPP_REPO = "https://github.com/CodSpeedHQ/codspeed-cpp.git" -CODSPEED_CPP_SHA = "d6b4111428ae1f1667fec9bff009522378d5d347" +CODSPEED_CPP_SHA = "e633aca00da3d0ad14e7bf424d9cb47165a29028" # v2.1.0 DEFAULT_OUTPUT_DIR = "/tmp/codspeed-cpp" From 6b88820400a82b041f82f3759240be7143b869a4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 21:59:36 -1000 Subject: [PATCH 18/61] Remove debug logging now that CodSpeed is working --- .github/workflows/ci.yml | 2 -- script/cpp_benchmark.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42817bf2bf..6f20a4a7d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -338,8 +338,6 @@ jobs: - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4 - env: - CODSPEED_LOG: debug with: run: ${{ steps.build.outputs.binary }} token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index e26668a055..d479938845 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -61,9 +61,6 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> **PLATFORMIO_OPTIONS, "build_flags": PLATFORMIO_OPTIONS["build_flags"] + codspeed_flags, } - print(f"CodSpeed library: {lib_config['lib_path']}", file=sys.stderr) - print(f"CodSpeed build flags: {codspeed_flags}", file=sys.stderr) - print(f"PlatformIO options: {pio_options}", file=sys.stderr) else: benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB From ea5f62f03055f8186927980aa2d834c9eb4b1210 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:03:16 -1000 Subject: [PATCH 19/61] Add -DNDEBUG to benchmark builds The benchmark library warns "Library was built as DEBUG" without NDEBUG. This enables debug assertions and disables optimizations in the benchmark framework, causing instrumentation overhead to show up in profiles. --- script/cpp_benchmark.py | 1 + script/setup_codspeed_lib.py | 1 + 2 files changed, 2 insertions(+) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index d479938845..7487815402 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -53,6 +53,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> # CODSPEED_ROOT_DIR is used to display relative file paths in reports. project_root = Path(__file__).resolve().parent.parent codspeed_flags = [ + "-DNDEBUG", "-DCODSPEED_ENABLED", "-DCODSPEED_ANALYSIS", f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index b34eed4816..b89f5ef780 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -134,6 +134,7 @@ def _write_library_json( f"-I{core_include}", f"-I{hooks_include}", # google benchmark build flags + "-DNDEBUG", "-DHAVE_STD_REGEX", "-DHAVE_STEADY_CLOCK", "-DBENCHMARK_STATIC_DEFINE", From 93991317c4e9d2bae2519900a960ba37f710e8d4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:08:22 -1000 Subject: [PATCH 20/61] Build benchmark library with -O2 to inline instrumentation hooks --- script/setup_codspeed_lib.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index b89f5ef780..214714d698 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -134,6 +134,9 @@ def _write_library_json( f"-I{core_include}", f"-I{hooks_include}", # google benchmark build flags + # -O2 is critical: without it, instrument_hooks_start_benchmark_inline + # doesn't get inlined and shows up as overhead in profiles + "-O2", "-DNDEBUG", "-DHAVE_STD_REGEX", "-DHAVE_STEADY_CLOCK", From 10e05b5efa75ea2d9c4cd3fbb1e127a0dc73c399 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:13:11 -1000 Subject: [PATCH 21/61] Build all benchmark code with -O2 instead of -Os --- script/cpp_benchmark.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 7487815402..f30054fbad 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -27,9 +27,13 @@ CORE_BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "core" BENCHMARK_CODEGEN_COMPONENTS = BASE_CODEGEN_COMPONENTS | {"json"} PLATFORMIO_OPTIONS = { + "build_unflags": [ + "-Os", # remove default size-opt + ], "build_flags": [ - USE_TIME_TIMEZONE_FLAG, + "-O2", # optimize for speed (CodSpeed recommends RelWithDebInfo) "-g", # debug symbols for profiling + USE_TIME_TIMEZONE_FLAG, ], # Use deep+ LDF mode to ensure PlatformIO detects the benchmark # library dependency from nested includes. From 456a7d740e816d69445952b81fb2686190bef2d6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:14:51 -1000 Subject: [PATCH 22/61] Remove application loop benchmark - doesn't test real loop path --- .../core/bench_application_loop.cpp | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 tests/benchmarks/core/bench_application_loop.cpp diff --git a/tests/benchmarks/core/bench_application_loop.cpp b/tests/benchmarks/core/bench_application_loop.cpp deleted file mode 100644 index 2489d4e9ac..0000000000 --- a/tests/benchmarks/core/bench_application_loop.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include - -#include "esphome/core/application.h" -#include "esphome/core/component.h" -#include "esphome/core/hal.h" - -namespace esphome::benchmarks { - -// A minimal component with an empty loop for benchmarking dispatch overhead -class NoopComponent : public Component { - public: - void loop() override {} - float get_setup_priority() const override { return 0.0f; } - // Expose protected method for benchmarking - void set_loop_state() { this->set_component_state_(COMPONENT_STATE_LOOP); } -}; - -// --- WarnIfComponentBlockingGuard overhead --- - -static void BM_WarnIfComponentBlockingGuard(benchmark::State &state) { - NoopComponent component; - uint32_t now = millis(); - - for (auto _ : state) { - WarnIfComponentBlockingGuard guard{&component, now}; - now = guard.finish(); - benchmark::DoNotOptimize(now); - } -} -BENCHMARK(BM_WarnIfComponentBlockingGuard); - -// --- Component virtual dispatch overhead --- - -static void BM_ComponentDispatch(benchmark::State &state) { - NoopComponent component; - component.set_loop_state(); - uint32_t now = millis(); - - for (auto _ : state) { - { - WarnIfComponentBlockingGuard guard{&component, now}; - component.loop(); - now = guard.finish(); - } - benchmark::DoNotOptimize(now); - } -} -BENCHMARK(BM_ComponentDispatch); - -} // namespace esphome::benchmarks From a92147e1c9be63b493c44a78d937647e2014a86e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:27:03 -1000 Subject: [PATCH 23/61] Remove BM_ prefix, drop application loop benchmark, fix core includes - Remove BM_ prefix from all benchmark names (unnecessary convention) - Remove bench_application_loop.cpp (App needs pre_setup/setup which requires full code generation; will revisit as integration benchmark) - Fix extra_include_dirs to use os.path.relpath for sibling directories --- script/test_helpers.py | 10 ++--- .../components/api/bench_proto_decode.cpp | 12 +++--- .../components/api/bench_proto_encode.cpp | 40 +++++++++---------- .../components/api/bench_proto_varint.cpp | 32 +++++++-------- tests/benchmarks/core/bench_helpers.cpp | 8 ++-- tests/benchmarks/core/bench_scheduler.cpp | 12 +++--- 6 files changed, 55 insertions(+), 59 deletions(-) diff --git a/script/test_helpers.py b/script/test_helpers.py index 25ff062e02..36a3e4b859 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -356,13 +356,9 @@ def build_and_run( if extra_include_dirs: for d in extra_include_dirs: if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h"))): - # Use path relative to tests_dir for PlatformIO includes - try: - rel = d.relative_to(tests_dir) - includes.append(str(rel)) - except ValueError: - # Not relative to tests_dir, use absolute - includes.append(str(d)) + # ESPHome includes are relative to the config directory (tests_dir) + rel = os.path.relpath(d, tests_dir) + includes.append(rel) # Discover platform sub-components try: diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp index 8b7215497d..c9313c2fca 100644 --- a/tests/benchmarks/components/api/bench_proto_decode.cpp +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -7,7 +7,7 @@ namespace esphome::api::benchmarks { // --- HelloRequest decode (string + varint fields) --- -static void BM_Decode_HelloRequest(benchmark::State &state) { +static void Decode_HelloRequest(benchmark::State &state) { // Manually encoded HelloRequest: // field 1 (string): "aioesphomeapi" // field 2 (varint): 1 (api_version_major) @@ -25,11 +25,11 @@ static void BM_Decode_HelloRequest(benchmark::State &state) { benchmark::DoNotOptimize(msg.api_version_major); } } -BENCHMARK(BM_Decode_HelloRequest); +BENCHMARK(Decode_HelloRequest); // --- SwitchCommandRequest decode (simple command) --- -static void BM_Decode_SwitchCommandRequest(benchmark::State &state) { +static void Decode_SwitchCommandRequest(benchmark::State &state) { // field 1 (fixed32): key = 0x12345678 // field 2 (varint): state = true uint8_t encoded[] = { @@ -43,11 +43,11 @@ static void BM_Decode_SwitchCommandRequest(benchmark::State &state) { benchmark::DoNotOptimize(msg.state); } } -BENCHMARK(BM_Decode_SwitchCommandRequest); +BENCHMARK(Decode_SwitchCommandRequest); // --- LightCommandRequest decode (complex command with many fields) --- -static void BM_Decode_LightCommandRequest(benchmark::State &state) { +static void Decode_LightCommandRequest(benchmark::State &state) { uint8_t encoded[] = { // field 1: key (fixed32) = 0x11223344 0x0D, @@ -114,6 +114,6 @@ static void BM_Decode_LightCommandRequest(benchmark::State &state) { benchmark::DoNotOptimize(msg.brightness); } } -BENCHMARK(BM_Decode_LightCommandRequest); +BENCHMARK(Decode_LightCommandRequest); } // namespace esphome::api::benchmarks diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index b5a89864e7..6fc1a5449d 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -7,7 +7,7 @@ namespace esphome::api::benchmarks { // --- SensorStateResponse (highest frequency message) --- -static void BM_Encode_SensorStateResponse(benchmark::State &state) { +static void Encode_SensorStateResponse(benchmark::State &state) { APIBuffer buffer; SensorStateResponse msg; msg.key = 0x12345678; @@ -22,9 +22,9 @@ static void BM_Encode_SensorStateResponse(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_SensorStateResponse); +BENCHMARK(Encode_SensorStateResponse); -static void BM_CalculateSize_SensorStateResponse(benchmark::State &state) { +static void CalculateSize_SensorStateResponse(benchmark::State &state) { SensorStateResponse msg; msg.key = 0x12345678; msg.state = 23.5f; @@ -34,9 +34,9 @@ static void BM_CalculateSize_SensorStateResponse(benchmark::State &state) { benchmark::DoNotOptimize(msg.calculate_size()); } } -BENCHMARK(BM_CalculateSize_SensorStateResponse); +BENCHMARK(CalculateSize_SensorStateResponse); -static void BM_CalcAndEncode_SensorStateResponse(benchmark::State &state) { +static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { APIBuffer buffer; SensorStateResponse msg; msg.key = 0x12345678; @@ -51,11 +51,11 @@ static void BM_CalcAndEncode_SensorStateResponse(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_CalcAndEncode_SensorStateResponse); +BENCHMARK(CalcAndEncode_SensorStateResponse); // --- BinarySensorStateResponse --- -static void BM_Encode_BinarySensorStateResponse(benchmark::State &state) { +static void Encode_BinarySensorStateResponse(benchmark::State &state) { APIBuffer buffer; BinarySensorStateResponse msg; msg.key = 0xAABBCCDD; @@ -70,11 +70,11 @@ static void BM_Encode_BinarySensorStateResponse(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_BinarySensorStateResponse); +BENCHMARK(Encode_BinarySensorStateResponse); // --- HelloResponse (string fields) --- -static void BM_Encode_HelloResponse(benchmark::State &state) { +static void Encode_HelloResponse(benchmark::State &state) { APIBuffer buffer; HelloResponse msg; msg.api_version_major = 1; @@ -90,11 +90,11 @@ static void BM_Encode_HelloResponse(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_HelloResponse); +BENCHMARK(Encode_HelloResponse); // --- LightStateResponse (complex multi-field message) --- -static void BM_Encode_LightStateResponse(benchmark::State &state) { +static void Encode_LightStateResponse(benchmark::State &state) { APIBuffer buffer; LightStateResponse msg; msg.key = 0x11223344; @@ -119,9 +119,9 @@ static void BM_Encode_LightStateResponse(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_LightStateResponse); +BENCHMARK(Encode_LightStateResponse); -static void BM_CalculateSize_LightStateResponse(benchmark::State &state) { +static void CalculateSize_LightStateResponse(benchmark::State &state) { LightStateResponse msg; msg.key = 0x11223344; msg.state = true; @@ -141,7 +141,7 @@ static void BM_CalculateSize_LightStateResponse(benchmark::State &state) { benchmark::DoNotOptimize(msg.calculate_size()); } } -BENCHMARK(BM_CalculateSize_LightStateResponse); +BENCHMARK(CalculateSize_LightStateResponse); // --- DeviceInfoResponse (nested submessages: 20 devices + 20 areas) --- @@ -170,16 +170,16 @@ static DeviceInfoResponse make_device_info_response() { return msg; } -static void BM_CalculateSize_DeviceInfoResponse(benchmark::State &state) { +static void CalculateSize_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { benchmark::DoNotOptimize(msg.calculate_size()); } } -BENCHMARK(BM_CalculateSize_DeviceInfoResponse); +BENCHMARK(CalculateSize_DeviceInfoResponse); -static void BM_Encode_DeviceInfoResponse(benchmark::State &state) { +static void Encode_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); APIBuffer buffer; uint32_t total_size = msg.calculate_size(); @@ -191,9 +191,9 @@ static void BM_Encode_DeviceInfoResponse(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_DeviceInfoResponse); +BENCHMARK(Encode_DeviceInfoResponse); -static void BM_CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { +static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); APIBuffer buffer; @@ -205,6 +205,6 @@ static void BM_CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_CalcAndEncode_DeviceInfoResponse); +BENCHMARK(CalcAndEncode_DeviceInfoResponse); } // namespace esphome::api::benchmarks diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp index dbe03cafb5..d33dd4b5ac 100644 --- a/tests/benchmarks/components/api/bench_proto_varint.cpp +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -7,7 +7,7 @@ namespace esphome::api::benchmarks { // --- ProtoVarInt::parse() benchmarks --- -static void BM_ProtoVarInt_Parse_SingleByte(benchmark::State &state) { +static void ProtoVarInt_Parse_SingleByte(benchmark::State &state) { // Single-byte varint (0-127) — the most common case (fast path) uint8_t buf[] = {0x42}; // value = 66 @@ -16,9 +16,9 @@ static void BM_ProtoVarInt_Parse_SingleByte(benchmark::State &state) { benchmark::DoNotOptimize(result); } } -BENCHMARK(BM_ProtoVarInt_Parse_SingleByte); +BENCHMARK(ProtoVarInt_Parse_SingleByte); -static void BM_ProtoVarInt_Parse_TwoByte(benchmark::State &state) { +static void ProtoVarInt_Parse_TwoByte(benchmark::State &state) { // Two-byte varint (128-16383) uint8_t buf[] = {0x80, 0x01}; // value = 128 @@ -27,9 +27,9 @@ static void BM_ProtoVarInt_Parse_TwoByte(benchmark::State &state) { benchmark::DoNotOptimize(result); } } -BENCHMARK(BM_ProtoVarInt_Parse_TwoByte); +BENCHMARK(ProtoVarInt_Parse_TwoByte); -static void BM_ProtoVarInt_Parse_FiveByte(benchmark::State &state) { +static void ProtoVarInt_Parse_FiveByte(benchmark::State &state) { // Five-byte varint (max uint32 = 4294967295) uint8_t buf[] = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F}; @@ -38,11 +38,11 @@ static void BM_ProtoVarInt_Parse_FiveByte(benchmark::State &state) { benchmark::DoNotOptimize(result); } } -BENCHMARK(BM_ProtoVarInt_Parse_FiveByte); +BENCHMARK(ProtoVarInt_Parse_FiveByte); // --- Varint encoding benchmarks --- -static void BM_Encode_Varint_Small(benchmark::State &state) { +static void Encode_Varint_Small(benchmark::State &state) { // Value < 128 — single byte fast path APIBuffer buffer; buffer.resize(16); @@ -53,9 +53,9 @@ static void BM_Encode_Varint_Small(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_Varint_Small); +BENCHMARK(Encode_Varint_Small); -static void BM_Encode_Varint_Large(benchmark::State &state) { +static void Encode_Varint_Large(benchmark::State &state) { // Value > 128 — multi-byte slow path APIBuffer buffer; buffer.resize(16); @@ -66,9 +66,9 @@ static void BM_Encode_Varint_Large(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_Varint_Large); +BENCHMARK(Encode_Varint_Large); -static void BM_Encode_Varint_MaxUint32(benchmark::State &state) { +static void Encode_Varint_MaxUint32(benchmark::State &state) { APIBuffer buffer; buffer.resize(16); @@ -78,22 +78,22 @@ static void BM_Encode_Varint_MaxUint32(benchmark::State &state) { benchmark::DoNotOptimize(buffer.data()); } } -BENCHMARK(BM_Encode_Varint_MaxUint32); +BENCHMARK(Encode_Varint_MaxUint32); // --- ProtoSize::varint() benchmarks --- -static void BM_ProtoSize_Varint_Small(benchmark::State &state) { +static void ProtoSize_Varint_Small(benchmark::State &state) { for (auto _ : state) { benchmark::DoNotOptimize(ProtoSize::varint(42)); } } -BENCHMARK(BM_ProtoSize_Varint_Small); +BENCHMARK(ProtoSize_Varint_Small); -static void BM_ProtoSize_Varint_Large(benchmark::State &state) { +static void ProtoSize_Varint_Large(benchmark::State &state) { for (auto _ : state) { benchmark::DoNotOptimize(ProtoSize::varint(0xFFFFFFFF)); } } -BENCHMARK(BM_ProtoSize_Varint_Large); +BENCHMARK(ProtoSize_Varint_Large); } // namespace esphome::api::benchmarks diff --git a/tests/benchmarks/core/bench_helpers.cpp b/tests/benchmarks/core/bench_helpers.cpp index 15e0bc5ab1..d337a27f61 100644 --- a/tests/benchmarks/core/bench_helpers.cpp +++ b/tests/benchmarks/core/bench_helpers.cpp @@ -7,20 +7,20 @@ namespace esphome::benchmarks { // --- random_float() --- // Ported from ol.yaml:148 "Random Float Benchmark" -static void BM_RandomFloat(benchmark::State &state) { +static void RandomFloat(benchmark::State &state) { for (auto _ : state) { benchmark::DoNotOptimize(random_float()); } } -BENCHMARK(BM_RandomFloat); +BENCHMARK(RandomFloat); // --- random_uint32() --- -static void BM_RandomUint32(benchmark::State &state) { +static void RandomUint32(benchmark::State &state) { for (auto _ : state) { benchmark::DoNotOptimize(random_uint32()); } } -BENCHMARK(BM_RandomUint32); +BENCHMARK(RandomUint32); } // namespace esphome::benchmarks diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 0bd343d94b..4ce24abf94 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -7,7 +7,7 @@ namespace esphome::benchmarks { // --- Scheduler fast path: no work to do --- -static void BM_Scheduler_Call_NoWork(benchmark::State &state) { +static void Scheduler_Call_NoWork(benchmark::State &state) { Scheduler scheduler; uint32_t now = millis(); @@ -16,11 +16,11 @@ static void BM_Scheduler_Call_NoWork(benchmark::State &state) { benchmark::DoNotOptimize(now); } } -BENCHMARK(BM_Scheduler_Call_NoWork); +BENCHMARK(Scheduler_Call_NoWork); // --- Scheduler with timers: call() when timers exist but aren't due --- -static void BM_Scheduler_Call_TimersNotDue(benchmark::State &state) { +static void Scheduler_Call_TimersNotDue(benchmark::State &state) { Scheduler scheduler; Component dummy_component; @@ -37,11 +37,11 @@ static void BM_Scheduler_Call_TimersNotDue(benchmark::State &state) { benchmark::DoNotOptimize(now); } } -BENCHMARK(BM_Scheduler_Call_TimersNotDue); +BENCHMARK(Scheduler_Call_TimersNotDue); // --- Scheduler: next_schedule_in() calculation --- -static void BM_Scheduler_NextScheduleIn(benchmark::State &state) { +static void Scheduler_NextScheduleIn(benchmark::State &state) { Scheduler scheduler; Component dummy_component; @@ -58,6 +58,6 @@ static void BM_Scheduler_NextScheduleIn(benchmark::State &state) { benchmark::DoNotOptimize(result); } } -BENCHMARK(BM_Scheduler_NextScheduleIn); +BENCHMARK(Scheduler_NextScheduleIn); } // namespace esphome::benchmarks From 68cec9cc283e56776b8d2828f41777a8034fd1d2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:32:36 -1000 Subject: [PATCH 24/61] Add real Application::loop() benchmark, call original_setup() in main - Call original_setup() in benchmark main.cpp so code-generated App initialization (pre_setup, area/device registration, looping_components_ init) runs before benchmarks - Restore ApplicationLoop_Empty benchmark that calls the actual App.loop() --- tests/benchmarks/components/main.cpp | 4 ++++ .../core/bench_application_loop.cpp | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/benchmarks/core/bench_application_loop.cpp diff --git a/tests/benchmarks/components/main.cpp b/tests/benchmarks/components/main.cpp index bae733cbc6..990db92f66 100644 --- a/tests/benchmarks/components/main.cpp +++ b/tests/benchmarks/components/main.cpp @@ -20,6 +20,10 @@ void original_setup() { } void setup() { + // Run auto-generated initialization (App.pre_setup, area/device registration, + // looping_components_.init, etc.) so benchmarks that use App work correctly. + original_setup(); + // Log functions call global_logger->log_vprintf_() without a null check, // so we must set up a Logger before any test that triggers logging. static esphome::logger::Logger test_logger(0); diff --git a/tests/benchmarks/core/bench_application_loop.cpp b/tests/benchmarks/core/bench_application_loop.cpp new file mode 100644 index 0000000000..5acbe8beaa --- /dev/null +++ b/tests/benchmarks/core/bench_application_loop.cpp @@ -0,0 +1,19 @@ +#include + +#include "esphome/core/application.h" + +namespace esphome::benchmarks { + +// Benchmark Application::loop() with no registered components. +// App is initialized by original_setup() in main.cpp (code-generated +// pre_setup, area/device registration, looping_components_.init). +// This measures the baseline overhead of the main loop: scheduler, +// timing, before/after loop tasks, and yield_with_select_. +static void ApplicationLoop_Empty(benchmark::State &state) { + for (auto _ : state) { + App.loop(); + } +} +BENCHMARK(ApplicationLoop_Empty); + +} // namespace esphome::benchmarks From 621ba4d5b4ff507eea56259045c153c7d7de87be Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:34:41 -1000 Subject: [PATCH 25/61] Set loop_interval to 0 in ApplicationLoop benchmark to skip sleep --- tests/benchmarks/core/bench_application_loop.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/benchmarks/core/bench_application_loop.cpp b/tests/benchmarks/core/bench_application_loop.cpp index 5acbe8beaa..dde78ae739 100644 --- a/tests/benchmarks/core/bench_application_loop.cpp +++ b/tests/benchmarks/core/bench_application_loop.cpp @@ -10,6 +10,9 @@ namespace esphome::benchmarks { // This measures the baseline overhead of the main loop: scheduler, // timing, before/after loop tasks, and yield_with_select_. static void ApplicationLoop_Empty(benchmark::State &state) { + // Set loop interval to 0 so yield_with_select_ returns immediately + // instead of sleeping. This benchmarks the loop overhead, not the sleep. + App.set_loop_interval(0); for (auto _ : state) { App.loop(); } From e34855b92edf035f4db34d6c753d170c219434c9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:44:16 -1000 Subject: [PATCH 26/61] Add _Fresh variants to measure heap allocation cost in encode benchmarks --- .../components/api/bench_proto_encode.cpp | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index 6fc1a5449d..2cd050f820 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -36,6 +36,7 @@ static void CalculateSize_SensorStateResponse(benchmark::State &state) { } BENCHMARK(CalculateSize_SensorStateResponse); +// Steady state: buffer already allocated from previous iteration static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { APIBuffer buffer; SensorStateResponse msg; @@ -53,6 +54,24 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { } BENCHMARK(CalcAndEncode_SensorStateResponse); +// Cold path: fresh buffer each iteration (measures heap allocation) +static void CalcAndEncode_SensorStateResponse_Fresh(benchmark::State &state) { + SensorStateResponse msg; + msg.key = 0x12345678; + msg.state = 23.5f; + msg.missing_state = false; + + for (auto _ : state) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(CalcAndEncode_SensorStateResponse_Fresh); + // --- BinarySensorStateResponse --- static void Encode_BinarySensorStateResponse(benchmark::State &state) { @@ -193,6 +212,7 @@ static void Encode_DeviceInfoResponse(benchmark::State &state) { } BENCHMARK(Encode_DeviceInfoResponse); +// Steady state: buffer already allocated from previous iteration static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); APIBuffer buffer; @@ -207,4 +227,19 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { } BENCHMARK(CalcAndEncode_DeviceInfoResponse); +// Cold path: fresh buffer each iteration (measures heap allocation) +static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) { + auto msg = make_device_info_response(); + + for (auto _ : state) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } +} +BENCHMARK(CalcAndEncode_DeviceInfoResponse_Fresh); + } // namespace esphome::api::benchmarks From 5b073ca520e1cd4d490fd047d873779a362f68ac Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 22:48:39 -1000 Subject: [PATCH 27/61] Fix stale comment in main.cpp, simplify CI binary extraction --- .github/workflows/ci.yml | 4 ++-- tests/benchmarks/components/main.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f20a4a7d2..f64c6ffbb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,8 +332,8 @@ jobs: run: | . venv/bin/activate export BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) - script/cpp_benchmark.py --all --build-only 2>&1 | tee /tmp/bench-build.log - BINARY=$(tail -1 /tmp/bench-build.log) + # --build-only prints the binary path as the last line of stdout + BINARY=$(script/cpp_benchmark.py --all --build-only | tail -1) echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/tests/benchmarks/components/main.cpp b/tests/benchmarks/components/main.cpp index 990db92f66..02fdf288a5 100644 --- a/tests/benchmarks/components/main.cpp +++ b/tests/benchmarks/components/main.cpp @@ -13,7 +13,7 @@ It replaces the default ESPHome main with a benchmark runner. // ========== AUTO GENERATED INCLUDE BLOCK END ===========" void original_setup() { - // This function won't be run. + // Code-generated App initialization (pre_setup, area/device registration, etc.) // ========== AUTO GENERATED CODE BEGIN =========== // =========== AUTO GENERATED CODE END ============ From dd52c9129005a5313d70ae175827936e5edde469 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 23:32:49 -1000 Subject: [PATCH 28/61] Add inner iteration loops to amortize CodSpeed instrumentation overhead Sub-microsecond benchmarks are dominated by the ~60ns per-iteration valgrind start/stop cost in CodSpeed simulation mode. Add kInnerIterations (1000) inner loops to all fast benchmarks so the actual work dominates. Move DoNotOptimize calls outside inner loops to prevent artificial overhead. Also address review feedback: - Use tokenless CodSpeed (public repo, no CODSPEED_TOKEN needed) - Fix warning message to show component-specific path - Fix stray ". :" in error message - Verify pinned SHA on re-runs to prevent stale checkouts --- .github/workflows/ci.yml | 1 - script/setup_codspeed_lib.py | 15 +++ script/test_helpers.py | 4 +- .../components/api/bench_proto_decode.cpp | 20 +++- .../components/api/bench_proto_encode.cpp | 98 ++++++++++++++----- .../components/api/bench_proto_varint.cpp | 63 +++++++++--- tests/benchmarks/core/bench_helpers.cpp | 19 +++- tests/benchmarks/core/bench_scheduler.cpp | 21 +++- 8 files changed, 187 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f64c6ffbb4..4a5ac0e29e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,7 +340,6 @@ jobs: uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4 with: run: ${{ steps.build.outputs.binary }} - token: ${{ secrets.CODSPEED_TOKEN }} mode: simulation clang-tidy-single: diff --git a/script/setup_codspeed_lib.py b/script/setup_codspeed_lib.py index 214714d698..959c89d05b 100755 --- a/script/setup_codspeed_lib.py +++ b/script/setup_codspeed_lib.py @@ -171,6 +171,21 @@ def setup_codspeed_lib(output_dir: Path) -> None: """ if not (output_dir / ".git").exists(): _clone_repo(output_dir) + else: + # Verify the existing checkout matches the pinned SHA + result = subprocess.run( + ["git", "-C", str(output_dir), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or result.stdout.strip() != CODSPEED_CPP_SHA: + print( + f"Stale codspeed-cpp checkout, re-cloning at {CODSPEED_CPP_SHA}", + file=sys.stderr, + ) + shutil.rmtree(output_dir) + _clone_repo(output_dir) benchmark_dir = output_dir / GOOGLE_BENCHMARK_SUBDIR lib_src = benchmark_dir / "src" diff --git a/script/test_helpers.py b/script/test_helpers.py index 36a3e4b859..db1c981b82 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -81,7 +81,7 @@ def filter_components_with_files(components: list[str], tests_dir: Path) -> list filtered_components.append(component) else: print( - f"WARNING: No files found for component '{component}' in {tests_dir}, skipping.", + f"WARNING: No files found for component '{component}' in {test_dir}, skipping.", file=sys.stderr, ) return filtered_components @@ -284,7 +284,7 @@ def compile_and_get_binary( print(f"Error compiling {label} for {', '.join(components)}") return exit_code, None except Exception as e: - print(f"Error compiling {label} for {', '.join(components)}. Check path. : {e}") + print(f"Error compiling {label} for {', '.join(components)}: {e}") return EXIT_COMPILE_ERROR, None # After a successful compilation, locate the executable: diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp index c9313c2fca..2208a61692 100644 --- a/tests/benchmarks/components/api/bench_proto_decode.cpp +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -5,6 +5,11 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- HelloRequest decode (string + varint fields) --- static void Decode_HelloRequest(benchmark::State &state) { @@ -21,9 +26,12 @@ static void Decode_HelloRequest(benchmark::State &state) { for (auto _ : state) { HelloRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.api_version_major); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_HelloRequest); @@ -39,9 +47,12 @@ static void Decode_SwitchCommandRequest(benchmark::State &state) { for (auto _ : state) { SwitchCommandRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.state); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_SwitchCommandRequest); @@ -110,9 +121,12 @@ static void Decode_LightCommandRequest(benchmark::State &state) { for (auto _ : state) { LightCommandRequest msg; - msg.decode(encoded, sizeof(encoded)); + for (int i = 0; i < kInnerIterations; i++) { + msg.decode(encoded, sizeof(encoded)); + } benchmark::DoNotOptimize(msg.brightness); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Decode_LightCommandRequest); diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index 2cd050f820..d58a2b01d2 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -5,6 +5,11 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- SensorStateResponse (highest frequency message) --- static void Encode_SensorStateResponse(benchmark::State &state) { @@ -17,10 +22,13 @@ static void Encode_SensorStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_SensorStateResponse); @@ -31,8 +39,13 @@ static void CalculateSize_SensorStateResponse(benchmark::State &state) { msg.missing_state = false; for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_SensorStateResponse); @@ -45,12 +58,15 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { msg.missing_state = false; for (auto _ : state) { - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_SensorStateResponse); @@ -84,10 +100,13 @@ static void Encode_BinarySensorStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_BinarySensorStateResponse); @@ -104,10 +123,13 @@ static void Encode_HelloResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_HelloResponse); @@ -133,10 +155,13 @@ static void Encode_LightStateResponse(benchmark::State &state) { buffer.resize(size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_LightStateResponse); @@ -157,8 +182,13 @@ static void CalculateSize_LightStateResponse(benchmark::State &state) { msg.effect = StringRef::from_lit("rainbow"); for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_LightStateResponse); @@ -193,8 +223,13 @@ static void CalculateSize_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - benchmark::DoNotOptimize(msg.calculate_size()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += msg.calculate_size(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalculateSize_DeviceInfoResponse); @@ -205,10 +240,13 @@ static void Encode_DeviceInfoResponse(benchmark::State &state) { buffer.resize(total_size); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_DeviceInfoResponse); @@ -218,12 +256,15 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { APIBuffer buffer; for (auto _ : state) { - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); + for (int i = 0; i < kInnerIterations; i++) { + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse); @@ -232,13 +273,16 @@ static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - APIBuffer buffer; - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); - benchmark::DoNotOptimize(buffer.data()); + for (int i = 0; i < kInnerIterations; i++) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse_Fresh); diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp index d33dd4b5ac..440769ea67 100644 --- a/tests/benchmarks/components/api/bench_proto_varint.cpp +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -5,66 +5,84 @@ namespace esphome::api::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- ProtoVarInt::parse() benchmarks --- static void ProtoVarInt_Parse_SingleByte(benchmark::State &state) { - // Single-byte varint (0-127) — the most common case (fast path) uint8_t buf[] = {0x42}; // value = 66 for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_SingleByte); static void ProtoVarInt_Parse_TwoByte(benchmark::State &state) { - // Two-byte varint (128-16383) uint8_t buf[] = {0x80, 0x01}; // value = 128 for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_TwoByte); static void ProtoVarInt_Parse_FiveByte(benchmark::State &state) { - // Five-byte varint (max uint32 = 4294967295) uint8_t buf[] = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F}; for (auto _ : state) { - auto result = ProtoVarInt::parse(buf, sizeof(buf)); + ProtoVarIntResult result{}; + for (int i = 0; i < kInnerIterations; i++) { + result = ProtoVarInt::parse(buf, sizeof(buf)); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoVarInt_Parse_FiveByte); // --- Varint encoding benchmarks --- static void Encode_Varint_Small(benchmark::State &state) { - // Value < 128 — single byte fast path APIBuffer buffer; buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(42); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(42); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_Small); static void Encode_Varint_Large(benchmark::State &state) { - // Value > 128 — multi-byte slow path APIBuffer buffer; buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(300); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(300); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_Large); @@ -73,10 +91,13 @@ static void Encode_Varint_MaxUint32(benchmark::State &state) { buffer.resize(16); for (auto _ : state) { - ProtoWriteBuffer writer(&buffer, 0); - writer.encode_varint_raw(0xFFFFFFFF); + for (int i = 0; i < kInnerIterations; i++) { + ProtoWriteBuffer writer(&buffer, 0); + writer.encode_varint_raw(0xFFFFFFFF); + } benchmark::DoNotOptimize(buffer.data()); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Encode_Varint_MaxUint32); @@ -84,15 +105,25 @@ BENCHMARK(Encode_Varint_MaxUint32); static void ProtoSize_Varint_Small(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(ProtoSize::varint(42)); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += ProtoSize::varint(42); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoSize_Varint_Small); static void ProtoSize_Varint_Large(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(ProtoSize::varint(0xFFFFFFFF)); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += ProtoSize::varint(0xFFFFFFFF); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(ProtoSize_Varint_Large); diff --git a/tests/benchmarks/core/bench_helpers.cpp b/tests/benchmarks/core/bench_helpers.cpp index d337a27f61..5610d833e3 100644 --- a/tests/benchmarks/core/bench_helpers.cpp +++ b/tests/benchmarks/core/bench_helpers.cpp @@ -4,13 +4,23 @@ namespace esphome::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- random_float() --- // Ported from ol.yaml:148 "Random Float Benchmark" static void RandomFloat(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(random_float()); + float result = 0.0f; + for (int i = 0; i < kInnerIterations; i++) { + result += random_float(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(RandomFloat); @@ -18,8 +28,13 @@ BENCHMARK(RandomFloat); static void RandomUint32(benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(random_uint32()); + uint32_t result = 0; + for (int i = 0; i < kInnerIterations; i++) { + result += random_uint32(); + } + benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(RandomUint32); diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 4ce24abf94..8382a4b228 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -5,6 +5,11 @@ namespace esphome::benchmarks { +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 1000; + // --- Scheduler fast path: no work to do --- static void Scheduler_Call_NoWork(benchmark::State &state) { @@ -12,9 +17,12 @@ static void Scheduler_Call_NoWork(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - scheduler.call(now); + for (int i = 0; i < kInnerIterations; i++) { + scheduler.call(now); + } benchmark::DoNotOptimize(now); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_Call_NoWork); @@ -33,9 +41,12 @@ static void Scheduler_Call_TimersNotDue(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - scheduler.call(now); + for (int i = 0; i < kInnerIterations; i++) { + scheduler.call(now); + } benchmark::DoNotOptimize(now); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_Call_TimersNotDue); @@ -54,9 +65,13 @@ static void Scheduler_NextScheduleIn(benchmark::State &state) { uint32_t now = millis(); for (auto _ : state) { - auto result = scheduler.next_schedule_in(now); + optional result; + for (int i = 0; i < kInnerIterations; i++) { + result = scheduler.next_schedule_in(now); + } benchmark::DoNotOptimize(result); } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_NextScheduleIn); From 8e4091baa3f3c6d720233ef1c6f50dbc81591715 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 23:40:18 -1000 Subject: [PATCH 29/61] Increase inner iterations from 1000 to 2000 to reduce jitter --- tests/benchmarks/components/api/bench_proto_decode.cpp | 2 +- tests/benchmarks/components/api/bench_proto_encode.cpp | 2 +- tests/benchmarks/components/api/bench_proto_varint.cpp | 2 +- tests/benchmarks/core/bench_helpers.cpp | 2 +- tests/benchmarks/core/bench_scheduler.cpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp index 2208a61692..a5ecf78cde 100644 --- a/tests/benchmarks/components/api/bench_proto_decode.cpp +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -8,7 +8,7 @@ namespace esphome::api::benchmarks { // Inner iteration count to amortize CodSpeed instrumentation overhead. // Without this, the ~60ns per-iteration valgrind start/stop cost dominates // sub-microsecond benchmarks. -static constexpr int kInnerIterations = 1000; +static constexpr int kInnerIterations = 2000; // --- HelloRequest decode (string + varint fields) --- diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index d58a2b01d2..c5dbd685be 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -8,7 +8,7 @@ namespace esphome::api::benchmarks { // Inner iteration count to amortize CodSpeed instrumentation overhead. // Without this, the ~60ns per-iteration valgrind start/stop cost dominates // sub-microsecond benchmarks. -static constexpr int kInnerIterations = 1000; +static constexpr int kInnerIterations = 2000; // --- SensorStateResponse (highest frequency message) --- diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp index 440769ea67..ff4a656980 100644 --- a/tests/benchmarks/components/api/bench_proto_varint.cpp +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -8,7 +8,7 @@ namespace esphome::api::benchmarks { // Inner iteration count to amortize CodSpeed instrumentation overhead. // Without this, the ~60ns per-iteration valgrind start/stop cost dominates // sub-microsecond benchmarks. -static constexpr int kInnerIterations = 1000; +static constexpr int kInnerIterations = 2000; // --- ProtoVarInt::parse() benchmarks --- diff --git a/tests/benchmarks/core/bench_helpers.cpp b/tests/benchmarks/core/bench_helpers.cpp index 5610d833e3..c6e1e6930e 100644 --- a/tests/benchmarks/core/bench_helpers.cpp +++ b/tests/benchmarks/core/bench_helpers.cpp @@ -7,7 +7,7 @@ namespace esphome::benchmarks { // Inner iteration count to amortize CodSpeed instrumentation overhead. // Without this, the ~60ns per-iteration valgrind start/stop cost dominates // sub-microsecond benchmarks. -static constexpr int kInnerIterations = 1000; +static constexpr int kInnerIterations = 2000; // --- random_float() --- // Ported from ol.yaml:148 "Random Float Benchmark" diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 8382a4b228..d9d1575ebd 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -8,7 +8,7 @@ namespace esphome::benchmarks { // Inner iteration count to amortize CodSpeed instrumentation overhead. // Without this, the ~60ns per-iteration valgrind start/stop cost dominates // sub-microsecond benchmarks. -static constexpr int kInnerIterations = 1000; +static constexpr int kInnerIterations = 2000; // --- Scheduler fast path: no work to do --- From 5d5a48c369a6087aaa165ac303d59c94d8e1e45c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 23:52:27 -1000 Subject: [PATCH 30/61] Fix _Fresh benchmark consistency and address review nits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove inner loop from CalcAndEncode_DeviceInfoResponse_Fresh to match SensorStateResponse_Fresh — both now measure single alloc+encode per iteration as intended for heap allocation cost benchmarking - Add comments documenting why _Fresh variants skip inner loops - Add first-wins comment to load_component_yaml_configs - Clean up .gitignore template comment --- script/test_helpers.py | 5 +++++ tests/benchmarks/components/.gitignore | 3 --- .../components/api/bench_proto_encode.cpp | 21 +++++++++---------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/script/test_helpers.py b/script/test_helpers.py index db1c981b82..24d99acaeb 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -139,6 +139,11 @@ def load_component_yaml_configs(components: list[str], tests_dir: Path) -> dict: Returns: Merged dict of component configs to add to the base config """ + # Note: components are processed in sorted order. For conflicting keys + # (e.g. two benchmark.yaml files both declaring sensor:), the first + # component alphabetically wins via setdefault(). This is fine for now + # with a single benchmark component (api) but would need a real merge + # strategy if multiple components declare overlapping configs. merged: dict = {} for component in components: yaml_path = tests_dir / component / BENCHMARK_YAML_FILENAME diff --git a/tests/benchmarks/components/.gitignore b/tests/benchmarks/components/.gitignore index d8b4157aef..163bec7b80 100644 --- a/tests/benchmarks/components/.gitignore +++ b/tests/benchmarks/components/.gitignore @@ -1,5 +1,2 @@ -# Gitignore settings for ESPHome -# This is an example and may include too much for your use-case. -# You can modify this file to suit your needs. /.esphome/ /secrets.yaml diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index c5dbd685be..11e4ccc4e3 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -70,7 +70,8 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { } BENCHMARK(CalcAndEncode_SensorStateResponse); -// Cold path: fresh buffer each iteration (measures heap allocation) +// Cold path: fresh buffer each iteration (measures heap allocation). +// No inner loop — the point is to measure one alloc+encode per iteration. static void CalcAndEncode_SensorStateResponse_Fresh(benchmark::State &state) { SensorStateResponse msg; msg.key = 0x12345678; @@ -268,21 +269,19 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { } BENCHMARK(CalcAndEncode_DeviceInfoResponse); -// Cold path: fresh buffer each iteration (measures heap allocation) +// Cold path: fresh buffer each iteration (measures heap allocation). +// No inner loop — the point is to measure one alloc+encode per iteration. static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - for (int i = 0; i < kInnerIterations; i++) { - APIBuffer buffer; - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); - benchmark::DoNotOptimize(buffer.data()); - } + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); } - state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse_Fresh); From 2061fa23935ddb832642f66aa3623b5de1945fd1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 23:53:19 -1000 Subject: [PATCH 31/61] Add inner loops to _Fresh benchmarks for CodSpeed consistency Both _Fresh variants now use kInnerIterations with fresh buffer creation inside the inner loop. This amortizes CodSpeed's per-iteration instrumentation overhead while still measuring alloc+calc+encode per item. --- .../components/api/bench_proto_encode.cpp | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index 11e4ccc4e3..656c1e17db 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -70,8 +70,10 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { } BENCHMARK(CalcAndEncode_SensorStateResponse); -// Cold path: fresh buffer each iteration (measures heap allocation). -// No inner loop — the point is to measure one alloc+encode per iteration. +// Cold path: fresh buffer each iteration (measures heap allocation cost). +// Inner loop still needed to amortize CodSpeed instrumentation overhead. +// Each inner iteration creates a fresh buffer, so this measures +// alloc+calc+encode per item. static void CalcAndEncode_SensorStateResponse_Fresh(benchmark::State &state) { SensorStateResponse msg; msg.key = 0x12345678; @@ -79,13 +81,16 @@ static void CalcAndEncode_SensorStateResponse_Fresh(benchmark::State &state) { msg.missing_state = false; for (auto _ : state) { - APIBuffer buffer; - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); - benchmark::DoNotOptimize(buffer.data()); + for (int i = 0; i < kInnerIterations; i++) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_SensorStateResponse_Fresh); @@ -269,19 +274,24 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { } BENCHMARK(CalcAndEncode_DeviceInfoResponse); -// Cold path: fresh buffer each iteration (measures heap allocation). -// No inner loop — the point is to measure one alloc+encode per iteration. +// Cold path: fresh buffer each iteration (measures heap allocation cost). +// Inner loop still needed to amortize CodSpeed instrumentation overhead. +// Each inner iteration creates a fresh buffer, so this measures +// alloc+calc+encode per item. static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) { auto msg = make_device_info_response(); for (auto _ : state) { - APIBuffer buffer; - uint32_t size = msg.calculate_size(); - buffer.resize(size); - ProtoWriteBuffer writer(&buffer, 0); - msg.encode(writer); - benchmark::DoNotOptimize(buffer.data()); + for (int i = 0; i < kInnerIterations; i++) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + benchmark::DoNotOptimize(buffer.data()); + } } + state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(CalcAndEncode_DeviceInfoResponse_Fresh); From ed539e17ff3ce8221d82a1c7290a90bc349dfd99 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 23:57:39 -1000 Subject: [PATCH 32/61] Add scheduler benchmark with 5 intervals firing per call Adds Scheduler_Call_5IntervalsFiring: 5 intervals with 1ms period, time advancing each inner iteration so all 5 fire every call(). This benchmarks the real scheduler hot path where callbacks execute. --- tests/benchmarks/core/bench_scheduler.cpp | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index d9d1575ebd..babcfc0be3 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -50,6 +50,34 @@ static void Scheduler_Call_TimersNotDue(benchmark::State &state) { } BENCHMARK(Scheduler_Call_TimersNotDue); +// --- Scheduler with 5 intervals firing every call --- + +static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { + Scheduler scheduler; + Component dummy_component; + int fire_count = 0; + + // Add 5 intervals with 1ms period — they fire every call when time advances + for (int i = 0; i < 5; i++) { + scheduler.set_interval(&dummy_component, static_cast(i), 1, [&fire_count]() { fire_count++; }); + } + scheduler.process_to_add(); + + // Start at a known time so intervals are immediately due + uint32_t now = millis() + 100; + + for (auto _ : state) { + for (int i = 0; i < kInnerIterations; i++) { + scheduler.call(now); + // Advance time by 1ms so intervals are due again next call + now++; + } + benchmark::DoNotOptimize(fire_count); + } + state.SetItemsProcessed(state.iterations() * kInnerIterations); +} +BENCHMARK(Scheduler_Call_5IntervalsFiring); + // --- Scheduler: next_schedule_in() calculation --- static void Scheduler_NextScheduleIn(benchmark::State &state) { From 3fdb201f592b4e68100a1fa6f2fc83f2721edada Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:16:44 -1000 Subject: [PATCH 33/61] Suppress blocking warnings in scheduler benchmarks Under valgrind, 2000 inner iterations take long enough in wall clock to trigger WarnIfComponentBlockingGuard. Use a BenchComponent subclass that sets warn_if_blocking_over_ to UINT16_MAX to prevent log noise. --- tests/benchmarks/core/bench_scheduler.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index babcfc0be3..c32bfebcde 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -10,6 +10,15 @@ namespace esphome::benchmarks { // sub-microsecond benchmarks. static constexpr int kInnerIterations = 2000; +// Component subclass that suppresses blocking warnings. +// Under valgrind, 2000 inner iterations take long enough in wall clock +// to trigger WarnIfComponentBlockingGuard. Setting the threshold to max +// prevents log noise without affecting the benchmarked code path. +class BenchComponent : public Component { + public: + BenchComponent() { this->warn_if_blocking_over_ = UINT16_MAX; } +}; + // --- Scheduler fast path: no work to do --- static void Scheduler_Call_NoWork(benchmark::State &state) { @@ -30,7 +39,7 @@ BENCHMARK(Scheduler_Call_NoWork); static void Scheduler_Call_TimersNotDue(benchmark::State &state) { Scheduler scheduler; - Component dummy_component; + BenchComponent dummy_component; // Add some timeouts far in the future for (int i = 0; i < 10; i++) { @@ -54,7 +63,7 @@ BENCHMARK(Scheduler_Call_TimersNotDue); static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { Scheduler scheduler; - Component dummy_component; + BenchComponent dummy_component; int fire_count = 0; // Add 5 intervals with 1ms period — they fire every call when time advances @@ -82,7 +91,7 @@ BENCHMARK(Scheduler_Call_5IntervalsFiring); static void Scheduler_NextScheduleIn(benchmark::State &state) { Scheduler scheduler; - Component dummy_component; + BenchComponent dummy_component; // Add some timeouts for (int i = 0; i < 10; i++) { From 8c058cd7257df1864cbff9dac4b8ff90f6e48f7d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:20:52 -1000 Subject: [PATCH 34/61] Replace hand-encoded protobuf bytes with programmatic encoding in decode benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encode messages once in setup using the real protobuf API, then decode the resulting bytes in the benchmark loop. This keeps decode benchmarks automatically in sync with the protobuf schema — hand-encoded byte arrays would silently break when fields change. --- .../components/api/bench_proto_decode.cpp | 116 ++++++------------ 1 file changed, 38 insertions(+), 78 deletions(-) diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp index a5ecf78cde..113201dd8a 100644 --- a/tests/benchmarks/components/api/bench_proto_decode.cpp +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -10,24 +10,32 @@ namespace esphome::api::benchmarks { // sub-microsecond benchmarks. static constexpr int kInnerIterations = 2000; +// Helper: encode a message into a buffer and return it. +// Benchmarks encode once in setup, then decode the resulting bytes in a loop. +// This keeps decode benchmarks in sync with the actual protobuf schema — +// hand-encoded byte arrays would silently break when fields change. +template static APIBuffer encode_message(const T &msg) { + APIBuffer buffer; + uint32_t size = msg.calculate_size(); + buffer.resize(size); + ProtoWriteBuffer writer(&buffer, 0); + msg.encode(writer); + return buffer; +} + // --- HelloRequest decode (string + varint fields) --- static void Decode_HelloRequest(benchmark::State &state) { - // Manually encoded HelloRequest: - // field 1 (string): "aioesphomeapi" - // field 2 (varint): 1 (api_version_major) - // field 3 (varint): 10 (api_version_minor) - uint8_t encoded[] = { - 0x0A, 0x0D, // field 1, length 13 - 'a', 'i', 'o', 'e', 's', 'p', 'h', 'o', 'm', 'e', 'a', 'p', 'i', // "aioesphomeapi" - 0x10, 0x01, // field 2, value 1 - 0x18, 0x0A, // field 3, value 10 - }; + HelloRequest source; + source.client_info = StringRef::from_lit("aioesphomeapi"); + source.api_version_major = 1; + source.api_version_minor = 10; + auto encoded = encode_message(source); for (auto _ : state) { HelloRequest msg; for (int i = 0; i < kInnerIterations; i++) { - msg.decode(encoded, sizeof(encoded)); + msg.decode(encoded.data(), encoded.size()); } benchmark::DoNotOptimize(msg.api_version_major); } @@ -38,17 +46,15 @@ BENCHMARK(Decode_HelloRequest); // --- SwitchCommandRequest decode (simple command) --- static void Decode_SwitchCommandRequest(benchmark::State &state) { - // field 1 (fixed32): key = 0x12345678 - // field 2 (varint): state = true - uint8_t encoded[] = { - 0x0D, 0x78, 0x56, 0x34, 0x12, // field 1, fixed32 - 0x10, 0x01, // field 2, varint true - }; + SwitchCommandRequest source; + source.key = 0x12345678; + source.state = true; + auto encoded = encode_message(source); for (auto _ : state) { SwitchCommandRequest msg; for (int i = 0; i < kInnerIterations; i++) { - msg.decode(encoded, sizeof(encoded)); + msg.decode(encoded.data(), encoded.size()); } benchmark::DoNotOptimize(msg.state); } @@ -59,70 +65,24 @@ BENCHMARK(Decode_SwitchCommandRequest); // --- LightCommandRequest decode (complex command with many fields) --- static void Decode_LightCommandRequest(benchmark::State &state) { - uint8_t encoded[] = { - // field 1: key (fixed32) = 0x11223344 - 0x0D, - 0x44, - 0x33, - 0x22, - 0x11, - // field 2: has_state (varint) = true - 0x10, - 0x01, - // field 3: state (varint) = true - 0x18, - 0x01, - // field 4: has_brightness (varint) = true - 0x20, - 0x01, - // field 5: brightness (fixed32/float) = 0.8 - 0x2D, - 0xCD, - 0xCC, - 0x4C, - 0x3F, - // field 9: has_rgb (varint) = true - 0x48, - 0x01, - // field 10: red (fixed32/float) = 1.0 - 0x55, - 0x00, - 0x00, - 0x80, - 0x3F, - // field 11: green (fixed32/float) = 0.5 - 0x5D, - 0x00, - 0x00, - 0x00, - 0x3F, - // field 12: blue (fixed32/float) = 0.2 - 0x65, - 0xCD, - 0xCC, - 0x4C, - 0x3E, - // field 20: has_effect (varint) = true - 0xA0, - 0x01, - 0x01, - // field 21: effect (string) = "rainbow" - 0xAA, - 0x01, - 0x07, - 'r', - 'a', - 'i', - 'n', - 'b', - 'o', - 'w', - }; + LightCommandRequest source; + source.key = 0x11223344; + source.has_state = true; + source.state = true; + source.has_brightness = true; + source.brightness = 0.8f; + source.has_rgb = true; + source.red = 1.0f; + source.green = 0.5f; + source.blue = 0.2f; + source.has_effect = true; + source.effect = StringRef::from_lit("rainbow"); + auto encoded = encode_message(source); for (auto _ : state) { LightCommandRequest msg; for (int i = 0; i < kInnerIterations; i++) { - msg.decode(encoded, sizeof(encoded)); + msg.decode(encoded.data(), encoded.size()); } benchmark::DoNotOptimize(msg.brightness); } From eb944651742354a623b7d8bd4b995b4a7a0c294a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:23:15 -1000 Subject: [PATCH 35/61] Add logger benchmarks for format string hot paths Three benchmarks covering the most common ESP_LOGW patterns: - Logger_NoFormat: plain string, no format specifiers (fastest path) - Logger_3Uint32: 3x uint32_t (common for status/diagnostics) - Logger_3Float: 3x float with precision (common for sensor values) --- tests/benchmarks/core/bench_logger.cpp | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/benchmarks/core/bench_logger.cpp diff --git a/tests/benchmarks/core/bench_logger.cpp b/tests/benchmarks/core/bench_logger.cpp new file mode 100644 index 0000000000..b7e9a1c4ea --- /dev/null +++ b/tests/benchmarks/core/bench_logger.cpp @@ -0,0 +1,54 @@ +#include + +#include "esphome/core/log.h" + +namespace esphome::benchmarks { + +// Inner iteration count to amortize CodSpeed instrumentation overhead. +// Without this, the ~60ns per-iteration valgrind start/stop cost dominates +// sub-microsecond benchmarks. +static constexpr int kInnerIterations = 2000; + +static const char *const TAG = "bench"; + +// --- Log a message with no format specifiers (fastest path) --- + +static void Logger_NoFormat(benchmark::State &state) { + for (auto _ : state) { + for (int i = 0; i < kInnerIterations; i++) { + ESP_LOGW(TAG, "Something happened"); + } + } + state.SetItemsProcessed(state.iterations() * kInnerIterations); +} +BENCHMARK(Logger_NoFormat); + +// --- Log a message with 3 uint32_t format specifiers --- + +static void Logger_3Uint32(benchmark::State &state) { + uint32_t a = 12345, b = 67890, c = 99999; + for (auto _ : state) { + for (int i = 0; i < kInnerIterations; i++) { + ESP_LOGW(TAG, "Values: %" PRIu32 " %" PRIu32 " %" PRIu32, a, b, c); + } + benchmark::DoNotOptimize(a); + } + state.SetItemsProcessed(state.iterations() * kInnerIterations); +} +BENCHMARK(Logger_3Uint32); + +// --- Log a message with 3 floats (common for sensor values) --- + +static void Logger_3Float(benchmark::State &state) { + float temp = 23.456f, humidity = 67.89f, pressure = 1013.25f; + for (auto _ : state) { + for (int i = 0; i < kInnerIterations; i++) { + ESP_LOGW(TAG, "Sensor: %.2f %.1f %.2f", temp, humidity, pressure); + } + benchmark::DoNotOptimize(temp); + } + state.SetItemsProcessed(state.iterations() * kInnerIterations); +} +BENCHMARK(Logger_3Float); + +} // namespace esphome::benchmarks From ca2cf4044c89b88ed86d4db638533ddf960492d9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:35:53 -1000 Subject: [PATCH 36/61] Fix stray quote in main.cpp and scheduler time overflow - Remove trailing " from AUTO GENERATED INCLUDE BLOCK END comment - Reset scheduler `now` at start of each outer iteration to avoid unbounded growth toward UINT32_MAX across benchmark iterations --- tests/benchmarks/components/main.cpp | 2 +- tests/benchmarks/core/bench_scheduler.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/benchmarks/components/main.cpp b/tests/benchmarks/components/main.cpp index 02fdf288a5..9bc0c31a15 100644 --- a/tests/benchmarks/components/main.cpp +++ b/tests/benchmarks/components/main.cpp @@ -10,7 +10,7 @@ It replaces the default ESPHome main with a benchmark runner. // Auto generated code by esphome // ========== AUTO GENERATED INCLUDE BLOCK BEGIN =========== -// ========== AUTO GENERATED INCLUDE BLOCK END ===========" +// ========== AUTO GENERATED INCLUDE BLOCK END =========== void original_setup() { // Code-generated App initialization (pre_setup, area/device registration, etc.) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index c32bfebcde..f49b19a626 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -72,10 +72,9 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { } scheduler.process_to_add(); - // Start at a known time so intervals are immediately due - uint32_t now = millis() + 100; - for (auto _ : state) { + // Reset each outer iteration to avoid unbounded growth toward UINT32_MAX + uint32_t now = 100; for (int i = 0; i < kInnerIterations; i++) { scheduler.call(now); // Advance time by 1ms so intervals are due again next call From e45cfc5451d52bfaa92cee48c25f4ff6a71f13b6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:37:58 -1000 Subject: [PATCH 37/61] =?UTF-8?q?Revert=20scheduler=20now=20reset=20?= =?UTF-8?q?=E2=80=94=20must=20be=20monotonic=20for=20millis=5F64=5Ffrom=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit millis_64_from_() tracks 32-bit rollovers, so going backwards in time would appear as a ~49 day forward jump. Keep now monotonically increasing across all iterations. With 2000 inner iterations per outer iteration, overflow is not a practical concern. --- tests/benchmarks/core/bench_scheduler.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index f49b19a626..c0f47b0fc4 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -72,9 +72,13 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { } scheduler.process_to_add(); + // Start at a known time so intervals are immediately due. + // now increases monotonically across all iterations — this is required + // because millis_64_from_() tracks rollovers and going backwards would + // appear as a 32-bit wrap (~49 day jump forward). + uint32_t now = millis() + 100; + for (auto _ : state) { - // Reset each outer iteration to avoid unbounded growth toward UINT32_MAX - uint32_t now = 100; for (int i = 0; i < kInnerIterations; i++) { scheduler.call(now); // Advance time by 1ms so intervals are due again next call From 662780bcc61753a9aec6d5052e5c072b3f869b79 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:38:48 -1000 Subject: [PATCH 38/61] Simplify scheduler now comment --- tests/benchmarks/core/bench_scheduler.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index c0f47b0fc4..897e3adc98 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -72,10 +72,7 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { } scheduler.process_to_add(); - // Start at a known time so intervals are immediately due. - // now increases monotonically across all iterations — this is required - // because millis_64_from_() tracks rollovers and going backwards would - // appear as a 32-bit wrap (~49 day jump forward). + // Must be monotonic — millis_64_from_() tracks rollovers. uint32_t now = millis() + 100; for (auto _ : state) { From 4565167bec8d98f664ecfed4115c9bcdb22ed336 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:53:10 -1000 Subject: [PATCH 39/61] Fix scheduler benchmark triggering warn_blocking WarnIfComponentBlockingGuard compares the `now` passed to scheduler.call() against real millis() in finish(). Using fake time ahead of real millis() caused uint32_t underflow in the guard, triggering blocking warnings. Fix by reading real millis() at the start of each outer iteration so fake time stays close to wall clock. --- tests/benchmarks/core/bench_scheduler.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 897e3adc98..40bed05327 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -72,13 +72,13 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { } scheduler.process_to_add(); - // Must be monotonic — millis_64_from_() tracks rollovers. - uint32_t now = millis() + 100; - for (auto _ : state) { + // Use real millis() each outer iteration so our fake time stays close + // to wall clock — WarnIfComponentBlockingGuard compares the `now` we + // pass to scheduler.call() against real millis() in finish(). + uint32_t now = millis(); for (int i = 0; i < kInnerIterations; i++) { scheduler.call(now); - // Advance time by 1ms so intervals are due again next call now++; } benchmark::DoNotOptimize(fire_count); From d7be19703bdcb56e8e7a7097bc6a568ccf7380c7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 00:54:29 -1000 Subject: [PATCH 40/61] Fix scheduler intervals not firing and warn_blocking Use interval=0 so all 5 intervals fire unconditionally every call(). Pass real millis() to scheduler.call() so WarnIfComponentBlockingGuard doesn't see fake time ahead of wall clock (which causes uint32_t underflow in the blocking time calculation). --- tests/benchmarks/core/bench_scheduler.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 40bed05327..6282cc1597 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -66,20 +66,18 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { BenchComponent dummy_component; int fire_count = 0; - // Add 5 intervals with 1ms period — they fire every call when time advances + // Add 5 intervals with 0ms period — they fire every call() unconditionally. + // WarnIfComponentBlockingGuard compares the `now` we pass against real + // millis() in finish(), so we must pass real millis() to avoid underflow. + // With interval=0, all 5 fire every call without needing to advance time. for (int i = 0; i < 5; i++) { - scheduler.set_interval(&dummy_component, static_cast(i), 1, [&fire_count]() { fire_count++; }); + scheduler.set_interval(&dummy_component, static_cast(i), 0, [&fire_count]() { fire_count++; }); } scheduler.process_to_add(); for (auto _ : state) { - // Use real millis() each outer iteration so our fake time stays close - // to wall clock — WarnIfComponentBlockingGuard compares the `now` we - // pass to scheduler.call() against real millis() in finish(). - uint32_t now = millis(); for (int i = 0; i < kInnerIterations; i++) { - scheduler.call(now); - now++; + scheduler.call(millis()); } benchmark::DoNotOptimize(fire_count); } From ec60e2c228177c4422d0879573030c61218b5105 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:00:06 -1000 Subject: [PATCH 41/61] Fix scheduler benchmark: use fake time with guard disabled at compile time interval=0 causes infinite loop (reschedules at same time, never breaks). interval=1 with millis() doesn't work (real time doesn't advance fast enough between inner iterations for intervals to re-fire). Solution: use interval=1 with monotonically increasing fake time (now++) and disable WarnIfComponentBlockingGuard at compile time via -DWARN_IF_BLOCKING_OVER_MS=UINT32_MAX in benchmark build flags. This prevents the guard's (millis() - started_) underflow when fake time exceeds real millis(). --- script/cpp_benchmark.py | 4 ++++ tests/benchmarks/core/bench_scheduler.cpp | 16 ++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index f30054fbad..17613c2842 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -34,6 +34,10 @@ PLATFORMIO_OPTIONS = { "-O2", # optimize for speed (CodSpeed recommends RelWithDebInfo) "-g", # debug symbols for profiling USE_TIME_TIMEZONE_FLAG, + # Disable WarnIfComponentBlockingGuard check — scheduler benchmarks + # use fake monotonic time ahead of real millis(), causing uint32_t + # underflow in the guard's (millis() - started_) calculation. + "-DWARN_IF_BLOCKING_OVER_MS=UINT32_MAX", ], # Use deep+ LDF mode to ensure PlatformIO detects the benchmark # library dependency from nested includes. diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 6282cc1597..4e3ef57084 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -66,18 +66,22 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { BenchComponent dummy_component; int fire_count = 0; - // Add 5 intervals with 0ms period — they fire every call() unconditionally. - // WarnIfComponentBlockingGuard compares the `now` we pass against real - // millis() in finish(), so we must pass real millis() to avoid underflow. - // With interval=0, all 5 fire every call without needing to advance time. + // Add 5 intervals with 1ms period — they fire every call when time advances. + // We use monotonically increasing fake time (now++) so intervals reliably fire. + // WARN_IF_BLOCKING_OVER_MS=UINT32_MAX in benchmark build flags prevents the + // WarnIfComponentBlockingGuard from triggering when fake time exceeds real millis(). + // Note: interval=0 causes an infinite loop (reschedules at same time, never breaks). for (int i = 0; i < 5; i++) { - scheduler.set_interval(&dummy_component, static_cast(i), 0, [&fire_count]() { fire_count++; }); + scheduler.set_interval(&dummy_component, static_cast(i), 1, [&fire_count]() { fire_count++; }); } scheduler.process_to_add(); + uint32_t now = millis() + 100; + for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { - scheduler.call(millis()); + scheduler.call(now); + now++; } benchmark::DoNotOptimize(fire_count); } From 1f9380ddc06a30e3cf2ddd7e92781942bd59b4be Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:05:04 -1000 Subject: [PATCH 42/61] Fix scheduler firing benchmark: no inner loop, warm-up call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Revert component.h change (no core changes for benchmarks) - Remove -DWARN_IF_BLOCKING_OVER_MS from build flags (can't shadow constexpr) - Drop inner loop — 5 heap pops + callbacks + pushes per call is well above CodSpeed's 60ns instrumentation overhead - Add warm-up call before benchmark loop to trigger the blocking guard once and ramp the threshold - interval=0 causes infinite loop, must use interval=1 with fake time --- script/cpp_benchmark.py | 4 ---- tests/benchmarks/core/bench_scheduler.cpp | 20 +++++++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 17613c2842..f30054fbad 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -34,10 +34,6 @@ PLATFORMIO_OPTIONS = { "-O2", # optimize for speed (CodSpeed recommends RelWithDebInfo) "-g", # debug symbols for profiling USE_TIME_TIMEZONE_FLAG, - # Disable WarnIfComponentBlockingGuard check — scheduler benchmarks - # use fake monotonic time ahead of real millis(), causing uint32_t - # underflow in the guard's (millis() - started_) calculation. - "-DWARN_IF_BLOCKING_OVER_MS=UINT32_MAX", ], # Use deep+ LDF mode to ensure PlatformIO detects the benchmark # library dependency from nested includes. diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 4e3ef57084..1b78c44376 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -67,25 +67,27 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { int fire_count = 0; // Add 5 intervals with 1ms period — they fire every call when time advances. - // We use monotonically increasing fake time (now++) so intervals reliably fire. - // WARN_IF_BLOCKING_OVER_MS=UINT32_MAX in benchmark build flags prevents the - // WarnIfComponentBlockingGuard from triggering when fake time exceeds real millis(). - // Note: interval=0 causes an infinite loop (reschedules at same time, never breaks). + // No inner loop needed: 5 heap pops + 5 callbacks + 5 heap pushes per call + // is well above CodSpeed's ~60ns instrumentation overhead. + // Note: interval=0 causes infinite loop (reschedules at same now, never breaks). for (int i = 0; i < 5; i++) { scheduler.set_interval(&dummy_component, static_cast(i), 1, [&fire_count]() { fire_count++; }); } scheduler.process_to_add(); + // Monotonically increasing fake time so intervals are due every call. + // Can't use real millis() — it doesn't advance fast enough between calls. + // Warm-up call outside the benchmark to trigger the blocking guard once + // and ramp the component's warn_if_blocking_over_ threshold to max. uint32_t now = millis() + 100; + scheduler.call(now); + now++; for (auto _ : state) { - for (int i = 0; i < kInnerIterations; i++) { - scheduler.call(now); - now++; - } + scheduler.call(now); + now++; benchmark::DoNotOptimize(fire_count); } - state.SetItemsProcessed(state.iterations() * kInnerIterations); } BENCHMARK(Scheduler_Call_5IntervalsFiring); From 885d7c3938c31d7d6c5afdef76368021e1eadf9b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:10:57 -1000 Subject: [PATCH 43/61] [core] Fix uint32_t underflow in WarnIfComponentBlockingGuard::finish() When curr_time (from millis()) is less than started_ (the `now` passed to scheduler.call()), the subtraction wraps to a huge value (~4 billion). This triggers spurious blocking warnings with nonsensical times. This can happen when the scheduler's execute_item_() returns a millis() value that subsequent items use as their guard start, but the next scheduler.call() passes a `now` value from a slightly different source. Fix by skipping the blocking check when curr_time < started_ (underflow). Also restore the scheduler firing benchmark to use intervals with monotonically increasing fake time, now that the guard handles underflow. --- esphome/core/component.h | 11 ++++++++--- tests/benchmarks/core/bench_scheduler.cpp | 11 +++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/esphome/core/component.h b/esphome/core/component.h index 5fdf23e128..64f9971627 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -594,12 +594,17 @@ class WarnIfComponentBlockingGuard { // Inlined: the fast path is just millis() + subtract + compare inline uint32_t HOT finish() { uint32_t curr_time = millis(); - uint32_t blocking_time = curr_time - this->started_; #ifdef USE_RUNTIME_STATS this->record_runtime_stats_(); #endif - if (blocking_time > WARN_IF_BLOCKING_OVER_MS) [[unlikely]] { - warn_blocking(this->component_, blocking_time); + // Guard against underflow: if curr_time < started_, the subtraction wraps + // to a huge value. This can happen when the scheduler passes a `now` value + // slightly ahead of real millis() (e.g. from execute_item_ return values). + if (curr_time >= this->started_) [[likely]] { + uint32_t blocking_time = curr_time - this->started_; + if (blocking_time > WARN_IF_BLOCKING_OVER_MS) [[unlikely]] { + warn_blocking(this->component_, blocking_time); + } } return curr_time; } diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 1b78c44376..160ab3c123 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -67,21 +67,16 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { int fire_count = 0; // Add 5 intervals with 1ms period — they fire every call when time advances. - // No inner loop needed: 5 heap pops + 5 callbacks + 5 heap pushes per call - // is well above CodSpeed's ~60ns instrumentation overhead. + // We use monotonically increasing fake time (now++) so intervals reliably fire. + // The underflow guard in WarnIfComponentBlockingGuard::finish() (curr_time >= started_) + // prevents warn_blocking from firing when fake time exceeds real millis(). // Note: interval=0 causes infinite loop (reschedules at same now, never breaks). for (int i = 0; i < 5; i++) { scheduler.set_interval(&dummy_component, static_cast(i), 1, [&fire_count]() { fire_count++; }); } scheduler.process_to_add(); - // Monotonically increasing fake time so intervals are due every call. - // Can't use real millis() — it doesn't advance fast enough between calls. - // Warm-up call outside the benchmark to trigger the blocking guard once - // and ramp the component's warn_if_blocking_over_ threshold to max. uint32_t now = millis() + 100; - scheduler.call(now); - now++; for (auto _ : state) { scheduler.call(now); From 2785edaac878d59bdfe6351e317465f881f5403f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:11:48 -1000 Subject: [PATCH 44/61] [core] Clamp underflowed blocking_time in warn_blocking cold path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When millis() < started_ (e.g. scheduler passes a now value slightly ahead of real millis()), the uint32_t subtraction in finish() wraps to ~4 billion. This caused warn_blocking to fire on every call since the underflowed value always exceeds the uint16_t threshold max (65535). Fix by clamping blocking_time to uint16_t max in the cold warn_blocking path. After one warning, should_warn_of_blocking() saturates the threshold to 65535 and subsequent clamped values (65535) don't exceed it. Zero cost on the hot path — the clamp is in the noinline cold function. --- esphome/core/component.cpp | 7 +++++++ esphome/core/component.h | 11 +++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index bfe9beb272..172842ee3d 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -512,6 +512,13 @@ void PollingComponent::set_update_interval(uint32_t update_interval) { this->upd void __attribute__((noinline, cold)) WarnIfComponentBlockingGuard::warn_blocking(Component *component, uint32_t blocking_time) { + // Clamp underflowed values: if millis() < started_ (e.g. scheduler passes + // a `now` slightly ahead of real millis()), the subtraction wraps to ~4 billion. + // Clamping to uint16_t max lets should_warn_of_blocking() saturate the + // threshold and suppress further warnings. + if (blocking_time > std::numeric_limits::max()) { + blocking_time = std::numeric_limits::max(); + } bool should_warn; if (component != nullptr) { should_warn = component->should_warn_of_blocking(blocking_time); diff --git a/esphome/core/component.h b/esphome/core/component.h index 64f9971627..5fdf23e128 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -594,17 +594,12 @@ class WarnIfComponentBlockingGuard { // Inlined: the fast path is just millis() + subtract + compare inline uint32_t HOT finish() { uint32_t curr_time = millis(); + uint32_t blocking_time = curr_time - this->started_; #ifdef USE_RUNTIME_STATS this->record_runtime_stats_(); #endif - // Guard against underflow: if curr_time < started_, the subtraction wraps - // to a huge value. This can happen when the scheduler passes a `now` value - // slightly ahead of real millis() (e.g. from execute_item_ return values). - if (curr_time >= this->started_) [[likely]] { - uint32_t blocking_time = curr_time - this->started_; - if (blocking_time > WARN_IF_BLOCKING_OVER_MS) [[unlikely]] { - warn_blocking(this->component_, blocking_time); - } + if (blocking_time > WARN_IF_BLOCKING_OVER_MS) [[unlikely]] { + warn_blocking(this->component_, blocking_time); } return curr_time; } From e01670eed897821979ede5ffdaa7788d63f835ce Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:14:39 -1000 Subject: [PATCH 45/61] Revert core changes, use intervals with fake time for scheduler benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warn_blocking underflow only happens with fake time in benchmarks, not in production (millis() is monotonic). Accept the consistent overhead from one warning per call — CodSpeed regression detection works on relative changes, not absolute values. --- esphome/core/component.cpp | 7 ------- tests/benchmarks/core/bench_scheduler.cpp | 11 ++++++----- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index 172842ee3d..bfe9beb272 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -512,13 +512,6 @@ void PollingComponent::set_update_interval(uint32_t update_interval) { this->upd void __attribute__((noinline, cold)) WarnIfComponentBlockingGuard::warn_blocking(Component *component, uint32_t blocking_time) { - // Clamp underflowed values: if millis() < started_ (e.g. scheduler passes - // a `now` slightly ahead of real millis()), the subtraction wraps to ~4 billion. - // Clamping to uint16_t max lets should_warn_of_blocking() saturate the - // threshold and suppress further warnings. - if (blocking_time > std::numeric_limits::max()) { - blocking_time = std::numeric_limits::max(); - } bool should_warn; if (component != nullptr) { should_warn = component->should_warn_of_blocking(blocking_time); diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 160ab3c123..0ae8b4add0 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -66,11 +66,12 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { BenchComponent dummy_component; int fire_count = 0; - // Add 5 intervals with 1ms period — they fire every call when time advances. - // We use monotonically increasing fake time (now++) so intervals reliably fire. - // The underflow guard in WarnIfComponentBlockingGuard::finish() (curr_time >= started_) - // prevents warn_blocking from firing when fake time exceeds real millis(). - // Note: interval=0 causes infinite loop (reschedules at same now, never breaks). + // Benchmarks the heap-based scheduler dispatch with 5 callbacks firing. + // Uses monotonically increasing fake time so intervals reliably fire every call. + // The first item per call triggers one WarnIfComponentBlockingGuard warning + // (fake now > real millis() causes underflow in finish()), but this is + // consistent overhead per iteration so CodSpeed regression detection works. + // interval=0 would cause an infinite loop (reschedules at same now). for (int i = 0; i < 5; i++) { scheduler.set_interval(&dummy_component, static_cast(i), 1, [&fire_count]() { fire_count++; }); } From 804e2330ecdc7fa002dba681d996c928c08436d5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:16:34 -1000 Subject: [PATCH 46/61] Ifdef out WarnIfComponentBlockingGuard for benchmark builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add USE_BENCHMARK define to benchmark build flags. Guard the warn_blocking call in finish() with #ifndef USE_BENCHMARK so scheduler benchmarks using fake monotonic time don't trigger the underflow (fake now > real millis()). Remove BenchComponent — no longer needed with the ifdef. --- esphome/core/component.h | 2 ++ script/cpp_benchmark.py | 1 + tests/benchmarks/core/bench_scheduler.cpp | 20 +++++--------------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/esphome/core/component.h b/esphome/core/component.h index 5fdf23e128..557ba09bbc 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -598,9 +598,11 @@ class WarnIfComponentBlockingGuard { #ifdef USE_RUNTIME_STATS this->record_runtime_stats_(); #endif +#ifndef USE_BENCHMARK if (blocking_time > WARN_IF_BLOCKING_OVER_MS) [[unlikely]] { warn_blocking(this->component_, blocking_time); } +#endif return curr_time; } diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index f30054fbad..bd92266ea6 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -34,6 +34,7 @@ PLATFORMIO_OPTIONS = { "-O2", # optimize for speed (CodSpeed recommends RelWithDebInfo) "-g", # debug symbols for profiling USE_TIME_TIMEZONE_FLAG, + "-DUSE_BENCHMARK", # disable WarnIfComponentBlockingGuard in finish() ], # Use deep+ LDF mode to ensure PlatformIO detects the benchmark # library dependency from nested includes. diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 0ae8b4add0..3d2cd0bda2 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -10,15 +10,6 @@ namespace esphome::benchmarks { // sub-microsecond benchmarks. static constexpr int kInnerIterations = 2000; -// Component subclass that suppresses blocking warnings. -// Under valgrind, 2000 inner iterations take long enough in wall clock -// to trigger WarnIfComponentBlockingGuard. Setting the threshold to max -// prevents log noise without affecting the benchmarked code path. -class BenchComponent : public Component { - public: - BenchComponent() { this->warn_if_blocking_over_ = UINT16_MAX; } -}; - // --- Scheduler fast path: no work to do --- static void Scheduler_Call_NoWork(benchmark::State &state) { @@ -39,7 +30,7 @@ BENCHMARK(Scheduler_Call_NoWork); static void Scheduler_Call_TimersNotDue(benchmark::State &state) { Scheduler scheduler; - BenchComponent dummy_component; + Component dummy_component; // Add some timeouts far in the future for (int i = 0; i < 10; i++) { @@ -63,14 +54,13 @@ BENCHMARK(Scheduler_Call_TimersNotDue); static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { Scheduler scheduler; - BenchComponent dummy_component; + Component dummy_component; int fire_count = 0; // Benchmarks the heap-based scheduler dispatch with 5 callbacks firing. // Uses monotonically increasing fake time so intervals reliably fire every call. - // The first item per call triggers one WarnIfComponentBlockingGuard warning - // (fake now > real millis() causes underflow in finish()), but this is - // consistent overhead per iteration so CodSpeed regression detection works. + // USE_BENCHMARK ifdef in component.h disables WarnIfComponentBlockingGuard + // (fake now > real millis() would cause underflow in finish()). // interval=0 would cause an infinite loop (reschedules at same now). for (int i = 0; i < 5; i++) { scheduler.set_interval(&dummy_component, static_cast(i), 1, [&fire_count]() { fire_count++; }); @@ -91,7 +81,7 @@ BENCHMARK(Scheduler_Call_5IntervalsFiring); static void Scheduler_NextScheduleIn(benchmark::State &state) { Scheduler scheduler; - BenchComponent dummy_component; + Component dummy_component; // Add some timeouts for (int i = 0; i < 10; i++) { From a577715d5a7cd6352880f3ccea460d4c5f6b0f23 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 16 Mar 2026 23:00:18 -1000 Subject: [PATCH 47/61] [scheduler] Skip lock in process_to_add/cleanup/defer when nothing to do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add fast-path checks in process_to_add(), cleanup_(), and process_defer_queue_() to skip mutex acquisition when there is nothing to process. Uses std::atomic counters on platforms with atomics support (ESP32, host), falls back to always taking the lock on platforms without atomics (LibreTiny). Single-threaded platforms (ESP8266, RP2040, nRF52) check directly since there are no concurrent writers. Also migrates existing to_remove_ to the same atomic pattern and moves the defer_queue_.size() snapshot under the lock — both were previously plain reads without the lock while being modified cross-thread. --- esphome/core/scheduler.cpp | 40 +++++++---- esphome/core/scheduler.h | 143 ++++++++++++++++++++++++++++++++++--- 2 files changed, 159 insertions(+), 24 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 72b183384e..a3a767a17b 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -212,6 +212,14 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type this->cancel_item_locked_(component, name_type, static_name, hash_or_id, type); } target->push_back(item); + if (target == &this->to_add_) { + this->to_add_count_increment_(); + } +#ifndef ESPHOME_THREAD_SINGLE + else { + this->defer_count_increment_(); + } +#endif } void HOT Scheduler::set_timeout(Component *component, const char *name, uint32_t timeout, @@ -422,7 +430,7 @@ void Scheduler::full_cleanup_removed_items_() { this->items_.erase(this->items_.begin() + write, this->items_.end()); // Rebuild the heap structure since items are no longer in heap order std::make_heap(this->items_.begin(), this->items_.end(), SchedulerItem::cmp); - this->to_remove_ = 0; + this->to_remove_clear_(); } #ifndef ESPHOME_THREAD_SINGLE @@ -503,7 +511,7 @@ void HOT Scheduler::call(uint32_t now) { // If we still have too many cancelled items, do a full cleanup // This only happens if cancelled items are stuck in the middle/bottom of the heap - if (this->to_remove_ >= MAX_LOGICALLY_DELETED_ITEMS) { + if (this->to_remove_count_() >= MAX_LOGICALLY_DELETED_ITEMS) { this->full_cleanup_removed_items_(); } while (!this->items_.empty()) { @@ -530,7 +538,7 @@ void HOT Scheduler::call(uint32_t now) { LockGuard guard{this->lock_}; if (is_item_removed_locked_(item)) { this->recycle_item_main_loop_(this->pop_raw_locked_()); - this->to_remove_--; + this->to_remove_decrement_(); continue; } } @@ -539,7 +547,7 @@ void HOT Scheduler::call(uint32_t now) { if (is_item_removed_(item)) { LockGuard guard{this->lock_}; this->recycle_item_main_loop_(this->pop_raw_locked_()); - this->to_remove_--; + this->to_remove_decrement_(); continue; } #endif @@ -567,7 +575,7 @@ void HOT Scheduler::call(uint32_t now) { if (this->is_item_removed_locked_(executed_item)) { // We were removed/cancelled in the function call, recycle and continue - this->to_remove_--; + this->to_remove_decrement_(); this->recycle_item_main_loop_(executed_item); continue; } @@ -577,6 +585,7 @@ void HOT Scheduler::call(uint32_t now) { // Add new item directly to to_add_ // since we have the lock held this->to_add_.push_back(executed_item); + this->to_add_count_increment_(); } else { // Timeout completed - recycle it this->recycle_item_main_loop_(executed_item); @@ -605,6 +614,10 @@ void HOT Scheduler::call(uint32_t now) { #endif } void HOT Scheduler::process_to_add() { + // Fast path: skip lock acquisition when nothing to add. + // Worst case is a one-loop-iteration delay before newly added items are processed. + if (this->to_add_empty_()) + return; LockGuard guard{this->lock_}; for (auto *&it : this->to_add_) { if (is_item_removed_locked_(it)) { @@ -618,16 +631,13 @@ void HOT Scheduler::process_to_add() { std::push_heap(this->items_.begin(), this->items_.end(), SchedulerItem::cmp); } this->to_add_.clear(); + this->to_add_count_clear_(); } size_t HOT Scheduler::cleanup_() { - // Fast path: if nothing to remove, just return the current size - // Reading to_remove_ without lock is safe because: - // 1. We only call this from the main thread during call() - // 2. If it's 0, there's definitely nothing to cleanup - // 3. If it becomes non-zero after we check, cleanup will happen on the next loop iteration - // 4. Not all platforms support atomics, so we accept this race in favor of performance - // 5. The worst case is a one-loop-iteration delay in cleanup, which is harmless - if (this->to_remove_ == 0) + // Fast path: if nothing to remove, just return the current size. + // Uses atomic load on platforms with atomics, falls back to always taking the lock otherwise. + // Worst case is a one-loop-iteration delay in cleanup. + if (this->to_remove_empty_()) return this->items_.size(); // We must hold the lock for the entire cleanup operation because: @@ -643,7 +653,7 @@ size_t HOT Scheduler::cleanup_() { SchedulerItem *item = this->items_[0]; if (!this->is_item_removed_locked_(item)) break; - this->to_remove_--; + this->to_remove_decrement_(); this->recycle_item_main_loop_(this->pop_raw_locked_()); } return this->items_.size(); @@ -699,7 +709,7 @@ bool HOT Scheduler::cancel_item_locked_(Component *component, NameType name_type size_t heap_cancelled = this->mark_matching_items_removed_locked_(this->items_, component, name_type, static_name, hash_or_id, type, match_retry); total_cancelled += heap_cancelled; - this->to_remove_ += heap_cancelled; + this->to_remove_add_(heap_cancelled); } // Cancel items in to_add_ diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 0476513bb9..f0589bef24 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -395,15 +395,9 @@ class Scheduler { // erase() on every pop, which would be O(n). The queue is processed once per loop - // any items added during processing are left for the next loop iteration. - // Snapshot the queue end point - only process items that existed at loop start - // Items added during processing (by callbacks or other threads) run next loop - // No lock needed: single consumer (main loop), stale read just means we process less this iteration - size_t defer_queue_end = this->defer_queue_.size(); - // Fast path: nothing to process, avoid lock entirely. - // Safe without lock: single consumer (main loop) reads front_, and a stale size() read - // from a concurrent push can only make us see fewer items — they'll be processed next loop. - if (this->defer_queue_front_ >= defer_queue_end) + // Worst case is a one-loop-iteration delay before newly deferred items are processed. + if (this->defer_empty_()) return; // Merge lock acquisitions: instead of separate locks for move-out and recycle (2N+1 total), @@ -412,6 +406,13 @@ class Scheduler { SchedulerItem *item; this->lock_.lock(); + // Reset counter and snapshot queue end under lock + this->defer_count_clear_(); + size_t defer_queue_end = this->defer_queue_.size(); + if (this->defer_queue_front_ >= defer_queue_end) { + this->lock_.unlock(); + return; + } while (this->defer_queue_front_ < defer_queue_end) { // Take ownership of the item, leaving nullptr in the vector slot. // This is safe because: @@ -527,14 +528,138 @@ class Scheduler { Mutex lock_; std::vector items_; std::vector to_add_; + + // Fast-path counter for process_to_add() to skip lock when nothing to add. + // Uses atomic on platforms that support it, plain uint32_t + lock otherwise. +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + std::atomic to_add_count_{0}; +#else + uint32_t to_add_count_{0}; +#endif + + // Lock-free check if to_add_ is empty (for fast-path in process_to_add) + bool to_add_empty_() const { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + return this->to_add_count_.load(std::memory_order_acquire) == 0; +#elif defined(ESPHOME_THREAD_SINGLE) + // Single-threaded: no concurrent writers, direct check is safe + return this->to_add_.empty(); +#else + // Multi-threaded without atomics: must take lock + return false; // Always take the lock path +#endif + } + + // Increment to_add_count_ (caller must hold lock on non-atomic platforms) + void to_add_count_increment_() { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + this->to_add_count_.fetch_add(1, std::memory_order_release); +#else + this->to_add_count_++; +#endif + } + + // Reset to_add_count_ (caller must hold lock) + void to_add_count_clear_() { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + this->to_add_count_.store(0, std::memory_order_relaxed); +#else + this->to_add_count_ = 0; +#endif + } + #ifndef ESPHOME_THREAD_SINGLE // Single-core platforms don't need the defer queue and save ~32 bytes of RAM // Using std::vector instead of std::deque avoids 512-byte chunked allocations // Index tracking avoids O(n) erase() calls when draining the queue each loop std::vector defer_queue_; // FIFO queue for defer() calls size_t defer_queue_front_{0}; // Index of first valid item in defer_queue_ (tracks consumed items) -#endif /* ESPHOME_THREAD_SINGLE */ + + // Fast-path counter for process_defer_queue_() to skip lock when nothing to process. +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + std::atomic defer_count_{0}; +#else + uint32_t defer_count_{0}; +#endif + + bool defer_empty_() const { + // defer_queue_ only exists on multi-threaded platforms, so no ESPHOME_THREAD_SINGLE path + // ESPHOME_THREAD_MULTI_NO_ATOMICS: always take the lock +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + return this->defer_count_.load(std::memory_order_acquire) == 0; +#else + return false; +#endif + } + + void defer_count_increment_() { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + this->defer_count_.fetch_add(1, std::memory_order_release); +#else + this->defer_count_++; +#endif + } + + void defer_count_clear_() { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + this->defer_count_.store(0, std::memory_order_relaxed); +#else + this->defer_count_ = 0; +#endif + } + +#endif /* ESPHOME_THREAD_SINGLE */ + + // Counter for items marked for removal. Incremented cross-thread in cancel_item_locked_(), + // read without lock in cleanup_() fast path. +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + std::atomic to_remove_{0}; +#else uint32_t to_remove_{0}; +#endif + + // Lock-free check if there are items to remove (for fast-path in cleanup_) + bool to_remove_empty_() const { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + return this->to_remove_.load(std::memory_order_acquire) == 0; +#elif defined(ESPHOME_THREAD_SINGLE) + return this->to_remove_ == 0; +#else + return false; // Always take the lock path +#endif + } + + void to_remove_add_(uint32_t count) { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + this->to_remove_.fetch_add(count, std::memory_order_release); +#else + this->to_remove_ += count; +#endif + } + + void to_remove_decrement_() { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + this->to_remove_.fetch_sub(1, std::memory_order_relaxed); +#else + this->to_remove_--; +#endif + } + + void to_remove_clear_() { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + this->to_remove_.store(0, std::memory_order_relaxed); +#else + this->to_remove_ = 0; +#endif + } + + uint32_t to_remove_count_() const { +#ifdef ESPHOME_THREAD_MULTI_ATOMICS + return this->to_remove_.load(std::memory_order_relaxed); +#else + return this->to_remove_; +#endif + } // Memory pool for recycling SchedulerItem objects to reduce heap churn. // Design decisions: From d5f2c93e68b4425a43a261f3573388c1b925f71d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:38:19 -1000 Subject: [PATCH 48/61] Add benchmarks for set_timeout, set_interval, and defer registration Measures the cost of registering scheduler items: - Scheduler_SetTimeout: set_timeout with 1s delay (heap path) - Scheduler_SetInterval: set_interval with 1s period (heap path + offset calc) - Scheduler_Defer: Component::defer (set_timeout with delay=0) Uses i%5 for IDs to exercise the cancel-existing-timer path that happens when re-registering with the same ID. --- tests/benchmarks/core/bench_scheduler.cpp | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 3d2cd0bda2..1b61223f75 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -77,6 +77,55 @@ static void Scheduler_Call_5IntervalsFiring(benchmark::State &state) { } BENCHMARK(Scheduler_Call_5IntervalsFiring); +// --- Scheduler: set_timeout registration --- + +static void Scheduler_SetTimeout(benchmark::State &state) { + Scheduler scheduler; + Component dummy_component; + + for (auto _ : state) { + for (int i = 0; i < kInnerIterations; i++) { + scheduler.set_timeout(&dummy_component, static_cast(i % 5), 1000, []() {}); + } + scheduler.process_to_add(); + benchmark::DoNotOptimize(scheduler); + } + state.SetItemsProcessed(state.iterations() * kInnerIterations); +} +BENCHMARK(Scheduler_SetTimeout); + +// --- Scheduler: set_interval registration --- + +static void Scheduler_SetInterval(benchmark::State &state) { + Scheduler scheduler; + Component dummy_component; + + for (auto _ : state) { + for (int i = 0; i < kInnerIterations; i++) { + scheduler.set_interval(&dummy_component, static_cast(i % 5), 1000, []() {}); + } + scheduler.process_to_add(); + benchmark::DoNotOptimize(scheduler); + } + state.SetItemsProcessed(state.iterations() * kInnerIterations); +} +BENCHMARK(Scheduler_SetInterval); + +// --- Scheduler: defer registration --- + +static void Scheduler_Defer(benchmark::State &state) { + Component dummy_component; + + for (auto _ : state) { + for (int i = 0; i < kInnerIterations; i++) { + dummy_component.defer(static_cast(i % 5), []() {}); + } + benchmark::DoNotOptimize(dummy_component); + } + state.SetItemsProcessed(state.iterations() * kInnerIterations); +} +BENCHMARK(Scheduler_Defer); + // --- Scheduler: next_schedule_in() calculation --- static void Scheduler_NextScheduleIn(benchmark::State &state) { From b87b4102e048da8007c429c73e3da83438a02c5d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:42:37 -1000 Subject: [PATCH 49/61] Fix binary path extraction and defer benchmark compile error - Use BENCHMARK_BINARY= marker for reliable binary path extraction instead of fragile tail -1 (PlatformIO can print warnings after path) - Fix Scheduler_Defer: defer() is protected on Component, use set_timeout(delay=0) directly on Scheduler instead --- .github/workflows/ci.yml | 4 ++-- script/test_helpers.py | 2 +- tests/benchmarks/core/bench_scheduler.cpp | 10 +++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a5ac0e29e..0a4a6302bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,8 +332,8 @@ jobs: run: | . venv/bin/activate export BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) - # --build-only prints the binary path as the last line of stdout - BINARY=$(script/cpp_benchmark.py --all --build-only | tail -1) + # --build-only prints BENCHMARK_BINARY= to stdout + BINARY=$(script/cpp_benchmark.py --all --build-only | grep '^BENCHMARK_BINARY=' | tail -1 | cut -d= -f2-) echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/script/test_helpers.py b/script/test_helpers.py index 24d99acaeb..ad290b7ceb 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -389,7 +389,7 @@ def build_and_run( return exit_code if build_only: - print(program_path) + print(f"BENCHMARK_BINARY={program_path}") return EXIT_OK # Run the binary diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 1b61223f75..f481bfc9e9 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -111,16 +111,20 @@ static void Scheduler_SetInterval(benchmark::State &state) { } BENCHMARK(Scheduler_SetInterval); -// --- Scheduler: defer registration --- +// --- Scheduler: defer registration (set_timeout with delay=0) --- static void Scheduler_Defer(benchmark::State &state) { + Scheduler scheduler; Component dummy_component; + // defer() is Component::defer which calls set_timeout(delay=0). + // Call set_timeout directly since defer() is protected. for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { - dummy_component.defer(static_cast(i % 5), []() {}); + scheduler.set_timeout(&dummy_component, static_cast(i % 5), 0, []() {}); } - benchmark::DoNotOptimize(dummy_component); + scheduler.process_to_add(); + benchmark::DoNotOptimize(scheduler); } state.SetItemsProcessed(state.iterations() * kInnerIterations); } From 587179143fcf34f9096df11c707aa548e3e94f81 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:43:58 -1000 Subject: [PATCH 50/61] Rename BENCHMARK_BINARY to BUILD_BINARY for generic use --- .github/workflows/ci.yml | 4 ++-- script/test_helpers.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a4a6302bf..1926ad5bf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,8 +332,8 @@ jobs: run: | . venv/bin/activate export BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) - # --build-only prints BENCHMARK_BINARY= to stdout - BINARY=$(script/cpp_benchmark.py --all --build-only | grep '^BENCHMARK_BINARY=' | tail -1 | cut -d= -f2-) + # --build-only prints BUILD_BINARY= to stdout + BINARY=$(script/cpp_benchmark.py --all --build-only | grep '^BUILD_BINARY=' | tail -1 | cut -d= -f2-) echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/script/test_helpers.py b/script/test_helpers.py index ad290b7ceb..e872bbc516 100644 --- a/script/test_helpers.py +++ b/script/test_helpers.py @@ -389,7 +389,7 @@ def build_and_run( return exit_code if build_only: - print(f"BENCHMARK_BINARY={program_path}") + print(f"BUILD_BINARY={program_path}") return EXIT_OK # Run the binary From e14443b9c3fcd119813ef1ac109404a063b7dee5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:50:36 -1000 Subject: [PATCH 51/61] Fix ProtoSize_Varint benchmarks: prevent constant folding The compiler constant-folds ProtoSize::varint(42) to 1 and optimizes the entire inner loop to a single addition, causing ~62ns jitter-dominated measurements. Use varying inputs (i & 0x7F for small, 0xFFFF0000 | i for large) so each call computes a real result. --- tests/benchmarks/components/api/bench_proto_varint.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp index ff4a656980..0b5ccc2b7d 100644 --- a/tests/benchmarks/components/api/bench_proto_varint.cpp +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -104,10 +104,12 @@ BENCHMARK(Encode_Varint_MaxUint32); // --- ProtoSize::varint() benchmarks --- static void ProtoSize_Varint_Small(benchmark::State &state) { + // Use varying input to prevent constant folding. + // Values 0-127 all take 1 byte but the compiler can't prove that. for (auto _ : state) { uint32_t result = 0; for (int i = 0; i < kInnerIterations; i++) { - result += ProtoSize::varint(42); + result += ProtoSize::varint(static_cast(i) & 0x7F); } benchmark::DoNotOptimize(result); } @@ -116,10 +118,11 @@ static void ProtoSize_Varint_Small(benchmark::State &state) { BENCHMARK(ProtoSize_Varint_Small); static void ProtoSize_Varint_Large(benchmark::State &state) { + // Use varying input to prevent constant folding. for (auto _ : state) { uint32_t result = 0; for (int i = 0; i < kInnerIterations; i++) { - result += ProtoSize::varint(0xFFFFFFFF); + result += ProtoSize::varint(0xFFFF0000 | static_cast(i)); } benchmark::DoNotOptimize(result); } From a1009f7a5c423dfa613b045ca09f3b77681480f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:51:33 -1000 Subject: [PATCH 52/61] Use relaxed memory ordering for all counter atomics The mutex already provides all necessary memory ordering for the counter operations. acquire/release fences on the fast-path reads were causing unnecessary cache flushes, regressing Scheduler_NextScheduleIn by ~10%. --- esphome/core/scheduler.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index f0589bef24..d00673f403 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -540,7 +540,7 @@ class Scheduler { // Lock-free check if to_add_ is empty (for fast-path in process_to_add) bool to_add_empty_() const { #ifdef ESPHOME_THREAD_MULTI_ATOMICS - return this->to_add_count_.load(std::memory_order_acquire) == 0; + return this->to_add_count_.load(std::memory_order_relaxed) == 0; #elif defined(ESPHOME_THREAD_SINGLE) // Single-threaded: no concurrent writers, direct check is safe return this->to_add_.empty(); @@ -553,7 +553,7 @@ class Scheduler { // Increment to_add_count_ (caller must hold lock on non-atomic platforms) void to_add_count_increment_() { #ifdef ESPHOME_THREAD_MULTI_ATOMICS - this->to_add_count_.fetch_add(1, std::memory_order_release); + this->to_add_count_.fetch_add(1, std::memory_order_relaxed); #else this->to_add_count_++; #endif @@ -586,7 +586,7 @@ class Scheduler { // defer_queue_ only exists on multi-threaded platforms, so no ESPHOME_THREAD_SINGLE path // ESPHOME_THREAD_MULTI_NO_ATOMICS: always take the lock #ifdef ESPHOME_THREAD_MULTI_ATOMICS - return this->defer_count_.load(std::memory_order_acquire) == 0; + return this->defer_count_.load(std::memory_order_relaxed) == 0; #else return false; #endif @@ -594,7 +594,7 @@ class Scheduler { void defer_count_increment_() { #ifdef ESPHOME_THREAD_MULTI_ATOMICS - this->defer_count_.fetch_add(1, std::memory_order_release); + this->defer_count_.fetch_add(1, std::memory_order_relaxed); #else this->defer_count_++; #endif @@ -621,7 +621,7 @@ class Scheduler { // Lock-free check if there are items to remove (for fast-path in cleanup_) bool to_remove_empty_() const { #ifdef ESPHOME_THREAD_MULTI_ATOMICS - return this->to_remove_.load(std::memory_order_acquire) == 0; + return this->to_remove_.load(std::memory_order_relaxed) == 0; #elif defined(ESPHOME_THREAD_SINGLE) return this->to_remove_ == 0; #else @@ -631,7 +631,7 @@ class Scheduler { void to_remove_add_(uint32_t count) { #ifdef ESPHOME_THREAD_MULTI_ATOMICS - this->to_remove_.fetch_add(count, std::memory_order_release); + this->to_remove_.fetch_add(count, std::memory_order_relaxed); #else this->to_remove_ += count; #endif From 55a76fcbca6cac5b743a374fd64b74a0fdc8a82d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 01:58:10 -1000 Subject: [PATCH 53/61] Change cleanup_() to return bool instead of size_t cleanup_() was computing items_.size() on every call even on the fast path where nothing was removed. All callers only check if items remain (== 0), so return bool and use items_.empty() instead. --- esphome/core/scheduler.cpp | 10 +++++----- esphome/core/scheduler.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index a3a767a17b..db40ede78c 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -396,7 +396,7 @@ optional HOT Scheduler::next_schedule_in(uint32_t now) { // safe when called from the main thread. Other threads must not call this method. // If no items, return empty optional - if (this->cleanup_() == 0) + if (!this->cleanup_()) return {}; SchedulerItem *item = this->items_[0]; @@ -633,12 +633,12 @@ void HOT Scheduler::process_to_add() { this->to_add_.clear(); this->to_add_count_clear_(); } -size_t HOT Scheduler::cleanup_() { - // Fast path: if nothing to remove, just return the current size. +bool HOT Scheduler::cleanup_() { + // Fast path: if nothing to remove, just check if items exist. // Uses atomic load on platforms with atomics, falls back to always taking the lock otherwise. // Worst case is a one-loop-iteration delay in cleanup. if (this->to_remove_empty_()) - return this->items_.size(); + return !this->items_.empty(); // We must hold the lock for the entire cleanup operation because: // 1. We're modifying items_ (via pop_raw_locked_) which requires exclusive access @@ -656,7 +656,7 @@ size_t HOT Scheduler::cleanup_() { this->to_remove_decrement_(); this->recycle_item_main_loop_(this->pop_raw_locked_()); } - return this->items_.size(); + return !this->items_.empty(); } Scheduler::SchedulerItem *HOT Scheduler::pop_raw_locked_() { std::pop_heap(this->items_.begin(), this->items_.end(), SchedulerItem::cmp); diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index d00673f403..32fa807473 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -284,9 +284,9 @@ class Scheduler { #endif } // Cleanup logically deleted items from the scheduler - // Returns the number of items remaining after cleanup + // Returns true if items remain after cleanup // IMPORTANT: This method should only be called from the main thread (loop task). - size_t cleanup_(); + bool cleanup_(); // Remove and return the front item from the heap as a raw pointer. // Caller takes ownership and must either recycle or delete the item. // IMPORTANT: Caller must hold the scheduler lock before calling this function. From f2ab81a0efe2a74728ff0cfbf3a782ea96c28176 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 02:00:17 -1000 Subject: [PATCH 54/61] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/core/scheduler.h | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 32fa807473..62c3583dca 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -529,15 +529,21 @@ class Scheduler { std::vector items_; std::vector to_add_; - // Fast-path counter for process_to_add() to skip lock when nothing to add. - // Uses atomic on platforms that support it, plain uint32_t + lock otherwise. + // Fast-path counter for process_to_add() to skip taking the lock when there is + // nothing to add. Uses std::atomic on platforms that support it, plain uint32_t + // otherwise. On non-atomic platforms, callers must hold the scheduler lock when + // mutating this counter. #ifdef ESPHOME_THREAD_MULTI_ATOMICS std::atomic to_add_count_{0}; #else uint32_t to_add_count_{0}; #endif - // Lock-free check if to_add_ is empty (for fast-path in process_to_add) + // Fast-path helper for process_to_add() to decide if it can try the lock-free path. + // - On ESPHOME_THREAD_MULTI_ATOMICS: performs a lock-free check via to_add_count_. + // - On ESPHOME_THREAD_SINGLE: direct container check is safe (no concurrent writers). + // - On ESPHOME_THREAD_MULTI_NO_ATOMICS: always returns false to force the caller + // down the locked path; this is NOT a lock-free emptiness check on that platform. bool to_add_empty_() const { #ifdef ESPHOME_THREAD_MULTI_ATOMICS return this->to_add_count_.load(std::memory_order_relaxed) == 0; @@ -545,8 +551,9 @@ class Scheduler { // Single-threaded: no concurrent writers, direct check is safe return this->to_add_.empty(); #else - // Multi-threaded without atomics: must take lock - return false; // Always take the lock path + // Multi-threaded without atomics: must take lock; always indicate "not empty" + // so the caller does not attempt a lock-free path. + return false; #endif } From 1c685a8a7787f8a696f70b76532d6bb685c07d54 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 02:00:37 -1000 Subject: [PATCH 55/61] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/core/scheduler.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 62c3583dca..20e9f8b49b 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -617,8 +617,9 @@ class Scheduler { #endif /* ESPHOME_THREAD_SINGLE */ - // Counter for items marked for removal. Incremented cross-thread in cancel_item_locked_(), - // read without lock in cleanup_() fast path. + // Counter for items marked for removal. Incremented cross-thread in cancel_item_locked_(). + // On ESPHOME_THREAD_MULTI_ATOMICS this is read without a lock in the cleanup_() fast path; + // on ESPHOME_THREAD_MULTI_NO_ATOMICS the fast path is disabled so cleanup_() always takes the lock. #ifdef ESPHOME_THREAD_MULTI_ATOMICS std::atomic to_remove_{0}; #else From d0c39d95ade5024e9f2f00bf75a4b35cab3ab874 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:02:06 +0000 Subject: [PATCH 56/61] [pre-commit.ci lite] apply automatic fixes --- esphome/core/scheduler.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 20e9f8b49b..613053dec6 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -551,9 +551,9 @@ class Scheduler { // Single-threaded: no concurrent writers, direct check is safe return this->to_add_.empty(); #else - // Multi-threaded without atomics: must take lock; always indicate "not empty" - // so the caller does not attempt a lock-free path. - return false; + // Multi-threaded without atomics: must take lock; always indicate "not empty" + // so the caller does not attempt a lock-free path. + return false; #endif } From a8bb3d8c6f8c24703d510c163ca578a61cf2af4e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 02:02:23 -1000 Subject: [PATCH 57/61] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/core/scheduler.h | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 613053dec6..d6f9ee2a44 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -620,6 +620,7 @@ class Scheduler { // Counter for items marked for removal. Incremented cross-thread in cancel_item_locked_(). // On ESPHOME_THREAD_MULTI_ATOMICS this is read without a lock in the cleanup_() fast path; // on ESPHOME_THREAD_MULTI_NO_ATOMICS the fast path is disabled so cleanup_() always takes the lock. + // on ESPHOME_THREAD_MULTI_NO_ATOMICS the fast path is disabled so cleanup_() always takes the lock. #ifdef ESPHOME_THREAD_MULTI_ATOMICS std::atomic to_remove_{0}; #else From 1ed30414c9f30dae79f674897abf75bf6d014c1d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 02:05:07 -1000 Subject: [PATCH 58/61] Fix duplicate comment line in to_remove_ documentation --- esphome/core/scheduler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index d6f9ee2a44..613053dec6 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -620,7 +620,6 @@ class Scheduler { // Counter for items marked for removal. Incremented cross-thread in cancel_item_locked_(). // On ESPHOME_THREAD_MULTI_ATOMICS this is read without a lock in the cleanup_() fast path; // on ESPHOME_THREAD_MULTI_NO_ATOMICS the fast path is disabled so cleanup_() always takes the lock. - // on ESPHOME_THREAD_MULTI_NO_ATOMICS the fast path is disabled so cleanup_() always takes the lock. #ifdef ESPHOME_THREAD_MULTI_ATOMICS std::atomic to_remove_{0}; #else From 4003406f8bdc95c4ac33383a443b1048fb6ee419 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 02:06:16 -1000 Subject: [PATCH 59/61] =?UTF-8?q?Remove=20Scheduler=5FNextScheduleIn=20ben?= =?UTF-8?q?chmark=20=E2=80=94=20too=20cheap,=20flaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/benchmarks/core/bench_scheduler.cpp | 25 ----------------------- 1 file changed, 25 deletions(-) diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index f481bfc9e9..764f17ed73 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -130,29 +130,4 @@ static void Scheduler_Defer(benchmark::State &state) { } BENCHMARK(Scheduler_Defer); -// --- Scheduler: next_schedule_in() calculation --- - -static void Scheduler_NextScheduleIn(benchmark::State &state) { - Scheduler scheduler; - Component dummy_component; - - // Add some timeouts - for (int i = 0; i < 10; i++) { - scheduler.set_timeout(&dummy_component, static_cast(i), 1000 * (i + 1), []() {}); - } - scheduler.process_to_add(); - - uint32_t now = millis(); - - for (auto _ : state) { - optional result; - for (int i = 0; i < kInnerIterations; i++) { - result = scheduler.next_schedule_in(now); - } - benchmark::DoNotOptimize(result); - } - state.SetItemsProcessed(state.iterations() * kInnerIterations); -} -BENCHMARK(Scheduler_NextScheduleIn); - } // namespace esphome::benchmarks From 9c9464f29f2e5cc27327a8904592f8a16eb0d8b5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 02:05:07 -1000 Subject: [PATCH 60/61] Skip to_add_count_ field on single-threaded platforms On ESPHOME_THREAD_SINGLE there are no concurrent writers, so to_add_empty_() checks to_add_.empty() directly. The counter field and its increment/clear operations are compiled out. --- esphome/core/scheduler.h | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 613053dec6..e545055fca 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -529,49 +529,53 @@ class Scheduler { std::vector items_; std::vector to_add_; +#ifndef ESPHOME_THREAD_SINGLE // Fast-path counter for process_to_add() to skip taking the lock when there is // nothing to add. Uses std::atomic on platforms that support it, plain uint32_t // otherwise. On non-atomic platforms, callers must hold the scheduler lock when - // mutating this counter. + // mutating this counter. Not needed on single-threaded platforms where we can + // check to_add_.empty() directly. #ifdef ESPHOME_THREAD_MULTI_ATOMICS std::atomic to_add_count_{0}; #else uint32_t to_add_count_{0}; #endif +#endif /* ESPHOME_THREAD_SINGLE */ // Fast-path helper for process_to_add() to decide if it can try the lock-free path. - // - On ESPHOME_THREAD_MULTI_ATOMICS: performs a lock-free check via to_add_count_. // - On ESPHOME_THREAD_SINGLE: direct container check is safe (no concurrent writers). + // - On ESPHOME_THREAD_MULTI_ATOMICS: performs a lock-free check via to_add_count_. // - On ESPHOME_THREAD_MULTI_NO_ATOMICS: always returns false to force the caller // down the locked path; this is NOT a lock-free emptiness check on that platform. bool to_add_empty_() const { -#ifdef ESPHOME_THREAD_MULTI_ATOMICS - return this->to_add_count_.load(std::memory_order_relaxed) == 0; -#elif defined(ESPHOME_THREAD_SINGLE) - // Single-threaded: no concurrent writers, direct check is safe +#ifdef ESPHOME_THREAD_SINGLE return this->to_add_.empty(); +#elif defined(ESPHOME_THREAD_MULTI_ATOMICS) + return this->to_add_count_.load(std::memory_order_relaxed) == 0; #else - // Multi-threaded without atomics: must take lock; always indicate "not empty" - // so the caller does not attempt a lock-free path. return false; #endif } - // Increment to_add_count_ (caller must hold lock on non-atomic platforms) + // Increment to_add_count_ (no-op on single-threaded platforms) void to_add_count_increment_() { -#ifdef ESPHOME_THREAD_MULTI_ATOMICS +#ifdef ESPHOME_THREAD_SINGLE + // No counter needed — to_add_empty_() checks the vector directly +#elif defined(ESPHOME_THREAD_MULTI_ATOMICS) this->to_add_count_.fetch_add(1, std::memory_order_relaxed); #else - this->to_add_count_++; + this->to_add_count_++; #endif } - // Reset to_add_count_ (caller must hold lock) + // Reset to_add_count_ (no-op on single-threaded platforms) void to_add_count_clear_() { -#ifdef ESPHOME_THREAD_MULTI_ATOMICS +#ifdef ESPHOME_THREAD_SINGLE + // No counter needed — to_add_empty_() checks the vector directly +#elif defined(ESPHOME_THREAD_MULTI_ATOMICS) this->to_add_count_.store(0, std::memory_order_relaxed); #else - this->to_add_count_ = 0; + this->to_add_count_ = 0; #endif } From 81788951da684335800c379f9ce75d17b999c37d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 17 Mar 2026 08:44:56 -1000 Subject: [PATCH 61/61] Exclude tests/benchmarks/ from clang-tidy file discovery --- script/clang-tidy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/clang-tidy b/script/clang-tidy index 9c2899026d..f2834b44ac 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -231,6 +231,9 @@ def main(): cwd = os.getcwd() files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])] + # Exclude benchmark files — they require google benchmark headers not + # available in the ESP32 toolchain and use different naming conventions. + files = [f for f in files if not f.startswith("tests/benchmarks/")] # Print initial file count if it's large if len(files) > 50: