github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-07-10 17:05:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

[core] Add CodSpeed C++ benchmarks for protobuf and main loop

Browse Source 
Add automated benchmarks using Google Benchmark to prevent performance
regressions in the API protobuf encoding/decoding and core loop paths.

Benchmarks cover:
- Protobuf encode: SensorState, BinarySensorState, HelloResponse,
  LightState, DeviceInfoResponse (20 nested devices + 20 areas)
- Protobuf decode: HelloRequest, SwitchCommand, LightCommand
- Protobuf calculate_size and full calc+encode send path
- Varint parse/encode/size for various value ranges
- Scheduler call/next_schedule_in with idle and active timers
- Application loop component dispatch and blocking guard overhead

Infrastructure:
- Extract shared build logic from cpp_unit_test.py into test_helpers.py
- Add cpp_benchmark.py mirroring the unit test build pattern
- Support benchmark.yaml per component dir for declaring dependencies
- Add CodSpeed CI workflow triggered on api/core changes
- Fix ProtoMessage protected destructor on host platform
This commit is contained in:
J. Nick Koston
2026-03-16 19:25:46 -10:00
parent bba11b3b1e
commit 38ff332fec
13 changed files with 1280 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci-benchmarks.yml
+87
View File
@@ -0,0 +1,87 @@
---
name: CodSpeed Benchmarks
on:
push:
branches: [dev]
pull_request:
paths:
- "esphome/components/api/**"
- "esphome/core/**"
- "tests/benchmarks/**"
- "script/cpp_benchmark.py"
- "script/test_helpers.py"
- ".github/workflows/ci-benchmarks.yml"
merge_group:
permissions:
contents: read
concurrency:
# yamllint disable-line rule:line-length
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
DEFAULT_PYTHON: "3.11"
jobs:
common:
name: Create common environment
runs-on: ubuntu-24.04
outputs:
cache-key: ${{ steps.cache-key.outputs.key }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Generate cache-key
id: cache-key
run: echo key="${{ hashFiles('requirements.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: venv
# yamllint disable-line rule:line-length
key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ steps.cache-key.outputs.key }}
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
python -m venv venv
. venv/bin/activate
python --version
pip install -r requirements.txt -r requirements_test.txt
pip install -e .
benchmarks:
name: Run CodSpeed Benchmarks
runs-on: ubuntu-24.04
needs:
- common
steps:
- name: Check out code from GitHub
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Restore Python
uses: ./.github/actions/restore-python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
cache-key: ${{ needs.common.outputs.cache-key }}
- name: Build benchmarks
id: build
run: |
. venv/bin/activate
BINARY=$(script/cpp_benchmark.py --all --build-only 2>&1 | tail -1)
echo "binary=$BINARY" >> $GITHUB_OUTPUT
- name: Run CodSpeed benchmarks
uses: CodSpeedHQ/action@v4
with:
run: ${{ steps.build.outputs.binary }}
token: ${{ secrets.CODSPEED_TOKEN }}
esphome/components/api/proto.h
+4
View File
@@ -442,8 +442,12 @@ class ProtoMessage {
virtual const char *message_name() const { return "unknown"; }
#endif
#ifndef USE_HOST
protected:
#endif
// Non-virtual destructor is protected to prevent polymorphic deletion.
// On host platform, made public to allow value-initialization of std::array
// members (e.g. DeviceInfoResponse::devices) without clang errors.
~ProtoMessage() = default;
};
script/cpp_benchmark.py
Executable
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Build and run C++ benchmarks for ESPHome components using Google Benchmark."""
import argparse
from pathlib import Path
import sys
from helpers import root_path
from test_helpers import PLATFORMIO_GOOGLE_BENCHMARK_LIB, build_and_run
# Path to /tests/benchmarks/components
BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "components"
# Components whose to_code should run during benchmark builds.
# core/host/logger are infrastructure. json is needed because its
# to_code adds the ArduinoJson library (it's auto-loaded by api but
# cpp_testing suppresses to_code for components not in this set).
BENCHMARK_CODEGEN_COMPONENTS = {"core", "host", "logger", "json"}
PLATFORMIO_OPTIONS = {
"build_flags": [
"-DUSE_TIME_TIMEZONE", # enable timezone code paths
"-g", # debug symbols for profiling
],
}
def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int:
return build_and_run(
selected_components=selected_components,
tests_dir=BENCHMARKS_DIR,
codegen_components=BENCHMARK_CODEGEN_COMPONENTS,
config_prefix="cppbench",
friendly_name="CPP Benchmarks",
libraries=PLATFORMIO_GOOGLE_BENCHMARK_LIB,
platformio_options=PLATFORMIO_OPTIONS,
main_entry="main.cpp",
label="benchmarks",
build_only=build_only,
)
def main() -> None:
parser = argparse.ArgumentParser(
description="Build and run C++ benchmarks for ESPHome components."
)
parser.add_argument(
"components",
nargs="*",
help="List of components to benchmark (must have files in tests/benchmarks/components/).",
)
parser.add_argument(
"--all",
action="store_true",
help="Benchmark all components with benchmark files.",
)
parser.add_argument(
"--build-only",
action="store_true",
help="Only build, print binary path without running.",
)
args = parser.parse_args()
if args.all:
# Find all component directories that have .cpp files
components: list[str] = (
sorted(
d.name
for d in BENCHMARKS_DIR.iterdir()
if d.is_dir()
and d.name != "__pycache__"
and (any(d.glob("*.cpp")) or any(d.glob("*.h")))
)
if BENCHMARKS_DIR.is_dir()
else []
)
else:
components: list[str] = args.components
sys.exit(run_benchmarks(components, build_only=args.build_only))
if __name__ == "__main__":
main()
script/cpp_unit_test.py
+31 -214
View File
@@ -1,22 +1,10 @@
#!/usr/bin/env python3
import argparse
import hashlib
import os
from pathlib import Path
import subprocess
import sys
from helpers import get_all_components, get_all_dependencies, root_path
from esphome.__main__ import command_compile, parse_args
from esphome.config import validate_config
from esphome.const import CONF_PLATFORM
from esphome.core import CORE
from esphome.loader import get_component
from esphome.platformio_api import get_idedata
# This must coincide with the version in /platformio.ini
PLATFORMIO_GOOGLE_TEST_LIB = "google/googletest@^1.15.2"
from helpers import get_all_components, root_path
from test_helpers import PLATFORMIO_GOOGLE_TEST_LIB, build_and_run
# Path to /tests/components
COMPONENTS_TESTS_DIR: Path = Path(root_path) / "tests" / "components"
@@ -28,211 +16,40 @@ COMPONENTS_TESTS_DIR: Path = Path(root_path) / "tests" / "components"
# which registers under package name "core" not "esphome".
CPP_TESTING_CODEGEN_COMPONENTS = {"core", "host", "logger"}
def hash_components(components: list[str]) -> str:
key = ",".join(components)
return hashlib.sha256(key.encode()).hexdigest()[:16]
def filter_components_without_tests(components: list[str]) -> list[str]:
"""Filter out components that do not have a corresponding test file.
This is done by checking if the component's directory contains at
least a .cpp or .h file.
"""
filtered_components: list[str] = []
for component in components:
test_dir = COMPONENTS_TESTS_DIR / component
if test_dir.is_dir() and (
any(test_dir.glob("*.cpp")) or any(test_dir.glob("*.h"))
):
filtered_components.append(component)
else:
print(
f"WARNING: No tests found for component '{component}', skipping.",
file=sys.stderr,
)
return filtered_components
def create_test_config(config_name: str, includes: list[str]) -> dict:
"""Create ESPHome test configuration for C++ unit tests.
Args:
config_name: Unique name for this test configuration
includes: List of include folders for the test build
Returns:
Configuration dict for ESPHome
"""
return {
"esphome": {
"name": config_name,
"friendly_name": "CPP Unit Tests",
"libraries": PLATFORMIO_GOOGLE_TEST_LIB,
"platformio_options": {
"build_type": "debug",
"build_unflags": [
"-Os", # remove size-opt flag
],
"build_flags": [
"-Og", # optimize for debug
"-DUSE_TIME_TIMEZONE", # enable timezone code paths for testing
"-DESPHOME_DEBUG", # enable debug assertions
# Enable the address and undefined behavior sanitizers
"-fsanitize=address",
"-fsanitize=undefined",
"-fno-omit-frame-pointer",
],
"debug_build_flags": [ # only for debug builds
"-g3", # max debug info
"-ggdb3",
],
},
"includes": includes,
},
"host": {},
"logger": {"level": "DEBUG"},
}
def get_platform_components(components: list[str]) -> list[str]:
"""Discover platform sub-components referenced by test directory structure.
For each component being tested, any sub-directory named after a platform
domain (e.g. ``sensor``, ``binary_sensor``) is treated as a request to
include that ``<domain>.<component>`` platform in the build. The sub-
directory must name a valid platform domain; anything else raises an error
so that typos are caught early.
Returns:
List of ``"domain.component"`` strings, one per discovered sub-directory.
"""
platform_components: list[str] = []
for component in components:
test_dir = COMPONENTS_TESTS_DIR / component
if not test_dir.is_dir():
continue
# Each sub-directory name is expected to be a platform domain
# (e.g. tests/components/bthome/sensor/ → sensor.bthome).
for domain_dir in test_dir.iterdir():
if not domain_dir.is_dir():
continue
domain = domain_dir.name
domain_module = get_component(domain)
if domain_module is None or not domain_module.is_platform_component:
raise ValueError(
f"Component tests for '{component}' reference non-existing or invalid domain '{domain}'"
f" in its directory structure. See ({COMPONENTS_TESTS_DIR / component / domain})."
)
platform_components.append(f"{domain}.{component}")
return platform_components
# Exit codes for run_tests
EXIT_OK = 0
EXIT_SKIPPED = 1
EXIT_COMPILE_ERROR = 2
EXIT_CONFIG_ERROR = 3
EXIT_NO_EXECUTABLE = 4
PLATFORMIO_OPTIONS = {
"build_type": "debug",
"build_unflags": [
"-Os", # remove size-opt flag
],
"build_flags": [
"-Og", # optimize for debug
"-DUSE_TIME_TIMEZONE", # enable timezone code paths for testing
"-DESPHOME_DEBUG", # enable debug assertions
# Enable the address and undefined behavior sanitizers
"-fsanitize=address",
"-fsanitize=undefined",
"-fno-omit-frame-pointer",
],
"debug_build_flags": [ # only for debug builds
"-g3", # max debug info
"-ggdb3",
],
}
def run_tests(selected_components: list[str]) -> int:
# Skip tests on Windows
if os.name == "nt":
print("Skipping esphome tests on Windows", file=sys.stderr)
return EXIT_SKIPPED
# Remove components that do not have tests
components = filter_components_without_tests(selected_components)
if len(components) == 0:
print(
"No components specified or no tests found for the specified components.",
file=sys.stderr,
)
return EXIT_OK
components = sorted(components)
# Build a list of include folders relative to COMPONENTS_TESTS_DIR. These folders will
# be added along with their subfolders.
# "main.cpp" is a special entry that points to /tests/components/main.cpp,
# which provides a custom test runner entry-point replacing the default one.
# Each remaining entry is a component folder whose *.cpp files are compiled.
includes: list[str] = ["main.cpp"] + components
# Obtain a list of platform components to be tested:
try:
platform_components = get_platform_components(components)
except ValueError as e:
print(f"Error obtaining platform components: {e}")
return EXIT_CONFIG_ERROR
components = sorted(components + platform_components)
# Create a unique name for this config based on the actual components being tested
# to maximize cache during testing
config_name: str = "cpptests-" + hash_components(components)
# Obtain possible dependencies for the requested components.
# Always include 'time' because USE_TIME_TIMEZONE is defined as a build flag,
# which causes core/time.h to include components/time/posix_tz.h.
components_with_dependencies: list[str] = sorted(
get_all_dependencies(set(components) | {"time"}, cpp_testing=True)
return build_and_run(
selected_components=selected_components,
tests_dir=COMPONENTS_TESTS_DIR,
codegen_components=CPP_TESTING_CODEGEN_COMPONENTS,
config_prefix="cpptests",
friendly_name="CPP Unit Tests",
libraries=PLATFORMIO_GOOGLE_TEST_LIB,
platformio_options=PLATFORMIO_OPTIONS,
main_entry="main.cpp",
label="unit tests",
)
config = create_test_config(config_name, includes)
CORE.config_path = COMPONENTS_TESTS_DIR / "dummy.yaml"
CORE.dashboard = None
CORE.cpp_testing = True
CORE.cpp_testing_codegen = CPP_TESTING_CODEGEN_COMPONENTS
# Validate config will expand the above with defaults:
config = validate_config(config, {})
# Add all components and dependencies to the base configuration after validation, so their files
# are added to the build.
for component_name in components_with_dependencies:
if "." in component_name:
# Format is always "domain.component" (exactly one dot),
# as produced by get_platform_components().
domain, component = component_name.split(".", maxsplit=1)
domain_list = config.setdefault(domain, [])
CORE.testing_ensure_platform_registered(domain)
domain_list.append({CONF_PLATFORM: component})
else:
config.setdefault(component_name, [])
dependencies = set(components_with_dependencies) - set(components)
deps_str = ", ".join(dependencies) if dependencies else "None"
print(f"Testing components: {', '.join(components)}. Dependencies: {deps_str}")
CORE.config = config
args = parse_args(["program", "compile", str(CORE.config_path)])
try:
exit_code: int = command_compile(args, config)
if exit_code != 0:
print(f"Error compiling unit tests for {', '.join(components)}")
return exit_code
except Exception as e:
print(
f"Error compiling unit tests for {', '.join(components)}. Check path. : {e}"
)
return EXIT_COMPILE_ERROR
# After a successful compilation, locate the executable and run it:
idedata = get_idedata(config)
if idedata is None:
print("Cannot find executable")
return EXIT_NO_EXECUTABLE
program_path: str = idedata.raw["prog_path"]
run_cmd: list[str] = [program_path]
run_proc = subprocess.run(run_cmd, check=False)
return run_proc.returncode
def main() -> None:
parser = argparse.ArgumentParser(
script/test_helpers.py
+375
View File
@@ -0,0 +1,375 @@
"""Shared helpers for C++ unit test and benchmark build scripts."""
from __future__ import annotations
import hashlib
import os
from pathlib import Path
import subprocess
import sys
from helpers import get_all_dependencies
import yaml
from esphome.__main__ import command_compile, parse_args
from esphome.config import validate_config
from esphome.const import CONF_PLATFORM
from esphome.core import CORE
from esphome.loader import get_component
from esphome.platformio_api import get_idedata
# This must coincide with the version in /platformio.ini
PLATFORMIO_GOOGLE_TEST_LIB = "google/googletest@^1.15.2"
# Google Benchmark library for PlatformIO
# Format: name=repository_url (see esphome/core/config.py library parsing)
PLATFORMIO_GOOGLE_BENCHMARK_LIB = (
"benchmark=https://github.com/google/benchmark.git#v1.9.1"
)
# Key names for the base config sections
ESPHOME_KEY = "esphome"
HOST_KEY = "host"
LOGGER_KEY = "logger"
# Base config keys that are always present and must not be fully overridden
# by component benchmark.yaml files. esphome: allows sub-key merging.
BASE_CONFIG_KEYS = frozenset({ESPHOME_KEY, HOST_KEY, LOGGER_KEY})
# Exit codes
EXIT_OK = 0
EXIT_SKIPPED = 1
EXIT_COMPILE_ERROR = 2
EXIT_CONFIG_ERROR = 3
EXIT_NO_EXECUTABLE = 4
# Name of the per-component YAML config file in benchmark directories
BENCHMARK_YAML_FILENAME = "benchmark.yaml"
def hash_components(components: list[str]) -> str:
"""Create a short hash of component names for unique config naming."""
key = ",".join(components)
return hashlib.sha256(key.encode()).hexdigest()[:16]
def filter_components_with_files(components: list[str], tests_dir: Path) -> list[str]:
"""Filter out components that do not have .cpp or .h files in the tests dir.
Args:
components: List of component names to check
tests_dir: Base directory containing component test/benchmark folders
Returns:
Filtered list of components that have test files
"""
filtered_components: list[str] = []
for component in components:
test_dir = tests_dir / component
if test_dir.is_dir() and (
any(test_dir.glob("*.cpp")) or any(test_dir.glob("*.h"))
):
filtered_components.append(component)
else:
print(
f"WARNING: No files found for component '{component}' in {tests_dir}, skipping.",
file=sys.stderr,
)
return filtered_components
def get_platform_components(components: list[str], tests_dir: Path) -> list[str]:
"""Discover platform sub-components referenced by test directory structure.
For each component, any sub-directory named after a platform domain
(e.g. ``sensor``, ``binary_sensor``) is treated as a request to include
that ``<domain>.<component>`` platform in the build.
Args:
components: List of component names to scan
tests_dir: Base directory containing component test/benchmark folders
Returns:
List of ``"domain.component"`` strings
"""
platform_components: list[str] = []
for component in components:
test_dir = tests_dir / component
if not test_dir.is_dir():
continue
for domain_dir in test_dir.iterdir():
if not domain_dir.is_dir():
continue
domain = domain_dir.name
domain_module = get_component(domain)
if domain_module is None or not domain_module.is_platform_component:
raise ValueError(
f"Component '{component}' references non-existing or invalid domain '{domain}'"
f" in its directory structure. See ({tests_dir / component / domain})."
)
platform_components.append(f"{domain}.{component}")
return platform_components
def load_component_yaml_configs(components: list[str], tests_dir: Path) -> dict:
"""Load and merge benchmark.yaml files from component directories.
Each component directory may contain a ``benchmark.yaml`` file that
declares additional ESPHome components needed for the build (e.g.
``api:``, ``sensor:``). These get merged into the base config before
validation so that dependencies are properly resolved with defaults.
The ``esphome:`` key is special: its sub-keys are merged into the
existing esphome config (e.g. to add ``areas:`` or ``devices:``).
Other base config keys (``host:``, ``logger:``) are not overridable.
Args:
components: List of component directory names
tests_dir: Base directory containing component folders
Returns:
Merged dict of component configs to add to the base config
"""
merged: dict = {}
for component in components:
yaml_path = tests_dir / component / BENCHMARK_YAML_FILENAME
if not yaml_path.is_file():
continue
with open(yaml_path) as f:
component_config = yaml.safe_load(f)
if component_config and isinstance(component_config, dict):
for key, value in component_config.items():
if key in BASE_CONFIG_KEYS - {ESPHOME_KEY}:
# host: and logger: are not overridable
continue
if key == ESPHOME_KEY and isinstance(value, dict):
# Merge esphome sub-keys rather than replacing
esphome_extra = merged.setdefault(ESPHOME_KEY, {})
for sub_key, sub_value in value.items():
esphome_extra.setdefault(sub_key, sub_value)
continue
merged.setdefault(key, value)
return merged
def create_host_config(
config_name: str,
friendly_name: str,
libraries: str | list[str],
includes: list[str],
platformio_options: dict,
) -> dict:
"""Create an ESPHome host configuration for C++ builds.
Args:
config_name: Unique name for this configuration
friendly_name: Human-readable name
libraries: PlatformIO library specification(s)
includes: List of include folders for the build
platformio_options: Dict of platformio_options to set
Returns:
Configuration dict for ESPHome
"""
return {
ESPHOME_KEY: {
"name": config_name,
"friendly_name": friendly_name,
"libraries": libraries,
"platformio_options": platformio_options,
"includes": includes,
},
HOST_KEY: {},
LOGGER_KEY: {"level": "DEBUG"},
}
def compile_and_get_binary(
config: dict,
components: list[str],
codegen_components: set[str],
tests_dir: Path,
label: str = "build",
) -> tuple[int, str | None]:
"""Compile an ESPHome configuration and return the binary path.
Args:
config: ESPHome configuration dict (already created via create_host_config)
components: List of components to include in the build
codegen_components: Set of component names whose to_code should run
tests_dir: Base directory for test files (used as config_path base)
label: Label for log messages (e.g. "unit tests", "benchmarks")
Returns:
Tuple of (exit_code, program_path_or_none)
"""
# Load any benchmark.yaml files from component directories and merge
# them into the config BEFORE dependency resolution and validation.
# This allows each benchmark/test dir to declare which ESPHome components
# it needs (e.g. api:) so they get proper config defaults.
extra_config = load_component_yaml_configs(components, tests_dir)
for key, value in extra_config.items():
if key == ESPHOME_KEY and isinstance(value, dict):
# Merge esphome sub-keys into existing esphome config.
# For list values (e.g. libraries), extend rather than replace.
for sub_key, sub_value in value.items():
existing = config[ESPHOME_KEY].get(sub_key)
if existing is not None and isinstance(sub_value, list):
# Ensure existing is a list, then extend
if not isinstance(existing, list):
config[ESPHOME_KEY][sub_key] = [existing]
config[ESPHOME_KEY][sub_key].extend(sub_value)
else:
config[ESPHOME_KEY].setdefault(sub_key, sub_value)
else:
config.setdefault(key, value)
# Obtain possible dependencies BEFORE validate_config, because
# get_all_dependencies calls CORE.reset() which clears build_path.
# Always include 'time' because USE_TIME_TIMEZONE is defined as a build flag,
# which causes core/time.h to include components/time/posix_tz.h.
components_with_dependencies: list[str] = sorted(
get_all_dependencies(set(components) | {"time"}, cpp_testing=True)
)
CORE.config_path = tests_dir / "dummy.yaml"
CORE.dashboard = None
CORE.cpp_testing = True
CORE.cpp_testing_codegen = codegen_components
# Validate config will expand the above with defaults:
config = validate_config(config, {})
# Add remaining components and dependencies to the configuration after
# validation, so their source files are included in the build.
for component_name in components_with_dependencies:
if "." in component_name:
domain, component = component_name.split(".", maxsplit=1)
domain_list = config.setdefault(domain, [])
CORE.testing_ensure_platform_registered(domain)
domain_list.append({CONF_PLATFORM: component})
else:
config.setdefault(component_name, [])
# Register platforms from the extra config (benchmark.yaml) so
# USE_SENSOR, USE_LIGHT, etc. defines are emitted without needing
# real entity instances.
for key in extra_config:
if key == ESPHOME_KEY:
continue
comp = get_component(key)
if comp is not None and comp.is_platform_component:
CORE.testing_ensure_platform_registered(key)
dependencies = set(components_with_dependencies) - set(components)
deps_str = ", ".join(dependencies) if dependencies else "None"
print(f"Building {label}: {', '.join(components)}. Dependencies: {deps_str}")
CORE.config = config
args = parse_args(["program", "compile", str(CORE.config_path)])
try:
exit_code: int = command_compile(args, config)
if exit_code != 0:
print(f"Error compiling {label} for {', '.join(components)}")
return exit_code, None
except Exception as e:
print(f"Error compiling {label} for {', '.join(components)}. Check path. : {e}")
return EXIT_COMPILE_ERROR, None
# After a successful compilation, locate the executable:
idedata = get_idedata(config)
if idedata is None:
print("Cannot find executable")
return EXIT_NO_EXECUTABLE, None
program_path: str = idedata.raw["prog_path"]
return EXIT_OK, program_path
def build_and_run(
selected_components: list[str],
tests_dir: Path,
codegen_components: set[str],
config_prefix: str,
friendly_name: str,
libraries: str | list[str],
platformio_options: dict,
main_entry: str,
label: str = "build",
build_only: bool = False,
extra_run_args: list[str] | None = None,
) -> int:
"""Build and optionally run a C++ test/benchmark binary.
This is the main orchestration function shared between unit tests
and benchmarks.
Args:
selected_components: Components to include (directory names in tests_dir)
tests_dir: Directory containing test/benchmark files
codegen_components: Components whose to_code should run
config_prefix: Prefix for the config name (e.g. "cpptests", "cppbench")
friendly_name: Human-readable name for the config
libraries: PlatformIO library specification(s)
platformio_options: PlatformIO options dict
main_entry: Name of the main entry file (e.g. "main.cpp")
label: Label for log messages
build_only: If True, print binary path and return without running
extra_run_args: Extra arguments to pass to the binary
Returns:
Exit code
"""
# Skip on Windows
if os.name == "nt":
print(f"Skipping {label} on Windows", file=sys.stderr)
return EXIT_SKIPPED
# Remove components that do not have files
components = filter_components_with_files(selected_components, tests_dir)
if len(components) == 0:
print(
f"No components specified or no files found for {label}.",
file=sys.stderr,
)
return EXIT_OK
components = sorted(components)
# Build include list: main entry point + component folders
includes: list[str] = [main_entry] + components
# Discover platform sub-components
try:
platform_components = get_platform_components(components, tests_dir)
except ValueError as e:
print(f"Error obtaining platform components: {e}")
return EXIT_CONFIG_ERROR
components = sorted(components + platform_components)
# Create unique config name
config_name: str = f"{config_prefix}-" + hash_components(components)
config = create_host_config(
config_name, friendly_name, libraries, includes, platformio_options
)
exit_code, program_path = compile_and_get_binary(
config, components, codegen_components, tests_dir, label
)
if exit_code != EXIT_OK or program_path is None:
return exit_code
if build_only:
print(program_path)
return EXIT_OK
# Run the binary
run_cmd: list[str] = [program_path]
if extra_run_args:
run_cmd.extend(extra_run_args)
run_proc = subprocess.run(run_cmd, check=False)
return run_proc.returncode
tests/benchmarks/components/.gitignore
+5
View File
@@ -0,0 +1,5 @@
# Gitignore settings for ESPHome
# This is an example and may include too much for your use-case.
# You can modify this file to suit your needs.
/.esphome/
/secrets.yaml
tests/benchmarks/components/api/bench_application_loop.cpp
+50
View File
@@ -0,0 +1,50 @@
#include <benchmark/benchmark.h>
#include "esphome/core/application.h"
#include "esphome/core/component.h"
#include "esphome/core/hal.h"
namespace esphome::benchmarks {
// A minimal component with an empty loop for benchmarking dispatch overhead
class NoopComponent : public Component {
public:
void loop() override {}
float get_setup_priority() const override { return 0.0f; }
// Expose protected method for benchmarking
void set_loop_state() { this->set_component_state_(COMPONENT_STATE_LOOP); }
};
// --- WarnIfComponentBlockingGuard overhead ---
static void BM_WarnIfComponentBlockingGuard(benchmark::State &state) {
NoopComponent component;
uint32_t now = millis();
for (auto _ : state) {
WarnIfComponentBlockingGuard guard{&component, now};
now = guard.finish();
benchmark::DoNotOptimize(now);
}
}
BENCHMARK(BM_WarnIfComponentBlockingGuard);
// --- Component virtual dispatch overhead ---
static void BM_ComponentDispatch(benchmark::State &state) {
NoopComponent component;
component.set_loop_state();
uint32_t now = millis();
for (auto _ : state) {
{
WarnIfComponentBlockingGuard guard{&component, now};
component.loop();
now = guard.finish();
}
benchmark::DoNotOptimize(now);
}
}
BENCHMARK(BM_ComponentDispatch);
} // namespace esphome::benchmarks
tests/benchmarks/components/api/bench_proto_decode.cpp
+119
View File
@@ -0,0 +1,119 @@
#include <benchmark/benchmark.h>
#include "esphome/components/api/api_pb2.h"
#include "esphome/components/api/api_buffer.h"
namespace esphome::api::benchmarks {
// --- HelloRequest decode (string + varint fields) ---
static void BM_Decode_HelloRequest(benchmark::State &state) {
// Manually encoded HelloRequest:
// field 1 (string): "aioesphomeapi"
// field 2 (varint): 1 (api_version_major)
// field 3 (varint): 10 (api_version_minor)
uint8_t encoded[] = {
0x0A, 0x0D, // field 1, length 13
'a', 'i', 'o', 'e', 's', 'p', 'h', 'o', 'm', 'e', 'a', 'p', 'i', // "aioesphomeapi"
0x10, 0x01, // field 2, value 1
0x18, 0x0A, // field 3, value 10
};
for (auto _ : state) {
HelloRequest msg;
msg.decode(encoded, sizeof(encoded));
benchmark::DoNotOptimize(msg.api_version_major);
}
}
BENCHMARK(BM_Decode_HelloRequest);
// --- SwitchCommandRequest decode (simple command) ---
static void BM_Decode_SwitchCommandRequest(benchmark::State &state) {
// field 1 (fixed32): key = 0x12345678
// field 2 (varint): state = true
uint8_t encoded[] = {
0x0D, 0x78, 0x56, 0x34, 0x12, // field 1, fixed32
0x10, 0x01, // field 2, varint true
};
for (auto _ : state) {
SwitchCommandRequest msg;
msg.decode(encoded, sizeof(encoded));
benchmark::DoNotOptimize(msg.state);
}
}
BENCHMARK(BM_Decode_SwitchCommandRequest);
// --- LightCommandRequest decode (complex command with many fields) ---
static void BM_Decode_LightCommandRequest(benchmark::State &state) {
uint8_t encoded[] = {
// field 1: key (fixed32) = 0x11223344
0x0D,
0x44,
0x33,
0x22,
0x11,
// field 2: has_state (varint) = true
0x10,
0x01,
// field 3: state (varint) = true
0x18,
0x01,
// field 4: has_brightness (varint) = true
0x20,
0x01,
// field 5: brightness (fixed32/float) = 0.8
0x2D,
0xCD,
0xCC,
0x4C,
0x3F,
// field 9: has_rgb (varint) = true
0x48,
0x01,
// field 10: red (fixed32/float) = 1.0
0x55,
0x00,
0x00,
0x80,
0x3F,
// field 11: green (fixed32/float) = 0.5
0x5D,
0x00,
0x00,
0x00,
0x3F,
// field 12: blue (fixed32/float) = 0.2
0x65,
0xCD,
0xCC,
0x4C,
0x3E,
// field 20: has_effect (varint) = true
0xA0,
0x01,
0x01,
// field 21: effect (string) = "rainbow"
0xAA,
0x01,
0x07,
'r',
'a',
'i',
'n',
'b',
'o',
'w',
};
for (auto _ : state) {
LightCommandRequest msg;
msg.decode(encoded, sizeof(encoded));
benchmark::DoNotOptimize(msg.brightness);
}
}
BENCHMARK(BM_Decode_LightCommandRequest);
} // namespace esphome::api::benchmarks
tests/benchmarks/components/api/bench_proto_encode.cpp
+210
View File
@@ -0,0 +1,210 @@
#include <benchmark/benchmark.h>
#include "esphome/components/api/api_pb2.h"
#include "esphome/components/api/api_buffer.h"
namespace esphome::api::benchmarks {
// --- SensorStateResponse (highest frequency message) ---
static void BM_Encode_SensorStateResponse(benchmark::State &state) {
APIBuffer buffer;
SensorStateResponse msg;
msg.key = 0x12345678;
msg.state = 23.5f;
msg.missing_state = false;
uint32_t size = msg.calculate_size();
buffer.resize(size);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
msg.encode(writer);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_SensorStateResponse);
static void BM_CalculateSize_SensorStateResponse(benchmark::State &state) {
SensorStateResponse msg;
msg.key = 0x12345678;
msg.state = 23.5f;
msg.missing_state = false;
for (auto _ : state) {
benchmark::DoNotOptimize(msg.calculate_size());
}
}
BENCHMARK(BM_CalculateSize_SensorStateResponse);
static void BM_CalcAndEncode_SensorStateResponse(benchmark::State &state) {
APIBuffer buffer;
SensorStateResponse msg;
msg.key = 0x12345678;
msg.state = 23.5f;
msg.missing_state = false;
for (auto _ : state) {
uint32_t size = msg.calculate_size();
buffer.resize(size);
ProtoWriteBuffer writer(&buffer, 0);
msg.encode(writer);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_CalcAndEncode_SensorStateResponse);
// --- BinarySensorStateResponse ---
static void BM_Encode_BinarySensorStateResponse(benchmark::State &state) {
APIBuffer buffer;
BinarySensorStateResponse msg;
msg.key = 0xAABBCCDD;
msg.state = true;
msg.missing_state = false;
uint32_t size = msg.calculate_size();
buffer.resize(size);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
msg.encode(writer);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_BinarySensorStateResponse);
// --- HelloResponse (string fields) ---
static void BM_Encode_HelloResponse(benchmark::State &state) {
APIBuffer buffer;
HelloResponse msg;
msg.api_version_major = 1;
msg.api_version_minor = 10;
msg.server_info = StringRef::from_lit("esphome v2026.3.0");
msg.name = StringRef::from_lit("living-room-sensor");
uint32_t size = msg.calculate_size();
buffer.resize(size);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
msg.encode(writer);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_HelloResponse);
// --- LightStateResponse (complex multi-field message) ---
static void BM_Encode_LightStateResponse(benchmark::State &state) {
APIBuffer buffer;
LightStateResponse msg;
msg.key = 0x11223344;
msg.state = true;
msg.brightness = 0.8f;
msg.color_mode = enums::COLOR_MODE_RGB_WHITE;
msg.color_brightness = 1.0f;
msg.red = 1.0f;
msg.green = 0.5f;
msg.blue = 0.2f;
msg.white = 0.0f;
msg.color_temperature = 4000.0f;
msg.cold_white = 0.0f;
msg.warm_white = 0.0f;
msg.effect = StringRef::from_lit("rainbow");
uint32_t size = msg.calculate_size();
buffer.resize(size);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
msg.encode(writer);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_LightStateResponse);
static void BM_CalculateSize_LightStateResponse(benchmark::State &state) {
LightStateResponse msg;
msg.key = 0x11223344;
msg.state = true;
msg.brightness = 0.8f;
msg.color_mode = enums::COLOR_MODE_RGB_WHITE;
msg.color_brightness = 1.0f;
msg.red = 1.0f;
msg.green = 0.5f;
msg.blue = 0.2f;
msg.white = 0.0f;
msg.color_temperature = 4000.0f;
msg.cold_white = 0.0f;
msg.warm_white = 0.0f;
msg.effect = StringRef::from_lit("rainbow");
for (auto _ : state) {
benchmark::DoNotOptimize(msg.calculate_size());
}
}
BENCHMARK(BM_CalculateSize_LightStateResponse);
// --- DeviceInfoResponse (nested submessages: 20 devices + 20 areas) ---
static DeviceInfoResponse make_device_info_response() {
DeviceInfoResponse msg;
msg.name = StringRef::from_lit("living-room-sensor");
msg.mac_address = StringRef::from_lit("AA:BB:CC:DD:EE:FF");
msg.esphome_version = StringRef::from_lit("2026.3.0");
msg.compilation_time = StringRef::from_lit("Mar 16 2026, 12:00:00");
msg.model = StringRef::from_lit("esp32-poe-iso");
msg.manufacturer = StringRef::from_lit("Olimex");
msg.friendly_name = StringRef::from_lit("Living Room Sensor");
#ifdef USE_DEVICES
for (uint32_t i = 0; i < ESPHOME_DEVICE_COUNT && i < 20; i++) {
msg.devices[i].device_id = i + 1;
msg.devices[i].name = StringRef::from_lit("device");
msg.devices[i].area_id = (i % 20) + 1;
}
#endif
#ifdef USE_AREAS
for (uint32_t i = 0; i < ESPHOME_AREA_COUNT && i < 20; i++) {
msg.areas[i].area_id = i + 1;
msg.areas[i].name = StringRef::from_lit("area");
}
#endif
return msg;
}
static void BM_CalculateSize_DeviceInfoResponse(benchmark::State &state) {
auto msg = make_device_info_response();
for (auto _ : state) {
benchmark::DoNotOptimize(msg.calculate_size());
}
}
BENCHMARK(BM_CalculateSize_DeviceInfoResponse);
static void BM_Encode_DeviceInfoResponse(benchmark::State &state) {
auto msg = make_device_info_response();
APIBuffer buffer;
uint32_t total_size = msg.calculate_size();
buffer.resize(total_size);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
msg.encode(writer);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_DeviceInfoResponse);
static void BM_CalcAndEncode_DeviceInfoResponse(benchmark::State &state) {
auto msg = make_device_info_response();
APIBuffer buffer;
for (auto _ : state) {
uint32_t size = msg.calculate_size();
buffer.resize(size);
ProtoWriteBuffer writer(&buffer, 0);
msg.encode(writer);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_CalcAndEncode_DeviceInfoResponse);
} // namespace esphome::api::benchmarks
tests/benchmarks/components/api/bench_proto_varint.cpp
+99
View File
@@ -0,0 +1,99 @@
#include <benchmark/benchmark.h>
#include "esphome/components/api/proto.h"
#include "esphome/components/api/api_buffer.h"
namespace esphome::api::benchmarks {
// --- ProtoVarInt::parse() benchmarks ---
static void BM_ProtoVarInt_Parse_SingleByte(benchmark::State &state) {
// Single-byte varint (0-127) — the most common case (fast path)
uint8_t buf[] = {0x42}; // value = 66
for (auto _ : state) {
auto result = ProtoVarInt::parse(buf, sizeof(buf));
benchmark::DoNotOptimize(result);
}
}
BENCHMARK(BM_ProtoVarInt_Parse_SingleByte);
static void BM_ProtoVarInt_Parse_TwoByte(benchmark::State &state) {
// Two-byte varint (128-16383)
uint8_t buf[] = {0x80, 0x01}; // value = 128
for (auto _ : state) {
auto result = ProtoVarInt::parse(buf, sizeof(buf));
benchmark::DoNotOptimize(result);
}
}
BENCHMARK(BM_ProtoVarInt_Parse_TwoByte);
static void BM_ProtoVarInt_Parse_FiveByte(benchmark::State &state) {
// Five-byte varint (max uint32 = 4294967295)
uint8_t buf[] = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F};
for (auto _ : state) {
auto result = ProtoVarInt::parse(buf, sizeof(buf));
benchmark::DoNotOptimize(result);
}
}
BENCHMARK(BM_ProtoVarInt_Parse_FiveByte);
// --- Varint encoding benchmarks ---
static void BM_Encode_Varint_Small(benchmark::State &state) {
// Value < 128 — single byte fast path
APIBuffer buffer;
buffer.resize(16);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
writer.encode_varint_raw(42);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_Varint_Small);
static void BM_Encode_Varint_Large(benchmark::State &state) {
// Value > 128 — multi-byte slow path
APIBuffer buffer;
buffer.resize(16);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
writer.encode_varint_raw(300);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_Varint_Large);
static void BM_Encode_Varint_MaxUint32(benchmark::State &state) {
APIBuffer buffer;
buffer.resize(16);
for (auto _ : state) {
ProtoWriteBuffer writer(&buffer, 0);
writer.encode_varint_raw(0xFFFFFFFF);
benchmark::DoNotOptimize(buffer.data());
}
}
BENCHMARK(BM_Encode_Varint_MaxUint32);
// --- ProtoSize::varint() benchmarks ---
static void BM_ProtoSize_Varint_Small(benchmark::State &state) {
for (auto _ : state) {
benchmark::DoNotOptimize(ProtoSize::varint(42));
}
}
BENCHMARK(BM_ProtoSize_Varint_Small);
static void BM_ProtoSize_Varint_Large(benchmark::State &state) {
for (auto _ : state) {
benchmark::DoNotOptimize(ProtoSize::varint(0xFFFFFFFF));
}
}
BENCHMARK(BM_ProtoSize_Varint_Large);
} // namespace esphome::api::benchmarks
tests/benchmarks/components/api/bench_scheduler.cpp
+63
View File
@@ -0,0 +1,63 @@
#include <benchmark/benchmark.h>
#include "esphome/core/scheduler.h"
#include "esphome/core/hal.h"
namespace esphome::benchmarks {
// --- Scheduler fast path: no work to do ---
static void BM_Scheduler_Call_NoWork(benchmark::State &state) {
Scheduler scheduler;
uint32_t now = millis();
for (auto _ : state) {
scheduler.call(now);
benchmark::DoNotOptimize(now);
}
}
BENCHMARK(BM_Scheduler_Call_NoWork);
// --- Scheduler with timers: call() when timers exist but aren't due ---
static void BM_Scheduler_Call_TimersNotDue(benchmark::State &state) {
Scheduler scheduler;
Component dummy_component;
// Add some timeouts far in the future
for (int i = 0; i < 10; i++) {
scheduler.set_timeout(&dummy_component, static_cast<uint32_t>(i), 1000000, []() {});
}
scheduler.process_to_add();
uint32_t now = millis();
for (auto _ : state) {
scheduler.call(now);
benchmark::DoNotOptimize(now);
}
}
BENCHMARK(BM_Scheduler_Call_TimersNotDue);
// --- Scheduler: next_schedule_in() calculation ---
static void BM_Scheduler_NextScheduleIn(benchmark::State &state) {
Scheduler scheduler;
Component dummy_component;
// Add some timeouts
for (int i = 0; i < 10; i++) {
scheduler.set_timeout(&dummy_component, static_cast<uint32_t>(i), 1000 * (i + 1), []() {});
}
scheduler.process_to_add();
uint32_t now = millis();
for (auto _ : state) {
auto result = scheduler.next_schedule_in(now);
benchmark::DoNotOptimize(result);
}
}
BENCHMARK(BM_Scheduler_NextScheduleIn);
} // namespace esphome::benchmarks
tests/benchmarks/components/api/benchmark.yaml
+114
View File
@@ -0,0 +1,114 @@
# Components needed for API protobuf benchmarks.
# Merged into the base config before validation so all
# dependencies get proper defaults.
#
# esphome: sub-keys are merged into the base config.
esphome:
areas:
- id: area_1
name: "Area 1"
- id: area_2
name: "Area 2"
- id: area_3
name: "Area 3"
- id: area_4
name: "Area 4"
- id: area_5
name: "Area 5"
- id: area_6
name: "Area 6"
- id: area_7
name: "Area 7"
- id: area_8
name: "Area 8"
- id: area_9
name: "Area 9"
- id: area_10
name: "Area 10"
- id: area_11
name: "Area 11"
- id: area_12
name: "Area 12"
- id: area_13
name: "Area 13"
- id: area_14
name: "Area 14"
- id: area_15
name: "Area 15"
- id: area_16
name: "Area 16"
- id: area_17
name: "Area 17"
- id: area_18
name: "Area 18"
- id: area_19
name: "Area 19"
- id: area_20
name: "Area 20"
devices:
- id: device_1
name: "Device 1"
area_id: area_1
- id: device_2
name: "Device 2"
area_id: area_2
- id: device_3
name: "Device 3"
area_id: area_3
- id: device_4
name: "Device 4"
area_id: area_4
- id: device_5
name: "Device 5"
area_id: area_5
- id: device_6
name: "Device 6"
area_id: area_6
- id: device_7
name: "Device 7"
area_id: area_7
- id: device_8
name: "Device 8"
area_id: area_8
- id: device_9
name: "Device 9"
area_id: area_9
- id: device_10
name: "Device 10"
area_id: area_10
- id: device_11
name: "Device 11"
area_id: area_11
- id: device_12
name: "Device 12"
area_id: area_12
- id: device_13
name: "Device 13"
area_id: area_13
- id: device_14
name: "Device 14"
area_id: area_14
- id: device_15
name: "Device 15"
area_id: area_15
- id: device_16
name: "Device 16"
area_id: area_16
- id: device_17
name: "Device 17"
area_id: area_17
- id: device_18
name: "Device 18"
area_id: area_18
- id: device_19
name: "Device 19"
area_id: area_19
- id: device_20
name: "Device 20"
area_id: area_20
api:
sensor:
binary_sensor:
light:
switch:
tests/benchmarks/components/main.cpp
+38
View File
@@ -0,0 +1,38 @@
#include <benchmark/benchmark.h>
#include "esphome/components/logger/logger.h"
/*
This special main.cpp provides the entry point for Google Benchmark.
It replaces the default ESPHome main with a benchmark runner.
See the codspeed_plan.md for more information.
*/
// Auto generated code by esphome
// ========== AUTO GENERATED INCLUDE BLOCK BEGIN ===========
// ========== AUTO GENERATED INCLUDE BLOCK END ==========="
void original_setup() {
// This function won't be run.
// ========== AUTO GENERATED CODE BEGIN ===========
// =========== AUTO GENERATED CODE END ============
}
void setup() {
// Log functions call global_logger->log_vprintf_() without a null check,
// so we must set up a Logger before any test that triggers logging.
static esphome::logger::Logger test_logger(0);
test_logger.set_log_level(ESPHOME_LOG_LEVEL);
test_logger.pre_setup();
int argc = 1;
char arg0[] = "benchmark";
char *argv[] = {arg0, nullptr};
::benchmark::Initialize(&argc, argv);
::benchmark::RunSpecifiedBenchmarks();
::benchmark::Shutdown();
exit(0);
}
void loop() {}