Merge branch 'dev' into runtime_stats

tests/integration/conftest.py

@@ -5,12 +5,14 @@ from __future__ import annotations
 import asyncio
 from collections.abc import AsyncGenerator, Callable, Generator
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
+import fcntl
 import logging
 import os
 from pathlib import Path
 import platform
 import signal
 import socket
+import subprocess
 import sys
 import tempfile
 from typing import TextIO
@@ -50,6 +52,66 @@ if platform.system() == "Windows":
 import pty  # not available on Windows
 
 
+def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
+    """Get environment variables for PlatformIO with shared cache."""
+    env = os.environ.copy()
+    env["PLATFORMIO_CORE_DIR"] = str(cache_dir)
+    env["PLATFORMIO_CACHE_DIR"] = str(cache_dir / ".cache")
+    env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps")
+    return env
+
+
+@pytest.fixture(scope="session")
+def shared_platformio_cache() -> Generator[Path]:
+    """Initialize a shared PlatformIO cache for all integration tests."""
+    # Use a dedicated directory for integration tests to avoid conflicts
+    test_cache_dir = Path.home() / ".esphome-integration-tests"
+    cache_dir = test_cache_dir / "platformio"
+
+    # Use a lock file in the home directory to ensure only one process initializes the cache
+    # This is needed when running with pytest-xdist
+    # The lock file must be in a directory that already exists to avoid race conditions
+    lock_file = Path.home() / ".esphome-integration-tests-init.lock"
+
+    # Always acquire the lock to ensure cache is ready before proceeding
+    with open(lock_file, "w") as lock_fd:
+        fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX)
+
+        # Check if cache needs initialization while holding the lock
+        if not cache_dir.exists() or not any(cache_dir.iterdir()):
+            # Create the test cache directory if it doesn't exist
+            test_cache_dir.mkdir(exist_ok=True)
+
+            with tempfile.TemporaryDirectory() as tmpdir:
+                # Create a basic host config
+                init_dir = Path(tmpdir)
+                config_path = init_dir / "cache_init.yaml"
+                config_path.write_text("""esphome:
+  name: cache-init
+host:
+api:
+  encryption:
+    key: "IIevImVI42I0FGos5nLqFK91jrJehrgidI0ArwMLr8w="
+logger:
+""")
+
+                # Run compilation to populate the cache
+                # We must succeed here to avoid race conditions where multiple
+                # tests try to populate the same cache directory simultaneously
+                env = _get_platformio_env(cache_dir)
+
+                subprocess.run(
+                    ["esphome", "compile", str(config_path)],
+                    check=True,
+                    cwd=init_dir,
+                    env=env,
+                )
+
+        # Lock is held until here, ensuring cache is fully populated before any test proceeds
+
+    yield cache_dir
+
+
 @pytest.fixture(scope="module", autouse=True)
 def enable_aioesphomeapi_debug_logging():
     """Enable debug logging for aioesphomeapi to help diagnose connection issues."""
@@ -161,22 +223,14 @@ async def write_yaml_config(
 @pytest_asyncio.fixture
 async def compile_esphome(
     integration_test_dir: Path,
+    shared_platformio_cache: Path,
 ) -> AsyncGenerator[CompileFunction]:
     """Compile an ESPHome configuration and return the binary path."""
 
     async def _compile(config_path: Path) -> Path:
-        # Create a unique PlatformIO directory for this test to avoid race conditions
-        platformio_dir = integration_test_dir / ".platformio"
-        platformio_dir.mkdir(parents=True, exist_ok=True)
-
-        # Create cache directory as well
-        platformio_cache_dir = platformio_dir / ".cache"
-        platformio_cache_dir.mkdir(parents=True, exist_ok=True)
-
-        # Set up environment with isolated PlatformIO directories
-        env = os.environ.copy()
-        env["PLATFORMIO_CORE_DIR"] = str(platformio_dir)
-        env["PLATFORMIO_CACHE_DIR"] = str(platformio_cache_dir)
+        # Use the shared PlatformIO cache for faster compilation
+        # This avoids re-downloading dependencies for each test
+        env = _get_platformio_env(shared_platformio_cache)
 
         # Retry compilation up to 3 times if we get a segfault
         max_retries = 3

tests/integration/fixtures/api_custom_services.yaml

@@ -0,0 +1,24 @@
+esphome:
+  name: api-custom-services-test
+host:
+
+# This is required for CustomAPIDevice to work
+api:
+  custom_services: true
+  # Also test that YAML services still work
+  actions:
+    - action: test_yaml_service
+      then:
+        - logger.log: "YAML service called"
+
+logger:
+  level: DEBUG
+
+# External component that uses CustomAPIDevice
+external_components:
+  - source:
+      type: local
+      path: EXTERNAL_COMPONENT_PATH
+    components: [custom_api_device_component]
+
+custom_api_device_component:

tests/integration/fixtures/external_components/custom_api_device_component/__init__.py

@@ -0,0 +1,19 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ID
+
+custom_api_device_component_ns = cg.esphome_ns.namespace("custom_api_device_component")
+CustomAPIDeviceComponent = custom_api_device_component_ns.class_(
+    "CustomAPIDeviceComponent", cg.Component
+)
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(CustomAPIDeviceComponent),
+    }
+).extend(cv.COMPONENT_SCHEMA)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)

tests/integration/fixtures/external_components/custom_api_device_component/custom_api_device_component.cpp

@@ -0,0 +1,53 @@
+#include "custom_api_device_component.h"
+#include "esphome/core/log.h"
+
+#ifdef USE_API
+namespace esphome {
+namespace custom_api_device_component {
+
+static const char *const TAG = "custom_api";
+
+void CustomAPIDeviceComponent::setup() {
+  // Register services using CustomAPIDevice
+  register_service(&CustomAPIDeviceComponent::on_test_service, "custom_test_service");
+
+  register_service(&CustomAPIDeviceComponent::on_service_with_args, "custom_service_with_args",
+                   {"arg_string", "arg_int", "arg_bool", "arg_float"});
+
+  // Test array types
+  register_service(&CustomAPIDeviceComponent::on_service_with_arrays, "custom_service_with_arrays",
+                   {"bool_array", "int_array", "float_array", "string_array"});
+}
+
+void CustomAPIDeviceComponent::on_test_service() { ESP_LOGI(TAG, "Custom test service called!"); }
+
+// NOLINTNEXTLINE(performance-unnecessary-value-param)
+void CustomAPIDeviceComponent::on_service_with_args(std::string arg_string, int32_t arg_int, bool arg_bool,
+                                                    float arg_float) {
+  ESP_LOGI(TAG, "Custom service called with: %s, %d, %d, %.2f", arg_string.c_str(), arg_int, arg_bool, arg_float);
+}
+
+void CustomAPIDeviceComponent::on_service_with_arrays(std::vector<bool> bool_array, std::vector<int32_t> int_array,
+                                                      std::vector<float> float_array,
+                                                      std::vector<std::string> string_array) {
+  ESP_LOGI(TAG, "Array service called with %zu bools, %zu ints, %zu floats, %zu strings", bool_array.size(),
+           int_array.size(), float_array.size(), string_array.size());
+
+  // Log first element of each array if not empty
+  if (!bool_array.empty()) {
+    ESP_LOGI(TAG, "First bool: %s", bool_array[0] ? "true" : "false");
+  }
+  if (!int_array.empty()) {
+    ESP_LOGI(TAG, "First int: %d", int_array[0]);
+  }
+  if (!float_array.empty()) {
+    ESP_LOGI(TAG, "First float: %.2f", float_array[0]);
+  }
+  if (!string_array.empty()) {
+    ESP_LOGI(TAG, "First string: %s", string_array[0].c_str());
+  }
+}
+
+}  // namespace custom_api_device_component
+}  // namespace esphome
+#endif  // USE_API

tests/integration/fixtures/external_components/custom_api_device_component/custom_api_device_component.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include "esphome/core/component.h"
+#include "esphome/components/api/custom_api_device.h"
+
+#ifdef USE_API
+namespace esphome {
+namespace custom_api_device_component {
+
+using namespace api;
+
+class CustomAPIDeviceComponent : public Component, public CustomAPIDevice {
+ public:
+  void setup() override;
+
+  void on_test_service();
+
+  // NOLINTNEXTLINE(performance-unnecessary-value-param)
+  void on_service_with_args(std::string arg_string, int32_t arg_int, bool arg_bool, float arg_float);
+
+  void on_service_with_arrays(std::vector<bool> bool_array, std::vector<int32_t> int_array,
+                              std::vector<float> float_array, std::vector<std::string> string_array);
+};
+
+}  // namespace custom_api_device_component
+}  // namespace esphome
+#endif  // USE_API

tests/integration/fixtures/external_components/scheduler_string_lifetime_component/string_lifetime_component.cpp

@@ -23,19 +23,6 @@ void SchedulerStringLifetimeComponent::run_string_lifetime_test() {
   test_vector_reallocation();
   test_string_move_semantics();
   test_lambda_capture_lifetime();
-
-  // Schedule final check
-  this->set_timeout("final_check", 200, [this]() {
-    ESP_LOGI(TAG, "Tests passed: %d", this->tests_passed_);
-    ESP_LOGI(TAG, "Tests failed: %d", this->tests_failed_);
-
-    if (this->tests_failed_ == 0) {
-      ESP_LOGI(TAG, "SUCCESS: All string lifetime tests passed!");
-    } else {
-      ESP_LOGE(TAG, "FAILURE: %d string lifetime tests failed!", this->tests_failed_);
-    }
-    ESP_LOGI(TAG, "String lifetime tests complete");
-  });
 }
 
 void SchedulerStringLifetimeComponent::run_test1() {
@@ -69,7 +56,6 @@ void SchedulerStringLifetimeComponent::run_test5() {
 }
 
 void SchedulerStringLifetimeComponent::run_final_check() {
-  ESP_LOGI(TAG, "String lifetime tests complete");
   ESP_LOGI(TAG, "Tests passed: %d", this->tests_passed_);
   ESP_LOGI(TAG, "Tests failed: %d", this->tests_failed_);
 
@@ -78,6 +64,7 @@ void SchedulerStringLifetimeComponent::run_final_check() {
   } else {
     ESP_LOGE(TAG, "FAILURE: %d string lifetime tests failed!", this->tests_failed_);
   }
+  ESP_LOGI(TAG, "String lifetime tests complete");
 }
 
 void SchedulerStringLifetimeComponent::test_temporary_string_lifetime() {

tests/integration/fixtures/scheduler_null_name.yaml

@@ -0,0 +1,43 @@
+esphome:
+  name: scheduler-null-name
+
+host:
+
+logger:
+  level: DEBUG
+
+api:
+  services:
+    - service: test_null_name
+      then:
+        - lambda: |-
+            // First, create a scenario that would trigger the crash
+            // The crash happens when defer() is called with a name that would be cancelled
+
+            // Test 1: Create a defer with a valid name
+            App.scheduler.set_timeout(nullptr, "test_defer", 0, []() {
+              ESP_LOGI("TEST", "First defer should be cancelled");
+            });
+
+            // Test 2: Create another defer with the same name - this triggers cancel_item_locked_
+            // In the unfixed code, this would crash if the name was NULL
+            App.scheduler.set_timeout(nullptr, "test_defer", 0, []() {
+              ESP_LOGI("TEST", "Second defer executed");
+            });
+
+            // Test 3: Now test with nullptr - this is the actual crash scenario
+            // Create a defer item without a name (like voice assistant does)
+            const char* null_name = nullptr;
+            App.scheduler.set_timeout(nullptr, null_name, 0, []() {
+              ESP_LOGI("TEST", "Defer with null name executed");
+            });
+
+            // Test 4: Create another defer with null name - this would trigger the crash
+            App.scheduler.set_timeout(nullptr, null_name, 0, []() {
+              ESP_LOGI("TEST", "Second null defer executed");
+            });
+
+            // Test 5: Verify scheduler still works
+            App.scheduler.set_timeout(nullptr, "valid_timeout", 50, []() {
+              ESP_LOGI("TEST", "Test completed successfully");
+            });

tests/integration/test_api_custom_services.py

@@ -0,0 +1,144 @@
+"""Integration test for API custom services using CustomAPIDevice."""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+import re
+
+from aioesphomeapi import UserService, UserServiceArgType
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_api_custom_services(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test CustomAPIDevice services work correctly with custom_services: true."""
+    # Get the path to the external components directory
+    external_components_path = str(
+        Path(__file__).parent / "fixtures" / "external_components"
+    )
+
+    # Replace the placeholder in the YAML config with the actual path
+    yaml_config = yaml_config.replace(
+        "EXTERNAL_COMPONENT_PATH", external_components_path
+    )
+
+    loop = asyncio.get_running_loop()
+
+    # Track log messages
+    yaml_service_future = loop.create_future()
+    custom_service_future = loop.create_future()
+    custom_args_future = loop.create_future()
+    custom_arrays_future = loop.create_future()
+
+    # Patterns to match in logs
+    yaml_service_pattern = re.compile(r"YAML service called")
+    custom_service_pattern = re.compile(r"Custom test service called!")
+    custom_args_pattern = re.compile(
+        r"Custom service called with: test_string, 456, 1, 78\.90"
+    )
+    custom_arrays_pattern = re.compile(
+        r"Array service called with 2 bools, 3 ints, 2 floats, 2 strings"
+    )
+
+    def check_output(line: str) -> None:
+        """Check log output for expected messages."""
+        if not yaml_service_future.done() and yaml_service_pattern.search(line):
+            yaml_service_future.set_result(True)
+        elif not custom_service_future.done() and custom_service_pattern.search(line):
+            custom_service_future.set_result(True)
+        elif not custom_args_future.done() and custom_args_pattern.search(line):
+            custom_args_future.set_result(True)
+        elif not custom_arrays_future.done() and custom_arrays_pattern.search(line):
+            custom_arrays_future.set_result(True)
+
+    # Run with log monitoring
+    async with run_compiled(yaml_config, line_callback=check_output):
+        async with api_client_connected() as client:
+            # Verify device info
+            device_info = await client.device_info()
+            assert device_info is not None
+            assert device_info.name == "api-custom-services-test"
+
+            # List services
+            _, services = await client.list_entities_services()
+
+            # Should have 4 services: 1 YAML + 3 CustomAPIDevice
+            assert len(services) == 4, f"Expected 4 services, found {len(services)}"
+
+            # Find our services
+            yaml_service: UserService | None = None
+            custom_service: UserService | None = None
+            custom_args_service: UserService | None = None
+            custom_arrays_service: UserService | None = None
+
+            for service in services:
+                if service.name == "test_yaml_service":
+                    yaml_service = service
+                elif service.name == "custom_test_service":
+                    custom_service = service
+                elif service.name == "custom_service_with_args":
+                    custom_args_service = service
+                elif service.name == "custom_service_with_arrays":
+                    custom_arrays_service = service
+
+            assert yaml_service is not None, "test_yaml_service not found"
+            assert custom_service is not None, "custom_test_service not found"
+            assert custom_args_service is not None, "custom_service_with_args not found"
+            assert custom_arrays_service is not None, (
+                "custom_service_with_arrays not found"
+            )
+
+            # Test YAML service
+            client.execute_service(yaml_service, {})
+            await asyncio.wait_for(yaml_service_future, timeout=5.0)
+
+            # Test simple CustomAPIDevice service
+            client.execute_service(custom_service, {})
+            await asyncio.wait_for(custom_service_future, timeout=5.0)
+
+            # Verify custom_args_service arguments
+            assert len(custom_args_service.args) == 4
+            arg_types = {arg.name: arg.type for arg in custom_args_service.args}
+            assert arg_types["arg_string"] == UserServiceArgType.STRING
+            assert arg_types["arg_int"] == UserServiceArgType.INT
+            assert arg_types["arg_bool"] == UserServiceArgType.BOOL
+            assert arg_types["arg_float"] == UserServiceArgType.FLOAT
+
+            # Test CustomAPIDevice service with arguments
+            client.execute_service(
+                custom_args_service,
+                {
+                    "arg_string": "test_string",
+                    "arg_int": 456,
+                    "arg_bool": True,
+                    "arg_float": 78.9,
+                },
+            )
+            await asyncio.wait_for(custom_args_future, timeout=5.0)
+
+            # Verify array service arguments
+            assert len(custom_arrays_service.args) == 4
+            array_arg_types = {arg.name: arg.type for arg in custom_arrays_service.args}
+            assert array_arg_types["bool_array"] == UserServiceArgType.BOOL_ARRAY
+            assert array_arg_types["int_array"] == UserServiceArgType.INT_ARRAY
+            assert array_arg_types["float_array"] == UserServiceArgType.FLOAT_ARRAY
+            assert array_arg_types["string_array"] == UserServiceArgType.STRING_ARRAY
+
+            # Test CustomAPIDevice service with arrays
+            client.execute_service(
+                custom_arrays_service,
+                {
+                    "bool_array": [True, False],
+                    "int_array": [1, 2, 3],
+                    "float_array": [1.1, 2.2],
+                    "string_array": ["hello", "world"],
+                },
+            )
+            await asyncio.wait_for(custom_arrays_future, timeout=5.0)

tests/integration/test_scheduler_null_name.py

@@ -0,0 +1,59 @@
+"""Test that scheduler handles NULL names safely without crashing."""
+
+import asyncio
+import re
+
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_scheduler_null_name(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test that scheduler handles NULL names safely without crashing."""
+
+    loop = asyncio.get_running_loop()
+    test_complete_future: asyncio.Future[bool] = loop.create_future()
+
+    # Pattern to match test completion
+    test_complete_pattern = re.compile(r"Test completed successfully")
+
+    def check_output(line: str) -> None:
+        """Check log output for test completion."""
+        if not test_complete_future.done() and test_complete_pattern.search(line):
+            test_complete_future.set_result(True)
+
+    async with run_compiled(yaml_config, line_callback=check_output):
+        async with api_client_connected() as client:
+            # Verify we can connect
+            device_info = await client.device_info()
+            assert device_info is not None
+            assert device_info.name == "scheduler-null-name"
+
+            # List services
+            _, services = await asyncio.wait_for(
+                client.list_entities_services(), timeout=5.0
+            )
+
+            # Find our test service
+            test_null_name_service = next(
+                (s for s in services if s.name == "test_null_name"), None
+            )
+            assert test_null_name_service is not None, (
+                "test_null_name service not found"
+            )
+
+            # Execute the test
+            client.execute_service(test_null_name_service, {})
+
+            # Wait for test completion
+            try:
+                await asyncio.wait_for(test_complete_future, timeout=10.0)
+            except asyncio.TimeoutError:
+                pytest.fail(
+                    "Test did not complete within timeout - likely crashed due to NULL name"
+                )