[host] Build natively with ninja instead of PlatformIO (#19214)

Co-authored-by: J. Nick Koston <nick@home-assistant.io>

tests/integration/conftest.py

@@ -16,7 +16,6 @@ import re
 import shutil
 import signal
 import socket
-import subprocess
 import sys
 import tempfile
 import time
@@ -35,7 +34,7 @@ from esphome.helpers import (
     write_file,
     write_file_if_changed,
 )
-from esphome.platformio.toolchain import get_idedata
+from esphome.host.toolchain import get_elf_path
 
 from .const import (
     API_CONNECTION_TIMEOUT,
@@ -82,16 +81,9 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent
 INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests"
 
 
-def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
-    """Get environment variables for PlatformIO with shared cache."""
+def _get_build_env() -> dict[str, str]:
+    """Environment for an ``esphome compile`` subprocess."""
     env = os.environ.copy()
-    env["PLATFORMIO_CORE_DIR"] = str(cache_dir)
-    env["PLATFORMIO_CACHE_DIR"] = str(cache_dir / ".cache")
-    # libdeps is keyed only by env name (the device name), and fixtures share
-    # names; two xdist workers first-compiling the same name race pio pkg
-    # install in the same directory. Keep libdeps per worker.
-    worker = os.environ.get("PYTEST_XDIST_WORKER", "master")
-    env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker)
     # Prevent cache cleaning during integration tests
     env["ESPHOME_SKIP_CLEAN_BUILD"] = "1"
     # Cap each compile's -j so several xdist workers do not each spawn a
@@ -110,53 +102,41 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
     return env
 
 
+# Registry libraries (noise-c, lvgl, ...) download into <data_dir>/pio_components,
+# and every integration test gets its own data dir. Share one download cache
+# per xdist worker instead: the converter has no cross-process lock, so
+# workers must never write the same dir, while a worker's later tests (and
+# later runs) reuse what it fetched.
+LIBRARY_CACHE_ROOT = INTEGRATION_TESTS_ROOT / "pio_components"
+
+
 @pytest.fixture(scope="session")
-def shared_platformio_cache() -> Generator[Path]:
-    """Initialize a shared PlatformIO cache for all integration tests."""
-    # Use a dedicated directory for integration tests to avoid conflicts.
-    test_cache_dir = INTEGRATION_TESTS_ROOT
-    cache_dir = test_cache_dir / "platformio"
+def shared_library_cache() -> Path:
+    """This worker's shared registry-library download cache."""
+    worker = os.environ.get("PYTEST_XDIST_WORKER", "master")
+    # Keyed by checkout too: two sessions on one machine share worker ids
+    cache_dir = LIBRARY_CACHE_ROOT / _REPO_KEY / worker
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir
 
-    # Use a lock file in the home directory to ensure only one process initializes the cache
-    # This is needed when running with pytest-xdist
-    # The lock file must be in a directory that already exists to avoid race conditions
-    lock_file = Path.home() / ".esphome-integration-tests-init.lock"
 
-    # Always acquire the lock to ensure cache is ready before proceeding
-    with lock_file.open("w") as lock_fd:
-        fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX)
-
-        # Check if the native platform is installed (the actual indicator of a populated cache)
-        native_platform = cache_dir / "platforms" / "native"
-        if not native_platform.exists():
-            # Create the test cache directory if it doesn't exist
-            test_cache_dir.mkdir(exist_ok=True)
-
-            with tempfile.TemporaryDirectory() as tmpdir:
-                # Use the cache_init fixture for initialization
-                init_dir = Path(tmpdir)
-                fixture_path = Path(__file__).parent / "fixtures" / "cache_init.yaml"
-                config_path = init_dir / "cache_init.yaml"
-                config_path.write_text(
-                    fixture_path.read_text(encoding="utf-8"), encoding="utf-8"
-                )
-
-                # Run compilation to populate the cache
-                # We must succeed here to avoid race conditions where multiple
-                # tests try to populate the same cache directory simultaneously
-                env = _get_platformio_env(cache_dir)
-
-                subprocess.run(
-                    [sys.executable, "-m", "esphome", "compile", str(config_path)],
-                    check=True,
-                    cwd=init_dir,
-                    env=env,
-                    close_fds=False,
-                )
-
-        # Lock is held until here, ensuring cache is fully populated before any test proceeds
-
-    yield cache_dir
+def _link_library_cache(config_dir: Path, cache_dir: Path) -> None:
+    """Point a config dir's library download cache at the shared one
+    (blocking, run in executor)."""
+    data_dir = config_dir / ".esphome"
+    data_dir.mkdir(exist_ok=True)
+    link = data_dir / "pio_components"
+    if link.is_symlink():
+        if link.resolve() == cache_dir.resolve():
+            return
+        # A shared build dir was linked by another worker; writing through
+        # it would put two workers in one cache
+        link.unlink()
+    elif link.exists():
+        # A real dir from a run predating the shared cache; nothing in it is
+        # worth more than a re-download
+        rmtree(link)
+    link.symlink_to(cache_dir, target_is_directory=True)
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -238,18 +218,17 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s
         content = content.replace("api:", f"api:\n  port: {unused_tcp_port}")
 
     # Add debug build flags for integration tests to enable assertions
-    if "esphome:" in content and "platformio_options:" not in content:
-        # Add platformio_options with debug flags after esphome:
+    if "esphome:" in content and "build_flags:" not in content:
+        # Add build_flags with debug flags after esphome:
         content = content.replace(
             "esphome:",
             "esphome:\n"
             "  # Enable assertions for integration tests\n"
-            "  platformio_options:\n"
-            "    build_flags:\n"
-            '      - "-DDEBUG"  # Enable assert() statements\n'
-            '      - "-DESPHOME_DEBUG"  # Enable ESPHOME_DEBUG_ASSERT checks\n'
-            '      - "-DESPHOME_DEBUG_API"  # Enable API protocol asserts\n'
-            '      - "-g"       # Add debug symbols',
+            "  build_flags:\n"
+            '    - "-DDEBUG"  # Enable assert() statements\n'
+            '    - "-DESPHOME_DEBUG"  # Enable ESPHOME_DEBUG_ASSERT checks\n'
+            '    - "-DESPHOME_DEBUG_API"  # Enable API protocol asserts\n'
+            '    - "-g"       # Add debug symbols',
         )
 
     # Replace external component path placeholder if present
@@ -283,8 +262,8 @@ async def write_yaml_config(
     yield _write_config
 
 
-# Deliberately not CI-cached (ci.yml caches only platformio/ subpaths); stale
-# dirs for a fixture are pruned when its content hash changes.
+# Deliberately not CI-cached; stale dirs for a fixture are pruned when its
+# content hash changes.
 SHARED_BUILDS_ROOT = INTEGRATION_TESTS_ROOT / "builds"
 
 # In the dir name (not just the hash) so pruning stays inside this checkout
@@ -470,8 +449,7 @@ def _resolve_compiled_binary(config_path: Path) -> Path:
     )
     if config is None:
         raise RuntimeError(f"Failed to read config from {config_path}")
-    idedata = get_idedata(config)
-    binary_path = Path(idedata.firmware_elf_path)
+    binary_path = get_elf_path()
     if not binary_path.exists():
         raise RuntimeError(f"Compiled binary not found at {binary_path}")
     return binary_path
@@ -480,19 +458,22 @@ def _resolve_compiled_binary(config_path: Path) -> Path:
 @pytest_asyncio.fixture
 async def compile_esphome(
     integration_test_dir: Path,
-    shared_platformio_cache: Path,
+    shared_library_cache: Path,
     request: pytest.FixtureRequest,
 ) -> AsyncGenerator[CompileFunction]:
     """Compile an ESPHome configuration and return the binary path."""
 
     async def _compile(config_path: Path) -> Path:
-        # Use the shared PlatformIO cache for faster compilation
-        # This avoids re-downloading dependencies for each test
-        env = _get_platformio_env(shared_platformio_cache)
+        env = _get_build_env()
         loop = asyncio.get_running_loop()
 
         name = _shared_yaml_name(request)
         if name is None:
+            # Share the library download cache so a test never re-fetches
+            # what an earlier one already pulled from the registry
+            await loop.run_in_executor(
+                None, _link_library_cache, integration_test_dir, shared_library_cache
+            )
             await _run_esphome_compile(config_path, integration_test_dir, env)
             return await loop.run_in_executor(
                 None, _resolve_compiled_binary, config_path
@@ -568,6 +549,9 @@ async def compile_esphome(
             await loop.run_in_executor(
                 None, write_file_if_changed, shared_config, content
             )
+            await loop.run_in_executor(
+                None, _link_library_cache, shared_dir, shared_library_cache
+            )
             await _run_esphome_compile(shared_config, shared_dir, env)
             if built is None or not built.exists():
                 built = await loop.run_in_executor(

tests/integration/fixtures/api_action_timeout.yaml

@@ -1,9 +1,8 @@
 esphome:
   name: api-action-timeout-test
   # Use a short timeout for testing (500ms instead of 30s)
-  platformio_options:
-    build_flags:
-      - "-DUSE_API_ACTION_CALL_TIMEOUT_MS=500"
+  build_flags:
+    - "-DUSE_API_ACTION_CALL_TIMEOUT_MS=500"
 
 host:
 

tests/integration/fixtures/cache_init.yaml

@@ -1,10 +0,0 @@
-esphome:
-  name: cache-init
-
-host:
-
-api:
-  encryption:
-    key: "IIevImVI42I0FGos5nLqFK91jrJehrgidI0ArwMLr8w="
-
-logger:

tests/integration/fixtures/fnv1_hash_object_id.yaml

@@ -1,8 +1,7 @@
 esphome:
   name: fnv1-hash-object-id-test
-  platformio_options:
-    build_flags:
-      - "-DDEBUG"
+  build_flags:
+    - "-DDEBUG"
   on_boot:
     - lambda: |-
         using esphome::fnv1_hash_object_id;

tests/integration/fixtures/fnv1a_hash.yaml

@@ -1,8 +1,7 @@
 esphome:
   name: fnv1a-hash-test
-  platformio_options:
-    build_flags:
-      - "-DDEBUG"
+  build_flags:
+    - "-DDEBUG"
   on_boot:
     - lambda: |-
         using esphome::fnv1a_hash;

tests/integration/fixtures/micros_to_millis.yaml

@@ -1,8 +1,7 @@
 esphome:
   name: micros-to-millis-test
-  platformio_options:
-    build_flags:
-      - "-DDEBUG"
+  build_flags:
+    - "-DDEBUG"
   on_boot:
     - lambda: |-
         using esphome::micros_to_millis;