Merge branch 'esp8266-native-framework-installer' into esp8266-native-library-backend

esphome/arduino8266/framework.py

@@ -22,8 +22,9 @@ import functools
 import logging
 import os
 from pathlib import Path
-import shutil
+from typing import NamedTuple
 
+from esphome.build_helpers.ninja import find_ninja
 from esphome.core import EsphomeError, Version
 from esphome.framework_helpers import (
     ccache_defaults_env,
@@ -78,31 +79,15 @@ def get_toolchain_path() -> Path:
     return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION
 
 
-def _find_ninja() -> Path:
-    """Locate the ninja binary: PATH first, else the ninja PyPI wheel.
+class InstalledPaths(NamedTuple):
+    """Locations of the installed framework, toolchain, and ninja binary."""
 
-    The wheel is a requirements.txt dependency, so pip has already
-    integrity-checked it; no download logic is needed here.
-    """
-    if binary := shutil.which("ninja"):
-        return Path(binary)
-    try:
-        import ninja
-    except ImportError:
-        wheel_binary = None
-    else:
-        wheel_binary = Path(ninja.BIN_DIR) / (
-            "ninja.exe" if os.name == "nt" else "ninja"
-        )
-    if wheel_binary is None or not wheel_binary.is_file():
-        raise EsphomeError(
-            "ninja not found on PATH or in the ninja package; reinstall the "
-            "esphome Python environment"
-        )
-    return wheel_binary
+    framework: Path
+    toolchain: Path
+    ninja: Path
 
 
-def check_and_install(framework_version: Version) -> dict[str, Path]:
+def check_and_install(framework_version: Version) -> InstalledPaths:
     """Ensure framework, toolchain, and ninja are installed; return their paths."""
     if framework_version < MIN_FRAMEWORK_VERSION:
         # Config validation enforces this too; keep the module honest when
@@ -112,7 +97,7 @@ def check_and_install(framework_version: Version) -> dict[str, Path]:
             f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}"
         )
     # Probe the cheap local dependency before ~110 MB of downloads
-    ninja_path = _find_ninja()
+    ninja_path = find_ninja()
     package_version = framework_package_version(framework_version)
     framework_path = get_framework_path(package_version)
     downloads_dir = get_arduino8266_tools_path() / "downloads"
@@ -133,11 +118,9 @@ def check_and_install(framework_version: Version) -> dict[str, Path]:
         downloads_dir,
         expect=("bin",),
     )
-    return {
-        "framework_path": framework_path,
-        "toolchain_path": toolchain_path,
-        "ninja_path": ninja_path,
-    }
+    return InstalledPaths(
+        framework=framework_path, toolchain=toolchain_path, ninja=ninja_path
+    )
 
 
 def get_build_env(toolchain_path: Path) -> dict[str, str]:

esphome/build_helpers/ninja.py

@@ -0,0 +1,33 @@
+"""Platform-neutral helpers for ninja-driven native builds."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import shutil
+
+from esphome.core import EsphomeError
+
+
+def find_ninja() -> Path:
+    """Locate the ninja binary: PATH first, else the ninja PyPI wheel.
+
+    The wheel is a requirements.txt dependency, so pip has already
+    integrity-checked it; no download logic is needed here.
+    """
+    if binary := shutil.which("ninja"):
+        return Path(binary)
+    try:
+        import ninja
+    except ImportError:
+        wheel_binary = None
+    else:
+        wheel_binary = Path(ninja.BIN_DIR) / (
+            "ninja.exe" if os.name == "nt" else "ninja"
+        )
+    if wheel_binary is None or not wheel_binary.is_file():
+        raise EsphomeError(
+            "ninja not found on PATH or in the ninja package; reinstall the "
+            "esphome Python environment"
+        )
+    return wheel_binary

esphome/components/esp32/__init__.py

@@ -1071,20 +1071,11 @@ def _check_esp_idf_versions(config: ConfigType) -> ConfigType:
     return config
 
 
-def _validate_toolchain(value) -> Toolchain:
-    return Toolchain(
-        cv.one_of(Toolchain.PLATFORMIO, Toolchain.ESP_IDF, lower=True)(value)
-    )
-
-
-def _resolve_toolchain(value: ConfigType) -> ConfigType:
-    # Resolve toolchain: CLI (already on CORE.toolchain) > YAML > default.
-    # Runs before _detect_variant so downstream validators can rely on
-    # CORE.toolchain instead of re-resolving it from the config dict.
-    if CORE.toolchain is None:
-        CORE.toolchain = value.get(CONF_TOOLCHAIN, Toolchain.ESP_IDF)
-    cv.check_supported_toolchain("ESP32", (Toolchain.PLATFORMIO, Toolchain.ESP_IDF))
-    return value
+_TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.ESP_IDF)
+_validate_toolchain = cv.toolchain_enum(_TOOLCHAINS)
+# Runs before _detect_variant so downstream validators can rely on
+# CORE.toolchain instead of re-resolving it from the config dict.
+_resolve_toolchain = cv.resolve_toolchain("ESP32", _TOOLCHAINS, Toolchain.ESP_IDF)
 
 
 def _check_versions(config: ConfigType) -> ConfigType:

esphome/components/nrf52/__init__.py

@@ -125,11 +125,8 @@ def set_core_data(config: ConfigType) -> ConfigType:
     return config
 
 
-def _resolve_toolchain(config: ConfigType) -> ConfigType:
-    if CORE.toolchain is None:
-        CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.SDK_NRF)
-    cv.check_supported_toolchain("nRF52", (Toolchain.PLATFORMIO, Toolchain.SDK_NRF))
-    return config
+_TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.SDK_NRF)
+_resolve_toolchain = cv.resolve_toolchain("nRF52", _TOOLCHAINS, Toolchain.SDK_NRF)
 
 
 def set_framework(config: ConfigType) -> ConfigType:
@@ -171,10 +168,7 @@ BOOTLOADERS = [
 ]
 
 
-def _validate_toolchain(value) -> Toolchain:
-    return Toolchain(
-        cv.one_of(Toolchain.PLATFORMIO, Toolchain.SDK_NRF, lower=True)(value)
-    )
+_validate_toolchain = cv.toolchain_enum(_TOOLCHAINS)
 
 
 def _detect_bootloader(config: ConfigType) -> ConfigType:

esphome/config_validation.py

@@ -53,6 +53,7 @@ from esphome.const import (
     CONF_SETUP_PRIORITY,
     CONF_STATE_TOPIC,
     CONF_SUBSCRIBE_QOS,
+    CONF_TOOLCHAIN,
     CONF_TOPIC,
     CONF_TYPE,
     CONF_TYPE_ID,
@@ -2555,20 +2556,43 @@ def check_supported_toolchain(
         )
 
 
+def toolchain_enum(supported: tuple[Toolchain, ...]):
+    """Schema validator for a platform's ``toolchain`` config key."""
+
+    def validator(value) -> Toolchain:
+        return Toolchain(one_of(*supported, lower=True)(value))
+
+    return validator
+
+
+def resolve_toolchain(
+    platform_name: str, supported: tuple[Toolchain, ...], default: Toolchain
+):
+    """Resolve ``CORE.toolchain`` (CLI > YAML > default) and reject one the
+    platform cannot serve.
+
+    Add to the platform's validation chain before anything that reads
+    ``CORE.toolchain``.
+    """
+
+    def validator(config: ConfigType) -> ConfigType:
+        if CORE.toolchain is None:
+            CORE.toolchain = config.get(CONF_TOOLCHAIN, default)
+        check_supported_toolchain(platform_name, supported)
+        return config
+
+    return validator
+
+
 def require_platformio_toolchain(platform_name: str):
     """Reject a CLI-selected toolchain other than PlatformIO.
 
     For platforms with only the PlatformIO backend; without this a
     ``--toolchain`` they cannot serve would silently build with PlatformIO.
     """
-
-    def validator(config: ConfigType) -> ConfigType:
-        if CORE.toolchain is None:
-            CORE.toolchain = Toolchain.PLATFORMIO
-        check_supported_toolchain(platform_name, (Toolchain.PLATFORMIO,))
-        return config
-
-    return validator
+    return resolve_toolchain(
+        platform_name, (Toolchain.PLATFORMIO,), Toolchain.PLATFORMIO
+    )
 
 
 def require_framework_version(

esphome/const.py

@@ -25,6 +25,12 @@ class Toolchain(StrEnum):
     ARDUINO = "arduino"
 
 
+# Toolchains that drive their build natively and never read platformio.ini.
+# SDK_NRF is absent on purpose: the zephyr backend keeps consuming
+# platformio_options.
+NATIVE_TOOLCHAINS = frozenset({Toolchain.ESP_IDF, Toolchain.ARDUINO})
+
+
 class Platform(StrEnum):
     """Platform identifiers for ESPHome."""
 

esphome/core/__init__.py

@@ -21,6 +21,7 @@ from esphome.const import (
     KEY_CORE,
     KEY_TARGET_FRAMEWORK,
     KEY_TARGET_PLATFORM,
+    NATIVE_TOOLCHAINS,
     PLATFORM_BK72XX,
     PLATFORM_ESP32,
     PLATFORM_ESP8266,
@@ -992,6 +993,12 @@ class EsphomeCore:
         """
         return self.toolchain == Toolchain.ARDUINO
 
+    @property
+    def using_native_toolchain(self):
+        """Whether the selected toolchain builds natively, without reading
+        ``platformio.ini`` (see ``NATIVE_TOOLCHAINS`` in ``esphome.const``)."""
+        return self.toolchain in NATIVE_TOOLCHAINS
+
     @property
     def using_zephyr(self):
         return self.target_framework == "zephyr"

esphome/core/config.py

@@ -557,9 +557,7 @@ def _add_library_str(lib: str) -> None:
 
 @coroutine_with_priority(CoroPriority.FINAL)
 async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> None:
-    # Every platform's toolchain validation rejects values it cannot serve,
-    # so using_toolchain_arduino by itself implies the native ESP8266 build.
-    if CORE.using_toolchain_esp_idf or CORE.using_toolchain_arduino:
+    if CORE.using_native_toolchain:
         # The native builds don't read platformio.ini; honor the options
         # with a native equivalent and warn about the rest, which would
         # otherwise be silently ignored.

esphome/espidf/framework.py

@@ -11,10 +11,11 @@ import re
 import shutil
 from typing import Any, NoReturn
 
-from esphome.core import CORE, Version
+from esphome.core import Version
 from esphome.framework_helpers import (
     PathType,
     archive_extract_all,
+    ccache_defaults_env,
     create_venv,
     download_from_mirrors,
     download_with_resume,
@@ -1156,25 +1157,12 @@ def _ccache_env() -> dict[str, str]:
         # ESP-IDF silently skips ccache without the binary; don't enable it.
         return {}
 
-    # ccache is enabled past here. build_path is set during preload for every
-    # config-loading command, so it being unset means a caller built the IDF env
-    # too early -- fail loudly rather than silently drop CCACHE_BASEDIR (which
-    # would quietly cost cross-device cache hits).
-    if CORE.build_path is None:
-        raise ValueError(
-            "CORE.build_path must be set before constructing the ESP-IDF build "
-            "environment"
-        )
-
-    defaults = {
-        "IDF_CCACHE_ENABLE": "1",
-        "CCACHE_DIR": str(get_idf_tools_path() / "ccache"),
-        "CCACHE_NOHASHDIR": "true",
-        "CCACHE_DEPEND": "1",
-        "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
-    }
-    # Don't override CCACHE_* values the user already set in their environment.
-    return {k: v for k, v in defaults.items() if k not in os.environ}
+    # ccache is enabled past here; the shared helper carries the CCACHE_*
+    # policy (and the fail-loud build_path guard).
+    env = ccache_defaults_env(get_idf_tools_path() / "ccache")
+    if "IDF_CCACHE_ENABLE" not in os.environ:
+        env["IDF_CCACHE_ENABLE"] = "1"
+    return env
 
 
 def get_framework_env(

tests/unit_tests/build_helpers/test_ninja.py

@@ -0,0 +1,50 @@
+"""Tests for esphome.build_helpers.ninja."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from esphome.build_helpers import ninja as ninja_helper
+from esphome.core import EsphomeError
+
+
+def test_find_ninja_prefers_path(tmp_path: Path) -> None:
+    with patch("shutil.which", return_value=str(tmp_path / "ninja")):
+        assert ninja_helper.find_ninja() == tmp_path / "ninja"
+
+
+def test_find_ninja_falls_back_to_wheel(tmp_path: Path) -> None:
+    """Without a PATH entry, the ninja PyPI wheel's binary is used."""
+    binary_name = "ninja.exe" if os.name == "nt" else "ninja"
+    (tmp_path / binary_name).touch()
+    wheel = MagicMock(BIN_DIR=str(tmp_path))
+    with (
+        patch("shutil.which", return_value=None),
+        patch.dict(sys.modules, {"ninja": wheel}),
+    ):
+        assert ninja_helper.find_ninja() == tmp_path / binary_name
+
+
+def test_find_ninja_package_not_installed() -> None:
+    """A missing ninja package raises the actionable message, not ImportError."""
+    with (
+        patch("shutil.which", return_value=None),
+        patch.dict(sys.modules, {"ninja": None}),
+        pytest.raises(EsphomeError, match="ninja not found"),
+    ):
+        ninja_helper.find_ninja()
+
+
+def test_find_ninja_missing_everywhere(tmp_path: Path) -> None:
+    wheel = MagicMock(BIN_DIR=str(tmp_path))
+    with (
+        patch("shutil.which", return_value=None),
+        patch.dict(sys.modules, {"ninja": wheel}),
+        pytest.raises(EsphomeError, match="ninja not found"),
+    ):
+        ninja_helper.find_ninja()

tests/unit_tests/test_arduino8266_framework.py

@@ -5,8 +5,7 @@ from __future__ import annotations
 import os
 from pathlib import Path
 import subprocess
-import sys
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 
@@ -36,55 +35,16 @@ def test_tools_path_default_and_prefix(tmp_path: Path) -> None:
     assert path != Path.cwd()
 
 
-def test_find_ninja_prefers_path(tmp_path: Path) -> None:
-    with patch("shutil.which", return_value=str(tmp_path / "ninja")):
-        assert framework._find_ninja() == tmp_path / "ninja"
-
-
-def test_find_ninja_falls_back_to_wheel(tmp_path: Path) -> None:
-    """Without a PATH entry, the ninja PyPI wheel's binary is used."""
-    binary_name = "ninja.exe" if os.name == "nt" else "ninja"
-    (tmp_path / binary_name).touch()
-    wheel = MagicMock(BIN_DIR=str(tmp_path))
-    with (
-        patch("shutil.which", return_value=None),
-        patch.dict(sys.modules, {"ninja": wheel}),
-    ):
-        assert framework._find_ninja() == tmp_path / binary_name
-
-
-def test_find_ninja_package_not_installed() -> None:
-    """A missing ninja package raises the actionable message, not ImportError."""
-    with (
-        patch("shutil.which", return_value=None),
-        patch.dict(sys.modules, {"ninja": None}),
-        pytest.raises(EsphomeError, match="ninja not found"),
-    ):
-        framework._find_ninja()
-
-
-def test_find_ninja_missing_everywhere(tmp_path: Path) -> None:
-    wheel = MagicMock(BIN_DIR=str(tmp_path))
-    with (
-        patch("shutil.which", return_value=None),
-        patch.dict(sys.modules, {"ninja": wheel}),
-        pytest.raises(EsphomeError, match="ninja not found"),
-    ):
-        framework._find_ninja()
-
-
 def test_check_and_install_returns_paths(tmp_path: Path) -> None:
     with (
         patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
         patch.object(framework, "install_package") as mock_install,
-        patch.object(framework, "_find_ninja", return_value=tmp_path / "ninja"),
+        patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"),
     ):
         paths = framework.check_and_install(cv.Version(3, 1, 2))
-    assert paths["framework_path"] == tmp_path / "frameworks" / "3.30102.0"
-    assert (
-        paths["toolchain_path"] == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION
-    )
-    assert paths["ninja_path"] == tmp_path / "ninja"
+    assert paths.framework == tmp_path / "frameworks" / "3.30102.0"
+    assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION
+    assert paths.ninja == tmp_path / "ninja"
     assert mock_install.call_count == 2
     # The layout checks cover the directories write_project needs, including
     # the bundled libraries/ tree

tests/unit_tests/test_espidf_framework.py

@@ -1400,8 +1400,9 @@ def _ccache_patches(tmp_path: Path, which: str | None, build_path: Path | None):
             "esphome.espidf.framework.get_idf_tools_path",
             return_value=tmp_path / "tools",
         ),
+        # ccache_defaults_env (framework_helpers) reads CORE at call time
         patch(
-            "esphome.espidf.framework.CORE",
+            "esphome.core.CORE",
             SimpleNamespace(build_path=build_path),
         ),
     )