[esp8266] Add the native framework and toolchain installer (#18557)

esphome/arduino8266/__init__.py

@@ -0,0 +1,9 @@
+"""Native (PlatformIO-free) build support for the ESP8266 Arduino core.
+
+This package downloads the Arduino ESP8266 core and the xtensa-lx106
+toolchain, generates a ninja build for them plus the ESPHome sources, and
+drives the build directly — the ESP8266 equivalent of ``esphome.espidf``.
+
+Deliberately importable without the esp8266 component to avoid circular
+imports; the component wires these modules in via lazy imports.
+"""

esphome/arduino8266/framework.py

@@ -0,0 +1,164 @@
+"""Download and install the Arduino ESP8266 core, toolchain, and ninja.
+
+Artifacts land in a machine-global cache (shared across projects, like the
+ESP-IDF install in ``esphome.espidf.framework``):
+
+    <cache>/arduino8266/frameworks/<version>/   framework-arduinoespressif8266
+    <cache>/arduino8266/toolchains/<version>/   toolchain-xtensa (gcc 10.3)
+
+Packages come from the PlatformIO registry (identical bits to the PlatformIO
+backend); ``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes
+from PATH or the ninja PyPI wheel.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import NamedTuple
+
+from esphome.build_helpers.ccache import ccache_defaults_env
+from esphome.build_helpers.ninja import find_ninja
+from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path
+from esphome.core import EsphomeError, Version
+from esphome.framework_helpers import str_to_lst_of_str
+from esphome.platformio.registry import install_package, prefetch_packages
+
+FRAMEWORK_PACKAGE = "framework-arduinoespressif8266"
+TOOLCHAIN_PACKAGE = "toolchain-xtensa"
+# gcc 10.3, the toolchain Arduino core 3.x builds with; the build
+# generator's compile flags are tuned to it.
+TOOLCHAIN_VERSION = "2.100300.220621"
+
+ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str(
+    os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "")
+)
+ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str(
+    os.environ.get("ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", "")
+)
+
+
+def get_arduino8266_tools_path() -> Path:
+    # Machine-global so all projects share one install; see
+    # espidf.framework.get_idf_tools_path for the location rationale.
+    return tools_cache_path(*ARDUINO8266_TOOLS_CACHE)
+
+
+# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the
+# encoder below cannot name 3.0.0/3.0.1 either (see its docstring)
+MIN_FRAMEWORK_VERSION = Version(3, 1, 1)
+
+
+def framework_package_version(ver: Version) -> str:
+    """Map an Arduino core version to its registry package version (3.1.2 ->
+    3.30102.0; the leading 3 is the package major).
+
+    Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor
+    at MIN_FRAMEWORK_VERSION.
+    """
+    if ver.major > 3:
+        raise EsphomeError(
+            f"Arduino core {ver} is not supported yet; "
+            "the newest known core series is 3.x"
+        )
+    if ver <= Version(2, 6, 2):
+        # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same
+        # boundary as _format_framework_arduino_version's era guard)
+        raise EsphomeError(
+            f"Arduino core {ver} uses an older package encoding than this "
+            "helper implements (newer than 2.6.2)"
+        )
+    return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+
+
+def get_framework_path(package_version: str) -> Path:
+    return get_arduino8266_tools_path() / "frameworks" / package_version
+
+
+def get_toolchain_path() -> Path:
+    return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION
+
+
+class InstalledPaths(NamedTuple):
+    """Locations of the installed framework, toolchain, and ninja binary."""
+
+    framework: Path
+    toolchain: Path
+    ninja: Path
+
+
+def check_and_install(framework_version: Version) -> InstalledPaths:
+    """Ensure framework, toolchain, and ninja are installed; return their paths."""
+    if framework_version < MIN_FRAMEWORK_VERSION:
+        # Config validation enforces this too; keep the module honest when
+        # called directly.
+        raise EsphomeError(
+            f"The native toolchain requires the Arduino core "
+            f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}"
+        )
+    # Probe the cheap local dependency before ~110 MB of downloads
+    ninja_path = find_ninja()
+    package_version = framework_package_version(framework_version)
+    framework_path = get_framework_path(package_version)
+    downloads_dir = get_arduino8266_tools_path() / "downloads"
+    toolchain_path = get_toolchain_path()
+    # One spec per package: the prefetch and the installs must agree
+    specs = (
+        (
+            FRAMEWORK_PACKAGE,
+            package_version,
+            framework_path,
+            ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
+            ("cores/esp8266", "tools/sdk", "libraries"),
+        ),
+        (
+            TOOLCHAIN_PACKAGE,
+            TOOLCHAIN_VERSION,
+            toolchain_path,
+            ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS,
+            # xtensa-lx106-elf pins the target: every gcc package has a bin/
+            ("bin", "xtensa-lx106-elf"),
+        ),
+    )
+    # Fetch both archives at once; the installs below verify and extract
+    prefetch_packages([spec[:4] for spec in specs], downloads_dir)
+    for name, version, dest, mirrors, expect in specs:
+        install_package(name, version, dest, mirrors, downloads_dir, expect=expect)
+    return InstalledPaths(
+        framework=framework_path, toolchain=toolchain_path, ninja=ninja_path
+    )
+
+
+def toolchain_tool(toolchain_path: Path, name: str) -> Path:
+    """Path to one toolchain tool (gcc, g++, ar, size, addr2line, ...).
+
+    The single owner of the ``bin/xtensa-lx106-elf-<name>`` layout and the
+    Windows suffix, so a toolchain package bump touches one spot.
+    """
+    suffix = ".exe" if os.name == "nt" else ""
+    return toolchain_path / "bin" / f"xtensa-lx106-elf-{name}{suffix}"
+
+
+def get_build_env(toolchain_path: Path, ccache: str | None) -> dict[str, str]:
+    env = os.environ.copy()
+    # Drop empty entries: a trailing separator from an absent PATH would
+    # make the shell search the current directory for tools
+    parts = [
+        str(toolchain_path / "bin"),
+        *filter(None, env.get("PATH", "").split(os.pathsep)),
+    ]
+    env["PATH"] = os.pathsep.join(parts)
+    env.update(ccache_env(ccache))
+    return env
+
+
+def ccache_env(ccache: str | None) -> dict[str, str]:
+    """Return ccache settings for the build subprocess (not os.environ).
+
+    ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None
+    when disabled. Values the user already set in the environment are
+    respected.
+    """
+    if ccache is None:
+        return {}
+    return ccache_defaults_env(get_arduino8266_tools_path() / "ccache")

esphome/components/esp8266/__init__.py

@@ -137,7 +137,16 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
         return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
     if ver <= cv.Version(2, 6, 2):
         return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-    return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+    # Same encoding the native toolchain uses for its package download, so a
+    # version bump cannot drift between the two paths.
+    from esphome.arduino8266.framework import framework_package_version
+
+    try:
+        return f"~{framework_package_version(ver)}"
+    except EsphomeError as err:
+        # Anchor the 4.x rejection to the framework version line instead of
+        # aborting with a bare traceback-level error
+        raise cv.Invalid(str(err), path=[CONF_VERSION]) from err
 
 
 # NOTE: Keep this in mind when updating the recommended version:

tests/unit_tests/test_arduino8266_framework.py

@@ -0,0 +1,170 @@
+"""Tests for esphome.arduino8266.framework (downloads and environment)."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from esphome.arduino8266 import framework
+import esphome.config_validation as cv
+from esphome.core import CORE, EsphomeError
+
+
+@pytest.fixture(autouse=True)
+def _build_path(tmp_path: Path) -> None:
+    CORE.build_path = tmp_path
+
+
+def test_framework_package_version() -> None:
+    assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0"
+    assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0"
+    # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path)
+    assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0"
+    # A future major bump needs its own encoding, not a doomed registry lookup
+    with pytest.raises(EsphomeError, match="not supported yet"):
+        framework.framework_package_version(cv.Version(4, 0, 0))
+    # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release
+    # keeps this encoding
+    with pytest.raises(EsphomeError, match="older package encoding"):
+        framework.framework_package_version(cv.Version(2, 6, 2))
+    assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0"
+    assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0"
+
+
+def test_format_framework_arduino_version_pins_all_series() -> None:
+    """The esp8266 component's PIO source formatter across every encoding
+    era, including the 4.x rejection it now shares with the installer."""
+    from esphome.components.esp8266 import _format_framework_arduino_version as fmt
+
+    assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0"
+    assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0"
+    assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0"
+    assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0"
+    # Anchored to the framework version line, not a bare EsphomeError
+    with pytest.raises(cv.Invalid, match="not supported yet") as excinfo:
+        fmt(cv.Version(4, 0, 0))
+    assert excinfo.value.path == ["version"]
+
+
+def test_tools_path_default_and_prefix(tmp_path: Path) -> None:
+    with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}):
+        assert framework.get_arduino8266_tools_path() == tmp_path.resolve()
+    # A blank prefix must be treated as unset, not as the CWD
+    with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": "  "}):
+        path = framework.get_arduino8266_tools_path()
+    assert path.name == "arduino8266"
+    assert path != Path.cwd()
+
+
+def test_check_and_install_returns_paths(tmp_path: Path) -> None:
+    with (
+        patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
+        patch.object(framework, "install_package") as mock_install,
+        patch.object(framework, "prefetch_packages") as mock_prefetch,
+        patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"),
+    ):
+        paths = framework.check_and_install(cv.Version(3, 1, 2))
+    assert paths.framework == tmp_path / "frameworks" / "3.30102.0"
+    assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION
+    assert paths.ninja == tmp_path / "ninja"
+    assert mock_install.call_count == 2
+    # Full argument pinning: a copy-paste swap between the two near-identical
+    # calls (mirrors, destination) must not stay green
+    fw_call, tc_call = mock_install.call_args_list
+    assert fw_call.args == (
+        framework.FRAMEWORK_PACKAGE,
+        "3.30102.0",
+        tmp_path / "frameworks" / "3.30102.0",
+        framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
+        tmp_path / "downloads",
+    )
+    assert fw_call.kwargs["expect"] == ("cores/esp8266", "tools/sdk", "libraries")
+    assert tc_call.args == (
+        framework.TOOLCHAIN_PACKAGE,
+        framework.TOOLCHAIN_VERSION,
+        tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION,
+        framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS,
+        tmp_path / "downloads",
+    )
+    assert tc_call.kwargs["expect"] == ("bin", "xtensa-lx106-elf")
+    # The prefetch sees the same package specs as the installs
+    assert mock_prefetch.call_args.args == (
+        [
+            (
+                framework.FRAMEWORK_PACKAGE,
+                "3.30102.0",
+                tmp_path / "frameworks" / "3.30102.0",
+                framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
+            ),
+            (
+                framework.TOOLCHAIN_PACKAGE,
+                framework.TOOLCHAIN_VERSION,
+                tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION,
+                framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS,
+            ),
+        ],
+        tmp_path / "downloads",
+    )
+
+
+def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None:
+    with patch.object(framework, "ccache_env", return_value={"CCACHE_DIR": "x"}):
+        env = framework.get_build_env(tmp_path, None)
+    assert env["PATH"].startswith(str(tmp_path / "bin") + os.pathsep)
+    assert env["CCACHE_DIR"] == "x"
+
+
+def test_ccache_env(tmp_path: Path) -> None:
+    assert framework.ccache_env(None) == {}
+    with patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True):
+        env = framework.ccache_env("/usr/bin/ccache")
+    # User-set values are respected; the rest get defaults
+    assert "CCACHE_NOHASHDIR" not in env
+    assert env["CCACHE_DEPEND"] == "1"
+    assert env["CCACHE_BASEDIR"] == str(Path(CORE.build_path).resolve())
+    assert env["CCACHE_DIR"].endswith("ccache")
+
+
+def test_check_and_install_rejects_old_core(tmp_path: Path) -> None:
+    """Calling the installer below the floor fails before any download."""
+    with pytest.raises(EsphomeError, match=">= 3.1.1"):
+        framework.check_and_install(cv.Version(3, 0, 2))
+
+
+def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None:
+    """An absent PATH must not leave a trailing separator (an empty entry
+    means the current directory to the shell)."""
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        patch.object(framework, "ccache_env", return_value={}),
+    ):
+        env = framework.get_build_env(tmp_path, None)
+    assert env["PATH"] == str(tmp_path / "bin")
+    with (
+        patch.dict(
+            os.environ, {"PATH": f"/usr/bin{os.pathsep}{os.pathsep}/bin"}, clear=True
+        ),
+        patch.object(framework, "ccache_env", return_value={}),
+    ):
+        env = framework.get_build_env(tmp_path, None)
+    assert env["PATH"].split(os.pathsep) == [str(tmp_path / "bin"), "/usr/bin", "/bin"]
+
+
+def test_ccache_env_accepts_a_preresolved_path() -> None:
+    """The caller resolves ccache once and threads it through; None means
+    resolved-and-disabled."""
+    with patch.dict(os.environ, {}, clear=True):
+        assert framework.ccache_env(None) == {}
+        env = framework.ccache_env("/usr/bin/ccache")
+    assert env["CCACHE_DIR"].endswith("ccache")
+
+
+def test_toolchain_tool_layout(tmp_path: Path) -> None:
+    """One owner for the bin/xtensa-lx106-elf-<name> layout."""
+    tool = framework.toolchain_tool(tmp_path, "addr2line")
+    assert tool.parent == tmp_path / "bin"
+    assert tool.name.startswith("xtensa-lx106-elf-addr2line")
+    assert (tool.suffix == ".exe") is (os.name == "nt")

tests/unit_tests/test_writer.py

@@ -1059,6 +1059,28 @@ def test_clean_all_removes_global_sdk_nrf_install(
     assert str(sdk_nrf_install.resolve()) in caplog.text
 
 
+@patch("esphome.writer.CORE")
+def test_clean_all_removes_global_arduino8266_install(
+    mock_core: MagicMock,
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """clean_all removes the machine-global native arduino8266 install dir."""
+    arduino8266_install = tmp_path / "arduino8266_install"
+    (arduino8266_install / "frameworks").mkdir(parents=True)
+    monkeypatch.setenv("ESPHOME_ARDUINO8266_PREFIX", str(arduino8266_install))
+
+    config_dir = tmp_path / "config"
+    config_dir.mkdir()
+
+    with caplog.at_level("INFO"):
+        clean_all([str(config_dir)])
+
+    assert not arduino8266_install.exists()
+    assert str(arduino8266_install.resolve()) in caplog.text
+
+
 @patch("esphome.writer.CORE")
 def test_clean_all_removes_default_cache_root(
     mock_core: MagicMock,