From 9e4c52989c6015be0a95ff0761c4c9a935b44ef1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 20:24:47 -0500 Subject: [PATCH] [esp8266] Add the native framework and toolchain installer (#18557) --- esphome/arduino8266/__init__.py | 9 + esphome/arduino8266/framework.py | 164 +++++++++++++++++ esphome/components/esp8266/__init__.py | 11 +- .../unit_tests/test_arduino8266_framework.py | 170 ++++++++++++++++++ tests/unit_tests/test_writer.py | 22 +++ 5 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 esphome/arduino8266/__init__.py create mode 100644 esphome/arduino8266/framework.py create mode 100644 tests/unit_tests/test_arduino8266_framework.py diff --git a/esphome/arduino8266/__init__.py b/esphome/arduino8266/__init__.py new file mode 100644 index 0000000000..8f403a8553 --- /dev/null +++ b/esphome/arduino8266/__init__.py @@ -0,0 +1,9 @@ +"""Native (PlatformIO-free) build support for the ESP8266 Arduino core. + +This package downloads the Arduino ESP8266 core and the xtensa-lx106 +toolchain, generates a ninja build for them plus the ESPHome sources, and +drives the build directly — the ESP8266 equivalent of ``esphome.espidf``. + +Deliberately importable without the esp8266 component to avoid circular +imports; the component wires these modules in via lazy imports. +""" diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py new file mode 100644 index 0000000000..1edbe4b36f --- /dev/null +++ b/esphome/arduino8266/framework.py @@ -0,0 +1,164 @@ +"""Download and install the Arduino ESP8266 core, toolchain, and ninja. + +Artifacts land in a machine-global cache (shared across projects, like the +ESP-IDF install in ``esphome.espidf.framework``): + + /arduino8266/frameworks// framework-arduinoespressif8266 + /arduino8266/toolchains// toolchain-xtensa (gcc 10.3) + +Packages come from the PlatformIO registry (identical bits to the PlatformIO +backend); ``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes +from PATH or the ninja PyPI wheel. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import NamedTuple + +from esphome.build_helpers.ccache import ccache_defaults_env +from esphome.build_helpers.ninja import find_ninja +from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path +from esphome.core import EsphomeError, Version +from esphome.framework_helpers import str_to_lst_of_str +from esphome.platformio.registry import install_package, prefetch_packages + +FRAMEWORK_PACKAGE = "framework-arduinoespressif8266" +TOOLCHAIN_PACKAGE = "toolchain-xtensa" +# gcc 10.3, the toolchain Arduino core 3.x builds with; the build +# generator's compile flags are tuned to it. +TOOLCHAIN_VERSION = "2.100300.220621" + +ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "") +) +ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", "") +) + + +def get_arduino8266_tools_path() -> Path: + # Machine-global so all projects share one install; see + # espidf.framework.get_idf_tools_path for the location rationale. + return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) + + +# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the +# encoder below cannot name 3.0.0/3.0.1 either (see its docstring) +MIN_FRAMEWORK_VERSION = Version(3, 1, 1) + + +def framework_package_version(ver: Version) -> str: + """Map an Arduino core version to its registry package version (3.1.2 -> + 3.30102.0; the leading 3 is the package major). + + Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor + at MIN_FRAMEWORK_VERSION. + """ + if ver.major > 3: + raise EsphomeError( + f"Arduino core {ver} is not supported yet; " + "the newest known core series is 3.x" + ) + if ver <= Version(2, 6, 2): + # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same + # boundary as _format_framework_arduino_version's era guard) + raise EsphomeError( + f"Arduino core {ver} uses an older package encoding than this " + "helper implements (newer than 2.6.2)" + ) + return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + + +def get_framework_path(package_version: str) -> Path: + return get_arduino8266_tools_path() / "frameworks" / package_version + + +def get_toolchain_path() -> Path: + return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION + + +class InstalledPaths(NamedTuple): + """Locations of the installed framework, toolchain, and ninja binary.""" + + framework: Path + toolchain: Path + ninja: Path + + +def check_and_install(framework_version: Version) -> InstalledPaths: + """Ensure framework, toolchain, and ninja are installed; return their paths.""" + if framework_version < MIN_FRAMEWORK_VERSION: + # Config validation enforces this too; keep the module honest when + # called directly. + raise EsphomeError( + f"The native toolchain requires the Arduino core " + f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}" + ) + # Probe the cheap local dependency before ~110 MB of downloads + ninja_path = find_ninja() + package_version = framework_package_version(framework_version) + framework_path = get_framework_path(package_version) + downloads_dir = get_arduino8266_tools_path() / "downloads" + toolchain_path = get_toolchain_path() + # One spec per package: the prefetch and the installs must agree + specs = ( + ( + FRAMEWORK_PACKAGE, + package_version, + framework_path, + ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + ("cores/esp8266", "tools/sdk", "libraries"), + ), + ( + TOOLCHAIN_PACKAGE, + TOOLCHAIN_VERSION, + toolchain_path, + ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + # xtensa-lx106-elf pins the target: every gcc package has a bin/ + ("bin", "xtensa-lx106-elf"), + ), + ) + # Fetch both archives at once; the installs below verify and extract + prefetch_packages([spec[:4] for spec in specs], downloads_dir) + for name, version, dest, mirrors, expect in specs: + install_package(name, version, dest, mirrors, downloads_dir, expect=expect) + return InstalledPaths( + framework=framework_path, toolchain=toolchain_path, ninja=ninja_path + ) + + +def toolchain_tool(toolchain_path: Path, name: str) -> Path: + """Path to one toolchain tool (gcc, g++, ar, size, addr2line, ...). + + The single owner of the ``bin/xtensa-lx106-elf-`` layout and the + Windows suffix, so a toolchain package bump touches one spot. + """ + suffix = ".exe" if os.name == "nt" else "" + return toolchain_path / "bin" / f"xtensa-lx106-elf-{name}{suffix}" + + +def get_build_env(toolchain_path: Path, ccache: str | None) -> dict[str, str]: + env = os.environ.copy() + # Drop empty entries: a trailing separator from an absent PATH would + # make the shell search the current directory for tools + parts = [ + str(toolchain_path / "bin"), + *filter(None, env.get("PATH", "").split(os.pathsep)), + ] + env["PATH"] = os.pathsep.join(parts) + env.update(ccache_env(ccache)) + return env + + +def ccache_env(ccache: str | None) -> dict[str, str]: + """Return ccache settings for the build subprocess (not os.environ). + + ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None + when disabled. Values the user already set in the environment are + respected. + """ + if ccache is None: + return {} + return ccache_defaults_env(get_arduino8266_tools_path() / "ccache") diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 6f29cd7774..63665e7681 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -137,7 +137,16 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" if ver <= cv.Version(2, 6, 2): return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + # Same encoding the native toolchain uses for its package download, so a + # version bump cannot drift between the two paths. + from esphome.arduino8266.framework import framework_package_version + + try: + return f"~{framework_package_version(ver)}" + except EsphomeError as err: + # Anchor the 4.x rejection to the framework version line instead of + # aborting with a bare traceback-level error + raise cv.Invalid(str(err), path=[CONF_VERSION]) from err # NOTE: Keep this in mind when updating the recommended version: diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py new file mode 100644 index 0000000000..bd0a620e10 --- /dev/null +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -0,0 +1,170 @@ +"""Tests for esphome.arduino8266.framework (downloads and environment).""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.arduino8266 import framework +import esphome.config_validation as cv +from esphome.core import CORE, EsphomeError + + +@pytest.fixture(autouse=True) +def _build_path(tmp_path: Path) -> None: + CORE.build_path = tmp_path + + +def test_framework_package_version() -> None: + assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" + assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" + # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path) + assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0" + # A future major bump needs its own encoding, not a doomed registry lookup + with pytest.raises(EsphomeError, match="not supported yet"): + framework.framework_package_version(cv.Version(4, 0, 0)) + # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release + # keeps this encoding + with pytest.raises(EsphomeError, match="older package encoding"): + framework.framework_package_version(cv.Version(2, 6, 2)) + assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0" + assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0" + + +def test_format_framework_arduino_version_pins_all_series() -> None: + """The esp8266 component's PIO source formatter across every encoding + era, including the 4.x rejection it now shares with the installer.""" + from esphome.components.esp8266 import _format_framework_arduino_version as fmt + + assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0" + assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0" + assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0" + assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" + # Anchored to the framework version line, not a bare EsphomeError + with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: + fmt(cv.Version(4, 0, 0)) + assert excinfo.value.path == ["version"] + + +def test_tools_path_default_and_prefix(tmp_path: Path) -> None: + with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}): + assert framework.get_arduino8266_tools_path() == tmp_path.resolve() + # A blank prefix must be treated as unset, not as the CWD + with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": " "}): + path = framework.get_arduino8266_tools_path() + assert path.name == "arduino8266" + assert path != Path.cwd() + + +def test_check_and_install_returns_paths(tmp_path: Path) -> None: + with ( + patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "install_package") as mock_install, + patch.object(framework, "prefetch_packages") as mock_prefetch, + patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), + ): + paths = framework.check_and_install(cv.Version(3, 1, 2)) + assert paths.framework == tmp_path / "frameworks" / "3.30102.0" + assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION + assert paths.ninja == tmp_path / "ninja" + assert mock_install.call_count == 2 + # Full argument pinning: a copy-paste swap between the two near-identical + # calls (mirrors, destination) must not stay green + fw_call, tc_call = mock_install.call_args_list + assert fw_call.args == ( + framework.FRAMEWORK_PACKAGE, + "3.30102.0", + tmp_path / "frameworks" / "3.30102.0", + framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + tmp_path / "downloads", + ) + assert fw_call.kwargs["expect"] == ("cores/esp8266", "tools/sdk", "libraries") + assert tc_call.args == ( + framework.TOOLCHAIN_PACKAGE, + framework.TOOLCHAIN_VERSION, + tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION, + framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + tmp_path / "downloads", + ) + assert tc_call.kwargs["expect"] == ("bin", "xtensa-lx106-elf") + # The prefetch sees the same package specs as the installs + assert mock_prefetch.call_args.args == ( + [ + ( + framework.FRAMEWORK_PACKAGE, + "3.30102.0", + tmp_path / "frameworks" / "3.30102.0", + framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + ), + ( + framework.TOOLCHAIN_PACKAGE, + framework.TOOLCHAIN_VERSION, + tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION, + framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + ), + ], + tmp_path / "downloads", + ) + + +def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: + with patch.object(framework, "ccache_env", return_value={"CCACHE_DIR": "x"}): + env = framework.get_build_env(tmp_path, None) + assert env["PATH"].startswith(str(tmp_path / "bin") + os.pathsep) + assert env["CCACHE_DIR"] == "x" + + +def test_ccache_env(tmp_path: Path) -> None: + assert framework.ccache_env(None) == {} + with patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True): + env = framework.ccache_env("/usr/bin/ccache") + # User-set values are respected; the rest get defaults + assert "CCACHE_NOHASHDIR" not in env + assert env["CCACHE_DEPEND"] == "1" + assert env["CCACHE_BASEDIR"] == str(Path(CORE.build_path).resolve()) + assert env["CCACHE_DIR"].endswith("ccache") + + +def test_check_and_install_rejects_old_core(tmp_path: Path) -> None: + """Calling the installer below the floor fails before any download.""" + with pytest.raises(EsphomeError, match=">= 3.1.1"): + framework.check_and_install(cv.Version(3, 0, 2)) + + +def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: + """An absent PATH must not leave a trailing separator (an empty entry + means the current directory to the shell).""" + with ( + patch.dict(os.environ, {}, clear=True), + patch.object(framework, "ccache_env", return_value={}), + ): + env = framework.get_build_env(tmp_path, None) + assert env["PATH"] == str(tmp_path / "bin") + with ( + patch.dict( + os.environ, {"PATH": f"/usr/bin{os.pathsep}{os.pathsep}/bin"}, clear=True + ), + patch.object(framework, "ccache_env", return_value={}), + ): + env = framework.get_build_env(tmp_path, None) + assert env["PATH"].split(os.pathsep) == [str(tmp_path / "bin"), "/usr/bin", "/bin"] + + +def test_ccache_env_accepts_a_preresolved_path() -> None: + """The caller resolves ccache once and threads it through; None means + resolved-and-disabled.""" + with patch.dict(os.environ, {}, clear=True): + assert framework.ccache_env(None) == {} + env = framework.ccache_env("/usr/bin/ccache") + assert env["CCACHE_DIR"].endswith("ccache") + + +def test_toolchain_tool_layout(tmp_path: Path) -> None: + """One owner for the bin/xtensa-lx106-elf- layout.""" + tool = framework.toolchain_tool(tmp_path, "addr2line") + assert tool.parent == tmp_path / "bin" + assert tool.name.startswith("xtensa-lx106-elf-addr2line") + assert (tool.suffix == ".exe") is (os.name == "nt") diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 9c20ee10d2..47feae3e3c 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -1059,6 +1059,28 @@ def test_clean_all_removes_global_sdk_nrf_install( assert str(sdk_nrf_install.resolve()) in caplog.text +@patch("esphome.writer.CORE") +def test_clean_all_removes_global_arduino8266_install( + mock_core: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """clean_all removes the machine-global native arduino8266 install dir.""" + arduino8266_install = tmp_path / "arduino8266_install" + (arduino8266_install / "frameworks").mkdir(parents=True) + monkeypatch.setenv("ESPHOME_ARDUINO8266_PREFIX", str(arduino8266_install)) + + config_dir = tmp_path / "config" + config_dir.mkdir() + + with caplog.at_level("INFO"): + clean_all([str(config_dir)]) + + assert not arduino8266_install.exists() + assert str(arduino8266_install.resolve()) in caplog.text + + @patch("esphome.writer.CORE") def test_clean_all_removes_default_cache_root( mock_core: MagicMock,