github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-10-01 08:50:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

[esp8266] Use our own Arduino core builds for the native toolchain (#19876)

Browse Source
This commit is contained in:
J. Nick Koston
2026-09-29 18:38:51 +02:00
committed by GitHub
parent 5be41dec15
commit a7e0afeb46
6 changed files with 152 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/cache-arduino8266/action.yml
+1 -1
View File
@@ -22,7 +22,7 @@ runs:
# cached path cannot diverge.
echo "ESPHOME_ARDUINO8266_PREFIX=$HOME/.esphome-arduino8266" >> "$GITHUB_ENV"
. venv/bin/activate
key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import TOOLCHAIN_VERSION as t; print(f"{f}-{t}")')
key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import FRAMEWORK_RELEASES, TOOLCHAIN_VERSION as t; print(f"{FRAMEWORK_RELEASES[f].tag}-{t}")')
[ -n "$key" ] || exit 1
echo "key=$key" >> "$GITHUB_OUTPUT"
- name: Cache the native toolchain (write on dev)
esphome/arduino8266/framework.py
+42 -37
View File
@@ -3,13 +3,12 @@
Artifacts land in a machine-global cache (shared across projects, like the
ESP-IDF install in ``esphome.espidf.framework``):
<cache>/arduino8266/frameworks/<version>/ framework-arduinoespressif8266
<cache>/arduino8266/toolchains/<version>/ xtensa-lx106-elf gcc 10.3
<cache>/arduino8266/frameworks/<tag>/ the Arduino core
<cache>/arduino8266/toolchains/<tag>/ xtensa-lx106-elf gcc 10.3
The framework comes from the PlatformIO registry, the toolchain from
https://github.com/esphome-libs/xtensa-lx106-elf-toolchain/releases/;
``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes from PATH
or the ninja PyPI wheel.
Both come from esphome-libs releases pinned below;
``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs, with ``{VERSION}``
standing for the release tag. ninja comes from PATH or the ninja PyPI wheel.
"""
from __future__ import annotations
@@ -31,7 +30,30 @@ from esphome.platformio.registry import (
prefetch_packages,
)
FRAMEWORK_PACKAGE = "framework-arduinoespressif8266"
FRAMEWORK_PACKAGE = "arduino-esp8266"
_FRAMEWORK_RELEASES = "https://github.com/esphome-libs/arduino-esp8266/releases/"
class FrameworkRelease(NamedTuple):
tag: str
sha256: str
size: int
def download(self) -> Download:
archive = f"{FRAMEWORK_PACKAGE}-{self.tag}.tar.gz"
url = f"{_FRAMEWORK_RELEASES}download/{self.tag}/{archive}"
return Download(url, self.sha256, self.size)
# Arduino core version -> its build in esphome-libs/arduino-esp8266
FRAMEWORK_RELEASES: dict[Version, FrameworkRelease] = {
Version(3, 1, 2): FrameworkRelease(
"3.1.2-esphome.1",
"e80751e3123676b967143e39c61f2d8693946db4c7806f2a83dcaaf797ecd582",
37189311,
),
}
TOOLCHAIN_PACKAGE = "toolchain-xtensa-lx106-elf"
# gcc 10.3, the toolchain Arduino core 3.x builds with; the build
# generator's compile flags are tuned to it.
@@ -77,30 +99,18 @@ def get_arduino8266_tools_path() -> Path:
return tools_cache_path(*ARDUINO8266_TOOLS_CACHE)
# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0
MIN_FRAMEWORK_VERSION = Version(3, 1, 1)
def framework_package_version(ver: Version) -> str:
"""Map an Arduino core version to its registry package version (3.1.2 ->
3.30102.0; the leading 3 is the package major).
Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION.
"""
if ver.major > 3:
def framework_release(version: Version) -> FrameworkRelease:
if (release := FRAMEWORK_RELEASES.get(version)) is None:
raise EsphomeError(
f"Arduino core {ver} is not supported yet; "
"the newest known core series is 3.x"
f"'toolchain: arduino' has no build of Arduino core {version}; "
f"available: {', '.join(str(v) for v in FRAMEWORK_RELEASES)}. "
"Use one of those or 'toolchain: platformio'"
)
if ver.major < 3:
raise EsphomeError(
f"Arduino core {ver} is not supported; ESPHome requires core 3.x"
)
return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
return release
def get_framework_path(package_version: str) -> Path:
return get_arduino8266_tools_path() / "frameworks" / package_version
def get_framework_path(tag: str) -> Path:
return get_arduino8266_tools_path() / "frameworks" / tag
def get_toolchain_path() -> Path:
@@ -133,24 +143,17 @@ class InstalledPaths(NamedTuple):
def check_and_install(framework_version: Version) -> InstalledPaths:
"""Ensure framework, toolchain, and ninja are installed; return their paths."""
if framework_version < MIN_FRAMEWORK_VERSION:
# Config validation enforces this too; keep the module honest when
# called directly.
raise EsphomeError(
f"The native toolchain requires the Arduino core "
f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}"
)
release = framework_release(framework_version)
# Probe the cheap local dependency before ~110 MB of downloads
ninja_path = find_ninja()
package_version = framework_package_version(framework_version)
framework_path = get_framework_path(package_version)
framework_path = get_framework_path(release.tag)
downloads_dir = get_arduino8266_tools_path() / "downloads"
toolchain_path = get_toolchain_path()
# One spec per package: the prefetch and the installs must agree
specs = (
(
FRAMEWORK_PACKAGE,
package_version,
release.tag,
framework_path,
ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
("cores/esp8266", "tools/sdk", "libraries"),
@@ -167,6 +170,8 @@ def check_and_install(framework_version: Version) -> InstalledPaths:
# Resolved only when a download is needed, so an installed toolchain
# keeps working on a host without a build; a mirror override wins
resolvers: dict[str, Resolver] = {}
if not ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS:
resolvers[FRAMEWORK_PACKAGE] = release.download
if not ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS:
resolvers[TOOLCHAIN_PACKAGE] = toolchain_download
# Fetch both archives at once; the installs below verify and extract
esphome/components/esp8266/__init__.py
+25 -12
View File
@@ -118,15 +118,14 @@ def _validate_native_toolchain(config: ConfigType) -> ConfigType:
"""Constraints of the native (non-PlatformIO) Arduino toolchain."""
if not CORE.using_toolchain_arduino:
return config
from esphome.arduino8266.framework import MIN_FRAMEWORK_VERSION
from esphome.arduino8266.framework import framework_release
conf = config[CONF_FRAMEWORK]
version = cv.Version.parse(conf[CONF_VERSION])
if version < MIN_FRAMEWORK_VERSION:
raise cv.Invalid(
"'toolchain: arduino' requires framework version "
f"{MIN_FRAMEWORK_VERSION} or newer"
)
try:
framework_release(version)
except EsphomeError as err:
raise cv.Invalid(str(err), path=[CONF_FRAMEWORK, CONF_VERSION]) from err
# platform_version is a PlatformIO concept; drop it, warning when a
# custom pin is discarded
if (
@@ -137,6 +136,8 @@ def _validate_native_toolchain(config: ConfigType) -> ConfigType:
"'platform_version' is ignored by 'toolchain: arduino'; the native "
"toolchain downloads the framework and compiler directly"
)
# The native path fetches its own build; source may only be the
# PlatformIO default the schema filled in
if conf[CONF_SOURCE] != _format_framework_arduino_version(version):
raise cv.Invalid(
"'toolchain: arduino' does not support a custom framework source; "
@@ -174,16 +175,27 @@ def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]:
]
def _framework_package_version(ver: cv.Version) -> str:
"""Map an Arduino core version to its PlatformIO registry package version
(3.1.2 -> 3.30102.0; the leading 3 is the package major)."""
if ver.major > 3:
raise EsphomeError(
f"Arduino core {ver} is not supported yet; "
"the newest known core series is 3.x"
)
if ver.major < 3:
raise EsphomeError(
f"Arduino core {ver} is not supported; ESPHome requires core 3.x"
)
return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
def _format_framework_arduino_version(ver: cv.Version) -> str:
# format the given arduino (https://github.com/esp8266/Arduino/releases) version to
# a PIO platformio/framework-arduinoespressif8266 value
# List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266
# Same encoding the native toolchain uses for its package download, so a
# custom-source check against this value cannot drift from what it fetches.
from esphome.arduino8266.framework import framework_package_version
try:
return f"~{framework_package_version(ver)}"
return f"~{_framework_package_version(ver)}"
except EsphomeError as err:
# Anchor the 4.x rejection to the framework version line instead of
# aborting with a bare traceback-level error
@@ -194,7 +206,8 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
# * New framework historically have had some regressions, especially for WiFi.
# The new version needs to be thoroughly validated before changing the
# recommended version as otherwise a bunch of devices could be bricked
# * For all constants below, update platformio.ini (in this repo)
# * For all constants below, update platformio.ini (in this repo) and
# FRAMEWORK_RELEASES in esphome/arduino8266/framework.py
# The default/recommended arduino framework version
# - https://github.com/esp8266/Arduino/releases
tests/unit_tests/components/esp8266/test_framework_version.py
+30 -1
View File
@@ -2,9 +2,14 @@
import pytest
from esphome.components.esp8266 import _arduino_check_versions
from esphome.components.esp8266 import (
_arduino_check_versions,
_format_framework_arduino_version,
_framework_package_version,
)
import esphome.config_validation as cv
from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION
from esphome.core import EsphomeError
def test_versions_before_3_are_rejected() -> None:
@@ -21,3 +26,27 @@ def test_supported_versions_pass() -> None:
value = _arduino_check_versions({CONF_VERSION: "recommended"})
assert value[CONF_VERSION] == "3.1.2"
assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION]
def test_framework_package_version() -> None:
assert _framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0"
assert _framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0"
# A future major bump needs its own encoding, not a doomed registry lookup
with pytest.raises(EsphomeError, match="not supported yet"):
_framework_package_version(cv.Version(4, 0, 0))
# Cores before 3.x cannot build ESPHome (C++20) and are rejected
with pytest.raises(EsphomeError, match="requires core 3"):
_framework_package_version(cv.Version(2, 7, 4))
def test_format_framework_arduino_version_pins_all_series() -> None:
"""The PlatformIO source formatter across every encoding era."""
assert _format_framework_arduino_version(cv.Version(3, 1, 2)) == "~3.30102.0"
# Pre-3 cores are rejected with the version line anchored
with pytest.raises(cv.Invalid, match="requires core 3"):
_format_framework_arduino_version(cv.Version(2, 7, 4))
# Anchored to the framework version line, not a bare EsphomeError
with pytest.raises(cv.Invalid, match="not supported yet") as excinfo:
_format_framework_arduino_version(cv.Version(4, 0, 0))
assert excinfo.value.path == ["version"]
tests/unit_tests/components/esp8266/test_toolchain_validation.py
+11 -8
View File
@@ -67,21 +67,24 @@ def test_valid_config_passes() -> None:
def test_platformio_toolchain_skips_checks() -> None:
# 3.0.2 is pio-legal (>= the global 3.0.0 floor) but below the native
# toolchain's own 3.1.1 floor; the bogus board only the native path checks
# 3.0.2 is pio-legal (>= the global 3.0.0 floor) but has no native build;
# the bogus board only the native path checks
CORE.toolchain = Toolchain.PLATFORMIO
config = _config(board="not_a_board", version="3.0.2")
assert _validate_native_toolchain(config) is config
def test_version_below_floor_rejected() -> None:
# 3.1.0 has no registry package, so the native floor is 3.1.1
with pytest.raises(cv.Invalid, match="3.1.1 or newer"):
_validate_native_toolchain(_config(version="3.1.0"))
def test_version_without_build_rejected() -> None:
"""Only the core versions built in esphome-libs/arduino-esp8266 work."""
with pytest.raises(
cv.Invalid, match=r"3\.1\.1.*available: 3\.1\.2.*platformio"
) as excinfo:
_validate_native_toolchain(_config(version="3.1.1"))
assert excinfo.value.path == [CONF_FRAMEWORK, CONF_VERSION]
def test_version_at_floor_accepted() -> None:
_validate_native_toolchain(_config(version="3.1.1"))
def test_built_version_accepted() -> None:
_validate_native_toolchain(_config(version="3.1.2"))
def test_custom_platform_version_warns_and_is_dropped(
tests/unit_tests/test_arduino8266_framework.py
+43 -40
View File
@@ -9,6 +9,7 @@ from unittest.mock import patch
import pytest
from esphome.arduino8266 import framework
from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION
import esphome.config_validation as cv
from esphome.core import CORE, EsphomeError
@@ -18,32 +19,6 @@ def _build_path(tmp_path: Path) -> None:
CORE.build_path = tmp_path
def test_framework_package_version() -> None:
assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0"
assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0"
# A future major bump needs its own encoding, not a doomed registry lookup
with pytest.raises(EsphomeError, match="not supported yet"):
framework.framework_package_version(cv.Version(4, 0, 0))
# Cores before 3.x cannot build ESPHome (C++20) and are rejected
with pytest.raises(EsphomeError, match="requires core 3"):
framework.framework_package_version(cv.Version(2, 7, 4))
def test_format_framework_arduino_version_pins_all_series() -> None:
"""The esp8266 component's PIO source formatter across every encoding
era, including the 4.x rejection it now shares with the installer."""
from esphome.components.esp8266 import _format_framework_arduino_version as fmt
assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0"
# Pre-3 cores are rejected with the version line anchored
with pytest.raises(cv.Invalid, match="requires core 3"):
fmt(cv.Version(2, 7, 4))
# Anchored to the framework version line, not a bare EsphomeError
with pytest.raises(cv.Invalid, match="not supported yet") as excinfo:
fmt(cv.Version(4, 0, 0))
assert excinfo.value.path == ["version"]
def test_tools_path_default_and_prefix(tmp_path: Path) -> None:
with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}):
assert framework.get_arduino8266_tools_path() == tmp_path.resolve()
@@ -54,6 +29,27 @@ def test_tools_path_default_and_prefix(tmp_path: Path) -> None:
assert path != Path.cwd()
def _recommended() -> framework.FrameworkRelease:
return framework.FRAMEWORK_RELEASES[RECOMMENDED_ARDUINO_FRAMEWORK_VERSION]
def test_framework_releases_are_pinned() -> None:
"""A release must not be pinned without its checksum, and the recommended
core must have one."""
assert RECOMMENDED_ARDUINO_FRAMEWORK_VERSION in framework.FRAMEWORK_RELEASES
for version, release in framework.FRAMEWORK_RELEASES.items():
assert release.tag.startswith(f"{version}-esphome.")
assert len(release.sha256) == 64
assert release.size > 0
def test_framework_download() -> None:
download = framework.FrameworkRelease("1.2.3-esphome.4", "a" * 64, 5).download()
releases = "https://github.com/esphome-libs/arduino-esp8266/releases/"
archive = "arduino-esp8266-1.2.3-esphome.4.tar.gz"
assert download == (f"{releases}download/1.2.3-esphome.4/{archive}", "a" * 64, 5)
def test_toolchain_builds_are_pinned() -> None:
"""A release must not be pinned without its checksums."""
for sha256, size in framework.TOOLCHAIN_BUILDS.values():
@@ -86,14 +82,15 @@ def test_toolchain_download_unsupported_system() -> None:
def _fake_framework(tmp_path: Path) -> None:
"""The layout install_package expects of an installed framework."""
for sub in ("cores/esp8266", "tools/sdk", "libraries"):
(tmp_path / "frameworks" / "3.30102.0" / sub).mkdir(parents=True)
(tmp_path / "frameworks" / "3.30102.0" / ".esphome_extracted").touch()
(tmp_path / "frameworks" / _recommended().tag / sub).mkdir(parents=True)
(tmp_path / "frameworks" / _recommended().tag / ".esphome_extracted").touch()
def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None:
"""With a mirror override an unsupported host can bring its own toolchain."""
with (
patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
patch.object(framework, "ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", ["http://f"]),
patch.object(framework, "ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", ["http://m"]),
patch.object(framework, "install_package") as mock_install,
patch.object(framework, "prefetch_packages") as mock_prefetch,
@@ -101,7 +98,10 @@ def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None
):
framework.check_and_install(cv.Version(3, 1, 2))
assert mock_prefetch.call_args.args[2] == {}
assert mock_install.call_args_list[1].kwargs["resolve"] is None
assert [call.kwargs["resolve"] for call in mock_install.call_args_list] == [
None,
None,
]
def test_check_and_install_installed_toolchain_on_unsupported_host(
@@ -147,7 +147,7 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None:
patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"),
):
paths = framework.check_and_install(cv.Version(3, 1, 2))
assert paths.framework == tmp_path / "frameworks" / "3.30102.0"
assert paths.framework == tmp_path / "frameworks" / _recommended().tag
assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION
assert paths.ninja == tmp_path / "ninja"
assert mock_install.call_count == 2
@@ -156,14 +156,14 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None:
fw_call, tc_call = mock_install.call_args_list
assert fw_call.args == (
framework.FRAMEWORK_PACKAGE,
"3.30102.0",
tmp_path / "frameworks" / "3.30102.0",
_recommended().tag,
tmp_path / "frameworks" / _recommended().tag,
framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
tmp_path / "downloads",
)
assert fw_call.kwargs == {
"expect": ("cores/esp8266", "tools/sdk", "libraries"),
"resolve": None,
"resolve": _recommended().download,
}
assert tc_call.args == (
framework.TOOLCHAIN_PACKAGE,
@@ -181,8 +181,8 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None:
[
(
framework.FRAMEWORK_PACKAGE,
"3.30102.0",
tmp_path / "frameworks" / "3.30102.0",
_recommended().tag,
tmp_path / "frameworks" / _recommended().tag,
framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
),
(
@@ -193,7 +193,10 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None:
),
],
tmp_path / "downloads",
{framework.TOOLCHAIN_PACKAGE: framework.toolchain_download},
{
framework.FRAMEWORK_PACKAGE: _recommended().download,
framework.TOOLCHAIN_PACKAGE: framework.toolchain_download,
},
)
@@ -204,10 +207,10 @@ def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None:
assert env["CCACHE_DIR"] == "x"
def test_check_and_install_rejects_old_core(tmp_path: Path) -> None:
"""Calling the installer below the floor fails before any download."""
with pytest.raises(EsphomeError, match=">= 3.1.1"):
framework.check_and_install(cv.Version(3, 0, 2))
def test_check_and_install_rejects_unbuilt_core(tmp_path: Path) -> None:
"""A core version without a build fails before any download."""
with pytest.raises(EsphomeError, match=r"3\.1\.1.*available: 3\.1\.2"):
framework.check_and_install(cv.Version(3, 1, 1))
def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: