diff --git a/.github/actions/cache-arduino8266/action.yml b/.github/actions/cache-arduino8266/action.yml index affe62b5aa..c9e194d667 100644 --- a/.github/actions/cache-arduino8266/action.yml +++ b/.github/actions/cache-arduino8266/action.yml @@ -22,7 +22,7 @@ runs: # cached path cannot diverge. echo "ESPHOME_ARDUINO8266_PREFIX=$HOME/.esphome-arduino8266" >> "$GITHUB_ENV" . venv/bin/activate - key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import TOOLCHAIN_VERSION as t; print(f"{f}-{t}")') + key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import FRAMEWORK_RELEASES, TOOLCHAIN_VERSION as t; print(f"{FRAMEWORK_RELEASES[f].tag}-{t}")') [ -n "$key" ] || exit 1 echo "key=$key" >> "$GITHUB_OUTPUT" - name: Cache the native toolchain (write on dev) diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 0c411a9d5d..a8f8c65c06 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -3,13 +3,12 @@ Artifacts land in a machine-global cache (shared across projects, like the ESP-IDF install in ``esphome.espidf.framework``): - /arduino8266/frameworks// framework-arduinoespressif8266 - /arduino8266/toolchains// xtensa-lx106-elf gcc 10.3 + /arduino8266/frameworks// the Arduino core + /arduino8266/toolchains// xtensa-lx106-elf gcc 10.3 -The framework comes from the PlatformIO registry, the toolchain from -https://github.com/esphome-libs/xtensa-lx106-elf-toolchain/releases/; -``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes from PATH -or the ninja PyPI wheel. +Both come from esphome-libs releases pinned below; +``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs, with ``{VERSION}`` +standing for the release tag. ninja comes from PATH or the ninja PyPI wheel. """ from __future__ import annotations @@ -31,7 +30,30 @@ from esphome.platformio.registry import ( prefetch_packages, ) -FRAMEWORK_PACKAGE = "framework-arduinoespressif8266" +FRAMEWORK_PACKAGE = "arduino-esp8266" +_FRAMEWORK_RELEASES = "https://github.com/esphome-libs/arduino-esp8266/releases/" + + +class FrameworkRelease(NamedTuple): + tag: str + sha256: str + size: int + + def download(self) -> Download: + archive = f"{FRAMEWORK_PACKAGE}-{self.tag}.tar.gz" + url = f"{_FRAMEWORK_RELEASES}download/{self.tag}/{archive}" + return Download(url, self.sha256, self.size) + + +# Arduino core version -> its build in esphome-libs/arduino-esp8266 +FRAMEWORK_RELEASES: dict[Version, FrameworkRelease] = { + Version(3, 1, 2): FrameworkRelease( + "3.1.2-esphome.1", + "e80751e3123676b967143e39c61f2d8693946db4c7806f2a83dcaaf797ecd582", + 37189311, + ), +} + TOOLCHAIN_PACKAGE = "toolchain-xtensa-lx106-elf" # gcc 10.3, the toolchain Arduino core 3.x builds with; the build # generator's compile flags are tuned to it. @@ -77,30 +99,18 @@ def get_arduino8266_tools_path() -> Path: return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) -# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0 -MIN_FRAMEWORK_VERSION = Version(3, 1, 1) - - -def framework_package_version(ver: Version) -> str: - """Map an Arduino core version to its registry package version (3.1.2 -> - 3.30102.0; the leading 3 is the package major). - - Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION. - """ - if ver.major > 3: +def framework_release(version: Version) -> FrameworkRelease: + if (release := FRAMEWORK_RELEASES.get(version)) is None: raise EsphomeError( - f"Arduino core {ver} is not supported yet; " - "the newest known core series is 3.x" + f"'toolchain: arduino' has no build of Arduino core {version}; " + f"available: {', '.join(str(v) for v in FRAMEWORK_RELEASES)}. " + "Use one of those or 'toolchain: platformio'" ) - if ver.major < 3: - raise EsphomeError( - f"Arduino core {ver} is not supported; ESPHome requires core 3.x" - ) - return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + return release -def get_framework_path(package_version: str) -> Path: - return get_arduino8266_tools_path() / "frameworks" / package_version +def get_framework_path(tag: str) -> Path: + return get_arduino8266_tools_path() / "frameworks" / tag def get_toolchain_path() -> Path: @@ -133,24 +143,17 @@ class InstalledPaths(NamedTuple): def check_and_install(framework_version: Version) -> InstalledPaths: """Ensure framework, toolchain, and ninja are installed; return their paths.""" - if framework_version < MIN_FRAMEWORK_VERSION: - # Config validation enforces this too; keep the module honest when - # called directly. - raise EsphomeError( - f"The native toolchain requires the Arduino core " - f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}" - ) + release = framework_release(framework_version) # Probe the cheap local dependency before ~110 MB of downloads ninja_path = find_ninja() - package_version = framework_package_version(framework_version) - framework_path = get_framework_path(package_version) + framework_path = get_framework_path(release.tag) downloads_dir = get_arduino8266_tools_path() / "downloads" toolchain_path = get_toolchain_path() # One spec per package: the prefetch and the installs must agree specs = ( ( FRAMEWORK_PACKAGE, - package_version, + release.tag, framework_path, ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, ("cores/esp8266", "tools/sdk", "libraries"), @@ -167,6 +170,8 @@ def check_and_install(framework_version: Version) -> InstalledPaths: # Resolved only when a download is needed, so an installed toolchain # keeps working on a host without a build; a mirror override wins resolvers: dict[str, Resolver] = {} + if not ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS: + resolvers[FRAMEWORK_PACKAGE] = release.download if not ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS: resolvers[TOOLCHAIN_PACKAGE] = toolchain_download # Fetch both archives at once; the installs below verify and extract diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 1016c49152..0e5303e8df 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -118,15 +118,14 @@ def _validate_native_toolchain(config: ConfigType) -> ConfigType: """Constraints of the native (non-PlatformIO) Arduino toolchain.""" if not CORE.using_toolchain_arduino: return config - from esphome.arduino8266.framework import MIN_FRAMEWORK_VERSION + from esphome.arduino8266.framework import framework_release conf = config[CONF_FRAMEWORK] version = cv.Version.parse(conf[CONF_VERSION]) - if version < MIN_FRAMEWORK_VERSION: - raise cv.Invalid( - "'toolchain: arduino' requires framework version " - f"{MIN_FRAMEWORK_VERSION} or newer" - ) + try: + framework_release(version) + except EsphomeError as err: + raise cv.Invalid(str(err), path=[CONF_FRAMEWORK, CONF_VERSION]) from err # platform_version is a PlatformIO concept; drop it, warning when a # custom pin is discarded if ( @@ -137,6 +136,8 @@ def _validate_native_toolchain(config: ConfigType) -> ConfigType: "'platform_version' is ignored by 'toolchain: arduino'; the native " "toolchain downloads the framework and compiler directly" ) + # The native path fetches its own build; source may only be the + # PlatformIO default the schema filled in if conf[CONF_SOURCE] != _format_framework_arduino_version(version): raise cv.Invalid( "'toolchain: arduino' does not support a custom framework source; " @@ -174,16 +175,27 @@ def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]: ] +def _framework_package_version(ver: cv.Version) -> str: + """Map an Arduino core version to its PlatformIO registry package version + (3.1.2 -> 3.30102.0; the leading 3 is the package major).""" + if ver.major > 3: + raise EsphomeError( + f"Arduino core {ver} is not supported yet; " + "the newest known core series is 3.x" + ) + if ver.major < 3: + raise EsphomeError( + f"Arduino core {ver} is not supported; ESPHome requires core 3.x" + ) + return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + + def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/esp8266/Arduino/releases) version to # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 - # Same encoding the native toolchain uses for its package download, so a - # custom-source check against this value cannot drift from what it fetches. - from esphome.arduino8266.framework import framework_package_version - try: - return f"~{framework_package_version(ver)}" + return f"~{_framework_package_version(ver)}" except EsphomeError as err: # Anchor the 4.x rejection to the framework version line instead of # aborting with a bare traceback-level error @@ -194,7 +206,8 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # * New framework historically have had some regressions, especially for WiFi. # The new version needs to be thoroughly validated before changing the # recommended version as otherwise a bunch of devices could be bricked -# * For all constants below, update platformio.ini (in this repo) +# * For all constants below, update platformio.ini (in this repo) and +# FRAMEWORK_RELEASES in esphome/arduino8266/framework.py # The default/recommended arduino framework version # - https://github.com/esp8266/Arduino/releases diff --git a/tests/unit_tests/components/esp8266/test_framework_version.py b/tests/unit_tests/components/esp8266/test_framework_version.py index 0107aff8dd..11590f77cf 100644 --- a/tests/unit_tests/components/esp8266/test_framework_version.py +++ b/tests/unit_tests/components/esp8266/test_framework_version.py @@ -2,9 +2,14 @@ import pytest -from esphome.components.esp8266 import _arduino_check_versions +from esphome.components.esp8266 import ( + _arduino_check_versions, + _format_framework_arduino_version, + _framework_package_version, +) import esphome.config_validation as cv from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION +from esphome.core import EsphomeError def test_versions_before_3_are_rejected() -> None: @@ -21,3 +26,27 @@ def test_supported_versions_pass() -> None: value = _arduino_check_versions({CONF_VERSION: "recommended"}) assert value[CONF_VERSION] == "3.1.2" assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION] + + +def test_framework_package_version() -> None: + assert _framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" + assert _framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" + # A future major bump needs its own encoding, not a doomed registry lookup + with pytest.raises(EsphomeError, match="not supported yet"): + _framework_package_version(cv.Version(4, 0, 0)) + # Cores before 3.x cannot build ESPHome (C++20) and are rejected + with pytest.raises(EsphomeError, match="requires core 3"): + _framework_package_version(cv.Version(2, 7, 4)) + + +def test_format_framework_arduino_version_pins_all_series() -> None: + """The PlatformIO source formatter across every encoding era.""" + + assert _format_framework_arduino_version(cv.Version(3, 1, 2)) == "~3.30102.0" + # Pre-3 cores are rejected with the version line anchored + with pytest.raises(cv.Invalid, match="requires core 3"): + _format_framework_arduino_version(cv.Version(2, 7, 4)) + # Anchored to the framework version line, not a bare EsphomeError + with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: + _format_framework_arduino_version(cv.Version(4, 0, 0)) + assert excinfo.value.path == ["version"] diff --git a/tests/unit_tests/components/esp8266/test_toolchain_validation.py b/tests/unit_tests/components/esp8266/test_toolchain_validation.py index 951217c8a1..d54c2c3533 100644 --- a/tests/unit_tests/components/esp8266/test_toolchain_validation.py +++ b/tests/unit_tests/components/esp8266/test_toolchain_validation.py @@ -67,21 +67,24 @@ def test_valid_config_passes() -> None: def test_platformio_toolchain_skips_checks() -> None: - # 3.0.2 is pio-legal (>= the global 3.0.0 floor) but below the native - # toolchain's own 3.1.1 floor; the bogus board only the native path checks + # 3.0.2 is pio-legal (>= the global 3.0.0 floor) but has no native build; + # the bogus board only the native path checks CORE.toolchain = Toolchain.PLATFORMIO config = _config(board="not_a_board", version="3.0.2") assert _validate_native_toolchain(config) is config -def test_version_below_floor_rejected() -> None: - # 3.1.0 has no registry package, so the native floor is 3.1.1 - with pytest.raises(cv.Invalid, match="3.1.1 or newer"): - _validate_native_toolchain(_config(version="3.1.0")) +def test_version_without_build_rejected() -> None: + """Only the core versions built in esphome-libs/arduino-esp8266 work.""" + with pytest.raises( + cv.Invalid, match=r"3\.1\.1.*available: 3\.1\.2.*platformio" + ) as excinfo: + _validate_native_toolchain(_config(version="3.1.1")) + assert excinfo.value.path == [CONF_FRAMEWORK, CONF_VERSION] -def test_version_at_floor_accepted() -> None: - _validate_native_toolchain(_config(version="3.1.1")) +def test_built_version_accepted() -> None: + _validate_native_toolchain(_config(version="3.1.2")) def test_custom_platform_version_warns_and_is_dropped( diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index 6f43f0bf40..a4193b9a74 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -9,6 +9,7 @@ from unittest.mock import patch import pytest from esphome.arduino8266 import framework +from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION import esphome.config_validation as cv from esphome.core import CORE, EsphomeError @@ -18,32 +19,6 @@ def _build_path(tmp_path: Path) -> None: CORE.build_path = tmp_path -def test_framework_package_version() -> None: - assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" - assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" - # A future major bump needs its own encoding, not a doomed registry lookup - with pytest.raises(EsphomeError, match="not supported yet"): - framework.framework_package_version(cv.Version(4, 0, 0)) - # Cores before 3.x cannot build ESPHome (C++20) and are rejected - with pytest.raises(EsphomeError, match="requires core 3"): - framework.framework_package_version(cv.Version(2, 7, 4)) - - -def test_format_framework_arduino_version_pins_all_series() -> None: - """The esp8266 component's PIO source formatter across every encoding - era, including the 4.x rejection it now shares with the installer.""" - from esphome.components.esp8266 import _format_framework_arduino_version as fmt - - assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" - # Pre-3 cores are rejected with the version line anchored - with pytest.raises(cv.Invalid, match="requires core 3"): - fmt(cv.Version(2, 7, 4)) - # Anchored to the framework version line, not a bare EsphomeError - with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: - fmt(cv.Version(4, 0, 0)) - assert excinfo.value.path == ["version"] - - def test_tools_path_default_and_prefix(tmp_path: Path) -> None: with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}): assert framework.get_arduino8266_tools_path() == tmp_path.resolve() @@ -54,6 +29,27 @@ def test_tools_path_default_and_prefix(tmp_path: Path) -> None: assert path != Path.cwd() +def _recommended() -> framework.FrameworkRelease: + return framework.FRAMEWORK_RELEASES[RECOMMENDED_ARDUINO_FRAMEWORK_VERSION] + + +def test_framework_releases_are_pinned() -> None: + """A release must not be pinned without its checksum, and the recommended + core must have one.""" + assert RECOMMENDED_ARDUINO_FRAMEWORK_VERSION in framework.FRAMEWORK_RELEASES + for version, release in framework.FRAMEWORK_RELEASES.items(): + assert release.tag.startswith(f"{version}-esphome.") + assert len(release.sha256) == 64 + assert release.size > 0 + + +def test_framework_download() -> None: + download = framework.FrameworkRelease("1.2.3-esphome.4", "a" * 64, 5).download() + releases = "https://github.com/esphome-libs/arduino-esp8266/releases/" + archive = "arduino-esp8266-1.2.3-esphome.4.tar.gz" + assert download == (f"{releases}download/1.2.3-esphome.4/{archive}", "a" * 64, 5) + + def test_toolchain_builds_are_pinned() -> None: """A release must not be pinned without its checksums.""" for sha256, size in framework.TOOLCHAIN_BUILDS.values(): @@ -86,14 +82,15 @@ def test_toolchain_download_unsupported_system() -> None: def _fake_framework(tmp_path: Path) -> None: """The layout install_package expects of an installed framework.""" for sub in ("cores/esp8266", "tools/sdk", "libraries"): - (tmp_path / "frameworks" / "3.30102.0" / sub).mkdir(parents=True) - (tmp_path / "frameworks" / "3.30102.0" / ".esphome_extracted").touch() + (tmp_path / "frameworks" / _recommended().tag / sub).mkdir(parents=True) + (tmp_path / "frameworks" / _recommended().tag / ".esphome_extracted").touch() def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None: """With a mirror override an unsupported host can bring its own toolchain.""" with ( patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", ["http://f"]), patch.object(framework, "ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", ["http://m"]), patch.object(framework, "install_package") as mock_install, patch.object(framework, "prefetch_packages") as mock_prefetch, @@ -101,7 +98,10 @@ def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None ): framework.check_and_install(cv.Version(3, 1, 2)) assert mock_prefetch.call_args.args[2] == {} - assert mock_install.call_args_list[1].kwargs["resolve"] is None + assert [call.kwargs["resolve"] for call in mock_install.call_args_list] == [ + None, + None, + ] def test_check_and_install_installed_toolchain_on_unsupported_host( @@ -147,7 +147,7 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), ): paths = framework.check_and_install(cv.Version(3, 1, 2)) - assert paths.framework == tmp_path / "frameworks" / "3.30102.0" + assert paths.framework == tmp_path / "frameworks" / _recommended().tag assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION assert paths.ninja == tmp_path / "ninja" assert mock_install.call_count == 2 @@ -156,14 +156,14 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: fw_call, tc_call = mock_install.call_args_list assert fw_call.args == ( framework.FRAMEWORK_PACKAGE, - "3.30102.0", - tmp_path / "frameworks" / "3.30102.0", + _recommended().tag, + tmp_path / "frameworks" / _recommended().tag, framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, tmp_path / "downloads", ) assert fw_call.kwargs == { "expect": ("cores/esp8266", "tools/sdk", "libraries"), - "resolve": None, + "resolve": _recommended().download, } assert tc_call.args == ( framework.TOOLCHAIN_PACKAGE, @@ -181,8 +181,8 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: [ ( framework.FRAMEWORK_PACKAGE, - "3.30102.0", - tmp_path / "frameworks" / "3.30102.0", + _recommended().tag, + tmp_path / "frameworks" / _recommended().tag, framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, ), ( @@ -193,7 +193,10 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: ), ], tmp_path / "downloads", - {framework.TOOLCHAIN_PACKAGE: framework.toolchain_download}, + { + framework.FRAMEWORK_PACKAGE: _recommended().download, + framework.TOOLCHAIN_PACKAGE: framework.toolchain_download, + }, ) @@ -204,10 +207,10 @@ def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: assert env["CCACHE_DIR"] == "x" -def test_check_and_install_rejects_old_core(tmp_path: Path) -> None: - """Calling the installer below the floor fails before any download.""" - with pytest.raises(EsphomeError, match=">= 3.1.1"): - framework.check_and_install(cv.Version(3, 0, 2)) +def test_check_and_install_rejects_unbuilt_core(tmp_path: Path) -> None: + """A core version without a build fails before any download.""" + with pytest.raises(EsphomeError, match=r"3\.1\.1.*available: 3\.1\.2"): + framework.check_and_install(cv.Version(3, 1, 1)) def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: