Get ninja from the PyPI wheel instead of hardcoded release URLs and checksums

esphome/arduino8266/framework.py

@@ -5,7 +5,9 @@ ESP-IDF install in ``esphome.espidf.framework``):
 
     <cache>/arduino8266/frameworks/<version>/   framework-arduinoespressif8266
     <cache>/arduino8266/toolchains/<version>/   toolchain-xtensa (gcc 10.3)
-    <cache>/arduino8266/tools/ninja/            ninja (only if not on PATH)
+
+ninja itself comes from PATH or the ninja PyPI wheel (a requirements.txt
+dependency), so only the two packages above are downloaded here.
 
 Sources default to the PlatformIO registry (the exact packages the PlatformIO
 toolchain has always used, so the bits are identical); the
@@ -21,7 +23,6 @@ import os
 from pathlib import Path
 import platform
 import shutil
-import stat
 
 import platformdirs
 
@@ -45,19 +46,6 @@ TOOLCHAIN_PACKAGE = "toolchain-xtensa"
 # reinstall (the install dir is keyed on the version).
 TOOLCHAIN_VERSION = "2.100300.220621"
 
-NINJA_VERSION = "1.12.1"
-
-# sha256 of the ninja release archives, so the binary we chmod and execute is
-# integrity-checked. Only applies to the default download source; a mirror
-# override via ESPHOME_ARDUINO8266_NINJA_MIRRORS is trusted as configured.
-_NINJA_SHA256 = {
-    "ninja-mac.zip": "89a287444b5b3e98f88a945afa50ce937b8ffd1dcc59c555ad9b1baf855298c9",
-    "ninja-win.zip": "f550fec705b6d6ff58f2db3c374c2277a37691678d6aba463adcbb129108467a",
-    "ninja-winarm64.zip": "79c96a50e0deafec212cfa85aa57c6b74003f52d9d1673ddcd1eab1c958c5900",
-    "ninja-linux.zip": "6f98805688d19672bd699fbbfa2c2cf0fc054ac3df1f0e6a47664d963d530255",
-    "ninja-linux-aarch64.zip": "5c25c6570b0155e95fce5918cb95f1ad9870df5768653afe128db822301a05a1",
-}
-
 _REGISTRY_URL = (
     "https://api.registry.platformio.org/v3/packages/platformio/tool/{package}"
 )
@@ -198,44 +186,22 @@ def _install_package(
     archive.unlink(missing_ok=True)
 
 
-def _ninja_archive_name() -> str:
-    sysname = platform.system().lower()
-    machine = platform.machine().lower()
-    if sysname == "darwin":
-        return "ninja-mac.zip"
-    if sysname == "windows":
-        return "ninja-winarm64.zip" if machine == "arm64" else "ninja-win.zip"
-    if machine in ("arm64", "aarch64"):
-        return "ninja-linux-aarch64.zip"
-    return "ninja-linux.zip"
+def _find_ninja() -> Path:
+    """Locate the ninja binary: PATH first, else the ninja PyPI wheel.
 
+    The wheel is a requirements.txt dependency, so pip has already
+    integrity-checked it; no download logic is needed here.
+    """
+    if binary := shutil.which("ninja"):
+        return Path(binary)
+    import ninja
 
-def _check_ninja_install() -> Path:
-    """Return a usable ninja binary, downloading one if none is on PATH."""
-    if ninja := shutil.which("ninja"):
-        return Path(ninja)
-    ninja_dir = get_arduino8266_tools_path() / "tools" / "ninja" / NINJA_VERSION
-    binary = ninja_dir / ("ninja.exe" if os.name == "nt" else "ninja")
-    if binary.is_file():
-        return binary
-    rmdir(ninja_dir, msg="Clean up incomplete ninja install")
-    archive_name = _ninja_archive_name()
-    archive = _downloads_path() / archive_name
-    _LOGGER.info("Downloading ninja %s ...", NINJA_VERSION)
-    if ESPHOME_ARDUINO8266_NINJA_MIRRORS:
-        download_from_mirrors(
-            ESPHOME_ARDUINO8266_NINJA_MIRRORS,
-            {"VERSION": NINJA_VERSION, "ARCHIVE": archive_name},
-            archive,
-        )
-    else:
-        url = _NINJA_URL.format(VERSION=NINJA_VERSION, ARCHIVE=archive_name)
-        download_with_resume(url, archive, sha256=_NINJA_SHA256[archive_name])
-    archive_extract_all(archive, ninja_dir)
-    archive.unlink(missing_ok=True)
+    binary = Path(ninja.BIN_DIR) / ("ninja.exe" if os.name == "nt" else "ninja")
     if not binary.is_file():
-        raise EsphomeError(f"ninja binary missing after extraction in {ninja_dir}")
-    binary.chmod(binary.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        raise EsphomeError(
+            "ninja not found on PATH or in the ninja package; reinstall the "
+            "esphome Python environment"
+        )
     return binary
 
 
@@ -259,7 +225,7 @@ def check_and_install(framework_version: cv.Version) -> dict[str, Path]:
     return {
         "framework_path": framework_path,
         "toolchain_path": toolchain_path,
-        "ninja_path": _check_ninja_install(),
+        "ninja_path": _find_ninja(),
     }
 
 

requirements.txt

@@ -28,6 +28,7 @@ smpclient==7.2.0
 requests==2.34.2
 py7zr==1.1.3
 platformdirs==4.11.3  # native esp-idf toolchain global cache dir
+ninja==1.13.0  # native esp8266 arduino toolchain build driver
 filelock==3.32.3  # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg
 
 # esp-idf >= 5.0 requires this

tests/unit_tests/test_arduino8266_framework.py

@@ -4,8 +4,8 @@ from __future__ import annotations
 
 import os
 from pathlib import Path
-import stat
 import subprocess
+import sys
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -165,103 +165,38 @@ def test_install_package_downloads_via_registry(tmp_path: Path) -> None:
     assert mock_download.call_args[1] == {"sha256": "abc123", "size": 42}
 
 
-@pytest.mark.parametrize(
-    ("system", "machine", "expected"),
-    [
-        ("Darwin", "arm64", "ninja-mac.zip"),
-        ("Windows", "AMD64", "ninja-win.zip"),
-        ("Windows", "arm64", "ninja-winarm64.zip"),
-        ("Linux", "x86_64", "ninja-linux.zip"),
-        ("Linux", "aarch64", "ninja-linux-aarch64.zip"),
-    ],
-)
-def test_ninja_archive_name(system: str, machine: str, expected: str) -> None:
-    with (
-        patch("platform.system", return_value=system),
-        patch("platform.machine", return_value=machine),
-    ):
-        assert framework._ninja_archive_name() == expected
-
-
-def test_check_ninja_install_prefers_path(tmp_path: Path) -> None:
+def test_find_ninja_prefers_path(tmp_path: Path) -> None:
     with patch("shutil.which", return_value=str(tmp_path / "ninja")):
-        assert framework._check_ninja_install() == tmp_path / "ninja"
+        assert framework._find_ninja() == tmp_path / "ninja"
 
 
-def test_check_ninja_install_cached_binary(tmp_path: Path) -> None:
-    binary = (
-        tmp_path
-        / "tools"
-        / "ninja"
-        / framework.NINJA_VERSION
-        / ("ninja.exe" if os.name == "nt" else "ninja")
-    )
-    binary.parent.mkdir(parents=True)
-    binary.touch()
+def test_find_ninja_falls_back_to_wheel(tmp_path: Path) -> None:
+    """Without a PATH entry, the ninja PyPI wheel's binary is used."""
+    binary_name = "ninja.exe" if os.name == "nt" else "ninja"
+    (tmp_path / binary_name).touch()
+    wheel = MagicMock(BIN_DIR=str(tmp_path))
     with (
         patch("shutil.which", return_value=None),
-        patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
+        patch.dict(sys.modules, {"ninja": wheel}),
     ):
-        assert framework._check_ninja_install() == binary
+        assert framework._find_ninja() == tmp_path / binary_name
 
 
-def _fake_ninja_extract(_archive, ninja_dir, **_kw) -> None:
-    ninja_dir.mkdir(parents=True, exist_ok=True)
-    (ninja_dir / ("ninja.exe" if os.name == "nt" else "ninja")).touch()
-
-
-def test_check_ninja_install_downloads(tmp_path: Path) -> None:
-    """The default source is integrity-checked against the pinned sha256."""
+def test_find_ninja_missing_everywhere(tmp_path: Path) -> None:
+    wheel = MagicMock(BIN_DIR=str(tmp_path))
     with (
         patch("shutil.which", return_value=None),
-        patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
-        patch.object(framework, "download_with_resume") as mock_download,
-        patch.object(framework, "archive_extract_all", side_effect=_fake_ninja_extract),
+        patch.dict(sys.modules, {"ninja": wheel}),
+        pytest.raises(EsphomeError, match="ninja not found"),
     ):
-        binary = framework._check_ninja_install()
-    assert binary.is_file()
-    # X_OK would consult mount flags (fails on noexec /tmp); check mode bits
-    assert binary.stat().st_mode & stat.S_IXUSR
-    archive_name = framework._ninja_archive_name()
-    assert mock_download.call_args[1]["sha256"] == framework._NINJA_SHA256[archive_name]
-
-
-def test_check_ninja_install_mirror_override_skips_checksum(tmp_path: Path) -> None:
-    """A mirror override is trusted as configured (no pinned checksum)."""
-    with (
-        patch("shutil.which", return_value=None),
-        patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
-        patch.object(
-            framework,
-            "ESPHOME_ARDUINO8266_NINJA_MIRRORS",
-            ["http://mirror/{ARCHIVE}"],
-        ),
-        patch.object(framework, "download_from_mirrors") as mock_download,
-        patch.object(framework, "archive_extract_all", side_effect=_fake_ninja_extract),
-    ):
-        binary = framework._check_ninja_install()
-    assert binary.is_file()
-    mock_download.assert_called_once()
-
-
-def test_check_ninja_install_missing_after_extract(tmp_path: Path) -> None:
-    with (
-        patch("shutil.which", return_value=None),
-        patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
-        patch.object(framework, "download_from_mirrors"),
-        patch.object(framework, "archive_extract_all"),
-        pytest.raises(EsphomeError, match="ninja binary missing"),
-    ):
-        framework._check_ninja_install()
+        framework._find_ninja()
 
 
 def test_check_and_install_returns_paths(tmp_path: Path) -> None:
     with (
         patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}),
         patch.object(framework, "_install_package") as mock_install,
-        patch.object(
-            framework, "_check_ninja_install", return_value=tmp_path / "ninja"
-        ),
+        patch.object(framework, "_find_ninja", return_value=tmp_path / "ninja"),
     ):
         paths = framework.check_and_install(cv.Version(3, 1, 2))
     assert paths["framework_path"] == tmp_path / "frameworks" / "3.30102.0"