diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 8d828dc936..5c2d9f4821 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -24,6 +24,7 @@ import os from pathlib import Path import platform import shutil +import time from esphome.core import EsphomeError, Version from esphome.framework_helpers import ( @@ -103,9 +104,15 @@ def _pio_system() -> str: sysname = platform.system().lower() machine = platform.machine().lower() if sysname == "darwin": - return "darwin_arm64" if machine == "arm64" else "darwin_x86_64" + if machine == "arm64": + return "darwin_arm64" + if machine == "x86_64": + return "darwin_x86_64" if sysname == "windows": - return "windows_amd64" if machine in ("amd64", "arm64") else "windows_x86" + if machine in ("amd64", "arm64"): + return "windows_amd64" + if machine in ("x86", "i686", "i386"): + return "windows_x86" if sysname == "linux": if machine in ("arm64", "aarch64"): return "linux_aarch64" @@ -129,7 +136,7 @@ def _registry_download(package: str, version: str) -> tuple[str, str, int | None url = _REGISTRY_URL.format(package=package) last_err: Exception | None = None - for _ in range(3): + for attempt in range(3): try: resp = requests.get(url, timeout=30) resp.raise_for_status() @@ -137,6 +144,8 @@ def _registry_download(package: str, version: str) -> tuple[str, str, int | None break except requests.RequestException as err: last_err = err + # Back off so the retries are not one burst against a hiccup + time.sleep(2**attempt) else: # A clean, retried error like the other download paths in the tree raise EsphomeError( @@ -186,7 +195,10 @@ def _install_package( # process cannot wipe the directory another is extracting into (same # filelock pattern as platformio/toolchain.py and git.py). dest.parent.mkdir(parents=True, exist_ok=True) - with FileLock(f"{dest}.lock"): + # fallback_to_soft would silently degrade to an existence lock on a + # flock-less filesystem; a hard-killed run would then hang every later + # build forever (same hazard git.py documents). + with FileLock(f"{dest}.lock", fallback_to_soft=False): if marker.is_file(): # Another process finished the install while we waited return @@ -248,6 +260,15 @@ def _find_ninja() -> Path: def check_and_install(framework_version: Version) -> dict[str, Path]: """Ensure framework, toolchain, and ninja are installed; return their paths.""" + if framework_version < MIN_FRAMEWORK_VERSION: + # Config validation enforces this too; keep the module honest when + # called directly. + raise EsphomeError( + f"The native toolchain requires the Arduino core " + f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}" + ) + # Probe the cheap local dependency before ~110 MB of downloads + ninja_path = _find_ninja() package_version = framework_package_version(framework_version) framework_path = get_framework_path(package_version) _install_package( @@ -268,7 +289,7 @@ def check_and_install(framework_version: Version) -> dict[str, Path]: return { "framework_path": framework_path, "toolchain_path": toolchain_path, - "ninja_path": _find_ninja(), + "ninja_path": ninja_path, } diff --git a/esphome/components/esp8266/boards.py b/esphome/components/esp8266/boards.py index e5a9628e90..2646682766 100644 --- a/esphome/components/esp8266/boards.py +++ b/esphome/components/esp8266/boards.py @@ -362,9 +362,10 @@ BOARDS = { } """ -ESP8266_BOARD_BUILD generate with: +ESP8266_BOARD_BUILD generate with (v4.2.1 is the platform version the +native toolchain mirrors; regenerate against the tag when bumping it): -git clone https://github.com/platformio/platform-espressif8266 +git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266 python3 - <<'EOF' import json, glob, os for f in sorted(glob.glob("platform-espressif8266/boards/*.json")): diff --git a/esphome/components/esp8266/build_surgery.py b/esphome/components/esp8266/build_surgery.py index c0034f7bad..47f9042f08 100644 --- a/esphome/components/esp8266/build_surgery.py +++ b/esphome/components/esp8266/build_surgery.py @@ -58,21 +58,33 @@ _TESTING_SEGMENT_SIZES = { def _segment_line_re(segment_name: str) -> re.Pattern[str]: - """The MEMORY line for one segment: `` : org = 0x..., len = 0x...``.""" + """The MEMORY line for one segment: `` : org = 0x..., len = 0x...``. + + Anchored to the start of the line so a name never matches inside a + longer one (``ram0_0_seg`` must not read ``dram0_0_seg``). The size + group stops at the hex digits, leaving any ``ul`` suffix (from the + preprocessed ``MMU_IRAM_SIZE``) in place. + """ return re.compile( - rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)" - r"(0x[0-9a-fA-F]+)" + rf"(^[ \t]*{re.escape(segment_name)}" + r"\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)" + r"(0x[0-9a-fA-F]+)", + re.MULTILINE, ) def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str: """Enlarge the named memory segments so grouped CI test builds can link. - Each caller passes the segments its linker script defines; a segment - that fails to match raises, since a silently kept real memory limit - would fail grouped builds far from the cause. + Each caller passes the segments its linker script defines: the + generated common ld carries ``iram1_0_seg``; the flash ld carries + ``dram0_0_seg`` and ``irom0_0_seg``. A segment that fails to match + raises, since a silently kept real memory limit would fail grouped + builds far from the cause. """ for segment in segments: + if segment not in _TESTING_SEGMENT_SIZES: + raise RuntimeError(f"Unknown testing-mode segment {segment!r}") content, count = _segment_line_re(segment).subn( rf"\g<1>{_TESTING_SEGMENT_SIZES[segment]}", content ) diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index f83593269e..6ba0b6e834 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -314,8 +314,8 @@ BASE_SCHEMA = cv.Schema( ) BASE_SCHEMA.add_extra(_detect_variant) -BASE_SCHEMA.add_extra(_update_core_data) BASE_SCHEMA.add_extra(cv.require_platformio_toolchain("LibreTiny")) +BASE_SCHEMA.add_extra(_update_core_data) def _configure_lwip(config: dict) -> None: diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 009b844986..df0c152b13 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -107,6 +107,9 @@ from esphome.util import parse_esphome_version # noqa: F401 from esphome.voluptuous_schema import _Schema from esphome.yaml_util import SensitiveStr, make_data_base +if typing.TYPE_CHECKING: + from esphome.types import ConfigType + _LOGGER = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -2533,17 +2536,21 @@ def platformio_version_constraint(value): return constraints -def check_supported_toolchain(platform_name: str, supported: tuple) -> None: +def check_supported_toolchain( + platform_name: str, supported: tuple[Toolchain, ...] +) -> None: """Raise when the resolved ``CORE.toolchain`` is not in ``supported``. One message shape for every platform, so a ``--toolchain`` a platform cannot serve always fails by name instead of silently building with a different backend. """ - if CORE.toolchain not in supported: + toolchain = CORE.toolchain + if toolchain is None or toolchain not in supported: names = ", ".join(f"'{tc.value}'" for tc in supported) raise Invalid( - f"Unsupported toolchain '{CORE.toolchain.value}' for " + f"Unsupported toolchain " + f"'{toolchain.value if toolchain else 'unresolved'}' for " f"{platform_name}. Supported: {names}." ) @@ -2555,7 +2562,7 @@ def require_platformio_toolchain(platform_name: str): ``--toolchain`` they cannot serve would silently build with PlatformIO. """ - def validator(config): + def validator(config: ConfigType) -> ConfigType: if CORE.toolchain is None: CORE.toolchain = Toolchain.PLATFORMIO check_supported_toolchain(platform_name, (Toolchain.PLATFORMIO,)) diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 8e9f8e9751..afe57f29b4 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -984,6 +984,12 @@ class EsphomeCore: @property def using_toolchain_arduino(self): + """The native (PlatformIO-free) ESP8266 Arduino build backend. + + Unlike ``using_arduino`` (the target *framework*, true for any + platform compiling Arduino code), this is a build *toolchain* + choice, like its ``using_toolchain_*`` siblings. + """ return self.toolchain == Toolchain.ARDUINO @property @@ -1099,6 +1105,8 @@ class EsphomeCore: return build_flag def add_build_unflag(self, build_unflag: str) -> None: + # No warning for using_toolchain_arduino: the native ESP8266 build + # honors build_unflags (token-level, matching PlatformIO). if self.using_toolchain_esp_idf: # The native ESP-IDF build generator does not consume build_unflags _LOGGER.warning( diff --git a/esphome/core/config.py b/esphome/core/config.py index 9fef173b48..350e4fd557 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -576,14 +576,15 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No for flag in vals: cg.add_build_flag(flag) elif key == "lib_deps": - # Routed through the regular library mechanism so the libraries - # are converted to IDF components like any other PIO library + # Routed through the regular library mechanism so the + # libraries reach the native backend's converter (IDF + # components, or the ESP8266 native library resolution) for lib in vals: _add_library_str(lib) elif key == "lib_ignore": - # Read by the PIO-library-to-IDF-component conversion - # (generate_idf_components); filters both top-level libraries - # and dependencies discovered during conversion + # Read by the shared library conversion (lib_ignore_set in + # platformio/library.py) on both native backends; filters + # top-level libraries and discovered dependencies cg.add_platformio_option(key, vals) elif key != "upload_speed": # upload_speed needs no handling: it is read from the raw diff --git a/tests/unit_tests/components/esp8266/test_build_surgery.py b/tests/unit_tests/components/esp8266/test_build_surgery.py index 64e36f1591..e62a2270fb 100644 --- a/tests/unit_tests/components/esp8266/test_build_surgery.py +++ b/tests/unit_tests/components/esp8266/test_build_surgery.py @@ -24,16 +24,26 @@ _COMMON_LD_SNIPPET = """\ } >dram0_0_seg :dram0_0_phdr """ +# Shaped like the real SDK flash ld scripts: no iram1_0_seg (that lives in +# the generated common ld only) _FLASH_LD_SNIPPET = """\ MEMORY { dport0_0_seg : org = 0x3FF00000, len = 0x10 dram0_0_seg : org = 0x3FFE8000, len = 0x14000 - iram1_0_seg : org = 0x40100000, len = 0x8000 irom0_0_seg : org = 0x40201010, len = 0xfeff0 } """ +# Shaped like the preprocessed common ld: MMU_IRAM_SIZE expands with a ul +# suffix the patcher must leave in place +_COMMON_LD_MEMORY_SNIPPET = """\ +MEMORY +{ + iram1_0_seg : org = 0x40100000, len = 0x8000ul +} +""" + def test_relocate_ratetable_inserts_after_data_start() -> None: patched = relocate_ratetable(_COMMON_LD_SNIPPET) @@ -52,15 +62,32 @@ def test_relocate_ratetable_requires_anchor() -> None: def test_testing_memory_patches_enlarge_segments() -> None: patched = apply_testing_memory_patches( - _FLASH_LD_SNIPPET, ("iram1_0_seg", "dram0_0_seg", "irom0_0_seg") + _FLASH_LD_SNIPPET, ("dram0_0_seg", "irom0_0_seg") ) - assert segment_length(patched, "iram1_0_seg") == 0x200000 assert segment_length(patched, "dram0_0_seg") == 0x200000 assert segment_length(patched, "irom0_0_seg") == 0x2000000 # Untouched segments keep their sizes assert segment_length(patched, "dport0_0_seg") == 0x10 +def test_testing_memory_patches_keep_ul_suffix() -> None: + """The common ld's preprocessed sizes carry a ul suffix; the patch must + replace only the hex digits, as testing_mode.py.script does.""" + patched = apply_testing_memory_patches(_COMMON_LD_MEMORY_SNIPPET, ("iram1_0_seg",)) + assert "len = 0x200000ul" in patched + assert segment_length(patched, "iram1_0_seg") == 0x200000 + + +def test_segment_length_requires_whole_name() -> None: + """A name must match its own line, never inside a longer segment name.""" + assert segment_length(_FLASH_LD_SNIPPET, "ram0_0_seg") is None + + +def test_testing_memory_patches_unknown_segment_raises() -> None: + with pytest.raises(RuntimeError, match="Unknown testing-mode segment"): + apply_testing_memory_patches("MEMORY { }", ("bogus_seg",)) + + def test_segment_length() -> None: assert segment_length(_FLASH_LD_SNIPPET, "irom0_0_seg") == 0xFEFF0 assert segment_length(_FLASH_LD_SNIPPET, "missing_seg") is None diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index 7602674c16..3ca9e70ee2 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -64,6 +64,9 @@ def test_pio_system(system: str, machine: str, expected: str) -> None: [ ("FreeBSD", "amd64"), ("Linux", "ppc64le"), + ("Darwin", "ppc"), + ("Darwin", ""), + ("Windows", "ia64"), ], ) def test_pio_system_unsupported_host_raises(system: str, machine: str) -> None: @@ -88,10 +91,13 @@ def test_registry_download_network_error_is_clean_and_retried() -> None: with ( patch("requests.get", side_effect=requests.ConnectionError("boom")) as mock_get, + patch.object(framework.time, "sleep") as mock_sleep, pytest.raises(EsphomeError, match="Could not query the package registry"), ): framework._registry_download("pkg", "1.0.0") assert mock_get.call_count == 3 + # Backed-off retries, not one burst + assert mock_sleep.call_count == 3 def test_registry_download_retries_transient_error() -> None: @@ -109,6 +115,7 @@ def test_registry_download_retries_transient_error() -> None: ) with ( patch("requests.get", side_effect=[requests.ConnectionError("boom"), resp]), + patch.object(framework.time, "sleep"), patch.object(framework, "_pio_system", return_value="linux_x86_64"), ): assert framework._registry_download("pkg", "1.0.0") == ( @@ -439,3 +446,23 @@ def test_ccache_env_requires_build_path() -> None: pytest.raises(ValueError, match="build_path"), ): framework.ccache_env() + + +def test_check_and_install_rejects_old_core(tmp_path: Path) -> None: + """Calling the installer below the floor fails before any download.""" + with pytest.raises(EsphomeError, match=">= 3.1.1"): + framework.check_and_install(cv.Version(3, 0, 2)) + + +def test_install_package_uses_hard_lock(tmp_path: Path) -> None: + """The install lock must never degrade to a soft (existence) lock.""" + dest = tmp_path / "pkg" + with ( + patch("filelock.FileLock") as mock_lock, + patch.object(framework, "download_from_mirrors"), + patch.object(framework, "archive_extract_all") as mock_extract, + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + ): + mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir(exist_ok=True) + framework._install_package("pkg", "1.0.0", dest, ["http://m"]) + assert mock_lock.call_args.kwargs["fallback_to_soft"] is False diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index a7ddb931b3..19106f4e1b 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -3181,3 +3181,28 @@ def test_require_platformio_toolchain() -> None: CORE.toolchain = Toolchain.ARDUINO with pytest.raises(Invalid, match="Unsupported toolchain 'arduino' for RP2"): validator(config) + + +@pytest.mark.parametrize( + ("platform", "minimal_config"), + [ + ("host", {}), + ("rp2", {"board": "rpipicow"}), + ("bk72xx", {"board": "generic-bk7231n-qfn32-tuya"}), + ], +) +def test_every_platformio_only_platform_rejects_arduino_toolchain( + platform: str, minimal_config: dict +) -> None: + """The invariant every native-toolchain gate relies on: a platform that + cannot serve a CLI toolchain rejects it at validation (esp32, esp8266, + and nrf52 pin this in their own suites).""" + import importlib + + from esphome.const import Toolchain + from esphome.core import CORE + + module = importlib.import_module(f"esphome.components.{platform}") + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(Invalid, match="Unsupported toolchain 'arduino'"): + module.CONFIG_SCHEMA(dict(minimal_config))