Merge branch 'esp8266-native-build-surgery' into esp8266-native-toolchain-plumbing

esphome/__main__.py

@@ -870,6 +870,7 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
             # Broad on purpose: the firmware already built; an idedata
             # failure must not fail a successful build.
             _LOGGER.warning("Could not generate idedata: %s", err)
+            _LOGGER.debug("Idedata failure detail", exc_info=True)
     else:
         from esphome.platformio import toolchain
 

esphome/espidf/framework.py

@@ -750,7 +750,7 @@ def _prefetch_idf_tool_archives(
         # tools.json always carries sizes; should one be missing the combined
         # bar could not be trusted, so show no bar at all (per-file bars from
         # several threads would interleave) rather than a wrong one.
-        sizes = [entry["size"] for entry in entries]
+        sizes = [entry.get("size") or 0 for entry in entries]
         progress = BatchDownloadProgress(
             "Downloading ESP-IDF tools", sum(sizes) if all(sizes) else 0
         )
@@ -764,8 +764,8 @@ def _prefetch_idf_tool_archives(
                 download_with_resume(
                     entry["url"],
                     dist_path / entry["dest"],
-                    sha256=entry["sha256"],
+                    sha256=entry.get("sha256"),
-                    size=entry["size"],
+                    size=entry.get("size"),
                     progress=tracker,
                 )
             except Exception as e:  # noqa: BLE001  # pylint: disable=broad-exception-caught

esphome/platformio/library.py

@@ -15,7 +15,6 @@ regardless of which toolchain consumes the result.
 from collections import deque
 from collections.abc import Callable, Iterable
 from concurrent.futures import ThreadPoolExecutor
-import contextlib
 from dataclasses import dataclass, field
 import glob
 import hashlib
@@ -905,16 +904,16 @@ def _content_lengths(urls: list[str]) -> list[int]:
     """Content-Length per URL via HEAD requests; 0 for any that fail."""
     import requests
 
-    def head(url: str) -> int:
+    def head(url: str) -> int | None:
         try:
             resp = requests.head(url, timeout=10, allow_redirects=True)
             if not resp.ok:
                 _LOGGER.debug("HEAD %s returned %s", url, resp.status_code)
-                return 0
+                return None
-            return int(resp.headers.get("content-length", 0))
+            return int(resp.headers.get("content-length", 0)) or None
-        except Exception as err:  # noqa: BLE001  # pylint: disable=broad-exception-caught
+        except requests.RequestException as err:
             _LOGGER.debug("HEAD %s failed: %s", url, err)
-            return 0
+            return None
 
     with ThreadPoolExecutor(max_workers=min(_DOWNLOAD_WORKERS, len(urls))) as ex:
         return list(ex.map(head, urls))
@@ -938,12 +937,17 @@ def _prefetch_wave(
         if component.source.url in seen:
             continue
         seen.add(component.source.url)
-        with contextlib.suppress(Exception):
+        try:
-            if component.source.is_cached(
+            cached = component.source.is_cached(
                 component.get_sanitized_name(), salt=salt, namespace=namespace
-            ):
+            )
-                # A completed extraction downloads nothing; a warm build
+        except Exception as err:  # noqa: BLE001  # pylint: disable=broad-exception-caught
-                # must stay silent
+            # Best-effort: a failing probe prefetches (and re-downloads)
+            _LOGGER.debug("Cache probe for %s failed: %s", component.name, err)
+            cached = False
+        if cached:
+            # A completed extraction downloads nothing; a warm build must
+            # stay silent
             continue
         components.append(component)
     if len(components) < 2:
@@ -953,12 +957,13 @@ def _prefetch_wave(
         len(components),
         ", ".join(c.name for c in components),
     )
-    # One combined bar over the batch; sizes come from HEAD requests so the
+    # One combined bar over the batch, sized by HEAD requests. An unknown
-    # bar can be trusted (no sizes -> no bar, per BatchDownloadProgress)
+    # size would mean a silent multi-MB download; fall back to sequential
+    # downloads with their per-file bars instead.
     sizes = _content_lengths([c.source.url for c in components])
-    progress = BatchDownloadProgress(
+    if not all(sizes):
-        "Downloading libraries", sum(sizes) if all(sizes) else 0
+        return
-    )
+    progress = BatchDownloadProgress("Downloading libraries", sum(sizes))
 
     def _fetch(component: ConvertedLibrary) -> None:
         tracker = progress.tracker()

script/determine-jobs.py

@@ -535,6 +535,7 @@ ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers
 ESP_IDF_INFRA_TRIGGER_FILES = frozenset(
     {
         "esphome/build_gen/espidf.py",
+        "esphome/framework_helpers.py",
         "esphome/platformio/library.py",
         "esphome/platformio/extra_script.py",
     }

tests/unit_tests/test_platformio_library.py

@@ -611,6 +611,7 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel(
             raise RuntimeError("boom")
 
     monkeypatch.setattr(ConvertedLibrary, "download", fake_download)
+    monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls))
     wave = [
         ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))),
         ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),
@@ -643,10 +644,12 @@ def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(
         lib.requests if hasattr(lib, "requests") else requests, "head", fake_head
     )
+    # None marks an unknown size (probe failure or non-2xx), distinct
+    # from a genuine zero
     assert lib._content_lengths(["https://x/a", "https://x/bad", "https://x/gone"]) == [
         123,
-        0,
+        None,
-        0,
+        None,
     ]
 
 
@@ -665,6 +668,7 @@ def test_prefetch_wave_cache_probe_failure_still_prefetches(
         "is_cached",
         lambda self, *a, **kw: (_ for _ in ()).throw(RuntimeError("no core")),
     )
+    monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls))
     wave = [
         ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))),
         ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),