From f2ff1cc9869066246290fb3049bb96702794def9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 22 Aug 2026 16:10:03 -0500 Subject: [PATCH] Keep per-file bars when sizes are unknown; harden the prefetch probes An unknown Content-Length (probe failure, non-2xx, or a CDN omitting it on HEAD) now skips the parallel prefetch so the sequential downloads keep their per-file bars, instead of a silent multi-MB batch; the HEAD helper returns None for unknown, distinct from a genuine zero, and catches only RequestException. The cache probe logs its failure at debug. The espidf tool prefetch reads sizes with .get so a missing size only drops the bar instead of aborting the resumable prefetch. The idedata warning logs its traceback at debug for parser regressions, and framework_helpers joins the ESP-IDF infra CI triggers it is imported by. --- esphome/__main__.py | 1 + esphome/espidf/framework.py | 6 ++-- esphome/platformio/library.py | 39 ++++++++++++--------- script/determine-jobs.py | 1 + tests/unit_tests/test_platformio_library.py | 8 +++-- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 17e636c417..f8896087f9 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -870,6 +870,7 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: # Broad on purpose: the firmware already built; an idedata # failure must not fail a successful build. _LOGGER.warning("Could not generate idedata: %s", err) + _LOGGER.debug("Idedata failure detail", exc_info=True) else: from esphome.platformio import toolchain diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index ee46364abf..13d99fd709 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -750,7 +750,7 @@ def _prefetch_idf_tool_archives( # tools.json always carries sizes; should one be missing the combined # bar could not be trusted, so show no bar at all (per-file bars from # several threads would interleave) rather than a wrong one. - sizes = [entry["size"] for entry in entries] + sizes = [entry.get("size") or 0 for entry in entries] progress = BatchDownloadProgress( "Downloading ESP-IDF tools", sum(sizes) if all(sizes) else 0 ) @@ -764,8 +764,8 @@ def _prefetch_idf_tool_archives( download_with_resume( entry["url"], dist_path / entry["dest"], - sha256=entry["sha256"], - size=entry["size"], + sha256=entry.get("sha256"), + size=entry.get("size"), progress=tracker, ) except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 537a8224ea..47c8d70fd6 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -15,7 +15,6 @@ regardless of which toolchain consumes the result. from collections import deque from collections.abc import Callable, Iterable from concurrent.futures import ThreadPoolExecutor -import contextlib from dataclasses import dataclass, field import glob import hashlib @@ -905,16 +904,16 @@ def _content_lengths(urls: list[str]) -> list[int]: """Content-Length per URL via HEAD requests; 0 for any that fail.""" import requests - def head(url: str) -> int: + def head(url: str) -> int | None: try: resp = requests.head(url, timeout=10, allow_redirects=True) if not resp.ok: _LOGGER.debug("HEAD %s returned %s", url, resp.status_code) - return 0 - return int(resp.headers.get("content-length", 0)) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + return None + return int(resp.headers.get("content-length", 0)) or None + except requests.RequestException as err: _LOGGER.debug("HEAD %s failed: %s", url, err) - return 0 + return None with ThreadPoolExecutor(max_workers=min(_DOWNLOAD_WORKERS, len(urls))) as ex: return list(ex.map(head, urls)) @@ -938,13 +937,18 @@ def _prefetch_wave( if component.source.url in seen: continue seen.add(component.source.url) - with contextlib.suppress(Exception): - if component.source.is_cached( + try: + cached = component.source.is_cached( component.get_sanitized_name(), salt=salt, namespace=namespace - ): - # A completed extraction downloads nothing; a warm build - # must stay silent - continue + ) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # Best-effort: a failing probe prefetches (and re-downloads) + _LOGGER.debug("Cache probe for %s failed: %s", component.name, err) + cached = False + if cached: + # A completed extraction downloads nothing; a warm build must + # stay silent + continue components.append(component) if len(components) < 2: return @@ -953,12 +957,13 @@ def _prefetch_wave( len(components), ", ".join(c.name for c in components), ) - # One combined bar over the batch; sizes come from HEAD requests so the - # bar can be trusted (no sizes -> no bar, per BatchDownloadProgress) + # One combined bar over the batch, sized by HEAD requests. An unknown + # size would mean a silent multi-MB download; fall back to sequential + # downloads with their per-file bars instead. sizes = _content_lengths([c.source.url for c in components]) - progress = BatchDownloadProgress( - "Downloading libraries", sum(sizes) if all(sizes) else 0 - ) + if not all(sizes): + return + progress = BatchDownloadProgress("Downloading libraries", sum(sizes)) def _fetch(component: ConvertedLibrary) -> None: tracker = progress.tracker() diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 4e97275563..3e11deeb9a 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -535,6 +535,7 @@ ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers ESP_IDF_INFRA_TRIGGER_FILES = frozenset( { "esphome/build_gen/espidf.py", + "esphome/framework_helpers.py", "esphome/platformio/library.py", "esphome/platformio/extra_script.py", } diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 1e4f91827c..b6f87bca88 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -611,6 +611,7 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( raise RuntimeError("boom") monkeypatch.setattr(ConvertedLibrary, "download", fake_download) + monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls)) wave = [ ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))), ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))), @@ -643,10 +644,12 @@ def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( lib.requests if hasattr(lib, "requests") else requests, "head", fake_head ) + # None marks an unknown size (probe failure or non-2xx), distinct + # from a genuine zero assert lib._content_lengths(["https://x/a", "https://x/bad", "https://x/gone"]) == [ 123, - 0, - 0, + None, + None, ] @@ -665,6 +668,7 @@ def test_prefetch_wave_cache_probe_failure_still_prefetches( "is_cached", lambda self, *a, **kw: (_ for _ in ()).throw(RuntimeError("no core")), ) + monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls)) wave = [ ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))), ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),