diff --git a/esphome/__main__.py b/esphome/__main__.py index 5dc9403ed9..acb7370bab 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -874,6 +874,7 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: # Broad on purpose: the firmware already built; an idedata # failure must not fail a successful build. _LOGGER.warning("Could not generate idedata: %s", err) + _LOGGER.debug("Idedata failure detail", exc_info=True) elif CORE.using_native_toolchain: raise EsphomeError( f"Toolchain '{CORE.toolchain.value}' resolved but no platform " diff --git a/esphome/arduino8266/toolchain.py b/esphome/arduino8266/toolchain.py index 76cf415748..947f75985f 100644 --- a/esphome/arduino8266/toolchain.py +++ b/esphome/arduino8266/toolchain.py @@ -124,6 +124,7 @@ def run_compile(config: ConfigType, verbose: bool) -> int: # Broad on purpose: idedata is a bonus artifact; nothing here may # fail a successful build. _LOGGER.warning("Could not generate idedata: %s", err) + _LOGGER.debug("Idedata failure detail", exc_info=True) else: if idedata is None: _LOGGER.warning( diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index badd174421..a4d48f8f84 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -742,7 +742,7 @@ def _prefetch_idf_tool_archives( # tools.json always carries sizes; should one be missing the combined # bar could not be trusted, so show no bar at all (per-file bars from # several threads would interleave) rather than a wrong one. - sizes = [entry["size"] for entry in entries] + sizes = [entry.get("size") or 0 for entry in entries] progress = BatchDownloadProgress( "Downloading ESP-IDF tools", sum(sizes) if all(sizes) else 0 ) @@ -756,8 +756,8 @@ def _prefetch_idf_tool_archives( download_with_resume( entry["url"], dist_path / entry["dest"], - sha256=entry["sha256"], - size=entry["size"], + sha256=entry.get("sha256"), + size=entry.get("size"), progress=tracker, ) except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index ce93943dae..10c61d98eb 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -15,7 +15,6 @@ regardless of which toolchain consumes the result. from collections import deque from collections.abc import Callable, Iterable from concurrent.futures import ThreadPoolExecutor -import contextlib from dataclasses import dataclass, field import glob import hashlib @@ -951,16 +950,16 @@ def _content_lengths(urls: list[str]) -> list[int]: """Content-Length per URL via HEAD requests; 0 for any that fail.""" import requests - def head(url: str) -> int: + def head(url: str) -> int | None: try: resp = requests.head(url, timeout=10, allow_redirects=True) if not resp.ok: _LOGGER.debug("HEAD %s returned %s", url, resp.status_code) - return 0 - return int(resp.headers.get("content-length", 0)) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + return None + return int(resp.headers.get("content-length", 0)) or None + except requests.RequestException as err: _LOGGER.debug("HEAD %s failed: %s", url, err) - return 0 + return None with ThreadPoolExecutor(max_workers=min(_DOWNLOAD_WORKERS, len(urls))) as ex: return list(ex.map(head, urls)) @@ -984,13 +983,18 @@ def _prefetch_wave( if component.source.url in seen: continue seen.add(component.source.url) - with contextlib.suppress(Exception): - if component.source.is_cached( + try: + cached = component.source.is_cached( component.get_sanitized_name(), salt=salt, namespace=namespace - ): - # A completed extraction downloads nothing; a warm build - # must stay silent - continue + ) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # Best-effort: a failing probe prefetches (and re-downloads) + _LOGGER.debug("Cache probe for %s failed: %s", component.name, err) + cached = False + if cached: + # A completed extraction downloads nothing; a warm build must + # stay silent + continue components.append(component) if len(components) < 2: return @@ -999,12 +1003,13 @@ def _prefetch_wave( len(components), ", ".join(c.name for c in components), ) - # One combined bar over the batch; sizes come from HEAD requests so the - # bar can be trusted (no sizes -> no bar, per BatchDownloadProgress) + # One combined bar over the batch, sized by HEAD requests. An unknown + # size would mean a silent multi-MB download; fall back to sequential + # downloads with their per-file bars instead. sizes = _content_lengths([c.source.url for c in components]) - progress = BatchDownloadProgress( - "Downloading libraries", sum(sizes) if all(sizes) else 0 - ) + if not all(sizes): + return + progress = BatchDownloadProgress("Downloading libraries", sum(sizes)) def _fetch(component: ConvertedLibrary) -> None: tracker = progress.tracker() diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 8fd541a6d3..3dc7737d9a 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -548,6 +548,7 @@ ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers ESP_IDF_INFRA_TRIGGER_FILES = frozenset( { "esphome/build_gen/espidf.py", + "esphome/framework_helpers.py", "esphome/platformio/library.py", "esphome/platformio/extra_script.py", } diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index e36a7fa353..3f952c26ae 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -645,6 +645,7 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( raise RuntimeError("boom") monkeypatch.setattr(ConvertedLibrary, "download", fake_download) + monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls)) wave = [ ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))), ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))), @@ -677,10 +678,12 @@ def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( lib.requests if hasattr(lib, "requests") else requests, "head", fake_head ) + # None marks an unknown size (probe failure or non-2xx), distinct + # from a genuine zero assert lib._content_lengths(["https://x/a", "https://x/bad", "https://x/gone"]) == [ 123, - 0, - 0, + None, + None, ] @@ -699,6 +702,7 @@ def test_prefetch_wave_cache_probe_failure_still_prefetches( "is_cached", lambda self, *a, **kw: (_ for _ in ()).throw(RuntimeError("no core")), ) + monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls)) wave = [ ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))), ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),