diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 3418aa4597..e6601eb767 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -735,11 +735,22 @@ def _prefetch_idf_tool_archives( ) return dist_path = get_idf_tools_path() / "dist" - entries = [ + pending = [ entry for entry in json.loads(stdout) if not (dist_path / entry["dest"]).is_file() ] + # tools.json always carries sha256 and size; an entry missing either + # must not be downloaded unverified here, so leave it to the + # installer (which fails loudly on a bad archive). + entries = [e for e in pending if e.get("sha256") and e.get("size")] + for entry in pending: + if entry not in entries: + _LOGGER.warning( + "Tool %s has no sha256/size in the download list; " + "leaving it to the installer", + entry["name"], + ) if not entries: return _LOGGER.info( @@ -747,14 +758,12 @@ def _prefetch_idf_tool_archives( len(entries), ", ".join(entry["name"] for entry in entries), ) - # tools.json always carries sizes; should one be missing the combined - # bar could not be trusted, so show no bar at all rather than a wrong - # one. Unlike the library prefetch there is no sequential fallback: - # per-file bars from several threads would interleave, and skipping - # the prefetch would lose the resume workaround for #17703. - sizes = [entry.get("size") or 0 for entry in entries] + # Every entry carries a size (checked above), so the combined bar can + # be trusted. Unlike the library prefetch there is no sequential + # fallback: per-file bars from several threads would interleave, and + # skipping the prefetch would lose the resume workaround for #17703. progress = BatchDownloadProgress( - "Downloading ESP-IDF tools", sum(sizes) if all(sizes) else 0 + "Downloading ESP-IDF tools", sum(entry["size"] for entry in entries) ) # Reported after the bar is done so the warnings do not land on # its row; list.append is atomic under the GIL. @@ -766,8 +775,8 @@ def _prefetch_idf_tool_archives( download_with_resume( entry["url"], dist_path / entry["dest"], - sha256=entry.get("sha256"), - size=entry.get("size"), + sha256=entry["sha256"], + size=entry["size"], progress=tracker, ) except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index d7e29d152e..140975a930 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -911,7 +911,7 @@ def _content_lengths(urls: list[str]) -> list[int | None]: _LOGGER.debug("HEAD %s returned %s", url, resp.status_code) return None return int(resp.headers.get("content-length", 0)) or None - except requests.RequestException as err: + except (requests.RequestException, ValueError) as err: _LOGGER.debug("HEAD %s failed: %s", url, err) return None diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index a453f6eece..70091fbf13 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -888,6 +888,58 @@ _PREFETCH_JSON = json.dumps( ) +def test_prefetch_leaves_unverifiable_entries_to_the_installer( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An entry missing sha256 or size must not download unverified; the + installer handles it and fails loudly on a bad archive.""" + entries = json.loads(_PREFETCH_JSON) + del entries[0]["sha256"] + del entries[1]["size"] + entries.append( + { + "name": "gcc@14.2.0", + "url": "https://example.com/gcc.tar.gz", + "size": 67, + "sha256": "ef" * 32, + "dest": "gcc.tar.gz", + } + ) + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, json.dumps(entries), ""), + ), + patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls, + ): + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + assert [call[0][0] for call in download.call_args_list] == [ + "https://example.com/gcc.tar.gz" + ] + assert download.call_args[1]["sha256"] == "ef" * 32 + progress_cls.assert_called_once_with("Downloading ESP-IDF tools", 67) + assert "cmake@3.30.2 has no sha256/size" in caplog.text + assert "ninja@1.12.1 has no sha256/size" in caplog.text + + +def test_prefetch_all_entries_unverifiable_is_a_noop(tmp_path: Path) -> None: + entries = json.loads(_PREFETCH_JSON) + for entry in entries: + del entry["sha256"] + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, json.dumps(entries), ""), + ), + patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + ): + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + download.assert_not_called() + + def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None: with ( patch( diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index f832d444cf..dfd8729bb3 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -657,6 +657,10 @@ def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None: raise requests.ConnectionError("down") if "gone" in url: return SimpleNamespace(ok=False, status_code=404, headers={}) + if "garbage" in url: + # A proxy/CDN doubling the header ("123, 123") or emitting junk + # must degrade to unknown, not ValueError the build + return SimpleNamespace(ok=True, headers={"content-length": "123, 123"}) return SimpleNamespace(ok=True, headers={"content-length": "123"}) monkeypatch.setattr( @@ -664,10 +668,13 @@ def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None: ) # None marks an unknown size (probe failure or non-2xx), distinct # from a genuine zero - assert lib._content_lengths(["https://x/a", "https://x/bad", "https://x/gone"]) == [ + assert lib._content_lengths( + ["https://x/a", "https://x/bad", "https://x/gone", "https://x/garbage"] + ) == [ 123, None, None, + None, ]