Merge branch 'esp8266-native-shared-helpers' into esp8266-native-build-surgery

esphome/espidf/framework.py

@@ -735,11 +735,22 @@ def _prefetch_idf_tool_archives(
             )
             return
         dist_path = get_idf_tools_path() / "dist"
-        entries = [
+        pending = [
             entry
             for entry in json.loads(stdout)
             if not (dist_path / entry["dest"]).is_file()
         ]
+        # tools.json always carries sha256 and size; an entry missing either
+        # must not be downloaded unverified here, so leave it to the
+        # installer (which fails loudly on a bad archive).
+        entries = [e for e in pending if e.get("sha256") and e.get("size")]
+        for entry in pending:
+            if entry not in entries:
+                _LOGGER.warning(
+                    "Tool %s has no sha256/size in the download list; "
+                    "leaving it to the installer",
+                    entry["name"],
+                )
         if not entries:
             return
         _LOGGER.info(
@@ -747,14 +758,12 @@ def _prefetch_idf_tool_archives(
             len(entries),
             ", ".join(entry["name"] for entry in entries),
         )
-        # tools.json always carries sizes; should one be missing the combined
+        # Every entry carries a size (checked above), so the combined bar can
-        # bar could not be trusted, so show no bar at all rather than a wrong
+        # be trusted. Unlike the library prefetch there is no sequential
-        # one. Unlike the library prefetch there is no sequential fallback:
+        # fallback: per-file bars from several threads would interleave, and
-        # per-file bars from several threads would interleave, and skipping
+        # skipping the prefetch would lose the resume workaround for #17703.
-        # the prefetch would lose the resume workaround for #17703.
-        sizes = [entry.get("size") or 0 for entry in entries]
         progress = BatchDownloadProgress(
-            "Downloading ESP-IDF tools", sum(sizes) if all(sizes) else 0
+            "Downloading ESP-IDF tools", sum(entry["size"] for entry in entries)
         )
         # Reported after the bar is done so the warnings do not land on
         # its row; list.append is atomic under the GIL.
@@ -766,8 +775,8 @@ def _prefetch_idf_tool_archives(
                 download_with_resume(
                     entry["url"],
                     dist_path / entry["dest"],
-                    sha256=entry.get("sha256"),
+                    sha256=entry["sha256"],
-                    size=entry.get("size"),
+                    size=entry["size"],
                     progress=tracker,
                 )
             except Exception as e:  # noqa: BLE001  # pylint: disable=broad-exception-caught

esphome/platformio/library.py

@@ -911,7 +911,7 @@ def _content_lengths(urls: list[str]) -> list[int | None]:
                 _LOGGER.debug("HEAD %s returned %s", url, resp.status_code)
                 return None
             return int(resp.headers.get("content-length", 0)) or None
-        except requests.RequestException as err:
+        except (requests.RequestException, ValueError) as err:
             _LOGGER.debug("HEAD %s failed: %s", url, err)
             return None
 

tests/unit_tests/test_espidf_framework.py

@@ -888,6 +888,58 @@ _PREFETCH_JSON = json.dumps(
 )
 
 
+def test_prefetch_leaves_unverifiable_entries_to_the_installer(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """An entry missing sha256 or size must not download unverified; the
+    installer handles it and fails loudly on a bad archive."""
+    entries = json.loads(_PREFETCH_JSON)
+    del entries[0]["sha256"]
+    del entries[1]["size"]
+    entries.append(
+        {
+            "name": "gcc@14.2.0",
+            "url": "https://example.com/gcc.tar.gz",
+            "size": 67,
+            "sha256": "ef" * 32,
+            "dest": "gcc.tar.gz",
+        }
+    )
+    with (
+        patch(
+            "esphome.espidf.framework.run_command",
+            return_value=(True, json.dumps(entries), ""),
+        ),
+        patch("esphome.espidf.framework.download_with_resume") as download,
+        patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
+        patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls,
+    ):
+        _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
+    assert [call[0][0] for call in download.call_args_list] == [
+        "https://example.com/gcc.tar.gz"
+    ]
+    assert download.call_args[1]["sha256"] == "ef" * 32
+    progress_cls.assert_called_once_with("Downloading ESP-IDF tools", 67)
+    assert "cmake@3.30.2 has no sha256/size" in caplog.text
+    assert "ninja@1.12.1 has no sha256/size" in caplog.text
+
+
+def test_prefetch_all_entries_unverifiable_is_a_noop(tmp_path: Path) -> None:
+    entries = json.loads(_PREFETCH_JSON)
+    for entry in entries:
+        del entry["sha256"]
+    with (
+        patch(
+            "esphome.espidf.framework.run_command",
+            return_value=(True, json.dumps(entries), ""),
+        ),
+        patch("esphome.espidf.framework.download_with_resume") as download,
+        patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
+    ):
+        _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
+    download.assert_not_called()
+
+
 def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None:
     with (
         patch(

tests/unit_tests/test_platformio_library.py

@@ -657,6 +657,10 @@ def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None:
             raise requests.ConnectionError("down")
         if "gone" in url:
             return SimpleNamespace(ok=False, status_code=404, headers={})
+        if "garbage" in url:
+            # A proxy/CDN doubling the header ("123, 123") or emitting junk
+            # must degrade to unknown, not ValueError the build
+            return SimpleNamespace(ok=True, headers={"content-length": "123, 123"})
         return SimpleNamespace(ok=True, headers={"content-length": "123"})
 
     monkeypatch.setattr(
@@ -664,10 +668,13 @@ def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None:
     )
     # None marks an unknown size (probe failure or non-2xx), distinct
     # from a genuine zero
-    assert lib._content_lengths(["https://x/a", "https://x/bad", "https://x/gone"]) == [
+    assert lib._content_lengths(
+        ["https://x/a", "https://x/bad", "https://x/gone", "https://x/garbage"]
+    ) == [
         123,
         None,
         None,
+        None,
     ]