Note why SSLError stays transient

2026.8.0

docker/Dockerfile

@@ -22,7 +22,7 @@ RUN \
     -r /requirements.txt
 
 # Install the ESPHome Device Builder dashboard.
-RUN uv pip install --no-cache-dir esphome-device-builder==1.12.0
+RUN uv pip install --no-cache-dir esphome-device-builder==1.12.1
 
 RUN \
     platformio settings set enable_telemetry No \

esphome/components/wifi/wifi_component_esp_idf.cpp

@@ -580,7 +580,14 @@ bool WiFiComponent::wifi_sta_ip_config_(const optional<ManualIP> &manual_ip) {
     // lwIP starts the SNTP client if it gets an SNTP server from DHCP. We don't need the time, and more importantly,
     // the built-in SNTP client has a memory leak in certain situations. Disable this feature.
     // https://github.com/esphome/issues/issues/2299
-    sntp_servermode_dhcp(false);
+    {
+#if SNTP_GET_SERVERS_FROM_DHCP || SNTP_GET_SERVERS_FROM_DHCPV6
+      // sntp_servermode_dhcp() is an empty macro unless lwIP is built with
+      // DHCP-supplied NTP servers, so only that build needs the core lock.
+      LwIPLock lock;
+#endif
+      sntp_servermode_dhcp(false);
+    }
 
     // No manual IP is set; use DHCP client
     if (dhcp_status != ESP_NETIF_DHCP_STARTED) {

esphome/espidf/framework.py

@@ -1,7 +1,6 @@
 """ESP-IDF framework tools for ESPHome."""
 
 from collections.abc import Callable
-from concurrent.futures import ThreadPoolExecutor
 from ctypes.util import find_library
 import json
 import logging
@@ -16,7 +15,6 @@ import platformdirs
 
 from esphome.core import CORE, Version
 from esphome.framework_helpers import (
-    BatchDownloadProgress,
     PathType,
     archive_extract_all,
     create_venv,
@@ -692,12 +690,6 @@ def _patch_tools_json_demote_unused_tools(framework_path: Path) -> None:
     )
 
 
-# Tool archives are large (tens to hundreds of MB) and served by GitHub /
-# dl.espressif.com; a few streams at once saturate most links without
-# hammering the host. Smaller than external_files' 8: those are tiny files.
-_PREFETCH_WORKERS = 4
-
-
 def _prefetch_idf_tool_archives(
     framework_path: Path,
     targets_str: str,
@@ -710,10 +702,10 @@ def _prefetch_idf_tool_archives(
     which makes large archives effectively impossible to fetch on unstable
     connections (#17703). This asks the framework's idf_tools (via
     ``get_tool_downloads.py``) which archives the coming install needs, then
-    downloads them into ``<IDF_TOOLS_PATH>/dist`` with
-    ``download_with_resume``, a few at a time under one combined progress
-    bar. The installer then finds the verified archives already in place
-    ("file ... is already downloaded") and never touches the network.
+    downloads each into ``<IDF_TOOLS_PATH>/dist`` with
+    ``download_with_resume``. The installer then finds the verified archives
+    already in place ("file ... is already downloaded") and never touches the
+    network.
 
     Strictly best-effort: any failure here just logs and returns, leaving
     ``idf_tools.py install`` to download whatever is missing exactly as
@@ -740,51 +732,21 @@ def _prefetch_idf_tool_archives(
             for entry in json.loads(stdout)
             if not (dist_path / entry["dest"]).is_file()
         ]
-        if not entries:
-            return
-        _LOGGER.info(
-            "Downloading %d ESP-IDF tool archive(s): %s",
-            len(entries),
-            ", ".join(entry["name"] for entry in entries),
-        )
-        # tools.json always carries sizes; should one be missing the combined
-        # bar could not be trusted, so show no bar at all (per-file bars from
-        # several threads would interleave) rather than a wrong one.
-        sizes = [entry["size"] for entry in entries]
-        progress = BatchDownloadProgress(
-            "Downloading ESP-IDF tools", sum(sizes) if all(sizes) else 0
-        )
-        # Reported after the bar is done so the warnings do not land on
-        # its row; list.append is atomic under the GIL.
-        failures: list[tuple[str, Exception]] = []
-
-        def _download(entry: dict) -> None:
-            tracker = progress.tracker()
+        for index, entry in enumerate(entries, start=1):
+            _LOGGER.info(
+                "Downloading %s (%d/%d) ...", entry["name"], index, len(entries)
+            )
             try:
                 download_with_resume(
                     entry["url"],
                     dist_path / entry["dest"],
                     sha256=entry["sha256"],
                     size=entry["size"],
-                    progress=tracker,
                 )
             except Exception as e:  # noqa: BLE001  # pylint: disable=broad-exception-caught
                 # Keep prefetching the remaining archives; the installer
                 # will retry this one itself (without resume).
-                tracker(0)
-                failures.append((entry["name"], e))
-
-        ex = ThreadPoolExecutor(max_workers=min(_PREFETCH_WORKERS, len(entries)))
-        try:
-            for future in [ex.submit(_download, entry) for entry in entries]:
-                future.result()
-        finally:
-            # On Ctrl-C drop the queued archives instead of downloading them
-            # all before the process can exit; in-flight ones still finish.
-            ex.shutdown(wait=True, cancel_futures=True)
-            progress.done()
-        for name, e in failures:
-            _LOGGER.warning("Could not prefetch %s: %s", name, e)
+                _LOGGER.warning("Could not prefetch %s: %s", entry["name"], e)
     except Exception as e:  # noqa: BLE001  # pylint: disable=broad-exception-caught
         # The installer downloads anything missing itself; never let the
         # prefetch become a new way for the install to fail.

esphome/external_files.py

@@ -16,6 +16,7 @@ from esphome.const import CONF_FILE, CONF_TYPE, CONF_URL, __version__
 from esphome.core import CORE, EsphomeError, TimePeriodSeconds
 from esphome.happy_eyeballs import ensure_happy_eyeballs
 from esphome.helpers import write_file
+from esphome.net_retry import fetch_with_retry
 from esphome.types import ConfigType
 
 _LOGGER = logging.getLogger(__name__)
@@ -157,8 +158,17 @@ def has_remote_file_changed(
             }
             if etag := _read_etag(local_file_path):
                 headers[IF_NONE_MATCH] = etag
-            response = requests.head(
-                url, headers=headers, timeout=timeout, allow_redirects=True
+            # Retried so allow_stale=False consumers don't hard-fail on a
+            # healed flake. Only connection-level failures retry: HEAD
+            # never raises on HTTP status (servers rejecting HEAD with
+            # 405/501 must fall through to the GET), so 5xx is handled by
+            # the GET's own retry.
+            response = fetch_with_retry(
+                url,
+                lambda: requests.head(
+                    url, headers=headers, timeout=timeout, allow_redirects=True
+                ),
+                what="Revalidation",
             )
 
             _LOGGER.debug(
@@ -293,7 +303,7 @@ def download_content(
     _LOGGER.info("Downloading %s", url)
     _LOGGER.debug("Saving to %s", path)
 
-    try:
+    def _fetch() -> tuple[requests.Response, bytes]:
         req = requests.get(
             url,
             timeout=timeout,
@@ -304,7 +314,10 @@ def download_content(
         # and mid-stream connection errors all surface here as
         # RequestException subclasses, so this needs the same fall-back
         # treatment as the request itself.
-        data = req.content
+        return req, req.content
+
+    try:
+        req, data = fetch_with_retry(url, _fetch)
     except requests.exceptions.RequestException as e:
         if path.exists():
             # Memoized so a flaky host warns once per run, not per consumer.

esphome/framework_helpers.py

@@ -1,6 +1,6 @@
 """Generic toolchain installation helpers shared across framework implementations."""
 
-from collections.abc import Callable, Iterable
+from collections.abc import Iterable
 from contextlib import ExitStack
 import hashlib
 import io
@@ -10,12 +10,12 @@ import os
 from pathlib import Path
 import subprocess
 import sys
-import threading
 import time
 from typing import IO, TYPE_CHECKING
 
 from esphome.happy_eyeballs import ensure_happy_eyeballs
 from esphome.helpers import ProgressBar, rmtree
+from esphome.net_retry import NETWORK_MAX_ATTEMPTS, is_transient_download_error
 
 if TYPE_CHECKING:
     import requests
@@ -30,8 +30,9 @@ _LOGGER = logging.getLogger(__name__)
 _MIRROR_ATTEMPTS = 3
 
 # Passes over the whole mirror list when a transient network error is in
-# the mix; matches git.py's _NETWORK_MAX_ATTEMPTS (3 tries, 2s/4s backoff).
-_MIRROR_SWEEP_ATTEMPTS = 3
+# the mix; shares net_retry's policy (3 tries, 2s/4s backoff), which in
+# turn matches git.py's _NETWORK_MAX_ATTEMPTS.
+_MIRROR_SWEEP_ATTEMPTS = NETWORK_MAX_ATTEMPTS
 
 
 def get_project_link_flags() -> list[str]:
@@ -698,11 +699,7 @@ def _response_validator(resp: "requests.Response") -> str | None:
 
 
 def _stream_response_to_file(
-    resp: "requests.Response",
-    f: IO[bytes],
-    offset: int,
-    size: int | None = None,
-    progress: Callable[[int], None] | None = None,
+    resp: "requests.Response", f: IO[bytes], offset: int, size: int | None = None
 ) -> None:
     """Stream an open ``_open_ranged`` response body into ``f`` at ``offset``.
 
@@ -710,72 +707,21 @@ def _stream_response_to_file(
     (effective offset 0) discards the stale bytes. ``offset`` also seeds the
     progress bar so a resumed download shows overall progress. ``size`` is
     the known full file size; when None it is derived from the response's
-    content-length, and without either there is no progress bar. With
-    ``progress`` set, no bar is drawn here; the callback gets the absolute
-    byte count, seeded with ``offset`` and then after each chunk.
+    content-length, and without either there is no progress bar.
     """
     f.seek(offset)
     f.truncate(offset)
     total_size = size or offset + _content_length(resp)
     downloaded = offset
-    own_bar: ProgressBar | None = None
-    if progress is None:
-        own_bar = ProgressBar("Downloading") if total_size > 0 else None
-        progress = (
-            (lambda done: own_bar.update(done / total_size))
-            if own_bar
-            else (lambda _: None)
-        )
-    progress(downloaded)
+    progress = ProgressBar("Downloading") if total_size > 0 else None
     for chunk in resp.iter_content(chunk_size=256 * 1024):
         if chunk:
             f.write(chunk)
             downloaded += len(chunk)
-        progress(downloaded)
-    if own_bar is not None:
-        own_bar.update(1)
-
-
-class BatchDownloadProgress:
-    """One progress bar across several concurrent ``download_with_resume`` calls.
-
-    Each ``tracker()`` is a ``progress`` callback for one download; it reports
-    that file's absolute byte count and the bar shows the sum over ``total``.
-    The lock also serialises the bar's stderr writes, so worker threads never
-    interleave frames. With an unknown ``total`` (0) nothing is drawn. Call
-    ``done()`` once every download has finished (or failed) so a bar that
-    never reached 100% still ends its line before the next log message.
-    """
-
-    def __init__(self, header: str, total: int) -> None:
-        self._bar = ProgressBar(header) if total > 0 else None
-        self._total = total
-        self._sum = 0
-        self._lock = threading.Lock()
-
-    def tracker(self) -> Callable[[int], None]:
-        last = 0
-
-        def update(done: int) -> None:
-            nonlocal last
-            if self._bar is None:
-                return
-            with self._lock:
-                self._sum += done - last
-                last = done
-                self._bar.update(min(self._sum / self._total, 1))
-
-        return update
-
-    def done(self) -> None:
-        # Nothing to end unless a frame was drawn and it was not the final
-        # one (update(1) already emitted its own newline).
-        if (
-            self._bar is not None
-            and self._bar.last_progress is not None
-            and self._bar.last_progress != 100
-        ):
-            self._bar.done()
+        if progress is not None:
+            progress.update(downloaded / total_size)
+    if progress is not None:
+        progress.update(1)
 
 
 def download_with_resume(
@@ -788,7 +734,6 @@ def download_with_resume(
     attempts: int = 5,
     timeout: int = 30,
     retry_connect_errors: bool = True,
-    progress: Callable[[int], None] | None = None,
 ) -> None:
     """Download ``url`` to ``dest``, resuming partial downloads.
 
@@ -811,12 +756,6 @@ def download_with_resume(
     of consuming attempts — for callers with their own fallback, like
     ``download_from_mirrors``.
 
-    ``progress``, when given, replaces the built-in progress bar: it is called
-    with the absolute number of bytes of ``dest`` obtained so far (including
-    a resumed prefix, and the final size once the file is verified), so a
-    caller running several downloads at once can draw one combined bar (see
-    ``BatchDownloadProgress``).
-
     Raises EsphomeError when all attempts are exhausted.
     """
     # Imported lazily: requests is a heavy import (~85ms) and is only needed
@@ -840,8 +779,6 @@ def download_with_resume(
     if dest.is_file() and (sha256 is not None or size is not None):
         try:
             _verify_file(dest, sha256, size)
-            if progress is not None:
-                progress(size if size is not None else dest.stat().st_size)
             return
         except EsphomeError:
             dest.unlink()
@@ -887,7 +824,7 @@ def download_with_resume(
                             # Recorded so a later run can prove an If-Range
                             # resume of this part file safe.
                             _write_download_meta(meta, url, validator, expected_total)
-                        _stream_response_to_file(resp, f, offset, size, progress)
+                        _stream_response_to_file(resp, f, offset, size)
             # else: a previous run already wrote every byte (or more) but
             # was killed before the rename below. Skip the network entirely
             # — a Range request past EOF would draw HTTP 416 — and let
@@ -896,10 +833,6 @@ def download_with_resume(
 
             expected_size = size if size is not None else expected_total
             _verify_file(part, sha256, expected_size or None)
-            if progress is not None:
-                # Also credits a part file an earlier run completed without
-                # streaming anything this time.
-                progress(expected_size or part.stat().st_size)
             if not expected_size and sha256 is None:
                 # No sha, no size, and the server sent no usable
                 # content-length: nothing can prove the download complete
@@ -972,30 +905,6 @@ def _spent_attempts_error(e: Exception, attempts: int) -> Exception:
     return err
 
 
-def _is_transient_download_error(e: Exception) -> bool:
-    """Return True when a download failure is worth retrying.
-
-    Connection-level failures and HTTP 429/5xx are transient. Other HTTP
-    errors, local errors, and exhausted-attempts EsphomeError wrappers
-    (their per-mirror retries are already spent) are permanent.
-    """
-    # Imported lazily: requests is a heavy import (~85ms) and is only
-    # needed when actually downloading, never during config validation.
-    import requests
-
-    if isinstance(e, requests.exceptions.HTTPError):
-        resp = e.response
-        return resp is not None and (resp.status_code == 429 or resp.status_code >= 500)
-    return isinstance(
-        e,
-        (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.Timeout,
-            requests.exceptions.ChunkedEncodingError,
-        ),
-    )
-
-
 def _try_mirrors_once(
     urls: list[str],
     path_target: Path | None,
@@ -1200,7 +1109,7 @@ def download_from_mirrors(
         # Permanent failures (404, verification mismatch) won't heal;
         # only retry when a transient error is in the mix (as git.py does).
         transient = next(
-            ((u, e) for u, e in sweep_failures if _is_transient_download_error(e)),
+            ((u, e) for u, e in sweep_failures if is_transient_download_error(e)),
             None,
         )
         if transient is None:

esphome/net_retry.py

@@ -0,0 +1,114 @@
+"""Retry policy for HTTP downloads.
+
+Kept import-light on purpose: this module is imported at config time, so it
+must not pull in requests (a heavy import, ~85ms) at module scope.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+import logging
+import time
+
+_LOGGER = logging.getLogger(__name__)
+
+# 3 tries with 2s/4s backoff, matching git.py's _NETWORK_MAX_ATTEMPTS.
+# Callers memoize failures so a flaky host pays this once per file per run.
+NETWORK_MAX_ATTEMPTS = 3
+
+
+def _is_permanent_dns_failure(e: BaseException) -> bool:
+    """Whether a hard socket.gaierror hides in ``e``'s exception chain.
+
+    EAI_AGAIN (flaky resolver) stays retryable; anything else is permanent
+    so offline builds fall back to their cache without sleeping first.
+    Narrower than git.py, which retries NXDOMAIN too.
+
+    Walks ``__cause__``, ``args`` (requests wraps MaxRetryError without
+    ``from``) and MaxRetryError's ``reason``, but not implicit
+    ``__context__``: an unrelated earlier attempt's resolution failure
+    must not reclassify an error it did not cause.
+    """
+    import socket
+
+    seen: set[int] = set()
+    stack: list[BaseException] = [e]
+    while stack:
+        exc = stack.pop()
+        if id(exc) in seen:
+            continue
+        if (
+            isinstance(exc, socket.gaierror)
+            and exc.errno is not None
+            and exc.errno != socket.EAI_AGAIN
+        ):
+            return True
+        seen.add(id(exc))
+        stack.extend(
+            nxt
+            for nxt in (
+                exc.__cause__,
+                getattr(exc, "reason", None),  # urllib3 MaxRetryError
+                *exc.args,
+            )
+            if isinstance(nxt, BaseException)
+        )
+    return False
+
+
+def is_transient_download_error(e: Exception) -> bool:
+    """Return True when a download failure is worth retrying.
+
+    Connection-level failures and HTTP 429/5xx are transient; hard DNS
+    failures, other HTTP errors, and local errors are permanent.
+    """
+    # Imported lazily: requests is a heavy import (~85ms) and is only
+    # needed when actually downloading, never during config validation.
+    import requests
+
+    if isinstance(e, requests.exceptions.HTTPError):
+        resp = e.response
+        return resp is not None and (resp.status_code == 429 or resp.status_code >= 500)
+    if isinstance(e, requests.exceptions.ConnectionError) and _is_permanent_dns_failure(
+        e
+    ):
+        return False
+    # SSLError (a ConnectionError subclass) stays transient on purpose: it
+    # also covers mid-handshake connection drops, not just bad certificates.
+    return isinstance(
+        e,
+        (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.Timeout,
+            requests.exceptions.ChunkedEncodingError,
+            requests.exceptions.ContentDecodingError,
+        ),
+    )
+
+
+def fetch_with_retry[T](url: str, fetch: Callable[[], T], what: str = "Download") -> T:
+    """Run ``fetch``, retrying transient failures with 2s/4s backoff.
+
+    Permanent failures and the final attempt propagate to the caller;
+    ``what`` names the operation in the retry warning.
+    """
+    import requests
+
+    for attempt in range(1, NETWORK_MAX_ATTEMPTS):
+        try:
+            return fetch()
+        except requests.exceptions.RequestException as e:
+            if not is_transient_download_error(e):
+                raise
+            delay = 2**attempt
+            _LOGGER.warning(
+                "%s of %s failed: %s. Retrying in %d seconds... (attempt %d/%d)",
+                what,
+                url,
+                e,
+                delay,
+                attempt + 1,
+                NETWORK_MAX_ATTEMPTS,
+            )
+            time.sleep(delay)
+    return fetch()

tests/integration/conftest.py

@@ -60,7 +60,11 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
     env = os.environ.copy()
     env["PLATFORMIO_CORE_DIR"] = str(cache_dir)
     env["PLATFORMIO_CACHE_DIR"] = str(cache_dir / ".cache")
-    env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps")
+    # libdeps is keyed only by env name (the device name), and fixtures share
+    # names; two xdist workers first-compiling the same name race pio pkg
+    # install in the same directory. Keep libdeps per worker.
+    worker = os.environ.get("PYTEST_XDIST_WORKER", "master")
+    env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker)
     # Prevent cache cleaning during integration tests
     env["ESPHOME_SKIP_CLEAN_BUILD"] = "1"
     # Compile with THIS tree's esphome sources, not wherever the venv's editable

tests/unit_tests/test_espidf_framework.py

@@ -2,7 +2,6 @@
 
 # pylint: disable=protected-access
 
-from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 import importlib.util
 import io
@@ -15,7 +14,7 @@ import subprocess
 import sys
 import tarfile
 from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 
@@ -896,72 +895,16 @@ def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None:
         ),
         patch("esphome.espidf.framework.download_with_resume") as download,
         patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
-        patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls,
     ):
         _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
 
     dist = get_idf_tools_path() / "dist"
-    # Archives download concurrently, so the call order is not fixed.
-    calls = {call[0]: call[1] for call in download.call_args_list}
-    assert set(calls) == {
-        ("https://example.com/cmake.tar.gz", dist / "cmake-3.30.2.tar.gz"),
-        ("https://example.com/ninja.zip", dist / "ninja.zip"),
-    }
-    kwargs = calls[("https://example.com/cmake.tar.gz", dist / "cmake-3.30.2.tar.gz")]
-    assert kwargs["sha256"] == "ab" * 32
-    assert kwargs["size"] == 123
-    # every archive reports into the one combined progress bar
-    progress_cls.assert_called_once_with("Downloading ESP-IDF tools", 123 + 45)
-    tracker = progress_cls.return_value.tracker.return_value
-    assert all(kw["progress"] is tracker for kw in calls.values())
-
-
-def test_prefetch_downloads_archives_concurrently(tmp_path: Path) -> None:
-    """More than one archive fans out over a bounded thread pool."""
-    entries = [
-        {
-            "name": f"tool{i}@1",
-            "url": f"https://example.com/tool{i}.tar.gz",
-            "size": 10,
-            "sha256": "ab" * 32,
-            "dest": f"tool{i}.tar.gz",
-        }
-        for i in range(6)
-    ]
-    with (
-        patch(
-            "esphome.espidf.framework.run_command",
-            return_value=(True, json.dumps(entries), ""),
-        ),
-        patch("esphome.espidf.framework.download_with_resume") as download,
-        patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
-        patch(
-            "esphome.espidf.framework.ThreadPoolExecutor", wraps=ThreadPoolExecutor
-        ) as pool,
-    ):
-        _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
-
-    pool.assert_called_once_with(max_workers=4)
-    assert download.call_count == 6
-
-
-def test_prefetch_single_archive_uses_one_worker(tmp_path: Path) -> None:
-    entries = json.loads(_PREFETCH_JSON)[:1]
-    with (
-        patch(
-            "esphome.espidf.framework.run_command",
-            return_value=(True, json.dumps(entries), ""),
-        ),
-        patch("esphome.espidf.framework.download_with_resume") as download,
-        patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
-        patch(
-            "esphome.espidf.framework.ThreadPoolExecutor", wraps=ThreadPoolExecutor
-        ) as pool,
-    ):
-        _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
-
-    pool.assert_called_once_with(max_workers=1)
-    assert download.call_count == 1
+    assert download.call_count == 2
+    assert download.call_args_list[0][0] == (
+        "https://example.com/cmake.tar.gz",
+        dist / "cmake-3.30.2.tar.gz",
+    )
+    assert download.call_args_list[0][1] == {"sha256": "ab" * 32, "size": 123}
 
 
 def test_prefetch_skips_already_downloaded_archives(tmp_path: Path) -> None:
@@ -1021,11 +964,6 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest(
 ) -> None:
     """A single archive failing its download must not abort the prefetch of
     the remaining archives."""
-
-    def _fail_cmake_download(url: str, *args, **kwargs) -> None:
-        if "cmake" in url:
-            raise OSError("network down")
-
     with (
         patch(
             "esphome.espidf.framework.run_command",
@@ -1033,7 +971,7 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest(
         ),
         patch(
             "esphome.espidf.framework.download_with_resume",
-            side_effect=_fail_cmake_download,
+            side_effect=[OSError("network down"), None],
         ) as download,
         patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
     ):
@@ -1043,29 +981,6 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest(
     assert "Could not prefetch cmake@3.30.2" in caplog.text
 
 
-def test_prefetch_finishes_progress_bar_and_cancels_queue(tmp_path: Path) -> None:
-    """The batch bar is closed out after the pool, and the pool is shut down
-    with cancel_futures so Ctrl-C does not drain every queued archive."""
-    with (
-        patch(
-            "esphome.espidf.framework.run_command",
-            return_value=(True, _PREFETCH_JSON, ""),
-        ),
-        patch("esphome.espidf.framework.download_with_resume"),
-        patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
-        patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls,
-        patch(
-            "esphome.espidf.framework.ThreadPoolExecutor", wraps=ThreadPoolExecutor
-        ) as pool_cls,
-    ):
-        pool = MagicMock(wraps=ThreadPoolExecutor(max_workers=2))
-        pool_cls.return_value = pool
-        _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
-
-    pool.shutdown.assert_called_once_with(wait=True, cancel_futures=True)
-    progress_cls.return_value.done.assert_called_once_with()
-
-
 def test_prefetch_passes_targets_and_tools_to_script(tmp_path: Path) -> None:
     with (
         patch(

tests/unit_tests/test_external_files.py

@@ -4,7 +4,7 @@ import os
 from pathlib import Path
 import time
 from typing import Any
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, call, patch
 
 import pytest
 import requests
@@ -81,6 +81,15 @@ def mock_download_content_many() -> MagicMock:
         yield m
 
 
+@pytest.fixture
+def mock_retry_sleep() -> MagicMock:
+    """Patch the retry backoff sleep (process-wide; net_retry.time is the
+    global module) so transient-error tests don't really wait 2s/4s.
+    """
+    with patch("esphome.net_retry.time.sleep") as m:
+        yield m
+
+
 def test_compute_local_file_dir(setup_core: Path) -> None:
     """Test compute_local_file_dir creates and returns correct path."""
     domain = "font"
@@ -495,6 +504,7 @@ class _BodyReadErrorResponse:
 def test_download_content_with_body_read_error_uses_cache(
     mock_has_remote_file_changed: MagicMock,
     mock_requests_get: MagicMock,
+    mock_retry_sleep: MagicMock,
     setup_core: Path,
 ) -> None:
     """Body-read errors (chunked-decode/gzip-decode/mid-stream connection
@@ -519,6 +529,7 @@ def test_download_content_with_body_read_error_uses_cache(
 def test_download_content_with_body_read_error_no_cache_fails(
     mock_has_remote_file_changed: MagicMock,
     mock_requests_get: MagicMock,
+    mock_retry_sleep: MagicMock,
     setup_core: Path,
 ) -> None:
     """A body-read failure with no cache available must surface as a
@@ -535,6 +546,131 @@ def test_download_content_with_body_read_error_no_cache_fails(
         external_files.download_content("https://example.com/file.txt", test_file)
 
 
+def test_download_content_retries_transient_error_then_succeeds(
+    mock_has_remote_file_changed: MagicMock,
+    mock_requests_get: MagicMock,
+    mock_retry_sleep: MagicMock,
+    setup_core: Path,
+) -> None:
+    """Transient failures (connection reset, timeout) are retried with 2s/4s
+    backoff before giving up; a late success downloads normally."""
+    test_file = setup_core / "downloads" / "file.txt"
+    mock_has_remote_file_changed.return_value = True
+
+    ok = MagicMock()
+    ok.content = b"downloaded"
+    ok.headers = {}
+    mock_requests_get.side_effect = [
+        requests.exceptions.ConnectionError("reset by peer"),
+        requests.exceptions.Timeout("timed out"),
+        ok,
+    ]
+
+    result = external_files.download_content("https://example.com/file.txt", test_file)
+
+    assert result == b"downloaded"
+    assert test_file.read_bytes() == b"downloaded"
+    assert mock_retry_sleep.call_args_list == [call(2), call(4)]
+
+
+def test_download_content_transient_error_exhausts_attempts(
+    mock_has_remote_file_changed: MagicMock,
+    mock_requests_get: MagicMock,
+    mock_retry_sleep: MagicMock,
+    setup_core: Path,
+) -> None:
+    """A persistent transient failure gives up after three attempts and then
+    follows the normal no-cache error path."""
+    test_file = setup_core / "nonexistent.txt"
+    mock_has_remote_file_changed.return_value = True
+    mock_requests_get.side_effect = requests.exceptions.ConnectionError("reset by peer")
+
+    with pytest.raises(Invalid, match="Could not download from.*reset by peer"):
+        external_files.download_content("https://example.com/file.txt", test_file)
+
+    assert mock_retry_sleep.call_args_list == [call(2), call(4)]
+
+
+def test_download_content_non_transient_error_not_retried(
+    mock_has_remote_file_changed: MagicMock,
+    mock_requests_get: MagicMock,
+    mock_retry_sleep: MagicMock,
+    setup_core: Path,
+) -> None:
+    """Permanent failures like a 404 fail on the first attempt."""
+    test_file = setup_core / "nonexistent.txt"
+    mock_has_remote_file_changed.return_value = True
+
+    response = MagicMock()
+    response.status_code = 404
+    mock_requests_get.side_effect = requests.exceptions.HTTPError(
+        "404 Client Error", response=response
+    )
+
+    with pytest.raises(Invalid, match="Could not download from.*404"):
+        external_files.download_content("https://example.com/file.txt", test_file)
+
+    assert mock_requests_get.call_count == 1
+    mock_retry_sleep.assert_not_called()
+
+
+def test_download_content_retries_body_read_error(
+    mock_has_remote_file_changed: MagicMock,
+    mock_requests_get: MagicMock,
+    mock_retry_sleep: MagicMock,
+    setup_core: Path,
+) -> None:
+    """Mid-stream failures surfacing from `.content` are retried too."""
+    test_file = setup_core / "downloads" / "file.txt"
+    mock_has_remote_file_changed.return_value = True
+
+    ok = MagicMock()
+    ok.content = b"downloaded"
+    ok.headers = {}
+    mock_requests_get.side_effect = [
+        _BodyReadErrorResponse(
+            requests.exceptions.ChunkedEncodingError("body truncated")
+        ),
+        ok,
+    ]
+
+    result = external_files.download_content("https://example.com/file.txt", test_file)
+
+    assert result == b"downloaded"
+    assert mock_requests_get.call_count == 2
+    assert mock_retry_sleep.call_args_list == [call(2)]
+
+
+def test_has_remote_file_changed_retries_transient_error(
+    mock_requests_head: MagicMock,
+    mock_retry_sleep: MagicMock,
+    setup_core: Path,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A HEAD revalidation that fails transiently then returns 304 does not
+    mark the cached copy stale, and the retry warning names the operation."""
+    test_file = setup_core / "cached.txt"
+    test_file.write_bytes(b"cached content")
+
+    ok = MagicMock()
+    ok.status_code = 304
+    ok.headers = {}
+    mock_requests_head.side_effect = [
+        requests.exceptions.ConnectionError("reset by peer"),
+        ok,
+    ]
+
+    changed = external_files.has_remote_file_changed(
+        "https://example.com/file.txt", test_file
+    )
+
+    assert changed is False
+    assert test_file not in external_files._run_data().stale_paths
+    assert mock_requests_head.call_count == 2
+    assert mock_retry_sleep.call_args_list == [call(2)]
+    assert "Revalidation of" in caplog.text
+
+
 def test_download_content_skip_external_update_uses_cache(
     mock_has_remote_file_changed: MagicMock,
     mock_requests_get: MagicMock,

tests/unit_tests/test_framework_helpers.py

@@ -21,10 +21,8 @@ import requests as req
 from esphome import framework_helpers
 from esphome.core import EsphomeError
 from esphome.framework_helpers import (
-    BatchDownloadProgress,
     _7z_extract_all,
     _detect_archive_root,
-    _is_transient_download_error,
     _rename_with_retry,
     _tar_extract_all,
     _zip_extract_all,
@@ -1113,108 +1111,6 @@ class TestDownloadWithResume:
         assert mock_get.call_args[1]["headers"] == {}
         assert dest.read_bytes() == b"data"
 
-    def test_progress_callback_reports_absolute_bytes(self, tmp_path: Path) -> None:
-        """With a callback no bar is drawn; the callback sees the running
-        byte count of this file, then its final verified size."""
-        dest = tmp_path / "tool.tar.gz"
-        resp = _mock_response(b"")
-        resp.headers = {"content-length": "7"}
-        resp.iter_content.return_value = [b"1234", b"567"]
-        seen: list[int] = []
-        with (
-            patch("requests.get", return_value=resp),
-            patch("esphome.framework_helpers.ProgressBar") as bar,
-        ):
-            download_with_resume(
-                "https://example.com/t", dest, size=7, progress=seen.append
-            )
-        assert seen == [0, 4, 7, 7]
-        bar.assert_not_called()
-
-    def test_progress_callback_seeds_with_resume_offset(self, tmp_path: Path) -> None:
-        dest = tmp_path / "tool.tar.gz"
-        (tmp_path / "tool.tar.gz.part").write_bytes(b"12345")
-        good = hashlib.sha256(b"12345678").hexdigest()
-        seen: list[int] = []
-        with patch("requests.get", return_value=_resumed_response(b"678")):
-            download_with_resume(
-                "https://example.com/t", dest, sha256=good, size=8, progress=seen.append
-            )
-        assert seen[0] == 5
-        assert seen[-1] == 8
-
-    def test_progress_callback_credits_already_complete_download(
-        self, tmp_path: Path
-    ) -> None:
-        """A verified dest from an earlier run still counts toward the batch."""
-        dest = tmp_path / "tool.tar.gz"
-        dest.write_bytes(b"12345678")
-        seen: list[int] = []
-        with patch("requests.get") as mock_get:
-            download_with_resume(
-                "https://example.com/t", dest, size=8, progress=seen.append
-            )
-        mock_get.assert_not_called()
-        assert seen == [8]
-
-
-class TestBatchDownloadProgress:
-    def test_sums_trackers_into_one_bar(self) -> None:
-        with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
-            progress = BatchDownloadProgress("Downloading", 100)
-            a = progress.tracker()
-            b = progress.tracker()
-            a(10)
-            b(20)
-            a(30)
-            a(0)  # a restart from zero takes that file's bytes back out
-        bar_cls.assert_called_once_with("Downloading")
-        updates = [c[0][0] for c in bar_cls.return_value.update.call_args_list]
-        assert updates == [0.1, 0.3, 0.5, 0.2]
-
-    def test_clamps_at_one(self) -> None:
-        """Sizes are advisory; an over-delivering server never pushes past 100%."""
-        with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
-            progress = BatchDownloadProgress("Downloading", 10)
-            progress.tracker()(25)
-        assert bar_cls.return_value.update.call_args[0][0] == 1
-
-    def test_unknown_total_draws_nothing(self) -> None:
-        with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
-            progress = BatchDownloadProgress("Downloading", 0)
-            progress.tracker()(5)
-            progress.done()
-        bar_cls.assert_not_called()
-
-    def test_done_ends_an_unfinished_bar(self) -> None:
-        """A batch that stops short of 100% (a failed archive) still ends its
-        line so the next log message starts on a fresh row."""
-        stream = io.StringIO()
-        stream.isatty = lambda: True  # type: ignore[method-assign]
-        with patch("esphome.helpers.sys.stderr", stream):
-            progress = BatchDownloadProgress("Downloading", 10)
-            progress.tracker()(5)
-            progress.done()
-        assert stream.getvalue().endswith("50% \n")
-
-    def test_done_before_any_frame_writes_nothing(self) -> None:
-        """A batch aborted before any tracker fired must not emit a stray
-        newline for a bar that was never drawn."""
-        stream = io.StringIO()
-        stream.isatty = lambda: True  # type: ignore[method-assign]
-        with patch("esphome.helpers.sys.stderr", stream):
-            BatchDownloadProgress("Downloading", 10).done()
-        assert stream.getvalue() == ""
-
-    def test_done_after_full_bar_adds_nothing(self) -> None:
-        stream = io.StringIO()
-        stream.isatty = lambda: True  # type: ignore[method-assign]
-        with patch("esphome.helpers.sys.stderr", stream):
-            progress = BatchDownloadProgress("Downloading", 10)
-            progress.tracker()(10)
-            progress.done()
-        assert stream.getvalue().endswith("100% Done...\r\n")
-
 
 class TestDownloadFromMirrors:
     def test_success_returns_url_and_writes_content(self, tmp_path: Path) -> None:
@@ -1697,43 +1593,6 @@ class TestDownloadFromMirrors:
         mock_sleep.assert_not_called()
 
 
-def _http_error(status: int) -> req.HTTPError:
-    """An HTTPError carrying a response with the given status, as raised by
-    ``raise_for_status`` on a real response."""
-    resp = MagicMock()
-    resp.status_code = status
-    return req.HTTPError(str(status), response=resp)
-
-
-class TestIsTransientDownloadError:
-    def test_connection_errors_are_transient(self) -> None:
-        assert _is_transient_download_error(req.ConnectionError("reset"))
-        assert _is_transient_download_error(req.Timeout("timed out"))
-        assert _is_transient_download_error(
-            req.exceptions.ChunkedEncodingError("dropped")
-        )
-
-    def test_http_statuses(self) -> None:
-        assert not _is_transient_download_error(_http_error(404))
-        assert not _is_transient_download_error(_http_error(403))
-        assert _is_transient_download_error(_http_error(429))
-        assert _is_transient_download_error(_http_error(503))
-
-    def test_http_error_without_response_is_permanent(self) -> None:
-        assert not _is_transient_download_error(req.HTTPError("boom"))
-
-    def test_exhausted_resume_attempts_are_permanent(self) -> None:
-        """download_with_resume already spent its own resume attempts; its
-        EsphomeError wrapper is not retried again at the sweep level."""
-        wrapped = EsphomeError("Failed to download after 3 attempts")
-        wrapped.__cause__ = req.ConnectionError("down")
-        assert not _is_transient_download_error(wrapped)
-
-    def test_unrelated_errors_are_permanent(self) -> None:
-        assert not _is_transient_download_error(OSError("disk full"))
-        assert not _is_transient_download_error(EsphomeError("size mismatch"))
-
-
 def test_importing_framework_helpers_does_not_import_requests() -> None:
     """Importing framework_helpers must not drag in requests.
 

tests/unit_tests/test_net_retry.py

@@ -0,0 +1,143 @@
+"""Tests for esphome.net_retry."""
+
+import socket
+from unittest.mock import MagicMock, call, patch
+
+import pytest
+import requests as req
+
+from esphome.core import EsphomeError
+from esphome.net_retry import fetch_with_retry, is_transient_download_error
+
+
+def _http_error(status: int) -> req.HTTPError:
+    """An HTTPError carrying a response with the given status, as raised by
+    ``raise_for_status`` on a real response."""
+    resp = MagicMock()
+    resp.status_code = status
+    return req.HTTPError(str(status), response=resp)
+
+
+class TestIsTransientDownloadError:
+    def test_connection_errors_are_transient(self) -> None:
+        assert is_transient_download_error(req.ConnectionError("reset"))
+        assert is_transient_download_error(req.Timeout("timed out"))
+        assert is_transient_download_error(
+            req.exceptions.ChunkedEncodingError("dropped")
+        )
+        assert is_transient_download_error(
+            req.exceptions.ContentDecodingError("gzip stream truncated")
+        )
+
+    def test_http_statuses(self) -> None:
+        assert not is_transient_download_error(_http_error(404))
+        assert not is_transient_download_error(_http_error(403))
+        assert is_transient_download_error(_http_error(429))
+        assert is_transient_download_error(_http_error(503))
+
+    def test_http_error_without_response_is_permanent(self) -> None:
+        assert not is_transient_download_error(req.HTTPError("boom"))
+
+    def test_hard_dns_failures_are_permanent(self) -> None:
+        """Hard resolution failures are permanent via both the cause chain
+        and MaxRetryError.reason."""
+        from urllib3.exceptions import MaxRetryError, NameResolutionError
+
+        gai = socket.gaierror(socket.EAI_NONAME, "nodename nor servname provided")
+
+        chained = req.ConnectionError("resolution failed")
+        chained.__cause__ = gai
+        assert not is_transient_download_error(chained)
+
+        # The real urllib3 shape: gaierror on NameResolutionError.__cause__,
+        # carried by MaxRetryError.reason.
+        try:
+            raise NameResolutionError("example.invalid", None, gai) from gai
+        except NameResolutionError as nre:
+            wrapped = req.ConnectionError(
+                MaxRetryError(None, "http://example.invalid/", reason=nre)
+            )
+        assert not is_transient_download_error(wrapped)
+
+        # A garden-variety connection reset stays transient.
+        assert is_transient_download_error(req.ConnectionError("reset by peer"))
+
+    def test_temporary_dns_failure_stays_transient(self) -> None:
+        """EAI_AGAIN (flaky resolver) stays retryable."""
+        gai = socket.gaierror(socket.EAI_AGAIN, "temporary failure in name resolution")
+        chained = req.ConnectionError("resolution failed")
+        chained.__cause__ = gai
+
+        assert is_transient_download_error(chained)
+
+    def test_implicit_context_does_not_reclassify(self) -> None:
+        """A gaierror riding along as implicit __context__ must not turn a
+        genuine connection reset permanent."""
+        try:
+            try:
+                raise socket.gaierror(socket.EAI_NONAME, "first attempt")
+            except socket.gaierror:
+                raise req.ConnectionError("reset by peer") from None
+        except req.ConnectionError as reset:
+            assert reset.__context__ is not None
+            assert is_transient_download_error(reset)
+
+    def test_gaierror_without_errno_stays_transient(self) -> None:
+        """A gaierror carrying no EAI code cannot prove a hard failure."""
+        chained = req.ConnectionError("resolution failed")
+        chained.__cause__ = socket.gaierror("no errno")
+
+        assert is_transient_download_error(chained)
+
+    def test_mixed_chain_hard_failure_wins(self) -> None:
+        """EAI_AGAIN in the chain does not mask a hard failure elsewhere."""
+        again = socket.gaierror(socket.EAI_AGAIN, "temporary failure")
+        hard = socket.gaierror(socket.EAI_NONAME, "unknown host")
+
+        outer = req.ConnectionError(hard)
+        outer.__cause__ = again
+        assert not is_transient_download_error(outer)
+
+        outer = req.ConnectionError(again)
+        outer.__cause__ = hard
+        assert not is_transient_download_error(outer)
+
+    def test_dns_walk_survives_exception_cycles(self) -> None:
+        """A cyclic cause chain must terminate (and stay transient when no
+        resolution failure is present)."""
+        outer = req.ConnectionError("a")
+        inner = ValueError("b")
+        outer.__cause__ = inner
+        inner.__cause__ = outer
+
+        assert is_transient_download_error(outer)
+
+    def test_exhausted_resume_attempts_are_permanent(self) -> None:
+        """download_with_resume already spent its own resume attempts; its
+        EsphomeError wrapper is not retried again at the sweep level."""
+        wrapped = EsphomeError("Failed to download after 3 attempts")
+        wrapped.__cause__ = req.ConnectionError("down")
+        assert not is_transient_download_error(wrapped)
+
+    def test_unrelated_errors_are_permanent(self) -> None:
+        assert not is_transient_download_error(OSError("disk full"))
+        assert not is_transient_download_error(EsphomeError("size mismatch"))
+
+
+class TestFetchWithRetry:
+    def test_logs_the_upcoming_attempt_number(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """The warning names the attempt about to run, not the failed one."""
+        with (
+            patch("esphome.net_retry.time.sleep") as mock_sleep,
+            pytest.raises(req.ConnectionError),
+        ):
+            fetch_with_retry(
+                "https://example.com/f",
+                lambda: (_ for _ in ()).throw(req.ConnectionError("reset")),
+            )
+
+        assert mock_sleep.call_args_list == [call(2), call(4)]
+        assert "(attempt 2/3)" in caplog.text
+        assert "(attempt 3/3)" in caplog.text