Merge branch 'esp8266-native-shared-helpers' into esp8266-native-build-surgery

esphome/espidf/framework.py

@@ -791,10 +791,17 @@ def _prefetch_idf_tool_archives(
             # failure_reason: a message-less exception must not log blank
             _LOGGER.warning("Could not prefetch %s: %s", name, failure_reason(e))
             _LOGGER.debug("Prefetch failure detail", exc_info=e)
+        if failures and len(failures) == len(entries):
+            # A systematic fault, not one flaky mirror: the resume
+            # workaround (#17703) is off for this whole install
+            _LOGGER.error(
+                "Every ESP-IDF tool prefetch failed; the installer will "
+                "download without resume"
+            )
     except Exception as e:  # noqa: BLE001  # pylint: disable=broad-exception-caught
         # The installer downloads anything missing itself; never let the
         # prefetch become a new way for the install to fail.
-        _LOGGER.warning("ESP-IDF tool prefetch failed: %s", e)
+        _LOGGER.warning("ESP-IDF tool prefetch failed: %s", failure_reason(e))
         _LOGGER.debug("Prefetch failure detail", exc_info=True)
 
 

esphome/framework_helpers.py

@@ -834,7 +834,10 @@ class _BatchDownloadProgress:
             with self._lock:
                 self._sum += done - last
                 last = done
-                self._bar.update(min(self._sum / self._total, 1))
+                # A bar-write failure (broken stderr pipe) must not surface
+                # as a download failure and cost the .part file
+                with suppress(Exception):
+                    self._bar.update(min(self._sum / self._total, 1))
 
         return update
 
@@ -859,7 +862,9 @@ class _BatchDownloadProgress:
 
         class _EndRow(logging.Filter):
             def filter(self, record: logging.LogRecord) -> bool:
-                with lock:
+                # Handler.handle() runs filters outside handleError's try; a
+                # stderr write failure must not escape through the log call
+                with lock, suppress(Exception):
                     the_bar.interrupt()
                 return True
 
@@ -926,7 +931,7 @@ def download_with_resume(
     ``download_from_mirrors``.
 
     ``progress`` replaces the built-in bar: it receives the absolute bytes of
-    ``dest`` obtained so far (see ``BatchDownloadProgress``).
+    ``dest`` obtained so far (see ``_BatchDownloadProgress``).
 
     Raises EsphomeError when all attempts are exhausted.
     """

esphome/platformio/library.py

@@ -925,9 +925,10 @@ def _prefetch_wave(
                 cached = source.is_cached(
                     component.get_sanitized_name(), salt=salt, namespace=namespace
                 )
-            except Exception as err:  # noqa: BLE001  # pylint: disable=broad-exception-caught
-                # Best-effort: a failing probe prefetches (and re-downloads)
-                _LOGGER.debug("Cache probe for %s failed: %s", component.name, err)
+            except OSError as err:
+                # Best-effort, but visibly: a systematic probe failure makes
+                # every warm build re-download every archive
+                _LOGGER.warning("Cache probe for %s failed: %s", component.name, err)
                 cached = False
             if cached:
                 # A warm build must stay silent
@@ -960,7 +961,7 @@ def _prefetch_wave(
     except Exception as err:  # noqa: BLE001  # pylint: disable=broad-exception-caught
         # Same policy as the ESP-IDF twin: the prefetch must never become a
         # new way for the build to fail
-        _LOGGER.warning("Library prefetch failed: %s", err)
+        _LOGGER.warning("Library prefetch failed: %s", failure_reason(err))
         _LOGGER.debug("Prefetch failure detail", exc_info=True)
 
 

tests/unit_tests/test_espidf_framework.py

@@ -1074,6 +1074,27 @@ def test_prefetch_failures_never_raise(
     assert expected_log in caplog.text
 
 
+def test_prefetch_total_failure_logs_error(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """Every archive failing is a systematic fault (proxy, bad kwarg), not
+    a flaky mirror; it must be distinguishable at ERROR because the resume
+    workaround is off for the whole install."""
+    with (
+        patch(
+            "esphome.espidf.framework.run_command",
+            return_value=(True, _PREFETCH_JSON, ""),
+        ),
+        patch(
+            "esphome.espidf.framework.download_with_resume",
+            side_effect=OSError("proxy refuses everything"),
+        ),
+        patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
+    ):
+        _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
+    assert "Every ESP-IDF tool prefetch failed" in caplog.text
+
+
 def test_prefetch_one_failed_archive_does_not_stop_the_rest(
     tmp_path: Path, caplog: pytest.LogCaptureFixture
 ) -> None:
@@ -1099,6 +1120,8 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest(
 
     assert download.call_count == 2
     assert "Could not prefetch cmake@3.30.2" in caplog.text
+    # One flaky archive is routine, never the systematic-fault ERROR
+    assert "Every ESP-IDF tool prefetch failed" not in caplog.text
 
 
 def test_prefetch_finishes_progress_bar_and_cancels_queue(tmp_path: Path) -> None:

tests/unit_tests/test_platformio_library.py

@@ -683,9 +683,11 @@ def test_join_flag_args_empty_argument_warns_and_drops(
 
 
 def test_prefetch_wave_cache_probe_failure_still_prefetches(
-    monkeypatch: pytest.MonkeyPatch,
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
 ) -> None:
-    """The cache probe is best-effort; a failing probe prefetches anyway."""
+    """A filesystem probe failure warns (a systematic one re-downloads
+    everything) but still prefetches; a programming error is NOT swallowed
+    here, it reaches the outer blanket guard."""
     calls: list[str] = []
     monkeypatch.setattr(
         URLSource,
@@ -695,7 +697,7 @@ def test_prefetch_wave_cache_probe_failure_still_prefetches(
     monkeypatch.setattr(
         URLSource,
         "is_cached",
-        lambda self, *a, **kw: (_ for _ in ()).throw(RuntimeError("no core")),
+        lambda self, *a, **kw: (_ for _ in ()).throw(OSError("cache root denied")),
     )
     wave = [
         ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
@@ -703,12 +705,14 @@ def test_prefetch_wave_cache_probe_failure_still_prefetches(
     ]
     lib._prefetch_wave(wave, "", "idf")
     assert sorted(calls) == ["https://x/a.tar.gz", "https://x/b.tar.gz"]
+    assert "Cache probe for a failed: cache root denied" in caplog.text
 
 
 def test_prefetch_wave_internal_error_never_fails_the_build(
     monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
 ) -> None:
     """The blanket guard keeps a prefetch bug from failing the walk."""
+    monkeypatch.setattr(URLSource, "is_cached", lambda self, *a, **kw: False)
     monkeypatch.setattr(
         lib,
         "run_batch_downloads",
@@ -752,6 +756,7 @@ def test_prefetch_wave_single_archive_uses_the_batch(
     through the same runner so there is one download method and one bar."""
     caplog.set_level("INFO")
     calls: list[str] = []
+    monkeypatch.setattr(URLSource, "is_cached", lambda self, *a, **kw: False)
     monkeypatch.setattr(
         URLSource,
         "download",