Merge branch 'esp8266-native-library-backend' into esp8266-native-build-spec

esphome/build_helpers/idedata.py

@@ -261,7 +261,7 @@ def load_or_build_idedata(
     if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime:
         try:
             cached = json.loads(cache.read_text(encoding="utf-8"))
-        except ValueError as err:
+        except (ValueError, OSError) as err:
             # A recurring cause (interrupted write, disk full) would otherwise
             # look like unexplained slow builds
             _LOGGER.warning("Discarding unreadable idedata cache %s: %s", cache, err)

esphome/platformio/library.py

@@ -1015,16 +1015,27 @@ def _prefetch_wave(
     # downloads with their per-file bars instead.
     sizes = _content_lengths([c.source.url for c in components])
     if not all(sizes):
+        # Name the culprits so the fallback is distinguishable from a hang
+        _LOGGER.debug(
+            "No Content-Length for %s; downloading sequentially",
+            ", ".join(
+                c.source.url
+                for c, size in zip(components, sizes, strict=True)
+                if not size
+            ),
+        )
         return
     progress = BatchDownloadProgress("Downloading libraries", sum(sizes))
+    # Reported after the bar is done so the warnings do not land on its
+    # row; list.append is atomic under the GIL.
+    failures: list[tuple[str, Exception]] = []
 
     def _fetch(component: ConvertedLibrary) -> None:
         tracker = progress.tracker()
         try:
             component.download(salt=salt, namespace=namespace, progress=tracker)
         except Exception as err:  # noqa: BLE001  # pylint: disable=broad-exception-caught
-            # The sequential call below retries and reports the failure
-            _LOGGER.debug("Prefetch of %s failed: %s", component.name, err)
+            failures.append((component.name, err))
             tracker(0)
 
     ex = ThreadPoolExecutor(max_workers=min(_DOWNLOAD_WORKERS, len(components)))
@@ -1036,6 +1047,9 @@ def _prefetch_wave(
         # all before the process can exit; in-flight ones still finish.
         ex.shutdown(wait=True, cancel_futures=True)
         progress.done()
+    for name, err in failures:
+        # The sequential call below retries and raises the real error
+        _LOGGER.warning("Prefetch of %s failed (retrying sequentially): %s", name, err)
 
 
 def convert_libraries(

esphome/writer.py

@@ -410,6 +410,11 @@ def _build_info_stale(
     except (json.JSONDecodeError, OSError):
         _LOGGER.debug("Build info JSON unreadable; regenerating")
         return True
+    if not isinstance(existing, dict):
+        # Valid JSON that is not an object (truncated or hand-edited) is
+        # stale, not a traceback
+        _LOGGER.debug("Build info JSON malformed; regenerating")
+        return True
     if (
         existing.get("config_hash") != config_hash
         or existing.get("esphome_version") != __version__

tests/unit_tests/build_helpers/test_idedata.py

@@ -428,6 +428,27 @@ def test_load_or_build_idedata_corrupted_cache_is_logged(
     assert "Discarding unreadable idedata cache" in caplog.text
 
 
+def test_load_or_build_idedata_discards_unreadable_cache_file(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """An OSError on the cache read (permissions, I/O) regenerates like a
+    parse failure instead of aborting the consumer."""
+    compile_commands = _write_compile_commands(tmp_path)
+    cache = tmp_path / "c.json"
+    cache.write_text("{}")
+    cache.chmod(0)
+    os.utime(cache, (compile_commands.stat().st_mtime + 5,) * 2)
+    try:
+        with patch.object(idedata, "get_toolchain_includes", return_value=[]):
+            data = idedata.load_or_build_idedata(
+                compile_commands, tmp_path / "f.elf", cache
+            )
+    finally:
+        cache.chmod(0o600)
+    assert data["cxx_path"] == "/tools/g++"
+    assert "Discarding unreadable idedata cache" in caplog.text
+
+
 def test_load_or_build_idedata_never_caches_a_launcher(tmp_path: Path) -> None:
     """A compile DB naming a launcher as the compiler is rejected, never cached."""
     compile_commands = tmp_path / "compile_commands.json"

tests/unit_tests/test_platformio_library.py

@@ -631,7 +631,7 @@ def test_lex_build_flags_dangling_flag_does_not_cross_entries(
 
 
 def test_prefetch_wave_downloads_registry_archives_in_parallel(
-    setup_core, monkeypatch: pytest.MonkeyPatch
+    setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
 ) -> None:
     """Registry archives in one wave download concurrently, deduped by URL;
     git/local sources and failures are left to the sequential call."""
@@ -661,10 +661,12 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel(
         "https://x/b.tar.gz",
         "https://x/boom.tar.gz",
     ]
+    # The failure surfaces at default verbosity, after the bar
+    assert "Prefetch of c failed (retrying sequentially)" in caplog.text
 
 
 def test_prefetch_wave_unknown_size_falls_back_to_sequential(
-    setup_core, monkeypatch: pytest.MonkeyPatch
+    setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
 ) -> None:
     """Any unknown HEAD size skips the parallel prefetch entirely so the
     sequential downloads keep their per-file bars."""
@@ -678,7 +680,10 @@ def test_prefetch_wave_unknown_size_falls_back_to_sequential(
         ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))),
         ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),
     ]
+    caplog.set_level("DEBUG")
     lib._prefetch_wave(wave, "", "idf")
+    # The culprit URL is named so the fallback is traceable
+    assert "No Content-Length for https://x/b.tar.gz" in caplog.text
 
 
 def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None:

tests/unit_tests/test_writer.py

@@ -2506,6 +2506,9 @@ def test_build_info_stale_branches(tmp_path: Path) -> None:
     assert _build_info_stale(h, cpp, info, 1) is True  # JSON unreadable
     info.write_text("not json")
     assert _build_info_stale(h, cpp, info, 1) is True
+    # Valid JSON that is not an object is stale, not an AttributeError
+    info.write_text("[]")
+    assert _build_info_stale(h, cpp, info, 1) is True
     info.write_text(json_mod.dumps({"config_hash": 2, "esphome_version": __version__}))
     assert _build_info_stale(h, cpp, info, 1) is True  # hash mismatch
     info.write_text(json_mod.dumps({"config_hash": 1, "esphome_version": "0.0.0"}))