diff --git a/esphome/build_helpers/idedata.py b/esphome/build_helpers/idedata.py index e81c5ccc1a..f58b0ec8f8 100644 --- a/esphome/build_helpers/idedata.py +++ b/esphome/build_helpers/idedata.py @@ -261,7 +261,7 @@ def load_or_build_idedata( if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime: try: cached = json.loads(cache.read_text(encoding="utf-8")) - except ValueError as err: + except (ValueError, OSError) as err: # A recurring cause (interrupted write, disk full) would otherwise # look like unexplained slow builds _LOGGER.warning("Discarding unreadable idedata cache %s: %s", cache, err) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index c39ddeedef..0e3a942724 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -1015,16 +1015,27 @@ def _prefetch_wave( # downloads with their per-file bars instead. sizes = _content_lengths([c.source.url for c in components]) if not all(sizes): + # Name the culprits so the fallback is distinguishable from a hang + _LOGGER.debug( + "No Content-Length for %s; downloading sequentially", + ", ".join( + c.source.url + for c, size in zip(components, sizes, strict=True) + if not size + ), + ) return progress = BatchDownloadProgress("Downloading libraries", sum(sizes)) + # Reported after the bar is done so the warnings do not land on its + # row; list.append is atomic under the GIL. + failures: list[tuple[str, Exception]] = [] def _fetch(component: ConvertedLibrary) -> None: tracker = progress.tracker() try: component.download(salt=salt, namespace=namespace, progress=tracker) except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught - # The sequential call below retries and reports the failure - _LOGGER.debug("Prefetch of %s failed: %s", component.name, err) + failures.append((component.name, err)) tracker(0) ex = ThreadPoolExecutor(max_workers=min(_DOWNLOAD_WORKERS, len(components))) @@ -1036,6 +1047,9 @@ def _prefetch_wave( # all before the process can exit; in-flight ones still finish. ex.shutdown(wait=True, cancel_futures=True) progress.done() + for name, err in failures: + # The sequential call below retries and raises the real error + _LOGGER.warning("Prefetch of %s failed (retrying sequentially): %s", name, err) def convert_libraries( diff --git a/esphome/writer.py b/esphome/writer.py index a65b5c173a..6e70f320a9 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -410,6 +410,11 @@ def _build_info_stale( except (json.JSONDecodeError, OSError): _LOGGER.debug("Build info JSON unreadable; regenerating") return True + if not isinstance(existing, dict): + # Valid JSON that is not an object (truncated or hand-edited) is + # stale, not a traceback + _LOGGER.debug("Build info JSON malformed; regenerating") + return True if ( existing.get("config_hash") != config_hash or existing.get("esphome_version") != __version__ diff --git a/tests/unit_tests/build_helpers/test_idedata.py b/tests/unit_tests/build_helpers/test_idedata.py index 52ffc88224..c60edb4a83 100644 --- a/tests/unit_tests/build_helpers/test_idedata.py +++ b/tests/unit_tests/build_helpers/test_idedata.py @@ -428,6 +428,27 @@ def test_load_or_build_idedata_corrupted_cache_is_logged( assert "Discarding unreadable idedata cache" in caplog.text +def test_load_or_build_idedata_discards_unreadable_cache_file( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An OSError on the cache read (permissions, I/O) regenerates like a + parse failure instead of aborting the consumer.""" + compile_commands = _write_compile_commands(tmp_path) + cache = tmp_path / "c.json" + cache.write_text("{}") + cache.chmod(0) + os.utime(cache, (compile_commands.stat().st_mtime + 5,) * 2) + try: + with patch.object(idedata, "get_toolchain_includes", return_value=[]): + data = idedata.load_or_build_idedata( + compile_commands, tmp_path / "f.elf", cache + ) + finally: + cache.chmod(0o600) + assert data["cxx_path"] == "/tools/g++" + assert "Discarding unreadable idedata cache" in caplog.text + + def test_load_or_build_idedata_never_caches_a_launcher(tmp_path: Path) -> None: """A compile DB naming a launcher as the compiler is rejected, never cached.""" compile_commands = tmp_path / "compile_commands.json" diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index ba7b35ef32..232f939dec 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -631,7 +631,7 @@ def test_lex_build_flags_dangling_flag_does_not_cross_entries( def test_prefetch_wave_downloads_registry_archives_in_parallel( - setup_core, monkeypatch: pytest.MonkeyPatch + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: """Registry archives in one wave download concurrently, deduped by URL; git/local sources and failures are left to the sequential call.""" @@ -661,10 +661,12 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( "https://x/b.tar.gz", "https://x/boom.tar.gz", ] + # The failure surfaces at default verbosity, after the bar + assert "Prefetch of c failed (retrying sequentially)" in caplog.text def test_prefetch_wave_unknown_size_falls_back_to_sequential( - setup_core, monkeypatch: pytest.MonkeyPatch + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: """Any unknown HEAD size skips the parallel prefetch entirely so the sequential downloads keep their per-file bars.""" @@ -678,7 +680,10 @@ def test_prefetch_wave_unknown_size_falls_back_to_sequential( ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))), ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))), ] + caplog.set_level("DEBUG") lib._prefetch_wave(wave, "", "idf") + # The culprit URL is named so the fallback is traceable + assert "No Content-Length for https://x/b.tar.gz" in caplog.text def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 6d1118ccf4..8249abd27b 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -2506,6 +2506,9 @@ def test_build_info_stale_branches(tmp_path: Path) -> None: assert _build_info_stale(h, cpp, info, 1) is True # JSON unreadable info.write_text("not json") assert _build_info_stale(h, cpp, info, 1) is True + # Valid JSON that is not an object is stale, not an AttributeError + info.write_text("[]") + assert _build_info_stale(h, cpp, info, 1) is True info.write_text(json_mod.dumps({"config_hash": 2, "esphome_version": __version__})) assert _build_info_stale(h, cpp, info, 1) is True # hash mismatch info.write_text(json_mod.dumps({"config_hash": 1, "esphome_version": "0.0.0"}))