diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index ef8c27c9aa..586ccbe220 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -354,14 +354,27 @@ def _registry_jobs( return jobs, failed, installable +def _is_vcs_spec_uri(url: str) -> bool: + """Whether pio's ``install_from_uri`` would clone this URI rather than + copy or download it (PackageSpec normalizes git URLs to ``git+``).""" + return not url.startswith(("file://", "symlink://", "http://", "https://")) + + +def _spec_name(spec: Any, url: str) -> str: + """The spec's name; the URL basename fallback is defensive only + (PackageSpec derives a name from the URI itself).""" + return spec.name or url.split("#", 1)[0].rsplit("/", 1)[-1] + + def _uri_jobs( manager: Any, specs: list[Any], seen: set[str] ) -> tuple[list[tuple[str, int, Any]], int, list[tuple[str, Any]]]: """Jobs for direct-URL specs; a HEAD sizes each for the combined bar. Also returns how many HEAD probes errored (an absent length is not an - error) and the ``(name, spec)`` pairs whose archives will be - installable. + error) and the ``(name, spec)`` pairs to pre-install: downloaded + archives, plus VCS specs, which have no archive -- the pre-install + itself clones them, in parallel instead of one at a time in pio run. """ from esphome.net_retry import fetch_with_retry, http_request @@ -369,13 +382,17 @@ def _uri_jobs( installable: list[tuple[str, Any]] = [] for spec in specs: url = spec.uri - if not url or not url.startswith(("http://", "https://")): - continue # git+/file specs are cloned/copied, not downloaded - if url.split("#", 1)[0].endswith(".git"): - continue # bare-URL VCS spec; PlatformIO clones it + if not url: + continue + is_vcs = _is_vcs_spec_uri(url) + if not is_vcs and not url.startswith(("http://", "https://")): + continue # file/symlink specs are copied in place by pio run if manager.get_package(spec): continue - name = spec.name or url.rsplit("/", 1)[-1] + name = _spec_name(spec, url) + if is_vcs: + installable.append((name, spec)) + continue # PlatformIO downloads URL specs with no checksum dl_path = Path(manager.compute_download_path(url, "")) if dl_path.is_file(): @@ -899,8 +916,14 @@ def _prefetch(build_dir: Path, env: str) -> None: if name not in failed_names } if to_install: + # Clones first: they wait on the network, so they must not + # queue behind CPU-bound archive extractions + ordered = sorted( + to_install.values(), + key=lambda entry: not ((url := entry[1].uri) and _is_vcs_spec_uri(url)), + ) try: - _preinstall(mgr, list(to_install.values())) + _preinstall(mgr, ordered) except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught # Each group degrades independently; pio run installs # whatever this one did not diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 91fb78c6af..9a112b29c0 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -587,7 +587,8 @@ def test_registry_jobs_one_bad_spec_keeps_the_rest(tmp_path: Path) -> None: def test_uri_jobs_head_sizes_the_bar(tmp_path: Path) -> None: - """HEAD sizes direct-URL specs; git and unreachable URLs are skipped.""" + """HEAD sizes direct-URL specs; VCS specs skip the download but are + still installable (the pre-install clones them in parallel).""" m = _fake_manager(tmp_path) resp = MagicMock() resp.headers = {"content-length": "2222"} @@ -597,14 +598,13 @@ def test_uri_jobs_head_sizes_the_bar(tmp_path: Path) -> None: [ _FakeSpec(uri="https://x/big.zip", name="big", custom_name=True), _FakeSpec(uri="git+https://x/repo.git", name="repo"), - _FakeSpec(uri="https://x/repo.git#v1", name="barevcs"), _FakeSpec(name="registry"), ], set(), ) assert failed == 0 assert [(n, s) for n, s, _ in jobs] == [("big", 2222)] - assert [n for n, _ in installable] == ["big"] + assert [n for n, _ in installable] == ["repo", "big"] # a successful HEAD with no Content-Length is a clean skip resp.headers = {} with patch("esphome.net_retry.http_request", return_value=resp): @@ -613,6 +613,35 @@ def test_uri_jobs_head_sizes_the_bar(tmp_path: Path) -> None: ) == ([], 0, []) +def test_uri_jobs_vcs_specs_installable_without_probe(tmp_path: Path) -> None: + """VCS specs never probe the network here (there is no archive); an + uninstalled one is handed to the pre-install, an installed one and + file/symlink specs are skipped.""" + m = _fake_manager(tmp_path) + with patch("esphome.net_retry.http_request") as mock_head: + jobs, failed, installable = pf._uri_jobs( + m, + [ + _FakeSpec(uri="git+https://x/tool.git#1.0", name="tool"), + _FakeSpec(uri="hg+https://x/old", name="mercurial"), + # Name falls back to the URL basename, fragment excluded + _FakeSpec(uri="git+https://x/noname#v2", name=None), + _FakeSpec(uri="file:///local/dir", name="local"), + _FakeSpec(uri="symlink:///local/dir", name="link"), + ], + set(), + ) + mock_head.assert_not_called() + assert (jobs, failed) == ([], 0) + assert [n for n, _ in installable] == ["tool", "mercurial", "noname"] + + m.get_package.return_value = object() # already installed: warm and silent + with patch("esphome.net_retry.http_request"): + assert pf._uri_jobs( + m, [_FakeSpec(uri="git+https://x/tool.git#1.0", name="tool")], set() + ) == ([], 0, []) + + def test_uri_jobs_head_failure_counts_as_unresolved( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: