[core] Download external_files in parallel

Each component that uses external_files (audio_file, speaker
media_player, ...) currently calls download_content once per file
inline inside a per-item config validator. With ~24 cached audio
files in a Home Assistant Voice PE config, that means ~24 sequential
HEAD round-trips, even when every response is a 304.

This adds download_content_many(items, ...) which fans the per-file
checks out across a ThreadPoolExecutor (capped at 16 workers so
configs with hundreds of files don't open hundreds of sockets), then
refactors audio_file and speaker.media_player to collect URLs at the
list level and call the batch helper once instead of downloading
inside each per-item validator.

Wall time for the validation phase drops from sum(latency) to roughly
max(latency) when the cache is warm.
This commit is contained in:
J. Nick Koston
2026-04-26 09:36:13 -05:00
parent e87e78c544
commit 2d9d1eabfa
4 changed files with 140 additions and 23 deletions
+66
View File
@@ -282,3 +282,69 @@ def test_download_content_skip_external_update_downloads_when_missing(
assert result == new_content
assert test_file.read_bytes() == new_content
@patch("esphome.external_files.download_content")
def test_download_content_many_empty_is_noop(
mock_download: MagicMock, setup_core: Path
) -> None:
"""Empty input shouldn't spin up a thread pool or call download_content."""
external_files.download_content_many([])
mock_download.assert_not_called()
@patch("esphome.external_files.download_content")
def test_download_content_many_single_item_avoids_pool(
mock_download: MagicMock, setup_core: Path
) -> None:
"""A single item should be downloaded inline (no thread pool overhead)."""
item = ("https://example.com/file.txt", setup_core / "f.txt")
external_files.download_content_many([item])
mock_download.assert_called_once_with(
item[0], item[1], external_files.NETWORK_TIMEOUT
)
@patch("esphome.external_files.download_content")
def test_download_content_many_runs_in_parallel(
mock_download: MagicMock, setup_core: Path
) -> None:
"""Multiple items should run concurrently — total wall time ≈ max latency."""
import threading
barrier = threading.Barrier(3)
def slow_download(url: str, path: Path, timeout: int) -> bytes:
# If calls were serial this would deadlock (third caller never arrives
# while the first is blocked at the barrier).
barrier.wait(timeout=2.0)
return b""
mock_download.side_effect = slow_download
items = [
("https://example.com/a", setup_core / "a"),
("https://example.com/b", setup_core / "b"),
("https://example.com/c", setup_core / "c"),
]
external_files.download_content_many(items, max_workers=4)
assert mock_download.call_count == 3
@patch("esphome.external_files.download_content")
def test_download_content_many_propagates_errors(
mock_download: MagicMock, setup_core: Path
) -> None:
"""An exception from any worker must propagate out of download_content_many."""
def fake_download(url: str, path: Path, timeout: int) -> bytes:
if url.endswith("bad"):
raise Invalid(f"could not download {url}")
return b""
mock_download.side_effect = fake_download
items = [
("https://example.com/ok", setup_core / "ok"),
("https://example.com/bad", setup_core / "bad"),
]
with pytest.raises(Invalid, match="could not download"):
external_files.download_content_many(items)