github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-08-31 10:06:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'esp8266-native-library-backend' into esp8266-native-build-spec

Browse Source
This commit is contained in:
J. Nick Koston
2026-08-23 13:10:39 -05:00
parent 7155d5446b 2d44dacbed
commit 8d45edbb3e
10 changed files with 261 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit_tests/test_espidf_component.py
+13 -5
View File
@@ -614,7 +614,7 @@ def _patch_registry(monkeypatch, versions):
def test_resolve_registry_version_intersects_constraints(monkeypatch):
_patch_registry(monkeypatch, ["1.10018.1", "1.10021.0", "1.10021.1"])
owner, name, version, url = _resolve_registry_version(
owner, name, version, url, _size = _resolve_registry_version(
"esphome", "libsodium", {"==1.10021.0", "^1.10018.1"}
)
assert (owner, name, version) == ("esphome", "libsodium", "1.10021.0")
@@ -623,7 +623,9 @@ def test_resolve_registry_version_intersects_constraints(monkeypatch):
def test_resolve_registry_version_picks_highest_satisfying(monkeypatch):
_patch_registry(monkeypatch, ["1.0.0", "1.5.0", "2.0.0"])
_owner, _name, version, _url = _resolve_registry_version("o", "p", {"^1.0.0"})
_owner, _name, version, _url, _size = _resolve_registry_version(
"o", "p", {"^1.0.0"}
)
assert version == "1.5.0"
@@ -673,7 +675,7 @@ def test_generate_idf_components_dedupes_shared_dependency(
resolve_calls.append(pkgname)
captured[f"{owner}/{pkgname}"] = set(requirements)
version = "1.10021.0" if pkgname == "C" else "1.0.0"
return owner, pkgname, version, f"http://x/{pkgname}.tar.gz"
return owner, pkgname, version, f"http://x/{pkgname}.tar.gz", None
monkeypatch.setattr(
esphome.platformio.library, "_resolve_registry_version", fake_resolve
@@ -732,7 +734,7 @@ def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies(
def fake_resolve(owner, pkgname, requirements):
resolve_calls.append(pkgname)
return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz"
return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None
monkeypatch.setattr(
esphome.platformio.library, "_resolve_registry_version", fake_resolve
@@ -788,6 +790,7 @@ def test_generate_idf_components_handles_dependency_cycle(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -845,6 +848,7 @@ def test_generate_idf_components_git_overrides_registry_warns(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -881,6 +885,7 @@ def test_generate_idf_components_missing_manifest_raises(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -925,6 +930,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -958,6 +964,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -994,6 +1001,7 @@ def test_generate_idf_components_incompatible_dependency_skipped(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -1066,7 +1074,7 @@ def test_idf_component_download_passes_salt() -> None:
c.download(force=True, salt="abcd1234", namespace="idf")
source.download.assert_called_once_with(
"owner/name", force=True, salt="abcd1234", namespace="idf", progress=None
"owner/name", force=True, salt="abcd1234", namespace="idf"
)
assert c.path == Path("/converted/owner/name")
tests/unit_tests/test_espidf_framework.py
+5 -26
View File
@@ -912,7 +912,7 @@ def test_prefetch_leaves_unverifiable_entries_to_the_installer(
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls,
patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls,
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
assert [call[0][0] for call in download.call_args_list] == [
@@ -953,7 +953,7 @@ def test_prefetch_dedupes_entries_by_dest(tmp_path: Path) -> None:
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.espidf.framework.BatchDownloadProgress"),
patch("esphome.framework_helpers._BatchDownloadProgress"),
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
dests = [call[0][1].name for call in download.call_args_list]
@@ -968,7 +968,7 @@ def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None:
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls,
patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls,
):
# Materialize the lazy mock before threads race its first creation
tracker = progress_cls.return_value.tracker.return_value
@@ -1022,25 +1022,6 @@ def test_prefetch_downloads_archives_concurrently(tmp_path: Path) -> None:
assert download.call_count == 6
def test_prefetch_single_archive_uses_one_worker(tmp_path: Path) -> None:
entries = json.loads(_PREFETCH_JSON)[:1]
with (
patch(
"esphome.espidf.framework.run_command",
return_value=(True, json.dumps(entries), ""),
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch(
"esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor
) as pool,
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
pool.assert_called_once_with(max_workers=1)
assert download.call_count == 1
def test_prefetch_skips_already_downloaded_archives(tmp_path: Path) -> None:
dist = get_idf_tools_path() / "dist"
dist.mkdir(parents=True)
@@ -1130,10 +1111,8 @@ def test_prefetch_finishes_progress_bar_and_cancels_queue(tmp_path: Path) -> Non
),
patch("esphome.espidf.framework.download_with_resume"),
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls,
patch(
"esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor
) as pool_cls,
patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls,
patch("esphome.framework_helpers.ThreadPoolExecutor") as pool_cls,
):
pool = MagicMock(wraps=ThreadPoolExecutor(max_workers=2))
pool_cls.return_value = pool
tests/unit_tests/test_espota2.py
+6 -2
View File
@@ -998,10 +998,10 @@ def test_progress_bar(capsys: CaptureFixture[str]) -> None:
assert "100%" in captured.err
assert "Done" in captured.err
# Test done method
# done() after the 100% frame adds nothing; that frame ended its line
progress.done()
captured = capsys.readouterr()
assert captured.err == "\n"
assert captured.err == ""
# Test same progress doesn't update
progress.update(0.5)
@@ -1010,6 +1010,10 @@ def test_progress_bar(capsys: CaptureFixture[str]) -> None:
# Should only see one update (second call shouldn't write)
assert captured.err.count("50%") == 1
# done() after a mid-way frame ends the line
progress.done()
assert capsys.readouterr().err == "\n"
# Tests for SHA256 authentication
@pytest.mark.usefixtures("mock_time")
tests/unit_tests/test_framework_helpers.py
+29 -15
View File
@@ -12,6 +12,8 @@ from pathlib import Path
import subprocess
import sys
import tarfile
import threading
import time
from unittest.mock import MagicMock, Mock, call, patch
import zipfile
@@ -21,8 +23,8 @@ import requests as req
from esphome import framework_helpers
from esphome.core import EsphomeError
from esphome.framework_helpers import (
BatchDownloadProgress,
_7z_extract_all,
_BatchDownloadProgress,
_detect_archive_root,
_is_transient_download_error,
_rename_with_retry,
@@ -38,6 +40,7 @@ from esphome.framework_helpers import (
get_python_env_executable_path,
get_system_python_path,
rmdir,
run_batch_downloads,
run_command,
run_command_ok,
str_to_lst_of_str,
@@ -1161,11 +1164,6 @@ class TestDownloadWithResume:
def test_run_batch_downloads_ctrl_c_aborts_in_flight_jobs() -> None:
"""Ctrl-C cancels in-flight downloads at their next tick instead of
letting non-daemon workers download to completion."""
import threading
import time
from esphome.framework_helpers import BatchDownloadProgress, run_batch_downloads
started = threading.Event()
ticks: list[int] = []
@@ -1183,8 +1181,8 @@ def test_run_batch_downloads_ctrl_c_aborts_in_flight_jobs() -> None:
t0 = time.monotonic()
with pytest.raises(KeyboardInterrupt):
run_batch_downloads(
BatchDownloadProgress("Downloading", 0),
[("boom", interrupter), ("slow", slow_download)],
"Downloading",
[("boom", 0, interrupter), ("slow", 0, slow_download)],
max_workers=2,
)
# Uncancelled, slow_download alone takes ~5s
@@ -1192,10 +1190,10 @@ def test_run_batch_downloads_ctrl_c_aborts_in_flight_jobs() -> None:
assert len(ticks) < 500
class TestBatchDownloadProgress:
class Test_BatchDownloadProgress:
def test_sums_trackers_into_one_bar(self) -> None:
with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
progress = BatchDownloadProgress("Downloading", 100)
progress = _BatchDownloadProgress("Downloading", 100)
a = progress.tracker()
b = progress.tracker()
a(10)
@@ -1209,13 +1207,13 @@ class TestBatchDownloadProgress:
def test_clamps_at_one(self) -> None:
"""Sizes are advisory; an over-delivering server never pushes past 100%."""
with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
progress = BatchDownloadProgress("Downloading", 10)
progress = _BatchDownloadProgress("Downloading", 10)
progress.tracker()(25)
assert bar_cls.return_value.update.call_args[0][0] == 1
def test_unknown_total_draws_nothing(self) -> None:
with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
progress = BatchDownloadProgress("Downloading", 0)
progress = _BatchDownloadProgress("Downloading", 0)
progress.tracker()(5)
progress.done()
bar_cls.assert_not_called()
@@ -1226,7 +1224,7 @@ class TestBatchDownloadProgress:
stream = io.StringIO()
stream.isatty = lambda: True # type: ignore[method-assign]
with patch("esphome.helpers.sys.stderr", stream):
progress = BatchDownloadProgress("Downloading", 10)
progress = _BatchDownloadProgress("Downloading", 10)
progress.tracker()(5)
progress.done()
assert stream.getvalue().endswith("50% \n")
@@ -1237,14 +1235,14 @@ class TestBatchDownloadProgress:
stream = io.StringIO()
stream.isatty = lambda: True # type: ignore[method-assign]
with patch("esphome.helpers.sys.stderr", stream):
BatchDownloadProgress("Downloading", 10).done()
_BatchDownloadProgress("Downloading", 10).done()
assert stream.getvalue() == ""
def test_done_after_full_bar_adds_nothing(self) -> None:
stream = io.StringIO()
stream.isatty = lambda: True # type: ignore[method-assign]
with patch("esphome.helpers.sys.stderr", stream):
progress = BatchDownloadProgress("Downloading", 10)
progress = _BatchDownloadProgress("Downloading", 10)
progress.tracker()(10)
progress.done()
assert stream.getvalue().endswith("100% Done...\r\n")
@@ -1261,6 +1259,22 @@ class TestDownloadFromMirrors:
assert url == "https://example.com/f"
assert target.read_bytes() == b"filedata"
def test_file_object_target_reports_progress(self) -> None:
"""The library prefetch's production path: a file-object target
streams through the mirror fallback and ticks the tracker."""
buf = io.BytesIO()
ticks: list[int] = []
with patch(
"requests.get",
return_value=_mock_response(b"filedata"),
):
url = download_from_mirrors(
["https://example.com/f"], {}, buf, progress=ticks.append
)
assert url == "https://example.com/f"
assert buf.getvalue() == b"filedata"
assert ticks and ticks[-1] == len(b"filedata")
def test_substitutions_applied_to_url(self, tmp_path: Path) -> None:
with patch(
"requests.get",
tests/unit_tests/test_platformio_library.py
+43 -67
View File
@@ -8,7 +8,6 @@ from contextlib import contextmanager
import json
import logging
from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -240,6 +239,7 @@ def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None:
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -280,8 +280,7 @@ def test_wave_requirement_growth_defers_the_superseded_download(tmp_path, monkey
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(ConvertedLibrary, "download", fake_download)
# Hermetic: unknown sizes take the sequential path instead of real HEADs
monkeypatch.setattr(lib, "_content_lengths", lambda urls: [None] * len(urls))
# Hermetic: the stubbed registry reports no size, so no batch prefetch
_patch_registry_resolve(monkeypatch)
top = convert_libraries(
[Library("esphome/A", "1.0.0", None), Library("esphome/B", None, None)],
@@ -674,22 +673,23 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel(
git/local sources and failures are left to the sequential call."""
calls: list[str] = []
def fake_download(self, force=False, salt="", namespace="", progress=None):
calls.append(self.source.url)
def fake_download(
self, dir_suffix, force=False, salt="", namespace="", progress=None
):
calls.append(self.url)
if progress is not None:
progress(0)
if "boom" in self.source.url:
if "boom" in self.url:
raise RuntimeError("boom")
monkeypatch.setattr(ConvertedLibrary, "download", fake_download)
monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls))
monkeypatch.setattr(URLSource, "download", fake_download)
wave = [
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))),
# Duplicate URL must prefetch once (two threads must never extract
# into the same cache directory)
("b2", ConvertedLibrary("b2", "1.0", URLSource("https://x/b.tar.gz"))),
("c", ConvertedLibrary("c", "1.0", URLSource("https://x/boom.tar.gz"))),
("b2", ConvertedLibrary("b2", "1.0", URLSource("https://x/b.tar.gz", 1))),
("c", ConvertedLibrary("c", "1.0", URLSource("https://x/boom.tar.gz", 1))),
("g", ConvertedLibrary("g", "*", lib.GitSource("https://x/g.git", None))),
]
lib._prefetch_wave(wave, "", "idf")
@@ -702,25 +702,27 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel(
assert "Prefetch of c failed (retrying sequentially)" in caplog.text
def test_prefetch_wave_unknown_size_falls_back_to_sequential(
setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
def test_prefetch_wave_unknown_size_left_to_sequential(
setup_core, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Any unknown HEAD size skips the parallel prefetch entirely so the
sequential downloads keep their per-file bars."""
def fail_download(self, force=False, salt="", namespace="", progress=None):
raise AssertionError("prefetched despite unknown size")
monkeypatch.setattr(ConvertedLibrary, "download", fail_download)
monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1, None])
"""Archives without a registry-reported size skip the batch (their
sequential per-file bars don't interleave); the known subset still
prefetches."""
calls: list[str] = []
monkeypatch.setattr(
URLSource,
"download",
lambda self, dir_suffix, force=False, salt="", namespace="", progress=None: (
calls.append(self.url)
),
)
wave = [
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))),
("u", ConvertedLibrary("u", "1.0", URLSource("https://x/u.tar.gz"))),
]
caplog.set_level("DEBUG")
lib._prefetch_wave(wave, "", "idf")
# The culprit URL is named so the fallback is traceable
assert "No Content-Length for https://x/b.tar.gz" in caplog.text
assert sorted(calls) == ["https://x/a.tar.gz", "https://x/b.tar.gz"]
def test_join_flag_args_empty_argument_warns_and_drops(
@@ -731,54 +733,24 @@ def test_join_flag_args_empty_argument_warns_and_drops(
assert "Ignoring '-D' with empty argument in build_flags" in caplog.text
def test_content_lengths_head_requests(monkeypatch: pytest.MonkeyPatch) -> None:
"""Sizes come from HEAD Content-Length; a failing HEAD reads as 0 so
the combined bar is skipped rather than wrong."""
import requests
def fake_head(url, timeout, allow_redirects):
if "bad" in url:
raise requests.ConnectionError("down")
if "gone" in url:
return SimpleNamespace(ok=False, status_code=404, headers={})
if "garbage" in url:
# A proxy/CDN doubling the header ("123, 123") or emitting junk
# must degrade to unknown, not ValueError the build
return SimpleNamespace(ok=True, headers={"content-length": "123, 123"})
return SimpleNamespace(ok=True, headers={"content-length": "123"})
monkeypatch.setattr(requests, "head", fake_head)
# None marks an unknown size (probe failure or non-2xx), distinct
# from a genuine zero
assert lib._content_lengths(
["https://x/a", "https://x/bad", "https://x/gone", "https://x/garbage"]
) == [
123,
None,
None,
None,
]
def test_prefetch_wave_cache_probe_failure_still_prefetches(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The cache probe is best-effort; a failing probe prefetches anyway."""
calls: list[str] = []
monkeypatch.setattr(
ConvertedLibrary,
URLSource,
"download",
lambda self, **kw: calls.append(self.source.url),
lambda self, dir_suffix, **kw: calls.append(self.url),
)
monkeypatch.setattr(
URLSource,
"is_cached",
lambda self, *a, **kw: (_ for _ in ()).throw(RuntimeError("no core")),
)
monkeypatch.setattr(lib, "_content_lengths", lambda urls: [1] * len(urls))
wave = [
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz"))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz"))),
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))),
]
lib._prefetch_wave(wave, "", "idf")
assert sorted(calls) == ["https://x/a.tar.gz", "https://x/b.tar.gz"]
@@ -790,13 +762,15 @@ def test_prefetch_wave_warm_cache_is_silent(
"""Already-extracted archives download nothing; a warm build must not
print a Downloading line or draw a bar."""
monkeypatch.setattr(
ConvertedLibrary,
URLSource,
"download",
lambda self, **kw: (_ for _ in ()).throw(AssertionError("downloaded")),
lambda self, dir_suffix, **kw: (_ for _ in ()).throw(
AssertionError("downloaded")
),
)
wave = []
for name in ("a", "b", "c"):
comp = ConvertedLibrary(name, "1.0", URLSource(f"https://x/{name}.tar.gz"))
comp = ConvertedLibrary(name, "1.0", URLSource(f"https://x/{name}.tar.gz", 1))
marker_dir = comp.source._cache_dir(comp.get_sanitized_name(), "", "idf")
marker_dir.mkdir(parents=True)
(marker_dir / ".esphome_extracted").touch()
@@ -811,12 +785,14 @@ def test_prefetch_wave_single_archive_skips_the_pool(
"""One archive gains nothing from a pool; the sequential call keeps its
progress bar."""
monkeypatch.setattr(
ConvertedLibrary,
URLSource,
"download",
lambda self, **kw: (_ for _ in ()).throw(AssertionError("prefetched")),
lambda self, dir_suffix, **kw: (_ for _ in ()).throw(
AssertionError("prefetched")
),
)
lib._prefetch_wave(
[("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz")))],
[("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1)))],
"",
"idf",
)