Merge remote-tracking branch 'upstream/dev' into noise-session-resume

This commit is contained in:
J. Nick Koston
2026-08-24 17:37:39 -05:00
44 changed files with 1994 additions and 377 deletions
@@ -13,17 +13,18 @@ namespace api {
class APIConnection;
} // namespace api
namespace uart {
enum class UARTFlushResult : uint8_t {
UART_FLUSH_RESULT_SUCCESS,
UART_FLUSH_RESULT_ASSUMED_SUCCESS,
UART_FLUSH_RESULT_TIMEOUT,
UART_FLUSH_RESULT_FAILED,
};
} // namespace uart
namespace serial_proxy {
enum class SerialProxyResult : uint8_t {
SERIAL_PROXY_RESULT_OK,
SERIAL_PROXY_RESULT_ASSUMED_SUCCESS,
SERIAL_PROXY_RESULT_PORT_IN_USE,
SERIAL_PROXY_RESULT_INVALID_ARGUMENT,
SERIAL_PROXY_RESULT_ERROR,
SERIAL_PROXY_RESULT_TIMEOUT,
SERIAL_PROXY_RESULT_NOT_SUPPORTED,
};
class SerialProxy {
public:
void set_instance_index(uint32_t index) { this->instance_index_ = index; }
@@ -31,13 +32,20 @@ class SerialProxy {
const char *get_name() const { return ""; }
api::enums::SerialProxyPortType get_port_type() const { return {}; }
api::APIConnection *get_api_connection() { return nullptr; }
void serial_proxy_request(api::APIConnection *conn, api::enums::SerialProxyRequestType type) {}
void configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity,
uint32_t stop_bits, uint32_t data_size) {}
SerialProxyResult serial_proxy_request(api::APIConnection *conn, api::enums::SerialProxyRequestType type) {
return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
}
SerialProxyResult configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity,
uint8_t stop_bits, uint8_t data_size) {
return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
}
void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {}
void set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) {}
SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) {
return SerialProxyResult::SERIAL_PROXY_RESULT_OK;
}
uint32_t get_modem_pins() const { return 0; }
uart::UARTFlushResult flush_port() { return uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS; }
uint32_t get_configured_modem_pins() const { return 0; }
SerialProxyResult flush_port(api::APIConnection *api_connection) { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; }
protected:
uint32_t instance_index_{0};
@@ -15,7 +15,9 @@ namespace zwave_proxy {
class ZWaveProxy {
public:
api::APIConnection *get_api_connection() { return nullptr; }
void zwave_proxy_request(api::APIConnection *conn, api::enums::ZWaveProxyRequestType type) {}
api::enums::ZWaveProxyStatus zwave_proxy_request(api::APIConnection *conn, api::enums::ZWaveProxyRequestType type) {
return api::enums::ZWAVE_PROXY_STATUS_OK;
}
void send_frame(api::APIConnection *api_connection, const uint8_t *data, size_t length) {}
void api_connection_authenticated(api::APIConnection *conn) {}
uint32_t get_feature_flags() const { return 0; }
@@ -0,0 +1,61 @@
#include <gtest/gtest.h>
#include <optional>
#include "esphome/components/runtime_image/runtime_image.h"
namespace esphome::runtime_image::testing {
TEST(RuntimeImageMime, FormatForKnownMimeTypes) {
EXPECT_EQ(get_format_for_mime_type("image/bmp"), BMP);
EXPECT_EQ(get_format_for_mime_type("image/x-ms-bmp"), BMP);
EXPECT_EQ(get_format_for_mime_type("image/x-bmp"), BMP);
EXPECT_EQ(get_format_for_mime_type("image/png"), PNG);
EXPECT_EQ(get_format_for_mime_type("image/x-png"), PNG);
#ifdef USE_RUNTIME_IMAGE_JPEG
EXPECT_EQ(get_format_for_mime_type("image/jpeg"), JPEG);
EXPECT_EQ(get_format_for_mime_type("image/jpg"), JPEG);
#endif // USE_RUNTIME_IMAGE_JPEG
}
TEST(RuntimeImageMime, FormatMatchingIsCaseInsensitive) {
EXPECT_EQ(get_format_for_mime_type("Image/PNG"), PNG);
EXPECT_EQ(get_format_for_mime_type("IMAGE/BMP"), BMP);
}
TEST(RuntimeImageMime, FormatMatchesContentTypeWithParameters) {
// Content-Type headers may carry parameters after the media type
EXPECT_EQ(get_format_for_mime_type("image/png; charset=binary"), PNG);
EXPECT_EQ(get_format_for_mime_type("image/bmp;name=\"a.bmp\""), BMP);
}
TEST(RuntimeImageMime, UnknownMimeTypeHasNoFormat) {
EXPECT_EQ(get_format_for_mime_type("text/html"), std::nullopt);
EXPECT_EQ(get_format_for_mime_type("application/octet-stream"), std::nullopt);
EXPECT_EQ(get_format_for_mime_type("image/*"), std::nullopt);
EXPECT_EQ(get_format_for_mime_type(""), std::nullopt);
EXPECT_EQ(get_format_for_mime_type(nullptr), std::nullopt);
}
TEST(RuntimeImageMime, MimeTypeForFormatRoundTrip) {
EXPECT_STREQ(get_mime_type_for_format(BMP), "image/bmp");
EXPECT_STREQ(get_mime_type_for_format(PNG), "image/png");
#ifdef USE_RUNTIME_IMAGE_JPEG
EXPECT_STREQ(get_mime_type_for_format(JPEG), "image/jpeg");
#endif // USE_RUNTIME_IMAGE_JPEG
// AUTO has no single MIME type and falls back to the wildcard
EXPECT_STREQ(get_mime_type_for_format(AUTO), "image/*");
// Every decodable format must resolve back to itself through its MIME type
for (ImageFormat format : {
BMP,
PNG,
#ifdef USE_RUNTIME_IMAGE_JPEG
JPEG,
#endif // USE_RUNTIME_IMAGE_JPEG
}) {
EXPECT_EQ(get_format_for_mime_type(get_mime_type_for_format(format)), format) << format;
}
}
} // namespace esphome::runtime_image::testing
+3
View File
@@ -41,3 +41,6 @@ number:
min_value: 2
max_value: 100
step: 1
time:
- platform: zigbee
@@ -10,6 +10,3 @@ zigbee:
on_start:
then:
- logger.log: "Started zigbee stack"
time:
- platform: zigbee
@@ -5,3 +5,6 @@ zigbee:
on_join:
then:
- logger.log: "Joined network"
time:
- platform: zigbee
@@ -263,6 +263,17 @@ def test_device_capabilities_response_has_id_150() -> None:
)
def test_z_wave_proxy_request_response_has_id_151() -> None:
body = _extract_proto_message(PROTO_TEXT, "ZWaveProxyRequestResponse")
match = re.search(r"option \(id\) = (\d+);", body)
assert match is not None, "ZWaveProxyRequestResponse is missing `option (id)`"
assert int(match.group(1)) == 151, (
f"ZWaveProxyRequestResponse has id {match.group(1)}, expected 151. "
"Message ids are part of the wire protocol and must not change once "
"assigned."
)
def test_superseded_fields_are_not_marked_deprecated_in_proto() -> None:
"""The six superseded fields must not carry `[deprecated = true]` in
api.proto, or the generator drops them and old clients stop receiving
+12 -4
View File
@@ -660,7 +660,7 @@ def _patch_registry(monkeypatch, versions):
def test_resolve_registry_version_intersects_constraints(monkeypatch):
_patch_registry(monkeypatch, ["1.10018.1", "1.10021.0", "1.10021.1"])
owner, name, version, url = _resolve_registry_version(
owner, name, version, url, _size = _resolve_registry_version(
"esphome", "libsodium", {"==1.10021.0", "^1.10018.1"}
)
assert (owner, name, version) == ("esphome", "libsodium", "1.10021.0")
@@ -669,7 +669,9 @@ def test_resolve_registry_version_intersects_constraints(monkeypatch):
def test_resolve_registry_version_picks_highest_satisfying(monkeypatch):
_patch_registry(monkeypatch, ["1.0.0", "1.5.0", "2.0.0"])
_owner, _name, version, _url = _resolve_registry_version("o", "p", {"^1.0.0"})
_owner, _name, version, _url, _size = _resolve_registry_version(
"o", "p", {"^1.0.0"}
)
assert version == "1.5.0"
@@ -719,7 +721,7 @@ def test_generate_idf_components_dedupes_shared_dependency(
resolve_calls.append(pkgname)
captured[f"{owner}/{pkgname}"] = set(requirements)
version = "1.10021.0" if pkgname == "C" else "1.0.0"
return owner, pkgname, version, f"http://x/{pkgname}.tar.gz"
return owner, pkgname, version, f"http://x/{pkgname}.tar.gz", None
monkeypatch.setattr(
esphome.platformio.library, "_resolve_registry_version", fake_resolve
@@ -778,7 +780,7 @@ def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies(
def fake_resolve(owner, pkgname, requirements):
resolve_calls.append(pkgname)
return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz"
return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None
monkeypatch.setattr(
esphome.platformio.library, "_resolve_registry_version", fake_resolve
@@ -834,6 +836,7 @@ def test_generate_idf_components_handles_dependency_cycle(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -891,6 +894,7 @@ def test_generate_idf_components_git_overrides_registry_warns(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -927,6 +931,7 @@ def test_generate_idf_components_missing_manifest_raises(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -971,6 +976,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -1004,6 +1010,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -1040,6 +1047,7 @@ def test_generate_idf_components_incompatible_dependency_skipped(
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
+175 -11
View File
@@ -2,6 +2,7 @@
# pylint: disable=protected-access
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
import importlib.util
import io
@@ -14,7 +15,7 @@ import subprocess
import sys
import tarfile
from types import SimpleNamespace
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
@@ -887,6 +888,78 @@ _PREFETCH_JSON = json.dumps(
)
def test_prefetch_leaves_unverifiable_entries_to_the_installer(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""An entry missing sha256 or size must not download unverified; the
installer handles it and fails loudly on a bad archive."""
entries = json.loads(_PREFETCH_JSON)
del entries[0]["sha256"]
del entries[1]["size"]
entries.append(
{
"name": "gcc@14.2.0",
"url": "https://example.com/gcc.tar.gz",
"size": 67,
"sha256": "ef" * 32,
"dest": "gcc.tar.gz",
}
)
with (
patch(
"esphome.espidf.framework.run_command",
return_value=(True, json.dumps(entries), ""),
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls,
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
assert [call[0][0] for call in download.call_args_list] == [
"https://example.com/gcc.tar.gz"
]
assert download.call_args[1]["sha256"] == "ef" * 32
progress_cls.assert_called_once_with("Downloading ESP-IDF tools", 67)
assert "cmake@3.30.2 has no sha256/size" in caplog.text
assert "ninja@1.12.1 has no sha256/size" in caplog.text
def test_prefetch_all_entries_unverifiable_is_a_noop(tmp_path: Path) -> None:
entries = json.loads(_PREFETCH_JSON)
for entry in entries:
del entry["sha256"]
with (
patch(
"esphome.espidf.framework.run_command",
return_value=(True, json.dumps(entries), ""),
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
download.assert_not_called()
def test_prefetch_dedupes_entries_by_dest(tmp_path: Path) -> None:
"""Two entries resolving to one dest would interleave writes into the
same .part file; only the first downloads."""
entries = json.loads(_PREFETCH_JSON)
dup = dict(entries[0]) | {"name": "cmake-alias@3.30.2"}
entries.append(dup)
with (
patch(
"esphome.espidf.framework.run_command",
return_value=(True, json.dumps(entries), ""),
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.framework_helpers._BatchDownloadProgress"),
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
dests = [call[0][1].name for call in download.call_args_list]
assert dests.count("cmake-3.30.2.tar.gz") == 1
def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None:
with (
patch(
@@ -895,16 +968,58 @@ def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None:
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls,
):
# Materialize the lazy mock before threads race its first creation
tracker = progress_cls.return_value.tracker.return_value
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
dist = get_idf_tools_path() / "dist"
assert download.call_count == 2
assert download.call_args_list[0][0] == (
"https://example.com/cmake.tar.gz",
dist / "cmake-3.30.2.tar.gz",
)
assert download.call_args_list[0][1] == {"sha256": "ab" * 32, "size": 123}
# Archives download concurrently, so the call order is not fixed.
calls = {call[0]: call[1] for call in download.call_args_list}
assert set(calls) == {
("https://example.com/cmake.tar.gz", dist / "cmake-3.30.2.tar.gz"),
("https://example.com/ninja.zip", dist / "ninja.zip"),
}
kwargs = calls[("https://example.com/cmake.tar.gz", dist / "cmake-3.30.2.tar.gz")]
assert kwargs["sha256"] == "ab" * 32
assert kwargs["size"] == 123
# every archive reports into the one combined progress bar via the
# cancellation-checked wrapper; verify it delegates to the tracker
progress_cls.assert_called_once_with("Downloading ESP-IDF tools", 123 + 45)
before = tracker.call_count
for kw in calls.values():
kw["progress"](7)
assert tracker.call_count == before + len(calls)
def test_prefetch_downloads_archives_concurrently(tmp_path: Path) -> None:
"""More than one archive fans out over a bounded thread pool."""
entries = [
{
"name": f"tool{i}@1",
"url": f"https://example.com/tool{i}.tar.gz",
"size": 10,
"sha256": "ab" * 32,
"dest": f"tool{i}.tar.gz",
}
for i in range(6)
]
with (
patch(
"esphome.espidf.framework.run_command",
return_value=(True, json.dumps(entries), ""),
),
patch("esphome.espidf.framework.download_with_resume") as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch(
"esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor
) as pool,
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
pool.assert_called_once_with(max_workers=4)
assert download.call_count == 6
def test_prefetch_skips_already_downloaded_archives(tmp_path: Path) -> None:
@@ -959,11 +1074,12 @@ def test_prefetch_failures_never_raise(
assert expected_log in caplog.text
def test_prefetch_one_failed_archive_does_not_stop_the_rest(
def test_prefetch_total_failure_logs_error(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A single archive failing its download must not abort the prefetch of
the remaining archives."""
"""Every archive failing is a systematic fault (proxy, bad kwarg), not
a flaky mirror; it must be distinguishable at ERROR because the resume
workaround is off for the whole install."""
with (
patch(
"esphome.espidf.framework.run_command",
@@ -971,7 +1087,32 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest(
),
patch(
"esphome.espidf.framework.download_with_resume",
side_effect=[OSError("network down"), None],
side_effect=OSError("proxy refuses everything"),
),
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
):
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
assert "Every ESP-IDF tool prefetch failed" in caplog.text
def test_prefetch_one_failed_archive_does_not_stop_the_rest(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A single archive failing its download must not abort the prefetch of
the remaining archives."""
def _fail_cmake_download(url: str, *args, **kwargs) -> None:
if "cmake" in url:
raise OSError("network down")
with (
patch(
"esphome.espidf.framework.run_command",
return_value=(True, _PREFETCH_JSON, ""),
),
patch(
"esphome.espidf.framework.download_with_resume",
side_effect=_fail_cmake_download,
) as download,
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
):
@@ -979,6 +1120,29 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest(
assert download.call_count == 2
assert "Could not prefetch cmake@3.30.2" in caplog.text
# One flaky archive is routine, never the systematic-fault ERROR
assert "Every ESP-IDF tool prefetch failed" not in caplog.text
def test_prefetch_finishes_progress_bar_and_cancels_queue(tmp_path: Path) -> None:
"""The batch bar is closed out after the pool, and the pool is shut down
with cancel_futures so Ctrl-C does not drain every queued archive."""
with (
patch(
"esphome.espidf.framework.run_command",
return_value=(True, _PREFETCH_JSON, ""),
),
patch("esphome.espidf.framework.download_with_resume"),
patch("esphome.espidf.framework.get_system_python_path", return_value="python"),
patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls,
patch("esphome.framework_helpers.ThreadPoolExecutor") as pool_cls,
):
pool = MagicMock(wraps=ThreadPoolExecutor(max_workers=2))
pool_cls.return_value = pool
_prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None)
pool.shutdown.assert_called_once_with(wait=True, cancel_futures=True)
progress_cls.return_value.done.assert_called_once_with()
def test_prefetch_passes_targets_and_tools_to_script(tmp_path: Path) -> None:
+49
View File
@@ -299,10 +299,13 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No
_setup_build(setup_core)
config = {CONF_ESPHOME: {}}
cmakecache = CORE.relative_build_path("build/CMakeCache.txt")
build_ninja = CORE.relative_build_path("build/build.ninja")
cmakecache.parent.mkdir(parents=True, exist_ok=True)
cmakecache.write_text("")
build_ninja.write_text("")
old = cmakecache.stat().st_mtime - 100
os.utime(cmakecache, (old, old))
os.utime(build_ninja, (old, old))
with (
patch.object(toolchain, "need_reconfigure", return_value=True),
@@ -314,6 +317,8 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No
assert toolchain.run_compile(config, verbose=False) == 0
assert cmakecache.stat().st_mtime > old
# build.ninja must not be older than the cache or ninja re-runs cmake
assert build_ninja.stat().st_mtime >= cmakecache.stat().st_mtime
def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None:
@@ -334,6 +339,50 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None:
assert not CORE.relative_build_path("build/CMakeCache.txt").exists()
def test_run_compile_reconfigures_after_full_write_outside_testing_mode(
setup_core: Path,
) -> None:
"""The full CMakeLists write is followed by a reconfigure (#18682); a
failure there stops the build and leaves the cache unstamped."""
_setup_build(setup_core)
config = {CONF_ESPHOME: {}}
cmakecache = CORE.relative_build_path("build/CMakeCache.txt")
cmakecache.parent.mkdir(parents=True, exist_ok=True)
cmakecache.write_text("")
old = cmakecache.stat().st_mtime - 100
os.utime(cmakecache, (old, old))
calls: list[tuple] = []
reconfigures = 0
def record_write(minimal: bool = False) -> None:
calls.append(("write_project", minimal))
def record_reconfigure() -> int:
nonlocal reconfigures
reconfigures += 1
calls.append(("run_reconfigure",))
return 1 if reconfigures == 2 else 0
with (
patch.object(toolchain, "need_reconfigure", return_value=True),
patch("esphome.build_gen.espidf.write_project", side_effect=record_write),
patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure),
patch.object(toolchain, "run_idf_py", return_value=0) as mock_build,
patch.object(toolchain, "print_summary"),
):
assert not CORE.testing_mode
assert toolchain.run_compile(config, verbose=False) == 1
assert calls == [
("write_project", True),
("run_reconfigure",),
("write_project", False),
("run_reconfigure",),
]
mock_build.assert_not_called()
assert cmakecache.stat().st_mtime == old
def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None:
"""compile_process_limit is forwarded to run_idf_py as the job limit."""
_setup_build(setup_core)
+6 -2
View File
@@ -998,10 +998,10 @@ def test_progress_bar(capsys: CaptureFixture[str]) -> None:
assert "100%" in captured.err
assert "Done" in captured.err
# Test done method
# done() after the 100% frame adds nothing; that frame ended its line
progress.done()
captured = capsys.readouterr()
assert captured.err == "\n"
assert captured.err == ""
# Test same progress doesn't update
progress.update(0.5)
@@ -1010,6 +1010,10 @@ def test_progress_bar(capsys: CaptureFixture[str]) -> None:
# Should only see one update (second call shouldn't write)
assert captured.err.count("50%") == 1
# done() after a mid-way frame ends the line
progress.done()
assert capsys.readouterr().err == "\n"
# Tests for SHA256 authentication
@pytest.mark.usefixtures("mock_time")
+275
View File
@@ -12,6 +12,8 @@ from pathlib import Path
import subprocess
import sys
import tarfile
import threading
import time
from unittest.mock import MagicMock, Mock, call, patch
import zipfile
@@ -22,6 +24,7 @@ from esphome import framework_helpers
from esphome.core import EsphomeError
from esphome.framework_helpers import (
_7z_extract_all,
_BatchDownloadProgress,
_detect_archive_root,
_rename_with_retry,
_tar_extract_all,
@@ -36,6 +39,7 @@ from esphome.framework_helpers import (
get_python_env_executable_path,
get_system_python_path,
rmdir,
run_batch_downloads,
run_command,
run_command_ok,
str_to_lst_of_str,
@@ -1111,6 +1115,218 @@ class TestDownloadWithResume:
assert mock_get.call_args[1]["headers"] == {}
assert dest.read_bytes() == b"data"
def test_progress_callback_reports_absolute_bytes(self, tmp_path: Path) -> None:
"""With a callback no bar is drawn; the callback sees the running
byte count of this file, then its final verified size."""
dest = tmp_path / "tool.tar.gz"
resp = _mock_response(b"")
resp.headers = {"content-length": "7"}
resp.iter_content.return_value = [b"1234", b"567"]
seen: list[int] = []
with (
patch("requests.get", return_value=resp),
patch("esphome.framework_helpers.ProgressBar") as bar_cls,
):
download_with_resume(
"https://example.com/t", dest, size=7, progress=seen.append
)
assert seen == [0, 4, 7, 7]
bar_cls.assert_not_called()
def test_progress_callback_seeds_with_resume_offset(self, tmp_path: Path) -> None:
dest = tmp_path / "tool.tar.gz"
(tmp_path / "tool.tar.gz.part").write_bytes(b"12345")
good = hashlib.sha256(b"12345678").hexdigest()
seen: list[int] = []
with patch("requests.get", return_value=_resumed_response(b"678")):
download_with_resume(
"https://example.com/t", dest, sha256=good, size=8, progress=seen.append
)
assert seen[0] == 5
assert seen[-1] == 8
def test_progress_callback_credits_already_complete_download(
self, tmp_path: Path
) -> None:
"""A verified dest from an earlier run still counts toward the batch."""
dest = tmp_path / "tool.tar.gz"
dest.write_bytes(b"12345678")
seen: list[int] = []
with patch("requests.get") as mock_get:
download_with_resume(
"https://example.com/t", dest, size=8, progress=seen.append
)
mock_get.assert_not_called()
assert seen == [8]
def test_run_batch_downloads_ctrl_c_aborts_in_flight_jobs() -> None:
"""Ctrl-C cancels in-flight downloads at their next tick instead of
letting non-daemon workers download to completion."""
started = threading.Event()
ticks: list[int] = []
def interrupter(tracker) -> None:
started.wait(5)
raise KeyboardInterrupt
def slow_download(tracker) -> None:
started.set()
for i in range(500):
tracker(i)
ticks.append(i)
time.sleep(0.01)
t0 = time.monotonic()
with pytest.raises(KeyboardInterrupt):
run_batch_downloads(
"Downloading",
[("boom", 0, interrupter), ("slow", 0, slow_download)],
max_workers=2,
)
# Uncancelled, slow_download alone takes ~5s
assert time.monotonic() - t0 < 3
assert len(ticks) < 500
def test_cancellation_escapes_broad_except_in_fetch() -> None:
"""A fetch that wraps its work in except Exception cannot swallow the
Ctrl-C sentinel (it is a BaseException)."""
from esphome.framework_helpers import _BatchDownloadCancelled
started = threading.Event()
swallowed = []
def interrupter(tracker) -> None:
started.wait(5)
raise KeyboardInterrupt
def greedy_fetch(tracker) -> None:
started.set()
try:
for i in range(500):
tracker(i)
time.sleep(0.01)
except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught
swallowed.append(err)
t0 = time.monotonic()
with pytest.raises(KeyboardInterrupt):
run_batch_downloads(
"Downloading",
[("boom", 0, interrupter), ("greedy", 0, greedy_fetch)],
max_workers=2,
)
assert time.monotonic() - t0 < 3
assert not swallowed
assert issubclass(_BatchDownloadCancelled, BaseException)
assert not issubclass(_BatchDownloadCancelled, Exception)
def test_logging_guard_ends_the_bar_row_before_a_record() -> None:
r"""A worker warning gets its own line instead of the bar's \r row."""
stream = io.StringIO()
stream.isatty = lambda: True # type: ignore[method-assign]
with patch("esphome.helpers.sys.stderr", stream):
progress = _BatchDownloadProgress("Downloading", 10)
progress.tracker()(5)
with progress.logging_guard():
logging.getLogger("esphome.test").warning("mirror retry")
# The partial 50% frame ended its line before the record was emitted
assert stream.getvalue().endswith("50% \n")
# And the next tick redraws the frame on a fresh row
progress.tracker()(2)
assert stream.getvalue().endswith("70% ")
def test_logging_guard_without_a_bar_is_a_no_op() -> None:
"""An unknown total draws no bar; the guard passes records through."""
progress = _BatchDownloadProgress("Downloading", 0)
with progress.logging_guard():
logging.getLogger("esphome.test").warning("plain record")
def test_cancellable_sleep_sleeps_between_ticks() -> None:
"""An uncancelled backoff actually waits out its delay in slices."""
from esphome.framework_helpers import _cancellable_sleep
ticks: list[int] = []
t0 = time.monotonic()
_cancellable_sleep(0.05, ticks.append, 3)
assert time.monotonic() - t0 >= 0.05
assert ticks and all(t == 3 for t in ticks)
def test_cancellable_sleep_aborts_at_the_tick() -> None:
"""A backoff sleep observes the cancellation raise promptly."""
from esphome.framework_helpers import _BatchDownloadCancelled, _cancellable_sleep
def cancelled_tick(done: int) -> None:
raise _BatchDownloadCancelled
t0 = time.monotonic()
with pytest.raises(_BatchDownloadCancelled):
_cancellable_sleep(30, cancelled_tick, 0)
assert time.monotonic() - t0 < 1
class Test_BatchDownloadProgress:
def test_sums_trackers_into_one_bar(self) -> None:
with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
progress = _BatchDownloadProgress("Downloading", 100)
a = progress.tracker()
b = progress.tracker()
a(10)
b(20)
a(30)
a(0) # a restart from zero takes that file's bytes back out
bar_cls.assert_called_once_with("Downloading")
updates = [c[0][0] for c in bar_cls.return_value.update.call_args_list]
assert updates == [0.1, 0.3, 0.5, 0.2]
def test_clamps_at_one(self) -> None:
"""Sizes are advisory; an over-delivering server never pushes past 100%."""
with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
progress = _BatchDownloadProgress("Downloading", 10)
progress.tracker()(25)
assert bar_cls.return_value.update.call_args[0][0] == 1
def test_unknown_total_draws_nothing(self) -> None:
with patch("esphome.framework_helpers.ProgressBar") as bar_cls:
progress = _BatchDownloadProgress("Downloading", 0)
progress.tracker()(5)
progress.done()
bar_cls.assert_not_called()
def test_done_ends_an_unfinished_bar(self) -> None:
"""A batch that stops short of 100% (a failed archive) still ends its
line so the next log message starts on a fresh row."""
stream = io.StringIO()
stream.isatty = lambda: True # type: ignore[method-assign]
with patch("esphome.helpers.sys.stderr", stream):
progress = _BatchDownloadProgress("Downloading", 10)
progress.tracker()(5)
progress.done()
assert stream.getvalue().endswith("50% \n")
def test_done_before_any_frame_writes_nothing(self) -> None:
"""A batch aborted before any tracker fired must not emit a stray
newline for a bar that was never drawn."""
stream = io.StringIO()
stream.isatty = lambda: True # type: ignore[method-assign]
with patch("esphome.helpers.sys.stderr", stream):
_BatchDownloadProgress("Downloading", 10).done()
assert stream.getvalue() == ""
def test_done_after_full_bar_adds_nothing(self) -> None:
stream = io.StringIO()
stream.isatty = lambda: True # type: ignore[method-assign]
with patch("esphome.helpers.sys.stderr", stream):
progress = _BatchDownloadProgress("Downloading", 10)
progress.tracker()(10)
progress.done()
assert stream.getvalue().endswith("100% Done...\r\n")
class TestDownloadFromMirrors:
def test_success_returns_url_and_writes_content(self, tmp_path: Path) -> None:
@@ -1123,6 +1339,22 @@ class TestDownloadFromMirrors:
assert url == "https://example.com/f"
assert target.read_bytes() == b"filedata"
def test_file_object_target_reports_progress(self) -> None:
"""The library prefetch's production path: a file-object target
streams through the mirror fallback and ticks the tracker."""
buf = io.BytesIO()
ticks: list[int] = []
with patch(
"requests.get",
return_value=_mock_response(b"filedata"),
):
url = download_from_mirrors(
["https://example.com/f"], {}, buf, progress=ticks.append
)
assert url == "https://example.com/f"
assert buf.getvalue() == b"filedata"
assert ticks and ticks[-1] == len(b"filedata")
def test_substitutions_applied_to_url(self, tmp_path: Path) -> None:
with patch(
"requests.get",
@@ -1468,6 +1700,49 @@ class TestDownloadFromMirrors:
assert mock_get.call_count == 2
mock_sleep.assert_called_once_with(2)
def test_backoff_tick_reports_filelike_bytes(self) -> None:
"""For a file-like target the backoff tick carries f.tell(), so the
combined bar holds steady through the sweep retry."""
target = io.BytesIO()
ticks: list[int] = []
with (
patch(
"requests.get",
side_effect=[
req.ConnectionError("down"),
_mock_response(b"data"),
],
),
patch("esphome.framework_helpers._cancellable_sleep") as mock_sleep,
):
download_from_mirrors(
["https://mirror1.com/f"], {}, target, progress=ticks.append
)
# No bytes had streamed at backoff time, so the tick carries 0
assert mock_sleep.call_args == call(2, ticks.append, 0)
assert target.getvalue() == b"data"
def test_backoff_tick_reports_partial_bytes(self, tmp_path: Path) -> None:
"""The backoff tick carries the bytes already in the part file, so a
combined bar holds steady instead of rewinding to zero."""
dest = tmp_path / "out.bin"
(tmp_path / "out.bin.part").write_bytes(b"12345")
ticks: list[int] = []
with (
patch(
"requests.get",
side_effect=[
req.ConnectionError("down"),
_mock_response(b"data"),
],
),
patch("esphome.framework_helpers._cancellable_sleep") as mock_sleep,
):
download_from_mirrors(
["https://mirror1.com/f"], {}, dest, progress=ticks.append
)
assert mock_sleep.call_args == call(2, ticks.append, 5)
def test_permanent_failure_does_not_retry_sweep(self, tmp_path: Path) -> None:
"""An HTTP 404 will not heal on its own; fail after a single pass."""
with (
+14
View File
@@ -1124,6 +1124,20 @@ def test_progressbar_enabled_on_pipe_with_dashboard(monkeypatch) -> None:
assert bar.enabled is True
def test_progressbar_interrupt_keeps_finished_bar_done(monkeypatch) -> None:
"""interrupt() on a bar whose 100% frame already ended its own line
must not reset it, or the next tick would redraw a second Done row."""
stream = MagicMock(spec=io.TextIOWrapper)
stream.isatty.return_value = True
monkeypatch.setattr(CORE, "dashboard", False)
bar = ProgressBar("Uploading", stream=stream)
bar.update(1)
assert bar.last_progress == 100
bar.interrupt()
assert bar.last_progress == 100
@pytest.mark.parametrize(
("seconds", "expected"),
[
+193 -2
View File
@@ -153,13 +153,15 @@ def test_localsource_download_returns_empty_build_dir(setup_core: Path) -> None:
assert plain != out
def test_urlsource_download_extracts_then_reuses_marker(setup_core, monkeypatch):
def test_urlsource_download_extracts_then_reuses_marker(
setup_core, monkeypatch, caplog
):
monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None)
dl_calls: list[list[str]] = []
monkeypatch.setattr(
lib,
"download_from_mirrors",
lambda urls, headers, f: dl_calls.append(urls),
lambda urls, headers, f, progress=None: dl_calls.append(urls),
)
def fake_extract(fileobj, path):
@@ -178,6 +180,12 @@ def test_urlsource_download_extracts_then_reuses_marker(setup_core, monkeypatch)
assert out2 == out
assert len(dl_calls) == 1
# A batch caller passes a tracker and owns the messaging; no per-file INFO
caplog.set_level("INFO")
src.download("mylib-batch", progress=lambda done: None)
assert len(dl_calls) == 2
assert "Downloading" not in caplog.text
def test_resolve_registry_version_raises_without_pkg_file(monkeypatch):
registry = lib._make_registry_client()
@@ -211,6 +219,7 @@ def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None:
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
None,
),
)
@@ -230,6 +239,38 @@ def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properti
_patch_registry_resolve(monkeypatch)
def test_wave_requirement_growth_defers_the_superseded_download(tmp_path, monkeypatch):
"""A's manifest constrains B while B sits in the same wave: B's
drain-time resolution is superseded, so its download defers to the
next wave instead of fetching a version that is immediately replaced."""
download_names: list[str] = []
manifests = {
"esphome/A": {
"name": "A",
"build": {},
"dependencies": {"esphome/B": ">=1.0"},
},
"esphome/B": {"name": "B", "build": {}},
}
def fake_download(self, force=False, salt="", namespace="", progress=None):
download_names.append(self.name)
self.path = tmp_path / self.get_require_name()
self.path.mkdir(parents=True, exist_ok=True)
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(ConvertedLibrary, "download", fake_download)
# Hermetic: the stubbed registry reports no size, so no batch prefetch
_patch_registry_resolve(monkeypatch)
top = convert_libraries(
[Library("esphome/A", "1.0.0", None), Library("esphome/B", None, None)],
_backend(),
)
assert sorted(c.name for c in top) == ["esphome/A", "esphome/B"]
# B downloads exactly once, after its requirement set stabilized
assert download_names.count("esphome/B") == 1
def test_convert_libraries_parses_library_properties(tmp_path, monkeypatch):
# A manifest provided as library.properties (Arduino style) instead of
# library.json must still be parsed and converted.
@@ -574,6 +615,65 @@ def test_lex_build_flags_dangling_flag_does_not_cross_entries(
assert "Ignoring trailing '-I'" in caplog.text
def test_prefetch_wave_downloads_registry_archives_in_parallel(
setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""Registry archives in one wave download concurrently, deduped by URL;
git/local sources and failures are left to the sequential call."""
calls: list[str] = []
def fake_download(
self, dir_suffix, force=False, salt="", namespace="", progress=None
):
calls.append(self.url)
if progress is not None:
progress(0)
if "boom" in self.url:
raise RuntimeError("boom")
monkeypatch.setattr(URLSource, "download", fake_download)
wave = [
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))),
# Duplicate URL must prefetch once (two threads must never extract
# into the same cache directory)
("b2", ConvertedLibrary("b2", "1.0", URLSource("https://x/b.tar.gz", 1))),
("c", ConvertedLibrary("c", "1.0", URLSource("https://x/boom.tar.gz", 1))),
("g", ConvertedLibrary("g", "*", lib.GitSource("https://x/g.git", None))),
]
lib._prefetch_wave(wave, "", "idf")
assert sorted(calls) == [
"https://x/a.tar.gz",
"https://x/b.tar.gz",
"https://x/boom.tar.gz",
]
# The failure surfaces at default verbosity, after the bar
assert "Prefetch of c failed (retrying sequentially)" in caplog.text
def test_prefetch_wave_unknown_size_left_to_sequential(
setup_core, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Archives without a registry-reported size skip the batch (their
sequential per-file bars don't interleave); the known subset still
prefetches."""
calls: list[str] = []
monkeypatch.setattr(
URLSource,
"download",
lambda self, dir_suffix, force=False, salt="", namespace="", progress=None: (
calls.append(self.url)
),
)
wave = [
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))),
("u", ConvertedLibrary("u", "1.0", URLSource("https://x/u.tar.gz"))),
]
lib._prefetch_wave(wave, "", "idf")
assert sorted(calls) == ["https://x/a.tar.gz", "https://x/b.tar.gz"]
def test_join_flag_args_empty_argument_warns_and_drops(
caplog: pytest.LogCaptureFixture,
) -> None:
@@ -582,6 +682,97 @@ def test_join_flag_args_empty_argument_warns_and_drops(
assert "Ignoring '-D' with empty argument in build_flags" in caplog.text
def test_prefetch_wave_cache_probe_failure_still_prefetches(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A filesystem probe failure warns (a systematic one re-downloads
everything) but still prefetches; a programming error is NOT swallowed
here, it reaches the outer blanket guard."""
calls: list[str] = []
monkeypatch.setattr(
URLSource,
"download",
lambda self, dir_suffix, **kw: calls.append(self.url),
)
monkeypatch.setattr(
URLSource,
"is_cached",
lambda self, *a, **kw: (_ for _ in ()).throw(OSError("cache root denied")),
)
wave = [
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))),
]
lib._prefetch_wave(wave, "", "idf")
assert sorted(calls) == ["https://x/a.tar.gz", "https://x/b.tar.gz"]
assert "Cache probe for a failed: cache root denied" in caplog.text
def test_prefetch_wave_internal_error_never_fails_the_build(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""The blanket guard keeps a prefetch bug from failing the walk."""
monkeypatch.setattr(URLSource, "is_cached", lambda self, *a, **kw: False)
monkeypatch.setattr(
lib,
"run_batch_downloads",
lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("bug")),
)
wave = [
("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))),
("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))),
]
lib._prefetch_wave(wave, "", "idf")
assert "Library prefetch failed: bug" in caplog.text
def test_prefetch_wave_warm_cache_is_silent(
setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""Already-extracted archives download nothing; a warm build must not
print a Downloading line or draw a bar."""
monkeypatch.setattr(
URLSource,
"download",
lambda self, dir_suffix, **kw: (_ for _ in ()).throw(
AssertionError("downloaded")
),
)
wave = []
for name in ("a", "b", "c"):
comp = ConvertedLibrary(name, "1.0", URLSource(f"https://x/{name}.tar.gz", 1))
marker_dir = comp.source._cache_dir(comp.get_sanitized_name(), "", "idf")
marker_dir.mkdir(parents=True)
(marker_dir / ".esphome_extracted").touch()
wave.append((name, comp))
lib._prefetch_wave(wave, "", "idf")
assert "Downloading" not in caplog.text
def test_prefetch_wave_single_archive_uses_the_batch(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A dependency chain discovers one archive per wave; it downloads
through the same runner so there is one download method and one bar."""
caplog.set_level("INFO")
calls: list[str] = []
monkeypatch.setattr(URLSource, "is_cached", lambda self, *a, **kw: False)
monkeypatch.setattr(
URLSource,
"download",
lambda self, dir_suffix, force=False, salt="", namespace="", progress=None: (
calls.append(self.url)
),
)
lib._prefetch_wave(
[("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1)))],
"",
"idf",
)
assert calls == ["https://x/a.tar.gz"]
assert "Downloading 1 library archive(s): a" in caplog.text
def test_normalize_dependencies_forms(caplog) -> None:
"""Every PIO-legal spelling normalizes; unrecognizable entries warn."""
from esphome.platformio.library import normalize_dependencies