From 9daae377fca5eea6d1f39d13fa33e790d50b2f9c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:03:14 -0500 Subject: [PATCH 01/16] [api] Bump noise-c to 0.1.20 (#18482) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index cdc0d97c49..3e69d5842c 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.19") + cg.add_library("esphome/noise-c", "0.1.20") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 39600d622a..13bb5a556f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.19 ; api + esphome/noise-c@0.1.20 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.19 ; api + esphome/noise-c@0.1.20 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.19 ; used by api + esphome/noise-c@0.1.20 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 200a1644a5d12c30e4f0d562f1e0132b96482dc6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:03:33 -0500 Subject: [PATCH 02/16] [ci] Fail the benchmark job when the C++ benchmark build fails (#18480) --- .github/workflows/ci.yml | 17 ++++++++++++++--- tests/benchmarks/components/api/__init__.py | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b603e68ad7..2075fde9ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -460,10 +460,21 @@ jobs: - name: Build benchmarks id: build run: | + # pipefail: without it a failed build is masked by the grep/cut + # pipeline below, leaving BINARY empty and silently dropping every + # C++ benchmark from the run while the job still reports success. + set -o pipefail . venv/bin/activate - export BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) - # --build-only prints BUILD_BINARY= to stdout - BINARY=$(script/cpp_benchmark.py --all --build-only | grep '^BUILD_BINARY=' | tail -1 | cut -d= -f2-) + BENCHMARK_LIB_CONFIG=$(python script/setup_codspeed_lib.py) + export BENCHMARK_LIB_CONFIG + # --build-only prints BUILD_BINARY= to stdout; the grep is + # non-fatal so a missing marker reaches the check below instead of + # tripping errexit at this assignment + BINARY=$(script/cpp_benchmark.py --all --build-only | { grep '^BUILD_BINARY=' || true; } | tail -1 | cut -d= -f2-) + if [ -z "$BINARY" ]; then + echo "::error::Benchmark build did not report a binary path" + exit 1 + fi echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks diff --git a/tests/benchmarks/components/api/__init__.py b/tests/benchmarks/components/api/__init__.py index 0d02e0b054..0565bc5330 100644 --- a/tests/benchmarks/components/api/__init__.py +++ b/tests/benchmarks/components/api/__init__.py @@ -15,6 +15,7 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # components have hardware dependencies (BLE/UART/RMT); lightweight # stub headers in tests/benchmarks/stubs/ satisfy the includes. cg.add_define("USE_BLUETOOTH_PROXY") + cg.add_define("USE_BLUETOOTH_PROXY_CONNECTIONS") cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", 3) cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) cg.add_define("USE_ZWAVE_PROXY") From a99a8f364e8bad032d89d65e18e2470fd2df2267 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 09:32:37 -0500 Subject: [PATCH 03/16] [api] Bump noise-c to 0.1.21 (#18484) --- esphome/components/api/__init__.py | 2 +- platformio.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3e69d5842c..912d580a0f 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -497,7 +497,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") - cg.add_library("esphome/noise-c", "0.1.20") + cg.add_library("esphome/noise-c", "0.1.21") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 13bb5a556f..4c372cc0bb 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.20 ; api + esphome/noise-c@0.1.21 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.20 ; api + esphome/noise-c@0.1.21 ; api ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.20 ; used by api + esphome/noise-c@0.1.21 ; used by api lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} From 10e592fa3a51b301644b5a742c75088cc3ab286a Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Tue, 18 Aug 2026 09:09:12 -0700 Subject: [PATCH 04/16] [modbus] CRC scan all unknown function codes (#18483) Co-authored-by: Claude Opus 4.8 (1M context) --- esphome/components/modbus/modbus.cpp | 33 ++-- esphome/components/modbus/modbus.h | 2 +- esphome/components/modbus/modbus_helpers.h | 32 ++++ tests/components/modbus/common.h | 36 +++++ .../modbus/modbus_unknown_function_test.cpp | 141 ++++++++++++++++++ 5 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 tests/components/modbus/modbus_unknown_function_test.cpp diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index 5305f6313f..e4bd51ad5a 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -219,14 +219,25 @@ void ModbusServerHub::parse_modbus_frames() { this->clear_rx_buffer_(LOG_STR("timeout after partial response"), true); } -uint16_t Modbus::find_custom_frame_end_(uint16_t min_length) const { - // Custom functions could be any length - we have to rely on the CRC to determine completeness. +uint16_t Modbus::find_frame_end_by_crc_(uint16_t min_length) const { + // Unknown-length functions (user-defined codes, unimplemented management codes, unassigned values) + // could be any length - we have to rely on the CRC to determine completeness. // If a CRC match is never found, the buffer will eventually overflow and be cleared. const uint8_t *raw = &this->rx_buffer_[0]; const size_t size = this->rx_buffer_.size(); - for (uint16_t len = min_length; len <= std::min(size, size_t(MAX_FRAME_SIZE)); len++) { - if (crc16(raw, len) == 0) - return len; + const auto max_len = static_cast(std::min(size, size_t(MAX_FRAME_SIZE))); + if (min_length > max_len) + return 0; + // The Modbus CRC (poly 0xa001, refin/refout false) keeps its running state in the returned value, + // so we seed once over the first min_length bytes and extend one byte at a time instead of + // recomputing the whole prefix for every candidate length. + uint16_t crc = crc16(raw, min_length); + if (crc == 0) + return min_length; + for (uint16_t len = min_length; len < max_len; len++) { + crc = crc16(&raw[len], 1, crc); + if (crc == 0) + return len + 1; } return 0; } @@ -241,11 +252,11 @@ bool Modbus::parse_modbus_server_frame_() { uint8_t address = this->rx_buffer_[0]; uint8_t function_code = this->rx_buffer_[1]; - if (helpers::is_function_code_custom(function_code)) { - frame_length = this->find_custom_frame_end_(frame_length); + if (helpers::is_function_code_unknown_length(function_code)) { + frame_length = this->find_frame_end_by_crc_(frame_length); if (frame_length == 0) return size < MAX_FRAME_SIZE; // Continue to parse until we hit max size - ESP_LOGD(TAG, "User-defined function %02X found", function_code); + ESP_LOGD(TAG, "Unknown-length function %02X found", function_code); } else { if (crc16(&this->rx_buffer_[0], frame_length) != 0) return false; @@ -272,11 +283,11 @@ bool ModbusServerHub::parse_modbus_client_frame_() { uint8_t address = this->rx_buffer_[0]; uint8_t function_code = this->rx_buffer_[1]; - if (helpers::is_function_code_custom(function_code)) { - frame_length = this->find_custom_frame_end_(frame_length); + if (helpers::is_function_code_unknown_length(function_code)) { + frame_length = this->find_frame_end_by_crc_(frame_length); if (frame_length == 0) return size < MAX_FRAME_SIZE; // Continue to parse until we hit max size - ESP_LOGD(TAG, "User-defined function %02X found", function_code); + ESP_LOGD(TAG, "Unknown-length function %02X found", function_code); } else { if (crc16(&this->rx_buffer_[0], frame_length) != 0) return false; diff --git a/esphome/components/modbus/modbus.h b/esphome/components/modbus/modbus.h index dfe4a4872d..bb303c43a8 100644 --- a/esphome/components/modbus/modbus.h +++ b/esphome/components/modbus/modbus.h @@ -82,7 +82,7 @@ class Modbus : public uart::UARTDevice, public Component { bool send_frame_(const ModbusFrame &frame); // Scans forward from min_length to find a frame boundary by CRC match for custom function codes. // Returns the matched frame length, or 0 if no valid CRC was found within MAX_FRAME_SIZE. - uint16_t find_custom_frame_end_(uint16_t min_length) const; + uint16_t find_frame_end_by_crc_(uint16_t min_length) const; uint32_t last_modbus_byte_{0}; uint32_t last_receive_check_{0}; diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index c737e206c0..b2454e6f14 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -55,6 +55,38 @@ inline bool is_function_code_custom(uint8_t function_code) { masked_function_code <= FUNCTION_CODE_USER_DEFINED_SPACE_2_END); } +/// True for any function code whose frame length the parsers cannot predict - everything the +/// server_pdu_length()/client_pdu_length() switches fall through to `default` on (keep the case list +/// in step with those switches). Deliberately wider than is_function_code_custom(): the user-defined +/// ranges are unknown to the parser too, but so are the assigned-but-unimplemented codes +/// (READ_EXCEPTION_STATUS, DIAGNOSTICS, GET_COMM_EVENT_*, REPORT_SERVER_ID) and every unassigned value. +/// The 0x80 exception flag is masked off first, so a frame with it set classifies by its base code - +/// even though a spec exception reply has a known 2-byte PDU. That is deliberate, matching what +/// is_function_code_custom() has always done: some vendors use codes with the 0x80 bit set as ordinary +/// codes with longer payloads, so the response parser CRC-scans these rather than assuming the spec +/// length. For an intact spec exception the scan matches at its first candidate, so only a corrupt one +/// pays (recovery by timeout instead of an immediate CRC failure). +inline bool is_function_code_unknown_length(uint8_t function_code) { + switch (static_cast(function_code & FUNCTION_CODE_MASK)) { + case FunctionCode::READ_COILS: + case FunctionCode::READ_DISCRETE_INPUTS: + case FunctionCode::READ_HOLDING_REGISTERS: + case FunctionCode::READ_INPUT_REGISTERS: + case FunctionCode::WRITE_SINGLE_COIL: + case FunctionCode::WRITE_SINGLE_REGISTER: + case FunctionCode::WRITE_MULTIPLE_COILS: + case FunctionCode::WRITE_MULTIPLE_REGISTERS: + case FunctionCode::READ_FILE_RECORD: + case FunctionCode::WRITE_FILE_RECORD: + case FunctionCode::MASK_WRITE_REGISTER: + case FunctionCode::READ_WRITE_MULTIPLE_REGISTERS: + case FunctionCode::READ_FIFO_QUEUE: + return false; + default: + return true; + } +} + // Returns the expected length of a server response PDU based on the function code. // If too few bytes have arrived to determine the length, returns the minimum length. `size` is the // number of bytes available so far, which may exceed the eventual PDU (e.g. include the frame's CRC diff --git a/tests/components/modbus/common.h b/tests/components/modbus/common.h index d03ccf8ec3..e6c37b0e6d 100644 --- a/tests/components/modbus/common.h +++ b/tests/components/modbus/common.h @@ -1,7 +1,10 @@ #pragma once #include +#include +#include #include #include "esphome/components/uart/uart_component.h" +#include "esphome/core/helpers.h" namespace esphome::modbus::testing { @@ -30,4 +33,37 @@ class RecordingUART : public NullUART { std::vector written; }; +// A UART the test can inject received bytes into, so frames travel the full receive path +// (receive_modbus_frames -> parse -> dispatch) through hub.loop(). Writes are recorded. +class InjectableUART : public RecordingUART { + public: + bool peek_byte(uint8_t *data) override { + if (this->rx_.empty()) + return false; + *data = this->rx_.front(); + return true; + } + bool read_array(uint8_t *data, size_t len) override { + if (len > this->rx_.size()) + return false; + memcpy(data, this->rx_.data(), len); + this->rx_.erase(this->rx_.begin(), this->rx_.begin() + len); + return true; + } + size_t available() override { return this->rx_.size(); } + + // Queues a complete wire frame: address + PDU + CRC16 (low byte first). + void inject_frame(uint8_t address, std::span pdu) { + size_t start = this->rx_.size(); + this->rx_.push_back(address); + this->rx_.insert(this->rx_.end(), pdu.begin(), pdu.end()); + uint16_t crc = crc16(this->rx_.data() + start, this->rx_.size() - start); + this->rx_.push_back(crc & 0xFF); + this->rx_.push_back(crc >> 8); + } + + private: + std::vector rx_; +}; + } // namespace esphome::modbus::testing diff --git a/tests/components/modbus/modbus_unknown_function_test.cpp b/tests/components/modbus/modbus_unknown_function_test.cpp new file mode 100644 index 0000000000..8b91d088b8 --- /dev/null +++ b/tests/components/modbus/modbus_unknown_function_test.cpp @@ -0,0 +1,141 @@ +#include + +#include +#include +#include + +#include "common.h" +#include "esphome/components/modbus/modbus.h" + +namespace esphome::modbus::testing { + +namespace { + +// Records custom-response dispatches so tests can assert an unknown-length frame reached the device. +class CustomRecordingDevice : public ModbusClientDevice { + public: + using ModbusClientDevice::ModbusClientDevice; + void on_custom_response(std::span request_pdu, std::span response_pdu, + ResponseStatus status) override { + this->requests.emplace_back(request_pdu.begin(), request_pdu.end()); + this->responses.emplace_back(response_pdu.begin(), response_pdu.end()); + this->statuses.push_back(status); + } + std::vector> requests; + std::vector> responses; + std::vector statuses; +}; + +// Every handler keeps its ILLEGAL_FUNCTION default; the hub's dispatch is what is under test. +class SilentServerDevice : public ModbusServerDevice {}; + +// Drives full client frames through the server hub's receive path (same shape as the broadcast tests). +class TestServerHub : public ModbusServerHub { + public: + bool tx_blocked() override { return false; } + + // Builds a complete client frame (address + FC + data + CRC) and runs the full receive-side parser. + // Returns true once the buffer has fully drained. + bool run_receive_parser_for_test(uint8_t address, uint8_t function_code, std::span data) { + this->rx_buffer_.clear(); + this->rx_buffer_.reserve(data.size() + 4); + this->rx_buffer_.push_back(address); + this->rx_buffer_.push_back(function_code); + this->rx_buffer_.insert(this->rx_buffer_.end(), data.begin(), data.end()); + uint16_t crc = crc16(this->rx_buffer_.data(), this->rx_buffer_.size()); + this->rx_buffer_.push_back(crc & 0xFF); + this->rx_buffer_.push_back(crc >> 8); + this->parse_modbus_frames(); + return this->rx_buffer_.empty(); + } +}; + +} // namespace + +// The frame-length parsers have explicit cases for exactly these 13 codes; every other value - the +// assigned-but-unimplemented management codes, both user-defined ranges, and all unassigned codes - +// must classify as unknown length. The exception flag masks off first. +TEST(ModbusUnknownFunction, HelperMatchesParserCoverage) { + for (uint8_t fc : {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x0F, 0x10, 0x14, 0x15, 0x16, 0x17, 0x18}) { + EXPECT_FALSE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); + } + for (uint8_t fc : {0x07, 0x08, 0x0B, 0x0C, 0x11, 0x2A, 0x41, 0x48, 0x49, 0x64, 0x6E, 0x00, 0x7F}) { + EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); + } + // Exception replies classify by their base code. + EXPECT_FALSE(helpers::is_function_code_unknown_length(0x83)); + EXPECT_TRUE(helpers::is_function_code_unknown_length(0x87)); + // Strictly wider than the user-defined ranges: every custom code is unknown-length, but not vice versa. + for (int fc = 0; fc <= 0xFF; fc++) { + if (helpers::is_function_code_custom(fc)) + EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << fc; + } + EXPECT_FALSE(helpers::is_function_code_custom(0x49)); + + // Derived contract check: the helper must say "unknown" exactly when both length parsers fall + // through to default. With a zero-filled max-size PDU every explicit case returns at least 2 + // (file records bottom out at 2, FIFO at 3) and only default returns MIN_PDU_SIZE, so comparing + // against MIN_PDU_SIZE detects a case added to either switch without updating the helper. The + // loop stops at 0x7F: above it the helper masks the exception flag off while client_pdu_length() + // switches on the unmasked byte and server_pdu_length() early-returns the exception length. + for (int fc = 0; fc <= 0x7F; fc++) { + const uint8_t pdu[MAX_PDU_SIZE] = {static_cast(fc)}; // zero header fields + EXPECT_EQ(helpers::is_function_code_unknown_length(fc), + helpers::client_pdu_length(pdu, sizeof(pdu)) == MIN_PDU_SIZE) + << "client_pdu_length disagrees for fc 0x" << std::hex << fc; + EXPECT_EQ(helpers::is_function_code_unknown_length(fc), + helpers::server_pdu_length(pdu, sizeof(pdu)) == MIN_PDU_SIZE) + << "server_pdu_length disagrees for fc 0x" << std::hex << fc; + } +} + +// A response with a function code outside the user-defined ranges (0x49) has no length case in +// server_pdu_length(), so the parser must find the frame end by CRC scan - the same way it already +// handles user-defined codes. Frame: address + FC 0x49 + 3 data bytes + CRC = 7 bytes. Without the +// scan the parser assumes a 4-byte frame, fails the CRC, and the response never reaches the device. +TEST(ModbusUnknownFunction, ClientParsesUnknownLengthResponse) { + InjectableUART uart; + ModbusClientHub hub; + hub.set_uart_parent(&uart); + hub.setup(); // computes frame timing from the baud rate + CustomRecordingDevice device(&hub, 0x02); + + const uint8_t request[] = {0x49, 0x01}; + ASSERT_TRUE(device.queue_pdu(request)); + hub.loop(); // transmit + ASSERT_FALSE(uart.written.empty()); + + const uint8_t response_pdu[] = {0x49, 0x02, 0xAA, 0xBB}; + uart.inject_frame(0x02, response_pdu); + hub.loop(); // receive + parse + match + dispatch + + ASSERT_EQ(device.responses.size(), 1u); + EXPECT_EQ(device.requests[0], std::vector(request, request + sizeof(request))); + EXPECT_EQ(device.responses[0], std::vector(response_pdu, response_pdu + sizeof(response_pdu))); + EXPECT_FALSE(device.statuses[0].has_value()); +} + +// The server side of the same gap: a request with FC 0x49 for a registered device must parse (CRC +// scan again) so the hub can answer ILLEGAL_FUNCTION per the spec. Without the scan the frame fails +// to parse and the client gets silence instead of the exception. +TEST(ModbusUnknownFunction, ServerRepliesIllegalFunctionToUnknownLengthRequest) { + TestServerHub hub; + RecordingUART uart; + hub.set_uart_parent(&uart); + + SilentServerDevice device; + device.set_address(0x02); + hub.register_device(&device); + + const uint8_t data[] = {0x02, 0xAA, 0xBB}; + ASSERT_TRUE(hub.run_receive_parser_for_test(0x02, 0x49, data)); + + // Expected reply: address + FC with exception flag + ILLEGAL_FUNCTION + CRC. + std::vector expected = {0x02, 0xC9, 0x01}; + uint16_t crc = crc16(expected.data(), expected.size()); + expected.push_back(crc & 0xFF); + expected.push_back(crc >> 8); + EXPECT_EQ(uart.written, expected); +} + +} // namespace esphome::modbus::testing From 2df953f3d7c0b022cd3a75c05ab2ca0d63cc39f9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:36:57 -0500 Subject: [PATCH 05/16] [platformio] Give the ccache wrapper a cmd.exe safe path (#18495) --- esphome/platformio/ccache.py.script | 9 +- esphome/platformio/toolchain.py | 63 +++-- tests/unit_tests/test_platformio_toolchain.py | 240 +++++++++++++++++- 3 files changed, 286 insertions(+), 26 deletions(-) diff --git a/esphome/platformio/ccache.py.script b/esphome/platformio/ccache.py.script index cc08a8c044..22592a2398 100644 --- a/esphome/platformio/ccache.py.script +++ b/esphome/platformio/ccache.py.script @@ -1,5 +1,4 @@ import os -import shutil # pylint: disable=E0602 Import("env") # noqa @@ -9,15 +8,17 @@ Import("env") # noqa # esphome/platformio/toolchain.py); this script only supplies the SCons-level # mechanism. # +# The binary comes pre-resolved in ESPHOME_CCACHE_PATH; _ccache_env() has +# already stripped the Windows \\?\ prefix that cmd.exe cannot run. +# # This is a "pre" script, so the platform's builder (which sets CC/CXX and # clones the construction environment for framework and library builds) runs # after it. Replacing CC/CXX here would be overwritten, and replacing them in # a "post" script would miss the already-cloned library environments. Wrapping # SPAWN instead is ordering-proof: clones copy the wrapper, and every compiler # invocation from every environment funnels through it at execution time. -if ( - os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" - and (ccache_path := shutil.which("ccache")) is not None +if os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" and ( + ccache_path := os.environ.get("ESPHOME_CCACHE_PATH") ): original_spawn = env["SPAWN"] diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 08a4fcff78..d76581d032 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -60,6 +60,9 @@ def _strip_win_long_path_prefix(path: str) -> str: "The system cannot find the path specified." Stripping the prefix early keeps the path shell-quotable. + Also applied to the ccache path exported by ``_ccache_env()``, which + ``shutil.which`` can return with the same prefix. + No-op on non-Windows platforms. """ if sys.platform != "win32": @@ -235,8 +238,8 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) -def _ccache_usable() -> bool: - """Return True when the ``ccache`` on PATH actually runs. +def _ccache_runs(ccache: str) -> bool: + """Return True when the ``ccache`` found on PATH actually runs. ``shutil.which`` proves existence, not runnability: on Windows it also matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose @@ -244,9 +247,6 @@ def _ccache_usable() -> bool: step with an opaque OS error, so probe once and fall back to compiling without ccache when the probe fails. """ - ccache = shutil.which("ccache") - if ccache is None: - return False try: subprocess.run( [ccache, "--version"], @@ -265,14 +265,29 @@ def _ccache_usable() -> bool: def _ccache_env() -> dict[str, str]: - """Return ccache settings for PlatformIO builds. + r"""Return ccache settings for PlatformIO builds. Enabled by default whenever the ``ccache`` binary is on PATH; set ``ESPHOME_CCACHE_ENABLE=0`` in the environment to opt out (or ``1`` to - force it on). The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` - so platform build scripts (e.g. the esp8266 ``ccache.py`` extra script, - which wraps compiler invocations inside SCons) only have to check for - ``"1"`` instead of re-implementing the policy. + force it on without the runnability probe; a binary is still needed). + The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` and the + binary's location into ``ESPHOME_CCACHE_PATH`` so platform build scripts + (the shared ``ccache.py`` extra script, which wraps compiler invocations + inside SCons) only have to check for ``"1"`` and use the path as given + instead of re-implementing the policy. + + The path is exported rather than looked up again inside SCons because + ``shutil.which`` can return a Windows extended-length ``\\?\`` path + (ESPHome Desktop puts its bundled ccache on PATH that way). Such a path + runs fine through ``CreateProcess``, which is how ESP-IDF invokes it, + but SCons runs every compile through ``cmd.exe``, which fails on it with + "The system cannot find the path specified." (#18399), so the prefix is + stripped here with ``_strip_win_long_path_prefix()`` before the + runnability probe, which therefore validates the exact string the build + will execute. + ``ESPHOME_CCACHE_PATH`` is an internal channel, not a user setting: the + script only honours it together with ``ESPHOME_CCACHE_ENABLE=1``, and this + function always sets both or neither. The returned values are merged into the environment of the PlatformIO subprocess only, never into ``os.environ``: a long-running process @@ -293,13 +308,27 @@ def _ccache_env() -> dict[str, str]: build dir. The other ``CCACHE_*`` values the user already set in the environment are respected. """ - if "ESPHOME_CCACHE_ENABLE" in os.environ: - enabled = get_bool_env("ESPHOME_CCACHE_ENABLE") - else: - enabled = _ccache_usable() - env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"} - if not enabled: - return env + explicit = "ESPHOME_CCACHE_ENABLE" in os.environ + if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): + return {"ESPHOME_CCACHE_ENABLE": "0"} + ccache_path = shutil.which("ccache") + if ccache_path is None: + if explicit: + _LOGGER.warning( + "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " + "compiling without ccache" + ) + return {"ESPHOME_CCACHE_ENABLE": "0"} + # Strip before probing so the probe validates (and the failure warning + # names) the exact string the build will execute through cmd.exe. + ccache_path = _strip_win_long_path_prefix(ccache_path) + # An explicit opt-in skips the runnability probe. + if not explicit and not _ccache_runs(ccache_path): + return {"ESPHOME_CCACHE_ENABLE": "0"} + env = { + "ESPHOME_CCACHE_ENABLE": "1", + "ESPHOME_CCACHE_PATH": ccache_path, + } # build_path is set during preload for every config-loading command, so it # being unset means a caller built the environment too early; fail loudly # rather than with an opaque TypeError from Path(None). diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index eebb0b8cd7..172b288c25 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -2,7 +2,7 @@ # pylint: disable=protected-access -from collections.abc import Generator +from collections.abc import Callable, Generator from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json @@ -437,6 +437,7 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: env = toolchain._ccache_env() assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) assert env["CCACHE_DIR"].endswith("platformio-ccache") assert env["CCACHE_NOHASHDIR"] == "true" @@ -446,17 +447,35 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: assert "ESPHOME_CCACHE_ENABLE" not in os.environ -def test_ccache_env_disabled_without_binary(setup_core: Path) -> None: - """Ccache stays off when the binary is not on PATH.""" +@pytest.mark.parametrize( + ("env_vars", "expect_warning"), + [ + pytest.param({}, False, id="default"), + pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, True, id="forced-on"), + ], +) +def test_ccache_env_disabled_without_binary( + setup_core: Path, + caplog: pytest.LogCaptureFixture, + env_vars: dict[str, str], + expect_warning: bool, +) -> None: + """Ccache stays off when the binary is not on PATH, even when forced on. + + A deliberate opt-in that finds no binary is downgraded with a warning so + the user can tell why it had no effect; the default path stays quiet. + """ CORE.build_path = setup_core / "build" / "test" with ( - patch.dict(os.environ, {}, clear=True), + patch.dict(os.environ, env_vars, clear=True), patch.object(toolchain.shutil, "which", return_value=None), + caplog.at_level("WARNING"), ): env = toolchain._ccache_env() assert env == {"ESPHOME_CCACHE_ENABLE": "0"} + assert ("no ccache binary is on PATH" in caplog.text) is expect_warning @pytest.mark.parametrize( @@ -489,14 +508,47 @@ def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), patch.object(toolchain.subprocess, "run") as mock_probe, ): env = toolchain._ccache_env() assert env["ESPHOME_CCACHE_ENABLE"] == "1" + # The binary's location is still handed to the build script. + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" mock_probe.assert_not_called() +def test_ccache_env_strips_win_long_path_prefix(setup_core: Path) -> None: + r"""A ``\\?\`` ccache path from PATH is exported without the prefix. + + That is the shape ESPHome Desktop puts on PATH (#18399); see ``_ccache_env``. + """ + CORE.build_path = setup_core / "build" / "test" + prefixed = ( + "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder" + "\\ccache\\ccache.exe" + ) + stripped = ( + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder\\ccache\\ccache.exe" + ) + + with ( + patch.dict(os.environ, {}, clear=True), + # shutil.which is patched, so the win32 code path of the real + # implementation (which crashes on a POSIX host) is never reached. + patch("esphome.platformio.toolchain.sys.platform", "win32"), + patch.object(toolchain.shutil, "which", return_value=prefixed), + patch.object(toolchain.subprocess, "run") as mock_probe, + ): + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == stripped + # The probe validates the exact string the build will execute. + assert mock_probe.call_args[0][0] == [stripped, "--version"] + + def test_ccache_env_opt_out(setup_core: Path) -> None: """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present.""" CORE.build_path = setup_core / "build" / "test" @@ -516,7 +568,7 @@ def test_ccache_env_normalizes_enable_value(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "yes"}, clear=True), - patch.object(toolchain.shutil, "which", return_value=None), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), ): env = toolchain._ccache_env() @@ -563,8 +615,10 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only( env = mock_run_external_process.call_args[1]["env"] assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache" assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) assert "ESPHOME_CCACHE_ENABLE" not in os.environ + assert "ESPHOME_CCACHE_PATH" not in os.environ assert "CCACHE_BASEDIR" not in os.environ @@ -613,6 +667,182 @@ def test_copy_ccache_script(setup_core: Path) -> None: assert dest.read_text() == source.read_text() +class _FakeSConsEnv(dict): + """Just enough of a SCons construction environment for ccache.py.""" + + def Replace(self, **kwargs: object) -> None: # noqa: N802 + self.update(kwargs) + + +def _load_ccache_script( + env_vars: dict[str, str], original_spawn: Callable[..., int] | None = None +) -> tuple[_FakeSConsEnv, Callable[..., int]]: + """Run ccache.py.script against a fake SCons env and return (env, original SPAWN).""" + if original_spawn is None: + original_spawn = Mock(name="original_spawn", return_value=0) + scons_env = _FakeSConsEnv(SPAWN=original_spawn) + source = (Path(toolchain.__file__).parent / "ccache.py.script").read_text() + with patch.dict(os.environ, env_vars, clear=True): + exec( # noqa: S102 + compile(source, "ccache.py", "exec"), + {"Import": lambda *_names: None, "env": scons_env}, + ) + return scons_env, original_spawn + + +def _scons_win32_escape(x: str) -> str: + """Copy of ``SCons.Platform.win32.escape``: quote, guarding a trailing backslash.""" + if x[-1] == "\\": + x = x + "\\" + return '"' + x + '"' + + +def test_ccache_script_wraps_compiles_with_exported_path() -> None: + """The SCons script uses ESPHOME_CCACHE_PATH as given, without a PATH lookup.""" + ccache_path = "C:\\Users\\jesse\\ESPHome Device Builder\\ccache\\ccache.exe" + scons_env, original_spawn = _load_ccache_script( + {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": ccache_path} + ) + spawn = scons_env["SPAWN"] + assert spawn is not original_spawn + + # A compile step is routed through ccache, with the same path used for + # the program and (escaped) as the first argument. + compile_args = ["xtensa-lx106-elf-g++", "-o", "main.o", "-c", "main.cpp"] + spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", compile_args, {}) + original_spawn.assert_called_once_with( + "cmd.exe", + _scons_win32_escape, + ccache_path, + [_scons_win32_escape(ccache_path), *compile_args], + {}, + ) + + # Link steps pass through untouched. + original_spawn.reset_mock() + link_args = ["xtensa-lx106-elf-g++", "-o", "firmware.elf", "main.o"] + spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {}) + original_spawn.assert_called_once_with( + "cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {} + ) + + +@pytest.mark.parametrize( + "env_vars", + [ + pytest.param({"ESPHOME_CCACHE_ENABLE": "0"}, id="disabled"), + pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, id="enabled-without-path"), + pytest.param({}, id="unset"), + ], +) +def test_ccache_script_leaves_spawn_alone_without_path( + env_vars: dict[str, str], +) -> None: + """Without both the enable flag and a path, SPAWN is not replaced.""" + scons_env, original_spawn = _load_ccache_script(env_vars) + assert scons_env["SPAWN"] is original_spawn + + +def _scons_win32_spawn( + sh: str, escape: Callable[[str], str], cmd: str, args: list[str], env: dict +) -> int: + r"""Mirror of ``SCons.Platform.win32.spawn``: every command runs via ``cmd.exe /C``. + + SCons is not importable in the test environment (PlatformIO fetches it at + build time), so the lines that matter are mirrored here. The command line + SCons hands ``os.spawnve`` goes to ``CreateProcess`` via ``subprocess`` + instead (identical on Windows, where a string passes through untouched); + ``spawnve`` itself crashes inside pytest. + """ + return subprocess.run( + " ".join([sh, "/C", escape(" ".join(args))]), env=env, check=False + ).returncode + + +_MARKER_ENV = "ESPHOME_TEST_CCACHE_MARKER" +# Stands in for a compile: the "ccache" is really the Python interpreter, and +# the compile "flags" make it write a marker file so the test can tell whether +# the wrapped command actually ran to completion. +_FAKE_COMPILE_ARGS = [ + "-c", + f"import os, pathlib; pathlib.Path(os.environ['{_MARKER_ENV}']).write_text('compiled')", +] + + +def _spawn_fake_compile_via_cmd_exe(scons_env: _FakeSConsEnv, marker: Path) -> int: + """Run one wrapped compile step the way SCons does on Windows.""" + child_env = {**os.environ, _MARKER_ENV: str(marker)} + return scons_env["SPAWN"]( + os.environ.get("COMSPEC", "cmd.exe"), + _scons_win32_escape, + "xtensa-lx106-elf-gcc", + [_scons_win32_escape(arg) if " " in arg else arg for arg in _FAKE_COMPILE_ARGS], + child_env, + ) + + +_WINDOWS_ONLY = pytest.mark.skipif( + sys.platform != "win32", reason="drives cmd.exe, which SCons uses only on Windows" +) + + +@_WINDOWS_ONLY +def test_ccache_env_real_probe_runs_stripped_path(setup_core: Path) -> None: + r"""With a ``\\?\`` which result, the real probe runs the stripped binary. + + The probe therefore validates the exact string the build will execute + through ``cmd.exe``; probing the verbatim path instead would pass even + when the stripped path is unusable (``CreateProcess`` accepts + extended-length paths, ``cmd.exe`` does not). + """ + CORE.build_path = setup_core / "build" / "test" + assert not sys.executable.startswith("\\\\?\\") + + with ( + patch.dict(os.environ, {}, clear=False), + patch.object( + toolchain.shutil, "which", return_value="\\\\?\\" + sys.executable + ), + ): + os.environ.pop("ESPHOME_CCACHE_ENABLE", None) + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["ESPHOME_CCACHE_PATH"] == sys.executable + + +@_WINDOWS_ONLY +@pytest.mark.parametrize( + ("prefix", "expect_ok"), + [ + pytest.param("", True, id="stripped-path-compiles"), + pytest.param("\\\\?\\", False, id="verbatim-path-fails"), + ], +) +def test_ccache_wrapper_through_cmd_exe( + tmp_path: Path, prefix: str, expect_ok: bool +) -> None: + r"""End to end through ``cmd.exe``: the exported path works, a ``\\?\`` one does not. + + The interpreter stands in for ccache; the spawn mirrors SCons on Windows. + The failing case is the mechanism behind #18399 ("The system cannot find + the path specified." on every compile step); should it ever start passing, + ``cmd.exe`` learned extended-length paths and the strip is no longer needed. + """ + marker = tmp_path / "compiled.txt" + scons_env, _ = _load_ccache_script( + {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": prefix + sys.executable}, + original_spawn=_scons_win32_spawn, + ) + assert scons_env["SPAWN"] is not _scons_win32_spawn + + rc = _spawn_fake_compile_via_cmd_exe(scons_env, marker) + assert (rc == 0) is expect_ok + assert marker.exists() is expect_ok + if expect_ok: + assert marker.read_text() == "compiled" + + @pytest.mark.parametrize( ("platform", "input_path", "expected"), [ From 6084314cc9c029b4b6b131a92665d98d4046e464 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Aug 2026 19:37:10 -0500 Subject: [PATCH 06/16] [vscode] Report the origin of an unexpected exception during validation (#18494) --- esphome/vscode.py | 20 +++++++++- tests/unit_tests/test_vscode.py | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/esphome/vscode.py b/esphome/vscode.py index f404f02f00..ba7b4e727b 100644 --- a/esphome/vscode.py +++ b/esphome/vscode.py @@ -3,12 +3,14 @@ from __future__ import annotations from io import StringIO import json from pathlib import Path +import sys +import traceback from typing import Any from esphome.config import Config, _format_vol_invalid, validate_config import esphome.config_validation as cv from esphome.const import __version__ as ESPHOME_VERSION -from esphome.core import CORE, DocumentRange +from esphome.core import CORE, DocumentRange, EsphomeError from esphome.yaml_util import parse_yaml @@ -97,6 +99,16 @@ def _ace_loader(fname: Path) -> dict[str, Any]: return parse_yaml(fname, raw_yaml_stream) +def _format_unexpected_error(err: Exception) -> str: + """Describe a crash inside validation with the frame it came from.""" + message = f"Unexpected error while validating: {type(err).__name__}: {err}" + frames = traceback.extract_tb(err.__traceback__) + if not frames: + return message + frame = frames[-1] + return f"{message} ({frame.filename}:{frame.lineno} in {frame.name})" + + def _print_version(): """Print ESPHome version.""" print( @@ -134,8 +146,12 @@ def read_config(args): try: config = loader(file_name) res = validate_config(config, command_line_substitutions) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + except (EsphomeError, cv.Invalid) as err: vs.add_yaml_error(str(err)) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + # stdout carries the JSON protocol; the full chain goes to stderr. + traceback.print_exc(file=sys.stderr) + vs.add_yaml_error(_format_unexpected_error(err)) else: for err in res.errors: try: diff --git a/tests/unit_tests/test_vscode.py b/tests/unit_tests/test_vscode.py index 63bdf3e255..9b7d1e9504 100644 --- a/tests/unit_tests/test_vscode.py +++ b/tests/unit_tests/test_vscode.py @@ -3,6 +3,8 @@ from pathlib import Path from unittest.mock import Mock, patch from esphome import vscode +import esphome.config_validation as cv +from esphome.core import EsphomeError def _run_repl_test(input_data): @@ -126,3 +128,67 @@ packages: assert range["start_col"] == 2 assert range["end_line"] == 1 assert range["end_col"] == 7 + + +def _explode(*_args: object, **_kwargs: object) -> None: + raise AttributeError("'NoneType' object has no attribute 'get'") + + +def test_unexpected_error_reports_origin() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", _explode): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["validation_errors"] == [] + (error,) = result["yaml_errors"] + assert error["message"].startswith( + "Unexpected error while validating: AttributeError: " + "'NoneType' object has no attribute 'get' (" + ) + assert "test_vscode.py" in error["message"] + assert error["message"].endswith(" in _explode)") + + +def test_esphome_error_stays_plain() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", side_effect=EsphomeError("boom")): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["yaml_errors"] == [{"message": "boom"}] + + +def test_invalid_stays_plain() -> None: + source_path = str(Path("dir_path", "x.yaml")) + with patch("esphome.vscode.validate_config", side_effect=cv.Invalid("bad value")): + output_lines = _run_repl_test( + [ + _validate(source_path), + _file_response("""esphome: + name: test1 +"""), + ] + ) + + result = json.loads(output_lines[-1]) + assert result["yaml_errors"] == [{"message": "bad value"}] + + +def test_format_unexpected_error_without_traceback() -> None: + message = vscode._format_unexpected_error(ValueError("boom")) + assert message == "Unexpected error while validating: ValueError: boom" From b768e2a1ce796f7055f9fcba8e8b3494798ce8fa Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:25:09 -0400 Subject: [PATCH 07/16] [esp32] Fix ESP32-P4 bootloop on rev3 (v3.x) chips when only variant is set (#18500) --- esphome/components/esp32/__init__.py | 78 +++++++++++++++------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index ada6d25db5..2c06ebac9a 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1070,6 +1070,26 @@ def _parse_pio_platform_version(value): return value +def _normalize_p4_engineering_sample(value: ConfigType) -> bool: + """Fill in CONF_ENGINEERING_SAMPLE when unset, warning that production + silicon (rev3) is assumed. Returns the normalized flag.""" + if (engineering_sample := value.get(CONF_ENGINEERING_SAMPLE)) is None: + _LOGGER.warning( + "Defaulting to ESP32-P4 production silicon (rev3).\n" + "If you have an early engineering sample (pre-rev3), add this to your config:\n" + "\n" + " esp32:\n" + " engineering_sample: true\n" + "\n" + "To check your chip revision, look for 'chip revision: vX.Y' in the boot log.\n" + "Engineering samples will show a revision below v3.0.\n" + "The 'debug:' component also reports the revision (e.g. Revision: 100 = v1.0, 300 = v3.0)." + ) + engineering_sample = False + value[CONF_ENGINEERING_SAMPLE] = engineering_sample + return engineering_sample + + def _detect_variant(value): board = value.get(CONF_BOARD) variant = value.get(CONF_VARIANT) @@ -1082,6 +1102,8 @@ def _detect_variant(value): # name rather than carrying a PIO board name through the IDF build. if CORE.using_toolchain_esp_idf: value = value.copy() + if variant == VARIANT_ESP32P4: + _normalize_p4_engineering_sample(value) value[CONF_BOARD] = VARIANT_FRIENDLY[variant].lower() return value if variant not in STANDARD_BOARDS: @@ -1092,22 +1114,8 @@ def _detect_variant(value): ) value = value.copy() value[CONF_BOARD] = STANDARD_BOARDS[variant] - if variant == VARIANT_ESP32P4: - engineering_sample = value.get(CONF_ENGINEERING_SAMPLE) - if engineering_sample is None: - _LOGGER.warning( - "No board specified for ESP32-P4. Defaulting to production silicon (rev3).\n" - "If you have an early engineering sample (pre-rev3), add this to your config:\n" - "\n" - " esp32:\n" - " engineering_sample: true\n" - "\n" - "To check your chip revision, look for 'chip revision: vX.Y' in the boot log.\n" - "Engineering samples will show a revision below v3.0.\n" - "The 'debug:' component also reports the revision (e.g. Revision: 100 = v1.0, 300 = v3.0)." - ) - elif engineering_sample: - value[CONF_BOARD] = "esp32-p4-evboard" + if variant == VARIANT_ESP32P4 and _normalize_p4_engineering_sample(value): + value[CONF_BOARD] = "esp32-p4-evboard" elif board in BOARDS: variant = variant or BOARDS[board][KEY_VARIANT] if variant != BOARDS[board][KEY_VARIANT]: @@ -1117,6 +1125,14 @@ def _detect_variant(value): ) value = value.copy() value[CONF_VARIANT] = variant + if variant == VARIANT_ESP32P4: + board_is_es = BOARDS[board].get("engineering_sample", False) + engineering_sample = value.setdefault(CONF_ENGINEERING_SAMPLE, board_is_es) + if engineering_sample != board_is_es: + raise cv.Invalid( + f"'{CONF_ENGINEERING_SAMPLE}' does not match board '{board}'", + path=[CONF_ENGINEERING_SAMPLE], + ) elif not variant: raise cv.Invalid( "This board is unknown, if you are sure you want to compile with this board selection, " @@ -1128,6 +1144,9 @@ def _detect_variant(value): "This board is unknown; the specified variant '%s' will be used but this may not work as expected.", variant, ) + if variant == VARIANT_ESP32P4: + value = value.copy() + _normalize_p4_engineering_sample(value) return value @@ -1431,20 +1450,6 @@ def final_validate(config): path=[CONF_ENGINEERING_SAMPLE], ) ) - if ( - config[CONF_VARIANT] == VARIANT_ESP32P4 - and config.get(CONF_ENGINEERING_SAMPLE) is not None - ): - board_is_es = BOARDS.get(config[CONF_BOARD], {}).get( - "engineering_sample", False - ) - if config[CONF_ENGINEERING_SAMPLE] != board_is_es: - errs.append( - cv.Invalid( - f"'{CONF_ENGINEERING_SAMPLE}' does not match board '{config[CONF_BOARD]}'", - path=[CONF_ENGINEERING_SAMPLE], - ) - ) if advanced[CONF_EXECUTE_FROM_PSRAM]: if config[CONF_VARIANT] not in {VARIANT_ESP32S3, VARIANT_ESP32P4}: errs.append( @@ -2517,15 +2522,14 @@ async def to_code(config): f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_frequency[:-3]}M", True ) - # ESP32-P4: ESP-IDF 5.5.3 changed the default of ESP32P4_SELECTS_REV_LESS_V3 - # from y to n. PlatformIO uses sections.ld.in (for rev <3) or - # sections.rev3.ld.in (for rev >=3) based on board definition. - # Set the sdkconfig option to match the board's chip revision. + # ESP32-P4: pre-v3 and rev3 (v3.0+) silicon are not binary compatible. + # CONFIG_ESP32P4_SELECTS_REV_LESS_V3 selects which layout ESP-IDF links; + # validation normalizes CONF_ENGINEERING_SAMPLE from the board when unset. if variant == VARIANT_ESP32P4: - is_eng_sample = BOARDS.get(config[CONF_BOARD], {}).get( - "engineering_sample", False + add_idf_sdkconfig_option( + "CONFIG_ESP32P4_SELECTS_REV_LESS_V3", + config.get(CONF_ENGINEERING_SAMPLE, False), ) - add_idf_sdkconfig_option("CONFIG_ESP32P4_SELECTS_REV_LESS_V3", is_eng_sample) # Set minimum chip revision for ESP32 variant # Setting this to 3.0 or higher reduces flash size by excluding workaround code, From 7418fcce8d8f154bceb088f1ad10782c6dadb4ca Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:38:18 -0400 Subject: [PATCH 08/16] [ci] Stop persisting the integration test ccache (#18504) --- .github/workflows/ci.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2075fde9ef..0d8f35ed83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -341,18 +341,6 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -y --no-install-recommends ccache - - name: Restore ccache (restore-only) - # esphome stores the PlatformIO ccache under the machine-global cache - # dir (see _ccache_env() in esphome/platformio/toolchain.py). The - # bucket-name prefix prefers a same-bucket seed; the bare prefix falls - # back to any seed when the bucket layout differs from dev. - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/esphome/platformio-ccache - key: integration-ccache-${{ matrix.bucket.name }}-${{ github.sha }} - restore-keys: | - integration-ccache-${{ matrix.bucket.name }}- - integration-ccache- - name: Set up Python 3.13 id: python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -401,14 +389,6 @@ jobs: # esphome stores the PlatformIO ccache under the machine-global cache # dir (see _ccache_env() in esphome/platformio/toolchain.py). run: CCACHE_DIR="$HOME/.cache/esphome/platformio-ccache" ccache -s - - name: Save ccache - # Pull request saves land in per-PR scopes nothing else can reuse; - # dev pushes seed the shared copy instead. - if: github.event_name != 'pull_request' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/esphome/platformio-ccache - key: integration-ccache-${{ matrix.bucket.name }}-${{ github.sha }} import-time: name: Check import esphome.__main__ time From e9e77d02a00d6d9b8f0661b0e4c4a025b4f697b9 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:57:51 -0500 Subject: [PATCH 09/16] Bump bundled esphome-device-builder to 1.11.3 (#18505) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4a8daeaaf6..50b698224c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.3 RUN \ platformio settings set enable_telemetry No \ From 74e22b5ad74308fbed86738bf63fbcaed9f0fd03 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:24:23 -0500 Subject: [PATCH 10/16] Bump bundled esphome-device-builder to 1.11.4 (#18506) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 50b698224c..5c21e07618 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.4 RUN \ platformio settings set enable_telemetry No \ From 2c92a2498e5fb5632554f485eedd3446155d7e83 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:42:05 -0500 Subject: [PATCH 11/16] Bump bundled esphome-device-builder to 1.11.5 (#18507) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5c21e07618..1be10db3af 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.11.5 RUN \ platformio settings set enable_telemetry No \ From 4a85c98285c1a2c38b2e5e9115fb4793b5b1d69f Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:21:58 -0500 Subject: [PATCH 12/16] Bump bundled esphome-device-builder to 1.12.0 (#18514) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1be10db3af..18f705b501 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.11.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.12.0 RUN \ platformio settings set enable_telemetry No \ From b3fda9973ebd67fcafb26b4f3b7a831427ecafff Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:30:12 -0700 Subject: [PATCH 13/16] [image] Restore defaults:/files: support for platform entries (#18032) Co-authored-by: Claude Sonnet 5 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/components/file/image.py | 3 +- esphome/components/image/__init__.py | 152 +++++++- esphome/components/runtime_image/__init__.py | 5 +- esphome/config.py | 17 + esphome/loader.py | 8 + tests/component_tests/image/test_init.py | 328 +++++++++++++++++- .../validate-platform-defaults.host.yaml | 21 ++ .../validate-platform-defaults.host.yaml | 24 ++ tests/unit_tests/test_config_normalization.py | 124 ++++++- 9 files changed, 657 insertions(+), 25 deletions(-) create mode 100644 tests/components/animation/validate-platform-defaults.host.yaml create mode 100644 tests/components/image/validate-platform-defaults.host.yaml diff --git a/esphome/components/file/image.py b/esphome/components/file/image.py index b54c3f2adf..212c778763 100644 --- a/esphome/components/file/image.py +++ b/esphome/components/file/image.py @@ -23,6 +23,7 @@ from esphome.components.image import ( get_image_type_enum, get_transparency_enum, is_svg_file, + validate_byte_order, validate_settings, validate_transparency, validate_type, @@ -200,7 +201,7 @@ OPTIONS_SCHEMA = { "NONE", "FLOYDSTEINBERG", upper=True ), cv.Optional(CONF_INVERT_ALPHA, default=False): cv.boolean, - cv.Optional(CONF_BYTE_ORDER): cv.one_of("BIG_ENDIAN", "LITTLE_ENDIAN", upper=True), + cv.Optional(CONF_BYTE_ORDER): validate_byte_order, cv.Optional(CONF_TRANSPARENCY, default=CONF_OPAQUE): validate_transparency(), } diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 37a9afb84d..eaee31a1c7 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -10,7 +10,14 @@ from PIL import Image, UnidentifiedImageError import esphome.codegen as cg from esphome.components.const import CONF_BYTE_ORDER, KEY_METADATA import esphome.config_validation as cv -from esphome.const import CONF_DEFAULTS, CONF_FILE, CONF_ID, CONF_PLATFORM, CONF_TYPE +from esphome.const import ( + CONF_DEFAULTS, + CONF_FILE, + CONF_FILES, + CONF_ID, + CONF_PLATFORM, + CONF_TYPE, +) from esphome.core import CORE from esphome.types import ConfigType @@ -48,6 +55,9 @@ TRANSPARENCY_TYPES = ( CONF_ALPHA_CHANNEL, ) +# Shared validator for the image platform schemas and `_drop_incompatible_byte_order`. +validate_byte_order = cv.one_of("BIG_ENDIAN", "LITTLE_ENDIAN", upper=True) + def get_image_type_enum(type): return getattr(ImageType, f"IMAGE_TYPE_{type.upper()}") @@ -404,6 +414,120 @@ def get_image_metadata(image_id: str) -> ImageMetaData | None: return get_all_image_metadata().get(image_id) +# --------------------------------------------------------------------------- +# `defaults:`/`files:` expansion: a `platform:` entry merges shared `defaults:` +# into every `files:` entry; the platform's CONFIG_SCHEMA validates each. +# Permanent, unlike the legacy migration below. +# --------------------------------------------------------------------------- + + +def _drop_incompatible_byte_order( + merged: dict, explicit: dict, *, index: int | None = None +) -> dict: + """Drop `byte_order` when the resolved type doesn't support it, unless written directly on `explicit`. + + With `index`, inherited values are validated before being dropped (the legacy flattener always drops). + """ + if CONF_BYTE_ORDER in explicit: + return merged + type_class = IMAGE_TYPE.get(str(merged.get(CONF_TYPE, "")).upper()) + if ( + CONF_BYTE_ORDER in merged + and isinstance(type_class, type) + and issubclass(type_class, ImageEncoder) + and not type_class.is_endian() + ): + if index is not None: + try: + validate_byte_order(merged[CONF_BYTE_ORDER]) + except cv.Invalid as exc: + exc.prepend([index]) + raise + del merged[CONF_BYTE_ORDER] + return merged + + +def _expand_platform_entry(index: int, entry: dict) -> list[dict]: + if CONF_FILES not in entry: + if CONF_DEFAULTS in entry: + raise cv.Invalid( + f"'{CONF_DEFAULTS}' may only be used together with '{CONF_FILES}'", + path=[index], + ) + return [entry] + + extra_keys = set(entry) - {CONF_PLATFORM, CONF_DEFAULTS, CONF_FILES} + if extra_keys: + raise cv.Invalid( + f"'{CONF_FILES}' cannot be combined with " + f"{', '.join(sorted(extra_keys))} on the same entry", + path=[index], + ) + + files = entry[CONF_FILES] + if files is None: + raise cv.Invalid(f"'{CONF_FILES}' must not be empty", path=[index]) + if not isinstance(files, list): + raise cv.Invalid(f"'{CONF_FILES}' must be a list", path=[index]) + if not files: + raise cv.Invalid(f"'{CONF_FILES}' must not be empty", path=[index]) + + defaults = entry.get(CONF_DEFAULTS, {}) + if defaults is None: + defaults = {} + if not isinstance(defaults, dict): + raise cv.Invalid(f"'{CONF_DEFAULTS}' must be a mapping", path=[index]) + # Neither `id:` nor `platform:` makes sense inside `defaults:`. + for disallowed in (CONF_ID, CONF_PLATFORM): + if disallowed in defaults: + raise cv.Invalid( + f"'{disallowed}' is not allowed inside '{CONF_DEFAULTS}'", + path=[index], + ) + + from esphome import yaml_util + + platform = entry[CONF_PLATFORM] + result: list[dict] = [] + for file_entry in files: + if not isinstance(file_entry, dict): + raise cv.Invalid( + f"each entry in '{CONF_FILES}' must be a mapping", path=[index] + ) + # The platform is chosen by the entry's own `platform:` key, not per file. + if CONF_PLATFORM in file_entry: + raise cv.Invalid( + f"'{CONF_PLATFORM}' is not allowed inside '{CONF_FILES}'", + path=[index], + ) + # Keep the `files:` item's source range so whole-entry errors anchor there; + # `make_data_base` needs a real ESPHomeDataBase, so skip it for plain dicts. + source = ( + file_entry if isinstance(file_entry, yaml_util.ESPHomeDataBase) else None + ) + merged = yaml_util.make_data_base( + {CONF_PLATFORM: platform, **defaults, **file_entry}, source + ) + result.append(_drop_incompatible_byte_order(merged, file_entry, index=index)) + return result + + +def expand_platform_config(config: list) -> list: + """Expand `defaults:`/`files:` entries; the platform's own CONFIG_SCHEMA validates each result.""" + result = [] + for i, entry in enumerate(config): + if isinstance(entry, dict) and CONF_PLATFORM in entry: + result.extend(_expand_platform_entry(i, entry)) + else: + result.append(entry) + return result + + +EXPAND_PLATFORM_CONFIG = expand_platform_config + +# --------------------- end defaults/files expansion ------------------------- + + # --------------------------------------------------------------------------- # Legacy top-level component -> `image:` platform deprecation helpers # -- REMOVE after 2027.1.0 together with the `animation:`/`online_image:` shims. @@ -496,11 +620,17 @@ def _is_legacy_image_format(config: object) -> bool: proper error instead of the migration silently dropping the input. """ if isinstance(config, list): - # A bare list of (not-yet-platform-tagged) image dicts. + # Exclude `files:` entries -- the list branch would otherwise silently + # migrate them to `platform: file` instead of raising the missing-platform error. return bool(config) and all( - isinstance(entry, dict) and CONF_PLATFORM not in entry for entry in config + isinstance(entry, dict) + and CONF_PLATFORM not in entry + and CONF_FILES not in entry + for entry in config ) - if not isinstance(config, dict): + if not isinstance(config, dict) or CONF_PLATFORM in config or CONF_FILES in config: + # `platform:`/`files:` dicts are new-format (left for list-wrapping + + # expansion); the legacy flattener has no `files:` branch and would drop them. return False # A single image dict, or the grouped `defaults:`/`images:`/type-key form. return ( @@ -532,18 +662,8 @@ def _flatten_legacy_image_config(config: object) -> list[dict]: def _add(entry: dict, extra: dict) -> None: merged = {**defaults, **extra, **entry} - # The legacy `defaults:`/type-grouped forms only applied `byte_order` to - # types that support it. Replicate that so an endian default merged into - # e.g. a binary image stays valid. - type_class = IMAGE_TYPE.get(str(merged.get(CONF_TYPE, "")).upper()) - if ( - CONF_BYTE_ORDER in merged - and isinstance(type_class, type) - and issubclass(type_class, ImageEncoder) - and not type_class.is_endian() - ): - del merged[CONF_BYTE_ORDER] - result.append(merged) + # Always drop, matching the pre-platform behavior -- see `_drop_incompatible_byte_order`. + result.append(_drop_incompatible_byte_order(merged, {})) def _add_entries(entries: object, extra: dict) -> None: # `entries` may be a single image dict or a list of them; non-dict diff --git a/esphome/components/runtime_image/__init__.py b/esphome/components/runtime_image/__init__.py index d8517d4493..9fa32a5a65 100644 --- a/esphome/components/runtime_image/__init__.py +++ b/esphome/components/runtime_image/__init__.py @@ -5,6 +5,7 @@ from esphome.components.const import CONF_BYTE_ORDER from esphome.components.image import ( IMAGE_TYPE, Image_, + validate_byte_order, validate_settings, validate_transparency, validate_type, @@ -128,9 +129,7 @@ def runtime_image_schema(image_class: cg.MockObjClass = RuntimeImage) -> cv.Sche cv.Required(CONF_FORMAT): cv.one_of(*IMAGE_FORMATS, upper=True), cv.Optional(CONF_RESIZE): cv.dimensions, cv.Required(CONF_TYPE): validate_type(IMAGE_TYPE), - cv.Optional(CONF_BYTE_ORDER): cv.one_of( - "BIG_ENDIAN", "LITTLE_ENDIAN", upper=True - ), + cv.Optional(CONF_BYTE_ORDER): validate_byte_order, cv.Optional(CONF_TRANSPARENCY, default="OPAQUE"): validate_transparency(), cv.Optional(CONF_PLACEHOLDER): cv.use_id(Image_), } diff --git a/esphome/config.py b/esphome/config.py index 987bb9c96a..13ec744ce4 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -620,6 +620,23 @@ class LoadValidationStep(ConfigValidationStep): elif not isinstance(self.conf, list): result[self.domain] = self.conf = [self.conf] + # Permanent expansion hook: a platform-tagged entry may expand into + # several (e.g. `image`'s `defaults:`/`files:`), for `platform:`-tagged dicts only. + if (expand := component.expand_platform_config) is not None and all( + isinstance(entry, dict) and CONF_PLATFORM in entry + for entry in self.conf + ): + with result.catch_error(path): + expanded = expand(self.conf) + if not isinstance(expanded, list): + # A non-list return is a component bug (not a user error): + # raise explicitly (survives -O/-OO) so it escapes catch_error. + raise TypeError( + f"{self.domain}: EXPAND_PLATFORM_CONFIG must " + f"return a list, got {type(expanded).__name__}" + ) + result[self.domain] = self.conf = expanded + # Process AUTO_LOAD _process_auto_load(result, component, path) diff --git a/esphome/loader.py b/esphome/loader.py index f994f0c5eb..23c6d1bfa5 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -164,6 +164,14 @@ class ComponentManifest: """ return getattr(self.module, "LEGACY_CONFIG_MIGRATE", None) + @property + def expand_platform_config( + self, + ) -> Callable[[list[ConfigType]], list[ConfigType]] | None: + """Optional `EXPAND_PLATFORM_CONFIG` callable; runs on the normalized `platform:`-tagged + entry list before per-entry CONFIG_SCHEMA. Must return a list (raise `cv.Invalid` for user errors).""" + return getattr(self.module, "EXPAND_PLATFORM_CONFIG", None) + @property def resources(self) -> list[FileResource]: """Return a list of all file resources defined in the package of this component. diff --git a/tests/component_tests/image/test_init.py b/tests/component_tests/image/test_init.py index 78462463b1..846c152cab 100644 --- a/tests/component_tests/image/test_init.py +++ b/tests/component_tests/image/test_init.py @@ -21,16 +21,20 @@ from esphome.components.image import ( CONF_OPAQUE, CONF_TRANSPARENCY, PLATFORM_FILE, + _expand_platform_entry, _flatten_legacy_image_config, _is_legacy_image_format, _is_new_image_format, _migrate_legacy_image_config, + expand_platform_config, get_all_image_metadata, get_image_metadata, ) from esphome.const import ( + CONF_DEFAULTS, CONF_DITHER, CONF_FILE, + CONF_FILES, CONF_ID, CONF_PLATFORM, CONF_RAW_DATA_ID, @@ -259,6 +263,15 @@ def test_flatten_keeps_byte_order_for_endian_type() -> None: assert out[0][CONF_BYTE_ORDER] == "little_endian" +def test_flatten_drops_byte_order_written_directly_on_legacy_entry() -> None: + """The legacy flattener drops an incompatible byte_order even when written directly on the entry.""" + out = _flatten_legacy_image_config( + {"binary": [{"id": "a", "file": "x.png", "byte_order": "little_endian"}]} + ) + assert out == [{"id": "a", "file": "x.png", "type": "binary"}] + assert CONF_BYTE_ORDER not in out[0] + + def test_flatten_skips_meta_and_unknown_keys() -> None: out = _flatten_legacy_image_config( { @@ -342,6 +355,42 @@ def test_migrate_legacy_warns_and_prepends_platform( ), pytest.param({"foo": 1}, False, id="dict_unknown_keys"), pytest.param("a string", False, id="scalar"), + # A `platform:`-tagged dict is the new format written without list brackets. + pytest.param( + {CONF_PLATFORM: "file", "id": "a", "file": "x.png"}, + False, + id="platform_tagged_flat_dict", + ), + pytest.param( + { + CONF_PLATFORM: "file", + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + }, + False, + id="platform_tagged_defaults_files_dict", + ), + # `files:` without `platform:` is not legacy either -- the flattener has no branch for it. + pytest.param( + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + }, + False, + id="defaults_files_dict_without_platform", + ), + # Same as above in a list -- without this exclusion it would be silently + # migrated to a hard-coded `platform: file` instead of raising the error. + pytest.param( + [ + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "x.png"}], + } + ], + False, + id="defaults_files_list_entry_without_platform", + ), ], ) def test_is_legacy_image_format(config: object, expected: bool) -> None: @@ -359,17 +408,290 @@ def test_is_legacy_image_format(config: object, expected: bool) -> None: def test_migrate_returns_none_for_invalid_legacy_shapes( config: object, caplog: pytest.LogCaptureFixture ) -> None: - """Unrecognised shapes are not migrated (and emit no warning) so normal - platform validation surfaces a proper error instead of silently dropping - the offending input.""" + """Unrecognised shapes are not migrated (and emit no warning), so normal platform validation reports them.""" with caplog.at_level(logging.WARNING): assert _migrate_legacy_image_config(config) is None assert "deprecated" not in caplog.text +def test_migrate_returns_none_for_mapping_form_defaults_files() -> None: + """A `platform:`-tagged `defaults:`/`files:` mapping must not be swallowed by the legacy migrator.""" + config = { + CONF_PLATFORM: "file", + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + assert _migrate_legacy_image_config(config) is None + + +def test_migrate_returns_none_for_defaults_files_dict_without_platform() -> None: + """`defaults:`/`files:` without `platform:` must not be swallowed either -- the flattener has + no `files:` branch and would silently return `[]`.""" + config = { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + assert _migrate_legacy_image_config(config) is None + + +def test_migrate_returns_none_for_defaults_files_list_entry_without_platform() -> None: + """Same, in a list -- previously the list branch migrated it to a hard-coded + `platform: file` instead of raising a missing-platform error.""" + config = [ + { + "defaults": {"type": "rgb565"}, + "files": [{"id": "a", "file": "a.png"}], + } + ] + assert _migrate_legacy_image_config(config) is None + + # --------------------------- end legacy migration -------------------------- +def test_expand_platform_entry_passes_through_plain_entry() -> None: + entry = {CONF_PLATFORM: "file", "id": "a", "file": "x.png"} + assert _expand_platform_entry(0, entry) == [entry] + + +def test_expand_platform_entry_expands_files_with_defaults() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565", "transparency": "opaque"}, + CONF_FILES: [ + {"id": "img1", "file": "foo.png"}, + {"id": "img2", "file": "bar.png", "type": "GRAYSCALE"}, + ], + } + assert _expand_platform_entry(0, entry) == [ + { + CONF_PLATFORM: "file", + "id": "img1", + "file": "foo.png", + "type": "RGB565", + "transparency": "opaque", + }, + { + CONF_PLATFORM: "file", + "id": "img2", + "file": "bar.png", + "type": "GRAYSCALE", + "transparency": "opaque", + }, + ] + + +def test_expand_platform_entry_files_without_defaults() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "img1", "file": "foo.png"}], + } + assert _expand_platform_entry(0, entry) == [ + {CONF_PLATFORM: "file", "id": "img1", "file": "foo.png"} + ] + + +def test_expand_platform_entry_preserves_source_range() -> None: + """A merged entry keeps the source range of its `files:` item so whole-entry errors anchor there.""" + from esphome import yaml_util + + file_entry = yaml_util.make_data_base({"id": "img1", "file": "foo.png"}) + file_entry._esp_range = "sentinel-range" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [file_entry], + } + [out] = _expand_platform_entry(0, entry) + assert isinstance(out, yaml_util.ESPHomeDataBase) + assert out.esp_range == "sentinel-range" + + +def test_expand_platform_entry_plain_dict_file_entry_has_no_source_range() -> None: + """Plain-dict `files:` items must not crash -- `from_database` reads `.esp_range` unconditionally.""" + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "img1", "file": "foo.png"}], + } + [out] = _expand_platform_entry(0, entry) + assert out == {CONF_PLATFORM: "file", "id": "img1", "file": "foo.png"} + + +def test_expand_platform_entry_per_file_overrides_win() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [{"id": "img1", "file": "foo.png", "type": "BINARY"}], + } + [out] = _expand_platform_entry(0, entry) + assert out["type"] == "BINARY" + + +def test_expand_platform_entry_drops_byte_order_for_non_endian_override() -> None: + """A `byte_order` default merged into a non-endian override is dropped, as the legacy flattener did.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "little_endian"}, + CONF_FILES: [ + {"id": "a", "file": "x.png"}, + {"id": "b", "file": "y.png", "type": "binary"}, + ], + } + out = _expand_platform_entry(0, entry) + assert out[0]["byte_order"] == "little_endian" + assert "byte_order" not in out[1] + + +def test_expand_platform_entry_invalid_byte_order_in_defaults_raises() -> None: + """A dropped `byte_order` inherited from `defaults:` is still validated, so a typo raises.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "little_andian"}, + CONF_FILES: [{"id": "a", "file": "x.png", "type": "binary"}], + } + with pytest.raises(cv.Invalid, match="did you mean") as excinfo: + _expand_platform_entry(0, entry) + assert excinfo.value.path == [0] + + +def test_expand_platform_entry_keeps_byte_order_for_endian_override() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565", "byte_order": "big_endian"}, + CONF_FILES: [{"id": "a", "file": "x.png", "type": "rgb565"}], + } + [out] = _expand_platform_entry(0, entry) + assert out["byte_order"] == "big_endian" + + +def test_expand_platform_entry_keeps_explicit_byte_order_conflict() -> None: + """A `byte_order` written directly on the entry is kept so validate_settings raises the normal error.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "rgb565"}, + CONF_FILES: [ + { + "id": "a", + "file": "x.png", + "type": "binary", + "byte_order": "little_endian", + } + ], + } + [out] = _expand_platform_entry(0, entry) + assert out["byte_order"] == "little_endian" + + +def test_expand_platform_entry_defaults_without_files_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_DEFAULTS: {"type": "RGB565"}} + with pytest.raises(cv.Invalid, match="may only be used together with") as excinfo: + _expand_platform_entry(0, entry) + assert excinfo.value.path == [0] + + +def test_expand_platform_entry_null_files_raises_not_empty() -> None: + """A `files:` key with no value parses to `None` and must be reported clearly.""" + entry = {CONF_PLATFORM: "file", CONF_DEFAULTS: {"type": "RGB565"}, CONF_FILES: None} + with pytest.raises(cv.Invalid, match="must not be empty"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_empty_files_list_raises_not_empty() -> None: + """An explicit `files: []` must not silently drop the whole platform entry.""" + entry = {CONF_PLATFORM: "file", CONF_FILES: []} + with pytest.raises(cv.Invalid, match="must not be empty"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_files_with_stray_key_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "a", "file": "x.png"}], + "extra": 1, + } + with pytest.raises(cv.Invalid, match="cannot be combined with"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_id_in_defaults_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {CONF_ID: "a"}, + CONF_FILES: [{"file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_platform_in_defaults_raises() -> None: + """`platform:` inside `defaults:` would silently reassign every file's platform.""" + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {CONF_PLATFORM: "animation"}, + CONF_FILES: [{"id": "a", "file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_platform_in_file_entry_raises() -> None: + """`platform:` on a `files:` item must not silently override the entry's platform.""" + entry = { + CONF_PLATFORM: "file", + CONF_FILES: [{"id": "a", "file": "x.png", CONF_PLATFORM: "animation"}], + } + with pytest.raises(cv.Invalid, match="not allowed inside"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_files_not_list_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_FILES: "not-a-list"} + with pytest.raises(cv.Invalid, match="must be a list"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_defaults_not_mapping_raises() -> None: + entry = { + CONF_PLATFORM: "file", + CONF_DEFAULTS: "not-a-mapping", + CONF_FILES: [{"id": "a", "file": "x.png"}], + } + with pytest.raises(cv.Invalid, match="must be a mapping"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_entry_file_item_not_mapping_raises() -> None: + entry = {CONF_PLATFORM: "file", CONF_FILES: [1, 2]} + with pytest.raises(cv.Invalid, match="must be a mapping"): + _expand_platform_entry(0, entry) + + +def test_expand_platform_config_mixes_plain_and_expanded_entries() -> None: + config = [ + { + CONF_PLATFORM: "file", + CONF_DEFAULTS: {"type": "RGB565"}, + CONF_FILES: [ + {"id": "img1", "file": "foo.png"}, + {"id": "img2", "file": "bar.png"}, + ], + }, + {CONF_PLATFORM: "file", "id": "plain", "file": "baz.png", "type": "BINARY"}, + ] + out = expand_platform_config(config) + assert [entry["id"] for entry in out] == ["img1", "img2", "plain"] + + +def test_expand_platform_config_ignores_non_platform_entries() -> None: + # Not expanded here -- legacy_config_migrate runs before this hook and is + # responsible for tagging/flattening pre-platform shapes. + config = ["not-a-platform-entry"] + assert expand_platform_config(config) == config + + +# --------------------- end defaults/files expansion ------------------------- + + def test_validate_image_final_defaults_to_little_endian() -> None: out = validate_image_final({CONF_FILE: "x.png"}) assert out[CONF_BYTE_ORDER] == "LITTLE_ENDIAN" diff --git a/tests/components/animation/validate-platform-defaults.host.yaml b/tests/components/animation/validate-platform-defaults.host.yaml new file mode 100644 index 0000000000..034497c548 --- /dev/null +++ b/tests/components/animation/validate-platform-defaults.host.yaml @@ -0,0 +1,21 @@ +# `platform: animation` entry exercising the shared `defaults:`/`files:` expansion. +display: + - platform: sdl + id: animation_display + auto_clear_enabled: false + dimensions: + width: 480 + height: 480 + +image: + - platform: animation + defaults: + type: rgb565 + transparency: opaque + resize: 50x50 + files: + - id: platform_defaults_animation + file: $component_dir/anim.gif + - id: platform_defaults_animation_rgb + file: $component_dir/anim.apng + type: rgb diff --git a/tests/components/image/validate-platform-defaults.host.yaml b/tests/components/image/validate-platform-defaults.host.yaml new file mode 100644 index 0000000000..e1b3037cc3 --- /dev/null +++ b/tests/components/image/validate-platform-defaults.host.yaml @@ -0,0 +1,24 @@ +# `platform: file` entry using the `defaults:`/`files:` shape, including the +# per-type byte_order drop when an entry overrides to a non-endian type. +display: + - platform: sdl + id: image_display + auto_clear_enabled: false + dimensions: + width: 480 + height: 480 + +image: + - platform: file + defaults: + type: rgb565 + transparency: opaque + byte_order: little_endian + resize: 50x50 + dither: FloydSteinberg + files: + - id: platform_defaults_image + file: ../../pnglogo.png + - id: platform_defaults_binary + file: ../../pnglogo.png + type: binary diff --git a/tests/unit_tests/test_config_normalization.py b/tests/unit_tests/test_config_normalization.py index c8b7b63094..04363ad45b 100644 --- a/tests/unit_tests/test_config_normalization.py +++ b/tests/unit_tests/test_config_normalization.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest -from esphome import config, yaml_util +from esphome import config, config_validation as cv, yaml_util from esphome.core import CORE, AutoLoad from esphome.types import ConfigType @@ -127,12 +127,14 @@ def _run_load_step( domain: str, conf: object, migrate: Callable[[ConfigType], list | None] | None, + expand: Callable[[list], list] | None = None, ) -> config.Config: - """Run a LoadValidationStep for a platform component with a given migrate hook.""" + """Run a LoadValidationStep for a platform component with given hooks.""" component = Mock() component.is_platform_component = True component.multi_conf_no_default = False component.legacy_config_migrate = migrate + component.expand_platform_config = expand result = config.Config() with ( @@ -197,6 +199,124 @@ def test_legacy_migrate_skipped_for_autoload() -> None: assert result["image"] == [auto] +# --------------------------------------------------------------------------- +# EXPAND_PLATFORM_CONFIG hook on LoadValidationStep -- permanent counterpart +# to legacy_config_migrate; runs after legacy migration/list normalization. +# --------------------------------------------------------------------------- + + +def test_expand_hook_rewrites_conf() -> None: + """A config the expand hook rewrites is replaced with the expanded list.""" + expanded = [{"platform": "file", "id": "a"}, {"platform": "file", "id": "b"}] + expand = Mock(return_value=expanded) + + result = _run_load_step("image", [{"platform": "file", "id": "a"}], None, expand) + + expand.assert_called_once_with([{"platform": "file", "id": "a"}]) + assert result["image"] == expanded + + +def test_expand_hook_absent_is_noop() -> None: + """A platform component without the hook is left as normalized by the + existing list-wrapping logic.""" + result = _run_load_step("image", [{"platform": "file", "id": "a"}], None, None) + + assert result["image"] == [{"platform": "file", "id": "a"}] + + +def test_expand_hook_runs_after_legacy_migrate() -> None: + """The expand hook sees the already-migrated list, not the raw legacy conf.""" + migrated = [{"platform": "file", "id": "a"}] + migrate = Mock(return_value=migrated) + expand = Mock(side_effect=lambda conf: conf) + + _run_load_step("image", [{"id": "a", "file": "x.png"}], migrate, expand) + + expand.assert_called_once_with(migrated) + + +def test_expand_hook_skipped_for_non_dict_entry() -> None: + """Malformed entries are left alone; the hook only sees `platform:`-tagged dicts.""" + expand = Mock(side_effect=lambda conf: conf) + + result = _run_load_step("image", ["not-a-dict"], None, expand) + + expand.assert_not_called() + assert result["image"] == ["not-a-dict"] + + +def test_expand_hook_skipped_for_entry_missing_platform_key() -> None: + """A dict entry missing the `platform:` key is left alone -- the normal + per-entry error reporting further down catches this case instead.""" + expand = Mock(side_effect=lambda conf: conf) + + result = _run_load_step("image", [{"id": "a"}], None, expand) + + expand.assert_not_called() + assert result["image"] == [{"id": "a"}] + + +def test_expand_hook_skipped_for_autoload() -> None: + """A non-empty AutoLoad reaching the hook stage is left alone.""" + expand = Mock(side_effect=lambda conf: conf) + auto = AutoLoad() + auto["id"] = "a" + + result = _run_load_step("image", auto, None, expand) + + expand.assert_not_called() + assert result["image"] == [auto] + + +def test_expand_hook_runs_when_all_entries_are_platform_tagged_dicts() -> None: + """The guard does not block the normal, well-formed case.""" + expand = Mock(side_effect=lambda conf: conf) + conf = [{"platform": "file", "id": "a"}, {"platform": "animation", "id": "b"}] + + result = _run_load_step("image", conf, None, expand) + + expand.assert_called_once_with(conf) + assert result["image"] == conf + + +def test_expand_hook_invalid_reports_single_error_at_domain_path() -> None: + """A `cv.Invalid` from the hook is reported once with the domain path prepended; no further validation runs.""" + expand = Mock(side_effect=cv.Invalid("bad shape")) + pre_expand_conf = [{"platform": "file", "id": "a"}] + + result = _run_load_step("image", pre_expand_conf, None, expand) + + assert len(result.errors) == 1 + assert result.errors[0].path == ["image"] + assert "bad shape" in str(result.errors[0]) + assert result["image"] == pre_expand_conf + + +def test_expand_hook_final_external_invalid_reports_without_path_prepend() -> None: + """`cv.FinalExternalInvalid` keeps its already-resolved path (no domain path prepended).""" + already_resolved_error = cv.FinalExternalInvalid( + "bad shape", path=["image", 3, "files"] + ) + expand = Mock(side_effect=already_resolved_error) + pre_expand_conf = [{"platform": "file", "id": "a"}] + + result = _run_load_step("image", pre_expand_conf, None, expand) + + assert len(result.errors) == 1 + assert result.errors[0] is already_resolved_error + assert result.errors[0].path == ["image", 3, "files"] + assert result["image"] == pre_expand_conf + + +def test_expand_hook_non_list_return_raises_type_error() -> None: + """A non-list return is a component bug: it escapes as an uncaught TypeError + (explicit raise survives -O/-OO).""" + expand = Mock(return_value={"not": "a list"}) + + with pytest.raises(TypeError, match="must return a list"): + _run_load_step("image", [{"platform": "file", "id": "a"}], None, expand) + + def _write_merge_conflict_config(tmp_path: Path, *, suppress: bool) -> Path: """Create a config where two `<<` includes both define `logger:`. From 78a65eabdc6f33e6ac7f398a905217f61f779b64 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 16:30:34 -0500 Subject: [PATCH 14/16] [ci] Stop jobs hanging on apt by restoring the cached apt action and bounding raw apt calls (#18518) --- .github/workflows/ci-api-proto.yml | 28 +++++++++- .github/workflows/ci.yml | 89 +++++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-api-proto.yml b/.github/workflows/ci-api-proto.yml index 820081cc46..771b4cd94f 100644 --- a/.github/workflows/ci-api-proto.yml +++ b/.github/workflows/ci-api-proto.yml @@ -41,10 +41,32 @@ jobs: version: "0.11.15" - name: Install apt dependencies + # PR-only workflow, so nothing on dev could seed a shared apt cache + # entry; the cached apt action would save one copy per PR. Plain apt + # with every call bounded: the apt.conf.d timeouts make a dead + # mirror fail over in seconds, and timeout runs under sudo so it can + # kill apt-get itself. Install without update first: image lists are + # fresh, and the index refresh is what a congested mirror makes slow. + timeout-minutes: 15 run: | - sudo apt update - sudo apt-cache show protobuf-compiler - sudo apt install -y protobuf-compiler + sudo tee /etc/apt/apt.conf.d/99ci-acquire-timeouts >/dev/null <<'EOF' + Acquire::Retries "1"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + EOF + # Common path: the image's package lists are fresh enough. + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 \ + apt-get install -y protobuf-compiler; then + protoc --version + exit 0 + fi + # Rescue path: refresh the lists once with a generous bound; the + # apt config already fails a stalled mirror over quickly. + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 30 \ + dpkg --configure -a || true + sudo timeout -k 15 300 apt-get update + sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 \ + apt-get install -y protobuf-compiler protoc --version - name: Install python dependencies run: uv pip install --system aioesphomeapi -c requirements.txt -r requirements_dev.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d8f35ed83..d2d4c7a2a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,22 @@ jobs: uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . + seed-apt-cache: + name: Seed apt package cache + runs-on: ubuntu-24.04 + # PR-branch cache saves are invisible to other PRs, so dev/beta/release + # pushes seed the one shared entry PR jobs restore. The key is derived + # only from the package list and version; keep both identical in every + # step that restores it. In ci-status needs so a broken seed fails dev. + if: github.event_name == 'push' + timeout-minutes: 10 + steps: + - name: Install apt packages (cached) + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 + determine-jobs: name: Determine which jobs to run runs-on: ubuntu-24.04 @@ -323,7 +339,8 @@ jobs: integration-tests: name: Run integration tests (${{ matrix.bucket.name }}) - runs-on: ubuntu-latest + # Must match seed-apt-cache's image: the apt cache key has no OS in it. + runs-on: ubuntu-24.04 needs: - common - determine-jobs @@ -335,12 +352,16 @@ jobs: steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install ccache - # Speeds up the host compiles: tests in a bucket compile overlapping - # component sets, so later tests reuse earlier tests' objects. - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends ccache + - name: Install apt packages (cached) + # ccache speeds up the host compiles. A cache hit never touches apt + # (mirror outages cannot hang the job); the timeout bounds the cold + # path. Packages and version must match seed-apt-cache exactly; + # libsdl2-dev is unused here and carried only for cache-key parity. + timeout-minutes: 10 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 - name: Set up Python 3.13 id: python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -421,6 +442,7 @@ jobs: benchmarks: name: Run CodSpeed benchmarks runs-on: ubuntu-24.04 + timeout-minutes: 30 needs: - common - determine-jobs @@ -457,6 +479,41 @@ jobs: fi echo "binary=$BINARY" >> $GITHUB_OUTPUT + - name: Bound apt fetches and pre-install libc6-dbg + # The CodSpeed runner installs valgrind + libc6-dbg via its own + # unbounded apt-get update; per-invocation apt options cannot reach + # it. The apt.conf.d timeouts below bound every later apt call in + # this job, the runner's included. Pre-installing libc6-dbg lets the + # runner skip apt once its valgrind cache is restored (it checks + # ``dpkg -s libc6-dbg``, so the cache action's unregistered restores + # would not count). Install without update first: image lists are + # fresh, and the index refresh is what a congested mirror makes + # slow. Best effort; the job timeout is the last backstop. + timeout-minutes: 15 + continue-on-error: true + run: | + sudo tee /etc/apt/apt.conf.d/99ci-acquire-timeouts >/dev/null <<'EOF' + Acquire::Retries "1"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + EOF + if dpkg -s libc6-dbg >/dev/null 2>&1; then + echo "libc6-dbg already installed" + exit 0 + fi + # Common path: the image's package lists are fresh enough. + if sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 \ + apt-get install -y libc6-dbg; then + exit 0 + fi + # Rescue path: refresh the lists once with a generous bound; the + # apt config already fails a stalled mirror over quickly. + sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 30 \ + dpkg --configure -a || true + sudo timeout -k 15 300 apt-get update + sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 \ + apt-get install -y libc6-dbg + - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3 with: @@ -875,12 +932,17 @@ jobs: - name: List components run: echo ${{ matrix.batch.components }} - - name: Install apt packages - # Not cached: this job is pull-request-only, so a cache save could - # never be shared and would only consume quota. - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends libsdl2-dev ccache + - name: Install apt packages (cached) + # A cache hit (seeded on dev by seed-apt-cache) never touches apt, + # so mirror outages cannot hang this PR-only job; the timeout bounds + # the cold path. Packages and version must match seed-apt-cache + # exactly. The action has no --no-install-recommends; same package + # set this job used before #17463. + timeout-minutes: 10 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 + with: + packages: libsdl2-dev ccache + version: 1.1 - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1415,6 +1477,7 @@ jobs: # this check. needs: - common + - seed-apt-cache - determine-jobs - ci-custom - pylint From f735dcadc0c38f25eec83cf4f5eba97bc62e30d6 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:33:05 +1200 Subject: [PATCH 15/16] Bump version to 2026.8.0b6 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 3dad4629be..c83d95d0ef 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.0b5 +PROJECT_NUMBER = 2026.8.0b6 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index e86465f9a0..2296f8c0b7 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.0b5" +__version__ = "2026.8.0b6" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From c45599196235345e2f13415eccb537b2d6e13b49 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 19 Aug 2026 11:39:01 -0500 Subject: [PATCH 16/16] [ci] Key PlatformIO cache on the Python version so a runner image bump does not serve a broken LibreTiny venv (#18512) --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2d4c7a2a1..fa119fb6d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -598,24 +598,29 @@ jobs: fetch-depth: 2 - name: Restore Python + id: restore-python uses: ./.github/actions/restore-python with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + # Key on the exact Python version as well: LibreTiny creates a venv under + # ~/.platformio/penv whose interpreter is a symlink into the runner's + # hosted toolcache, so a cache saved on an older runner image breaks once + # a new image ships a newer patch release and drops the old interpreter. - name: Cache platformio if: github.ref == 'refs/heads/dev' && matrix.pio_cache_key uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.platformio - key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} + key: platformio-${{ matrix.pio_cache_key }}-${{ steps.restore-python.outputs.python-version }}-${{ hashFiles('platformio.ini') }} - name: Cache platformio if: github.ref != 'refs/heads/dev' && matrix.pio_cache_key uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.platformio - key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }} + key: platformio-${{ matrix.pio_cache_key }}-${{ steps.restore-python.outputs.python-version }}-${{ hashFiles('platformio.ini') }} - name: Cache ESP-IDF install if: matrix.cache_idf