From 5ed20805ced8d67e3b27117ab0ba231c8571b9ab Mon Sep 17 00:00:00 2001 From: n-IA-hane <49248235+n-IA-hane@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:35:59 +0200 Subject: [PATCH 01/65] [audio_http] Add persistent ring buffer option (#18708) Co-authored-by: n-IA-hane --- esphome/components/audio_http/audio_http_media_source.cpp | 4 +++- esphome/components/audio_http/audio_http_media_source.h | 2 ++ esphome/components/audio_http/media_source.py | 4 ++++ tests/components/audio_http/common.yaml | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/esphome/components/audio_http/audio_http_media_source.cpp b/esphome/components/audio_http/audio_http_media_source.cpp index 04b7d046e6..fb8620f7d9 100644 --- a/esphome/components/audio_http/audio_http_media_source.cpp +++ b/esphome/components/audio_http/audio_http_media_source.cpp @@ -30,8 +30,9 @@ void AudioHTTPMediaSource::dump_config() { ESP_LOGCONFIG(TAG, "Audio HTTP Media Source:\n" " Buffer Size: %zu bytes\n" + " Persistent Ring Buffer: %s\n" " Decoder Task Stack in PSRAM: %s", - this->buffer_size_, YESNO(this->decoder_task_stack_in_psram_)); + this->buffer_size_, YESNO(this->persistent_ring_buffer_), YESNO(this->decoder_task_stack_in_psram_)); } void AudioHTTPMediaSource::setup() { @@ -39,6 +40,7 @@ void AudioHTTPMediaSource::setup() { micro_decoder::DecoderConfig config; config.ring_buffer_size = this->buffer_size_; + config.persistent_ring_buffer = this->persistent_ring_buffer_; // Keep the transfer buffer smaller than the ring buffer so the reader can top up the ring // while the decoder is still draining it, instead of oscillating between empty and full. config.transfer_buffer_size = std::min(DEFAULT_TRANSFER_BUFFER_SIZE, this->buffer_size_ / 2); diff --git a/esphome/components/audio_http/audio_http_media_source.h b/esphome/components/audio_http/audio_http_media_source.h index f794aa1f02..a97025e53e 100644 --- a/esphome/components/audio_http/audio_http_media_source.h +++ b/esphome/components/audio_http/audio_http_media_source.h @@ -33,6 +33,7 @@ class AudioHTTPMediaSource final : public Component, void set_buffer_size(size_t buffer_size) { this->buffer_size_ = buffer_size; } void set_task_stack_in_psram(bool task_stack_in_psram) { this->decoder_task_stack_in_psram_ = task_stack_in_psram; } + void set_persistent_ring_buffer(bool persistent) { this->persistent_ring_buffer_ = persistent; } // MediaSource interface implementation bool play_uri(const std::string &uri) override; @@ -54,6 +55,7 @@ class AudioHTTPMediaSource final : public Component, // on_audio_write(). Must be atomic to avoid a data race. std::atomic pause_{false}; bool decoder_task_stack_in_psram_{false}; + bool persistent_ring_buffer_{false}; }; } // namespace esphome::audio_http diff --git a/esphome/components/audio_http/media_source.py b/esphome/components/audio_http/media_source.py index e8acbc81af..14543957e9 100644 --- a/esphome/components/audio_http/media_source.py +++ b/esphome/components/audio_http/media_source.py @@ -7,6 +7,8 @@ from esphome.types import ConfigType CODEOWNERS = ["@kahrendt"] AUTO_LOAD = ["audio"] +CONF_PERSISTENT_RING_BUFFER = "persistent_ring_buffer" + audio_http_ns = cg.esphome_ns.namespace("audio_http") AudioHTTPMediaSource = audio_http_ns.class_( "AudioHTTPMediaSource", cg.Component, media_source.MediaSource @@ -28,6 +30,7 @@ CONFIG_SCHEMA = cv.All( min=5000, max=1000000 ), cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, + cv.Optional(CONF_PERSISTENT_RING_BUFFER, default=False): cv.boolean, } ) .extend(cv.COMPONENT_SCHEMA), @@ -45,3 +48,4 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_task_stack_in_psram(True)) psram.request_external_task_stack() cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) + cg.add(var.set_persistent_ring_buffer(config[CONF_PERSISTENT_RING_BUFFER])) diff --git a/tests/components/audio_http/common.yaml b/tests/components/audio_http/common.yaml index b7457165a5..7bee370c90 100644 --- a/tests/components/audio_http/common.yaml +++ b/tests/components/audio_http/common.yaml @@ -4,4 +4,5 @@ media_source: - platform: audio_http id: audio_http_source buffer_size: 100000 + persistent_ring_buffer: true task_stack_in_psram: true From b11a34af5cf81c0ca227b2be71908773a9433e0a Mon Sep 17 00:00:00 2001 From: Boris Krivonog Date: Mon, 24 Aug 2026 17:51:09 +0200 Subject: [PATCH 02/65] [mitsubishi_cn105] Defer status requests after responses (#18227) --- .../mitsubishi_cn105/mitsubishi_cn105.cpp | 37 ++++++++---- .../mitsubishi_cn105/mitsubishi_cn105.h | 1 + .../mitsubishi_cn105_climate_tests.cpp | 11 ++-- .../climate/mitsubishi_cn105_tests.cpp | 60 +++++++++++++++++-- 4 files changed, 88 insertions(+), 21 deletions(-) diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.cpp b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.cpp index 6683a9a25b..3d30d1a25f 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.cpp +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.cpp @@ -49,6 +49,13 @@ void MitsubishiCN105::initialize() { this->set_state_(State::CONNECTING); } bool MitsubishiCN105::update() { switch (this->state_) { + case State::DEFERRED_STATUS_REQUEST: + // Defer the next request to a later loop iteration; some units might not respond if a request is sent + // immediately after a response. See https://github.com/esphome/esphome/issues/18099. No minimum RX-to-TX delay + // is enforced. + this->set_state_(State::UPDATING_STATUS); + return false; + case State::WAITING_FOR_SCHEDULED_STATUS_UPDATE: if (this->pending_updates_.any()) { this->status_update_wait_credit_ms_ = @@ -101,12 +108,14 @@ bool MitsubishiCN105::should_transition(State from, State to) { return from == State::CONNECTING; case State::UPDATING_STATUS: - return from == State::CONNECTED || from == State::STATUS_UPDATED || - from == State::WAITING_FOR_SCHEDULED_STATUS_UPDATE; + return from == State::DEFERRED_STATUS_REQUEST || from == State::WAITING_FOR_SCHEDULED_STATUS_UPDATE; case State::STATUS_UPDATED: return from == State::UPDATING_STATUS; + case State::DEFERRED_STATUS_REQUEST: + return from == State::CONNECTED || from == State::STATUS_UPDATED; + case State::SCHEDULE_NEXT_STATUS_UPDATE: return from == State::STATUS_UPDATED || from == State::SETTINGS_APPLIED; @@ -114,7 +123,7 @@ bool MitsubishiCN105::should_transition(State from, State to) { return from == State::SCHEDULE_NEXT_STATUS_UPDATE; case State::APPLYING_SETTINGS: - return from == State::WAITING_FOR_SCHEDULED_STATUS_UPDATE || from == State::STATUS_UPDATED; + return from == State::WAITING_FOR_SCHEDULED_STATUS_UPDATE; case State::SETTINGS_APPLIED: return from == State::APPLYING_SETTINGS; @@ -122,9 +131,10 @@ bool MitsubishiCN105::should_transition(State from, State to) { case State::READ_TIMEOUT: return from == State::UPDATING_STATUS || from == State::APPLYING_SETTINGS || from == State::CONNECTING; - default: + case State::NOT_CONNECTED: return false; } + return false; } void MitsubishiCN105::did_transition_(State to) { @@ -135,7 +145,7 @@ void MitsubishiCN105::did_transition_(State to) { case State::CONNECTED: this->current_status_msg_type_ = STATUS_MSG_SETTINGS; - this->set_state_(State::UPDATING_STATUS); + this->set_state_(State::DEFERRED_STATUS_REQUEST); break; case State::UPDATING_STATUS: @@ -143,11 +153,14 @@ void MitsubishiCN105::did_transition_(State to) { break; case State::STATUS_UPDATED: { - if (this->pending_updates_.any() && this->is_status_initialized()) { - this->set_state_(State::APPLYING_SETTINGS); - } else if (this->current_status_msg_type_ == STATUS_MSG_SETTINGS && this->should_request_telemetry_()) { + // When present, pending settings are applied from WAITING_FOR_SCHEDULED_STATUS_UPDATE during the next update(), + // deferring transmission to a later loop iteration; some units might not respond if a request is sent + // immediately after a response, causing the request to time out. + const bool should_apply_pending_settings = this->pending_updates_.any() && this->is_status_initialized(); + if (!should_apply_pending_settings && this->current_status_msg_type_ == STATUS_MSG_SETTINGS && + this->should_request_telemetry_()) { this->current_status_msg_type_ = STATUS_MSG_TELEMETRY; - this->set_state_(State::UPDATING_STATUS); + this->set_state_(State::DEFERRED_STATUS_REQUEST); } else { this->set_state_(State::SCHEDULE_NEXT_STATUS_UPDATE); } @@ -175,7 +188,9 @@ void MitsubishiCN105::did_transition_(State to) { this->set_state_(State::CONNECTING); break; - default: + case State::NOT_CONNECTED: + case State::DEFERRED_STATUS_REQUEST: + case State::WAITING_FOR_SCHEDULED_STATUS_UPDATE: break; } } @@ -359,6 +374,8 @@ const LogString *MitsubishiCN105::state_to_string(State state) { return LOG_STR("UpdatingStatus"); case State::STATUS_UPDATED: return LOG_STR("StatusUpdated"); + case State::DEFERRED_STATUS_REQUEST: + return LOG_STR("DeferredStatusRequest"); case State::SCHEDULE_NEXT_STATUS_UPDATE: return LOG_STR("ScheduleNextStatusUpdate"); case State::WAITING_FOR_SCHEDULED_STATUS_UPDATE: diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h index 4d3f899dee..0fee90dfc1 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h @@ -101,6 +101,7 @@ class MitsubishiCN105 { CONNECTED, UPDATING_STATUS, STATUS_UPDATED, + DEFERRED_STATUS_REQUEST, SCHEDULE_NEXT_STATUS_UPDATE, WAITING_FOR_SCHEDULED_STATUS_UPDATE, APPLYING_SETTINGS, diff --git a/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp index 669345f576..10d935a775 100644 --- a/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp +++ b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp @@ -15,11 +15,10 @@ struct MitsubishiCN105ClimateTestContext { TEST(MitsubishiCN105ClimateTests, CelsiusTemperatureMappingAndTraitsMatchExpectedValues) { MitsubishiCN105ClimateTestContext context; - const auto mapping = TemperatureMapping(); for (int temperature = 16; temperature <= 31; ++temperature) { - EXPECT_EQ(mapping.to_mitsubishi(temperature), temperature); - EXPECT_EQ(mapping.from_mitsubishi(temperature), temperature); + EXPECT_EQ(context.component.get_temperature_mapping().to_mitsubishi(temperature), temperature); + EXPECT_EQ(context.component.get_temperature_mapping().from_mitsubishi(temperature), temperature); } const auto traits = context.sut.traits(); @@ -32,8 +31,6 @@ TEST(MitsubishiCN105ClimateTests, CelsiusTemperatureMappingAndTraitsMatchExpecte TEST(MitsubishiCN105ClimateTests, FahrenheitTemperatureMappingAndTraitsMatchExpectedValues) { MitsubishiCN105ClimateTestContext context; - auto mapping = TemperatureMapping(); - mapping.set_use_fahrenheit(true); context.component.set_use_fahrenheit(true); const std::array cases{ @@ -46,8 +43,8 @@ TEST(MitsubishiCN105ClimateTests, FahrenheitTemperatureMappingAndTraitsMatchExpe }; for (const auto &[fahrenheit, mitsubishi_celsius] : cases) { - EXPECT_FLOAT_EQ(mapping.to_mitsubishi(fahrenheit), mitsubishi_celsius); - EXPECT_FLOAT_EQ(mapping.from_mitsubishi(mitsubishi_celsius), fahrenheit); + EXPECT_FLOAT_EQ(context.component.get_temperature_mapping().to_mitsubishi(fahrenheit), mitsubishi_celsius); + EXPECT_FLOAT_EQ(context.component.get_temperature_mapping().from_mitsubishi(mitsubishi_celsius), fahrenheit); } const auto traits = context.sut.traits(); EXPECT_EQ(traits.get_temperature_unit(), TemperatureUnit::FAHRENHEIT); diff --git a/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_tests.cpp b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_tests.cpp index 3bc6d5b2b8..fcd09bb18e 100644 --- a/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_tests.cpp +++ b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_tests.cpp @@ -42,11 +42,17 @@ TEST(MitsubishiCN105Tests, ConnectAndUpdateStatus) { // All bytes from UART should be consumed EXPECT_TRUE(ctx.uart.rx.empty()); - // After successful connect we request status, first settings (0x02) + // Defer the first settings request (0x02) until the next update. + EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::DEFERRED_STATUS_REQUEST); + EXPECT_TRUE(ctx.uart.tx.empty()); + + ctx.sut.set_current_time(201); + ASSERT_FALSE(ctx.sut.update()); + EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::UPDATING_STATUS); EXPECT_THAT(ctx.uart.tx, ::testing::ElementsAre(0xFC, 0x42, 0x01, 0x30, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7B)); - EXPECT_EQ(ctx.sut.operation_start_ms_, 200); + EXPECT_EQ(ctx.sut.operation_start_ms_, 201); // Clear TX bytes. ctx.uart.tx.clear(); @@ -75,15 +81,24 @@ TEST(MitsubishiCN105Tests, ConnectAndUpdateStatus) { EXPECT_EQ(ctx.sut.status().vane_mode, MitsubishiCN105::VaneMode::POSITION_4); EXPECT_EQ(ctx.sut.status().wide_vane_mode, MitsubishiCN105::WideVaneMode::SWING); - // Now fetch telemetry (0x03) + // Defer the telemetry request (0x03) until the next update. + EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::DEFERRED_STATUS_REQUEST); + EXPECT_TRUE(ctx.uart.tx.empty()); + + ctx.sut.set_current_time(301); + ASSERT_FALSE(ctx.sut.update()); + EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::UPDATING_STATUS); EXPECT_THAT(ctx.uart.tx, ::testing::ElementsAre(0xFC, 0x42, 0x01, 0x30, 0x10, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7A)); - EXPECT_EQ(ctx.sut.operation_start_ms_, 300); + EXPECT_EQ(ctx.sut.operation_start_ms_, 301); // Clear TX bytes. ctx.uart.tx.clear(); + // Queue a setting while waiting for telemetry. + ctx.sut.set_power(true); + // Telemetry response ctx.uart.push_rx({0xFC, 0x62, 0x01, 0x30, 0x10, 0x03, 0x00, 0x00, 0x0B, 0x00, 0x00, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA5}); @@ -103,6 +118,13 @@ TEST(MitsubishiCN105Tests, ConnectAndUpdateStatus) { EXPECT_TRUE(ctx.uart.tx.empty()); EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::WAITING_FOR_SCHEDULED_STATUS_UPDATE); EXPECT_EQ(ctx.sut.operation_start_ms_, 400); + + // Apply the pending setting on the next update, outside RX processing. + ctx.sut.set_current_time(401); + ASSERT_FALSE(ctx.sut.update()); + EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::APPLYING_SETTINGS); + EXPECT_FALSE(ctx.uart.tx.empty()); + EXPECT_EQ(ctx.sut.operation_start_ms_, 401); } TEST(MitsubishiCN105Tests, NoResponseTriggersReconnect) { @@ -469,6 +491,36 @@ TEST(MitsubishiCN105Tests, WriteInterruptsWaitingForNextStatusUpdate) { EXPECT_EQ(ctx.sut.status_update_wait_credit_ms_, 0); } +TEST(MitsubishiCN105Tests, PendingSettingsTakePriorityOverDueTelemetry) { + MitsubishiCN105TestsContext ctx; + + ctx.sut.status_.target_temperature = 24.0f; + ctx.sut.status_.room_temperature = 21.0f; + ASSERT_TRUE(ctx.sut.is_status_initialized()); + + ctx.sut.state_ = TestableMitsubishiCN105::State::STATUS_UPDATED; + ctx.sut.set_state(TestableMitsubishiCN105::State::SCHEDULE_NEXT_STATUS_UPDATE); + ctx.sut.set_current_time(1000); + ASSERT_FALSE(ctx.sut.update()); + ASSERT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::UPDATING_STATUS); + ctx.uart.tx.clear(); + + ctx.sut.set_power(true); + ctx.uart.push_rx({0xFC, 0x62, 0x01, 0x30, 0x10, 0x02, 0x00, 0x00, 0x00, 0x08, 0x07, + 0x00, 0x04, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3C}); + + ctx.sut.set_current_time(1001); + ASSERT_TRUE(ctx.sut.update()); + EXPECT_TRUE(ctx.uart.tx.empty()); + EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::WAITING_FOR_SCHEDULED_STATUS_UPDATE); + + ctx.sut.set_current_time(1002); + ASSERT_FALSE(ctx.sut.update()); + EXPECT_EQ(ctx.sut.state_, TestableMitsubishiCN105::State::APPLYING_SETTINGS); + EXPECT_THAT(ctx.uart.tx, ::testing::ElementsAre(0xFC, 0x41, 0x01, 0x30, 0x10, 0x01, 0x01, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7B)); +} + TEST(MitsubishiCN105Tests, SetAndClearRemoteRoomTemp) { MitsubishiCN105TestsContext ctx; From b8408e14b9c5bdd90e167d6b1a3bf129ac502798 Mon Sep 17 00:00:00 2001 From: Gafielt <57099610+Gafielt@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:36:23 +0200 Subject: [PATCH 03/65] [uart] Delete ESP32 UART driver on shutdown to keep ROM output off the bus (#18684) Co-authored-by: J. Nick Koston --- esphome/components/uart/uart_component_esp_idf.cpp | 11 +++++++++++ esphome/components/uart/uart_component_esp_idf.h | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/esphome/components/uart/uart_component_esp_idf.cpp b/esphome/components/uart/uart_component_esp_idf.cpp index a61339feb4..bbeb86bcdb 100644 --- a/esphome/components/uart/uart_component_esp_idf.cpp +++ b/esphome/components/uart/uart_component_esp_idf.cpp @@ -393,5 +393,16 @@ void IRAM_ATTR IDFUARTComponent::uart_rx_isr_callback(uart_port_t uart_num, uart } #endif // USE_UART_WAKE_LOOP_ON_RX +void IDFUARTComponent::on_shutdown() { + if (this->uart_num_ == UART_NUM_MAX || !uart_is_driver_installed(this->uart_num_)) + return; + uart_wait_tx_done(this->uart_num_, pdMS_TO_TICKS(100)); + // Keep the peripheral quiet across a soft reset so ROM output does not reach the attached device (#15472) + esp_err_t err = uart_driver_delete(this->uart_num_); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_driver_delete failed: %s", esp_err_to_name(err)); + } +} + } // namespace esphome::uart #endif // USE_ESP32 diff --git a/esphome/components/uart/uart_component_esp_idf.h b/esphome/components/uart/uart_component_esp_idf.h index 649dd3aa46..a761d80f04 100644 --- a/esphome/components/uart/uart_component_esp_idf.h +++ b/esphome/components/uart/uart_component_esp_idf.h @@ -52,9 +52,11 @@ class IDFUARTComponent final : public UARTComponent, public Component { void load_settings(bool dump_config) override; using UARTComponent::load_settings; // also bring in the no-arg overload for convenience + void on_shutdown() override; + protected: void check_logger_conflict() override; - uart_port_t uart_num_; + uart_port_t uart_num_{UART_NUM_MAX}; uart_config_t get_config_(); bool has_peek_{false}; From 49fc4be861cf4fdbf946489c5e96fd1373b891ad Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 24 Aug 2026 12:18:17 -0500 Subject: [PATCH 04/65] [ci] Run C++ unit tests when a component's Python or test override changes (#18706) --- script/build_helpers.py | 9 +-- script/determine-jobs.py | 14 +++-- script/helpers.py | 70 +++++++++++++++-------- script/list-components.py | 13 ++--- tests/script/test_determine_jobs.py | 50 ++++++++++++++++ tests/script/test_helpers.py | 89 +++++++++++++++++++++++++++++ 6 files changed, 202 insertions(+), 43 deletions(-) diff --git a/script/build_helpers.py b/script/build_helpers.py index 50830c221e..b4b25924c3 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -10,7 +10,7 @@ from pathlib import Path import subprocess import sys -from helpers import get_all_dependencies, root_path as _root_path +from helpers import get_all_dependencies, has_cpp_unit_tests, root_path as _root_path import yaml # Ensure the repo root is on sys.path so that ``tests.testing_helpers`` and @@ -131,14 +131,11 @@ def filter_components_with_files(components: list[str], tests_dir: Path) -> list """ filtered_components: list[str] = [] for component in components: - test_dir = tests_dir / component - if test_dir.is_dir() and ( - any(test_dir.glob("*.cpp")) or any(test_dir.glob("*.h")) - ): + if has_cpp_unit_tests(component, tests_dir): filtered_components.append(component) else: print( - f"WARNING: No files found for component '{component}' in {test_dir}, skipping.", + f"WARNING: No files found for component '{component}' in {tests_dir / component}, skipping.", file=sys.stderr, ) return filtered_components diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 3e11deeb9a..2bdf7807a9 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -66,7 +66,6 @@ from helpers import ( base_python_changed, changed_files, core_changed, - filter_component_and_test_cpp_files, filter_component_and_test_files, get_changed_components, get_component_from_path, @@ -628,12 +627,17 @@ def determine_cpp_unit_tests( C++ unit tests will run when any of the following conditions are met: - 1. Any C++ core source files changed (esphome/core/*), in which case + 1. Any core C++ or Python files changed (esphome/core/*), in which case all cpp unit tests run. 2. A test file for a component changed, which triggers tests for that component. 3. The code for a component changed, which triggers tests for that - component and all components that depend on it. + component and all components that depend on it. Python files count + too: a component's Python decides which sources and defines go into + the host test build, so a Python-only change can break the link. + + Components without C++ test sources are dropped from the list, so the + job is only scheduled when there is something to build. Args: branch: Branch to compare against. If None, uses default. @@ -647,9 +651,7 @@ def determine_cpp_unit_tests( if core_changed(files): return (True, []) - # Filter to only C++ files - cpp_files = list(filter(filter_component_and_test_cpp_files, files)) - return (False, get_cpp_changed_components(cpp_files)) + return (False, get_cpp_changed_components(files)) # Paths within tests/benchmarks/ that contain component benchmark files diff --git a/script/helpers.py b/script/helpers.py index 8132ee49e5..9e3969e5ce 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -1150,17 +1150,41 @@ def filter_component_and_test_files(file_path: str) -> bool: ) -def filter_component_and_test_cpp_files(file_path: str) -> bool: - """Check if a file is a C++ source file in component or test directories. +def filter_cpp_unit_test_files(file_path: str) -> bool: + """Check if a file can affect a component's C++ unit test build. + + Besides C++ sources, a component's Python code (defines, source file + filters, libraries) and the ``__init__.py`` manifest overrides under + ``tests/components//`` decide what the host test binary + compiles and links. Other Python files under ``tests/components/`` + (pytest conftest.py, fixtures) do not. Args: file_path: Path to check Returns: - True if the file is a C++ source/header file in component or test directories + True if the file is a C++ or Python file in a component directory, or + a C++ file or ``__init__.py`` in a component test directory """ - return file_path.endswith(CPP_FILE_EXTENSIONS) and file_path.startswith( - COMPONENT_AND_TESTS_PATHS + if file_path.startswith(ESPHOME_COMPONENTS_PATH): + return file_path.endswith(CPP_AND_PYTHON_FILE_EXTENSIONS) + if file_path.startswith(ESPHOME_TESTS_COMPONENTS_PATH): + return file_path.endswith(CPP_FILE_EXTENSIONS) or file_path.endswith( + "/__init__.py" + ) + return False + + +def has_cpp_unit_tests(component: str, tests_dir: Path) -> bool: + """Check if a component has C++ test or benchmark sources in ``tests_dir``. + + Shared by CI job selection and the build itself + (``build_helpers.filter_components_with_files``) so both agree on + which components have something to build. + """ + component_dir = tests_dir / component + return component_dir.is_dir() and ( + any(component_dir.glob("*.cpp")) or any(component_dir.glob("*.h")) ) @@ -1486,41 +1510,41 @@ def base_python_changed(files: list[str]) -> bool: def get_cpp_changed_components(files: list[str]) -> list[str]: - """Get components that have changed C++ files or tests. + """Get components whose C++ unit tests are affected by changed files. This function analyzes a list of changed files and determines which components are affected. It handles two scenarios: - 1. Test files changed (tests/components//*.cpp): + 1. Test files changed (tests/components//*.cpp or __init__.py): - Adds the component to the affected list - Only that component needs to be tested - 2. Component C++ files changed (esphome/components//*): + 2. Component files changed (esphome/components//*.cpp or *.py): - Adds the component to the affected list - Also adds all components that depend on this component (recursively) - This ensures that changes propagate to dependent components + Python files count because a component's Python code decides which + sources and defines end up in the host test build. Components without + C++ test sources are dropped so CI does not schedule the job for nothing. + Args: - files: List of file paths to analyze (should be C++ files) + files: List of changed file paths; irrelevant ones are ignored Returns: Sorted list of component names that need C++ unit tests run """ components_graph = create_components_graph() + tests_dir = Path(root_path) / ESPHOME_TESTS_COMPONENTS_PATH affected: set[str] = set() for file in files: - if not file.endswith(CPP_FILE_EXTENSIONS): + if not filter_cpp_unit_test_files(file): continue - if file.startswith(ESPHOME_TESTS_COMPONENTS_PATH): - parts = file.split("/") - if len(parts) >= 4: - component_dir = Path(ESPHOME_TESTS_COMPONENTS_PATH) / parts[2] - if component_dir.is_dir(): - affected.add(parts[2]) - elif file.startswith(ESPHOME_COMPONENTS_PATH): - parts = file.split("/") - if len(parts) >= 4: - component = parts[2] - affected.update(find_children_of_component(components_graph, component)) - affected.add(component) - return sorted(affected) + parts = file.split("/") + if len(parts) < 4: + continue + component = parts[2] + affected.add(component) + if file.startswith(ESPHOME_COMPONENTS_PATH): + affected.update(find_children_of_component(components_graph, component)) + return sorted(c for c in affected if has_cpp_unit_tests(c, tests_dir)) diff --git a/script/list-components.py b/script/list-components.py index 31a1609f88..45efccb133 100755 --- a/script/list-components.py +++ b/script/list-components.py @@ -3,7 +3,6 @@ import argparse from helpers import ( changed_files, - filter_component_and_test_cpp_files, filter_component_and_test_files, get_all_component_files, get_components_with_dependencies, @@ -38,7 +37,7 @@ def main(): parser.add_argument( "--cpp-changed", action="store_true", - help="List components with changed C++ files", + help="List components whose C++ unit tests are affected by changed files", ) args = parser.parse_args() @@ -78,9 +77,9 @@ def main(): # Returns: Components with code changes + their dependencies (not infrastructure) # Reason: CI needs to test changed components and their dependents # - # - --cpp-changed: Used by CI to determine if any C++ files changed (script/determine-jobs.py) - # Returns: Only components with changed C++ files - # Reason: Only components with C++ changes need C++ testing + # - --cpp-changed: Mirrors the C++ unit test selection in script/determine-jobs.py + # Returns: Components with changed C++ or Python files (plus dependents) + # Reason: Python decides which sources and defines go into the host test build base_test_changed = any( "tests/test_build_components" in file for file in changed @@ -115,9 +114,7 @@ def main(): for c in get_components_with_dependencies(files, False): print(c) elif args.cpp_changed: - # Only look at changed cpp files - files = list(filter(filter_component_and_test_cpp_files, changed)) - for c in get_cpp_changed_components(files): + for c in get_cpp_changed_components(changed): print(c) else: # Return all changed components (with dependencies) - default behavior diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index b42c33de96..565f8c563f 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -1214,6 +1214,56 @@ def test_count_changed_cpp_files_with_branch() -> None: mock_changed.assert_called_once_with("release") +@pytest.mark.parametrize( + ("changed_files", "expected"), + [ + # Core C++ change runs everything + (["esphome/core/helpers.cpp"], (True, [])), + # Core Python change runs everything too + (["esphome/core/config.py"], (True, [])), + # Component C++ change: component plus dependents with C++ tests + (["esphome/components/time/posix_tz.cpp"], (False, ["sntp", "time"])), + # Component Python change shapes the host build (defines, source + # filters), so it must trigger the same tests as a C++ change + (["esphome/components/time/__init__.py"], (False, ["sntp", "time"])), + # Nothing to build when no selected component has C++ tests + (["esphome/components/homeassistant/__init__.py"], (False, [])), + # Test manifest override changes only that component + (["tests/components/time/__init__.py"], (False, ["time"])), + # Test source change only that component + (["tests/components/time/posix_tz.cpp"], (False, ["time"])), + # pytest files and YAML build tests do not affect the test binary + (["tests/components/socket/conftest.py"], (False, [])), + (["tests/components/time/test.esp32-idf.yaml"], (False, [])), + (["README.md", "script/helpers.py"], (False, [])), + ([], (False, [])), + ], +) +def test_determine_cpp_unit_tests( + changed_files: list[str], + expected: tuple[bool, list[str]], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test which C++ unit tests a set of changed files selects.""" + tests_dir = tmp_path / "tests" / "components" + for component in ("time", "sntp"): + (tests_dir / component).mkdir(parents=True) + (tests_dir / component / f"{component}.cpp").write_text("") + (tests_dir / "homeassistant").mkdir() + (tests_dir / "socket").mkdir() + monkeypatch.setattr(helpers, "root_path", str(tmp_path)) + with ( + patch.object(determine_jobs, "changed_files", return_value=changed_files), + patch.object( + helpers, + "create_components_graph", + return_value={"time": ["homeassistant", "sntp"]}, + ), + ): + assert determine_jobs.determine_cpp_unit_tests() == expected + + def test_main_filters_components_without_tests( mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 2c3ae95655..38b8c57368 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -2031,3 +2031,92 @@ def test_get_changed_files_from_command_gh_failure_keeps_stderr() -> None: pytest.raises(Exception, match="maximum number of changed files"), ): _get_changed_files_from_command(["gh", "pr", "diff", "123", "--name-only"]) + + +@pytest.mark.parametrize( + ("file_path", "expected"), + [ + ("esphome/components/time/posix_tz.cpp", True), + ("esphome/components/time/posix_tz.h", True), + ("esphome/components/time/__init__.py", True), + ("esphome/components/sntp/time.py", True), + ("tests/components/time/posix_tz.cpp", True), + ("tests/components/time/__init__.py", True), + # Platform override: tests/components///__init__.py + ("tests/components/template/sensor/__init__.py", True), + # pytest-only files do not shape the C++ test binary + ("tests/components/socket/conftest.py", False), + ("tests/components/socket/test_socket.py", False), + ("tests/components/time/test.esp32-idf.yaml", False), + ("esphome/core/time.cpp", False), + ("esphome/config.py", False), + ("script/helpers.py", False), + ("README.md", False), + ], +) +def test_filter_cpp_unit_test_files(file_path: str, expected: bool) -> None: + """Test which changed files can affect a component's C++ unit test build.""" + assert helpers.filter_cpp_unit_test_files(file_path) is expected + + +@pytest.fixture +def cpp_unit_test_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Fake repo root where time, sntp and api have C++ unit tests. + + homeassistant depends on time but has no C++ tests, so it must be + dropped from the selection; socket has only pytest files. + """ + tests_dir = tmp_path / "tests" / "components" + for component in ("time", "sntp", "api"): + (tests_dir / component).mkdir(parents=True) + (tests_dir / component / f"{component}.cpp").write_text("") + (tests_dir / "homeassistant").mkdir() + (tests_dir / "homeassistant" / "__init__.py").write_text("") + (tests_dir / "socket").mkdir() + (tests_dir / "socket" / "conftest.py").write_text("") + monkeypatch.setattr(helpers, "root_path", str(tmp_path)) + monkeypatch.setattr( + helpers, + "create_components_graph", + lambda: {"time": ["homeassistant", "sntp"]}, + ) + return tmp_path + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + # Component changes expand to dependents with C++ tests + (["esphome/components/time/posix_tz.cpp"], ["sntp", "time"]), + (["esphome/components/time/__init__.py"], ["sntp", "time"]), + # Dependent without C++ tests is dropped + (["esphome/components/homeassistant/__init__.py"], []), + # Test changes select only that component + (["tests/components/time/posix_tz.cpp"], ["time"]), + (["tests/components/time/__init__.py"], ["time"]), + (["tests/components/homeassistant/__init__.py"], []), + (["tests/components/socket/conftest.py"], []), + (["tests/components/time/test.esp32-idf.yaml"], []), + ( + ["esphome/components/time/__init__.py", "tests/components/api/api.cpp"], + ["api", "sntp", "time"], + ), + ([], []), + ], +) +@pytest.mark.usefixtures("cpp_unit_test_tree") +def test_get_cpp_changed_components(files: list[str], expected: list[str]) -> None: + """Test that C++ and Python component changes select the right unit tests.""" + assert helpers.get_cpp_changed_components(files) == expected + + +def test_get_cpp_changed_components_independent_of_cwd( + cpp_unit_test_tree: Path, + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test directories resolve against root_path, not the current directory.""" + monkeypatch.chdir(tmp_path_factory.mktemp("elsewhere")) + assert helpers.get_cpp_changed_components( + ["tests/components/time/__init__.py"] + ) == ["time"] From c2a153a9469acf53dc74b8a303ae0ccd8065b9e4 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Mon, 24 Aug 2026 11:26:26 -0700 Subject: [PATCH 05/65] [modbus_controller] Add continuous polling option (#18080) --- esphome/components/modbus/__init__.py | 33 +++++ esphome/components/modbus_client/__init__.py | 8 +- .../components/modbus_controller/__init__.py | 44 ++++++- .../binary_sensor/__init__.py | 4 +- .../modbus_controller/modbus_controller.cpp | 20 +-- .../modbus_controller/modbus_controller.h | 10 +- .../modbus_controller/number/__init__.py | 4 +- .../modbus_controller/sensor/__init__.py | 4 +- .../modbus_controller/switch/__init__.py | 4 +- .../modbus_controller/text_sensor/__init__.py | 4 +- .../modbus_controller/test_custom_pdu.py | 63 +++++++++- .../components/modbus_controller/common.yaml | 1 + .../fixtures/uart_mock_modbus_continuous.yaml | 115 ++++++++++++++++++ tests/integration/test_uart_mock_modbus.py | 62 ++++++++++ 14 files changed, 345 insertions(+), 31 deletions(-) create mode 100644 tests/integration/fixtures/uart_mock_modbus_continuous.yaml diff --git a/esphome/components/modbus/__init__.py b/esphome/components/modbus/__init__.py index c9ba00f111..769858e72a 100644 --- a/esphome/components/modbus/__init__.py +++ b/esphome/components/modbus/__init__.py @@ -45,6 +45,7 @@ ModbusClient = modbus_ns.class_("ModbusClientHub", Modbus) ModbusDevice = modbus_ns.class_("ModbusDevice") ModbusClientDevice = modbus_ns.class_("ModbusClientDevice") ModbusServerDevice = modbus_ns.class_("ModbusServerDevice") +CommandOptions = modbus_ns.struct("CommandOptions") MULTI_CONF = True CONF_ROLE = "role" @@ -81,6 +82,19 @@ def _command_options(direction: str) -> list[_CommandOption]: raise ValueError(f"unknown command-options direction {direction!r}") from None +# The write (mutating) function codes, matching modbus::helpers::is_function_code_write(). 0x17 +# (read/write multiple) is included: it mutates, so the hub treats it as a write despite its read half. +_WRITE_FUNCTION_CODES = frozenset({0x05, 0x06, 0x0F, 0x10, 0x16, 0x17}) + + +def is_function_code_write(function_code: int) -> bool: + """True if the Modbus function code writes (mutates). The exception bit (0x80) is masked off first, + so an exception-flagged code still classifies by its base code - stricter than the runtime hub, + whose classify() treats an exception-flagged code as a read. Keep in sync with + modbus::helpers::is_function_code_write().""" + return function_code & 0x7F in _WRITE_FUNCTION_CODES + + def command_options_schema( *, direction: Literal["read", "write"], templatable: bool = False ) -> dict[cv.Optional, Any]: @@ -98,6 +112,25 @@ def command_options_schema( } +def command_options_expression( + config: ConfigType, *, direction: Literal["read", "write"] +) -> cg.StructInitializer: + """Build the modbus::CommandOptions initializer for a config validated with + command_options_schema() of the same direction. For static (non-templatable) options only; + actions with lambda values use register_templatable_command_options() instead. + """ + return cg.StructInitializer( + CommandOptions, + *( + # Construct the value as its declared cpp_type, so a future non-bool option (enum, + # uint16_t, ...) is emitted with the right type instead of whatever safe_exp() infers. + (option.field, option.cpp_type(config[option.conf_key])) + for option in _command_options(direction) + if option.conf_key in config + ), + ) + + async def register_templatable_command_options( var: MockObj, config: ConfigType, args: TemplateArgsType, direction: str ) -> None: diff --git a/esphome/components/modbus_client/__init__.py b/esphome/components/modbus_client/__init__.py index 48d7c1df4f..a59eb91066 100644 --- a/esphome/components/modbus_client/__init__.py +++ b/esphome/components/modbus_client/__init__.py @@ -157,10 +157,6 @@ _ACTION_BASE_SCHEMA = cv.Schema( } ) -# The write codes recognised by modbus::helpers::is_function_code_write() - keep in sync. 0x17 -# (read/write multiple) is included: it mutates, so the hub treats it as a write despite its read half. -_WRITE_FUNCTION_CODES = frozenset({0x05, 0x06, 0x0F, 0x10, 0x16, 0x17}) - def _no_continuous_on_write(config: ConfigType) -> ConfigType: """Reject `continuous: true` on a static write PDU: continuous polling only applies to reads. @@ -170,9 +166,7 @@ def _no_continuous_on_write(config: ConfigType) -> ConfigType: if ( isinstance(pdu, list) and config.get(CONF_CONTINUOUS) is True - # Masking the exception bit (0x90 -> 0x10) makes this check stricter than the runtime hub, - # whose classify() treats an exception-flagged code as a read and leaves continuous in place. - and pdu[0] & 0x7F in _WRITE_FUNCTION_CODES + and modbus.is_function_code_write(pdu[0]) ): raise cv.Invalid( f"'{CONF_CONTINUOUS}: true' does not apply to a write PDU (function code " diff --git a/esphome/components/modbus_controller/__init__.py b/esphome/components/modbus_controller/__init__.py index 188b552a3c..924a260d37 100644 --- a/esphome/components/modbus_controller/__init__.py +++ b/esphome/components/modbus_controller/__init__.py @@ -11,7 +11,14 @@ from esphome.components.modbus.helpers import ( EntityType, ) import esphome.config_validation as cv -from esphome.const import CONF_ADDRESS, CONF_ID, CONF_LAMBDA, CONF_NAME, CONF_OFFSET +from esphome.const import ( + CONF_ADDRESS, + CONF_CONTINUOUS, + CONF_ID, + CONF_LAMBDA, + CONF_NAME, + CONF_OFFSET, +) from esphome.core import CORE from esphome.cpp_helpers import logging import esphome.final_validate as fv @@ -125,6 +132,7 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_MAX_CMD_RETRIES, default=4): cv.positive_int, cv.Optional(CONF_OFFLINE_SKIP_UPDATES, default=0): cv.positive_int, + **modbus.command_options_schema(direction="read"), cv.Optional( CONF_SERVER_REGISTERS, ): cv.invalid( @@ -234,6 +242,35 @@ def migrate_custom_command(config: ConfigType) -> None: del config[CONF_CUSTOM_COMMAND] +def _reject_continuous_write_custom_pdu(config: ConfigType) -> None: + """Final-validate: a custom_pdu whose function code writes (e.g. 0x17 read/write-multiple) cannot be + polled continuously - the hub ignores continuous for mutating codes and would warn on every update + while that range silently does not stream. Reject the combination instead. Runs after + migrate_custom_command, so it sees custom_pdu whether written directly or migrated from + custom_command.""" + pdu = config.get(CONF_CUSTOM_PDU) + if pdu is None or not modbus.is_function_code_write(pdu[0]): + return + fconf = fv.full_config.get() + path = fconf.get_path_for_id(config[CONF_MODBUS_CONTROLLER_ID])[:-1] + controller = fconf.get_config_for_path(path) + if controller.get(CONF_CONTINUOUS) is True: + raise cv.Invalid( + f"a '{CONF_CUSTOM_PDU}' with a write function code (0x{pdu[0] & 0x7F:02X}) can't be polled " + f"continuously: the hub ignores 'continuous' for mutating codes. Remove 'continuous: true' " + f"from the '{controller[CONF_ID]}' modbus_controller, or use a read function code.", + [CONF_CUSTOM_PDU], + ) + + +def validate_custom_pdu_item(config: ConfigType) -> None: + """Final-validate for the read platforms that accept custom_pdu (sensor, binary_sensor, + text_sensor): migrate the deprecated custom_command, then reject a write-coded custom_pdu under a + continuously-polling controller.""" + migrate_custom_command(config) + _reject_continuous_write_custom_pdu(config) + + def _final_validate(config: ConfigType) -> None: modbus.final_validate_modbus_device("modbus_controller", role="client")(config) @@ -314,6 +351,11 @@ async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) cg.add(var.set_max_cmd_retries(config[CONF_MAX_CMD_RETRIES])) cg.add(var.set_offline_skip_updates(config[CONF_OFFLINE_SKIP_UPDATES])) + cg.add( + var.set_read_options( + modbus.command_options_expression(config, direction="read") + ) + ) await register_modbus_device(var, config) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) diff --git a/esphome/components/modbus_controller/binary_sensor/__init__.py b/esphome/components/modbus_controller/binary_sensor/__init__.py index 6ff1975b1e..366dab6062 100644 --- a/esphome/components/modbus_controller/binary_sensor/__init__.py +++ b/esphome/components/modbus_controller/binary_sensor/__init__.py @@ -8,9 +8,9 @@ from .. import ( ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, - migrate_custom_command, modbus_calc_properties, modbus_controller_ns, + validate_custom_pdu_item, validate_modbus_register, ) from ..const import ( @@ -40,7 +40,7 @@ CONFIG_SCHEMA = cv.All( validate_modbus_register, ) -FINAL_VALIDATE_SCHEMA = migrate_custom_command +FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): diff --git a/esphome/components/modbus_controller/modbus_controller.cpp b/esphome/components/modbus_controller/modbus_controller.cpp index 21fe4ef45f..20b8f516e9 100644 --- a/esphome/components/modbus_controller/modbus_controller.cpp +++ b/esphome/components/modbus_controller/modbus_controller.cpp @@ -167,6 +167,7 @@ void ModbusController::queue_command(ModbusCommandItem command) { this->one_shot_command_items_.push_back(make_unique(std::move(command))); // A refused frame gets no terminal callback (see the hub contract), so reclaim the item here. auto &item = this->one_shot_command_items_.back(); + // We intentionally do not pass read_options_ here, because one-shot commands are usually writes, and are non-polling. if (!item->send()) { // The caller (e.g. a write entity) has usually already published optimistically - surface the loss. ESP_LOGW(TAG, "Command refused by hub: type=0x%X address=0x%X", static_cast(item->register_type()), @@ -203,7 +204,9 @@ void ModbusController::update() { ESP_LOGV(TAG, "Module offline - retrying"); this->cmd_non_responses_ = 0; // allow the probe through can_send() for (auto &cmd : this->polling_command_items_) { - if (!cmd.send()) { + // Probes carry the read-side options too, so a recovering device resumes streaming on the + // probe itself rather than waiting for the next update_interval. + if (!cmd.send(this->read_options_)) { ESP_LOGD(TAG, "Probe refused by hub for range 0x%X", cmd.register_address()); } } @@ -217,8 +220,9 @@ void ModbusController::update() { if (this->can_send()) { for (auto &cmd : this->polling_command_items_) { ESP_LOGVV(TAG, "Updating range 0x%X", cmd.register_address()); + // read_options_ carries the controller's continuous flag (the offline probe above sends it too). // A refusal is already logged by the hub; note the affected range for controller-level diagnostics. - if (!cmd.send()) { + if (!cmd.send(this->read_options_)) { ESP_LOGD(TAG, "Poll refused by hub for range 0x%X", cmd.register_address()); } } @@ -496,16 +500,18 @@ ModbusCommandItem ModbusCommandItem::create_custom_command( return cmd; } -bool ModbusCommandItem::send() { +bool ModbusCommandItem::send(modbus::CommandOptions options) { + // Options pass straight through to the hub bool accepted; if (this->custom_pdu_ != nullptr) { // Custom polling command: send the sensor's ready-made PDU (function code + data, no address byte) // to this controller's own device address; the hub prepends the address and appends the CRC. - accepted = modbus::ModbusClientDevice::queue_pdu(std::span(*this->custom_pdu_)); + accepted = modbus::ModbusClientDevice::queue_pdu(std::span(*this->custom_pdu_), options); } else if (this->function_code_ != FunctionCode::CUSTOM) { accepted = this->queue_pdu(modbus::helpers::create_client_pdu( - this->function_code_, this->start_address_, this->register_count_, - this->payload.empty() ? nullptr : this->payload.data(), this->payload.size())); + this->function_code_, this->start_address_, this->register_count_, + this->payload.empty() ? nullptr : this->payload.data(), this->payload.size()), + options); } else { // Factory custom command: payload holds a complete raw frame (address + PDU). Send the PDU to the // frame's own address (which may differ from this controller's); the hub appends the CRC and routes @@ -515,7 +521,7 @@ bool ModbusCommandItem::send() { ESP_LOGW(TAG, "Empty custom command frame, not sent"); accepted = false; } else { - accepted = this->parent_->queue_pdu(frame[0], frame.subspan(1), this); + accepted = this->parent_->queue_pdu(frame[0], frame.subspan(1), this, options); } } // The on_command_sent trigger fires from on_sent() when the frame actually reaches the wire. diff --git a/esphome/components/modbus_controller/modbus_controller.h b/esphome/components/modbus_controller/modbus_controller.h index f36705cda4..1db07f1ee8 100644 --- a/esphome/components/modbus_controller/modbus_controller.h +++ b/esphome/components/modbus_controller/modbus_controller.h @@ -284,7 +284,9 @@ class ModbusCommandItem : public modbus::ModbusClientDevice { /// Queue this command's frame on the hub. Returns false when refused, in which case no callback ever comes. /// The item is the hub device, so it must stay alive until its terminal callback; a destroyed item's /// pending frame is silently retired. - bool send(); + /// Options pass straight through to the hub; the polling path passes the controller's read-side + /// options so reads re-queue after each success, one-shot commands keep the default. + bool send(modbus::CommandOptions options = {}); /// factory methods /** Create modbus read command @@ -452,6 +454,10 @@ class ModbusController final : public PollingComponent { void set_max_cmd_retries(uint8_t max_cmd_retries) { this->max_cmd_retries_ = max_cmd_retries; } /// get how many times a command will be (re)sent if no response is received uint8_t get_max_cmd_retries() { return this->max_cmd_retries_; } + /// called by esphome generated code with the read-side command options applied to every poll + void set_read_options(modbus::CommandOptions options) { this->read_options_ = options; } + /// the read-side command options applied to every poll + const modbus::CommandOptions &read_options() const { return this->read_options_; } protected: /// parse sensormap_ and create range of sequential addresses @@ -497,6 +503,8 @@ class ModbusController final : public PollingComponent { uint16_t offline_skip_updates_{0}; /// How many times we will retry a command if we get no response uint8_t max_cmd_retries_{4}; + /// read-side command options applied to every poll + modbus::CommandOptions read_options_{}; /// Command sent callback CallbackManager command_sent_callback_{}; /// Server online callback diff --git a/esphome/components/modbus_controller/number/__init__.py b/esphome/components/modbus_controller/number/__init__.py index 39d04e8d91..a43e10a51e 100644 --- a/esphome/components/modbus_controller/number/__init__.py +++ b/esphome/components/modbus_controller/number/__init__.py @@ -18,9 +18,9 @@ from .. import ( ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, - migrate_custom_command, modbus_calc_properties, modbus_controller_ns, + validate_custom_pdu_item, ) from ..const import ( CONF_BITMASK, @@ -86,7 +86,7 @@ CONFIG_SCHEMA = cv.All( validate_modbus_number, ) -FINAL_VALIDATE_SCHEMA = migrate_custom_command +FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): diff --git a/esphome/components/modbus_controller/sensor/__init__.py b/esphome/components/modbus_controller/sensor/__init__.py index c3c9bd4718..2c34ef04b4 100644 --- a/esphome/components/modbus_controller/sensor/__init__.py +++ b/esphome/components/modbus_controller/sensor/__init__.py @@ -8,9 +8,9 @@ from .. import ( ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, - migrate_custom_command, modbus_calc_properties, modbus_controller_ns, + validate_custom_pdu_item, validate_modbus_register, ) from ..const import ( @@ -44,7 +44,7 @@ CONFIG_SCHEMA = cv.All( validate_modbus_register, ) -FINAL_VALIDATE_SCHEMA = migrate_custom_command +FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): diff --git a/esphome/components/modbus_controller/switch/__init__.py b/esphome/components/modbus_controller/switch/__init__.py index 35ad12087c..dedd2ceedf 100644 --- a/esphome/components/modbus_controller/switch/__init__.py +++ b/esphome/components/modbus_controller/switch/__init__.py @@ -8,9 +8,9 @@ from .. import ( ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, - migrate_custom_command, modbus_calc_properties, modbus_controller_ns, + validate_custom_pdu_item, validate_modbus_register, ) from ..const import ( @@ -45,7 +45,7 @@ CONFIG_SCHEMA = cv.All( validate_modbus_register, ) -FINAL_VALIDATE_SCHEMA = migrate_custom_command +FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): diff --git a/esphome/components/modbus_controller/text_sensor/__init__.py b/esphome/components/modbus_controller/text_sensor/__init__.py index e8447658e2..31f5f87a98 100644 --- a/esphome/components/modbus_controller/text_sensor/__init__.py +++ b/esphome/components/modbus_controller/text_sensor/__init__.py @@ -8,9 +8,9 @@ from .. import ( ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, - migrate_custom_command, modbus_calc_properties, modbus_controller_ns, + validate_custom_pdu_item, validate_modbus_register, ) from ..const import ( @@ -55,7 +55,7 @@ CONFIG_SCHEMA = cv.All( validate_modbus_register, ) -FINAL_VALIDATE_SCHEMA = migrate_custom_command +FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): diff --git a/tests/component_tests/modbus_controller/test_custom_pdu.py b/tests/component_tests/modbus_controller/test_custom_pdu.py index a5d065c965..a3a18da07f 100644 --- a/tests/component_tests/modbus_controller/test_custom_pdu.py +++ b/tests/component_tests/modbus_controller/test_custom_pdu.py @@ -1,19 +1,27 @@ -"""Schema-level config validation for custom_pdu and the deprecated custom_command alias. +"""Config validation for custom_pdu and the deprecated custom_command alias. custom_command took a raw frame with a leading device address byte; custom_pdu takes the PDU only. -The old key is still accepted at the schema level and auto-migrated later in final validate (which a -bare-schema test can't reach), so these tests only cover what the schema itself enforces: the two keys -are mutually exclusive, and custom_pdu takes byte-sized values. +Most of these tests cover what the schema itself enforces (the two keys are mutually exclusive, and +custom_pdu takes byte-sized values). The last two reach the final-validate step that a bare-schema +test cannot: a write-coded custom_pdu polled continuously is rejected there. """ import pytest from voluptuous import Invalid, MultipleInvalid -from esphome.components.modbus_controller import ModbusItemBaseSchema +from esphome.components.modbus_controller import ( + ModbusItemBaseSchema, + validate_custom_pdu_item, +) from esphome.components.modbus_controller.const import ( CONF_CUSTOM_COMMAND, CONF_CUSTOM_PDU, + CONF_MODBUS_CONTROLLER_ID, ) +from esphome.config import Config +from esphome.const import CONF_ADDRESS, CONF_CONTINUOUS, CONF_ID +from esphome.core import ID +import esphome.final_validate as fv def test_custom_command_accepted_at_schema_level() -> None: @@ -45,3 +53,48 @@ def test_custom_pdu_rejects_non_byte_values() -> None: """PDU entries are bytes; a word-sized value is a sign the old raw format is being used.""" with pytest.raises((Invalid, MultipleInvalid)): ModbusItemBaseSchema({CONF_CUSTOM_PDU: [0x0103, 0x002A]}) + + +def _controller_full_config(*, continuous: bool) -> Config: + """A minimal full-config graph with one modbus_controller declaring id 'ctl', enough for the + final-validate to resolve the controller (and its continuous flag) from an item's + modbus_controller_id.""" + ctl_id = ID("ctl", is_declaration=True) + config = Config() + config["modbus_controller"] = [ + {CONF_ID: ctl_id, CONF_ADDRESS: 1, CONF_CONTINUOUS: continuous} + ] + config.declare_ids.append((ctl_id, ["modbus_controller", 0, CONF_ID])) + return config + + +@pytest.fixture +def reset_full_config(): + token = fv.full_config.set(Config()) + yield + fv.full_config.reset(token) + + +def test_continuous_write_custom_pdu_rejected(reset_full_config) -> None: + """A write-coded custom_pdu (0x17 = read/write-multiple) under a continuous controller is + rejected at final validate: the hub would strip continuous from the mutating code and warn on + every update.""" + fv.full_config.set(_controller_full_config(continuous=True)) + with pytest.raises(Invalid, match="can't be polled continuously"): + validate_custom_pdu_item( + { + CONF_MODBUS_CONTROLLER_ID: ID("ctl"), + CONF_CUSTOM_PDU: [0x17, 0x00, 0x03, 0x00, 0x01], + } + ) + + +def test_continuous_read_custom_pdu_allowed(reset_full_config) -> None: + """A read-coded custom_pdu (0x03) under a continuous controller is fine - only writes stream.""" + fv.full_config.set(_controller_full_config(continuous=True)) + validate_custom_pdu_item( + { + CONF_MODBUS_CONTROLLER_ID: ID("ctl"), + CONF_CUSTOM_PDU: [0x03, 0x00, 0x2A, 0x00, 0x01], + } + ) diff --git a/tests/components/modbus_controller/common.yaml b/tests/components/modbus_controller/common.yaml index 9c35a2f868..78bec522cf 100644 --- a/tests/components/modbus_controller/common.yaml +++ b/tests/components/modbus_controller/common.yaml @@ -2,6 +2,7 @@ modbus_controller: - id: modbus_controller1 address: 0x2 modbus_id: modbus_bus + continuous: true on_online: then: logger.log: "Module Online" diff --git a/tests/integration/fixtures/uart_mock_modbus_continuous.yaml b/tests/integration/fixtures/uart_mock_modbus_continuous.yaml new file mode 100644 index 0000000000..62b0b4c2cf --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_continuous.yaml @@ -0,0 +1,115 @@ +esphome: + name: uart-mock-modbus-continuous + +host: +api: +logger: + level: VERBOSE + +# When set, the mock server stops forwarding its replies to the controller, so the controller sees +# timeouts - used by the recovery test to drive a live continuous poll offline and back. +globals: + - id: silence_server + type: bool + initial_value: "false" + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - if: + condition: + lambda: "return !id(silence_server);" + then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + # Short timeout so the recovery test drives the poll offline quickly; when the server answers, + # replies arrive within turnaround_time, so this does not slow the streaming path. + send_wait_time: 100ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + # A long update_interval means that without continuous polling only the boot poll would run in the + # test window. continuous: true re-queues the read after each success, so it streams as fast as the + # bus allows. + update_interval: 30s + continuous: true + # One retry so a silenced device trips offline fast (initial send + 1 retry, each 100ms). + max_cmd_retries: 1 + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + # Each read returns the next counter value, so every poll publishes a distinct state the test can + # count (proving the read actually ran, not just that the state changed once). + - address: 0x01 + value_type: U_WORD + read_lambda: |- + static uint16_t counter = 0; + return counter++; + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "continuous_reg" + address: 0x01 + register_type: holding + value_type: U_WORD + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # Trigger the first poll deterministically. PollingComponent's first update() would otherwise land + # somewhere in the 30s update_interval; once this one read completes, continuous re-queuing takes over. + on_press: + - lambda: "id(modbus_controller_1)->update();" + +switch: + # Toggles whether the mock server forwards its replies. On = silence (controller sees timeouts); + # off = answer again. The recovery test uses it to drive a live continuous poll offline and back. + - platform: template + name: "Silence Server" + id: silence_server_switch + optimistic: true + turn_on_action: + - lambda: "id(silence_server) = true;" + turn_off_action: + - lambda: "id(silence_server) = false;" diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index 09e841b4bb..c84fb34e70 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -736,6 +736,68 @@ async def test_uart_mock_modbus_custom_pdu( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.asyncio +async def test_uart_mock_modbus_continuous( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that `continuous: true` polls faster than the update_interval. + + The controller's update_interval is 30s, so without continuous polling only the boot poll would + run during the short test window. With continuous the read is re-queued after each success, filling + idle bus time, so many reads arrive. The server returns an incrementing counter, so every read is a + distinct published state the tracker can count. (Bus warnings are not asserted here: continuous + polling deliberately saturates the bus, so the occasional timing hiccup is expected and off-topic; + the other tests cover clean operation at normal poll rates.) + """ + + tracker = SensorTracker(["continuous_reg"]) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + # setup_and_start_scenario presses the Start Scenario button, whose on_press triggers the + # controller's first update(). With continuous that one read re-queues and streams; without it + # the next poll would not run until the 30s update_interval elapses. + entities = await tracker.setup_and_start_scenario(client) + # Count reads over a window far shorter than the update_interval. Absent continuous polling we + # would see ~1 (the triggered poll); continuous re-queues, so the bus fills with reads. + await asyncio.sleep(3.0) + reads = len(tracker.sensor_states["continuous_reg"]) + assert reads >= 5, ( + "expected many continuous reads within the window (update_interval is 30s, so absent " + f"continuous polling we would see ~1), got {reads}" + ) + + # Recovery path: a live continuous poll that starts failing goes offline, and the next update() + # re-arms it once the device answers again. Silence the server so the poll's reads time out; with + # max_cmd_retries=1 and send_wait_time=100ms the device trips offline quickly and streaming stops. + silence = find_entity(entities, "silence_server", SwitchInfo) + assert silence is not None, "Silence Server switch not found" + start = find_entity(entities, "start_scenario", ButtonInfo) + assert start is not None, "Start Scenario button not found" + + client.switch_command(silence.key, True) + await asyncio.sleep(1.0) # let the poll fail and the device trip offline + plateau = len(tracker.sensor_states["continuous_reg"]) + await asyncio.sleep(1.0) # offline: no polls should land + assert len(tracker.sensor_states["continuous_reg"]) == plateau, ( + "reads kept arriving after the server was silenced - the failed continuous poll did not stop" + ) + + # Answer again and trigger update(): the offline probe recovers the device and the continuous + # poll re-arms, so streaming resumes. + client.switch_command(silence.key, False) + client.button_command(start.key) + await asyncio.sleep(3.0) + resumed = len(tracker.sensor_states["continuous_reg"]) - plateau + assert resumed >= 5, ( + f"continuous polling did not resume after the device recovered (got {resumed} new reads)" + ) + + @pytest.mark.asyncio async def test_uart_mock_modbus_offline( yaml_config: str, From b103df1bb1e57175fcd632521f645096538d69fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:33:24 -0500 Subject: [PATCH 06/65] Bump ruff from 0.16.3 to 0.16.4 (#18734) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 079c375c01..cf4b028b0e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.16.3 # also change in .pre-commit-config.yaml when updating +ruff==0.16.4 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating prek==0.4.14 # also change in .github/workflows/ci.yml when updating From 32a0a5c55cb8a9c5b392272d2d04b3dbf40401f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:33:39 -0500 Subject: [PATCH 07/65] Bump github/codeql-action/analyze from 4.37.7 to 4.37.8 (#18735) Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 103cecc1f9..46eed02656 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 with: category: "/language:${{matrix.language}}" From ad508d79afdb578503ae7768b8d6bd14fd0474c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:33:52 -0500 Subject: [PATCH 08/65] Bump github/codeql-action/init from 4.37.7 to 4.37.8 (#18736) Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 46eed02656..b46f9adab6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} From 3b6c6e3dadf6edb8902feb72de945d30ea8e47a6 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:50:35 -0500 Subject: [PATCH 09/65] Bump aioesphomeapi from 46.0.0 to 46.1.0 (#18737) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 822eebc1f2..c10ddc3f53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==46.0.0 +aioesphomeapi==46.1.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From ca9636514362b5f270dc98c896d04c9da98edb00 Mon Sep 17 00:00:00 2001 From: guillempages Date: Mon, 24 Aug 2026 22:07:01 +0200 Subject: [PATCH 10/65] [runtime_image] Add MIME types to image formats (#16361) Co-authored-by: J. Nick Koston Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston --- .../components/online_image/online_image.cpp | 62 +++++-------------- .../components/runtime_image/image_format.cpp | 44 +++++++++++++ .../components/runtime_image/image_format.h | 7 +++ .../runtime_image/runtime_image.cpp | 1 - .../runtime_image/test_mime_types.cpp | 61 ++++++++++++++++++ 5 files changed, 127 insertions(+), 48 deletions(-) create mode 100644 esphome/components/runtime_image/image_format.cpp create mode 100644 tests/components/runtime_image/test_mime_types.cpp diff --git a/esphome/components/online_image/online_image.cpp b/esphome/components/online_image/online_image.cpp index a2662ff0e3..3f2382accd 100644 --- a/esphome/components/online_image/online_image.cpp +++ b/esphome/components/online_image/online_image.cpp @@ -3,6 +3,7 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include +#include static const char *const TAG = "online_image"; static const char *const CONTENT_TYPE_HEADER_NAME = "content-type"; @@ -62,30 +63,11 @@ void OnlineImage::update() { headers.push_back({IF_MODIFIED_SINCE_HEADER_NAME, this->last_modified_}); } - // Add Accept header based on image format - const char *accept_mime_type; runtime_image::ImageFormat format = this->get_format(); - switch (format) { -#ifdef USE_RUNTIME_IMAGE_BMP - case runtime_image::BMP: - accept_mime_type = "image/bmp,*/*;q=0.8"; - break; -#endif -#ifdef USE_RUNTIME_IMAGE_JPEG - case runtime_image::JPEG: - accept_mime_type = "image/jpeg,*/*;q=0.8"; - break; -#endif -#ifdef USE_RUNTIME_IMAGE_PNG - case runtime_image::PNG: - accept_mime_type = "image/png,*/*;q=0.8"; - break; -#endif - default: - accept_mime_type = "image/*,*/*;q=0.8"; - break; - } - headers.push_back({"Accept", accept_mime_type}); + // Accept: ",*/*;q=0.8"; 32 covers the longest MIME type plus the suffix + char accept_header[32]; + snprintf(accept_header, sizeof(accept_header), "%s,*/*;q=0.8", runtime_image::get_mime_type_for_format(format)); + headers.push_back({"Accept", accept_header}); // User headers last so they can override any of the above for (auto &header : this->request_headers_) { @@ -122,32 +104,18 @@ void OnlineImage::update() { if (format == runtime_image::AUTO) { // Try to auto-detect format from Content-Type header - auto content_type_header = this->downloader_->get_response_header(CONTENT_TYPE_HEADER_NAME); - const char *content_type = content_type_header.c_str(); - ESP_LOGV(TAG, "Content-Type: %s", content_type); - // Includes aliases seen from real servers (older IIS, CDNs, S3) - if (str_contains_ignore_case(content_type, "image/bmp") || - str_contains_ignore_case(content_type, "image/x-ms-bmp") || - str_contains_ignore_case(content_type, "image/x-bmp")) { - format = runtime_image::BMP; - } else if (str_contains_ignore_case(content_type, "image/jpeg") || - str_contains_ignore_case(content_type, "image/jpg")) { - format = runtime_image::JPEG; - } else if (str_contains_ignore_case(content_type, "image/png") || - str_contains_ignore_case(content_type, "image/x-png")) { - format = runtime_image::PNG; - } else if (str_contains_ignore_case(content_type, "image/")) { - ESP_LOGW(TAG, "Unsupported image type: '%s'", content_type); - this->end_connection_(); - this->download_error_callback_.call(); - return; + auto content_type = this->downloader_->get_response_header(CONTENT_TYPE_HEADER_NAME); + ESP_LOGV(TAG, "Content-Type: %s", content_type.c_str()); + auto mime_format = esphome::runtime_image::get_format_for_mime_type(content_type.c_str()); + if (mime_format.has_value()) { + format = *mime_format; } else { - // TODO: implement auto-detection in runtime_image by sniffing the first few bytes of the image data - if (content_type_header.empty()) { - ESP_LOGW(TAG, "Server sent no Content-Type header; cannot determine image format. Set `format:` explicitly"); + if (content_type.empty()) { + ESP_LOGE(TAG, "Server sent no Content-Type header; cannot determine image format. Set `format:` explicitly"); + } else if (str_contains_ignore_case(content_type.c_str(), "image/")) { + ESP_LOGE(TAG, "Image format '%s' not supported.", content_type.c_str()); } else { - ESP_LOGE(TAG, "Could not determine image format from Content-Type: '%s'. Set `format:` explicitly", - content_type); + ESP_LOGE(TAG, "Server did not return an image (Content-Type: '%s')", content_type.c_str()); } this->end_connection_(); this->download_error_callback_.call(); diff --git a/esphome/components/runtime_image/image_format.cpp b/esphome/components/runtime_image/image_format.cpp new file mode 100644 index 0000000000..3ba8871862 --- /dev/null +++ b/esphome/components/runtime_image/image_format.cpp @@ -0,0 +1,44 @@ +#include "esphome/core/helpers.h" +#include "image_format.h" +#include "image_decoder.h" + +namespace esphome::runtime_image { + +struct MimeLookup { + const char *mime_type; + ImageFormat format; +}; + +// The first entry per format is its canonical MIME type; the rest are aliases +// seen from real servers (older IIS, CDNs, S3) +static constexpr MimeLookup MIME_LOOKUP_TABLE[] = { +#ifdef USE_RUNTIME_IMAGE_BMP + {"image/bmp", ImageFormat::BMP}, {"image/x-ms-bmp", ImageFormat::BMP}, {"image/x-bmp", ImageFormat::BMP}, +#endif +#ifdef USE_RUNTIME_IMAGE_JPEG + {"image/jpeg", ImageFormat::JPEG}, {"image/jpg", ImageFormat::JPEG}, +#endif +#ifdef USE_RUNTIME_IMAGE_PNG + {"image/png", ImageFormat::PNG}, {"image/x-png", ImageFormat::PNG}, +#endif +}; + +const char *get_mime_type_for_format(ImageFormat format) { + for (const auto &entry : MIME_LOOKUP_TABLE) { + if (entry.format == format) { + return entry.mime_type; + } + } + return "image/*"; // AUTO or compiled-out format +} + +std::optional get_format_for_mime_type(const char *mime_type) { + for (const auto &entry : MIME_LOOKUP_TABLE) { + if (str_contains_ignore_case(mime_type, entry.mime_type)) { + return entry.format; + } + } + return std::nullopt; +} + +} // namespace esphome::runtime_image diff --git a/esphome/components/runtime_image/image_format.h b/esphome/components/runtime_image/image_format.h index ca6e0782b9..aff0c026b9 100644 --- a/esphome/components/runtime_image/image_format.h +++ b/esphome/components/runtime_image/image_format.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace esphome::runtime_image { /** @@ -17,4 +19,9 @@ enum ImageFormat { BMP, }; +/// Canonical MIME type for a format; "image/*" for AUTO/unknown +const char *get_mime_type_for_format(ImageFormat format); +/// Case-insensitive substring match of known media types; nullopt if none found +std::optional get_format_for_mime_type(const char *mime_type); + } // namespace esphome::runtime_image diff --git a/esphome/components/runtime_image/runtime_image.cpp b/esphome/components/runtime_image/runtime_image.cpp index 254624caf4..f7417c2c8e 100644 --- a/esphome/components/runtime_image/runtime_image.cpp +++ b/esphome/components/runtime_image/runtime_image.cpp @@ -1,7 +1,6 @@ #include "runtime_image.h" #include "image_decoder.h" #include "esphome/core/log.h" -#include "esphome/core/helpers.h" #include #include #include diff --git a/tests/components/runtime_image/test_mime_types.cpp b/tests/components/runtime_image/test_mime_types.cpp new file mode 100644 index 0000000000..de8bbc76be --- /dev/null +++ b/tests/components/runtime_image/test_mime_types.cpp @@ -0,0 +1,61 @@ +#include + +#include + +#include "esphome/components/runtime_image/runtime_image.h" + +namespace esphome::runtime_image::testing { + +TEST(RuntimeImageMime, FormatForKnownMimeTypes) { + EXPECT_EQ(get_format_for_mime_type("image/bmp"), BMP); + EXPECT_EQ(get_format_for_mime_type("image/x-ms-bmp"), BMP); + EXPECT_EQ(get_format_for_mime_type("image/x-bmp"), BMP); + EXPECT_EQ(get_format_for_mime_type("image/png"), PNG); + EXPECT_EQ(get_format_for_mime_type("image/x-png"), PNG); +#ifdef USE_RUNTIME_IMAGE_JPEG + EXPECT_EQ(get_format_for_mime_type("image/jpeg"), JPEG); + EXPECT_EQ(get_format_for_mime_type("image/jpg"), JPEG); +#endif // USE_RUNTIME_IMAGE_JPEG +} + +TEST(RuntimeImageMime, FormatMatchingIsCaseInsensitive) { + EXPECT_EQ(get_format_for_mime_type("Image/PNG"), PNG); + EXPECT_EQ(get_format_for_mime_type("IMAGE/BMP"), BMP); +} + +TEST(RuntimeImageMime, FormatMatchesContentTypeWithParameters) { + // Content-Type headers may carry parameters after the media type + EXPECT_EQ(get_format_for_mime_type("image/png; charset=binary"), PNG); + EXPECT_EQ(get_format_for_mime_type("image/bmp;name=\"a.bmp\""), BMP); +} + +TEST(RuntimeImageMime, UnknownMimeTypeHasNoFormat) { + EXPECT_EQ(get_format_for_mime_type("text/html"), std::nullopt); + EXPECT_EQ(get_format_for_mime_type("application/octet-stream"), std::nullopt); + EXPECT_EQ(get_format_for_mime_type("image/*"), std::nullopt); + EXPECT_EQ(get_format_for_mime_type(""), std::nullopt); + EXPECT_EQ(get_format_for_mime_type(nullptr), std::nullopt); +} + +TEST(RuntimeImageMime, MimeTypeForFormatRoundTrip) { + EXPECT_STREQ(get_mime_type_for_format(BMP), "image/bmp"); + EXPECT_STREQ(get_mime_type_for_format(PNG), "image/png"); +#ifdef USE_RUNTIME_IMAGE_JPEG + EXPECT_STREQ(get_mime_type_for_format(JPEG), "image/jpeg"); +#endif // USE_RUNTIME_IMAGE_JPEG + // AUTO has no single MIME type and falls back to the wildcard + EXPECT_STREQ(get_mime_type_for_format(AUTO), "image/*"); + + // Every decodable format must resolve back to itself through its MIME type + for (ImageFormat format : { + BMP, + PNG, +#ifdef USE_RUNTIME_IMAGE_JPEG + JPEG, +#endif // USE_RUNTIME_IMAGE_JPEG + }) { + EXPECT_EQ(get_format_for_mime_type(get_mime_type_for_format(format)), format) << format; + } +} + +} // namespace esphome::runtime_image::testing From b97c87799255ce62eed22bcdcb22aae30e168a63 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Mon, 24 Aug 2026 15:27:24 -0500 Subject: [PATCH 11/65] [api] Acknowledge proxy subscribe and configuration requests (#18312) Co-authored-by: J. Nick Koston --- esphome/components/api/api.proto | 29 ++++- esphome/components/api/api_connection.cpp | 109 ++++++++++++------ esphome/components/api/api_pb2.cpp | 16 +++ esphome/components/api/api_pb2.h | 30 ++++- esphome/components/api/api_pb2_dump.cpp | 28 +++++ .../components/serial_proxy/serial_proxy.cpp | 74 ++++++++---- .../components/serial_proxy/serial_proxy.h | 28 ++++- .../components/zwave_proxy/zwave_proxy.cpp | 16 +-- esphome/components/zwave_proxy/zwave_proxy.h | 3 +- .../components/serial_proxy/serial_proxy.h | 36 +++--- .../components/zwave_proxy/zwave_proxy.h | 4 +- .../components/api/test_api_proto.py | 11 ++ 12 files changed, 298 insertions(+), 86 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 6ea124d155..1942ff568b 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -232,6 +232,7 @@ enum SerialProxyPortType { message SerialProxyInfo { string name = 1; // Human-readable port name SerialProxyPortType port_type = 2; // Port type (RS232, RS485) + uint32 configured_line_states = 3; // Bitmask of SerialProxyLineStateFlags this instance can drive } // DeviceInfoResponse max_data_length values: @@ -2626,6 +2627,22 @@ message ZWaveProxyRequest { bytes data = 2; } +enum ZWaveProxyStatus { + ZWAVE_PROXY_STATUS_OK = 0; // Request completed successfully + ZWAVE_PROXY_STATUS_IN_USE = 1; // Denied: another client is already subscribed + ZWAVE_PROXY_STATUS_NOT_SUPPORTED = 2; // Request type not supported +} + +// Acknowledges a ZWaveProxyRequest (subscribe/unsubscribe). Sent since API 1.16. +message ZWaveProxyRequestResponse { + option (id) = 151; + option (source) = SOURCE_SERVER; + option (ifdef) = "USE_ZWAVE_PROXY"; + + ZWaveProxyRequestType type = 1; // Which request type this responds to + ZWaveProxyStatus status = 2; // Result status +} + // ==================== INFRARED ==================== // Note: Feature and capability flag enums are defined in // esphome/components/infrared/infrared.h @@ -2769,12 +2786,18 @@ message SerialProxyGetModemPinsResponse { uint32 instance = 1; // Instance index (0-based) uint32 line_states = 2; // Bitmask of SerialProxyLineStateFlags + SerialProxyStatus status = 3; // INVALID_ARGUMENT if the instance index is out of range (since API 1.16) } enum SerialProxyRequestType { SERIAL_PROXY_REQUEST_TYPE_SUBSCRIBE = 0; // Subscribe to receive data from this serial proxy instance SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE = 1; // Unsubscribe from this serial proxy instance SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2; // Flush the serial port (block until all TX data is sent) + // Values below are only valid in SerialProxyRequestResponse.type, identifying which + // operation is being acknowledged. Sending them in SerialProxyRequest.type is an + // error the device answers with INVALID_ARGUMENT. + SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3; // Acknowledges a SerialProxyConfigureRequest + SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4; // Acknowledges a SerialProxySetModemPinsRequest } enum SerialProxyStatus { @@ -2783,6 +2806,8 @@ enum SerialProxyStatus { SERIAL_PROXY_STATUS_ERROR = 2; // Driver or hardware error SERIAL_PROXY_STATUS_TIMEOUT = 3; // Timed out before TX completed SERIAL_PROXY_STATUS_NOT_SUPPORTED = 4; // Request type not supported by this instance + SERIAL_PROXY_STATUS_PORT_IN_USE = 5; // Denied: another client holds the port + SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6; // Invalid instance index or parameter value } // Generic request message for simple serial proxy operations @@ -2795,7 +2820,9 @@ message SerialProxyRequest { SerialProxyRequestType type = 2; // Request type } -// Response to a SerialProxyRequest (e.g. flush completion or failure) +// Acknowledges a serial proxy operation; the type field identifies which +// operation is being acknowledged. Flush has been acknowledged since the +// message was introduced; all other acknowledgements are sent since API 1.16. message SerialProxyRequestResponse { option (id) = 147; option (source) = SOURCE_SERVER; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index bb6c1695dd..05abbf0b75 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1381,7 +1381,12 @@ void APIConnection::on_z_wave_proxy_frame(const ZWaveProxyFrame &msg) { } void APIConnection::on_z_wave_proxy_request(const ZWaveProxyRequest &msg) { - zwave_proxy::global_zwave_proxy->zwave_proxy_request(this, msg.type); + ZWaveProxyRequestResponse resp{}; + resp.type = msg.type; + resp.status = zwave_proxy::global_zwave_proxy->zwave_proxy_request(this, msg.type); + if (!this->send_message(resp)) { + API_LOG_MSG_DROPPED(TAG, "Z-Wave proxy response"); + } } #endif @@ -1550,15 +1555,50 @@ void APIConnection::send_infrared_rf_receive_event(const InfraredRFReceiveEvent #endif #ifdef USE_SERIAL_PROXY +static enums::SerialProxyStatus serial_proxy_result_to_status(serial_proxy::SerialProxyResult result) { + switch (result) { + case serial_proxy::SerialProxyResult::SERIAL_PROXY_RESULT_OK: + return enums::SERIAL_PROXY_STATUS_OK; + case serial_proxy::SerialProxyResult::SERIAL_PROXY_RESULT_ASSUMED_SUCCESS: + return enums::SERIAL_PROXY_STATUS_ASSUMED_SUCCESS; + case serial_proxy::SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE: + return enums::SERIAL_PROXY_STATUS_PORT_IN_USE; + case serial_proxy::SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT: + return enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT; + case serial_proxy::SerialProxyResult::SERIAL_PROXY_RESULT_TIMEOUT: + return enums::SERIAL_PROXY_STATUS_TIMEOUT; + case serial_proxy::SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED: + return enums::SERIAL_PROXY_STATUS_NOT_SUPPORTED; + case serial_proxy::SerialProxyResult::SERIAL_PROXY_RESULT_ERROR: + return enums::SERIAL_PROXY_STATUS_ERROR; + } + return enums::SERIAL_PROXY_STATUS_ERROR; // Unreachable; all enum values handled above +} + +static void send_serial_proxy_ack(APIConnection *conn, uint32_t instance, enums::SerialProxyRequestType type, + enums::SerialProxyStatus status) { + SerialProxyRequestResponse resp{}; + resp.instance = instance; + resp.type = type; + resp.status = status; + if (!conn->send_message(resp)) { + API_LOG_MSG_DROPPED(TAG, "Serial proxy response"); + } +} + void APIConnection::on_serial_proxy_configure_request(const SerialProxyConfigureRequest &msg) { auto &proxies = App.get_serial_proxies(); if (msg.instance >= proxies.size()) { ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range (max %" PRIu32 ")", msg.instance, static_cast(proxies.size())); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE, + enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT); return; } - proxies[msg.instance]->configure(this, msg.baudrate, msg.flow_control, static_cast(msg.parity), - msg.stop_bits, msg.data_size); + serial_proxy::SerialProxyResult result = proxies[msg.instance]->configure( + this, msg.baudrate, msg.flow_control, static_cast(msg.parity), msg.stop_bits, msg.data_size); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE, + serial_proxy_result_to_status(result)); } void APIConnection::on_serial_proxy_write_request(const SerialProxyWriteRequest &msg) { @@ -1574,20 +1614,30 @@ void APIConnection::on_serial_proxy_set_modem_pins_request(const SerialProxySetM auto &proxies = App.get_serial_proxies(); if (msg.instance >= proxies.size()) { ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS, + enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT); return; } - proxies[msg.instance]->set_modem_pins(this, msg.line_states); + serial_proxy::SerialProxyResult result = proxies[msg.instance]->set_modem_pins(this, msg.line_states); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS, + serial_proxy_result_to_status(result)); } void APIConnection::on_serial_proxy_get_modem_pins_request(const SerialProxyGetModemPinsRequest &msg) { auto &proxies = App.get_serial_proxies(); - if (msg.instance >= proxies.size()) { - ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance); - return; - } SerialProxyGetModemPinsResponse resp{}; resp.instance = msg.instance; - resp.line_states = proxies[msg.instance]->get_modem_pins(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance); + // Pre-1.16 clients do not read the status field and would take this error + // for a successful "both pins deasserted" answer; let them time out as before + if (!this->client_supports_api_version(1, 16)) { + return; + } + resp.status = enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT; + } else { + resp.line_states = proxies[msg.instance]->get_modem_pins(); + } if (!this->send_message(resp)) { API_LOG_MSG_DROPPED(TAG, "Serial proxy response"); } @@ -1597,40 +1647,31 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { auto &proxies = App.get_serial_proxies(); if (msg.instance >= proxies.size()) { ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance); + send_serial_proxy_ack(this, msg.instance, msg.type, enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT); return; } + auto *proxy = proxies[msg.instance]; + enums::SerialProxyStatus status; switch (msg.type) { case enums::SERIAL_PROXY_REQUEST_TYPE_SUBSCRIBE: case enums::SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE: - proxies[msg.instance]->serial_proxy_request(this, msg.type); + status = serial_proxy_result_to_status(proxy->serial_proxy_request(this, msg.type)); break; - case enums::SERIAL_PROXY_REQUEST_TYPE_FLUSH: { - SerialProxyRequestResponse resp{}; - resp.instance = msg.instance; - resp.type = enums::SERIAL_PROXY_REQUEST_TYPE_FLUSH; - switch (proxies[msg.instance]->flush_port()) { - case uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS: - resp.status = enums::SERIAL_PROXY_STATUS_OK; - break; - case uart::UARTFlushResult::UART_FLUSH_RESULT_ASSUMED_SUCCESS: - resp.status = enums::SERIAL_PROXY_STATUS_ASSUMED_SUCCESS; - break; - case uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT: - resp.status = enums::SERIAL_PROXY_STATUS_TIMEOUT; - break; - case uart::UARTFlushResult::UART_FLUSH_RESULT_FAILED: - resp.status = enums::SERIAL_PROXY_STATUS_ERROR; - break; - } - if (!this->send_message(resp)) { - API_LOG_MSG_DROPPED(TAG, "Serial proxy response"); - } + case enums::SERIAL_PROXY_REQUEST_TYPE_FLUSH: + status = serial_proxy_result_to_status(proxy->flush_port(this)); + break; + case enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE: + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: + // Response-only discriminators; never valid in a request + ESP_LOGW(TAG, "Response-only serial proxy request type: %" PRIu32, static_cast(msg.type)); + status = enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT; break; - } default: ESP_LOGW(TAG, "Unknown serial proxy request type: %" PRIu32, static_cast(msg.type)); + status = enums::SERIAL_PROXY_STATUS_NOT_SUPPORTED; break; } + send_serial_proxy_ack(this, msg.instance, msg.type, status); } void APIConnection::send_serial_proxy_data(const SerialProxyDataReceived &msg) { @@ -1757,7 +1798,7 @@ bool APIConnection::send_hello_response_(const HelloRequest &msg) { HelloResponse resp; resp.api_version_major = 1; - resp.api_version_minor = 15; + resp.api_version_minor = 16; // Send only the version string - the client only logs this for debugging and doesn't use it otherwise resp.server_info = ESPHOME_VERSION_REF; resp.name = StringRef(App.get_name()); @@ -1891,6 +1932,7 @@ bool APIConnection::send_device_info_response_() { auto &info = resp.serial_proxies[serial_proxy_index++]; info.name = StringRef(proxy->get_name()); info.port_type = proxy->get_port_type(); + info.configured_line_states = proxy->get_configured_modem_pins(); } #endif #ifdef USE_API_NOISE @@ -1951,6 +1993,7 @@ bool APIConnection::send_device_capabilities_response_() { auto &info = resp.serial_proxies[serial_proxy_index++]; info.name = StringRef(proxy->get_name()); info.port_type = proxy->get_port_type(); + info.configured_line_states = proxy->get_configured_modem_pins(); } #endif return this->send_message(resp); diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 33611c5ee1..b5062f9e9f 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -102,12 +102,14 @@ uint8_t *SerialProxyInfo::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PAR uint8_t *__restrict__ pos = buffer.get_pos(); ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 1, this->name); ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 2, static_cast(this->port_type)); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 3, this->configured_line_states); return pos; } uint32_t SerialProxyInfo::calculate_size() const { uint32_t size = 0; size += ProtoSize::calc_length(1, this->name.size()); size += this->port_type ? 2 : 0; + size += ProtoSize::calc_uint32(1, this->configured_line_states); return size; } #endif @@ -3942,6 +3944,18 @@ uint32_t ZWaveProxyRequest::calculate_size() const { size += ProtoSize::calc_length(1, this->data_len); return size; } +uint8_t *ZWaveProxyRequestResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { + uint8_t *__restrict__ pos = buffer.get_pos(); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 1, static_cast(this->type)); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 2, static_cast(this->status)); + return pos; +} +uint32_t ZWaveProxyRequestResponse::calculate_size() const { + uint32_t size = 0; + size += this->type ? 2 : 0; + size += this->status ? 2 : 0; + return size; +} #endif #ifdef USE_INFRARED uint8_t *ListEntitiesInfraredResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { @@ -4184,12 +4198,14 @@ uint8_t *SerialProxyGetModemPinsResponse::encode(ProtoWriteBuffer &buffer PROTO_ uint8_t *__restrict__ pos = buffer.get_pos(); ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 1, this->instance); ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 2, this->line_states); + ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 3, static_cast(this->status)); return pos; } uint32_t SerialProxyGetModemPinsResponse::calculate_size() const { uint32_t size = 0; size += ProtoSize::calc_uint32(1, this->instance); size += ProtoSize::calc_uint32(1, this->line_states); + size += this->status ? 2 : 0; return size; } bool SerialProxyRequest::decode_varint(uint32_t field_id, proto_varint_value_t value) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 13db857467..cd2f32deaf 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -334,6 +334,11 @@ enum ZWaveProxyRequestType : uint32_t { ZWAVE_PROXY_REQUEST_TYPE_UNSUBSCRIBE = 1, ZWAVE_PROXY_REQUEST_TYPE_HOME_ID_CHANGE = 2, }; +enum ZWaveProxyStatus : uint32_t { + ZWAVE_PROXY_STATUS_OK = 0, + ZWAVE_PROXY_STATUS_IN_USE = 1, + ZWAVE_PROXY_STATUS_NOT_SUPPORTED = 2, +}; #endif #ifdef USE_SERIAL_PROXY enum SerialProxyParity : uint32_t { @@ -345,6 +350,8 @@ enum SerialProxyRequestType : uint32_t { SERIAL_PROXY_REQUEST_TYPE_SUBSCRIBE = 0, SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE = 1, SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2, + SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3, + SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4, }; enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_OK = 0, @@ -352,6 +359,8 @@ enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_ERROR = 2, SERIAL_PROXY_STATUS_TIMEOUT = 3, SERIAL_PROXY_STATUS_NOT_SUPPORTED = 4, + SERIAL_PROXY_STATUS_PORT_IN_USE = 5, + SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6, }; #endif @@ -523,6 +532,7 @@ class SerialProxyInfo final : public ProtoMessage { public: StringRef name{}; enums::SerialProxyPortType port_type{}; + uint32_t configured_line_states{0}; uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; uint32_t calculate_size() const; #ifdef HAS_PROTO_MESSAGE_DUMP @@ -3130,6 +3140,23 @@ class ZWaveProxyRequest final : public ProtoDecodableMessage { bool decode_length(uint32_t field_id, ProtoLengthDelimited value) override; bool decode_varint(uint32_t field_id, proto_varint_value_t value) override; }; +class ZWaveProxyRequestResponse final : public ProtoMessage { + public: + static constexpr uint8_t MESSAGE_TYPE = 151; + static constexpr uint8_t ESTIMATED_SIZE = 4; +#ifdef HAS_PROTO_MESSAGE_DUMP + const LogString *message_name() const override { return LOG_STR("z_wave_proxy_request_response"); } +#endif + enums::ZWaveProxyRequestType type{}; + enums::ZWaveProxyStatus status{}; + uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; + uint32_t calculate_size() const; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: +}; #endif #ifdef USE_INFRARED class ListEntitiesInfraredResponse final : public InfoResponseProtoMessage { @@ -3314,12 +3341,13 @@ class SerialProxyGetModemPinsRequest final : public ProtoDecodableMessage { class SerialProxyGetModemPinsResponse final : public ProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 143; - static constexpr uint8_t ESTIMATED_SIZE = 8; + static constexpr uint8_t ESTIMATED_SIZE = 10; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_get_modem_pins_response"); } #endif uint32_t instance{0}; uint32_t line_states{0}; + enums::SerialProxyStatus status{}; uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; uint32_t calculate_size() const; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index d54215ba2e..9f53438531 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -816,6 +816,18 @@ template<> const char *proto_enum_to_string(enums: return ESPHOME_PSTR("UNKNOWN"); } } +template<> const char *proto_enum_to_string(enums::ZWaveProxyStatus value) { + switch (value) { + case enums::ZWAVE_PROXY_STATUS_OK: + return ESPHOME_PSTR("ZWAVE_PROXY_STATUS_OK"); + case enums::ZWAVE_PROXY_STATUS_IN_USE: + return ESPHOME_PSTR("ZWAVE_PROXY_STATUS_IN_USE"); + case enums::ZWAVE_PROXY_STATUS_NOT_SUPPORTED: + return ESPHOME_PSTR("ZWAVE_PROXY_STATUS_NOT_SUPPORTED"); + default: + return ESPHOME_PSTR("UNKNOWN"); + } +} #endif #ifdef USE_SERIAL_PROXY template<> const char *proto_enum_to_string(enums::SerialProxyParity value) { @@ -838,6 +850,10 @@ template<> const char *proto_enum_to_string(enums return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE"); case enums::SERIAL_PROXY_REQUEST_TYPE_FLUSH: return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_FLUSH"); + case enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE: + return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_CONFIGURE"); + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: + return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS"); default: return ESPHOME_PSTR("UNKNOWN"); } @@ -854,6 +870,10 @@ template<> const char *proto_enum_to_string(enums::Ser return ESPHOME_PSTR("SERIAL_PROXY_STATUS_TIMEOUT"); case enums::SERIAL_PROXY_STATUS_NOT_SUPPORTED: return ESPHOME_PSTR("SERIAL_PROXY_STATUS_NOT_SUPPORTED"); + case enums::SERIAL_PROXY_STATUS_PORT_IN_USE: + return ESPHOME_PSTR("SERIAL_PROXY_STATUS_PORT_IN_USE"); + case enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT: + return ESPHOME_PSTR("SERIAL_PROXY_STATUS_INVALID_ARGUMENT"); default: return ESPHOME_PSTR("UNKNOWN"); } @@ -914,6 +934,7 @@ const char *SerialProxyInfo::dump_to(DumpBuffer &out) const { MessageDumpHelper helper(out, ESPHOME_PSTR("SerialProxyInfo")); dump_field(out, ESPHOME_PSTR("name"), this->name); dump_field(out, ESPHOME_PSTR("port_type"), static_cast(this->port_type)); + dump_field(out, ESPHOME_PSTR("configured_line_states"), this->configured_line_states); return out.c_str(); } #endif @@ -2644,6 +2665,12 @@ const char *ZWaveProxyRequest::dump_to(DumpBuffer &out) const { dump_bytes_field(out, ESPHOME_PSTR("data"), this->data, this->data_len); return out.c_str(); } +const char *ZWaveProxyRequestResponse::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("ZWaveProxyRequestResponse")); + dump_field(out, ESPHOME_PSTR("type"), static_cast(this->type)); + dump_field(out, ESPHOME_PSTR("status"), static_cast(this->status)); + return out.c_str(); +} #endif #ifdef USE_INFRARED const char *ListEntitiesInfraredResponse::dump_to(DumpBuffer &out) const { @@ -2753,6 +2780,7 @@ const char *SerialProxyGetModemPinsResponse::dump_to(DumpBuffer &out) const { MessageDumpHelper helper(out, ESPHOME_PSTR("SerialProxyGetModemPinsResponse")); dump_field(out, ESPHOME_PSTR("instance"), this->instance); dump_field(out, ESPHOME_PSTR("line_states"), this->line_states); + dump_field(out, ESPHOME_PSTR("status"), static_cast(this->status)); return out.c_str(); } const char *SerialProxyRequest::dump_to(DumpBuffer &out) const { diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index 4b3a907416..94cefc8700 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -89,12 +89,12 @@ void SerialProxy::dump_config() { this->dtr_pin_ != nullptr ? "configured" : "not configured"); } -void SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, - uint8_t stop_bits, uint8_t data_size) { +SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, + uint8_t parity, uint8_t stop_bits, uint8_t data_size) { #ifdef USE_API if (this->port_claimed_by_other_(api_connection)) { ESP_LOGW(TAG, "Ignoring configure request from client without port access [%" PRIu32 "]", this->instance_index_); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif ESP_LOGD(TAG, @@ -105,25 +105,29 @@ void SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrat auto *uart_comp = this->parent_; if (uart_comp == nullptr) { ESP_LOGE(TAG, "UART component not available"); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_ERROR; } // Validate all parameters before applying any (values come from a remote client) if (baudrate == 0) { ESP_LOGW(TAG, "Invalid baud rate: 0"); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT; } if (stop_bits < 1 || stop_bits > 2) { ESP_LOGW(TAG, "Invalid stop bits: %u (must be 1 or 2)", stop_bits); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT; } if (data_size < 5 || data_size > 8) { ESP_LOGW(TAG, "Invalid data bits: %u (must be 5-8)", data_size); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT; } if (parity > 2) { ESP_LOGW(TAG, "Invalid parity: %u (must be 0-2)", parity); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT; + } + if (flow_control) { + ESP_LOGW(TAG, "Hardware flow control requested but is not yet supported"); + return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; } // Apply validated parameters @@ -143,10 +147,7 @@ void SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrat #if defined(USE_ESP8266) || defined(USE_ESP32) uart_comp->load_settings(true); #endif - - if (flow_control) { - ESP_LOGW(TAG, "Hardware flow control requested but is not yet supported"); - } + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } void SerialProxy::write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) { @@ -163,13 +164,20 @@ void SerialProxy::write_from_client(api::APIConnection *api_connection, const ui this->write_array(data, len); } -void SerialProxy::set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { +SerialProxyResult SerialProxy::set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { #ifdef USE_API if (this->port_claimed_by_other_(api_connection)) { ESP_LOGW(TAG, "Ignoring modem pin request from client without port access [%" PRIu32 "]", this->instance_index_); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif + // Asserting a pin that is not configured must fail so the client learns the signal never + // reached the wire; deasserting an absent pin is harmless and stays allowed. Clients can + // avoid this by masking against SerialProxyInfo.configured_line_states. + if ((line_states & ~this->get_configured_modem_pins()) != 0) { + ESP_LOGW(TAG, "Requested modem pin not configured on serial proxy [%" PRIu32 "]", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; + } const bool rts = (line_states & SERIAL_PROXY_LINE_STATE_FLAG_RTS) != 0; const bool dtr = (line_states & SERIAL_PROXY_LINE_STATE_FLAG_DTR) != 0; ESP_LOGV(TAG, "Setting modem pins [%" PRIu32 "]: RTS=%s, DTR=%s", this->instance_index_, ONOFF(rts), ONOFF(dtr)); @@ -182,6 +190,7 @@ void SerialProxy::set_modem_pins(api::APIConnection *api_connection, uint32_t li this->dtr_state_ = dtr; this->dtr_pin_->digital_write(dtr); } + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } uint32_t SerialProxy::get_modem_pins() const { @@ -189,9 +198,26 @@ uint32_t SerialProxy::get_modem_pins() const { (this->dtr_state_ ? static_cast(SERIAL_PROXY_LINE_STATE_FLAG_DTR) : 0u); } -uart::UARTFlushResult SerialProxy::flush_port() { +SerialProxyResult SerialProxy::flush_port(api::APIConnection *api_connection) { +#ifdef USE_API + // Flushing stalls the port, so it gets the same ownership check as writes + if (this->port_claimed_by_other_(api_connection)) { + ESP_LOGW(TAG, "Ignoring flush from client without port access [%" PRIu32 "]", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; + } +#endif ESP_LOGV(TAG, "Flushing serial proxy [%" PRIu32 "]", this->instance_index_); - return this->flush(); + switch (this->flush()) { + case uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS: + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + case uart::UARTFlushResult::UART_FLUSH_RESULT_ASSUMED_SUCCESS: + return SerialProxyResult::SERIAL_PROXY_RESULT_ASSUMED_SUCCESS; + case uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT: + return SerialProxyResult::SERIAL_PROXY_RESULT_TIMEOUT; + case uart::UARTFlushResult::UART_FLUSH_RESULT_FAILED: + return SerialProxyResult::SERIAL_PROXY_RESULT_ERROR; + } + return SerialProxyResult::SERIAL_PROXY_RESULT_ERROR; // Unreachable; all enum values handled above } #ifdef USE_API @@ -200,12 +226,13 @@ bool SerialProxy::port_claimed_by_other_(api::APIConnection *api_connection) con this->api_connection_->is_connection_setup(); } -void SerialProxy::serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type) { +SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_connection, + api::enums::SerialProxyRequestType type) { switch (type) { case api::enums::SERIAL_PROXY_REQUEST_TYPE_SUBSCRIBE: if (this->api_connection_ == api_connection) { ESP_LOGV(TAG, "API connection is already subscribed to serial proxy [%" PRIu32 "]", this->instance_index_); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } if (this->api_connection_ != nullptr) { // A living subscriber keeps exclusive access. Its connection may be dead without @@ -213,26 +240,27 @@ void SerialProxy::serial_proxy_request(api::APIConnection *api_connection, api:: // in that case let the new client take over instead of locking it out. if (this->api_connection_->is_connection_setup()) { ESP_LOGE(TAG, "Only one API subscription is allowed at a time"); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } ESP_LOGW(TAG, "Previous subscriber disconnected; taking over subscription"); } this->api_connection_ = api_connection; this->enable_loop(); ESP_LOGV(TAG, "API connection subscribed to serial proxy [%" PRIu32 "]", this->instance_index_); - break; + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; case api::enums::SERIAL_PROXY_REQUEST_TYPE_UNSUBSCRIBE: + // Unsubscribe is idempotent: not being subscribed is not an error if (this->api_connection_ != api_connection) { ESP_LOGV(TAG, "API connection is not subscribed to serial proxy [%" PRIu32 "]", this->instance_index_); - return; + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } this->api_connection_ = nullptr; this->disable_loop(); ESP_LOGV(TAG, "API connection unsubscribed from serial proxy [%" PRIu32 "]", this->instance_index_); - break; + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; default: ESP_LOGW(TAG, "Unknown serial proxy request type: %" PRIu32, static_cast(type)); - break; + return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; } } #endif diff --git a/esphome/components/serial_proxy/serial_proxy.h b/esphome/components/serial_proxy/serial_proxy.h index 268c1b52be..a0e47ee686 100644 --- a/esphome/components/serial_proxy/serial_proxy.h +++ b/esphome/components/serial_proxy/serial_proxy.h @@ -38,6 +38,17 @@ enum SerialProxyLineStateFlag : uint32_t { SERIAL_PROXY_LINE_STATE_FLAG_DTR = 1 << 1, ///< DTR (Data Terminal Ready) }; +/// Result of a client-initiated operation; mapped to api::enums::SerialProxyStatus by the API layer +enum class SerialProxyResult : uint8_t { + SERIAL_PROXY_RESULT_OK, ///< Operation completed or request accepted + SERIAL_PROXY_RESULT_ASSUMED_SUCCESS, ///< Platform cannot confirm TX drain; success assumed + SERIAL_PROXY_RESULT_PORT_IN_USE, ///< Denied: another live client holds the port + SERIAL_PROXY_RESULT_INVALID_ARGUMENT, ///< A parameter value is out of range + SERIAL_PROXY_RESULT_ERROR, ///< Driver or hardware error + SERIAL_PROXY_RESULT_TIMEOUT, ///< Timed out before TX completed + SERIAL_PROXY_RESULT_NOT_SUPPORTED, ///< Requested feature is not available on this instance +}; + /// Maximum bytes to read from UART in a single loop iteration inline constexpr size_t SERIAL_PROXY_MAX_READ_SIZE = 256; @@ -73,14 +84,14 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// @param parity Parity setting (0=none, 1=even, 2=odd) /// @param stop_bits Number of stop bits (1 or 2) /// @param data_size Number of data bits (5-8) - void configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, - uint8_t stop_bits, uint8_t data_size); + SerialProxyResult configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, + uint8_t stop_bits, uint8_t data_size); /// Get the currently subscribed API connection (nullptr if none) api::APIConnection *get_api_connection() { return this->api_connection_; } /// Handle a subscribe/unsubscribe request from an API client - void serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type); + SerialProxyResult serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type); /// Write data received from an API client to the serial device /// @param api_connection The API connection sending the data @@ -89,13 +100,20 @@ class SerialProxy final : public uart::UARTDevice, public Component { void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len); /// Set modem pin states from a bitmask of SerialProxyLineStateFlag values - void set_modem_pins(api::APIConnection *api_connection, uint32_t line_states); + SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states); /// Get current modem pin states as a bitmask of SerialProxyLineStateFlag values uint32_t get_modem_pins() const; + /// Get the modem pins this instance can drive as a bitmask of SerialProxyLineStateFlag values + uint32_t get_configured_modem_pins() const { + return (this->rts_pin_ != nullptr ? static_cast(SERIAL_PROXY_LINE_STATE_FLAG_RTS) : 0u) | + (this->dtr_pin_ != nullptr ? static_cast(SERIAL_PROXY_LINE_STATE_FLAG_DTR) : 0u); + } + /// Flush the serial port (block until all TX data is sent) - uart::UARTFlushResult flush_port(); + /// @param api_connection The API connection requesting the flush + SerialProxyResult flush_port(api::APIConnection *api_connection); /// Set the RTS GPIO pin (from YAML configuration) void set_rts_pin(GPIOPin *pin) { this->rts_pin_ = pin; } diff --git a/esphome/components/zwave_proxy/zwave_proxy.cpp b/esphome/components/zwave_proxy/zwave_proxy.cpp index 6e3f109ca1..68750295e1 100644 --- a/esphome/components/zwave_proxy/zwave_proxy.cpp +++ b/esphome/components/zwave_proxy/zwave_proxy.cpp @@ -194,12 +194,13 @@ void ZWaveProxy::api_connection_authenticated(api::APIConnection *conn) { } } -void ZWaveProxy::zwave_proxy_request(api::APIConnection *api_connection, api::enums::ZWaveProxyRequestType type) { +api::enums::ZWaveProxyStatus ZWaveProxy::zwave_proxy_request(api::APIConnection *api_connection, + api::enums::ZWaveProxyRequestType type) { switch (type) { case api::enums::ZWAVE_PROXY_REQUEST_TYPE_SUBSCRIBE: if (this->api_connection_ == api_connection) { ESP_LOGV(TAG, "API connection is already subscribed"); - return; + return api::enums::ZWAVE_PROXY_STATUS_OK; } if (this->api_connection_ != nullptr) { // A living subscriber keeps exclusive access. Its connection may be dead without @@ -207,25 +208,26 @@ void ZWaveProxy::zwave_proxy_request(api::APIConnection *api_connection, api::en // in that case let the new client take over instead of locking it out. if (this->api_connection_->is_connection_setup()) { ESP_LOGE(TAG, "Only one API subscription is allowed at a time"); - return; + return api::enums::ZWAVE_PROXY_STATUS_IN_USE; } ESP_LOGW(TAG, "Previous subscriber disconnected; taking over subscription"); } this->api_connection_ = api_connection; ESP_LOGV(TAG, "API connection is now subscribed"); - break; + return api::enums::ZWAVE_PROXY_STATUS_OK; case api::enums::ZWAVE_PROXY_REQUEST_TYPE_UNSUBSCRIBE: + // Unsubscribe is idempotent: not being subscribed is not an error if (this->api_connection_ != api_connection) { ESP_LOGV(TAG, "API connection is not subscribed"); - return; + return api::enums::ZWAVE_PROXY_STATUS_OK; } this->api_connection_ = nullptr; - break; + return api::enums::ZWAVE_PROXY_STATUS_OK; default: ESP_LOGW(TAG, "Unknown request type: %" PRIu32, static_cast(type)); - break; + return api::enums::ZWAVE_PROXY_STATUS_NOT_SUPPORTED; } } diff --git a/esphome/components/zwave_proxy/zwave_proxy.h b/esphome/components/zwave_proxy/zwave_proxy.h index cb60139ef8..75225d84a8 100644 --- a/esphome/components/zwave_proxy/zwave_proxy.h +++ b/esphome/components/zwave_proxy/zwave_proxy.h @@ -60,7 +60,8 @@ class ZWaveProxy final : public uart::UARTDevice, public Component { bool can_proceed() override; void api_connection_authenticated(api::APIConnection *conn); - void zwave_proxy_request(api::APIConnection *api_connection, api::enums::ZWaveProxyRequestType type); + api::enums::ZWaveProxyStatus zwave_proxy_request(api::APIConnection *api_connection, + api::enums::ZWaveProxyRequestType type); api::APIConnection *get_api_connection() { return this->api_connection_; } uint32_t get_feature_flags() const { return ZWaveProxyFeature::FEATURE_ZWAVE_PROXY_ENABLED; } diff --git a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h index d8b068fb36..6fc20f3350 100644 --- a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h +++ b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h @@ -13,17 +13,18 @@ namespace api { class APIConnection; } // namespace api -namespace uart { -enum class UARTFlushResult : uint8_t { - UART_FLUSH_RESULT_SUCCESS, - UART_FLUSH_RESULT_ASSUMED_SUCCESS, - UART_FLUSH_RESULT_TIMEOUT, - UART_FLUSH_RESULT_FAILED, -}; -} // namespace uart - namespace serial_proxy { +enum class SerialProxyResult : uint8_t { + SERIAL_PROXY_RESULT_OK, + SERIAL_PROXY_RESULT_ASSUMED_SUCCESS, + SERIAL_PROXY_RESULT_PORT_IN_USE, + SERIAL_PROXY_RESULT_INVALID_ARGUMENT, + SERIAL_PROXY_RESULT_ERROR, + SERIAL_PROXY_RESULT_TIMEOUT, + SERIAL_PROXY_RESULT_NOT_SUPPORTED, +}; + class SerialProxy { public: void set_instance_index(uint32_t index) { this->instance_index_ = index; } @@ -31,13 +32,20 @@ class SerialProxy { const char *get_name() const { return ""; } api::enums::SerialProxyPortType get_port_type() const { return {}; } api::APIConnection *get_api_connection() { return nullptr; } - void serial_proxy_request(api::APIConnection *conn, api::enums::SerialProxyRequestType type) {} - void configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, - uint32_t stop_bits, uint32_t data_size) {} + SerialProxyResult serial_proxy_request(api::APIConnection *conn, api::enums::SerialProxyRequestType type) { + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } + SerialProxyResult configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, + uint8_t stop_bits, uint8_t data_size) { + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {} - void set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) {} + SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } uint32_t get_modem_pins() const { return 0; } - uart::UARTFlushResult flush_port() { return uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS; } + uint32_t get_configured_modem_pins() const { return 0; } + SerialProxyResult flush_port(api::APIConnection *api_connection) { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } protected: uint32_t instance_index_{0}; diff --git a/tests/benchmarks/stubs/esphome/components/zwave_proxy/zwave_proxy.h b/tests/benchmarks/stubs/esphome/components/zwave_proxy/zwave_proxy.h index b4ccd8fd00..3abbf7d257 100644 --- a/tests/benchmarks/stubs/esphome/components/zwave_proxy/zwave_proxy.h +++ b/tests/benchmarks/stubs/esphome/components/zwave_proxy/zwave_proxy.h @@ -15,7 +15,9 @@ namespace zwave_proxy { class ZWaveProxy { public: api::APIConnection *get_api_connection() { return nullptr; } - void zwave_proxy_request(api::APIConnection *conn, api::enums::ZWaveProxyRequestType type) {} + api::enums::ZWaveProxyStatus zwave_proxy_request(api::APIConnection *conn, api::enums::ZWaveProxyRequestType type) { + return api::enums::ZWAVE_PROXY_STATUS_OK; + } void send_frame(api::APIConnection *api_connection, const uint8_t *data, size_t length) {} void api_connection_authenticated(api::APIConnection *conn) {} uint32_t get_feature_flags() const { return 0; } diff --git a/tests/unit_tests/components/api/test_api_proto.py b/tests/unit_tests/components/api/test_api_proto.py index 35aa5ff529..31297911f5 100644 --- a/tests/unit_tests/components/api/test_api_proto.py +++ b/tests/unit_tests/components/api/test_api_proto.py @@ -263,6 +263,17 @@ def test_device_capabilities_response_has_id_150() -> None: ) +def test_z_wave_proxy_request_response_has_id_151() -> None: + body = _extract_proto_message(PROTO_TEXT, "ZWaveProxyRequestResponse") + match = re.search(r"option \(id\) = (\d+);", body) + assert match is not None, "ZWaveProxyRequestResponse is missing `option (id)`" + assert int(match.group(1)) == 151, ( + f"ZWaveProxyRequestResponse has id {match.group(1)}, expected 151. " + "Message ids are part of the wire protocol and must not change once " + "assigned." + ) + + def test_superseded_fields_are_not_marked_deprecated_in_proto() -> None: """The six superseded fields must not carry `[deprecated = true]` in api.proto, or the generator drops them and old clients stop receiving From 7c5105b16ea3f44a500494884415de6d3d6e3111 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:36:06 +1000 Subject: [PATCH 12/65] [key_collector][mipi_rgb] Suppress compilation warnings (#18721) --- esphome/components/key_collector/key_collector.cpp | 2 ++ esphome/components/mipi_rgb/mipi_rgb.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/esphome/components/key_collector/key_collector.cpp b/esphome/components/key_collector/key_collector.cpp index cb7d47b7f0..69b7a6a7c6 100644 --- a/esphome/components/key_collector/key_collector.cpp +++ b/esphome/components/key_collector/key_collector.cpp @@ -14,6 +14,7 @@ void KeyCollector::loop() { } void KeyCollector::dump_config() { +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_CONFIG ESP_LOGCONFIG(TAG, "Key Collector:"); if (this->min_length_ > 0) ESP_LOGCONFIG(TAG, " min length: %d", this->min_length_); @@ -35,6 +36,7 @@ void KeyCollector::dump_config() { ESP_LOGCONFIG(TAG, " allowed keys '%s'", this->allowed_keys_.c_str()); if (this->timeout_ > 0) ESP_LOGCONFIG(TAG, " entry timeout: %0.1f", this->timeout_ / 1000.0); +#endif } void KeyCollector::add_provider(key_provider::KeyProvider *provider) { diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index b07460fdba..7421d8ad83 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -345,7 +345,7 @@ int MipiRgb::get_height() { } } -static const char *get_pin_name(GPIOPin *pin, std::span buffer) { +[[maybe_unused]] static const char *get_pin_name(GPIOPin *pin, std::span buffer) { if (pin == nullptr) return "None"; pin->dump_summary(buffer.data(), buffer.size()); From 3f0ec0b1572376d8234ba9e61f710b21e2d0043e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 24 Aug 2026 15:56:59 -0500 Subject: [PATCH 13/65] [espidf] Always reconfigure after component discovery (#18730) --- esphome/espidf/toolchain.py | 38 ++++++++---------- tests/unit_tests/test_espidf_toolchain.py | 49 +++++++++++++++++++++++ 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 3c5c4803c2..baf316a4ee 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -398,27 +398,23 @@ def run_compile(config, verbose: bool) -> int: return rc _LOGGER.info("Regenerating CMakeLists.txt with discovered components...") write_project(minimal=False) - # Restamp the reference file has_outdated_files() compares against. - # A reconfigure that only changes properties or plain variables - # (sdkconfig options, the exclusion set) does not rewrite - # CMakeCache.txt, so without this the watched inputs stay newer - # forever and every subsequent build repeats the discovery pass. - # Done after the full write so an interrupt cannot leave a minimal - # CMakeLists behind that is already marked fresh. - cmakecache = CORE.relative_build_path("build/CMakeCache.txt") - if cmakecache.is_file(): - os.utime(cmakecache) - if CORE.testing_mode: - # Reconfigure again so cmake is up to date with the full - # component list before the build's idf.py invocation runs -- - # idf.py build would otherwise re-run cmake and regenerate - # memory.ld, wiping the DRAM/IRAM patches applied below. - # Outside testing mode ninja's own configure-time dep on - # CMakeLists.txt handles the re-run as part of the build step. - rc = run_reconfigure() - if rc != 0: - _LOGGER.error("Reconfigure with discovered components failed") - return rc + # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt + # is strictly newer than build.ninja, which fails on coarse-mtime + # filesystems (#18682). Also keeps idf.py from regenerating memory.ld + # in testing mode. + rc = run_reconfigure() + if rc != 0: + _LOGGER.error("Reconfigure with discovered components failed") + return rc + # cmake does not rewrite CMakeCache.txt when only properties change, + # so restamp it or every build repeats discovery. Only after success, + # or a failed reconfigure would be marked fresh. build.ninja is + # restamped too so the cache is not newer and ninja does not + # re-run cmake. + for name in ("build/CMakeCache.txt", "build/build.ninja"): + path = CORE.relative_build_path(name) + if path.is_file(): + os.utime(path) # In testing mode, generate the linker script first, patch DRAM/IRAM sizes, # then build. memory.ld is regenerated by ninja during the build phase, diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 6c11a74d48..c54daec6a4 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -299,10 +299,13 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No _setup_build(setup_core) config = {CONF_ESPHOME: {}} cmakecache = CORE.relative_build_path("build/CMakeCache.txt") + build_ninja = CORE.relative_build_path("build/build.ninja") cmakecache.parent.mkdir(parents=True, exist_ok=True) cmakecache.write_text("") + build_ninja.write_text("") old = cmakecache.stat().st_mtime - 100 os.utime(cmakecache, (old, old)) + os.utime(build_ninja, (old, old)) with ( patch.object(toolchain, "need_reconfigure", return_value=True), @@ -314,6 +317,8 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No assert toolchain.run_compile(config, verbose=False) == 0 assert cmakecache.stat().st_mtime > old + # build.ninja must not be older than the cache or ninja re-runs cmake + assert build_ninja.stat().st_mtime >= cmakecache.stat().st_mtime def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: @@ -334,6 +339,50 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: assert not CORE.relative_build_path("build/CMakeCache.txt").exists() +def test_run_compile_reconfigures_after_full_write_outside_testing_mode( + setup_core: Path, +) -> None: + """The full CMakeLists write is followed by a reconfigure (#18682); a + failure there stops the build and leaves the cache unstamped.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + cmakecache = CORE.relative_build_path("build/CMakeCache.txt") + cmakecache.parent.mkdir(parents=True, exist_ok=True) + cmakecache.write_text("") + old = cmakecache.stat().st_mtime - 100 + os.utime(cmakecache, (old, old)) + calls: list[tuple] = [] + reconfigures = 0 + + def record_write(minimal: bool = False) -> None: + calls.append(("write_project", minimal)) + + def record_reconfigure() -> int: + nonlocal reconfigures + reconfigures += 1 + calls.append(("run_reconfigure",)) + return 1 if reconfigures == 2 else 0 + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch("esphome.build_gen.espidf.write_project", side_effect=record_write), + patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure), + patch.object(toolchain, "run_idf_py", return_value=0) as mock_build, + patch.object(toolchain, "print_summary"), + ): + assert not CORE.testing_mode + assert toolchain.run_compile(config, verbose=False) == 1 + + assert calls == [ + ("write_project", True), + ("run_reconfigure",), + ("write_project", False), + ("run_reconfigure",), + ] + mock_build.assert_not_called() + assert cmakecache.stat().st_mtime == old + + def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None: """compile_process_limit is forwarded to run_idf_py as the job limit.""" _setup_build(setup_core) From 19866c60ee7b88cfbc5417ebea453029e7326422 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 24 Aug 2026 15:59:00 -0500 Subject: [PATCH 14/65] [core] Parallelize registry and tool downloads (#18662) --- esphome/espidf/framework.py | 100 +++-- esphome/framework_helpers.py | 237 +++++++++++- esphome/helpers.py | 13 +- esphome/platformio/library.py | 393 +++++++++++++------- tests/unit_tests/test_espidf_component.py | 16 +- tests/unit_tests/test_espidf_framework.py | 186 ++++++++- tests/unit_tests/test_espota2.py | 8 +- tests/unit_tests/test_framework_helpers.py | 275 ++++++++++++++ tests/unit_tests/test_helpers.py | 14 + tests/unit_tests/test_platformio_library.py | 195 +++++++++- 10 files changed, 1242 insertions(+), 195 deletions(-) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 0f6ef873b8..179346e072 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -2,6 +2,7 @@ from collections.abc import Callable from ctypes.util import find_library +from functools import partial import json import logging import os @@ -20,9 +21,11 @@ from esphome.framework_helpers import ( create_venv, download_from_mirrors, download_with_resume, + failure_reason, get_python_env_executable_path, get_system_python_path, rmdir, + run_batch_downloads, run_command, run_command_ok, str_to_lst_of_str, @@ -690,6 +693,18 @@ def _patch_tools_json_demote_unused_tools(framework_path: Path) -> None: ) +def _download_tool( + dist_path: Path, entry: dict, tracker: Callable[[int], None] +) -> None: + download_with_resume( + entry["url"], + dist_path / entry["dest"], + sha256=entry["sha256"], + size=entry["size"], + progress=tracker, + ) + + def _prefetch_idf_tool_archives( framework_path: Path, targets_str: str, @@ -702,10 +717,10 @@ def _prefetch_idf_tool_archives( which makes large archives effectively impossible to fetch on unstable connections (#17703). This asks the framework's idf_tools (via ``get_tool_downloads.py``) which archives the coming install needs, then - downloads each into ``/dist`` with - ``download_with_resume``. The installer then finds the verified archives - already in place ("file ... is already downloaded") and never touches the - network. + downloads them into ``/dist`` with + ``download_with_resume``, a few at a time under one combined progress + bar. The installer then finds the verified archives already in place + ("file ... is already downloaded") and never touches the network. Strictly best-effort: any failure here just logs and returns, leaving ``idf_tools.py install`` to download whatever is missing exactly as @@ -727,30 +742,67 @@ def _prefetch_idf_tool_archives( ) return dist_path = get_idf_tools_path() / "dist" - entries = [ - entry - for entry in json.loads(stdout) - if not (dist_path / entry["dest"]).is_file() - ] - for index, entry in enumerate(entries, start=1): - _LOGGER.info( - "Downloading %s (%d/%d) ...", entry["name"], index, len(entries) - ) - try: - download_with_resume( - entry["url"], - dist_path / entry["dest"], - sha256=entry["sha256"], - size=entry["size"], + entries = [] + seen_dests: set[str] = set() + for entry in json.loads(stdout): + if (dist_path / entry["dest"]).is_file(): + continue + # Never download unverified: an entry without sha256/size is + # left to the installer, which fails loudly on a bad archive. + # Checked before the dedupe so it cannot shadow a verifiable + # duplicate of the same dest. + if not (entry.get("sha256") and entry.get("size")): + _LOGGER.warning( + "Tool %s has no sha256/size in the download list; " + "leaving it to the installer", + entry["name"], ) - except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught - # Keep prefetching the remaining archives; the installer - # will retry this one itself (without resume). - _LOGGER.warning("Could not prefetch %s: %s", entry["name"], e) + continue + if entry["dest"] in seen_dests: + # Two workers on one .part file would interleave + # seek/truncate writes; mirror the library prefetch's dedupe + continue + seen_dests.add(entry["dest"]) + entries.append(entry) + if not entries: + return + _LOGGER.info( + "Downloading %d ESP-IDF tool archive(s): %s", + len(entries), + ", ".join(entry["name"] for entry in entries), + ) + + # No sequential fallback here: skipping the prefetch would lose the + # resume workaround for #17703, and every entry has a size (above). + # A failed archive is retried by the installer itself (without + # resume); keep prefetching the rest. + failures = run_batch_downloads( + "Downloading ESP-IDF tools", + [ + ( + entry["name"], + entry["size"], + partial(_download_tool, dist_path, entry), + ) + for entry in entries + ], + ) + for name, e in failures: + # failure_reason: a message-less exception must not log blank + _LOGGER.warning("Could not prefetch %s: %s", name, failure_reason(e)) + _LOGGER.debug("Prefetch failure detail", exc_info=e) + if len(failures) == len(entries): + # A systematic fault, not one flaky mirror: the resume + # workaround (#17703) is off for this whole install + _LOGGER.error( + "Every ESP-IDF tool prefetch failed; the installer will " + "download without resume" + ) except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught # The installer downloads anything missing itself; never let the # prefetch become a new way for the install to fail. - _LOGGER.warning("ESP-IDF tool prefetch failed: %s", e) + _LOGGER.warning("ESP-IDF tool prefetch failed: %s", failure_reason(e)) + _LOGGER.debug("Prefetch failure detail", exc_info=True) def _check_esphome_idf_framework_install( diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 105791c518..2a2ce6dacf 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -1,7 +1,8 @@ """Generic toolchain installation helpers shared across framework implementations.""" -from collections.abc import Iterable -from contextlib import ExitStack +from collections.abc import Callable, Iterable, Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack, contextmanager, suppress import hashlib import io import json @@ -10,6 +11,7 @@ import os from pathlib import Path import subprocess import sys +import threading import time from typing import IO, TYPE_CHECKING @@ -24,6 +26,7 @@ PathType = str | os.PathLike _LOGGER = logging.getLogger(__name__) + # Attempts per mirror URL before falling through to the next mirror; only # mid-stream drops retry (resuming when the server gave a validator), # connect errors move on to the next mirror immediately. @@ -699,7 +702,11 @@ def _response_validator(resp: "requests.Response") -> str | None: def _stream_response_to_file( - resp: "requests.Response", f: IO[bytes], offset: int, size: int | None = None + resp: "requests.Response", + f: IO[bytes], + offset: int, + size: int | None = None, + progress: Callable[[int], None] | None = None, ) -> None: """Stream an open ``_open_ranged`` response body into ``f`` at ``offset``. @@ -707,21 +714,187 @@ def _stream_response_to_file( (effective offset 0) discards the stale bytes. ``offset`` also seeds the progress bar so a resumed download shows overall progress. ``size`` is the known full file size; when None it is derived from the response's - content-length, and without either there is no progress bar. + content-length, and without either there is no bar. With ``progress`` + set no bar is drawn here; the callback gets the absolute byte count. """ f.seek(offset) f.truncate(offset) total_size = size or offset + _content_length(resp) downloaded = offset - progress = ProgressBar("Downloading") if total_size > 0 else None + own_bar: ProgressBar | None = None + if progress is None: + own_bar = ProgressBar("Downloading") if total_size > 0 else None + progress = ( + (lambda done: own_bar.update(done / total_size)) + if own_bar + else (lambda _: None) + ) + progress(downloaded) for chunk in resp.iter_content(chunk_size=256 * 1024): if chunk: f.write(chunk) downloaded += len(chunk) - if progress is not None: - progress.update(downloaded / total_size) - if progress is not None: - progress.update(1) + progress(downloaded) + if own_bar is not None: + own_bar.update(1) + + +# Concurrent downloads per batch; enough to hide latency without +# hammering the host or the mirrors. +BATCH_DOWNLOAD_WORKERS = 4 + + +def run_batch_downloads( + header: str, + jobs: list[tuple[str, int, Callable[[Callable[[int], None]], None]]], + max_workers: int = BATCH_DOWNLOAD_WORKERS, +) -> list[tuple[str, BaseException]]: + """Run ``(name, size, fetch)`` download jobs concurrently under one bar. + + Each ``fetch(tracker)`` reports absolute byte counts; the bar total is + the sum of the sizes. Failures are returned after the bar is done so + warnings never land on its row. Ctrl-C drops queued jobs and aborts + in-flight ones at their next progress tick or backoff boundary (a + parked socket read defers that by its timeout, and an in-progress + archive extraction runs to completion); resumable destinations + (``download_with_resume``) keep their fetched ``.part`` bytes. + ``jobs`` must be non-empty. + """ + progress = _BatchDownloadProgress(header, sum(size for _, size, _ in jobs)) + cancelled = threading.Event() + + def _run( + name: str, fetch: Callable[[Callable[[int], None]], None] + ) -> tuple[str, BaseException] | None: + tracker = progress.tracker() + + def checked(done: int) -> None: + if cancelled.is_set(): + raise _BatchDownloadCancelled + tracker(done) + + try: + fetch(checked) + except (_BatchDownloadCancelled, Exception) as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # The cancelled arm exists for the tracker rollback below; the + # batch re-raises the interrupt, so the list is never returned + # after Ctrl-C. A bar-frame write failure must not displace the + # download error. + with suppress(Exception): + tracker(0) + failure = (name, err) + else: + failure = None + return failure + + ex = ThreadPoolExecutor(max_workers=max_workers) + try: + with progress.logging_guard(): + futures = [ex.submit(_run, name, fetch) for name, _, fetch in jobs] + return [failure for f in futures if (failure := f.result()) is not None] + except BaseException: + # Without this the non-daemon workers download to completion before + # the interpreter can exit, making Ctrl-C ineffective for minutes + cancelled.set() + raise + finally: + ex.shutdown(wait=True, cancel_futures=True) + progress.done() + + +class _BatchDownloadCancelled(BaseException): + """Raised inside a download job to abandon it after Ctrl-C. + + BaseException, like KeyboardInterrupt: a broad ``except Exception`` in + the download layers must not convert an abort into a retry. + """ + + +class _BatchDownloadProgress: + """One bar across several concurrent downloads, summing tracker bytes. + + The lock also serialises stderr writes so workers never interleave + frames; a ``total`` of 0 draws nothing. Call ``done()`` at the end so a + bar short of 100% still ends its line. + """ + + def __init__(self, header: str, total: int) -> None: + self._bar = ProgressBar(header) if total > 0 else None + self._total = total + self._sum = 0 + self._lock = threading.Lock() + + def tracker(self) -> Callable[[int], None]: + if self._bar is None: + return lambda _: None + last = 0 + + def update(done: int) -> None: + nonlocal last + with self._lock: + self._sum += done - last + last = done + # A bar-write failure (broken stderr pipe) must not surface + # as a download failure and cost the .part file + with suppress(Exception): + self._bar.update(min(self._sum / self._total, 1)) + + return update + + def done(self) -> None: + if self._bar is not None: + self._bar.done() + + @contextmanager + def logging_guard(self) -> Iterator[None]: + r"""End a partial bar row before any log record while active. + + Worker warnings (mirror retries) share stderr with the bar's \r + frames; without this the record lands mid-row and the next frame + overwrites it. A handler-level filter runs just before emit, so + only a tiny window remains for a concurrent frame. + """ + the_bar = self._bar + if the_bar is None: + yield + return + lock = self._lock + + class _EndRow(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + # Handler.handle() runs filters outside handleError's try; a + # stderr write failure must not escape through the log call + with lock, suppress(Exception): + the_bar.interrupt() + return True + + end_row = _EndRow() + handlers = logging.getLogger().handlers + for handler in handlers: + handler.addFilter(end_row) + try: + yield + finally: + for handler in handlers: + handler.removeFilter(end_row) + + +def _part_path(dest: Path) -> Path: + """The in-progress sidecar ``download_with_resume`` streams into.""" + return dest.with_name(dest.name + ".part") + + +def _cancellable_sleep( + delay: float, progress: Callable[[int], None] | None, done: int +) -> None: + """Backoff sleep that still observes a batch cancellation tick.""" + if progress is None: + time.sleep(delay) + return + end = time.monotonic() + delay + while (remaining := end - time.monotonic()) > 0: + progress(done) # raises when the batch was cancelled + time.sleep(min(0.5, remaining)) def download_with_resume( @@ -734,6 +907,7 @@ def download_with_resume( attempts: int = 5, timeout: int = 30, retry_connect_errors: bool = True, + progress: Callable[[int], None] | None = None, ) -> None: """Download ``url`` to ``dest``, resuming partial downloads. @@ -756,6 +930,9 @@ def download_with_resume( of consuming attempts — for callers with their own fallback, like ``download_from_mirrors``. + ``progress`` replaces the built-in bar: it receives the absolute bytes of + ``dest`` obtained so far (see ``_BatchDownloadProgress``). + Raises EsphomeError when all attempts are exhausted. """ # Imported lazily: requests is a heavy import (~85ms) and is only needed @@ -767,7 +944,7 @@ def download_with_resume( ensure_happy_eyeballs() dest = Path(dest) - part = dest.with_name(dest.name + ".part") + part = _part_path(dest) meta = part.with_name(part.name + ".meta") dest.parent.mkdir(parents=True, exist_ok=True) last_error: Exception | None = None @@ -779,6 +956,8 @@ def download_with_resume( if dest.is_file() and (sha256 is not None or size is not None): try: _verify_file(dest, sha256, size) + if progress is not None: + progress(size if size is not None else dest.stat().st_size) return except EsphomeError: dest.unlink() @@ -824,7 +1003,7 @@ def download_with_resume( # Recorded so a later run can prove an If-Range # resume of this part file safe. _write_download_meta(meta, url, validator, expected_total) - _stream_response_to_file(resp, f, offset, size) + _stream_response_to_file(resp, f, offset, size, progress) # else: a previous run already wrote every byte (or more) but # was killed before the rename below. Skip the network entirely # — a Range request past EOF would draw HTTP 416 — and let @@ -833,6 +1012,10 @@ def download_with_resume( expected_size = size if size is not None else expected_total _verify_file(part, sha256, expected_size or None) + if progress is not None: + # Also credits a part file an earlier run completed without + # streaming anything this time. + progress(expected_size or part.stat().st_size) if not expected_size and sha256 is None: # No sha, no size, and the server sent no usable # content-length: nothing can prove the download complete @@ -880,11 +1063,11 @@ def download_with_resume( raise EsphomeError( f"Failed to download {url} after {attempts} attempts: " - f"{_failure_reason(last_error)}" + f"{failure_reason(last_error)}" ) from last_error -def _failure_reason(e: Exception) -> str: +def failure_reason(e: BaseException) -> str: """Format a download exception for the aggregated error message. ``requests`` appends " for url: " to HTTP errors; the URL is already @@ -900,7 +1083,7 @@ def _spent_attempts_error(e: Exception, attempts: int) -> Exception: the sweep classifies it as permanent.""" from esphome.core import EsphomeError - err = EsphomeError(f"failed after {attempts} attempts: {_failure_reason(e)}") + err = EsphomeError(f"failed after {attempts} attempts: {failure_reason(e)}") err.__cause__ = e return err @@ -911,6 +1094,7 @@ def _try_mirrors_once( f: IO[bytes] | None, timeout: int, failures: list[tuple[str, Exception]], + progress: Callable[[int], None] | None = None, ) -> str | None: """Single pass over the resolved mirror ``urls``, one try per URL. @@ -939,6 +1123,7 @@ def _try_mirrors_once( # next mirror immediately; only mid-stream drops # retry-with-resume on the same URL. retry_connect_errors=False, + progress=progress, ) return url except (requests.RequestException, OSError, EsphomeError) as e: @@ -980,7 +1165,7 @@ def _try_mirrors_once( if offset == 0: validator = _response_validator(resp) expected_total = _content_length(resp) - _stream_response_to_file(resp, f, offset) + _stream_response_to_file(resp, f, offset, progress=progress) if expected_total and f.tell() != expected_total: raise EsphomeError( @@ -1029,6 +1214,7 @@ def download_from_mirrors( substitutions: dict[str, str], target: io.RawIOBase | IO[bytes] | PathType, timeout: int = 30, + progress: Callable[[int], None] | None = None, ) -> str: """ Download file from multiple mirrors with substitution support. @@ -1038,6 +1224,8 @@ def download_from_mirrors( substitutions: Dictionary of substitutions to apply to URLs target: Target file path or file-like object timeout: Download timeout in seconds + progress: Passed through to the download (see ``download_with_resume``); + replaces the built-in per-file bar Returns: The source URL. @@ -1102,7 +1290,9 @@ def download_from_mirrors( for sweep in range(1, _MIRROR_SWEEP_ATTEMPTS + 1): sweep_failures: list[tuple[str, Exception]] = [] if ( - url := _try_mirrors_once(urls, path_target, f, timeout, sweep_failures) + url := _try_mirrors_once( + urls, path_target, f, timeout, sweep_failures, progress + ) ) is not None: return url failures.extend(sweep_failures) @@ -1119,12 +1309,21 @@ def download_from_mirrors( _LOGGER.warning( "Download of %s failed (%s); retrying in %d seconds (attempt %d/%d)", transient[0], - _failure_reason(transient[1]), + failure_reason(transient[1]), delay, sweep + 1, _MIRROR_SWEEP_ATTEMPTS, ) - time.sleep(delay) + # Tick with the bytes already on disk so a combined bar holds + # steady during the backoff instead of rewinding to zero + done = 0 + if progress is not None: + if f is not None: + done = f.tell() + else: + part = _part_path(path_target) + done = part.stat().st_size if part.is_file() else 0 + _cancellable_sleep(delay, progress, done) # 4. Report every attempted URL if all mirrors failed. failures spans # all sweeps (deduplicated by URL and reason), so neither an early @@ -1133,7 +1332,7 @@ def download_from_mirrors( seen: set[tuple[str, str]] = set() attempts = "" for url, e in failures: - reason = _failure_reason(e) + reason = failure_reason(e) if (url, reason) not in seen: seen.add((url, reason)) attempts += f"\n {url}\n {reason}" diff --git a/esphome/helpers.py b/esphome/helpers.py index b3102ca277..d30e9b16a2 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -738,11 +738,22 @@ class ProgressBar: sys.stderr.flush() def done(self) -> None: - if not self.enabled: + # No frame drawn, or the 100% frame already ended its own line + if not self.enabled or self.last_progress is None or self.last_progress == 100: return sys.stderr.write("\n") sys.stderr.flush() + def interrupt(self) -> None: + """End a mid-row frame so the next write starts on its own row. + + The next ``update()`` redraws the bar; a finished bar stays done. + """ + if self.last_progress == 100: + return + self.done() + self.last_progress = None + def docs_url(path: str) -> str: """Return the URL to the documentation for a given path.""" diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index a3899fa860..bf9c323b84 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -15,6 +15,7 @@ regardless of which toolchain consumes the result. from collections import deque from collections.abc import Callable, Iterable from dataclasses import dataclass, field +from functools import partial import glob import hashlib import itertools @@ -30,7 +31,13 @@ from urllib.request import url2pathname from esphome import git from esphome.core import CORE, EsphomeError, Library -from esphome.framework_helpers import archive_extract_all, download_from_mirrors, rmdir +from esphome.framework_helpers import ( + archive_extract_all, + download_from_mirrors, + failure_reason, + rmdir, + run_batch_downloads, +) _LOGGER = logging.getLogger(__name__) @@ -70,6 +77,10 @@ SRC_FILE_EXTENSIONS = list(SOURCE_KIND_FOR_SUFFIX) DOMAIN = "pio_components" +# Marks a cache dir whose archive finished extracting; a missing marker +# means a torn extraction that must be redone +_EXTRACTED_MARKER = ".esphome_extracted" + ESPHOME_DATA_KEY = "ESPHOME" ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE" # Captured extra-script LINKFLAGS; kept apart from build.flags so they reach @@ -93,12 +104,13 @@ class Source: class URLSource(Source): - def __init__(self, url: str): + def __init__(self, url: str, size: int | None = None): self.url = url + # Archive size as reported by the registry, when known; sizes the + # combined prefetch bar without any extra network probe + self.size = size - def download( - self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = "" - ) -> Path: + def _cache_dir(self, dir_suffix: str, salt: str, namespace: str) -> Path: # Namespace the cache per backend (e.g. pio_components/idf, .../zephyr) so # the build files each backend writes into the library dir can't collide. base_dir = Path(CORE.data_dir) / DOMAIN @@ -108,22 +120,40 @@ class URLSource(Source): h.update(self.url.encode()) if salt: h.update(salt.encode()) - path = base_dir / h.hexdigest()[:8] / dir_suffix + return base_dir / h.hexdigest()[:8] / dir_suffix + + def is_cached(self, dir_suffix: str, salt: str = "", namespace: str = "") -> bool: + """Whether a completed extraction already exists for this source.""" + return ( + self._cache_dir(dir_suffix, salt, namespace) / _EXTRACTED_MARKER + ).is_file() + + def download( + self, + dir_suffix: str, + force: bool = False, + salt: str = "", + namespace: str = "", + progress: Callable[[int], None] | None = None, + ) -> Path: + path = self._cache_dir(dir_suffix, salt, namespace) # Marker file written last to signal a complete extraction. Using a # marker (instead of just `path.is_dir()`) means an interrupted # extraction is correctly detected and re-run on the next invocation, # and lets us extract directly into ``path`` — avoiding a # post-extraction rename that races with antivirus on Windows. - extracted_marker = path / ".esphome_extracted" + extracted_marker = path / _EXTRACTED_MARKER if not extracted_marker.is_file() or force: rmdir(path, msg=f"Clean up library directory {path}") # Download in temporary file with tempfile.NamedTemporaryFile() as tmp: - _LOGGER.info("Downloading %s ...", self.url) + if progress is None: + # A batch caller draws one combined bar and logs the list + _LOGGER.info("Downloading %s ...", self.url) _LOGGER.debug("Location: %s", path) - download_from_mirrors([self.url], {}, tmp.file) + download_from_mirrors([self.url], {}, tmp.file, progress=progress) _LOGGER.debug("Extracting archive to %s ...", path) archive_extract_all(tmp.file, path) @@ -415,6 +445,27 @@ def split_list_by_condition( return matched, non_matched +def _valid_manifest_shape(data: Any) -> bool: + """Whether the manifest has the dict shapes every backend dereferences. + + A bare json.load imposes no shape; validating once here means a + malformed third-party manifest fails by library name instead of a raw + TypeError/AttributeError in a backend. + """ + if not isinstance(data, dict): + return False + build = data.get("build", {}) + esphome_data = data.get(ESPHOME_DATA_KEY, {}) + return ( + isinstance(build, dict) + and isinstance(esphome_data, dict) + and isinstance(esphome_data.get(ESPHOME_DATA_LINK_FLAGS_KEY, []), list) + and isinstance(build.get("srcDir", ""), str) + and isinstance(build.get("includeDir", ""), str) + and isinstance(build.get("srcFilter", ""), (str, list)) + ) + + def check_library_data(data: dict, platform: str | None, framework: str): """ Check whether a library manifest is compatible with the target toolchain. @@ -537,9 +588,10 @@ def _make_registry_client() -> Any: def _resolve_registry_version( owner: str | None, pkgname: str, requirements: set[str] -) -> tuple[str, str, str, str]: +) -> tuple[str, str, str, str, int | None]: """Resolve a registry package to the single highest version satisfying ALL - the given requirements; return ``(owner, name, version, download_url)``. + the given requirements; return ``(owner, name, version, download_url, + size)`` (``size`` is None when the registry omits it). Intersecting every requirement (rather than resolving each consumer in isolation) makes the result independent of processing order and guarantees @@ -569,7 +621,7 @@ def _resolve_registry_version( pkgfile = registry.pick_compatible_pkg_file(best["files"]) if not pkgfile: raise RuntimeError(f"No package file for {owner}/{name}@{best['name']}") - return owner, name, best["name"], pkgfile["download_url"] + return owner, name, best["name"], pkgfile["download_url"], pkgfile.get("size") def split_flag_entry(entry: Any, owner: str) -> list[str]: @@ -859,6 +911,85 @@ def is_lib_ignored(name: str | None, lib_ignore: set[str]) -> bool: ) +def _fetch_source( + component: ConvertedLibrary, + salt: str, + namespace: str, + tracker: Callable[[int], None], +) -> None: + # Straight to URLSource: only it takes progress, and mutating the + # shared component from a worker is the authoritative loop's job + component.source.download( + component.get_sanitized_name(), salt=salt, namespace=namespace, progress=tracker + ) + + +def _prefetch_wave( + wave: list[tuple[str, ConvertedLibrary]], salt: str, namespace: str +) -> None: + """Best-effort parallel download of a wave's registry archives. + + The walk's own ``download()`` stays authoritative; duplicate URLs + prefetch once so two threads never share a cache directory. Archives + whose size the registry did not report are left to the sequential + loop, whose per-file bars don't interleave. A node a sibling in the + same wave supersedes has its archive fetched in vain (knowing better + would need the manifests being downloaded). + """ + try: + components: list[ConvertedLibrary] = [] + seen: set[str] = set() + for _key, component in wave: + source = component.source + if not isinstance(source, URLSource) or not source.size: + continue + if source.url in seen: + continue + seen.add(source.url) + try: + cached = source.is_cached( + component.get_sanitized_name(), salt=salt, namespace=namespace + ) + except OSError as err: + # Best-effort, but visibly: a systematic probe failure makes + # every warm build re-download every archive + _LOGGER.warning("Cache probe for %s failed: %s", component.name, err) + cached = False + if cached: + # A warm build must stay silent + continue + components.append(component) + if not components: + return + # Single-item waves (a dependency chain discovers one archive per + # wave) go through the same runner: one download method, one bar + _LOGGER.info( + "Downloading %d library archive(s): %s", + len(components), + ", ".join(c.name for c in components), + ) + failures = run_batch_downloads( + "Downloading libraries", + [ + (c.name, c.source.size, partial(_fetch_source, c, salt, namespace)) + for c in components + ], + ) + for name, err in failures: + # The sequential call below retries and raises the real error + _LOGGER.warning( + "Prefetch of %s failed (retrying sequentially): %s", + name, + failure_reason(err), + ) + _LOGGER.debug("Prefetch failure detail", exc_info=err) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # Same policy as the ESP-IDF twin: the prefetch must never become a + # new way for the build to fail + _LOGGER.warning("Library prefetch failed: %s", failure_reason(err)) + _LOGGER.debug("Prefetch failure detail", exc_info=True) + + def convert_libraries( libraries: list[Library], backend: LibraryBackend ) -> list[ConvertedLibrary]: @@ -955,136 +1086,134 @@ def convert_libraries( top_level_keys = set(top_level) worklist = deque(dict.fromkeys(top_level)) while worklist: - key = worklist.popleft() - node = nodes[key] + # Drain the frontier sequentially (spec resolution mutates shared + # state), then prefetch the wave in parallel + wave: list[tuple[str, ConvertedLibrary]] = [] + while worklist: + key = worklist.popleft() + node = nodes[key] - # Re-resolve only when the requirement set grew; requirements - # only ever grow, so the fixpoint converges and cycles terminate - requirements = frozenset(node.requirements) - if resolved_requirements.get(key) == requirements: - continue - resolved_requirements[key] = requirements + # Re-resolve only when the requirement set grew; requirements + # only ever grow, so the fixpoint converges and cycles terminate + requirements = frozenset(node.requirements) + if resolved_requirements.get(key) == requirements: + continue + resolved_requirements[key] = requirements - if node.is_git: - component = ConvertedLibrary(key, "*", GitSource(node.url, node.ref)) - elif node.is_local: - component = ConvertedLibrary(key, "*", LocalSource(node.local_path)) - else: - owner, name, version, url = _resolve_registry_version( - node.owner, node.pkgname, node.requirements - ) - component = ConvertedLibrary( - _owner_pkgname_to_name(owner, name), version, URLSource(url) - ) - component.download(salt=salt, namespace=backend.cache_key) + if node.is_git: + component = ConvertedLibrary(key, "*", GitSource(node.url, node.ref)) + elif node.is_local: + component = ConvertedLibrary(key, "*", LocalSource(node.local_path)) + else: + owner, name, version, url, size = _resolve_registry_version( + node.owner, node.pkgname, node.requirements + ) + component = ConvertedLibrary( + _owner_pkgname_to_name(owner, name), version, URLSource(url, size) + ) + wave.append((key, component)) + _prefetch_wave(wave, salt, backend.cache_key) + for key, component in wave: + node = nodes[key] + if frozenset(node.requirements) != resolved_requirements[key]: + # Requirements grew mid-wave: skip parsing a manifest the + # next wave will re-resolve and replace + worklist.append(key) + continue + component.download(salt=salt, namespace=backend.cache_key) - source_dir = component.source_dir - library_json_path = source_dir / "library.json" - library_properties_path = source_dir / "library.properties" - has_json = library_json_path.is_file() - has_properties = library_properties_path.is_file() - if not has_json and not has_properties and not node.is_local: - # An interrupted clone/extraction self-heals with one forced - # re-download; a local source has nothing to re-download - _LOGGER.warning( - "Library %s at %s is missing library.json and library.properties; " - "re-downloading", - key, - source_dir, - ) - component.download(force=True, salt=salt, namespace=backend.cache_key) + source_dir = component.source_dir + library_json_path = source_dir / "library.json" + library_properties_path = source_dir / "library.properties" has_json = library_json_path.is_file() has_properties = library_properties_path.is_file() - if has_json: - component.data = parse_library_json(library_json_path) - elif has_properties: - component.data = parse_library_properties(library_properties_path) - else: - # Local sources are user input (EsphomeError); a registry/git - # miss means a corrupt cache (RuntimeError) - error_cls = EsphomeError if node.is_local else RuntimeError - raise error_cls( - f"Invalid PIO library {key}: missing library.json and " - f"library.properties in {source_dir}" - ) - - # A bare json.load imposes no shape; every backend dereferences - # these fields, so validate once here and name the library - malformed = not isinstance(component.data, dict) - if not malformed: - build = component.data.get("build", {}) - esphome_data = component.data.get(ESPHOME_DATA_KEY, {}) - malformed = ( - not isinstance(build, dict) - or not isinstance(esphome_data, dict) - or not isinstance( - esphome_data.get(ESPHOME_DATA_LINK_FLAGS_KEY, []), list + if not has_json and not has_properties and not node.is_local: + # An interrupted clone/extraction self-heals with one forced + # re-download; a local source has nothing to re-download + _LOGGER.warning( + "Library %s at %s is missing library.json and library.properties; " + "re-downloading", + key, + source_dir, ) - or not isinstance(build.get("srcDir", ""), str) - or not isinstance(build.get("includeDir", ""), str) - or not isinstance(build.get("srcFilter", ""), (str, list)) - ) - if malformed: - # Fail fast only for a library the user asked for; a defect in - # an unrequested corner of the graph must not block the build - if key in top_level_keys: - raise EsphomeError(f"Library {key} has a malformed manifest") - _LOGGER.warning("Skipping dependency %s: malformed manifest", key) - continue - warn_properties_depends(component.name, component.data) - - try: - check_library_data(component.data, backend.platform, backend.framework) - except InvalidLibrary as e: - # An explicitly requested library fails fast; the routine - # cross-platform skip stays at debug, other causes warn - if key in top_level_keys: - reason = ( - f"is not compatible with {backend.framework}" - if isinstance(e, IncompatiblePlatform) - else "has a malformed manifest" - ) - raise RuntimeError(f"Requested library {key} {reason}: {e}") from e - if isinstance(e, IncompatiblePlatform): - _LOGGER.debug("Skip incompatible dependency %s: %s", key, str(e)) + component.download(force=True, salt=salt, namespace=backend.cache_key) + has_json = library_json_path.is_file() + has_properties = library_properties_path.is_file() + if has_json: + component.data = parse_library_json(library_json_path) + elif has_properties: + component.data = parse_library_properties(library_properties_path) else: - _LOGGER.warning("Skipping dependency %s: %s", key, str(e)) - continue - components[key] = component - - # Requirements changed (we got past the short-circuit above), so - # (re)walk this component's dependencies. - node.edges = set() - for dependency in normalize_dependencies( - component.data.get("dependencies"), component.name - ): - if "version" not in dependency: - # Cannot resolve from the registry; common for bundled - # names (Wire, SPI) -- unactionable noise above debug - _LOGGER.debug( - "Skip version-less dependency %r of %s", - dependency.get("name"), - component.name, + # Local sources are user input (EsphomeError); a registry/git + # miss means a corrupt cache (RuntimeError) + error_cls = EsphomeError if node.is_local else RuntimeError + raise error_cls( + f"Invalid PIO library {key}: missing library.json and " + f"library.properties in {source_dir}" ) + + if not _valid_manifest_shape(component.data): + # Fail fast only for a library the user asked for; a defect + # in an unrequested corner of the graph must not block the + # build + if key in top_level_keys: + raise EsphomeError(f"Library {key} has a malformed manifest") + _LOGGER.warning("Skipping dependency %s: malformed manifest", key) continue - if not dependency_is_usable( - dependency, backend.platform, backend.framework, component.name + warn_properties_depends(component.name, component.data) + + try: + check_library_data(component.data, backend.platform, backend.framework) + except InvalidLibrary as e: + # An explicitly requested library fails fast; the routine + # cross-platform skip stays at debug, other causes warn + if key in top_level_keys: + reason = ( + f"is not compatible with {backend.framework}" + if isinstance(e, IncompatiblePlatform) + else "has a malformed manifest" + ) + raise RuntimeError(f"Requested library {key} {reason}: {e}") from e + if isinstance(e, IncompatiblePlatform): + _LOGGER.debug("Skip incompatible dependency %s: %s", key, str(e)) + else: + _LOGGER.warning("Skipping dependency %s: %s", key, str(e)) + continue + components[key] = component + + # Requirements changed (we got past the short-circuit above), so + # (re)walk this component's dependencies. + node.edges = set() + for dependency in normalize_dependencies( + component.data.get("dependencies"), component.name ): - continue - dep_name = _owner_pkgname_to_name( - dependency.get("owner"), dependency.get("name") - ) - if is_lib_ignored(dep_name, lib_ignore): - _LOGGER.debug("Skip ignored dependency %s", dep_name) - continue - # The version field may actually be a URL (git/archive dependency). - dep_version = dependency["version"] - dep_url = _url_or_none(dep_version) - if dep_url is not None: - dep_version = None - dep_key = add_spec(dep_name, dep_version, dep_url) - node.edges.add(dep_key) - worklist.append(dep_key) + if "version" not in dependency: + # Cannot resolve from the registry; common for bundled + # names (Wire, SPI) -- unactionable noise above debug + _LOGGER.debug( + "Skip version-less dependency %r of %s", + dependency.get("name"), + component.name, + ) + continue + if not dependency_is_usable( + dependency, backend.platform, backend.framework, component.name + ): + continue + dep_name = _owner_pkgname_to_name( + dependency.get("owner"), dependency.get("name") + ) + if is_lib_ignored(dep_name, lib_ignore): + _LOGGER.debug("Skip ignored dependency %s", dep_name) + continue + # The version field may actually be a URL (git/archive dependency). + dep_version = dependency["version"] + dep_url = _url_or_none(dep_version) + if dep_url is not None: + dep_version = None + dep_key = add_spec(dep_name, dep_version, dep_url) + node.edges.add(dep_key) + worklist.append(dep_key) # A git or local source wins over the same component requested from the # registry. That's intentional, but warn so the dropped registry spec isn't diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 7b4f848979..3789eefc64 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -660,7 +660,7 @@ def _patch_registry(monkeypatch, versions): def test_resolve_registry_version_intersects_constraints(monkeypatch): _patch_registry(monkeypatch, ["1.10018.1", "1.10021.0", "1.10021.1"]) - owner, name, version, url = _resolve_registry_version( + owner, name, version, url, _size = _resolve_registry_version( "esphome", "libsodium", {"==1.10021.0", "^1.10018.1"} ) assert (owner, name, version) == ("esphome", "libsodium", "1.10021.0") @@ -669,7 +669,9 @@ def test_resolve_registry_version_intersects_constraints(monkeypatch): def test_resolve_registry_version_picks_highest_satisfying(monkeypatch): _patch_registry(monkeypatch, ["1.0.0", "1.5.0", "2.0.0"]) - _owner, _name, version, _url = _resolve_registry_version("o", "p", {"^1.0.0"}) + _owner, _name, version, _url, _size = _resolve_registry_version( + "o", "p", {"^1.0.0"} + ) assert version == "1.5.0" @@ -719,7 +721,7 @@ def test_generate_idf_components_dedupes_shared_dependency( resolve_calls.append(pkgname) captured[f"{owner}/{pkgname}"] = set(requirements) version = "1.10021.0" if pkgname == "C" else "1.0.0" - return owner, pkgname, version, f"http://x/{pkgname}.tar.gz" + return owner, pkgname, version, f"http://x/{pkgname}.tar.gz", None monkeypatch.setattr( esphome.platformio.library, "_resolve_registry_version", fake_resolve @@ -778,7 +780,7 @@ def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies( def fake_resolve(owner, pkgname, requirements): resolve_calls.append(pkgname) - return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz" + return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None monkeypatch.setattr( esphome.platformio.library, "_resolve_registry_version", fake_resolve @@ -834,6 +836,7 @@ def test_generate_idf_components_handles_dependency_cycle( pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", + None, ), ) @@ -891,6 +894,7 @@ def test_generate_idf_components_git_overrides_registry_warns( pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", + None, ), ) @@ -927,6 +931,7 @@ def test_generate_idf_components_missing_manifest_raises( pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", + None, ), ) @@ -971,6 +976,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate( pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", + None, ), ) @@ -1004,6 +1010,7 @@ def test_generate_idf_components_incompatible_top_level_raises( pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", + None, ), ) @@ -1040,6 +1047,7 @@ def test_generate_idf_components_incompatible_dependency_skipped( pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", + None, ), ) diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index d8e7738569..6288933a6a 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -2,6 +2,7 @@ # pylint: disable=protected-access +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager import importlib.util import io @@ -14,7 +15,7 @@ import subprocess import sys import tarfile from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest @@ -887,6 +888,78 @@ _PREFETCH_JSON = json.dumps( ) +def test_prefetch_leaves_unverifiable_entries_to_the_installer( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An entry missing sha256 or size must not download unverified; the + installer handles it and fails loudly on a bad archive.""" + entries = json.loads(_PREFETCH_JSON) + del entries[0]["sha256"] + del entries[1]["size"] + entries.append( + { + "name": "gcc@14.2.0", + "url": "https://example.com/gcc.tar.gz", + "size": 67, + "sha256": "ef" * 32, + "dest": "gcc.tar.gz", + } + ) + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, json.dumps(entries), ""), + ), + patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls, + ): + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + assert [call[0][0] for call in download.call_args_list] == [ + "https://example.com/gcc.tar.gz" + ] + assert download.call_args[1]["sha256"] == "ef" * 32 + progress_cls.assert_called_once_with("Downloading ESP-IDF tools", 67) + assert "cmake@3.30.2 has no sha256/size" in caplog.text + assert "ninja@1.12.1 has no sha256/size" in caplog.text + + +def test_prefetch_all_entries_unverifiable_is_a_noop(tmp_path: Path) -> None: + entries = json.loads(_PREFETCH_JSON) + for entry in entries: + del entry["sha256"] + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, json.dumps(entries), ""), + ), + patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + ): + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + download.assert_not_called() + + +def test_prefetch_dedupes_entries_by_dest(tmp_path: Path) -> None: + """Two entries resolving to one dest would interleave writes into the + same .part file; only the first downloads.""" + entries = json.loads(_PREFETCH_JSON) + dup = dict(entries[0]) | {"name": "cmake-alias@3.30.2"} + entries.append(dup) + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, json.dumps(entries), ""), + ), + patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + patch("esphome.framework_helpers._BatchDownloadProgress"), + ): + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + dests = [call[0][1].name for call in download.call_args_list] + assert dests.count("cmake-3.30.2.tar.gz") == 1 + + def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None: with ( patch( @@ -895,16 +968,58 @@ def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None: ), patch("esphome.espidf.framework.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls, ): + # Materialize the lazy mock before threads race its first creation + tracker = progress_cls.return_value.tracker.return_value _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) dist = get_idf_tools_path() / "dist" - assert download.call_count == 2 - assert download.call_args_list[0][0] == ( - "https://example.com/cmake.tar.gz", - dist / "cmake-3.30.2.tar.gz", - ) - assert download.call_args_list[0][1] == {"sha256": "ab" * 32, "size": 123} + # Archives download concurrently, so the call order is not fixed. + calls = {call[0]: call[1] for call in download.call_args_list} + assert set(calls) == { + ("https://example.com/cmake.tar.gz", dist / "cmake-3.30.2.tar.gz"), + ("https://example.com/ninja.zip", dist / "ninja.zip"), + } + kwargs = calls[("https://example.com/cmake.tar.gz", dist / "cmake-3.30.2.tar.gz")] + assert kwargs["sha256"] == "ab" * 32 + assert kwargs["size"] == 123 + # every archive reports into the one combined progress bar via the + # cancellation-checked wrapper; verify it delegates to the tracker + progress_cls.assert_called_once_with("Downloading ESP-IDF tools", 123 + 45) + before = tracker.call_count + for kw in calls.values(): + kw["progress"](7) + assert tracker.call_count == before + len(calls) + + +def test_prefetch_downloads_archives_concurrently(tmp_path: Path) -> None: + """More than one archive fans out over a bounded thread pool.""" + entries = [ + { + "name": f"tool{i}@1", + "url": f"https://example.com/tool{i}.tar.gz", + "size": 10, + "sha256": "ab" * 32, + "dest": f"tool{i}.tar.gz", + } + for i in range(6) + ] + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, json.dumps(entries), ""), + ), + patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + patch( + "esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor + ) as pool, + ): + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + + pool.assert_called_once_with(max_workers=4) + assert download.call_count == 6 def test_prefetch_skips_already_downloaded_archives(tmp_path: Path) -> None: @@ -959,11 +1074,12 @@ def test_prefetch_failures_never_raise( assert expected_log in caplog.text -def test_prefetch_one_failed_archive_does_not_stop_the_rest( +def test_prefetch_total_failure_logs_error( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: - """A single archive failing its download must not abort the prefetch of - the remaining archives.""" + """Every archive failing is a systematic fault (proxy, bad kwarg), not + a flaky mirror; it must be distinguishable at ERROR because the resume + workaround is off for the whole install.""" with ( patch( "esphome.espidf.framework.run_command", @@ -971,7 +1087,32 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest( ), patch( "esphome.espidf.framework.download_with_resume", - side_effect=[OSError("network down"), None], + side_effect=OSError("proxy refuses everything"), + ), + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + ): + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + assert "Every ESP-IDF tool prefetch failed" in caplog.text + + +def test_prefetch_one_failed_archive_does_not_stop_the_rest( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A single archive failing its download must not abort the prefetch of + the remaining archives.""" + + def _fail_cmake_download(url: str, *args, **kwargs) -> None: + if "cmake" in url: + raise OSError("network down") + + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, _PREFETCH_JSON, ""), + ), + patch( + "esphome.espidf.framework.download_with_resume", + side_effect=_fail_cmake_download, ) as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), ): @@ -979,6 +1120,29 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest( assert download.call_count == 2 assert "Could not prefetch cmake@3.30.2" in caplog.text + # One flaky archive is routine, never the systematic-fault ERROR + assert "Every ESP-IDF tool prefetch failed" not in caplog.text + + +def test_prefetch_finishes_progress_bar_and_cancels_queue(tmp_path: Path) -> None: + """The batch bar is closed out after the pool, and the pool is shut down + with cancel_futures so Ctrl-C does not drain every queued archive.""" + with ( + patch( + "esphome.espidf.framework.run_command", + return_value=(True, _PREFETCH_JSON, ""), + ), + patch("esphome.espidf.framework.download_with_resume"), + patch("esphome.espidf.framework.get_system_python_path", return_value="python"), + patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls, + patch("esphome.framework_helpers.ThreadPoolExecutor") as pool_cls, + ): + pool = MagicMock(wraps=ThreadPoolExecutor(max_workers=2)) + pool_cls.return_value = pool + _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) + + pool.shutdown.assert_called_once_with(wait=True, cancel_futures=True) + progress_cls.return_value.done.assert_called_once_with() def test_prefetch_passes_targets_and_tools_to_script(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index e0e9185e1c..8867e2c215 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -998,10 +998,10 @@ def test_progress_bar(capsys: CaptureFixture[str]) -> None: assert "100%" in captured.err assert "Done" in captured.err - # Test done method + # done() after the 100% frame adds nothing; that frame ended its line progress.done() captured = capsys.readouterr() - assert captured.err == "\n" + assert captured.err == "" # Test same progress doesn't update progress.update(0.5) @@ -1010,6 +1010,10 @@ def test_progress_bar(capsys: CaptureFixture[str]) -> None: # Should only see one update (second call shouldn't write) assert captured.err.count("50%") == 1 + # done() after a mid-way frame ends the line + progress.done() + assert capsys.readouterr().err == "\n" + # Tests for SHA256 authentication @pytest.mark.usefixtures("mock_time") diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 500705ef67..5916a2fd60 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -12,6 +12,8 @@ from pathlib import Path import subprocess import sys import tarfile +import threading +import time from unittest.mock import MagicMock, Mock, call, patch import zipfile @@ -22,6 +24,7 @@ from esphome import framework_helpers from esphome.core import EsphomeError from esphome.framework_helpers import ( _7z_extract_all, + _BatchDownloadProgress, _detect_archive_root, _rename_with_retry, _tar_extract_all, @@ -36,6 +39,7 @@ from esphome.framework_helpers import ( get_python_env_executable_path, get_system_python_path, rmdir, + run_batch_downloads, run_command, run_command_ok, str_to_lst_of_str, @@ -1111,6 +1115,218 @@ class TestDownloadWithResume: assert mock_get.call_args[1]["headers"] == {} assert dest.read_bytes() == b"data" + def test_progress_callback_reports_absolute_bytes(self, tmp_path: Path) -> None: + """With a callback no bar is drawn; the callback sees the running + byte count of this file, then its final verified size.""" + dest = tmp_path / "tool.tar.gz" + resp = _mock_response(b"") + resp.headers = {"content-length": "7"} + resp.iter_content.return_value = [b"1234", b"567"] + seen: list[int] = [] + with ( + patch("requests.get", return_value=resp), + patch("esphome.framework_helpers.ProgressBar") as bar_cls, + ): + download_with_resume( + "https://example.com/t", dest, size=7, progress=seen.append + ) + assert seen == [0, 4, 7, 7] + bar_cls.assert_not_called() + + def test_progress_callback_seeds_with_resume_offset(self, tmp_path: Path) -> None: + dest = tmp_path / "tool.tar.gz" + (tmp_path / "tool.tar.gz.part").write_bytes(b"12345") + good = hashlib.sha256(b"12345678").hexdigest() + seen: list[int] = [] + with patch("requests.get", return_value=_resumed_response(b"678")): + download_with_resume( + "https://example.com/t", dest, sha256=good, size=8, progress=seen.append + ) + assert seen[0] == 5 + assert seen[-1] == 8 + + def test_progress_callback_credits_already_complete_download( + self, tmp_path: Path + ) -> None: + """A verified dest from an earlier run still counts toward the batch.""" + dest = tmp_path / "tool.tar.gz" + dest.write_bytes(b"12345678") + seen: list[int] = [] + with patch("requests.get") as mock_get: + download_with_resume( + "https://example.com/t", dest, size=8, progress=seen.append + ) + mock_get.assert_not_called() + assert seen == [8] + + +def test_run_batch_downloads_ctrl_c_aborts_in_flight_jobs() -> None: + """Ctrl-C cancels in-flight downloads at their next tick instead of + letting non-daemon workers download to completion.""" + started = threading.Event() + ticks: list[int] = [] + + def interrupter(tracker) -> None: + started.wait(5) + raise KeyboardInterrupt + + def slow_download(tracker) -> None: + started.set() + for i in range(500): + tracker(i) + ticks.append(i) + time.sleep(0.01) + + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + run_batch_downloads( + "Downloading", + [("boom", 0, interrupter), ("slow", 0, slow_download)], + max_workers=2, + ) + # Uncancelled, slow_download alone takes ~5s + assert time.monotonic() - t0 < 3 + assert len(ticks) < 500 + + +def test_cancellation_escapes_broad_except_in_fetch() -> None: + """A fetch that wraps its work in except Exception cannot swallow the + Ctrl-C sentinel (it is a BaseException).""" + from esphome.framework_helpers import _BatchDownloadCancelled + + started = threading.Event() + swallowed = [] + + def interrupter(tracker) -> None: + started.wait(5) + raise KeyboardInterrupt + + def greedy_fetch(tracker) -> None: + started.set() + try: + for i in range(500): + tracker(i) + time.sleep(0.01) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + swallowed.append(err) + + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + run_batch_downloads( + "Downloading", + [("boom", 0, interrupter), ("greedy", 0, greedy_fetch)], + max_workers=2, + ) + assert time.monotonic() - t0 < 3 + assert not swallowed + assert issubclass(_BatchDownloadCancelled, BaseException) + assert not issubclass(_BatchDownloadCancelled, Exception) + + +def test_logging_guard_ends_the_bar_row_before_a_record() -> None: + r"""A worker warning gets its own line instead of the bar's \r row.""" + stream = io.StringIO() + stream.isatty = lambda: True # type: ignore[method-assign] + with patch("esphome.helpers.sys.stderr", stream): + progress = _BatchDownloadProgress("Downloading", 10) + progress.tracker()(5) + with progress.logging_guard(): + logging.getLogger("esphome.test").warning("mirror retry") + # The partial 50% frame ended its line before the record was emitted + assert stream.getvalue().endswith("50% \n") + # And the next tick redraws the frame on a fresh row + progress.tracker()(2) + assert stream.getvalue().endswith("70% ") + + +def test_logging_guard_without_a_bar_is_a_no_op() -> None: + """An unknown total draws no bar; the guard passes records through.""" + progress = _BatchDownloadProgress("Downloading", 0) + with progress.logging_guard(): + logging.getLogger("esphome.test").warning("plain record") + + +def test_cancellable_sleep_sleeps_between_ticks() -> None: + """An uncancelled backoff actually waits out its delay in slices.""" + from esphome.framework_helpers import _cancellable_sleep + + ticks: list[int] = [] + t0 = time.monotonic() + _cancellable_sleep(0.05, ticks.append, 3) + assert time.monotonic() - t0 >= 0.05 + assert ticks and all(t == 3 for t in ticks) + + +def test_cancellable_sleep_aborts_at_the_tick() -> None: + """A backoff sleep observes the cancellation raise promptly.""" + from esphome.framework_helpers import _BatchDownloadCancelled, _cancellable_sleep + + def cancelled_tick(done: int) -> None: + raise _BatchDownloadCancelled + + t0 = time.monotonic() + with pytest.raises(_BatchDownloadCancelled): + _cancellable_sleep(30, cancelled_tick, 0) + assert time.monotonic() - t0 < 1 + + +class Test_BatchDownloadProgress: + def test_sums_trackers_into_one_bar(self) -> None: + with patch("esphome.framework_helpers.ProgressBar") as bar_cls: + progress = _BatchDownloadProgress("Downloading", 100) + a = progress.tracker() + b = progress.tracker() + a(10) + b(20) + a(30) + a(0) # a restart from zero takes that file's bytes back out + bar_cls.assert_called_once_with("Downloading") + updates = [c[0][0] for c in bar_cls.return_value.update.call_args_list] + assert updates == [0.1, 0.3, 0.5, 0.2] + + def test_clamps_at_one(self) -> None: + """Sizes are advisory; an over-delivering server never pushes past 100%.""" + with patch("esphome.framework_helpers.ProgressBar") as bar_cls: + progress = _BatchDownloadProgress("Downloading", 10) + progress.tracker()(25) + assert bar_cls.return_value.update.call_args[0][0] == 1 + + def test_unknown_total_draws_nothing(self) -> None: + with patch("esphome.framework_helpers.ProgressBar") as bar_cls: + progress = _BatchDownloadProgress("Downloading", 0) + progress.tracker()(5) + progress.done() + bar_cls.assert_not_called() + + def test_done_ends_an_unfinished_bar(self) -> None: + """A batch that stops short of 100% (a failed archive) still ends its + line so the next log message starts on a fresh row.""" + stream = io.StringIO() + stream.isatty = lambda: True # type: ignore[method-assign] + with patch("esphome.helpers.sys.stderr", stream): + progress = _BatchDownloadProgress("Downloading", 10) + progress.tracker()(5) + progress.done() + assert stream.getvalue().endswith("50% \n") + + def test_done_before_any_frame_writes_nothing(self) -> None: + """A batch aborted before any tracker fired must not emit a stray + newline for a bar that was never drawn.""" + stream = io.StringIO() + stream.isatty = lambda: True # type: ignore[method-assign] + with patch("esphome.helpers.sys.stderr", stream): + _BatchDownloadProgress("Downloading", 10).done() + assert stream.getvalue() == "" + + def test_done_after_full_bar_adds_nothing(self) -> None: + stream = io.StringIO() + stream.isatty = lambda: True # type: ignore[method-assign] + with patch("esphome.helpers.sys.stderr", stream): + progress = _BatchDownloadProgress("Downloading", 10) + progress.tracker()(10) + progress.done() + assert stream.getvalue().endswith("100% Done...\r\n") + class TestDownloadFromMirrors: def test_success_returns_url_and_writes_content(self, tmp_path: Path) -> None: @@ -1123,6 +1339,22 @@ class TestDownloadFromMirrors: assert url == "https://example.com/f" assert target.read_bytes() == b"filedata" + def test_file_object_target_reports_progress(self) -> None: + """The library prefetch's production path: a file-object target + streams through the mirror fallback and ticks the tracker.""" + buf = io.BytesIO() + ticks: list[int] = [] + with patch( + "requests.get", + return_value=_mock_response(b"filedata"), + ): + url = download_from_mirrors( + ["https://example.com/f"], {}, buf, progress=ticks.append + ) + assert url == "https://example.com/f" + assert buf.getvalue() == b"filedata" + assert ticks and ticks[-1] == len(b"filedata") + def test_substitutions_applied_to_url(self, tmp_path: Path) -> None: with patch( "requests.get", @@ -1468,6 +1700,49 @@ class TestDownloadFromMirrors: assert mock_get.call_count == 2 mock_sleep.assert_called_once_with(2) + def test_backoff_tick_reports_filelike_bytes(self) -> None: + """For a file-like target the backoff tick carries f.tell(), so the + combined bar holds steady through the sweep retry.""" + target = io.BytesIO() + ticks: list[int] = [] + with ( + patch( + "requests.get", + side_effect=[ + req.ConnectionError("down"), + _mock_response(b"data"), + ], + ), + patch("esphome.framework_helpers._cancellable_sleep") as mock_sleep, + ): + download_from_mirrors( + ["https://mirror1.com/f"], {}, target, progress=ticks.append + ) + # No bytes had streamed at backoff time, so the tick carries 0 + assert mock_sleep.call_args == call(2, ticks.append, 0) + assert target.getvalue() == b"data" + + def test_backoff_tick_reports_partial_bytes(self, tmp_path: Path) -> None: + """The backoff tick carries the bytes already in the part file, so a + combined bar holds steady instead of rewinding to zero.""" + dest = tmp_path / "out.bin" + (tmp_path / "out.bin.part").write_bytes(b"12345") + ticks: list[int] = [] + with ( + patch( + "requests.get", + side_effect=[ + req.ConnectionError("down"), + _mock_response(b"data"), + ], + ), + patch("esphome.framework_helpers._cancellable_sleep") as mock_sleep, + ): + download_from_mirrors( + ["https://mirror1.com/f"], {}, dest, progress=ticks.append + ) + assert mock_sleep.call_args == call(2, ticks.append, 5) + def test_permanent_failure_does_not_retry_sweep(self, tmp_path: Path) -> None: """An HTTP 404 will not heal on its own; fail after a single pass.""" with ( diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index 3160469063..683fef22cf 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -1124,6 +1124,20 @@ def test_progressbar_enabled_on_pipe_with_dashboard(monkeypatch) -> None: assert bar.enabled is True +def test_progressbar_interrupt_keeps_finished_bar_done(monkeypatch) -> None: + """interrupt() on a bar whose 100% frame already ended its own line + must not reset it, or the next tick would redraw a second Done row.""" + stream = MagicMock(spec=io.TextIOWrapper) + stream.isatty.return_value = True + monkeypatch.setattr(CORE, "dashboard", False) + + bar = ProgressBar("Uploading", stream=stream) + bar.update(1) + assert bar.last_progress == 100 + bar.interrupt() + assert bar.last_progress == 100 + + @pytest.mark.parametrize( ("seconds", "expected"), [ diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index bf8340cea0..792d7dab61 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -153,13 +153,15 @@ def test_localsource_download_returns_empty_build_dir(setup_core: Path) -> None: assert plain != out -def test_urlsource_download_extracts_then_reuses_marker(setup_core, monkeypatch): +def test_urlsource_download_extracts_then_reuses_marker( + setup_core, monkeypatch, caplog +): monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None) dl_calls: list[list[str]] = [] monkeypatch.setattr( lib, "download_from_mirrors", - lambda urls, headers, f: dl_calls.append(urls), + lambda urls, headers, f, progress=None: dl_calls.append(urls), ) def fake_extract(fileobj, path): @@ -178,6 +180,12 @@ def test_urlsource_download_extracts_then_reuses_marker(setup_core, monkeypatch) assert out2 == out assert len(dl_calls) == 1 + # A batch caller passes a tracker and owns the messaging; no per-file INFO + caplog.set_level("INFO") + src.download("mylib-batch", progress=lambda done: None) + assert len(dl_calls) == 2 + assert "Downloading" not in caplog.text + def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): registry = lib._make_registry_client() @@ -211,6 +219,7 @@ def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", + None, ), ) @@ -230,6 +239,38 @@ def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properti _patch_registry_resolve(monkeypatch) +def test_wave_requirement_growth_defers_the_superseded_download(tmp_path, monkeypatch): + """A's manifest constrains B while B sits in the same wave: B's + drain-time resolution is superseded, so its download defers to the + next wave instead of fetching a version that is immediately replaced.""" + download_names: list[str] = [] + manifests = { + "esphome/A": { + "name": "A", + "build": {}, + "dependencies": {"esphome/B": ">=1.0"}, + }, + "esphome/B": {"name": "B", "build": {}}, + } + + def fake_download(self, force=False, salt="", namespace="", progress=None): + download_names.append(self.name) + self.path = tmp_path / self.get_require_name() + self.path.mkdir(parents=True, exist_ok=True) + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(ConvertedLibrary, "download", fake_download) + # Hermetic: the stubbed registry reports no size, so no batch prefetch + _patch_registry_resolve(monkeypatch) + top = convert_libraries( + [Library("esphome/A", "1.0.0", None), Library("esphome/B", None, None)], + _backend(), + ) + assert sorted(c.name for c in top) == ["esphome/A", "esphome/B"] + # B downloads exactly once, after its requirement set stabilized + assert download_names.count("esphome/B") == 1 + + def test_convert_libraries_parses_library_properties(tmp_path, monkeypatch): # A manifest provided as library.properties (Arduino style) instead of # library.json must still be parsed and converted. @@ -574,6 +615,65 @@ def test_lex_build_flags_dangling_flag_does_not_cross_entries( assert "Ignoring trailing '-I'" in caplog.text +def test_prefetch_wave_downloads_registry_archives_in_parallel( + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Registry archives in one wave download concurrently, deduped by URL; + git/local sources and failures are left to the sequential call.""" + calls: list[str] = [] + + def fake_download( + self, dir_suffix, force=False, salt="", namespace="", progress=None + ): + calls.append(self.url) + if progress is not None: + progress(0) + if "boom" in self.url: + raise RuntimeError("boom") + + monkeypatch.setattr(URLSource, "download", fake_download) + wave = [ + ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))), + ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))), + # Duplicate URL must prefetch once (two threads must never extract + # into the same cache directory) + ("b2", ConvertedLibrary("b2", "1.0", URLSource("https://x/b.tar.gz", 1))), + ("c", ConvertedLibrary("c", "1.0", URLSource("https://x/boom.tar.gz", 1))), + ("g", ConvertedLibrary("g", "*", lib.GitSource("https://x/g.git", None))), + ] + lib._prefetch_wave(wave, "", "idf") + assert sorted(calls) == [ + "https://x/a.tar.gz", + "https://x/b.tar.gz", + "https://x/boom.tar.gz", + ] + # The failure surfaces at default verbosity, after the bar + assert "Prefetch of c failed (retrying sequentially)" in caplog.text + + +def test_prefetch_wave_unknown_size_left_to_sequential( + setup_core, monkeypatch: pytest.MonkeyPatch +) -> None: + """Archives without a registry-reported size skip the batch (their + sequential per-file bars don't interleave); the known subset still + prefetches.""" + calls: list[str] = [] + monkeypatch.setattr( + URLSource, + "download", + lambda self, dir_suffix, force=False, salt="", namespace="", progress=None: ( + calls.append(self.url) + ), + ) + wave = [ + ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))), + ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))), + ("u", ConvertedLibrary("u", "1.0", URLSource("https://x/u.tar.gz"))), + ] + lib._prefetch_wave(wave, "", "idf") + assert sorted(calls) == ["https://x/a.tar.gz", "https://x/b.tar.gz"] + + def test_join_flag_args_empty_argument_warns_and_drops( caplog: pytest.LogCaptureFixture, ) -> None: @@ -582,6 +682,97 @@ def test_join_flag_args_empty_argument_warns_and_drops( assert "Ignoring '-D' with empty argument in build_flags" in caplog.text +def test_prefetch_wave_cache_probe_failure_still_prefetches( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A filesystem probe failure warns (a systematic one re-downloads + everything) but still prefetches; a programming error is NOT swallowed + here, it reaches the outer blanket guard.""" + calls: list[str] = [] + monkeypatch.setattr( + URLSource, + "download", + lambda self, dir_suffix, **kw: calls.append(self.url), + ) + monkeypatch.setattr( + URLSource, + "is_cached", + lambda self, *a, **kw: (_ for _ in ()).throw(OSError("cache root denied")), + ) + wave = [ + ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))), + ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))), + ] + lib._prefetch_wave(wave, "", "idf") + assert sorted(calls) == ["https://x/a.tar.gz", "https://x/b.tar.gz"] + assert "Cache probe for a failed: cache root denied" in caplog.text + + +def test_prefetch_wave_internal_error_never_fails_the_build( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """The blanket guard keeps a prefetch bug from failing the walk.""" + monkeypatch.setattr(URLSource, "is_cached", lambda self, *a, **kw: False) + monkeypatch.setattr( + lib, + "run_batch_downloads", + lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("bug")), + ) + wave = [ + ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))), + ("b", ConvertedLibrary("b", "1.0", URLSource("https://x/b.tar.gz", 1))), + ] + lib._prefetch_wave(wave, "", "idf") + assert "Library prefetch failed: bug" in caplog.text + + +def test_prefetch_wave_warm_cache_is_silent( + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Already-extracted archives download nothing; a warm build must not + print a Downloading line or draw a bar.""" + monkeypatch.setattr( + URLSource, + "download", + lambda self, dir_suffix, **kw: (_ for _ in ()).throw( + AssertionError("downloaded") + ), + ) + wave = [] + for name in ("a", "b", "c"): + comp = ConvertedLibrary(name, "1.0", URLSource(f"https://x/{name}.tar.gz", 1)) + marker_dir = comp.source._cache_dir(comp.get_sanitized_name(), "", "idf") + marker_dir.mkdir(parents=True) + (marker_dir / ".esphome_extracted").touch() + wave.append((name, comp)) + lib._prefetch_wave(wave, "", "idf") + assert "Downloading" not in caplog.text + + +def test_prefetch_wave_single_archive_uses_the_batch( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A dependency chain discovers one archive per wave; it downloads + through the same runner so there is one download method and one bar.""" + caplog.set_level("INFO") + calls: list[str] = [] + monkeypatch.setattr(URLSource, "is_cached", lambda self, *a, **kw: False) + monkeypatch.setattr( + URLSource, + "download", + lambda self, dir_suffix, force=False, salt="", namespace="", progress=None: ( + calls.append(self.url) + ), + ) + lib._prefetch_wave( + [("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1)))], + "", + "idf", + ) + assert calls == ["https://x/a.tar.gz"] + assert "Downloading 1 library archive(s): a" in caplog.text + + def test_normalize_dependencies_forms(caplog) -> None: """Every PIO-legal spelling normalizes; unrecognizable entries warn.""" from esphome.platformio.library import normalize_dependencies From 2a05372a5193cfaaf70de782365c64a0ce3922c8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 24 Aug 2026 16:20:25 -0500 Subject: [PATCH 15/65] [gpio] Fix one_wire reset busy-waiting with interrupts off when delay wraps (#18733) --- esphome/components/gpio/one_wire/gpio_one_wire.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/gpio/one_wire/gpio_one_wire.cpp b/esphome/components/gpio/one_wire/gpio_one_wire.cpp index 1fecfbf0dd..f445efeca3 100644 --- a/esphome/components/gpio/one_wire/gpio_one_wire.cpp +++ b/esphome/components/gpio/one_wire/gpio_one_wire.cpp @@ -55,8 +55,11 @@ int HOT IRAM_ATTR GPIOOneWireBus::reset_int() { delayMicroseconds(1); } - // delay J - delayMicroseconds(start + 480 - micros()); + // delay J: finish the 480us slot, but never spin if it already elapsed + // (unsigned wrap here would busy-wait for minutes with interrupts off) + uint32_t elapsed = micros() - start; + if (elapsed < 480) + delayMicroseconds(480 - elapsed); this->pin_.digital_write(true); this->pin_.pin_mode(gpio::FLAG_OUTPUT); return r ? 1 : 0; From c866add3209df7a500b51ffc0caafccef835163d Mon Sep 17 00:00:00 2001 From: guillempages Date: Mon, 24 Aug 2026 23:23:07 +0200 Subject: [PATCH 16/65] [runtime_image] Fix include (#18738) Co-authored-by: J. Nick Koston --- esphome/components/runtime_image/image_format.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/runtime_image/image_format.cpp b/esphome/components/runtime_image/image_format.cpp index 3ba8871862..9575b30887 100644 --- a/esphome/components/runtime_image/image_format.cpp +++ b/esphome/components/runtime_image/image_format.cpp @@ -1,6 +1,6 @@ +#include "esphome/core/defines.h" #include "esphome/core/helpers.h" #include "image_format.h" -#include "image_decoder.h" namespace esphome::runtime_image { From a15fae6a1e55918253fe6d38a128025341b723c2 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:15:09 +0000 Subject: [PATCH 17/65] Bump aioesphomeapi from 46.1.0 to 46.2.0 (#18740) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c10ddc3f53..4f6bdbad4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==46.1.0 +aioesphomeapi==46.2.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 37b78ad46a4a76ad930d58b4105a98d358dc2e5b Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:24:57 +0200 Subject: [PATCH 18/65] [zigbee][time] Add zigbee time component on esp32 (#18656) Co-authored-by: J. Nick Koston --- esphome/components/zigbee/const_esp32.py | 3 + esphome/components/zigbee/time/__init__.py | 48 +++++-- .../zigbee/time/zigbee_time_esp32.cpp | 118 ++++++++++++++++++ .../zigbee/time/zigbee_time_esp32.h | 34 +++++ esphome/components/zigbee/zigbee_ep_esp32.py | 53 ++++++-- .../components/zigbee/zigbee_helpers_esp32.c | 4 + tests/components/zigbee/common.yaml | 3 + tests/components/zigbee/common_nrf52.yaml | 3 - .../zigbee/test-router.esp32-c6-idf.yaml | 3 + 9 files changed, 249 insertions(+), 20 deletions(-) create mode 100644 esphome/components/zigbee/time/zigbee_time_esp32.cpp create mode 100644 esphome/components/zigbee/time/zigbee_time_esp32.h diff --git a/esphome/components/zigbee/const_esp32.py b/esphome/components/zigbee/const_esp32.py index bfc4d93d5b..2e1b09fb22 100644 --- a/esphome/components/zigbee/const_esp32.py +++ b/esphome/components/zigbee/const_esp32.py @@ -9,6 +9,7 @@ SCALE = "scale" CONF_ATTRIBUTE_ID = "attribute_id" KEY_ZIGBEE_EP = "zigbee_ep" KEY_ZIGBEE_EP_NO_NUM = "zigbee_ep_no_num" +KEY_ZIGBEE_FIRST_EP_CL = "zigbee_first_ep_cl" DEVICE_ID = { "RANGE_EXTENDER": cg.RawExpression("EZB_ZHA_RANGE_EXTENDER_DEVICE_ID"), @@ -18,11 +19,13 @@ DEVICE_ID = { cluster_id = cg.esphome_ns.enum("ezb_zcl_cluster_id_e") CLUSTER_ID = { "BASIC": cluster_id.EZB_ZCL_CLUSTER_ID_BASIC, + "TIME": cluster_id.EZB_ZCL_CLUSTER_ID_TIME, "BINARY_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_BINARY_INPUT, "ANALOG_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_ANALOG_INPUT, } CLUSTER_ROLE = { "SERVER": cg.RawExpression("EZB_ZCL_CLUSTER_SERVER"), + "CLIENT": cg.RawExpression("EZB_ZCL_CLUSTER_CLIENT"), } attr_type = cg.esphome_ns.enum("ezb_zcl_attr_type_e") ATTR_TYPE = { diff --git a/esphome/components/zigbee/time/__init__.py b/esphome/components/zigbee/time/__init__.py index 3acab0076f..74f81df2e9 100644 --- a/esphome/components/zigbee/time/__init__.py +++ b/esphome/components/zigbee/time/__init__.py @@ -1,13 +1,15 @@ import esphome.codegen as cg from esphome.components import time as time_ import esphome.config_validation as cv -from esphome.const import CONF_ID +from esphome.const import CONF_ID, CONF_UPDATE_INTERVAL from esphome.core import CORE from esphome.types import ConfigType from .. import consume_endpoint from ..const import zigbee_ns +from ..const_esp32 import ROLE from ..const_zephyr import CONF_ZIGBEE_ID +from ..zigbee_ep_esp32 import add_clusters_to_first_ep, get_first_ep_num from ..zigbee_zephyr import ( ZigbeeClusterDesc, ZigbeeComponent, @@ -22,26 +24,52 @@ DEPENDENCIES = ["zigbee"] ZigbeeTime = zigbee_ns.class_("ZigbeeTime", time_.RealTimeClock) + +def _validate_zigbee_time(config: ConfigType) -> ConfigType: + if CORE.is_nrf52: + return consume_endpoint(config) + if CORE.is_esp32: + cl = [ + { + CONF_ID: "TIME", + ROLE: "CLIENT", + }, + { + CONF_ID: "TIME", + ROLE: "SERVER", + }, + ] + add_clusters_to_first_ep(cl) + return config + + CONFIG_SCHEMA = cv.All( time_.TIME_SCHEMA.extend( { cv.GenerateID(): cv.declare_id(ZigbeeTime), - cv.OnlyWith(CONF_ZIGBEE_ID, ["nrf52", "zigbee"]): cv.use_id( - ZigbeeComponent - ), + cv.GenerateID(CONF_ZIGBEE_ID): cv.use_id(ZigbeeComponent), + cv.SplitDefault( + CONF_UPDATE_INTERVAL, + nrf52="1s", + esp32="15min", + ): cv.update_interval, # override default from TIME_SCHEMA. Remove once nrf52 implementation is aligned. } - ) - .extend(cv.COMPONENT_SCHEMA) - .extend(cv.polling_component_schema("1s")), - consume_endpoint, + ).extend(cv.COMPONENT_SCHEMA), + _validate_zigbee_time, ) async def to_code(config: ConfigType) -> None: - CORE.add_job(_add_time, config) + if CORE.using_zephyr: + CORE.add_job(_add_time_zephyr, config) + if CORE.is_esp32: + zb = await cg.get_variable(config[CONF_ZIGBEE_ID]) + var = cg.new_Pvariable(config[CONF_ID], zb, get_first_ep_num()) + await cg.register_component(var, config) + await time_.register_time(var, config) -async def _add_time(config: ConfigType) -> None: +async def _add_time_zephyr(config: ConfigType) -> None: slot_index = get_slot_index() # Create unique names for this sensor's variables based on slot index diff --git a/esphome/components/zigbee/time/zigbee_time_esp32.cpp b/esphome/components/zigbee/time/zigbee_time_esp32.cpp new file mode 100644 index 0000000000..5567295782 --- /dev/null +++ b/esphome/components/zigbee/time/zigbee_time_esp32.cpp @@ -0,0 +1,118 @@ +#include "zigbee_time_esp32.h" +#if defined(USE_ZIGBEE) && defined(USE_ESP32) && defined(USE_TIME) +#include "esphome/core/log.h" +#include "esphome/core/application.h" + +namespace esphome::zigbee { + +static const char *const TAG = "zigbee.time"; + +// This time standard is the number of +// seconds since 0 hrs 0 mins 0 sec on 1st January 2000 UTC (Universal Coordinated Time). +constexpr time_t EPOCH_2000 = 946684800; + +static ZigbeeTime *global_time = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +void ZigbeeTime::setup() { + global_time = this; + if (this->parent_->is_started()) { + this->register_zb_time_(); + } else { + this->parent_->add_on_start_callback([this]() { this->register_zb_time_(); }); + } +} + +void ZigbeeTime::register_zb_time_() { + ezb_zcl_time_interface_t time_interface = { + .get_utc_time = esphome::zigbee::ZigbeeTime::get_utc_time, + .set_utc_time = esphome::zigbee::ZigbeeTime::set_utc_time, + }; + ezb_err_t ret; + if (!esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { + this->set_timeout("zb_time_register", 100, [this]() { this->register_zb_time_(); }); + return; + } + ret = ezb_zcl_time_server_interface_register(this->endpoint_, time_interface); + esp_zigbee_lock_release(); + if (ret != EZB_ERR_NONE) { + ESP_LOGW(TAG, "Setup failed: %d", ret); + this->mark_failed(); + return; + } + this->registered_ = true; + this->parent_->add_on_join_callback([this](bool x) { this->update(); }); + if (this->parent_->is_joined()) { + this->update(); + } +} + +void ZigbeeTime::status_cb(ezb_err_t status) { + if (status == EZB_ERR_NONE) { + ESP_LOGV(TAG, "Time synchronization successful"); + } else if (status == EZB_ERR_TIMEOUT) { + ESP_LOGW(TAG, "Time synchronization timed out"); + } else { + ESP_LOGW(TAG, "Time synchronization failed with error: %d", status); + } +} + +void ZigbeeTime::update() { + if (this->parent_->is_joined() && this->registered_) { + if (esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { + ESP_LOGV(TAG, "Updating time sync from Zigbee network..."); + ezb_zcl_time_server_synchronize_time(this->endpoint_, 10, esphome::zigbee::ZigbeeTime::status_cb, + EZB_ZCL_TIME_SERVER_RANK_MASTER); + esp_zigbee_lock_release(); + this->retry_count_ = 0; + } else { + if (this->retry_count_ == 0) { + ESP_LOGW(TAG, "Could not acquire Zigbee lock to synchronize time, will retry maximum 3 times"); + } + if (this->retry_count_ < 3) { + this->set_timeout("zb_time_sync", 100, [this]() { this->update(); }); + this->retry_count_++; + } else { + ESP_LOGW(TAG, "Could not acquire Zigbee lock to synchronize time"); + this->retry_count_ = 0; + } + } + } else { + ESP_LOGD(TAG, "Not connected to Zigbee network, cannot synchronize time"); + } +} + +uint32_t ZigbeeTime::get_utc_time() { + const time_t now = global_time->timestamp_now(); + if (now < EPOCH_2000) { + return 0xFFFFFFFF; // ZCL invalid UTCTime + } + return (uint32_t) (now - EPOCH_2000); +} + +void ZigbeeTime::set_utc_time(uint32_t utc) { + // prevent overflow + if (utc <= (std::numeric_limits::max() - EPOCH_2000)) { + global_time->set_epoch_time(utc + EPOCH_2000); + } +} + +void ZigbeeTime::set_epoch_time(uint32_t utc) { + // called from zigbee task, defer to main loop + this->defer([this, utc]() { + ESP_LOGV(TAG, "Setting device time to UTC: %u", static_cast(utc)); + this->synchronize_epoch_(utc); + }); + App.wake_loop_threadsafe(); +} + +void ZigbeeTime::dump_config() { + ESP_LOGCONFIG(TAG, + "Zigbee Time\n" + " Endpoint: %u", + this->endpoint_); + RealTimeClock::dump_config(); +} + +} // namespace esphome::zigbee + +#endif diff --git a/esphome/components/zigbee/time/zigbee_time_esp32.h b/esphome/components/zigbee/time/zigbee_time_esp32.h new file mode 100644 index 0000000000..4a47137745 --- /dev/null +++ b/esphome/components/zigbee/time/zigbee_time_esp32.h @@ -0,0 +1,34 @@ +#pragma once +#include "esphome/core/defines.h" +#if defined(USE_ZIGBEE) && defined(USE_ESP32) && defined(USE_TIME) + +#include "esphome/core/component.h" +#include "esphome/components/time/real_time_clock.h" +#include "../zigbee_esp32.h" + +namespace esphome::zigbee { + +class ZigbeeComponent; + +class ZigbeeTime final : public time::RealTimeClock { + public: + ZigbeeTime(ZigbeeComponent *parent, uint8_t ep) : parent_(parent), endpoint_(ep) {} + void setup() override; + void update() override; + void dump_config() override; + void set_epoch_time(uint32_t utc); + + protected: + void register_zb_time_(); + static void set_utc_time(uint32_t utc); + static uint32_t get_utc_time(); + static void status_cb(ezb_err_t status); + + ZigbeeComponent *parent_; + uint8_t endpoint_; + uint8_t retry_count_{0}; + bool registered_{false}; +}; + +} // namespace esphome::zigbee +#endif diff --git a/esphome/components/zigbee/zigbee_ep_esp32.py b/esphome/components/zigbee/zigbee_ep_esp32.py index c2001c66d6..700267ef50 100644 --- a/esphome/components/zigbee/zigbee_ep_esp32.py +++ b/esphome/components/zigbee/zigbee_ep_esp32.py @@ -18,6 +18,7 @@ from .const_esp32 import ( DEVICE_TYPE, KEY_ZIGBEE_EP, KEY_ZIGBEE_EP_NO_NUM, + KEY_ZIGBEE_FIRST_EP_CL, ROLE, ) @@ -95,11 +96,11 @@ def _get_next_ep_num(eps: list[int]) -> int: def _compare_clusters( - existing_ep: dict[str, Any], - ep: dict[str, Any], + existing_cl_list: list[dict[str, Any]], + cl_list: list[dict[str, Any]], ) -> tuple[str | int, str] | None: - existing_clusters = [(cl[CONF_ID], cl[ROLE]) for cl in existing_ep[CONF_CLUSTERS]] - for cl in [(cl[CONF_ID], cl[ROLE]) for cl in ep[CONF_CLUSTERS]]: + existing_clusters = [(cl[CONF_ID], cl[ROLE]) for cl in existing_cl_list] + for cl in [(cl[CONF_ID], cl[ROLE]) for cl in cl_list]: if cl in existing_clusters: return cl return None @@ -110,7 +111,7 @@ def _merge_endpoints( ep: dict[str, Any], use_type: bool | None, ) -> bool: - if _compare_clusters(existing_ep, ep): + if _compare_clusters(existing_ep.get(CONF_CLUSTERS, []), ep.get(CONF_CLUSTERS, [])): return False if ( ep.get(DEVICE_TYPE) @@ -200,6 +201,17 @@ def create_ep(router: bool) -> None: # clear list so that it is not processed again del zb_data[KEY_ZIGBEE_EP_NO_NUM] + # Add clusters to first ep + cl_list: list[dict] = zb_data.setdefault(KEY_ZIGBEE_FIRST_EP_CL, []) + if cl_list: + first_ep = ep_dict[get_first_ep_num()] + first_ep.setdefault(CONF_CLUSTERS, []) + if cl := _compare_clusters(first_ep[CONF_CLUSTERS], cl_list): + raise cv.Invalid( + f"Endpoint {get_first_ep_num()} has more than one cluster with cluster id {cl[0]} and role {cl[1]}." + ) + first_ep[CONF_CLUSTERS] += cl_list + del zb_data[KEY_ZIGBEE_FIRST_EP_CL] # Add default device type to endpoints that have none for ep in ep_dict.values(): @@ -207,6 +219,15 @@ def create_ep(router: bool) -> None: ep[DEVICE_TYPE] = "CUSTOM_ATTR" +def get_first_ep_num() -> int | None: + """Return the number of the first endpoint.""" + zb_data = CORE.data.setdefault(KEY_ZIGBEE, {}) + ep_dict: dict[int, dict] = zb_data.setdefault(KEY_ZIGBEE_EP, {}) + if ep_dict: + return min(ep_dict.keys()) + return None + + def add_ep(ep: dict[str, Any], ep_num: int | None, use_type: bool | None) -> None: """Add a Zigbee endpoint configuration to CORE.data. @@ -230,8 +251,8 @@ def add_ep(ep: dict[str, Any], ep_num: int | None, use_type: bool | None) -> Non # check if the existing endpoint has same clusters existing_ep = ep_dict[ep_num] if cl := _compare_clusters( - existing_ep, - ep, + existing_ep.get(CONF_CLUSTERS, []), + ep.get(CONF_CLUSTERS, []), ): raise cv.Invalid( f"Endpoint {ep_num} has more than one cluster with cluster id {cl[0]} and role {cl[1]}." @@ -245,3 +266,21 @@ def add_ep(ep: dict[str, Any], ep_num: int | None, use_type: bool | None) -> Non if use_type or ep.get(DEVICE_TYPE): ep[CONF_USE_DEVICE_TYPE] = {ep.get(DEVICE_TYPE): use_type} ep_dict[ep_num] = ep + + +def add_clusters_to_first_ep(cl: list[dict[str, Any]]) -> None: + """Add a list of Zigbee clusters to CORE.data. + + Args: + cl: list of cluster dictonaries. + """ + zb_data = CORE.data.setdefault(KEY_ZIGBEE, {}) + cl_list: list[dict] = zb_data.setdefault(KEY_ZIGBEE_FIRST_EP_CL, []) + if cluster := _compare_clusters( + cl_list, + cl, + ): + raise cv.Invalid( + f"Only one cluster with cluster id {cluster[0]} and role {cluster[1]} can be added to first endpoint." + ) + cl_list += cl diff --git a/esphome/components/zigbee/zigbee_helpers_esp32.c b/esphome/components/zigbee/zigbee_helpers_esp32.c index 150be612f6..0793669955 100644 --- a/esphome/components/zigbee/zigbee_helpers_esp32.c +++ b/esphome/components/zigbee/zigbee_helpers_esp32.c @@ -30,6 +30,8 @@ ezb_zcl_cluster_desc_t esphome_zb_default_cluster_dscr_create(uint16_t cluster_i return ezb_zcl_basic_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_IDENTIFY: return ezb_zcl_identify_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_TIME: + return ezb_zcl_time_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_ANALOG_INPUT: return ezb_zcl_analog_input_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: @@ -49,6 +51,8 @@ ezb_err_t esphome_zb_cluster_add_attr(uint16_t cluster_id, ezb_zcl_cluster_desc_ return ezb_zcl_basic_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_IDENTIFY: return ezb_zcl_identify_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_TIME: + return ezb_zcl_time_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_ANALOG_INPUT: return ezb_zcl_analog_input_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: diff --git a/tests/components/zigbee/common.yaml b/tests/components/zigbee/common.yaml index cc0d28ea61..4518e39060 100644 --- a/tests/components/zigbee/common.yaml +++ b/tests/components/zigbee/common.yaml @@ -41,3 +41,6 @@ number: min_value: 2 max_value: 100 step: 1 + +time: + - platform: zigbee diff --git a/tests/components/zigbee/common_nrf52.yaml b/tests/components/zigbee/common_nrf52.yaml index c05c4053a5..2da6570618 100644 --- a/tests/components/zigbee/common_nrf52.yaml +++ b/tests/components/zigbee/common_nrf52.yaml @@ -10,6 +10,3 @@ zigbee: on_start: then: - logger.log: "Started zigbee stack" - -time: - - platform: zigbee diff --git a/tests/components/zigbee/test-router.esp32-c6-idf.yaml b/tests/components/zigbee/test-router.esp32-c6-idf.yaml index 228fe331e5..557cc08f04 100644 --- a/tests/components/zigbee/test-router.esp32-c6-idf.yaml +++ b/tests/components/zigbee/test-router.esp32-c6-idf.yaml @@ -5,3 +5,6 @@ zigbee: on_join: then: - logger.log: "Joined network" + +time: + - platform: zigbee From acd3c4f156aa79d9dcf3630b40935b7a9fffe1d4 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:20:20 +1000 Subject: [PATCH 19/65] [lvgl] Implement list widget (#18018) Co-authored-by: Claude Sonnet 5 Co-authored-by: J. Nick Koston --- esphome/components/lvgl/__init__.py | 33 +- esphome/components/lvgl/automation.py | 2 +- esphome/components/lvgl/defines.py | 15 + esphome/components/lvgl/lvcode.py | 41 +- esphome/components/lvgl/lvgl_esphome.cpp | 37 +- esphome/components/lvgl/lvgl_esphome.h | 25 +- esphome/components/lvgl/schemas.py | 20 + esphome/components/lvgl/trigger.py | 27 +- esphome/components/lvgl/widgets/__init__.py | 30 +- esphome/components/lvgl/widgets/lv_list.py | 553 ++++++++++++++++++ .../config/list_on_add_lvgl_action_test.yaml | 36 ++ .../lvgl/config/list_outside_block_test.yaml | 41 ++ .../lvgl/config/list_test.yaml | 77 +++ tests/component_tests/lvgl/test_list.py | 403 +++++++++++++ .../lvgl/test_list_on_add_lvgl_action.py | 47 ++ .../lvgl/test_list_outside_block.py | 84 +++ tests/components/lvgl/lvgl-package.yaml | 58 ++ tests/components/lvgl/validate.host.yaml | 73 +++ 18 files changed, 1548 insertions(+), 54 deletions(-) create mode 100644 esphome/components/lvgl/widgets/lv_list.py create mode 100644 tests/component_tests/lvgl/config/list_on_add_lvgl_action_test.yaml create mode 100644 tests/component_tests/lvgl/config/list_outside_block_test.yaml create mode 100644 tests/component_tests/lvgl/config/list_test.yaml create mode 100644 tests/component_tests/lvgl/test_list.py create mode 100644 tests/component_tests/lvgl/test_list_on_add_lvgl_action.py create mode 100644 tests/component_tests/lvgl/test_list_outside_block.py create mode 100644 tests/components/lvgl/validate.host.yaml diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index bfe91eedd5..2d2f1d6288 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -57,7 +57,6 @@ from .defines import ( CONF_ALIGN_TO_LAMBDA_ID, CONF_ANIMATIONS, LOGGER, - add_lv_use, get_focused_widgets, get_lv_images_used, get_refreshed_widgets, @@ -74,7 +73,6 @@ from .keypads import KEYPADS_CONFIG, keypads_to_code from .lv_validation import lv_bool from .lvcode import LvContext, LvglComponent, lv_event_t_ptr, lvgl_static from .schemas import ( - BASE_PROPS, DISP_BG_SCHEMA, FULL_STYLE_SCHEMA, SET_STATE_SCHEMA, @@ -83,6 +81,7 @@ from .schemas import ( STYLE_SCHEMA, WIDGET_TYPES, any_widget_schema, + apply_style_driven_defines, container_schema, container_schema_value, theme_schema, @@ -108,7 +107,6 @@ from .widgets import ( get_screen_active, set_obj_properties, ) -from .widgets.img import CONF_IMAGE # Import only what we actually use directly in this file from .widgets.msgbox import MSGBOX_SCHEMA, msgboxes_to_code @@ -455,6 +453,15 @@ async def to_code(configs): # Mark all widgets as completed so awaiters of ``wait_for_widgets`` proceed. set_widgets_completed(True) async with LvContext(): + # Local import: lv_list imports meter, which imports obj_spec/set_obj_properties + # from this module's own namespace - a top-level import here would be circular. + from .widgets.lv_list import finish_list_triggers + + # Must run before generate_triggers(): that's what actually processes other + # widgets' on_click etc. automations, which can include lvgl.list.add/remove/ + # clear actions that fire a list's on_add/on_remove triggers - those need to + # already exist by then, not still be pending. + await finish_list_triggers() await generate_triggers() await generate_align_tos(configs[0]) for config in configs: @@ -481,34 +488,16 @@ async def to_code(configs): # This must be done after all widgets are created styles_used = df.get_styles_used() - if any(BASE_PROPS.get(x) is lvalid.lv_image for x in styles_used): - add_lv_use(CONF_IMAGE) + apply_style_driven_defines(styles_used) for use in df.get_lv_uses(): df.add_define(f"LV_USE_{use.upper()}") cg.add_define(f"USE_LVGL_{use.upper()}") - if { - "transform_rotation", - "transform_scale", - "transform_scale_x", - "transform_scale_y", - } & styles_used: - df.add_define("LV_COLOR_SCREEN_TRANSP", "1") - if configs[0].get(df.CONF_THEME, {}).get(df.CONF_DARK_MODE): df.add_define("LV_THEME_DEFAULT_DARK", "1") # Currently always need RGB565 for the display buffer, and ARGB8888 is used for layer blending lv_image_formats = {"RGB565", "ARGB8888"} - if { - "drop_shadow_color", - "drop_shadow_offset_x", - "drop_shadow_offset_y", - "drop_shadow_opa", - "drop_shadow_quality", - "drop_shadow_radius", - } & styles_used: - lv_image_formats.add("A8") for image_id in get_lv_images_used(): await cg.get_variable(image_id) diff --git a/esphome/components/lvgl/automation.py b/esphome/components/lvgl/automation.py index cad065adee..a62f466413 100644 --- a/esphome/components/lvgl/automation.py +++ b/esphome/components/lvgl/automation.py @@ -416,7 +416,7 @@ async def obj_set_z_index_to_code(config, action_id, template_arg, args): widget.obj, literal(f"{lv_expr.obj_get_index(widget.obj)} + 1") ) elif position == "DOWN": - with LvConditional(f"{lv_expr.obj_get_index(widget.obj)} > 0"): + with LvConditional(literal(f"{lv_expr.obj_get_index(widget.obj)} > 0")): lv_obj.move_to_index( widget.obj, literal(f"{lv_expr.obj_get_index(widget.obj)} - 1") ) diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 65e975ad6d..81a4d2b4ab 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -585,6 +585,21 @@ FLEX_FLOWS = LvConstant( "COLUMN_WRAP_REVERSE", ) +TRANSFORM_STYLE_PROPS = frozenset( + {"transform_rotation", "transform_scale", "transform_scale_x", "transform_scale_y"} +) + +DROP_SHADOW_STYLE_PROPS = frozenset( + { + "drop_shadow_color", + "drop_shadow_offset_x", + "drop_shadow_offset_y", + "drop_shadow_opa", + "drop_shadow_quality", + "drop_shadow_radius", + } +) + OBJ_FLAGS = ( "hidden", "clickable", diff --git a/esphome/components/lvgl/lvcode.py b/esphome/components/lvgl/lvcode.py index de00593773..850b63a26f 100644 --- a/esphome/components/lvgl/lvcode.py +++ b/esphome/components/lvgl/lvcode.py @@ -242,7 +242,7 @@ class LocalVariable(MockObj): self.base.type, self.modifier, self.base.id ) ) - return MockObj(self.base) + return MockObj(self.base, "->" if self.modifier == "*" else ".") def __exit__(self, *args): CodeContext.end_block() @@ -283,7 +283,15 @@ class MockLv: class LvConditional: def __init__(self, condition): - self.condition = condition + # Condition is embedded directly into a raw `if (...)` statement below, rather than + # going through the argument-list machinery (ExpressionList) that would otherwise + # convert a native Python value (e.g. a plain bool) to a proper Expression. + if isinstance(condition, str): + raise ValueError( + "LvConditional condition must not be a raw str; wrap it in literal() " + "if a string literal condition is really intended" + ) + self.condition = cg.safe_exp(condition) if condition is not None else None def __enter__(self): if self.condition is not None: @@ -303,6 +311,35 @@ class LvConditional: CodeContext.code_context.indent() +class LvCountdown: + """ + Emits a C++ `for` loop that counts an int variable down from `count - 1` to `0` inclusive. + Used to iterate over a widget's children in reverse, e.g. to fire a trigger once per child + before they're all removed. + """ + + def __init__(self, var_name: str, count): + self.var_name = var_name + self.count = count + + def __enter__(self): + # Cast explicitly rather than relying on `count`'s (typically unsigned) type to wrap + # and then narrow back to a negative int when count is 0 -- true in practice on every + # toolchain ESPHome targets, but not worth leaning on. + CodeContext.append( + RawStatement( + f"for (int {self.var_name} = (int) ({self.count}) - 1; {self.var_name} >= 0; " + f"{self.var_name}--) {{" + ) + ) + CodeContext.code_context.indent() + return literal(self.var_name) + + def __exit__(self, *args): + CodeContext.code_context.detent() + CodeContext.append(RawStatement("}")) + + class ReturnStatement(ExpressionStatement): def __str__(self): return f"return {self.expression};" diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index acd5a9bdef..22fccdd92a 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -208,21 +208,21 @@ void LvglComponent::esphome_lvgl_init() { lv_update_event = static_cast(lv_event_register_id()); } -void LvglComponent::add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event) { - lv_obj_add_event_cb(obj, callback, event, nullptr); +void LvglComponent::add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event, void *user_data) { + lv_obj_add_event_cb(obj, callback, event, user_data); } void LvglComponent::add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event1, - lv_event_code_t event2) { - add_event_cb(obj, callback, event1); - add_event_cb(obj, callback, event2); + lv_event_code_t event2, void *user_data) { + add_event_cb(obj, callback, event1, user_data); + add_event_cb(obj, callback, event2, user_data); } void LvglComponent::add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event1, - lv_event_code_t event2, lv_event_code_t event3) { - add_event_cb(obj, callback, event1); - add_event_cb(obj, callback, event2); - add_event_cb(obj, callback, event3); + lv_event_code_t event2, lv_event_code_t event3, void *user_data) { + add_event_cb(obj, callback, event1, user_data); + add_event_cb(obj, callback, event2, user_data); + add_event_cb(obj, callback, event3, user_data); } void LvglComponent::add_page(LvPageType *page) { @@ -963,6 +963,25 @@ lv_obj_t *lv_container_create(lv_obj_t *parent) { lv_obj_class_init_obj(obj); return obj; } + +#ifdef USE_LVGL_LIST +int lv_list_get_row_index(lv_obj_t *list, lv_obj_t *child) { + for (lv_obj_t *obj = child; obj != nullptr; obj = lv_obj_get_parent(obj)) { + if (lv_obj_get_parent(obj) == list) + return lv_obj_get_index(obj); + } + ESP_LOGW(TAG, "lvgl.list: entry is not inside the list it was added to"); + return -1; +} + +lv_obj_t *lv_list_get_row_for_remove(lv_obj_t *list, int index) { + lv_obj_t *child = index < 0 ? nullptr : lv_obj_get_child(list, index); + if (child == nullptr) { + ESP_LOGW(TAG, "lvgl.list.remove: index %d is out of range, ignoring", index); + } + return child; +} +#endif // USE_LVGL_LIST } // namespace esphome::lvgl lv_result_t lv_mem_test_core() { return LV_RESULT_OK; } diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index 9221ab9542..98b97e26d7 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -116,6 +116,18 @@ inline void lv_animimg_set_src(lv_obj_t *img, std::vector images int16_t lv_get_needle_angle_for_value(lv_obj_t *obj, int32_t value); #endif +#ifdef USE_LVGL_LIST +// Returns the index, within `list`, of the entry that contains `child`: `child` itself if it's a +// direct child of `list`, or the ancestor of `child` that is, when `child` is nested inside a +// widget hierarchy added via `lvgl.list.add`. Returns -1 if `child` isn't inside `list` at all. +int lv_list_get_row_index(lv_obj_t *list, lv_obj_t *child); + +// Returns the entry at `index` within `list`, or nullptr (logging why) if `index` is out of +// range -- shared by every `lvgl.list.remove` call site, since a templatable index can go out of +// range at runtime in ways config validation can't catch (e.g. driven by a sensor value). +lv_obj_t *lv_list_get_row_for_remove(lv_obj_t *list, int index); +#endif + #ifdef USE_LVGL_GRADIENT /** * @@ -135,6 +147,12 @@ class LvCompound { lv_obj_t *obj{}; }; +// Frees a heap-allocated LvCompound wrapper on LV_EVENT_DELETE, since lv_obj_del() only knows how to destroy LVGL's own +// object tree, not a separate C++ object paired with one of its nodes. +template void delete_lv_compound_on_delete(lv_event_t *e) { + delete static_cast(lv_event_get_user_data(e)); +} + class LvglComponent; class LvPageType : public Parented { @@ -241,10 +259,11 @@ class LvglComponent final : public PollingComponent { static void esphome_lvgl_init(); // Convenience overloads for adding a callback for one or more events - static void add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event); - static void add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event1, lv_event_code_t event2); + static void add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event, void *user_data = nullptr); static void add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event1, lv_event_code_t event2, - lv_event_code_t event3); + void *user_data = nullptr); + static void add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event1, lv_event_code_t event2, + lv_event_code_t event3, void *user_data = nullptr); // change the state of a widget and fire an event if changed (only needed for CHECKED) diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index e400dae50f..bbc977dca5 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -726,6 +726,26 @@ ALL_STYLES = { } +def apply_style_driven_defines(props: set[str]) -> None: + """Given a set of style-property names in use, registers everything their use + drives: add_lv_use(image) if any of them is image-typed (per BASE_PROPS), and + the LV_COLOR_SCREEN_TRANSP / LV_DRAW_SW_SUPPORT_A8 defines. Shared between + __init__.py (driven by df.get_styles_used(), for statically-declared widgets) + and lv_list.py's _register_dynamic_widget_style_uses (driven by scanning a + dynamically-added widget's own config), so a future style-driven define added + to one can't be missed in the other. + """ + # Local import: avoids a module-load-time cycle (widgets.img -> ... -> schemas). + from .widgets.img import CONF_IMAGE + + if any(BASE_PROPS.get(prop) is lvalid.lv_image for prop in props): + df.add_lv_use(CONF_IMAGE) + if df.TRANSFORM_STYLE_PROPS & props: + df.add_define("LV_COLOR_SCREEN_TRANSP", "1") + if df.DROP_SHADOW_STYLE_PROPS & props: + df.add_define("LV_DRAW_SW_SUPPORT_A8", "1") + + def strip_defaults(schema: cv.Schema): """ Take a schema and remove any default values, also convert Required to Optional. diff --git a/esphome/components/lvgl/trigger.py b/esphome/components/lvgl/trigger.py index 5f524969e2..56dcf81a79 100644 --- a/esphome/components/lvgl/trigger.py +++ b/esphome/components/lvgl/trigger.py @@ -59,7 +59,10 @@ async def generate_triggers(): all_triggers = ( LV_EVENT_TRIGGERS + LV_DISPLAY_EVENT_TRIGGERS + LV_SCREEN_EVENT_TRIGGERS ) - for w in get_widget_map().values(): + # Snapshot: building a trigger below can recurse into widget creation (e.g. a + # buttonmatrix's or tabview's to_code registers its own child widgets), which + # would otherwise mutate this dict mid-iteration. + for w in list(get_widget_map().values()): config = w.config if isinstance(w.type, LvScrActType): w = get_screen_active(w.var) @@ -141,7 +144,21 @@ def _get_event_literal(trigger: str | MockObj) -> MockObj: return literal("LV_EVENT_" + TRIGGER_MAP[trigger.upper()]) -async def add_trigger(conf, w, *events: str | MockObj, is_selected=None): +async def add_trigger( + conf, w, *events: str | MockObj, is_selected=None, attach_obj=None, user_data=None +): + """ + :param attach_obj: The object to actually register the callback on, if different + from `w.obj` - used when `w.obj` isn't valid at the point the callback gets + registered (e.g. a local variable that's only in scope inside the very + block this is called from, not from within the callback body itself; see + widgets/lv_list.py's dynamic widget creation). Defaults to `w.obj`. + :param user_data: Opaque pointer passed through to the registered event callback, + retrievable inside it via `lv_event_get_user_data(event)` - used to recover a + compound widget's C++ wrapper, which a captureless callback has no other way + to reach when it isn't a global variable (see widgets/lv_list.py). Defaults to + `nullptr`. + """ is_selected = is_selected or w.is_selected() tid = conf[CONF_TRIGGER_ID] trigger = cg.new_Pvariable(tid) @@ -158,12 +175,14 @@ async def add_trigger(conf, w, *events: str | MockObj, is_selected=None): lv_add(trigger.trigger(*value, literal("event"))) callback = await context.get_lambda() event_literals = [_get_event_literal(event) for event in events] + attach_obj = w.obj if attach_obj is None else attach_obj + user_data = nullptr if user_data is None else user_data if str(events[0]) in DISPLAY_TRIGGERS: assert len(events) == 1 lv.display_add_event_cb( - lv_expr.obj_get_display(w.obj), callback, event_literals[0], nullptr + lv_expr.obj_get_display(attach_obj), callback, event_literals[0], user_data ) else: lv_add( - lvgl_static.add_event_cb(w.obj, await context.get_lambda(), *event_literals) + lvgl_static.add_event_cb(attach_obj, callback, *event_literals, user_data) ) diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index 968db46adc..c9099e3c3a 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -190,18 +190,7 @@ class WidgetType: await self.on_create(var, config) w = Widget.create(wid, var, self, config) - if theme := get_theme_widget_map().get(self.name): - for part, states in theme.items(): - part = "LV_PART_" + part.upper() - for state, style in states.items(): - state = "LV_STATE_" + state.upper() - if state == "LV_STATE_DEFAULT": - lv_state = literal(part) - elif part == "LV_PART_MAIN": - lv_state = literal(state) - else: - lv_state = join_enums((state, part)) - w.add_style(style, lv_state) + apply_theme_styles(w) await set_obj_properties(w, config) await add_widgets(w, config) await self.to_code(w, config) @@ -230,7 +219,7 @@ class WidgetType: :param config: Its configuration """ - def get_uses(self): + def get_uses(self) -> tuple: """ Get a list of other widgets used by this one :return: @@ -267,6 +256,21 @@ class WidgetType: """ +def apply_theme_styles(w: "Widget") -> None: + """Apply the current theme's styles for this widget's type""" + for part, states in get_theme_widget_map().get(w.type.name, {}).items(): + part = "LV_PART_" + part.upper() + for state, style in states.items(): + state = "LV_STATE_" + state.upper() + if state == "LV_STATE_DEFAULT": + lv_state = literal(part) + elif part == "LV_PART_MAIN": + lv_state = literal(state) + else: + lv_state = join_enums((state, part)) + w.add_style(style, lv_state) + + class Widget: """ Represents a Widget. diff --git a/esphome/components/lvgl/widgets/lv_list.py b/esphome/components/lvgl/widgets/lv_list.py new file mode 100644 index 0000000000..83cbfb5ef9 --- /dev/null +++ b/esphome/components/lvgl/widgets/lv_list.py @@ -0,0 +1,553 @@ +from collections.abc import Generator +from dataclasses import dataclass, field +from typing import Any + +from esphome import automation +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import ( + CONF_BUTTON, + CONF_ID, + CONF_INDEX, + CONF_ON_BOOT, + CONF_ON_UPDATE, + CONF_ON_VALUE, + CONF_TEXT, + CONF_TRIGGER_ID, +) +from esphome.core import CORE +from esphome.coroutine import FakeAwaitable +from esphome.cpp_generator import MockObj +from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor + +from ..automation import action_to_code +from ..defines import ( + CONF_ALIGN_TO, + CONF_MAIN, + CONF_PAD_ROW, + CONF_SCROLLBAR, + CONF_WIDGETS, + LV_EVENT_TRIGGERS, + SWIPE_TRIGGERS, + TYPE_FLEX, + add_lv_use, + literal, +) +from ..lv_validation import lv_int, lv_text, padding +from ..lvcode import ( + UPDATE_EVENT, + LocalVariable, + LvConditional, + LvCountdown, + lv, + lv_add, + lv_expr, + lv_obj, +) +from ..schemas import ( + ALL_STYLES, + WIDGET_TYPES, + any_widget_schema, + apply_style_driven_defines, + container_schema_value, + remap_property, +) +from ..trigger import add_trigger +from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t +from . import ( + Widget, + WidgetType, + apply_theme_styles, + collect_parts, + get_widgets, + set_obj_properties, +) +from .buttonmatrix import CONF_BUTTONMATRIX +from .canvas import CONF_CANVAS +from .label import CONF_LABEL +from .meter import CONF_METER +from .tabview import CONF_TABVIEW +from .tileview import CONF_TILEVIEW + +CONF_LIST = "list" +CONF_WIDGET = "widget" +CONF_ON_ADD = "on_add" +CONF_ON_REMOVE = "on_remove" + +DOMAIN = "lvgl_list" + +lv_list_t = LvType("lv_list_t") + + +@dataclass +class ListTriggers: + on_add: list = field(default_factory=list) + on_remove: list = field(default_factory=list) + + +def _get_list_triggers(list_id) -> ListTriggers: + """ + Trigger Pvariables built for a given list's `on_add`/`on_remove` config, indexed by the + list's own ID. + """ + triggers_by_list = CORE.data.setdefault(DOMAIN, {}) + return triggers_by_list.setdefault(list_id, ListTriggers()) + + +def _get_pending_list_triggers(list_id) -> ListTriggers: + """ + Same shape as _get_list_triggers(), but holding raw on_add/on_remove automation + configs, not yet built. + """ + pending_by_list = CORE.data.setdefault(DOMAIN + "_pending", {}) + return pending_by_list.setdefault(list_id, ListTriggers()) + + +def _list_triggers_completed_flag() -> list[bool]: + return CORE.data.setdefault(DOMAIN + "_completed", [False]) + + +def _list_triggers_completed_generator() -> Generator[None, None, None]: + while True: + if _list_triggers_completed_flag()[0]: + return + yield + + +async def _wait_list_triggers_completed() -> None: + """Waits until finish_list_triggers() has built every list's on_add/on_remove automations.""" + if _list_triggers_completed_flag()[0]: + return + await FakeAwaitable(_list_triggers_completed_generator()) + + +async def finish_list_triggers() -> None: + """ + Builds every list's on_add/on_remove automations, collected by ListType.to_code() + instead of being built there directly. Must run after set_widgets_completed(True). + """ + for list_id, pending in CORE.data.get(DOMAIN + "_pending", {}).items(): + triggers = _get_list_triggers(list_id) + for conf in pending.on_add: + trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID]) + await automation.build_automation(trigger, [(cg.int_, "list_index")], conf) + triggers.on_add.append(trigger) + for conf in pending.on_remove: + trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID]) + await automation.build_automation(trigger, [(cg.int_, "list_index")], conf) + triggers.on_remove.append(trigger) + _list_triggers_completed_flag()[0] = True + + +def _fire_index_triggers(triggers: list, index) -> None: + for trigger in triggers: + lv_add(trigger.trigger(index)) + + +async def _fire_on_add(list_id, list_obj, entry_obj) -> None: + await _wait_list_triggers_completed() + triggers = _get_list_triggers(list_id).on_add + if not triggers: + return + index = cg.RawExpression(f"lvgl::lv_list_get_row_index({list_obj}, {entry_obj})") + _fire_index_triggers(triggers, index) + + +async def _fire_on_remove(list_id, index) -> None: + await _wait_list_triggers_completed() + _fire_index_triggers(_get_list_triggers(list_id).on_remove, index) + + +LIST_SCHEMA = cv.Schema( + { + cv.Optional(CONF_PAD_ROW): padding, + } +) + +LIST_CREATE_SCHEMA = LIST_SCHEMA.extend( + { + cv.Optional(CONF_ON_ADD): automation.validate_automation( + { + cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id( + automation.Trigger.template(cg.int_) + ), + } + ), + cv.Optional(CONF_ON_REMOVE): automation.validate_automation( + { + cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id( + automation.Trigger.template(cg.int_) + ), + } + ), + } +) + + +class ListType(WidgetType): + """A plain wrapper around LVGL's native `lv_list`""" + + def __init__(self): + super().__init__( + CONF_LIST, + lv_list_t, + (CONF_MAIN, CONF_SCROLLBAR), + LIST_CREATE_SCHEMA, + modify_schema=LIST_SCHEMA, + ) + + def get_uses(self): + return TYPE_FLEX, CONF_LABEL, CONF_BUTTON + + async def to_code(self, w: Widget, config: dict): + on_add = config.get(CONF_ON_ADD, ()) + on_remove = config.get(CONF_ON_REMOVE, ()) + if not on_add and not on_remove: + return + pending = _get_pending_list_triggers(w.config[CONF_ID]) + pending.on_add.extend(on_add) + pending.on_remove.extend(on_remove) + + +list_spec = ListType() + +LIST_ID_SCHEMA = cv.Schema({cv.Required(CONF_ID): cv.use_id(lv_list_t)}) + + +@automation.register_action( + "lvgl.list.add_text", + ObjUpdateAction, + LIST_ID_SCHEMA.extend( + { + cv.Required(CONF_TEXT): lv_text, + cv.Optional(CONF_INDEX): cv.templatable(cv.int_), + } + ), + synchronous=True, +) +async def list_add_text_to_code(config, action_id, template_arg, args): + widgets = await get_widgets(config) + + async def do_add_text(w: Widget): + text = await lv_text.process(config[CONF_TEXT]) + with LocalVariable( + "list_entry", lv_obj_t, lv_expr.list_add_text(w.obj, text) + ) as entry: + if (idx := config.get(CONF_INDEX)) is not None: + lv.obj_move_to_index(entry, await lv_int.process(idx)) + await _fire_on_add(config[CONF_ID], w.obj, entry) + + return await action_to_code( + widgets, do_add_text, action_id, template_arg, args, config + ) + + +_DYNAMIC_WIDGET_UNSUPPORTED = ( + CONF_BUTTONMATRIX, + CONF_TABVIEW, + CONF_TILEVIEW, + CONF_METER, + CONF_CANVAS, +) + + +def _check_dynamic_widget_supported(w_type_name: str, w_conf: dict) -> None: + # Each of these allocates a Pvariable, or registers children into the global widget + # map, once at boot - rebuilding them on every lvgl.list.add call would break that. + if w_type_name in _DYNAMIC_WIDGET_UNSUPPORTED: + raise cv.Invalid( + f"'{w_type_name}' cannot be used with lvgl.list.add - it manages its own " + "child widgets in a way that isn't compatible with widgets created at runtime" + ) + for child in w_conf.get(CONF_WIDGETS, ()): + [(child_type, child_conf)] = child.items() + _check_dynamic_widget_supported(child_type, child_conf) + + +_UNSUPPORTED_DYNAMIC_KEYS = SWIPE_TRIGGERS + (CONF_ON_BOOT, CONF_ALIGN_TO) + + +def _check_no_unsupported_triggers(w_type_name: str, w_conf: dict) -> None: + # These triggers currently aren't supporte for dynamic widgets + for key in _UNSUPPORTED_DYNAMIC_KEYS: + if key in w_conf: + raise cv.Invalid( + f"'{key}' is not supported on a widget added via lvgl.list.add - it " + "would validate but generate nothing, since it's only wired for " + "widgets that exist at boot", + path=[w_type_name, key], + ) + for child in w_conf.get(CONF_WIDGETS, ()): + [(child_type, child_conf)] = child.items() + _check_no_unsupported_triggers(child_type, child_conf) + + +def _check_no_explicit_widget_id(raw_value: dict) -> None: + for w_type_name, w_conf in raw_value.items(): + if not isinstance(w_conf, dict): + continue + if CONF_ID in w_conf: + raise cv.Invalid( + "'id' is not allowed on a widget added via lvgl.list.add - it is " + "rebuilt fresh on every call and never registered anywhere it " + "could be looked up by", + path=[w_type_name, CONF_ID], + ) + for child in w_conf.get(CONF_WIDGETS, ()): + if isinstance(child, dict): + _check_no_explicit_widget_id(child) + + +@schema_extractor("schema") +def list_add_schema(value: Any) -> Any: + # A plain cv.Schema can't express "id, an optional index, plus exactly one arbitrary + # widget-type key", since the set of widget types isn't fixed until validation time. + if value is SCHEMA_EXTRACT: + return LIST_ID_SCHEMA.extend( + { + cv.Optional(CONF_INDEX): cv.templatable(cv.int_), + **{ + cv.Optional(name): container_schema_value(widget_type) + for name, widget_type in WIDGET_TYPES.items() + }, + } + ) + if not isinstance(value, dict): + raise cv.Invalid("Expected a mapping") + value = value.copy() + if CONF_ID not in value: + raise cv.Invalid(f"required key '{CONF_ID}' not provided") + with cv.prepend_path([CONF_ID]): + list_id = cv.use_id(lv_list_t)(value.pop(CONF_ID)) + result = {CONF_ID: list_id} + if CONF_INDEX in value: + with cv.prepend_path([CONF_INDEX]): + result[CONF_INDEX] = cv.templatable(cv.int_)(value.pop(CONF_INDEX)) + if len(value) != 1: + raise cv.Invalid( + "lvgl.list.add takes exactly one widget definition, e.g. 'label:' or 'button:', alongside 'id' and optional 'index'" + ) + _check_no_explicit_widget_id(value) + result[CONF_WIDGET] = any_widget_schema()(value) + [(w_type_name, w_conf)] = result[CONF_WIDGET][0].items() + _check_dynamic_widget_supported(w_type_name, w_conf) + _check_no_unsupported_triggers(w_type_name, w_conf) + return result + + +def _register_lv_uses(w_type_name: str, w_conf: dict) -> None: + # Must run before this coroutine's first await. + widget_type = WIDGET_TYPES[w_type_name] + add_lv_use(w_type_name) + add_lv_use(*widget_type.get_uses()) + for child in w_conf.get(CONF_WIDGETS, ()): + [(child_type, child_conf)] = child.items() + _register_lv_uses(child_type, child_conf) + + +def _register_dynamic_widget_style_uses(w_conf: dict) -> None: + props = { + remap_property(prop) + for part_states in collect_parts(w_conf).values() + for state_props in part_states.values() + for prop in state_props + if prop in ALL_STYLES + } + apply_style_driven_defines(props) + for child in w_conf.get(CONF_WIDGETS, ()): + [(_, child_conf)] = child.items() + _register_dynamic_widget_style_uses(child_conf) + + +@automation.register_action( + "lvgl.list.add", + ObjUpdateAction, + list_add_schema, + synchronous=True, +) +async def list_add_to_code(config, action_id, template_arg, args): + [(w_type_name, w_conf)] = config[CONF_WIDGET][0].items() + _register_lv_uses(w_type_name, w_conf) + _register_dynamic_widget_style_uses(w_conf) + widgets = await get_widgets(config) + + async def do_add(w: Widget): + index = None + if (idx := config.get(CONF_INDEX)) is not None: + index = await lv_int.process(idx) + await _build_dynamic_widget( + w_type_name, + w_conf, + w.obj, + config[CONF_ID], + w.obj, + top_level=True, + index=index, + ) + + return await action_to_code(widgets, do_add, action_id, template_arg, args, config) + + +async def _build_dynamic_widget( + w_type_name: str, + w_conf: dict, + parent, + list_id, + list_obj, + top_level: bool = False, + index=None, + depth: int = 0, +) -> None: + # Builds one widget (recursively, with children and triggers) as a LocalVariable + # instead of a global Pvariable. Compound + # widgets are heap-allocated and freed via LV_EVENT_DELETE. + # `depth` suffixes the local variable's name below the row's top level. + widget_type = WIDGET_TYPES[w_type_name] + var_name = f"dyn_{w_type_name}" if depth == 0 else f"dyn_{w_type_name}_{depth}" + add_lv_use(w_type_name) + add_lv_use(*widget_type.get_uses()) + + async def finish_and_fire(w: Widget) -> None: + # Shared tail for both branches below - must run while var's LocalVariable + # block (opened by whichever branch calls this) is still open + await _finish_dynamic_widget(w, w_conf, list_id, list_obj, depth) + if top_level: + if index is not None: + lv.obj_move_to_index(w.obj, index) + await _fire_on_add(list_id, list_obj, w.obj) + + if widget_type.is_compound(): + with LocalVariable( + var_name, widget_type.w_type, widget_type.w_type.new() + ) as var: + creator = await widget_type.obj_creator(parent, w_conf) + lv_add(var.set_obj(creator)) + w = Widget(var, widget_type, w_conf) + lv_obj.add_event_cb( + w.obj, + literal(f"lvgl::delete_lv_compound_on_delete<{widget_type.w_type}>"), + literal("LV_EVENT_DELETE"), + var, + ) + await finish_and_fire(w) + else: + creator = await widget_type.obj_creator(parent, w_conf) + with LocalVariable(var_name, lv_obj_t, creator) as var: + w = Widget(var, widget_type, w_conf) + await finish_and_fire(w) + + +async def _finish_dynamic_widget( + w: Widget, w_conf: dict, list_id, list_obj, depth: int = 0 +) -> None: + await w.type.on_create(w.obj, w_conf) + apply_theme_styles(w) + await set_obj_properties(w, w_conf) + await w.type.to_code(w, w_conf) + await _wire_dynamic_triggers(w, w_conf) + for child in w_conf.get(CONF_WIDGETS, ()): + [(child_type, child_conf)] = child.items() + await _build_dynamic_widget( + child_type, child_conf, w.obj, list_id, list_obj, depth=depth + 1 + ) + + +async def _wire_dynamic_triggers(w: Widget, config: dict) -> None: + # Mirrors generate_triggers(), but runs immediately + if w.type.is_compound(): + event_var = MockObj( + f"static_cast<{w.type.w_type} *>(lv_event_get_user_data(event))", "->" + ) + user_data = w.var + else: + event_var = literal("static_cast(lv_event_get_target(event))") + user_data = None + event_target = Widget(event_var, w.type, config) + for event, conf in { + event: conf for event, conf in config.items() if event in LV_EVENT_TRIGGERS + }.items(): + w.add_flag("LV_OBJ_FLAG_CLICKABLE") + await add_trigger( + conf[0], event_target, event, attach_obj=w.obj, user_data=user_data + ) + for conf in config.get(CONF_ON_VALUE, ()): + await add_trigger( + conf, + event_target, + LV_EVENT.VALUE_CHANGED, + UPDATE_EVENT, + attach_obj=w.obj, + user_data=user_data, + ) + for conf in config.get(CONF_ON_UPDATE, ()): + await add_trigger( + conf, event_target, UPDATE_EVENT, attach_obj=w.obj, user_data=user_data + ) + + +LIST_REMOVE_SCHEMA = LIST_ID_SCHEMA.extend( + { + # positive_int, not int_: a negative index would silently delete the *last* + # row (lv_obj_get_child() counts back from the end) while reporting that + # same bogus value to on_remove's list_index. + cv.Required(CONF_INDEX): cv.templatable(cv.positive_int), + } +) + + +@automation.register_action( + "lvgl.list.remove", + ObjUpdateAction, + LIST_REMOVE_SCHEMA, + synchronous=True, +) +async def list_remove_to_code(config, action_id, template_arg, args): + widgets = await get_widgets(config) + + async def do_remove(w: Widget): + index = await lv_int.process(config[CONF_INDEX]) + # Materialised into a local since index is needed at two call sites below, and + # a lambda's body gets re-emitted (and re-run) at every point it's used. + with ( + LocalVariable("list_index", cg.int_, index, modifier="") as idx, + # Out-of-range lookup/log lives in a shared C++ helper, not inline here: + # a config can have many lvgl.list.remove call sites. + LocalVariable( + "list_child", + lv_obj_t, + cg.RawExpression(f"lvgl::lv_list_get_row_for_remove({w.obj}, {idx})"), + ) as child, + LvConditional(child), + ): + await _fire_on_remove(config[CONF_ID], idx) + # Recursively destroys the whole subtree + lv.obj_del(child) + + return await action_to_code( + widgets, do_remove, action_id, template_arg, args, config + ) + + +@automation.register_action( + "lvgl.list.clear", + ObjUpdateAction, + LIST_ID_SCHEMA, + synchronous=True, +) +async def list_clear_to_code(config, action_id, template_arg, args): + widgets = await get_widgets(config) + + async def do_clear(w: Widget): + await _wait_list_triggers_completed() + triggers = _get_list_triggers(config[CONF_ID]).on_remove + if triggers: + # Fire on_remove for every entry, newest to oldest, before wiping them all out, + # so on_remove's semantics ("an entry left the list") hold + with LvCountdown("list_index", lv_expr.obj_get_child_count(w.obj)) as index: + _fire_index_triggers(triggers, index) + # lv_obj_clean recursively destroys every child's whole subtree + lv.obj_clean(w.obj) + + return await action_to_code( + widgets, do_clear, action_id, template_arg, args, config + ) diff --git a/tests/component_tests/lvgl/config/list_on_add_lvgl_action_test.yaml b/tests/component_tests/lvgl/config/list_on_add_lvgl_action_test.yaml new file mode 100644 index 0000000000..875974a227 --- /dev/null +++ b/tests/component_tests/lvgl/config/list_on_add_lvgl_action_test.yaml @@ -0,0 +1,36 @@ +esphome: + name: test-list-on-add-lvgl-action + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +lvgl: + widgets: + - label: + id: later_label + text: orig + - list: + id: test_list + on_add: + - lvgl.label.update: + id: later_label + text: "changed" + on_remove: + - lvgl.label.update: + id: later_label + text: "removed" diff --git a/tests/component_tests/lvgl/config/list_outside_block_test.yaml b/tests/component_tests/lvgl/config/list_outside_block_test.yaml new file mode 100644 index 0000000000..4440734899 --- /dev/null +++ b/tests/component_tests/lvgl/config/list_outside_block_test.yaml @@ -0,0 +1,41 @@ +esphome: + name: test-list-outside-block + on_boot: + priority: -100 + then: + - lvgl.list.add: + id: test_list + switch: + transform_rotation: 100 + drop_shadow_color: 0x000000 + bg_image_src: my_image + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +image: + - platform: file + file: mdi:battery + id: my_image + resize: 8x8 + type: binary + +lvgl: + widgets: + - list: + id: test_list diff --git a/tests/component_tests/lvgl/config/list_test.yaml b/tests/component_tests/lvgl/config/list_test.yaml new file mode 100644 index 0000000000..2b5b73478d --- /dev/null +++ b/tests/component_tests/lvgl/config/list_test.yaml @@ -0,0 +1,77 @@ +esphome: + name: test-list + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +lvgl: + theme: + label: + bg_color: 0xFF0000 + widgets: + - list: + id: test_list + pad_row: 4 + on_add: + - delay: 10ms + - delay: 20ms + on_remove: + - delay: 10ms + - button: + id: trigger_button + text: "Trigger" + on_click: + - lvgl.list.add_text: + id: test_list + text: "Header" + - lvgl.list.add_text: + id: test_list + text: "Pinned" + index: 0 + - lvgl.list.add: + id: test_list + button: + text: "Entry" + checkable: true + - lvgl.list.add: + id: test_list + index: 1 + obj: + widgets: + - label: + text: "Nested" + - dropdown: + options: + - "One" + - "Two" + - lvgl.list.add: + id: test_list + obj: + widgets: + - obj: + widgets: + - label: + text: "Grandchild" + - lvgl.list.remove: + id: test_list + index: 0 + - lvgl.list.clear: + id: test_list + - lvgl.list.update: + id: test_list + pad_row: 8 diff --git a/tests/component_tests/lvgl/test_list.py b/tests/component_tests/lvgl/test_list.py new file mode 100644 index 0000000000..93ee162f1a --- /dev/null +++ b/tests/component_tests/lvgl/test_list.py @@ -0,0 +1,403 @@ +"""Tests for the LVGL ``list`` widget: schema validation for its actions +(``lvgl.list.add_text``/``add``/``remove``/``clear``) and the code they generate. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.components.lvgl.widgets.lv_list import ( + LIST_CREATE_SCHEMA, + LIST_REMOVE_SCHEMA, + LIST_SCHEMA, + list_add_schema, +) +from esphome.config import read_config +import esphome.config_validation as cv +from esphome.core import CORE + +# --------------------------------------------------------------------------- +# lvgl.list.add schema: id + optional index + exactly one widget-type key +# --------------------------------------------------------------------------- + + +class TestListAddSchema: + def test_valid_single_widget(self) -> None: + result = list_add_schema({"id": "my_list", "label": {"text": "hi"}}) + assert result["id"].id == "my_list" + assert "widget" in result + + def test_index_optional_and_templatable(self) -> None: + result = list_add_schema({"id": "my_list", "index": 2, "label": {"text": "hi"}}) + assert result["index"] == 2 + + def test_index_omitted_when_not_given(self) -> None: + result = list_add_schema({"id": "my_list", "label": {"text": "hi"}}) + assert "index" not in result + + def test_missing_id_rejected(self) -> None: + with pytest.raises(cv.Invalid, match="required key 'id' not provided"): + list_add_schema({"label": {"text": "hi"}}) + + def test_no_widget_key_rejected(self) -> None: + with pytest.raises(cv.Invalid, match="exactly one widget definition"): + list_add_schema({"id": "my_list"}) + + def test_two_widget_keys_rejected(self) -> None: + with pytest.raises(cv.Invalid, match="exactly one widget definition"): + list_add_schema( + { + "id": "my_list", + "label": {"text": "a"}, + "button": {"text": "b"}, + } + ) + + def test_non_mapping_rejected(self) -> None: + with pytest.raises(cv.Invalid, match="Expected a mapping"): + list_add_schema("not_a_mapping") + + def test_any_registered_widget_type_accepted(self) -> None: + for widget_key, widget_conf in ( + ("checkbox", {"text": "Option"}), + ("switch", {}), + ("spinner", {}), + ("obj", {}), + ("dropdown", {"options": ["a", "b"]}), + ): + result = list_add_schema({"id": "my_list", widget_key: widget_conf}) + assert widget_key in result["widget"][0] + + @pytest.mark.parametrize( + ("widget_key", "widget_conf"), + [ + ("buttonmatrix", {"rows": [{"buttons": [{"text": "A"}]}]}), + ("tabview", {"tabs": [{"name": "Tab1"}]}), + ("tileview", {"tiles": [{"row": 0, "column": 0}]}), + ("meter", {"scales": [{"range_from": 0, "range_to": 100}]}), + ("canvas", {"width": 20, "height": 20}), + ], + ) + def test_dynamic_widget_unsupported_rejected( + self, widget_key: str, widget_conf: dict + ) -> None: + """buttonmatrix/tabview/tileview all register their own child widgets into + the global widget map from inside their to_code - fine for a widget built + once at boot, but broken if lvgl.list.add re-enters that on every call. + meter/canvas are rejected for a related but distinct reason: they declare a + Pvariable (meter's scale/indicator objects; canvas's draw buffer) with + cg.Pvariable()/cg.new_Pvariable(), which emits its assignment wherever code + is currently being generated -- fine at the top level of a boot-time + to_code, but lvgl.list.add's do_add runs inside a lambda. meter's assignment + would then end up outside the very lambda that declares the local object it + refers to (doesn't compile); canvas's Pvariable is declared once per config + site rather than per call, so every call overwrites its one draw buffer + (compiles, but leaks the old buffer and shares one buffer across every row). + """ + with pytest.raises(cv.Invalid, match="cannot be used with lvgl.list.add"): + list_add_schema({"id": "my_list", widget_key: widget_conf}) + + def test_dynamic_widget_unsupported_rejected_when_nested(self) -> None: + """The check must recurse into `widgets:` so a tabview hidden a few levels + deep inside another widget is caught too, not just at the top level. + """ + with pytest.raises(cv.Invalid, match="cannot be used with lvgl.list.add"): + list_add_schema( + { + "id": "my_list", + "obj": {"widgets": [{"tabview": {"tabs": [{"name": "Tab1"}]}}]}, + } + ) + + def test_explicit_id_rejected(self) -> None: + """A dynamically-added widget is LocalVariable-scoped and rebuilt fresh + on every call, never registered anywhere an id could be looked up by -- + an explicit id: would otherwise validate fine and then fail confusingly + (an uncaught traceback, not a clean config error) the moment anything + tries to reference it. + """ + with pytest.raises(cv.Invalid, match="'id' is not allowed"): + list_add_schema( + {"id": "my_list", "label": {"id": "dyn_label", "text": "hi"}} + ) + + def test_explicit_id_rejected_when_nested(self) -> None: + with pytest.raises(cv.Invalid, match="'id' is not allowed"): + list_add_schema( + { + "id": "my_list", + "obj": {"widgets": [{"label": {"id": "dyn_label", "text": "hi"}}]}, + } + ) + + def test_no_explicit_id_still_valid(self) -> None: + """An id is auto-generated (and simply unused) when none is given -- + only an explicit one is rejected.""" + result = list_add_schema({"id": "my_list", "label": {"text": "hi"}}) + assert "id" in result["widget"][0]["label"] + + @pytest.mark.parametrize( + ("key", "conf"), + [ + ("on_swipe_left", [{"logger.log": "swiped"}]), + ("on_swipe_right", [{"logger.log": "swiped"}]), + ("on_swipe_up", [{"logger.log": "swiped"}]), + ("on_swipe_down", [{"logger.log": "swiped"}]), + ("on_boot", [{"logger.log": "booted"}]), + ("align_to", {"id": "some_other_widget", "align": "OUT_LEFT_TOP"}), + ], + ) + def test_unsupported_trigger_rejected(self, key: str, conf: list) -> None: + """_wire_dynamic_triggers only wires LV_EVENT_TRIGGERS/on_value/on_update -- + on_swipe_*/on_boot would otherwise validate fine and then silently generate + nothing at all for a widget added via lvgl.list.add. align_to is in the same + bucket: it's only ever consumed by generate_triggers() reading + get_widget_map(), which a widget built via lvgl.list.add never enters. + """ + with pytest.raises(cv.Invalid, match="is not supported"): + list_add_schema({"id": "my_list", "obj": {key: conf}}) + + def test_unsupported_trigger_rejected_when_nested(self) -> None: + with pytest.raises(cv.Invalid, match="is not supported"): + list_add_schema( + { + "id": "my_list", + "obj": { + "widgets": [ + { + "label": { + "text": "hi", + "on_swipe_left": [{"logger.log": "swiped"}], + } + } + ] + }, + } + ) + + +# --------------------------------------------------------------------------- +# lvgl.list.remove: index must be non-negative -- LVGL treats a negative index as +# counting back from the end, which would silently delete the wrong row while +# reporting a list_index that matches nothing real to on_remove. +# --------------------------------------------------------------------------- + + +class TestListRemoveSchema: + def test_negative_index_rejected(self) -> None: + with pytest.raises(cv.Invalid, match="at least 0"): + LIST_REMOVE_SCHEMA({"id": "my_list", "index": -1}) + + def test_zero_index_accepted(self) -> None: + result = LIST_REMOVE_SCHEMA({"id": "my_list", "index": 0}) + assert result["index"] == 0 + + +# --------------------------------------------------------------------------- +# The list widget's own schema: pad_row is shared between create/update, but +# on_add/on_remove only make sense at creation time. +# --------------------------------------------------------------------------- + + +class TestListCreateVsModifySchema: + def test_create_schema_has_pad_row_and_triggers(self) -> None: + keys = {str(k) for k in LIST_CREATE_SCHEMA.schema} + assert "pad_row" in keys + assert "on_add" in keys + assert "on_remove" in keys + + def test_modify_schema_has_pad_row_but_not_triggers(self) -> None: + """``lvgl.list.update`` can change pad_row but can't (re-)declare triggers.""" + keys = {str(k) for k in LIST_SCHEMA.schema} + assert "pad_row" in keys + assert "on_add" not in keys + assert "on_remove" not in keys + + def test_on_add_single_automation_with_multiple_actions(self) -> None: + """A bare action list under on_add: is one automation with a multi-step + `then:`, not multiple independent automations. + """ + config = LIST_CREATE_SCHEMA({"on_add": [{"delay": "10ms"}, {"delay": "20ms"}]}) + assert len(config["on_add"]) == 1 + assert len(config["on_add"][0]["then"]) == 2 + + def test_on_add_accepts_multiple_independent_automations(self) -> None: + """Each explicit `then:` entry gets its own Trigger, so on_add can fire + more than one independent automation. + """ + config = LIST_CREATE_SCHEMA( + { + "on_add": [ + {"then": [{"delay": "10ms"}]}, + {"then": [{"delay": "20ms"}]}, + ] + } + ) + assert len(config["on_add"]) == 2 + + +# --------------------------------------------------------------------------- +# Code generation +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + """Generate the C++ output for the shared list-widget YAML config once per + module -- see test_widget_state.py for why this is module-scoped and + inlines the generate_main fixture logic rather than depending on it. + """ + config_path = Path(request.fspath).parent / "config" / "list_test.yaml" + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_global_section + CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_pad_row_set_at_creation(main_cpp: str) -> None: + assert "lv_obj_set_style_pad_row(test_list, 4, LV_PART_MAIN);" in main_cpp + + +def test_pad_row_updated_via_update_action(main_cpp: str) -> None: + assert "lv_obj_set_style_pad_row(test_list, 8, LV_PART_MAIN);" in main_cpp + + +def test_add_text_appends(main_cpp: str) -> None: + assert 'lv_list_add_text(test_list, "Header");' in main_cpp + + +def test_add_text_with_index_moves_before_firing_on_add(main_cpp: str) -> None: + """The index move must happen before on_add fires, so the reported + list_index reflects the entry's final position, not where it was appended. + """ + assert ( + 'lv_obj_t *list_entry_VAR_ = lv_list_add_text(test_list, "Pinned");\n' + " lv_obj_move_to_index(list_entry_VAR_, 0);\n" + " triggerint_id->trigger(lvgl::lv_list_get_row_index(test_list, list_entry_VAR_));" + ) in main_cpp + + +def test_add_button_with_checkable_flag(main_cpp: str) -> None: + assert "lv_obj_t *dyn_button_VAR_ = lv_btn_create(test_list);" in main_cpp + assert ( + "lv_obj_add_flag(dyn_button_VAR_, (lv_obj_flag_t)(LV_OBJ_FLAG_CHECKABLE));" + in main_cpp + ) + assert ( + 'lv_label_set_text(lv_obj_get_child(dyn_button_VAR_, 0), "Entry");' in main_cpp + ) + + +def test_add_nested_hierarchy_with_compound_child(main_cpp: str) -> None: + """`obj: {widgets: [label, dropdown]}` builds a plain label child and a + heap-allocated (compound) dropdown child, both parented to the new row. + + The child variable names carry a `_1` (depth) suffix, distinguishing them + from the row's own top-level variable -- necessary so that a child of the + *same* widget type as its parent (e.g. `obj: {widgets: [{obj: {...}}]}`) + doesn't declare a C++ variable that shadows its own not-yet-initialized + self, silently parenting the child to garbage. + """ + assert "lv_obj_t *dyn_obj_VAR_ = lv_obj_create(test_list);" in main_cpp + assert ( + "lv_obj_t *dyn_label_1_VAR_ = lv_label_create(dyn_obj_VAR_);\n" + " lv_obj_add_style(dyn_label_1_VAR_, _lv_theme_style_label_main_default, " + "(lv_state_t)(LV_PART_MAIN));\n" + ' lv_label_set_text(dyn_label_1_VAR_, "Nested");' + ) in main_cpp + + +def test_add_applies_theme_styles_to_dynamic_widget(main_cpp: str) -> None: + """A widget added via lvgl.list.add must pick up the same `theme:` styling a + statically-declared widget of the same type gets, not render unthemed. + """ + assert ( + "lv_obj_add_style(dyn_label_1_VAR_, _lv_theme_style_label_main_default, " + "(lv_state_t)(LV_PART_MAIN));" + ) in main_cpp + assert "LvDropdownType *dyn_dropdown_1_VAR_ = new LvDropdownType();" in main_cpp + assert "lv_dropdown_create(dyn_obj_VAR_)" in main_cpp + assert ( + "lvgl::delete_lv_compound_on_delete, LV_EVENT_DELETE, " + "dyn_dropdown_1_VAR_);" + ) in main_cpp + + +def test_add_nested_same_type_child_does_not_shadow_parent(main_cpp: str) -> None: + """A child of the same widget type as its parent (`obj: {widgets: [{obj: + ...}]}`) must get a distinct C++ variable name (or the child's declaration + would shadow its own not-yet-initialized self, parenting it to garbage -- + compiling clean but for a -Wuninitialized warning). A grandchild of a third + type proves depth, not just type, drives the disambiguating suffix. + """ + assert "lv_obj_t *dyn_obj_VAR_ = lv_obj_create(test_list);" in main_cpp + assert "lv_obj_t *dyn_obj_1_VAR_ = lv_obj_create(dyn_obj_VAR_);" in main_cpp + assert ( + "lv_obj_t *dyn_label_2_VAR_ = lv_label_create(dyn_obj_1_VAR_);\n" + " lv_obj_add_style(dyn_label_2_VAR_, _lv_theme_style_label_main_default, " + "(lv_state_t)(LV_PART_MAIN));\n" + ' lv_label_set_text(dyn_label_2_VAR_, "Grandchild");' + ) in main_cpp + + +def test_add_moves_row_to_given_index_before_firing_on_add(main_cpp: str) -> None: + assert ( + "lv_obj_move_to_index(dyn_obj_VAR_, 1);\n" + " triggerint_id->trigger(lvgl::lv_list_get_row_index(test_list, dyn_obj_VAR_));" + ) in main_cpp + + +def test_on_add_fires_once_per_entry_via_shared_trigger(main_cpp: str) -> None: + """A single on_add: automation means a single Trigger instance, reused by + every lvgl.list.add_text/add call site. + """ + assert main_cpp.count("triggerint_id->trigger(lvgl::lv_list_get_row_index(") == 5 + + +def test_remove_guards_against_missing_child_and_fires_before_delete( + main_cpp: str, +) -> None: + """The index is materialised into a local once (list_index_VAR_) and reused for + both the child lookup and the on_remove trigger, so a templatable index isn't + evaluated twice. + """ + assert ( + "int list_index_VAR_ = 0;\n" + " {\n" + " lv_obj_t *list_child_VAR_ = lvgl::lv_list_get_row_for_remove(test_list, list_index_VAR_);\n" + " if (list_child_VAR_) {\n" + " triggerint_id_2->trigger(list_index_VAR_);\n" + " lv_obj_del(list_child_VAR_);" + ) in main_cpp + + +def test_remove_out_of_range_lookup_uses_shared_cpp_helper(main_cpp: str) -> None: + """The out-of-range lookup (and its log line) live in a single C++ helper -- + lvgl::lv_list_get_row_for_remove() in lvgl_esphome.cpp -- rather than being + generated inline at every lvgl.list.remove call site, since a config can + have many of them and duplicating that logic (and its log string) at each + one would waste flash for no benefit. + """ + assert ( + "lv_obj_t *list_child_VAR_ = lvgl::lv_list_get_row_for_remove(test_list, list_index_VAR_);" + in main_cpp + ) + assert "ESP_LOGV" not in main_cpp + + +def test_clear_fires_on_remove_for_every_entry_then_cleans(main_cpp: str) -> None: + assert ( + "for (int list_index = (int) (lv_obj_get_child_count(test_list)) - 1; " + "list_index >= 0; list_index--) {\n" + " triggerint_id_2->trigger(list_index);\n" + " }\n" + " lv_obj_clean(test_list);" + ) in main_cpp diff --git a/tests/component_tests/lvgl/test_list_on_add_lvgl_action.py b/tests/component_tests/lvgl/test_list_on_add_lvgl_action.py new file mode 100644 index 0000000000..2383adc251 --- /dev/null +++ b/tests/component_tests/lvgl/test_list_on_add_lvgl_action.py @@ -0,0 +1,47 @@ +"""Regression test: on_add:/on_remove: containing an lvgl action must not deadlock. + +ListType.to_code() used to build the on_add/on_remove automations directly, during +widget creation. Every lvgl action's to_code awaits wait_for_widgets(), which only +resolves once *all* widgets - including the list itself - have finished being +created. Building an automation containing an lvgl action from inside that same +widget-creation walk therefore could never complete: codegen deadlocked with +"Circular dependency detected!". Fixed by deferring the actual build_automation() +call to finish_list_triggers(), run after set_widgets_completed(True) - and, +critically, before generate_triggers(), which is what processes other widgets' +on_click etc. automations that might reference this list (e.g. via lvgl.list.add), +and which therefore need the list's own on_add/on_remove triggers to already exist. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + config_path = ( + Path(request.fspath).parent / "config" / "list_on_add_lvgl_action_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_on_add_with_lvgl_action_does_not_deadlock(main_cpp: str) -> None: + assert 'lv_label_set_text(later_label, "changed");' in main_cpp + + +def test_on_remove_with_lvgl_action_does_not_deadlock(main_cpp: str) -> None: + assert 'lv_label_set_text(later_label, "removed");' in main_cpp diff --git a/tests/component_tests/lvgl/test_list_outside_block.py b/tests/component_tests/lvgl/test_list_outside_block.py new file mode 100644 index 0000000000..34c93a18e2 --- /dev/null +++ b/tests/component_tests/lvgl/test_list_outside_block.py @@ -0,0 +1,84 @@ +"""Regression test for lvgl.list.add called from outside the lvgl: block. + +lv_list.py's list_add_to_code() must call _register_lv_uses() and +_register_dynamic_widget_style_uses() before its first await (get_widgets(), +which can block until the target list is defined) -- for an action referenced +outside the lvgl: block, that wait can outlast lvgl's own to_code, which reads +get_lv_uses()/get_styles_used() and flushes everything they drive (USE_LVGL_* +defines, plus add_lv_use(image)/screen-transparency/A8-draw-support triggered +by style properties) just once, near the end of its run. Every existing +list_test.yaml call site lives inside lvgl: widgets:, so neither ordering +requirement had any coverage. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@dataclass +class GeneratedOutput: + main_cpp: str + define_names: set[str] + lv_define_names: set[str] + + +@pytest.fixture(scope="module") +def generated(request: pytest.FixtureRequest) -> GeneratedOutput: + config_path = ( + Path(request.fspath).parent / "config" / "list_outside_block_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + # Copy out before CORE.reset() below clears these out from under us. + from esphome.components.lvgl import defines as df + + return GeneratedOutput( + main_cpp=CORE.cpp_global_section + CORE.cpp_main_section, + define_names={d.name for d in CORE.defines}, + lv_define_names=set(df.get_defines()), + ) + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_dynamic_widget_creates_correctly(generated: GeneratedOutput) -> None: + assert ( + "lv_obj_t *dyn_switch_VAR_ = lv_switch_create(test_list);" in generated.main_cpp + ) + + +def test_dynamic_widget_type_use_define_is_registered( + generated: GeneratedOutput, +) -> None: + """The switch type is only ever referenced via the on_boot lvgl.list.add call + (never declared as a static widget), so USE_LVGL_SWITCH can only be present + if _register_lv_uses() ran in time for lvgl's own to_code to flush it. + """ + assert "USE_LVGL_SWITCH" in generated.define_names + assert "USE_LVGL_LIST" in generated.define_names + + +def test_dynamic_widget_style_use_defines_are_registered( + generated: GeneratedOutput, +) -> None: + """bg_image_src/transform_rotation/drop_shadow_color are only ever set on + the dynamically-added switch (never on a static widget), so + USE_LVGL_IMAGE/LV_COLOR_SCREEN_TRANSP/LV_DRAW_SW_SUPPORT_A8 can only be + present if _register_dynamic_widget_style_uses() ran in time for lvgl's own + to_code to flush them. + """ + assert "USE_LVGL_IMAGE" in generated.define_names + assert "LV_COLOR_SCREEN_TRANSP" in generated.lv_define_names + assert "LV_DRAW_SW_SUPPORT_A8" in generated.lv_define_names diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 46c1fd362a..c78e910bc8 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -1214,6 +1214,64 @@ lvgl: id: checkbox_id text: Checkbox align: bottom_right + - list: + id: test_list_id + align: top_right + width: 150px + height: 120px + pad_row: 4 + on_add: + - logger.log: + format: "list entry added at %d" + args: [list_index] + on_remove: + - logger.log: + format: "list entry removed at %d" + args: [list_index] + on_click: + - lvgl.list.add_text: + id: test_list_id + text: !lambda return "Section"; + - lvgl.list.add_text: + id: test_list_id + text: "Pinned section" + index: 0 + - lvgl.list.add: + id: test_list_id + button: + text: "Entry" + checkable: true + - lvgl.list.add: + id: test_list_id + index: 1 + obj: + widgets: + - label: + text: !lambda return "Dynamic row " + std::to_string(millis()); + - button: + widgets: + - label: + text: "Tap" + on_click: + - lambda: |- + ESP_LOGD("lvgl", "dynamic row button clicked, row %d", + lvgl::lv_list_get_row_index(id(test_list_id), static_cast(lv_event_get_target(event)))); + - dropdown: + options: + - "One" + - "Two" + on_value: + - lambda: |- + ESP_LOGD("lvgl", "dynamic row dropdown changed, row %d", + lvgl::lv_list_get_row_index(id(test_list_id), static_cast(lv_event_get_target(event)))); + - lvgl.list.remove: + id: test_list_id + index: 0 + - lvgl.list.clear: + id: test_list_id + - lvgl.list.update: + id: test_list_id + pad_row: 8 - slider: id: slider_id align: top_mid diff --git a/tests/components/lvgl/validate.host.yaml b/tests/components/lvgl/validate.host.yaml new file mode 100644 index 0000000000..77db404a6e --- /dev/null +++ b/tests/components/lvgl/validate.host.yaml @@ -0,0 +1,73 @@ +esphome: + name: lvgl-list-validate + +host: + +logger: + +display: + - platform: sdl + id: sdl0 + dimensions: + width: 320 + height: 240 + +lvgl: + displays: sdl0 + widgets: + # Two independent lists, each with their own on_add/on_remove and, for list_a, + # more than one automation under the same trigger key -- checks that the + # per-list trigger bookkeeping is keyed correctly and doesn't require exactly + # one automation. + - list: + id: validate_list_a + align: center + pad_row: 6 + on_add: + - logger.log: + format: "a: added %d" + args: [list_index] + - logger.log: + format: "a: also added %d" + args: [list_index] + on_remove: + - logger.log: + format: "a: removed %d" + args: [list_index] + on_boot: + # lvgl.list.add_text and lvgl.list.add both take an optional, templatable index. + - lvgl.list.add_text: + id: validate_list_a + text: "Header" + index: !lambda return 0; + # any registered widget type is valid as the single lvgl.list.add key. + - lvgl.list.add: + id: validate_list_a + checkbox: + align: center + text: "Option" + - lvgl.list.add: + id: validate_list_a + index: !lambda return 0; + switch: + align: center + - lvgl.list.add: + id: validate_list_a + spinner: + align: center + - lvgl.list.add: + id: validate_list_a + obj: + align: center + - lvgl.list.remove: + id: validate_list_a + index: !lambda return 0; + - lvgl.list.clear: + id: validate_list_a + - list: + id: validate_list_b + align: center + on_remove: + - logger.log: + format: "b: removed %d" + args: [list_index] From b3d9aa2d8459fd0bb30b8bbcf5f5ff8e35ac1937 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 24 Aug 2026 18:46:43 -0500 Subject: [PATCH 20/65] [core] Log which source change triggered a rebuild (#18552) --- esphome/platformio/library.py | 13 +-- esphome/writer.py | 88 ++++++++++++++------- tests/unit_tests/test_platformio_library.py | 8 +- tests/unit_tests/test_writer.py | 36 +++++++++ 4 files changed, 107 insertions(+), 38 deletions(-) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index bf9c323b84..50e1408b13 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -55,8 +55,9 @@ DEFAULT_BUILD_SRC_DIRS = "src" DEFAULT_BUILD_INCLUDE_DIR = "include" DEFAULT_BUILD_FLAGS = [] # Suffix -> compiler kind (PlatformIO's CSUFFIXES/CXXSUFFIXES/ASSUFFIXES); -# "asm" merges SCons's AS and ASPP sets. Per CXXSUFFIXES .C/.C++ are C++ -# here, even where SCons demotes .C on case-insensitive filesystems. +# "aspp" is SCons's preprocessed-assembly set, "asm" its plain-assembler +# set (no preprocessor, defines, or includes). Per CXXSUFFIXES .C/.C++ are +# C++ here, even where SCons demotes .C on case-insensitive filesystems. SOURCE_KIND_FOR_SUFFIX: dict[str, str] = { ".c": "c", ".cpp": "cxx", @@ -65,10 +66,10 @@ SOURCE_KIND_FOR_SUFFIX: dict[str, str] = { ".c++": "cxx", ".C": "cxx", ".C++": "cxx", - ".S": "asm", - ".spp": "asm", - ".SPP": "asm", - ".sx": "asm", + ".S": "aspp", + ".spp": "aspp", + ".SPP": "aspp", + ".sx": "aspp", ".s": "asm", ".asm": "asm", ".ASM": "asm", diff --git a/esphome/writer.py b/esphome/writer.py index d614204603..85c0642774 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -288,11 +288,13 @@ def copy_src_tree(): # Source file removed, delete target p.unlink() if target not in generated_files: + _LOGGER.debug("Source removed: %s", target) sources_changed = True else: src_file = source_files_copy.pop(target) with src_file.path() as src_path: if copy_file_if_changed(src_path, p) and target not in generated_files: + _LOGGER.debug("Source changed: %s", target) sources_changed = True # Now copy new files @@ -303,21 +305,25 @@ def copy_src_tree(): copy_file_if_changed(src_path, dst_path) and target not in generated_files ): + _LOGGER.debug("Source added: %s", target) sources_changed = True # Finally copy defines if write_file_if_changed( CORE.relative_src_path("esphome", "core", "defines.h"), generate_defines_h() ): + _LOGGER.debug("Source changed: esphome/core/defines.h") sources_changed = True write_file_if_changed(CORE.relative_build_path("README.txt"), ESPHOME_README_TXT) if write_file_if_changed( CORE.relative_src_path("esphome.h"), ESPHOME_H_FORMAT.format(include_s) ): + _LOGGER.debug("Source changed: esphome.h") sources_changed = True if write_file_if_changed( CORE.relative_src_path("esphome", "core", "version.h"), generate_version_h() ): + _LOGGER.debug("Source changed: esphome/core/version.h") sources_changed = True # Generate new build_info files if needed @@ -332,35 +338,13 @@ def copy_src_tree(): # Defensively force a rebuild if the build_info files don't exist, or if # there was a config change which didn't actually cause a source change - if not build_info_data_h_path.exists() or not build_info_data_cpp_path.exists(): + if _build_info_stale( + build_info_data_h_path, + build_info_data_cpp_path, + build_info_json_path, + config_hash, + ): sources_changed = True - else: - try: - existing = json.loads(build_info_json_path.read_text(encoding="utf-8")) - if not isinstance(existing, dict) or ( - existing.get("config_hash") != config_hash - or existing.get("esphome_version") != __version__ - ): - # Non-object JSON is stale like every other damage case - sources_changed = True - except FileNotFoundError: - # An absent build_info.json is stale, not damaged; rebuild quietly - sources_changed = True - except (ValueError, OSError) as err: - # ValueError covers both JSONDecodeError and UnicodeDecodeError; - # unlink so the regenerating write never re-reads the bad copy. - # "Unreadable" not "damaged": EACCES/EISDIR land here too - _LOGGER.warning("Regenerating unreadable build_info.json: %s", err) - try: - # missing_ok: a concurrent clean may have removed it already - build_info_json_path.unlink(missing_ok=True) - except OSError as unlink_err: - # The later write re-reads the file, so a kept unreadable copy - # fails again with a misattributed error; name the real cause - _LOGGER.warning( - "Could not remove unreadable build_info.json: %s", unlink_err - ) - sources_changed = True # Write build_info header and JSON metadata if sources_changed: @@ -414,6 +398,54 @@ def generate_version_h(): ) +def _build_info_stale( + h_path: Path, cpp_path: Path, json_path: Path, config_hash: int +) -> bool: + """Whether the build-info sources must regenerate (missing or stale).""" + if not h_path.exists() or not cpp_path.exists(): + _LOGGER.debug("Build info files missing; regenerating") + return True + try: + existing = json.loads(json_path.read_text(encoding="utf-8")) + except FileNotFoundError: + # An absent build_info.json is stale, not damaged; rebuild quietly + _LOGGER.debug("Build info JSON missing; regenerating") + return True + except (ValueError, OSError) as err: + # ValueError covers both JSONDecodeError and UnicodeDecodeError; + # unlink so the regenerating write never re-reads the bad copy. + # "Unreadable" not "damaged": EACCES/EISDIR land here too + _LOGGER.warning("Regenerating unreadable build_info.json: %s", err) + try: + # missing_ok: a concurrent clean may have removed it already + json_path.unlink(missing_ok=True) + except OSError as unlink_err: + # The later write re-reads the file, so a kept unreadable copy + # fails again with a misattributed error; name the real cause + _LOGGER.warning( + "Could not remove unreadable build_info.json: %s", unlink_err + ) + return True + if not isinstance(existing, dict): + # Valid JSON that is not an object (truncated or hand-edited) is + # stale, not a traceback + _LOGGER.debug("Build info JSON malformed; regenerating") + return True + if ( + existing.get("config_hash") != config_hash + or existing.get("esphome_version") != __version__ + ): + _LOGGER.debug( + "Build info stale (config_hash %s -> %s, version %s -> %s)", + existing.get("config_hash"), + config_hash, + existing.get("esphome_version"), + __version__, + ) + return True + return False + + def get_build_info() -> tuple[int, int, str, str]: """Calculate build_info values from current config. diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 792d7dab61..ef24f99953 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -929,11 +929,11 @@ def test_split_flag_entry_non_string_is_clean() -> None: def test_source_kind_map_shape() -> None: - """The kind values the native compile rules key on, and the deliberate - AS/ASPP merge (.s and .S both map to asm).""" + """The kind values the native compile rules key on; the AS/ASPP split + matches SCons (.S preprocessed, .s plain assembler).""" - assert set(SOURCE_KIND_FOR_SUFFIX.values()) == {"c", "cxx", "asm"} + assert set(SOURCE_KIND_FOR_SUFFIX.values()) == {"c", "cxx", "asm", "aspp"} assert SOURCE_KIND_FOR_SUFFIX[".s"] == "asm" - assert SOURCE_KIND_FOR_SUFFIX[".S"] == "asm" + assert SOURCE_KIND_FOR_SUFFIX[".S"] == "aspp" assert SOURCE_KIND_FOR_SUFFIX[".c"] == "c" assert SOURCE_KIND_FOR_SUFFIX[".cpp"] == "cxx" diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index c73a5c5789..0a53dba9c2 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -2441,3 +2441,39 @@ def test_copy_src_tree_ignores_removed_generated_file( # file was removed and regenerated, not that it triggered sources_changed. new_json = json.loads(build_info_json_path.read_text()) assert new_json["config_hash"] == 0xDEADBEEF + + +@pytest.mark.parametrize( + ("case", "content", "expected"), + [ + ("files missing", None, True), + ("json missing", "ABSENT", True), + ("json unreadable", "not json", True), + # Valid JSON that is not an object is stale, not an AttributeError + ("json not an object", "[]", True), + ("hash mismatch", {"config_hash": 2, "esphome_version": "CURRENT"}, True), + ("version mismatch", {"config_hash": 1, "esphome_version": "0.0.0"}, True), + ("matching record", {"config_hash": 1, "esphome_version": "CURRENT"}, False), + ], +) +def test_build_info_stale_branches( + tmp_path: Path, case: str, content, expected: bool +) -> None: + """Missing files, an unreadable JSON, a hash or version mismatch each + regenerate; a matching record does not.""" + from esphome.const import __version__ + from esphome.writer import _build_info_stale + + h = tmp_path / "build_info_data.h" + cpp = tmp_path / "build_info_data.cpp" + info = tmp_path / "build_info.json" + if content is not None: + h.write_text("") + cpp.write_text("") + if isinstance(content, dict): + if content.get("esphome_version") == "CURRENT": + content["esphome_version"] = __version__ + info.write_text(json.dumps(content)) + elif isinstance(content, str) and content != "ABSENT": + info.write_text(content) + assert _build_info_stale(h, cpp, info, 1) is expected, case From 8623b6836d3d4a87cb670c2b7fb81409d593df7d Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:44:28 +0000 Subject: [PATCH 21/65] Bump bundled esphome-device-builder to 1.13.0 (#18743) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f27d51059..d46f01838e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.12.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.13.0 RUN \ platformio settings set enable_telemetry No \ From 169c0550119a96f1677029b1478542c4e38390a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edvard=20Filistovi=C4=8D?= Date: Tue, 25 Aug 2026 04:25:44 +0300 Subject: [PATCH 22/65] [wifi] libretiny: reset the STA state on synchronous connect failure (#18719) --- esphome/components/wifi/wifi_component.h | 8 +++++ .../wifi/wifi_component_libretiny.cpp | 35 ++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index ada7be4ba4..382d3d5932 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -951,6 +951,14 @@ class WiFiComponent final : public Component { // On ESP8266, written from SDK system context (wifi_event_callback) — // uint8_t writes are atomic on Xtensa LX106 so no synchronization is needed. uint8_t sta_state_{0}; +#endif +#ifdef USE_LIBRETINY + // First attempt since STA-up (re-armed on every STA off->on); the + // pre-attempt teardown is skipped then. + bool lt_first_connect_attempt_{true}; + // A self-inflicted disconnect from that teardown is pending; it must not + // consume an ignored-disconnect slot. + bool lt_teardown_event_pending_{false}; #endif RetryHiddenMode retry_hidden_mode_{RetryHiddenMode::BLIND_RETRY}; RoamingState roaming_state_{RoamingState::IDLE}; diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index e3c08416e8..63a63e7342 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -115,6 +115,8 @@ bool WiFiComponent::wifi_mode_(optional sta, optional ap) { if (enable_sta && !current_sta) { ESP_LOGV(TAG, "Enabling STA"); + // Fresh STA stack: skip the pre-attempt teardown again. + this->lt_first_connect_attempt_ = true; } else if (!enable_sta && current_sta) { ESP_LOGV(TAG, "Disabling STA"); } @@ -202,10 +204,21 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { if (!this->wifi_mode_(true, {})) return false; - String ssid = WiFi.SSID(); - if (ssid && strcmp(ssid.c_str(), ap.ssid_.c_str()) != 0) { - WiFi.disconnect(); + // Tear down any live session so begin() re-fires its events; skipped on the + // first attempt after STA-up (nothing to tear down, and BK7231N on the older + // Beken SDK did not come back from it). The flag is per-attempt and armed + // only for a live session: an idle disconnect may emit no event, and a stale + // flag would swallow this attempt's first real failure. + this->lt_teardown_event_pending_ = false; + if (!this->lt_first_connect_attempt_) { + const bool was_live = WiFi.status() == WL_CONNECTED; + if (WiFi.disconnect()) { + this->lt_teardown_event_pending_ = was_live; + } else { + ESP_LOGD(TAG, "Pre-connect teardown returned false"); + } } + this->lt_first_connect_attempt_ = false; #ifdef USE_WIFI_MANUAL_IP if (!this->wifi_sta_ip_config_(ap.get_manual_ip())) { @@ -227,7 +240,10 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { ap.get_channel(), // 0 = auto ap.has_bssid() ? ap.get_bssid().data() : NULL); if (status != WL_CONNECTED) { - ESP_LOGW(TAG, "esp_wifi_connect failed: %d", status); + ESP_LOGW(TAG, "WiFi.begin failed: %d", status); + // Without this reset the state machine stays at CONNECTING and each retry + // stalls for the full connect timeout (46 s). + this->sta_state_ = static_cast(LTWiFiSTAState::ERROR_FAILED); return false; } @@ -455,6 +471,9 @@ void WiFiComponent::wifi_process_event_(LTWiFiEvent *event) { break; } case ESPHOME_EVENT_ID_WIFI_STA_CONNECTED: { + // Processed in queue order, so a teardown event still ahead of this + // CONNECTED was already consumed; a leftover flag is stale. + this->lt_teardown_event_pending_ = false; auto &it = event->data.sta_connected; char bssid_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; format_mac_addr_upper(it.bssid, bssid_buf); @@ -482,6 +501,14 @@ void WiFiComponent::wifi_process_event_(LTWiFiEvent *event) { case ESPHOME_EVENT_ID_WIFI_STA_DISCONNECTED: { auto &it = event->data.sta_disconnected; + // Consume the disconnect our own teardown queued, without spending an + // ignore slot. Ungated on SSID and state: the flag is armed only for this + // attempt's teardown of a live session. + if (this->lt_teardown_event_pending_ && it.reason != WIFI_REASON_NO_AP_FOUND) { + this->lt_teardown_event_pending_ = false; + break; + } + // LibreTiny can send spurious disconnect events with empty ssid/bssid during connection. // These are typically "Association Leave" events that don't indicate actual failures: // [W][wifi_lt]: Disconnected ssid='' bssid=00:00:00:00:00:00 reason='Association Leave' From 90927ba788d2c00dbb505d16d0b43c64ba630875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edvard=20Filistovi=C4=8D?= Date: Tue, 25 Aug 2026 06:57:01 +0300 Subject: [PATCH 23/65] [rp2_ble_tracker] Automation triggers and scan actions (#18717) --- .../components/rp2_ble_tracker/__init__.py | 110 ++++++++++++- .../components/rp2_ble_tracker/automation.h | 47 ++++++ .../rp2_ble_tracker/rp2_ble_tracker.cpp | 152 ++++++++++++------ .../rp2_ble_tracker/rp2_ble_tracker.h | 47 ++++-- .../rp2_ble_tracker/__init__.py | 0 .../config/test_automations.yaml | 45 ++++++ .../test_automations_codegen.py | 60 +++++++ .../rp2_ble_tracker/common-automations.yaml | 54 +++++++ .../test-automations.rp2040-ard.yaml | 3 + 9 files changed, 446 insertions(+), 72 deletions(-) create mode 100644 esphome/components/rp2_ble_tracker/automation.h create mode 100644 tests/component_tests/rp2_ble_tracker/__init__.py create mode 100644 tests/component_tests/rp2_ble_tracker/config/test_automations.yaml create mode 100644 tests/component_tests/rp2_ble_tracker/test_automations_codegen.py create mode 100644 tests/components/rp2_ble_tracker/common-automations.yaml create mode 100644 tests/components/rp2_ble_tracker/test-automations.rp2040-ard.yaml diff --git a/esphome/components/rp2_ble_tracker/__init__.py b/esphome/components/rp2_ble_tracker/__init__.py index 7709df9899..b744aee31c 100644 --- a/esphome/components/rp2_ble_tracker/__init__.py +++ b/esphome/components/rp2_ble_tracker/__init__.py @@ -4,14 +4,15 @@ Scan modes: continuous: true — scan runs forever; never stops automatically. continuous: false — a started scan runs for `duration`, then stops. The first start is external too; nothing starts a non-continuous - scan on boot. Until start/stop automation actions land - (follow-up PR), starting means a lambda: - `id(my_tracker).start_scan();`. + scan on boot — use the rp2_ble_tracker.start_scan action + (e.g. from api: on_client_connected:). """ +from esphome import automation import esphome.codegen as cg from esphome.components import ble_device_base, ota, rp2040_ble -from esphome.components.const import CONF_SCAN_PARAMETERS, CONF_WINDOW +from esphome.components.ble_device_base import automation as ble_automation +from esphome.components.const import CONF_ON_SCAN_END, CONF_SCAN_PARAMETERS, CONF_WINDOW from esphome.components.rp2040_ble import CONF_RP2040_BLE_ID import esphome.config_validation as cv from esphome.const import ( @@ -20,7 +21,13 @@ from esphome.const import ( CONF_DURATION, CONF_ID, CONF_INTERVAL, + CONF_MANUFACTURER_ID, + CONF_ON_BLE_ADVERTISE, + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, + CONF_ON_BLE_SERVICE_DATA_ADVERTISE, + CONF_SERVICE_UUID, ) +from esphome.core import ID from esphome.types import ConfigType DEPENDENCIES = ["rp2"] @@ -34,6 +41,14 @@ RP2BLETracker = rp2_ble_tracker_ns.class_( "RP2BLETracker", ble_device_base.BLEHub, cg.Component ) +StartScanAction = rp2_ble_tracker_ns.class_("StartScanAction", automation.Action) +StopScanAction = rp2_ble_tracker_ns.class_("StopScanAction", automation.Action) + +ESPBTAdvertiseTrigger = ble_automation.ESPBTAdvertiseTrigger +BLEServiceDataAdvertiseTrigger = ble_automation.BLEServiceDataAdvertiseTrigger +BLEManufacturerDataAdvertiseTrigger = ble_automation.BLEManufacturerDataAdvertiseTrigger +BLEEndOfScanTrigger = ble_automation.BLEEndOfScanTrigger + # interval defaults to 100 ms with the shared 30 ms window, a 30 % duty cycle — # the same defaults as bk72xx_ble_tracker, leaving the radio mostly free for @@ -48,6 +63,24 @@ CONFIG_SCHEMA = cv.Schema( cv.GenerateID(): cv.declare_id(RP2BLETracker), cv.GenerateID(CONF_RP2040_BLE_ID): cv.use_id(rp2040_ble.RP2040BLE), cv.Optional(CONF_SCAN_PARAMETERS, default={}): SCAN_PARAMETERS_SCHEMA, + cv.Optional(CONF_ON_BLE_ADVERTISE): ble_automation.advertise_trigger_schema( + ESPBTAdvertiseTrigger + ), + cv.Optional( + CONF_ON_BLE_SERVICE_DATA_ADVERTISE + ): ble_automation.uuid_trigger_schema( + BLEServiceDataAdvertiseTrigger, + {cv.Required(CONF_SERVICE_UUID): ble_device_base.bt_uuid}, + ), + cv.Optional( + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE + ): ble_automation.uuid_trigger_schema( + BLEManufacturerDataAdvertiseTrigger, + {cv.Required(CONF_MANUFACTURER_ID): ble_device_base.bt_uuid}, + ), + cv.Optional(CONF_ON_SCAN_END): ble_automation.scan_end_trigger_schema( + BLEEndOfScanTrigger + ), } ).extend(cv.COMPONENT_SCHEMA) @@ -76,4 +109,71 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_scan_window(ble_device_base.to_ble_units(scan[CONF_WINDOW]))) cg.add(var.set_scan_duration(scan[CONF_DURATION].total_milliseconds)) cg.add(var.set_scan_active(scan[CONF_ACTIVE])) - cg.add(var.set_scan_continuous(scan[CONF_CONTINUOUS])) + cg.add(var.set_configured_continuous(scan[CONF_CONTINUOUS])) + + for conf in config.get(CONF_ON_BLE_ADVERTISE, []): + await ble_automation.advertise_trigger_to_code(conf, var) + + for trigger_key, uuid_key, setter_prefix in ( + (CONF_ON_BLE_SERVICE_DATA_ADVERTISE, CONF_SERVICE_UUID, "set_service_uuid"), + ( + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, + CONF_MANUFACTURER_ID, + "set_manufacturer_uuid", + ), + ): + for conf in config.get(trigger_key, []): + await ble_automation.uuid_trigger_to_code( + conf, var, uuid_key, setter_prefix + ) + + for conf in config.get(CONF_ON_SCAN_END, []): + await ble_automation.scan_end_trigger_to_code(conf, var) + + +@automation.register_action( + "rp2_ble_tracker.start_scan", + StartScanAction, + cv.Schema( + { + cv.GenerateID(): cv.use_id(RP2BLETracker), + cv.Optional(CONF_CONTINUOUS): cv.templatable(cv.boolean), + } + ), + synchronous=True, +) +async def start_scan_action_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: list, +) -> cg.MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + if (continuous := config.get(CONF_CONTINUOUS)) is not None: + template_ = await cg.templatable(continuous, args, cg.bool_) + cg.add(var.set_continuous(template_)) + return var + + +@automation.register_action( + "rp2_ble_tracker.stop_scan", + StopScanAction, + automation.maybe_simple_id( + cv.Schema( + { + cv.GenerateID(): cv.use_id(RP2BLETracker), + } + ) + ), + synchronous=True, +) +async def stop_scan_action_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: list, +) -> cg.MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + return var diff --git a/esphome/components/rp2_ble_tracker/automation.h b/esphome/components/rp2_ble_tracker/automation.h new file mode 100644 index 0000000000..a3cede236a --- /dev/null +++ b/esphome/components/rp2_ble_tracker/automation.h @@ -0,0 +1,47 @@ +// Scan-control actions for rp2_ble_tracker. The automation triggers are the +// neutral ble_device_base classes (ble_device_base/automation.h). + +#pragma once + +#ifdef USE_RP2 + +#include "rp2_ble_tracker.h" + +#include "esphome/core/automation.h" +#include "esphome/core/helpers.h" + +namespace esphome::rp2_ble_tracker { + +template class StartScanAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(bool, continuous) + void play(const Ts &...x) override { + // With continuous: set, the action wins. Without it, the configured value + // is used - stop_scan() clears the runtime flag permanently, so a bare + // stop_scan/start_scan pair would otherwise never resume continuous mode. + const bool want = + this->continuous_.has_value() ? this->continuous_.value(x...) : this->parent_->configured_continuous(); + if (this->parent_->scan_running()) { + // Same mode on a running scan is a no-op (esp32 parity): re-anchoring + // the duration window here would let a repeated action keep a one-shot + // scan alive forever. A real mode switch re-anchors so a change to + // one-shot runs a full duration from now. + if (want != this->parent_->scan_continuous()) { + this->parent_->set_scan_continuous(want); + this->parent_->restart_scan_duration(); + } + return; + } + this->parent_->set_scan_continuous(want); + this->parent_->start_scan(); + } +}; + +template class StopScanAction final : public Action, public Parented { + public: + void play(const Ts &...x) override { this->parent_->stop_scan(); } +}; + +} // namespace esphome::rp2_ble_tracker + +#endif // USE_RP2 diff --git a/esphome/components/rp2_ble_tracker/rp2_ble_tracker.cpp b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.cpp index 06beb186ae..a0a04b981d 100644 --- a/esphome/components/rp2_ble_tracker/rp2_ble_tracker.cpp +++ b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.cpp @@ -11,10 +11,8 @@ namespace esphome::rp2_ble_tracker { static const char *const TAG = "rp2_ble_tracker"; -// Minimum interval between scan start attempts on an active stack. The -// controller start has no failure mode once HCI is WORKING, so this fires at -// most once per enable cycle today; the floor is insurance against a future -// scan_start() failure being retried every main-loop iteration. +// Floor between controller start attempts; insurance against a failing +// scan_start() being retried every loop. static constexpr uint32_t SCAN_START_RETRY_MS = 1000; // One BLE scan unit in milliseconds; the controller programs interval/window in these units. @@ -32,7 +30,8 @@ void RP2BLETracker::setup() { // the OTA download on the shared CYW43 radio. Mirrors esp32_ble_tracker. ota::get_global_ota_callback()->add_global_state_listener(this); #endif - if (!this->scan_continuous_) { + // An on_boot start_scan runs before setup(); parking here would strand it. + if (!this->scan_continuous_ && !this->scan_running_ && !this->pending_start_) { // Nothing to do until an external start_scan(); the loop is re-enabled there. this->disable_loop(); } @@ -41,12 +40,21 @@ void RP2BLETracker::setup() { #ifdef USE_OTA_STATE_LISTENER void RP2BLETracker::on_ota_global_state(ota::OTAState state, float progress, uint8_t error, ota::OTAComponent *comp) { if (state == ota::OTA_STARTED) { + // Set before stop_scan(): its on_scan_end automations run synchronously and + // may call start_scan(), which must defer instead of resuming the radio. + this->ota_in_progress_ = true; this->scan_continuous_before_ota_ = this->scan_continuous_; - // A one-shot scan counts as pending when it is running or still retrying - // its start (loop enabled); captured before stop_scan() disables the loop. - this->scan_pending_before_ota_ = !this->scan_continuous_ && (this->scan_running_ || this->is_in_loop_state()); + // A one-shot scan counts as pending when it is running, latched, or still + // retrying its start (loop enabled); captured before stop_scan() parks it. + this->scan_pending_before_ota_ = + !this->scan_continuous_ && (this->scan_running_ || this->pending_start_ || this->is_in_loop_state()); + // The pause's own stop is not a user stop, so it must not clear the latches + // captured just above. + this->ota_pausing_ = true; this->stop_scan(); + this->ota_pausing_ = false; } else if (state == ota::OTA_ERROR || state == ota::OTA_ABORT) { + this->ota_in_progress_ = false; // On success the device reboots, so restore only on a failed/aborted update; // loop()'s retry branch restarts the scan on its next iteration. if (this->scan_continuous_before_ota_) { @@ -54,9 +62,7 @@ void RP2BLETracker::on_ota_global_state(ota::OTAState state, float progress, uin this->scan_continuous_ = true; this->enable_loop(); } - // A one-shot scan interrupted by the OTA resumes for a fresh duration - // rather than silently staying idle — an OTA failure does not reboot, so - // nothing external would restart it. + // A failed OTA does not reboot, so nothing else would restart a one-shot. if (this->scan_pending_before_ota_) { this->scan_pending_before_ota_ = false; this->enable_loop(); @@ -66,27 +72,34 @@ void RP2BLETracker::on_ota_global_state(ota::OTAState state, float progress, uin #endif // USE_OTA_STATE_LISTENER void RP2BLETracker::loop() { +#ifdef USE_OTA_STATE_LISTENER + // Keeps "no radio during an OTA" local instead of emergent from the + // parking sites. + if (this->ota_in_progress_) + return; +#endif const uint32_t now = App.get_loop_component_start_time(); + if (this->pending_start_ && this->parent_->is_active()) { + // Latched start, applied once the stack is ACTIVE; earlier attempts would + // fail and arm the retry floor for nothing. + this->pending_start_ = false; + if (!this->scan_running_) + this->start_scan_(); + } // Deliver held scannable advertisements whose scan response never arrived — // unmerged after the merger's timeout. if (!this->merger_.empty()) this->merger_.sweep(now); if (this->scan_running_ && !this->parent_->is_active()) { - // The controller was disabled underneath us (e.g. a lambda calling - // rp2040_ble's disable()); the scan died with the stack. Reconcile so the - // retry branch below takes over once the user re-enables the stack. + // Stack disabled underneath us; reconcile so the retry branch takes over. this->scan_running_ = false; this->fire_scan_end_(); } if (!this->scan_running_) { - // A scan should be running but is not: continuous mode is always in this - // state until the start succeeds, and non-continuous mode only reaches - // here between start_scan() and a successful controller start, because - // stop_scan_() disables the loop otherwise. + // Should be scanning but is not: continuous until the start succeeds, + // one-shot only between start_scan() and a successful controller start. if (!this->parent_->is_active()) { - // Stack not up (still booting, or the user called disable()) — - // scan_start() cannot succeed, so there is nothing to attempt; scanning - // starts on the first iteration after HCI reaches WORKING. + // Stack not up: scan_start() cannot succeed yet. return; } if (now - this->last_scan_start_attempt_ >= SCAN_START_RETRY_MS) { @@ -107,7 +120,7 @@ void RP2BLETracker::loop() { // Non-continuous mode: run for scan_duration_ ms, then stop and fire on_scan_end. // Restart is driven externally (e.g. api: on_client_connected:). - if (now - this->scan_period_start_ >= this->scan_duration_) { + if (now - this->scan_start_time_ >= this->scan_duration_) { this->stop_scan_(); } } @@ -126,24 +139,20 @@ void RP2BLETracker::dump_config() { YESNO(this->scan_continuous_)); } -// GAP advertising event types as BTstack reports them (Core spec advertising -// report event types; the tracker deliberately does not include BTstack -// headers). ADV_IND and ADV_SCAN_IND are the scannable types. +// Core spec advertising report event types (BTstack headers stay out of this +// TU). ADV_IND and ADV_SCAN_IND are the scannable ones. static constexpr uint8_t ADV_EVENT_TYPE_ADV_IND = 0; static constexpr uint8_t ADV_EVENT_TYPE_ADV_SCAN_IND = 2; static constexpr uint8_t ADV_EVENT_TYPE_SCAN_RSP = 4; -// Demux advertisements vs scan responses into the shared merger: BTstack -// delivers the pair as separate reports; a scannable advertisement is held -// until its scan response arrives and delivered as one merged frame. +// BTstack delivers the pair as separate reports; the merger holds a scannable +// advertisement until its response arrives. void RP2BLETracker::on_scan_report(const rp2040_ble::BLEScanReport &report) { if (report.adv_event_type == ADV_EVENT_TYPE_SCAN_RSP) { this->merger_.submit_scan_rsp(report.mac, report.rssi, report.addr_type, report.data, report.data_len); return; } - // Stash only while an active scan runs: a passive scan never gets a - // response, and after a stop nothing would sweep the merger, so a late - // report would surface minutes later as a fresh advertisement. + // Only while an active scan runs: nothing sweeps the merger after a stop. if (this->scan_running_ && this->scan_active_ && (report.adv_event_type == ADV_EVENT_TYPE_ADV_IND || report.adv_event_type == ADV_EVENT_TYPE_ADV_SCAN_IND)) { this->merger_.stash_adv(report.mac, report.rssi, report.addr_type, report.data, report.data_len, @@ -157,20 +166,49 @@ void RP2BLETracker::on_scan_report(const rp2040_ble::BLEScanReport &report) { void RP2BLETracker::start_scan() { // Mirrors esp32_ble_tracker::start_scan(): caller sets scan_continuous_ via // set_scan_continuous() first, then calls start_scan() to begin scanning. +#ifdef USE_OTA_STATE_LISTENER + if (this->ota_in_progress_) { + // Defer to the post-OTA resume path, carrying the requested mode. Not + // while ota_pausing_: scan_continuous_ is an artefact of the pause's own + // stop there, not intent. + if (!this->ota_pausing_) { + this->scan_continuous_before_ota_ = this->scan_continuous_; + this->scan_pending_before_ota_ = !this->scan_continuous_; + } + return; + } +#endif this->enable_loop(); + if (!this->is_ready() || !this->parent_->is_active()) { + // Pre-setup or stack not ACTIVE: latch, loop() applies it. + this->pending_start_ = true; + return; + } + // bk72xx force semantics: a user start jumps the floor only while the + // controller is healthy. loop()'s retry branch picks the request up. + if (this->last_start_failed_ && + App.get_loop_component_start_time() - this->last_scan_start_attempt_ < SCAN_START_RETRY_MS) { + return; + } this->start_scan_(); } +void RP2BLETracker::restart_scan_duration() { + if (!this->scan_running_) + return; // start_scan_() anchors the clock itself on the next real start + // One-shot clock only (bk72xx parity); re-anchoring the period would let + // repeated actions starve on_scan_end. Same clock as loop()'s now. + this->scan_start_time_ = App.get_loop_component_start_time(); +} + bool RP2BLETracker::request_scan_mode(bool active) { if (this->scan_active_ == active) return true; this->scan_active_ = active; // V: the proxy's "Setting scanner mode" line already narrates this at D. ESP_LOGV(TAG, "Scan mode %s", active ? "active" : "passive"); - // Apply to a running scan by restarting the CONTROLLER scan with the new - // mode, bypassing the tracker's stop/start bookkeeping: no on_scan_end (the - // scan logically continues, only the request mode changes), no period reset. - // An idle scanner picks the mode up on its next start. + // Restart the controller scan only: the scan logically continues, so no + // on_scan_end and no period reset. An idle scanner applies it on next start. if (this->scan_running_) { this->parent_->scan_stop(); if (!this->controller_scan_start_()) { @@ -184,20 +222,34 @@ bool RP2BLETracker::request_scan_mode(bool active) { } void RP2BLETracker::stop_scan() { + // Cancel a start latched before setup(); without this an on_boot + // start_scan/stop_scan pair would still start at the first loop(). + this->pending_start_ = false; this->scan_continuous_ = false; +#ifdef USE_OTA_STATE_LISTENER + // A user stop during the OTA is the latest intent; the pause's own stop + // (ota_pausing_) is exempt - it armed that state. + if (this->ota_in_progress_ && !this->ota_pausing_) { + this->scan_pending_before_ota_ = false; + this->scan_continuous_before_ota_ = false; + } +#endif this->stop_scan_(); - // stop_scan_() early-returns when no scan is running, so disable the loop - // here too: a scan that never came up (stack still powering on at OTA start) - // must not keep attempting scan_start() from the loop's retry branch. - this->disable_loop(); + // stop_scan_() early-returns when idle, so park here too - once set up, and + // re-checked: its synchronous on_scan_end may have restarted the scan. + if (this->is_ready() && !this->scan_running_ && !this->pending_start_) { + this->disable_loop(); + } } // Stamp-and-start for every controller scan attempt: the stamp keeps the // SCAN_START_RETRY_MS floor covering all callers, not only loop()'s retry. bool RP2BLETracker::controller_scan_start_() { this->last_scan_start_attempt_ = App.get_loop_component_start_time(); - return this->parent_->scan_start(static_cast(this->scan_interval_), - static_cast(this->scan_window_), this->scan_active_); + const bool ok = this->parent_->scan_start(static_cast(this->scan_interval_), + static_cast(this->scan_window_), this->scan_active_); + this->last_start_failed_ = !ok; + return ok; } void RP2BLETracker::start_scan_() { @@ -208,19 +260,15 @@ void RP2BLETracker::start_scan_() { return; this->scan_running_ = true; - // Log every explicit start at DEBUG — stop_scan_() logs every stop at DEBUG, and - // in non-continuous mode each period is an explicit start, so asymmetric logging - // would read as the scanner failing to come back up. + // Symmetric with stop_scan_()'s stop log; asymmetry would read as the + // scanner failing to come back. ESP_LOGD(TAG, "Scan started (%s, window=%.0fms, interval=%.0fms)", this->scan_active_ ? LOG_STR_LITERAL("active") : LOG_STR_LITERAL("passive"), this->scan_window_ * BLE_SCAN_UNIT_MS, this->scan_interval_ * BLE_SCAN_UNIT_MS); - // Re-anchor the scan period to every successful start — first start (so the - // period counts from the scan, not from boot) and every restart after a stop (so - // resuming after longer than scan_duration, e.g. a failed OTA restoring continuous - // mode 10 minutes later, does not fire on_scan_end before an advertisement can - // arrive). Same clock as loop()'s `now`: a fresh millis() here would be ahead of - // the cached loop time and make the same-iteration period check underflow. + // Anchor the period to the scan, not to boot, so a restart after a long gap + // does not fire on_scan_end immediately. Same clock as loop()'s now. this->scan_period_start_ = App.get_loop_component_start_time(); + this->scan_start_time_ = this->scan_period_start_; } void RP2BLETracker::stop_scan_() { @@ -232,7 +280,9 @@ void RP2BLETracker::stop_scan_() { this->fire_scan_end_(); // Reset the period clock so on_scan_end does not double-fire; same clock as loop(). this->scan_period_start_ = App.get_loop_component_start_time(); - if (!this->scan_continuous_) { + // on_scan_end runs synchronously and may restart the scan; re-check before + // parking or that scan runs untimed. + if (!this->scan_continuous_ && !this->scan_running_ && !this->pending_start_) { // Nothing left to time; start_scan() re-enables the loop. this->disable_loop(); } diff --git a/esphome/components/rp2_ble_tracker/rp2_ble_tracker.h b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.h index 431f2daec7..af5250fe7e 100644 --- a/esphome/components/rp2_ble_tracker/rp2_ble_tracker.h +++ b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.h @@ -44,11 +44,20 @@ class RP2BLETracker : public Component, void set_scan_duration(uint32_t scan_duration) { this->scan_duration_ = scan_duration; } void set_scan_active(bool scan_active) { this->scan_active_ = scan_active; } void set_scan_continuous(bool scan_continuous) { this->scan_continuous_ = scan_continuous; } + void set_configured_continuous(bool scan_continuous) { + this->configured_continuous_ = scan_continuous; + this->scan_continuous_ = scan_continuous; + } + bool scan_continuous() const { return this->scan_continuous_; } + bool configured_continuous() const { return this->configured_continuous_; } // ---- Public scan control ---- // Mirrors esp32_ble_tracker: set_scan_continuous() + start_scan() / stop_scan(). void start_scan(); void stop_scan(); + // Re-anchors the one-shot duration clock only (bk72xx parity); no-op while + // idle. Policy lives in the action. + void restart_scan_duration(); // ---- ble_device_base::BLEHub contract ---- void register_listener(ble_device_base::ESPBTDeviceListener *listener) { @@ -58,10 +67,8 @@ class RP2BLETracker : public Component, this->dispatcher_.set_raw_advertisement_callback(callback); } static constexpr ble_device_base::HubCapabilities get_capabilities() { - // BTstack delivers scan responses as separate advertisement reports; this - // tracker merges the pair before delivery (shared ScanResponseMerger, - // Bluedroid semantics). GATT is available when the BTstack connection - // backend is compiled in (bluetooth_proxy active). + // Scan responses arrive separately and are merged before delivery + // (Bluedroid semantics). GATT needs the BTstack connection backend. #ifdef USE_BLE_GATT_CLIENT constexpr bool has_gatt = true; #else @@ -77,8 +84,7 @@ class RP2BLETracker : public Component, bool request_scan_mode(bool active); // ---- rp2040_ble::BLEScanListener ---- - // Delivered by the controller's loop() on the ESPHome main loop — the - // IRQ → main-loop handoff already happened in the controller's queue. + // Delivered on the main loop; the controller's queue did the IRQ handoff. void on_scan_report(const rp2040_ble::BLEScanReport &report) override; protected: @@ -93,20 +99,29 @@ class RP2BLETracker : public Component, uint32_t scan_window_{48}; // 48 × 0.625 ms = 30 ms (30/100 = 30 %) uint32_t scan_duration_{300000}; uint32_t last_scan_start_attempt_{0}; // loop time of last start_scan_() attempt; rate-limits retries - uint32_t scan_period_start_{0}; // loop time at start of current scan period; rate-limits on_scan_end() - bool scan_running_{false}; - bool scan_active_{true}; + uint32_t scan_period_start_{0}; // continuous-mode on_scan_end period clock + uint32_t scan_start_time_{0}; // one-shot duration clock (bk72xx parity: kept separate from the period) + // Bit-packed (C++20 default member initializers on bit-fields); + // scan_continuous_ stays a plain bool because the merger binds its address. + bool scan_running_ : 1 {false}; + bool pending_start_ : 1 {false}; // start_scan() latched before setup() or while the stack is + // not ACTIVE; loop() applies it once it is + bool last_start_failed_ : 1 {false}; // last controller start failed; gates the public start_scan() floor + bool scan_active_ : 1 {true}; + bool configured_continuous_ : 1 {true}; // YAML scan_parameters.continuous; runtime stop_scan() must not lose it bool scan_continuous_{true}; #ifdef USE_OTA_STATE_LISTENER - bool scan_continuous_before_ota_{false}; // continuous mode saved at OTA start, restored on OTA failure - bool scan_pending_before_ota_{false}; // one-shot scan in flight at OTA start, resumed on OTA failure + // Resume intent for a failed/aborted OTA: seeded at OTA start, overwritten + // by a start/stop during the download, except from the pause's own stop. + bool scan_continuous_before_ota_ : 1 {false}; // resume continuous + bool scan_pending_before_ota_ : 1 {false}; // resume a one-shot scan + bool ota_in_progress_ : 1 {false}; // OTA holds the radio; start_scan() defers to the resume path + bool ota_pausing_ : 1 {false}; // inside the OTA's own stop_scan(); its latch clear is skipped #endif - // Shared adv + scan-response merge and frame dispatch (ble_device_base). - // All calls run on the main loop. Merger clock: stash_adv() reads the - // PARENT's cached loop time (on_scan_report runs inside rp2040_ble's queue - // drain), sweep() this component's — same App.loop() pass, so the delta - // stays non-negative and the 300 ms timeout holds. + // Shared merge + dispatch (ble_device_base), all on the main loop. + // stash_adv() uses the parent's cached loop time and sweep() this one's - + // same App.loop() pass, so the merger delta stays non-negative. ble_device_base::ScanResponseMerger merger_; ble_device_base::AdvDispatcher dispatcher_; }; diff --git a/tests/component_tests/rp2_ble_tracker/__init__.py b/tests/component_tests/rp2_ble_tracker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/rp2_ble_tracker/config/test_automations.yaml b/tests/component_tests/rp2_ble_tracker/config/test_automations.yaml new file mode 100644 index 0000000000..cf0d4f1567 --- /dev/null +++ b/tests/component_tests/rp2_ble_tracker/config/test_automations.yaml @@ -0,0 +1,45 @@ +esphome: + name: rp2-trigger-codegen + on_boot: + then: + - rp2_ble_tracker.start_scan: + continuous: true + # Bare form: restores the configured scan_parameters mode — no + # set_continuous emitted (asserted in the codegen test). + - rp2_ble_tracker.start_scan: + - rp2_ble_tracker.stop_scan + +rp2: + board: rpipicow + +rp2_ble_tracker: + scan_parameters: + continuous: false + active: false + on_ble_advertise: + - mac_address: + - AC:37:43:77:5F:4C + - 11:22:33:44:55:66 + then: + - lambda: 'char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; ESP_LOGD("t", "%s", x.address_str_to(addr));' + on_ble_service_data_advertise: + - service_uuid: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + mac_address: AC:37:43:77:5F:4C + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - service_uuid: ABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + on_ble_manufacturer_data_advertise: + - manufacturer_id: ABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - manufacturer_id: ABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - manufacturer_id: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + on_scan_end: + - then: + - lambda: 'ESP_LOGD("t", "end");' diff --git a/tests/component_tests/rp2_ble_tracker/test_automations_codegen.py b/tests/component_tests/rp2_ble_tracker/test_automations_codegen.py new file mode 100644 index 0000000000..fdcf2bcf75 --- /dev/null +++ b/tests/component_tests/rp2_ble_tracker/test_automations_codegen.py @@ -0,0 +1,60 @@ +"""Codegen tests for the tracker automations. + +The shared trigger classes (ble_device_base/automation.h) are compiled by the +rp2040 compile fixtures, but the codegen accounting — the getattr-built setter +spellings, the single set_continuous pin and the listener-count define — is +only checkable from the generated main, mirroring the bk72xx/ln882h tests.""" + +from collections.abc import Callable +from pathlib import Path +import re + +from esphome.components import ble_device_base +from tests.component_tests.helpers import get_define_value + + +def test_trigger_codegen( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("test_automations.yaml")) + + # on_ble_advertise: multi-mac filter (two addresses in one initializer list) + assert "set_addresses({0xAC3743775F4CULL, 0x112233445566ULL})" in main_cpp + # 128-bit service uuid goes out reversed (BLE wire order); single-mac filter + assert ( + "set_service_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," + "0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB})" in main_cpp + ) + assert "set_address(0xAC3743775F4CULL)" in main_cpp + # 32-bit middle branch of the width dispatch + assert "set_service_uuid32(0xABCDABCDULL)" in main_cpp + # All three manufacturer widths: getattr() builds these names as strings, + # so a misspelling only ever fails here. + assert "set_manufacturer_uuid16(0xABCDULL)" in main_cpp + assert "set_manufacturer_uuid32(0xABCDABCDULL)" in main_cpp + assert ( + "set_manufacturer_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," + "0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB})" in main_cpp + ) + # scan-control actions: templatable continuous lambda + parented actions. + # Exactly one set_continuous: the bare start_scan emits none, pinning the + # restore-configured-mode divergence from esp32 against a future default=. + assert main_cpp.count("->set_continuous(") == 1 + assert "startscanaction_id->set_continuous(" in main_cpp + assert "stopscanaction_id->set_parent(" in main_cpp + # scan_parameters continuous: false reaches the YAML-mode setter, not the + # runtime override. + assert "->set_configured_continuous(false)" in main_cpp + # active: false (non-default) flows through to the setter. + assert "->set_scan_active(false)" in main_cpp + # Constructor call, not just the declaration: the parent argument is what + # registers the trigger as a listener. + assert re.search( + r"new\(\w+\) ble_device_base::BLEEndOfScanTrigger\(\w+\)", main_cpp + ) + + # Seven triggers register as listeners; an undercount silently drops the + # last trigger at runtime (StaticVector::push_back past capacity), so the + # define is the assertion that matters most. + assert get_define_value(ble_device_base.LISTENER_COUNT_DEFINE) == "7" diff --git a/tests/components/rp2_ble_tracker/common-automations.yaml b/tests/components/rp2_ble_tracker/common-automations.yaml new file mode 100644 index 0000000000..273cb5bb66 --- /dev/null +++ b/tests/components/rp2_ble_tracker/common-automations.yaml @@ -0,0 +1,54 @@ +esphome: + on_boot: + then: + - rp2_ble_tracker.start_scan + - rp2_ble_tracker.start_scan: + continuous: true + # Lambda arm of the templatable value — different codegen instantiation. + - rp2_ble_tracker.start_scan: + continuous: !lambda return false; + - rp2_ble_tracker.stop_scan + - rp2_ble_tracker.stop_scan: ble_tracker + +rp2_ble_tracker: + on_ble_advertise: + - mac_address: AC:37:43:77:5F:4C + then: + - lambda: |- + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address is %s", x.address_str_to(addr)); + - mac_address: + - AC:37:43:77:5F:4C + - AC:37:43:77:5F:4D + then: + - lambda: |- + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address is %s", x.address_str_to(addr)); + on_ble_service_data_advertise: + - service_uuid: ABCD + # mac_address exercises the UUID triggers' set_address() codegen branch. + mac_address: AC:37:43:77:5F:4C + then: + - lambda: |- + ESP_LOGD("main", "Length of service data is %zu", x.size()); + - service_uuid: ABCDABCD + then: + - lambda: |- + ESP_LOGD("main", "32-bit service data is %zu", x.size()); + - service_uuid: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + then: + - lambda: |- + ESP_LOGD("main", "128-bit service data is %zu", x.size()); + on_ble_manufacturer_data_advertise: + - manufacturer_id: ABCD + then: + - lambda: |- + ESP_LOGD("main", "Length of manufacturer data is %zu", x.size()); + - manufacturer_id: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + then: + - lambda: |- + ESP_LOGD("main", "128-bit manufacturer data is %zu", x.size()); + on_scan_end: + - then: + - lambda: |- + ESP_LOGD("main", "Scan ended"); diff --git a/tests/components/rp2_ble_tracker/test-automations.rp2040-ard.yaml b/tests/components/rp2_ble_tracker/test-automations.rp2040-ard.yaml new file mode 100644 index 0000000000..6710ee4373 --- /dev/null +++ b/tests/components/rp2_ble_tracker/test-automations.rp2040-ard.yaml @@ -0,0 +1,3 @@ +packages: + rp2_ble_tracker: !include common.yaml + automations: !include common-automations.yaml From 44860ff5125af133276f8a66ded1790c9fd23637 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" <3060199+jesserockz@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:22:41 -0500 Subject: [PATCH 24/65] [time] Skip posix_tz.cpp when no timezone is configured (#18680) --- esphome/components/time/__init__.py | 8 ++++++++ tests/benchmarks/components/time/__init__.py | 2 +- tests/components/time/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index 94ff6ab051..7ad084493c 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -9,6 +9,7 @@ from esphome import automation from esphome.automation import Condition import esphome.codegen as cg from esphome.components.zephyr import zephyr_add_prj_conf +from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv from esphome.const import ( CONF_AT, @@ -475,3 +476,10 @@ async def to_code(config): async def time_has_time_to_code(config, condition_id, template_arg, args): paren = await cg.get_variable(config[CONF_ID]) return cg.new_Pvariable(condition_id, template_arg, paren) + + +# posix_tz.cpp is fully #ifdef'd on USE_TIME_TIMEZONE, set only when a +# timezone is configured or detected. +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {"posix_tz.cpp": "USE_TIME_TIMEZONE"} +) diff --git a/tests/benchmarks/components/time/__init__.py b/tests/benchmarks/components/time/__init__.py index 7f68003e29..8b4fd955f2 100644 --- a/tests/benchmarks/components/time/__init__.py +++ b/tests/benchmarks/components/time/__init__.py @@ -4,6 +4,6 @@ from tests.testing_helpers import ComponentManifestOverride def override_manifest(manifest: ComponentManifestOverride) -> None: async def to_code(config): - cg.add_build_flag("-DUSE_TIME_TIMEZONE") + cg.add_define("USE_TIME_TIMEZONE") manifest.to_code = to_code diff --git a/tests/components/time/__init__.py b/tests/components/time/__init__.py index 7f68003e29..8b4fd955f2 100644 --- a/tests/components/time/__init__.py +++ b/tests/components/time/__init__.py @@ -4,6 +4,6 @@ from tests.testing_helpers import ComponentManifestOverride def override_manifest(manifest: ComponentManifestOverride) -> None: async def to_code(config): - cg.add_build_flag("-DUSE_TIME_TIMEZONE") + cg.add_define("USE_TIME_TIMEZONE") manifest.to_code = to_code From 036e5cda7ee9318a09e6e3e0c4aab6e34fe7932b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 00:24:14 -0500 Subject: [PATCH 25/65] [core] Remove empty automation.cpp files from sensor, switch and output (#18745) --- esphome/components/output/automation.cpp | 8 -------- esphome/components/sensor/automation.cpp | 8 -------- esphome/components/switch/automation.cpp | 8 -------- 3 files changed, 24 deletions(-) delete mode 100644 esphome/components/output/automation.cpp delete mode 100644 esphome/components/sensor/automation.cpp delete mode 100644 esphome/components/switch/automation.cpp diff --git a/esphome/components/output/automation.cpp b/esphome/components/output/automation.cpp deleted file mode 100644 index 610da897d9..0000000000 --- a/esphome/components/output/automation.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include "automation.h" -#include "esphome/core/log.h" - -namespace esphome::output { - -static const char *const TAG = "output.automation"; - -} // namespace esphome::output diff --git a/esphome/components/sensor/automation.cpp b/esphome/components/sensor/automation.cpp deleted file mode 100644 index 977719db9b..0000000000 --- a/esphome/components/sensor/automation.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include "automation.h" -#include "esphome/core/log.h" - -namespace esphome::sensor { - -static const char *const TAG = "sensor.automation"; - -} // namespace esphome::sensor diff --git a/esphome/components/switch/automation.cpp b/esphome/components/switch/automation.cpp deleted file mode 100644 index 9a0221fe56..0000000000 --- a/esphome/components/switch/automation.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include "automation.h" -#include "esphome/core/log.h" - -namespace esphome::switch_ { - -static const char *const TAG = "switch.automation"; - -} // namespace esphome::switch_ From 791590c8a34cd3b8f7a0d3f4d7007fd04ea78474 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 00:24:20 -0500 Subject: [PATCH 26/65] [uptime] Skip the timestamp sensor source unless a timestamp sensor is configured (#18746) --- esphome/components/uptime/sensor/__init__.py | 3 ++- .../uptime/sensor/uptime_timestamp_sensor.cpp | 4 +-- .../uptime/sensor/uptime_timestamp_sensor.h | 4 +-- esphome/core/defines.h | 1 + tests/component_tests/uptime/__init__.py | 0 .../uptime/config/seconds.yaml | 20 ++++++++++++++ .../uptime/config/timestamp.yaml | 20 ++++++++++++++ tests/component_tests/uptime/test_uptime.py | 27 +++++++++++++++++++ 8 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 tests/component_tests/uptime/__init__.py create mode 100644 tests/component_tests/uptime/config/seconds.yaml create mode 100644 tests/component_tests/uptime/config/timestamp.yaml create mode 100644 tests/component_tests/uptime/test_uptime.py diff --git a/esphome/components/uptime/sensor/__init__.py b/esphome/components/uptime/sensor/__init__.py index 4b611ffff3..a13d6cd7fd 100644 --- a/esphome/components/uptime/sensor/__init__.py +++ b/esphome/components/uptime/sensor/__init__.py @@ -61,8 +61,9 @@ async def to_code(config: ConfigType) -> None: if time_id_config := config.get(CONF_TIME_ID): time_id = await cg.get_variable(time_id_config) cg.add(var.set_time(time_id)) + cg.add_define("USE_UPTIME_TIMESTAMP") FILTER_SOURCE_FILES = filter_source_files_from_defines( - {"uptime_timestamp_sensor.cpp": "USE_TIME"} + {"uptime_timestamp_sensor.cpp": "USE_UPTIME_TIMESTAMP"} ) diff --git a/esphome/components/uptime/sensor/uptime_timestamp_sensor.cpp b/esphome/components/uptime/sensor/uptime_timestamp_sensor.cpp index 4e0f06be1c..e8b8a21562 100644 --- a/esphome/components/uptime/sensor/uptime_timestamp_sensor.cpp +++ b/esphome/components/uptime/sensor/uptime_timestamp_sensor.cpp @@ -1,6 +1,6 @@ #include "uptime_timestamp_sensor.h" -#ifdef USE_TIME +#ifdef USE_UPTIME_TIMESTAMP #include "esphome/core/hal.h" #include "esphome/core/helpers.h" @@ -34,4 +34,4 @@ void UptimeTimestampSensor::dump_config() { } // namespace esphome::uptime -#endif // USE_TIME +#endif // USE_UPTIME_TIMESTAMP diff --git a/esphome/components/uptime/sensor/uptime_timestamp_sensor.h b/esphome/components/uptime/sensor/uptime_timestamp_sensor.h index 5b837cbce1..c0e00a14fe 100644 --- a/esphome/components/uptime/sensor/uptime_timestamp_sensor.h +++ b/esphome/components/uptime/sensor/uptime_timestamp_sensor.h @@ -2,7 +2,7 @@ #include "esphome/core/defines.h" -#ifdef USE_TIME +#ifdef USE_UPTIME_TIMESTAMP #include "esphome/components/sensor/sensor.h" #include "esphome/components/time/real_time_clock.h" @@ -25,4 +25,4 @@ class UptimeTimestampSensor final : public sensor::Sensor, public Component { } // namespace esphome::uptime -#endif // USE_TIME +#endif // USE_UPTIME_TIMESTAMP diff --git a/esphome/core/defines.h b/esphome/core/defines.h index a8aab65d4a..115ef2145d 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -192,6 +192,7 @@ #define USE_UART_DEBUGGER #define USE_UART_WAKE_LOOP_ON_RX #define USE_UPDATE +#define USE_UPTIME_TIMESTAMP #define USE_VALVE #define USE_WATER_HEATER #define USE_WATER_HEATER_VISUAL_OVERRIDES diff --git a/tests/component_tests/uptime/__init__.py b/tests/component_tests/uptime/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/uptime/config/seconds.yaml b/tests/component_tests/uptime/config/seconds.yaml new file mode 100644 index 0000000000..4d81a28011 --- /dev/null +++ b/tests/component_tests/uptime/config/seconds.yaml @@ -0,0 +1,20 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +time: + - platform: sntp + id: sntp_time + +sensor: + - platform: uptime + name: Uptime Seconds + type: seconds diff --git a/tests/component_tests/uptime/config/timestamp.yaml b/tests/component_tests/uptime/config/timestamp.yaml new file mode 100644 index 0000000000..e2eea374bb --- /dev/null +++ b/tests/component_tests/uptime/config/timestamp.yaml @@ -0,0 +1,20 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +time: + - platform: sntp + id: sntp_time + +sensor: + - platform: uptime + name: Uptime Timestamp + type: timestamp diff --git a/tests/component_tests/uptime/test_uptime.py b/tests/component_tests/uptime/test_uptime.py new file mode 100644 index 0000000000..d77f510659 --- /dev/null +++ b/tests/component_tests/uptime/test_uptime.py @@ -0,0 +1,27 @@ +"""The timestamp uptime sensor source is only compiled when that type is used, +so the define must follow the configured sensor type rather than time: alone.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.core import CORE + + +@pytest.mark.parametrize( + ("fixture", "emits"), + [ + ("seconds.yaml", False), + ("timestamp.yaml", True), + ], +) +def test_timestamp_define_follows_sensor_type( + fixture: str, + emits: bool, + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path(fixture)) + defines = {define.name for define in CORE.defines} + assert ("USE_UPTIME_TIMESTAMP" in defines) is emits From c6d423159c9f48f86a81f2bde29fac013f8b9852 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 00:26:18 -0500 Subject: [PATCH 27/65] [esp32] Only build the mbedTLS certificate bundle when a component needs it (#18747) --- esphome/components/audio/__init__.py | 3 + esphome/components/esp32/__init__.py | 128 ++++++++++++------ esphome/components/esp32/const.py | 1 + esphome/components/http_request/__init__.py | 4 +- .../certificate_bundle_arduino_tls.yaml | 9 ++ .../esp32/config/certificate_bundle_full.yaml | 9 ++ .../certificate_bundle_http_request.yaml | 14 ++ .../config/certificate_bundle_sdkconfig.yaml | 9 ++ tests/component_tests/esp32/test_esp32.py | 58 ++++++++ tests/components/esp32/test.esp32-idf.yaml | 2 +- 10 files changed, 190 insertions(+), 47 deletions(-) create mode 100644 tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml create mode 100644 tests/component_tests/esp32/config/certificate_bundle_full.yaml create mode 100644 tests/component_tests/esp32/config/certificate_bundle_http_request.yaml create mode 100644 tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 277df0506a..2a5304be77 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -7,6 +7,7 @@ from esphome.components.esp32 import ( add_idf_component, add_idf_sdkconfig_option, include_builtin_idf_component, + require_certificate_bundle, ) import esphome.config_validation as cv from esphome.const import ( @@ -335,6 +336,8 @@ def _emit_memory_pair(value: str | None, psram_key: str, internal_key: str) -> N async def to_code(config: ConfigType) -> None: # Re-enable ESP-IDF's HTTP client (excluded by default to save compile time) include_builtin_idf_component("esp_http_client") + # HTTPS streams verify the server against the root certificate bundle + require_certificate_bundle() add_idf_component( name="esphome/esp-audio-libs", diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index cde0cfd68b..fca63e9a25 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -65,6 +65,7 @@ from .boards import BOARDS, STANDARD_BOARDS from .const import ( KEY_ARDUINO_LIBRARIES, KEY_BOARD, + KEY_CERT_BUNDLE, KEY_COMPONENTS, KEY_ESP32, KEY_EXCLUDE_COMPONENTS, @@ -343,6 +344,10 @@ ARDUINO_LIBRARY_IDF_COMPONENTS: dict[str, tuple[str, ...]] = { "Zigbee": ("espressif__esp-zigbee-lib", "espressif__esp-zboss-lib"), } +# Arduino libraries whose sources reference esp_crt_bundle_attach without a +# CONFIG_MBEDTLS_CERTIFICATE_BUNDLE guard, so enabling them needs the bundle. +ARDUINO_LIBRARIES_NEEDING_CERT_BUNDLE = frozenset({"NetworkClientSecure"}) + # Arduino library to Arduino library dependencies # When enabling one library, also enable its dependencies # Kconfig "select" statements don't work with CONFIG_ARDUINO_SELECTIVE_COMPILATION @@ -644,6 +649,17 @@ class RawSdkconfigValue: SdkconfigValueType = bool | int | HexInt | str | RawSdkconfigValue +def set_idf_sdkconfig_default(name: str, value: SdkconfigValueType) -> None: + """Set an sdkconfig option unless it is already set. + + For the FINAL priority reconcile jobs: they run after every to_code, + including the user's sdkconfig_options, and must not override an + existing value. + """ + if name not in CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]: + add_idf_sdkconfig_option(name, value) + + def add_idf_sdkconfig_option(name: str, value: SdkconfigValueType): """Set an esp-idf sdkconfig value.""" CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS][name] = value @@ -788,6 +804,10 @@ def _enable_arduino_library(name: str) -> None: # Also enable any required IDF components for idf_component in ARDUINO_LIBRARY_IDF_COMPONENTS.get(name, ()): include_builtin_idf_component(idf_component) + if not ARDUINO_LIBRARIES_NEEDING_CERT_BUNDLE.isdisjoint( + {name, *ARDUINO_LIBRARY_DEPENDENCIES.get(name, ())} + ): + require_certificate_bundle() def add_extra_script(stage: str, filename: str, path: Path): @@ -1735,6 +1755,16 @@ def require_vfs_termios() -> None: CORE.data[KEY_VFS_TERMIOS_REQUIRED] = True +def require_certificate_bundle() -> None: + """Enable the mbedTLS root certificate bundle for this build. + + The bundle is off by default; components that verify TLS server + certificates (http_request, audio streaming) call this so the bundle is + compiled and gen_crt_bundle runs only when something uses it. + """ + CORE.data[KEY_ESP32][KEY_CERT_BUNDLE] = True + + def require_full_certificate_bundle() -> None: """Request the full certificate bundle instead of the common-CAs-only bundle. @@ -1744,6 +1774,7 @@ def require_full_certificate_bundle() -> None: Call this from components that need to connect to services using uncommon CAs. """ + require_certificate_bundle() CORE.data[KEY_ESP32][KEY_FULL_CERT_BUNDLE] = True @@ -2218,6 +2249,31 @@ async def _set_libc_picolibc_newlib_compat() -> None: ) +@coroutine_with_priority(CoroPriority.FINAL) +async def _reconcile_certificate_bundle_sdkconfig() -> None: + """Enable the mbedTLS certificate bundle only when something asked for it. + + Runs at FINAL priority so every require_certificate_bundle() call has + happened. Without a request the bundle is disabled, which skips + esp_crt_bundle.c, the gen_crt_bundle step and the x509_crt_bundle.S embed. + A user-supplied sdkconfig_options value takes precedence. + """ + data = CORE.data[KEY_ESP32] + enabled = data.get(KEY_CERT_BUNDLE, False) + set_idf_sdkconfig_default("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", enabled) + if not enabled: + return + # Use CMN (common CAs) bundle by default to save ~51KB flash + # CMN covers CAs with >1% market share (~99% of websites) + # Components needing uncommon CAs can call require_full_certificate_bundle() + use_full_bundle = data.get(KEY_FULL_CERT_BUNDLE, False) + set_idf_sdkconfig_default( + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", use_full_bundle + ) + if not use_full_bundle: + set_idf_sdkconfig_default("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", True) + + @coroutine_with_priority(CoroPriority.FINAL) async def _reconcile_network_sdkconfig() -> None: """Reconcile WiFi/Ethernet/Bluetooth/coexistence sdkconfig flags. @@ -2229,37 +2285,31 @@ async def _reconcile_network_sdkconfig() -> None: always takes precedence. """ net = CORE.data[KEY_ESP32].get(KEY_NETWORK_SDKCONFIG, NetworkSdkconfigData()) - opts = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] is_arduino = CORE.using_arduino - def set_opt(name: str, value: SdkconfigValueType) -> None: - # User sdkconfig_options (applied during to_code) win. - if name not in opts: - add_idf_sdkconfig_option(name, value) - # Bluetooth: only ever enable when requested. The IDF default is off. # According to the IDF docs, only one of 4.2 or 5.0 should be enabled. if net.bluetooth: - set_opt("CONFIG_BT_ENABLED", True) - set_opt("CONFIG_BT_BLE_42_FEATURES_SUPPORTED", True) - set_opt("CONFIG_BT_BLE_50_FEATURES_SUPPORTED", False) + set_idf_sdkconfig_default("CONFIG_BT_ENABLED", True) + set_idf_sdkconfig_default("CONFIG_BT_BLE_42_FEATURES_SUPPORTED", True) + set_idf_sdkconfig_default("CONFIG_BT_BLE_50_FEATURES_SUPPORTED", False) # WiFi stack: disable only when Ethernet is present and WiFi is not. WiFi # relies on the IDF default (enabled), so it is never written True here. wifi_disabled = net.ethernet and not net.wifi if wifi_disabled: - set_opt("CONFIG_ESP_WIFI_ENABLED", False) + set_idf_sdkconfig_default("CONFIG_ESP_WIFI_ENABLED", False) # Software coexistence: enable when requested (the schema only allows it # alongside WiFi). Disable only in the Ethernet-without-WiFi case. if net.software_coexistence: - set_opt("CONFIG_SW_COEXIST_ENABLE", True) + set_idf_sdkconfig_default("CONFIG_SW_COEXIST_ENABLE", True) elif wifi_disabled: - set_opt("CONFIG_SW_COEXIST_ENABLE", False) + set_idf_sdkconfig_default("CONFIG_SW_COEXIST_ENABLE", False) # SoftAP support: drop it when WiFi is used without AP mode (IDF only). if not is_arduino and net.wifi and not net.wifi_ap: - set_opt("CONFIG_ESP_WIFI_SOFTAP_SUPPORT", False) + set_idf_sdkconfig_default("CONFIG_ESP_WIFI_SOFTAP_SUPPORT", False) # LWIP DHCP server: a WiFi-AP-mode / enable_lwip_dhcp_server concern (not # coexistence). Disable when WiFi has no AP (IDF) or the enable_lwip_dhcp_server @@ -2270,7 +2320,7 @@ async def _reconcile_network_sdkconfig() -> None: if ( wifi_wants_dhcps_off or dhcp_server_disabled_by_option ) and not arduino_eth_exclusion: - set_opt("CONFIG_LWIP_DHCPS", False) + set_idf_sdkconfig_default("CONFIG_LWIP_DHCPS", False) @coroutine_with_priority(CoroPriority.FINAL) @@ -2295,29 +2345,24 @@ async def _reconcile_vfs_fatfs_sdkconfig( """Reconcile VFS/FATFS sdkconfig flags after all require_*() calls; user sdkconfig_options win.""" opts = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] - def set_opt(name: str, value: SdkconfigValueType) -> None: - # User sdkconfig_options (applied during to_code) win. - if name not in opts: - add_idf_sdkconfig_option(name, value) - # USB Serial JTAG VFS needs termios (require_vfs_termios(), e.g. logger). ~1.8KB flash when off. if CORE.data.get(KEY_VFS_TERMIOS_REQUIRED, False): - set_opt("CONFIG_VFS_SUPPORT_TERMIOS", True) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_TERMIOS", True) else: - set_opt("CONFIG_VFS_SUPPORT_TERMIOS", not disable_vfs_termios) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_TERMIOS", not disable_vfs_termios) # VFS select is only needed for UART/eventfd fds (require_vfs_select(), e.g. openthread); # sockets use lwip_select() either way. ~2.7KB flash when off. if CORE.data.get(KEY_VFS_SELECT_REQUIRED, False): - set_opt("CONFIG_VFS_SUPPORT_SELECT", True) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_SELECT", True) else: - set_opt("CONFIG_VFS_SUPPORT_SELECT", not disable_vfs_select) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_SELECT", not disable_vfs_select) # Directory functions: opendir/readdir/mkdir etc. (require_vfs_dir()). ~0.5KB flash when off. if CORE.data.get(KEY_VFS_DIR_REQUIRED, False): - set_opt("CONFIG_VFS_SUPPORT_DIR", True) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_DIR", True) else: - set_opt("CONFIG_VFS_SUPPORT_DIR", not disable_vfs_dir) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_DIR", not disable_vfs_dir) # FATFS (require_fatfs()): LFN + one volume per esp_vfs_fat mount. Defaults only; # sdkconfig_options override. FATFS_LONG_FILENAMES is a Kconfig choice -- if the user set @@ -2330,15 +2375,15 @@ async def _reconcile_vfs_fatfs_sdkconfig( user_picked_lfn = any(k in opts for k in lfn_keys) if CORE.data[KEY_ESP32].get(KEY_FATFS_REQUIRED, False): if not user_picked_lfn: - set_opt("CONFIG_FATFS_LFN_NONE", False) - set_opt("CONFIG_FATFS_LFN_HEAP", True) - set_opt("CONFIG_FATFS_MAX_LFN", 255) - set_opt("CONFIG_FATFS_VOLUME_COUNT", 4) + set_idf_sdkconfig_default("CONFIG_FATFS_LFN_NONE", False) + set_idf_sdkconfig_default("CONFIG_FATFS_LFN_HEAP", True) + set_idf_sdkconfig_default("CONFIG_FATFS_MAX_LFN", 255) + set_idf_sdkconfig_default("CONFIG_FATFS_VOLUME_COUNT", 4) elif disable_fatfs: if not user_picked_lfn: - set_opt("CONFIG_FATFS_LFN_NONE", True) + set_idf_sdkconfig_default("CONFIG_FATFS_LFN_NONE", True) # Kconfig range is [1,10]; 0 gets clamped to the default. - set_opt("CONFIG_FATFS_VOLUME_COUNT", 1) + set_idf_sdkconfig_default("CONFIG_FATFS_VOLUME_COUNT", 1) @coroutine_with_priority(CoroPriority.FINAL - 1) @@ -2525,21 +2570,11 @@ async def to_code(config): ) add_idf_sdkconfig_option("CONFIG_MBEDTLS_PSK_MODES", True) - add_idf_sdkconfig_option("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", True) cg.add_build_flag("-Wno-nonnull-compare") - # Use CMN (common CAs) bundle by default to save ~51KB flash - # CMN covers CAs with >1% market share (~99% of websites) - # Components needing uncommon CAs can call require_full_certificate_bundle() - use_full_bundle = conf[CONF_ADVANCED].get( - CONF_USE_FULL_CERTIFICATE_BUNDLE, False - ) or CORE.data[KEY_ESP32].get(KEY_FULL_CERT_BUNDLE, False) - add_idf_sdkconfig_option( - "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", use_full_bundle - ) - if not use_full_bundle: - add_idf_sdkconfig_option("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", True) + if conf[CONF_ADVANCED].get(CONF_USE_FULL_CERTIFICATE_BUNDLE, False): + require_full_certificate_bundle() add_idf_sdkconfig_option(f"CONFIG_IDF_TARGET_{variant}", True) add_idf_sdkconfig_option( @@ -2929,6 +2964,9 @@ async def to_code(config): # FINAL priority: runs after every network/coexistence request_*() call CORE.add_job(_reconcile_network_sdkconfig) + # FINAL priority: runs after every require_certificate_bundle() call + CORE.add_job(_reconcile_certificate_bundle_sdkconfig) + # FINAL: require_*() calls can come from to_code at or below this priority, so an # inline read would be iteration-order-dependent; reconcile once after every job ran. CORE.add_job( @@ -2956,6 +2994,10 @@ async def to_code(config): for name, value in conf[CONF_SDKCONFIG_OPTIONS].items(): add_idf_sdkconfig_option(name, RawSdkconfigValue(value)) + # A bundle forced on through sdkconfig_options is a request like any other, + # so it still gets the CMN variant pinned. + if conf[CONF_SDKCONFIG_OPTIONS].get("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE") == "y": + require_certificate_bundle() # Components from YAML are added in a separate coroutine with FINAL priority # Schedule it to run after all other components diff --git a/esphome/components/esp32/const.py b/esphome/components/esp32/const.py index 09f458c64b..e7d8a66e7a 100644 --- a/esphome/components/esp32/const.py +++ b/esphome/components/esp32/const.py @@ -27,6 +27,7 @@ KEY_REFRESH = "refresh" KEY_PATH = "path" KEY_SUBMODULES = "submodules" KEY_EXTRA_BUILD_FILES = "extra_build_files" +KEY_CERT_BUNDLE = "cert_bundle" KEY_FULL_CERT_BUNDLE = "full_cert_bundle" KEY_NETWORK_SDKCONFIG = "network_sdkconfig" diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 8a5aae022a..2abf097aec 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -196,9 +196,7 @@ async def to_code(config: ConfigType) -> None: # framework: # advanced: # use_full_certificate_bundle: true - esp32.add_idf_sdkconfig_option( - "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", True - ) + esp32.require_certificate_bundle() esp32.add_idf_sdkconfig_option( "CONFIG_ESP_TLS_INSECURE", diff --git a/tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml b/tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml new file mode 100644 index 0000000000..68f9cf1d0f --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + libraries: + - NetworkClientSecure + +esp32: + board: esp32dev + framework: + type: arduino diff --git a/tests/component_tests/esp32/config/certificate_bundle_full.yaml b/tests/component_tests/esp32/config/certificate_bundle_full.yaml new file mode 100644 index 0000000000..179fc12f51 --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_full.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + advanced: + use_full_certificate_bundle: true diff --git a/tests/component_tests/esp32/config/certificate_bundle_http_request.yaml b/tests/component_tests/esp32/config/certificate_bundle_http_request.yaml new file mode 100644 index 0000000000..b29e5de2bd --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_http_request.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml b/tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml new file mode 100644 index 0000000000..af44693806 --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + sdkconfig_options: + CONFIG_MBEDTLS_CERTIFICATE_BUNDLE: y diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 7208318d3a..9925887c66 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -17,6 +17,7 @@ from esphome.components.esp32 import ( VARIANT_ESP32, VARIANTS, NetworkSdkconfigData, + RawSdkconfigValue, _ota_downgrade_protection_errors, _reconcile_network_sdkconfig, _reconcile_vfs_fatfs_sdkconfig, @@ -292,6 +293,63 @@ def test_default_exclusions_reincluded_by_owning_components( assert "fatfs" in excluded +_BUNDLE_OPTIONS = ( + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", +) + + +@pytest.mark.parametrize( + ("config_file", "expected"), + [ + pytest.param("exclusion_reincludes.yaml", (False, None, None), id="no_tls"), + pytest.param( + "certificate_bundle_http_request.yaml", + (True, True, False), + id="http_request", + ), + pytest.param( + "exclusion_reincludes_http_request.yaml", + (False, None, None), + id="http_request_no_verify", + ), + pytest.param( + "certificate_bundle_full.yaml", (True, None, True), id="full_option" + ), + pytest.param( + "certificate_bundle_arduino_tls.yaml", + (True, True, False), + id="arduino_network_client_secure", + ), + ], +) +def test_certificate_bundle_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + expected: tuple[bool | None, ...], +) -> None: + """The bundle and its CMN/FULL variant are written only when requested.""" + generate_main(component_config_path(config_file)) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert tuple(sdkconfig.get(name) for name in _BUNDLE_OPTIONS) == expected + + +def test_user_sdkconfig_certificate_bundle_wins( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A raw sdkconfig_options bundle setting is kept and still pins CMN.""" + generate_main(component_config_path("certificate_bundle_sdkconfig.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + value = sdkconfig["CONFIG_MBEDTLS_CERTIFICATE_BUNDLE"] + assert isinstance(value, RawSdkconfigValue) + assert value.value == "y" + assert sdkconfig.get("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN") is True + assert sdkconfig.get("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL") is False + + def test_execute_from_psram_s3_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], diff --git a/tests/components/esp32/test.esp32-idf.yaml b/tests/components/esp32/test.esp32-idf.yaml index 6b77a4e171..523e614e24 100644 --- a/tests/components/esp32/test.esp32-idf.yaml +++ b/tests/components/esp32/test.esp32-idf.yaml @@ -7,7 +7,7 @@ esp32: enable_lwip_mdns_queries: true enable_lwip_bridge_interface: true disable_libc_locks_in_iram: false # Test explicit opt-out of RAM optimization - use_full_certificate_bundle: false # Test CMN bundle (default) + use_full_certificate_bundle: false # Bundle stays off without a component that needs it include_builtin_idf_components: - freertos # Test escape hatch (freertos is always included anyway) enable_full_printf: false From 3f615f87cf88f44878b1bfe7ea85d3ca5a1396bf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 01:16:32 -0500 Subject: [PATCH 28/65] [esp32] Exclude esp_http_server and nvs_sec_provider from IDF builds by default (#18748) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/esp32/__init__.py | 16 ++++++++++ .../esp32_camera_web_server/__init__.py | 4 +++ esphome/components/web_server_idf/__init__.py | 1 + ...xclusion_reincludes_camera_web_server.yaml | 15 ++++++++++ .../exclusion_reincludes_nvs_sdkconfig.yaml | 11 +++++++ .../exclusion_stays_nvs_sdkconfig_off.yaml | 9 ++++++ tests/component_tests/esp32/test_esp32.py | 30 ++++++++++++++++++- 7 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/component_tests/esp32/config/exclusion_reincludes_camera_web_server.yaml create mode 100644 tests/component_tests/esp32/config/exclusion_reincludes_nvs_sdkconfig.yaml create mode 100644 tests/component_tests/esp32/config/exclusion_stays_nvs_sdkconfig_off.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index fca63e9a25..073d87402a 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -238,6 +238,7 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "esp_gdbstub", # GDB stub panic handler - unused by ESPHome; bt pulls it back "esp_hid", # HID host/device support - ESPHome doesn't implement HID functionality "esp_http_client", # HTTP client - only needed by http_request component + "esp_http_server", # HTTP server - re-included by web_server_idf, esp32_camera_web_server "esp_https_ota", # ESP-IDF HTTPS OTA - ESPHome has its own OTA implementation "esp_https_server", # HTTPS server - ESPHome has its own web server "esp_lcd", # LCD controller drivers - only needed by display component @@ -246,6 +247,7 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "fatfs", # FAT filesystem - ESPHome doesn't use filesystem storage "json", # cJSON library - ESPHome uses ArduinoJson instead "mqtt", # ESP-IDF MQTT library - ESPHome has its own MQTT implementation + "nvs_sec_provider", # NVS encryption key provider - re-included when CONFIG_NVS_ENCRYPTION is set "openthread", # Thread protocol - only needed by openthread component "perfmon", # Xtensa performance monitor - ESPHome has its own debug component "protobuf-c", # Protobuf runtime - only used by provisioning components (also excluded) @@ -649,6 +651,16 @@ class RawSdkconfigValue: SdkconfigValueType = bool | int | HexInt | str | RawSdkconfigValue +def is_idf_sdkconfig_option_enabled(name: str) -> bool: + """Return True when a bool sdkconfig option resolves to ``y``. + + Handles both the ``True`` a component sets and the raw ``y`` a user sets + in ``sdkconfig_options``. + """ + value = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS].get(name) + return value is not None and _format_sdkconfig_val(value) == "y" + + def set_idf_sdkconfig_default(name: str, value: SdkconfigValueType) -> None: """Set an sdkconfig option unless it is already set. @@ -2191,6 +2203,10 @@ def register_exclude_components_cmake_arg() -> None: @coroutine_with_priority(CoroPriority.FINAL) async def _write_exclude_components() -> None: """Write EXCLUDE_COMPONENTS cmake arg after all components have registered exclusions.""" + # NVS encryption needs nvs_sec_provider however it was enabled: the + # nvs_encryption option, raw sdkconfig_options or another component. + if is_idf_sdkconfig_option_enabled("CONFIG_NVS_ENCRYPTION"): + include_builtin_idf_component("nvs_sec_provider") register_exclude_components_cmake_arg() diff --git a/esphome/components/esp32_camera_web_server/__init__.py b/esphome/components/esp32_camera_web_server/__init__.py index 55ace66681..d54d5c6937 100644 --- a/esphome/components/esp32_camera_web_server/__init__.py +++ b/esphome/components/esp32_camera_web_server/__init__.py @@ -1,4 +1,5 @@ import esphome.codegen as cg +from esphome.components.esp32 import include_builtin_idf_component import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_MODE, CONF_PORT from esphome.types import ConfigType @@ -35,6 +36,7 @@ CONFIG_SCHEMA = cv.All( cv.Required(CONF_MODE): cv.enum(MODES, upper=True), }, ).extend(cv.COMPONENT_SCHEMA), + cv.only_on_esp32, _consume_camera_web_server_sockets, ) @@ -44,3 +46,5 @@ async def to_code(config: ConfigType) -> None: cg.add(server.set_port(config[CONF_PORT])) cg.add(server.set_mode(config[CONF_MODE])) await cg.register_component(server, config) + # esp_http_server is excluded from IDF builds by default to save compile time + include_builtin_idf_component("esp_http_server") diff --git a/esphome/components/web_server_idf/__init__.py b/esphome/components/web_server_idf/__init__.py index c16b0a2833..5a400dfbf3 100644 --- a/esphome/components/web_server_idf/__init__.py +++ b/esphome/components/web_server_idf/__init__.py @@ -20,6 +20,7 @@ async def to_code(config: ConfigType) -> None: # Re-enable esp-tls (excluded by default to save compile time); # web_server_idf.cpp includes for digest auth include_builtin_idf_component("esp-tls") + include_builtin_idf_component("esp_http_server") # multipart.cpp is fully #ifdef'd on USE_WEBSERVER_OTA (set by the diff --git a/tests/component_tests/esp32/config/exclusion_reincludes_camera_web_server.yaml b/tests/component_tests/esp32/config/exclusion_reincludes_camera_web_server.yaml new file mode 100644 index 0000000000..1bc5c936e0 --- /dev/null +++ b/tests/component_tests/esp32/config/exclusion_reincludes_camera_web_server.yaml @@ -0,0 +1,15 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +esp32_camera_web_server: + port: 8080 + mode: stream diff --git a/tests/component_tests/esp32/config/exclusion_reincludes_nvs_sdkconfig.yaml b/tests/component_tests/esp32/config/exclusion_reincludes_nvs_sdkconfig.yaml new file mode 100644 index 0000000000..a82d4aaba8 --- /dev/null +++ b/tests/component_tests/esp32/config/exclusion_reincludes_nvs_sdkconfig.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + sdkconfig_options: + CONFIG_NVS_ENCRYPTION: y + CONFIG_NVS_SEC_KEY_PROTECT_USING_HMAC: y + CONFIG_NVS_SEC_HMAC_EFUSE_KEY_ID: "0" diff --git a/tests/component_tests/esp32/config/exclusion_stays_nvs_sdkconfig_off.yaml b/tests/component_tests/esp32/config/exclusion_stays_nvs_sdkconfig_off.yaml new file mode 100644 index 0000000000..2580e77736 --- /dev/null +++ b/tests/component_tests/esp32/config/exclusion_stays_nvs_sdkconfig_off.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + sdkconfig_options: + CONFIG_NVS_ENCRYPTION: n diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 9925887c66..0ffbe16a17 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -261,9 +261,24 @@ def test_esp32_configuration_errors( ), pytest.param( "exclusion_reincludes_web_server.yaml", - ("esp-tls",), + ("esp-tls", "esp_http_server"), id="web_server_idf", ), + pytest.param( + "nvs_encryption_s3.yaml", + ("nvs_sec_provider",), + id="nvs_encryption", + ), + pytest.param( + "exclusion_reincludes_nvs_sdkconfig.yaml", + ("nvs_sec_provider",), + id="nvs_encryption_raw_sdkconfig", + ), + pytest.param( + "exclusion_reincludes_camera_web_server.yaml", + ("esp_http_server",), + id="esp32_camera_web_server", + ), pytest.param( "exclusion_reincludes_nextion.yaml", ("esp-tls", "esp_http_client"), @@ -291,6 +306,19 @@ def test_default_exclusions_reincluded_by_owning_components( # Components no part of this config touches stay excluded. assert "unity" in excluded assert "fatfs" in excluded + # The HTTP server only comes back for configs that run one. + assert ("esp_http_server" in excluded) == ("esp_http_server" not in reincluded) + + +def test_nvs_sec_provider_stays_excluded_when_encryption_is_off( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """An explicit CONFIG_NVS_ENCRYPTION=n keeps nvs_sec_provider excluded.""" + from esphome.components.esp32.const import KEY_EXCLUDE_COMPONENTS + + generate_main(component_config_path("exclusion_stays_nvs_sdkconfig_off.yaml")) + assert "nvs_sec_provider" in CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] _BUNDLE_OPTIONS = ( From bc8d0840ebafb904d600b9f474733cec8eb493b6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" <3060199+jesserockz@users.noreply.github.com> Date: Tue, 25 Aug 2026 02:50:43 -0500 Subject: [PATCH 29/65] [time] Skip automation.cpp when no on_time or on_time_sync automation is configured (#18750) --- esphome/components/time/__init__.py | 17 ++++++++--- esphome/components/time/automation.cpp | 3 ++ esphome/components/time/automation.h | 5 ++++ esphome/core/defines.h | 1 + .../time/config/no_triggers.yaml | 15 ++++++++++ .../component_tests/time/config/on_time.yaml | 21 ++++++++++++++ .../time/config/on_time_sync.yaml | 20 +++++++++++++ tests/component_tests/time/test_triggers.py | 28 +++++++++++++++++++ 8 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/time/config/no_triggers.yaml create mode 100644 tests/component_tests/time/config/on_time.yaml create mode 100644 tests/component_tests/time/config/on_time_sync.yaml create mode 100644 tests/component_tests/time/test_triggers.py diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index 7ad084493c..ecc448a96a 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -426,7 +426,12 @@ async def setup_time_core_(time_var, config): raise EsphomeError(f"Invalid timezone: {timezone}") from e _emit_parsed_timezone_fields(parsed) - for conf in config.get(CONF_ON_TIME, []): + on_time = config.get(CONF_ON_TIME, []) + on_time_sync = config.get(CONF_ON_TIME_SYNC, []) + if on_time or on_time_sync: + cg.add_define("USE_TIME_TRIGGERS") + + for conf in on_time: trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], time_var) seconds = conf.get(CONF_SECONDS, list(range(61))) @@ -445,7 +450,7 @@ async def setup_time_core_(time_var, config): await cg.register_component(trigger, conf) await automation.build_automation(trigger, [], conf) - for conf in config.get(CONF_ON_TIME_SYNC, []): + for conf in on_time_sync: trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], time_var) await cg.register_component(trigger, conf) @@ -479,7 +484,11 @@ async def time_has_time_to_code(config, condition_id, template_arg, args): # posix_tz.cpp is fully #ifdef'd on USE_TIME_TIMEZONE, set only when a -# timezone is configured or detected. +# timezone is configured or detected; automation.cpp holds the on_time and +# on_time_sync triggers and is #ifdef'd on USE_TIME_TRIGGERS. FILTER_SOURCE_FILES = filter_source_files_from_defines( - {"posix_tz.cpp": "USE_TIME_TIMEZONE"} + { + "posix_tz.cpp": "USE_TIME_TIMEZONE", + "automation.cpp": "USE_TIME_TRIGGERS", + } ) diff --git a/esphome/components/time/automation.cpp b/esphome/components/time/automation.cpp index 3242669343..b91f8b0360 100644 --- a/esphome/components/time/automation.cpp +++ b/esphome/components/time/automation.cpp @@ -1,4 +1,5 @@ #include "automation.h" +#ifdef USE_TIME_TRIGGERS #include "esphome/core/log.h" @@ -98,3 +99,5 @@ SyncTrigger::SyncTrigger(RealTimeClock *rtc) : rtc_(rtc) { } } // namespace esphome::time + +#endif // USE_TIME_TRIGGERS diff --git a/esphome/components/time/automation.h b/esphome/components/time/automation.h index 7be195903a..5f980690a4 100644 --- a/esphome/components/time/automation.h +++ b/esphome/components/time/automation.h @@ -1,5 +1,8 @@ #pragma once +#include "esphome/core/defines.h" +#ifdef USE_TIME_TRIGGERS + #include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/time.h" @@ -49,3 +52,5 @@ class SyncTrigger final : public Trigger<>, public Component { RealTimeClock *rtc_; }; } // namespace esphome::time + +#endif // USE_TIME_TRIGGERS diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 115ef2145d..971ad7c8d9 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -188,6 +188,7 @@ #define USE_TEXT_SENSOR #define USE_TEXT_SENSOR_FILTER #define USE_TIME +#define USE_TIME_TRIGGERS #define USE_TOUCHSCREEN #define USE_UART_DEBUGGER #define USE_UART_WAKE_LOOP_ON_RX diff --git a/tests/component_tests/time/config/no_triggers.yaml b/tests/component_tests/time/config/no_triggers.yaml new file mode 100644 index 0000000000..2df919df7e --- /dev/null +++ b/tests/component_tests/time/config/no_triggers.yaml @@ -0,0 +1,15 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +time: + - platform: sntp + id: sntp_time diff --git a/tests/component_tests/time/config/on_time.yaml b/tests/component_tests/time/config/on_time.yaml new file mode 100644 index 0000000000..919a94f52d --- /dev/null +++ b/tests/component_tests/time/config/on_time.yaml @@ -0,0 +1,21 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +logger: + +time: + - platform: sntp + id: sntp_time + on_time: + - seconds: 0 + then: + - logger.log: tick diff --git a/tests/component_tests/time/config/on_time_sync.yaml b/tests/component_tests/time/config/on_time_sync.yaml new file mode 100644 index 0000000000..b5ddedce14 --- /dev/null +++ b/tests/component_tests/time/config/on_time_sync.yaml @@ -0,0 +1,20 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +logger: + +time: + - platform: sntp + id: sntp_time + on_time_sync: + then: + - logger.log: synced diff --git a/tests/component_tests/time/test_triggers.py b/tests/component_tests/time/test_triggers.py new file mode 100644 index 0000000000..34fd5a3157 --- /dev/null +++ b/tests/component_tests/time/test_triggers.py @@ -0,0 +1,28 @@ +"""automation.cpp (CronTrigger and SyncTrigger) is only compiled when an +on_time or on_time_sync automation exists, so the define must follow them.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.core import CORE + + +@pytest.mark.parametrize( + ("fixture", "emits"), + [ + ("no_triggers.yaml", False), + ("on_time.yaml", True), + ("on_time_sync.yaml", True), + ], +) +def test_triggers_define_follows_automations( + fixture: str, + emits: bool, + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path(fixture)) + defines = {define.name for define in CORE.defines} + assert ("USE_TIME_TRIGGERS" in defines) is emits From b579751bdf142afccd74a97ffd87855ba79079ef Mon Sep 17 00:00:00 2001 From: NoQuarrel <278613315+NoQuarrel@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:55:19 +0800 Subject: [PATCH 30/65] [sfa40] Add SFA40 sensor support (#17815) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- CODEOWNERS | 1 + esphome/components/sfa40/__init__.py | 1 + esphome/components/sfa40/sensor.py | 79 +++++++++ esphome/components/sfa40/sfa40.cpp | 159 ++++++++++++++++++ esphome/components/sfa40/sfa40.h | 46 +++++ tests/components/sfa40/common.yaml | 12 ++ tests/components/sfa40/test.esp32-idf.yaml | 3 + tests/components/sfa40/test.esp8266-ard.yaml | 3 + tests/components/sfa40/test.rp2040-ard.yaml | 3 + .../components/sfa40/validate.esp32-idf.yaml | 9 + 10 files changed, 316 insertions(+) create mode 100644 esphome/components/sfa40/__init__.py create mode 100644 esphome/components/sfa40/sensor.py create mode 100644 esphome/components/sfa40/sfa40.cpp create mode 100644 esphome/components/sfa40/sfa40.h create mode 100644 tests/components/sfa40/common.yaml create mode 100644 tests/components/sfa40/test.esp32-idf.yaml create mode 100644 tests/components/sfa40/test.esp8266-ard.yaml create mode 100644 tests/components/sfa40/test.rp2040-ard.yaml create mode 100644 tests/components/sfa40/validate.esp32-idf.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 3047072ea2..b898788b1a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -476,6 +476,7 @@ esphome/components/sensirion_common/* @martgras esphome/components/sensor/* @esphome/core esphome/components/serial_proxy/* @kbx81 esphome/components/sfa30/* @ghsensdev +esphome/components/sfa40/* @NoQuarrel esphome/components/sgp40/* @SenexCrenshaw esphome/components/sgp4x/* @martgras @SenexCrenshaw esphome/components/sha256/* @esphome/core diff --git a/esphome/components/sfa40/__init__.py b/esphome/components/sfa40/__init__.py new file mode 100644 index 0000000000..79568d458a --- /dev/null +++ b/esphome/components/sfa40/__init__.py @@ -0,0 +1 @@ +CODEOWNERS = ["@NoQuarrel"] diff --git a/esphome/components/sfa40/sensor.py b/esphome/components/sfa40/sensor.py new file mode 100644 index 0000000000..701660eb8f --- /dev/null +++ b/esphome/components/sfa40/sensor.py @@ -0,0 +1,79 @@ +import esphome.codegen as cg +from esphome.components import i2c, sensirion_common, sensor +import esphome.config_validation as cv +from esphome.const import ( + CONF_FORMALDEHYDE, + CONF_HUMIDITY, + CONF_ID, + CONF_TEMPERATURE, + DEVICE_CLASS_GAS, + DEVICE_CLASS_HUMIDITY, + DEVICE_CLASS_TEMPERATURE, + ICON_FLASK_OUTLINE, + ICON_THERMOMETER, + ICON_WATER_PERCENT, + STATE_CLASS_MEASUREMENT, + UNIT_CELSIUS, + UNIT_PARTS_PER_BILLION, + UNIT_PERCENT, +) + +DEPENDENCIES = ["i2c"] +AUTO_LOAD = ["sensirion_common"] + +CONF_WAIT_FOR_READY = "wait_for_ready" + +sfa40_ns = cg.esphome_ns.namespace("sfa40") +SFA40Component = sfa40_ns.class_( + "SFA40Component", cg.PollingComponent, sensirion_common.SensirionI2CDevice +) + +CONFIG_SCHEMA = ( + cv.Schema( + { + cv.GenerateID(CONF_ID): cv.declare_id(SFA40Component), + cv.Optional(CONF_WAIT_FOR_READY, default=True): cv.boolean, + cv.Optional(CONF_FORMALDEHYDE): sensor.sensor_schema( + unit_of_measurement=UNIT_PARTS_PER_BILLION, + icon=ICON_FLASK_OUTLINE, + accuracy_decimals=1, + device_class=DEVICE_CLASS_GAS, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_TEMPERATURE): sensor.sensor_schema( + unit_of_measurement=UNIT_CELSIUS, + icon=ICON_THERMOMETER, + accuracy_decimals=2, + device_class=DEVICE_CLASS_TEMPERATURE, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_HUMIDITY): sensor.sensor_schema( + unit_of_measurement=UNIT_PERCENT, + icon=ICON_WATER_PERCENT, + accuracy_decimals=2, + device_class=DEVICE_CLASS_HUMIDITY, + state_class=STATE_CLASS_MEASUREMENT, + ), + } + ) + .extend(cv.polling_component_schema("60s")) + .extend(i2c.i2c_device_schema(0x5D)) +) + +SENSOR_MAP = { + CONF_FORMALDEHYDE: "set_formaldehyde_sensor", + CONF_TEMPERATURE: "set_temperature_sensor", + CONF_HUMIDITY: "set_humidity_sensor", +} + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await i2c.register_i2c_device(var, config) + cg.add(var.set_wait_for_ready(config[CONF_WAIT_FOR_READY])) + + for key, func_name in SENSOR_MAP.items(): + if sensor_config := config.get(key): + sens = await sensor.new_sensor(sensor_config) + cg.add(getattr(var, func_name)(sens)) diff --git a/esphome/components/sfa40/sfa40.cpp b/esphome/components/sfa40/sfa40.cpp new file mode 100644 index 0000000000..0d6bee3f9a --- /dev/null +++ b/esphome/components/sfa40/sfa40.cpp @@ -0,0 +1,159 @@ +#include "sfa40.h" +#include "esphome/core/log.h" +#include + +namespace esphome::sfa40 { + +static const char *const TAG = "sfa40"; + +// SFA40 Datasheet: https://sensirion.com/media/documents/5B06EDD9/69F84BD8/Sensirion_Datasheet_SFA40.pdf + +static const uint16_t SFA40_CMD_START_MEASUREMENT = 0x00AC; +static const uint16_t SFA40_CMD_STOP_MEASUREMENT = 0x50D2; +static const uint16_t SFA40_CMD_READ_MEASURE_PROD = 0xC0EB; +// B4 (engineering-sample) command codes. Commands from here: https://github.com/DFRobot/DFRobot_SFA40 +static const uint16_t SFA40_CMD_READ_MEASURE_B4 = 0xE06D; +static const uint16_t SFA40_CMD_READ_ID_PROD = 0x02CE; +static const uint16_t SFA40_CMD_READ_ID_B4 = 0x0559; +static const uint8_t STATUS_NOT_READY = 0x01; +static const uint8_t STATUS_OUT_OF_SPEC = 0x02; + +static uint64_t raw_to_serial(const uint16_t *raw, size_t words) { + uint64_t serial = 0; + for (size_t i = 0; i < words; i++) { + serial = (serial << 16) | raw[i]; + } + return serial; +} + +static void raw_to_marking(const uint16_t *raw, size_t words, char *out, size_t out_len) { + if (out_len < words * 2 + 1) { + return; + } + for (size_t i = 0; i < words; i++) { + out[i * 2] = static_cast(raw[i] >> 8); + out[i * 2 + 1] = static_cast(raw[i] & 0xFF); + } + out[words * 2] = '\0'; +} + +void SFA40Component::setup() { + this->write_command(SFA40_CMD_STOP_MEASUREMENT); + this->set_timeout(25, [this]() { + if (!this->detect_protocol_()) { + ESP_LOGE(TAG, "Failed to detect SFA40 protocol"); + this->error_code_ = PROTOCOL_DETECTION_FAILED; + this->mark_failed(); + return; + } + if (!this->write_command(SFA40_CMD_START_MEASUREMENT)) { + ESP_LOGE(TAG, "Failed to start measurements"); + this->error_code_ = MEASUREMENT_INIT_FAILED; + this->mark_failed(); + return; + } + this->initialized_ = true; + ESP_LOGD(TAG, "Measurement started"); + }); +} + +bool SFA40Component::detect_protocol_() { + uint16_t raw[5] = {}; + if (this->get_register(SFA40_CMD_READ_ID_PROD, raw, 3, 5)) { + this->protocol_version_ = ProtocolVersion::PRODUCTION; + this->serial_number_ = raw_to_serial(raw, 3); + ESP_LOGD(TAG, "Detected production SFA40, serial number: %012" PRIX64, this->serial_number_); + return true; + } + if (this->get_register(SFA40_CMD_READ_ID_B4, raw, 5, 5)) { + this->protocol_version_ = ProtocolVersion::PROTOTYPE; + raw_to_marking(raw, 5, this->device_marking_, sizeof(this->device_marking_)); + ESP_LOGD(TAG, "Detected engineering-sample SFA40, marking: '%s'", this->device_marking_); + return true; + } + return false; +} + +void SFA40Component::dump_config() { + ESP_LOGCONFIG(TAG, "sfa40:"); + LOG_I2C_DEVICE(this); + if (this->is_failed()) { + switch (this->error_code_) { + case PROTOCOL_DETECTION_FAILED: + ESP_LOGW(TAG, "Protocol detection failed!"); + break; + case MEASUREMENT_INIT_FAILED: + ESP_LOGW(TAG, "Measurement initialization failed!"); + break; + default: + ESP_LOGW(TAG, "Unknown setup error!"); + break; + } + } + LOG_UPDATE_INTERVAL(this); + switch (this->protocol_version_) { + case ProtocolVersion::PRODUCTION: + ESP_LOGCONFIG(TAG, " Protocol: production\n Serial Number: %012" PRIX64, this->serial_number_); + break; + case ProtocolVersion::PROTOTYPE: + ESP_LOGCONFIG(TAG, " Protocol: prototype (B4)\n Marking: '%s'", this->device_marking_); + break; + default: + ESP_LOGCONFIG(TAG, " Protocol: (detecting...)"); + break; + } + ESP_LOGCONFIG(TAG, " Wait for ready: %s", YESNO(this->wait_for_ready_)); + LOG_SENSOR(" ", "Formaldehyde", this->formaldehyde_sensor_); + LOG_SENSOR(" ", "Temperature", this->temperature_sensor_); + LOG_SENSOR(" ", "Humidity", this->humidity_sensor_); +} + +void SFA40Component::update() { + if (!this->initialized_ || this->protocol_version_ == ProtocolVersion::UNKNOWN) { + return; + } + + const uint16_t read_cmd = (this->protocol_version_ == ProtocolVersion::PRODUCTION) ? SFA40_CMD_READ_MEASURE_PROD + : SFA40_CMD_READ_MEASURE_B4; + + if (!this->write_command(read_cmd)) { + ESP_LOGW(TAG, "Error reading measurement"); + this->status_set_warning(); + return; + } + + this->set_timeout(5, [this]() { + uint16_t raw[4]; + if (!this->read_data(raw, 4)) { + ESP_LOGW(TAG, "Error reading measurement data"); + this->status_set_warning(); + return; + } + + const uint8_t status = raw[3] >> 8; + const bool sensor_not_ready = (status & STATUS_NOT_READY) != 0; + const bool sensor_out_of_spec = (status & STATUS_OUT_OF_SPEC) != 0; + + if (this->formaldehyde_sensor_ != nullptr) { + if (sensor_out_of_spec) { + ESP_LOGW(TAG, "Skipping formaldehyde publish: sensor out of spec (status=0x%02X)", status); + } else if (this->wait_for_ready_ && sensor_not_ready) { + ESP_LOGD(TAG, "Skipping formaldehyde publish: sensor warming up"); + } else { + this->formaldehyde_sensor_->publish_state(static_cast(raw[0]) / 10.0f); + } + } + + if (this->humidity_sensor_ != nullptr) { + this->humidity_sensor_->publish_state(clamp(125.0f * static_cast(raw[1]) / 65535.0f - 6.0f, 0.0f, 100.0f)); + } + + if (this->temperature_sensor_ != nullptr) { + this->temperature_sensor_->publish_state(175.0f * (static_cast(raw[2]) / 65535.0f) - 45.0f); + } + + this->status_clear_warning(); + }); +} + +} // namespace esphome::sfa40 diff --git a/esphome/components/sfa40/sfa40.h b/esphome/components/sfa40/sfa40.h new file mode 100644 index 0000000000..6f67607753 --- /dev/null +++ b/esphome/components/sfa40/sfa40.h @@ -0,0 +1,46 @@ +#pragma once + +#include "esphome/core/component.h" +#include "esphome/components/sensor/sensor.h" +#include "esphome/components/sensirion_common/i2c_sensirion.h" + +namespace esphome::sfa40 { + +// SFA40 Datasheet: https://sensirion.com/media/documents/5B06EDD9/69F84BD8/Sensirion_Datasheet_SFA40.pdf + +class SFA40Component final : public PollingComponent, public sensirion_common::SensirionI2CDevice { + public: + void setup() override; + void dump_config() override; + void update() override; + + void set_formaldehyde_sensor(sensor::Sensor *formaldehyde) { this->formaldehyde_sensor_ = formaldehyde; } + void set_temperature_sensor(sensor::Sensor *temperature) { this->temperature_sensor_ = temperature; } + void set_humidity_sensor(sensor::Sensor *humidity) { this->humidity_sensor_ = humidity; } + void set_wait_for_ready(bool wait_for_ready) { this->wait_for_ready_ = wait_for_ready; } + + protected: + enum ProtocolVersion : uint8_t { + UNKNOWN = 0, + PRODUCTION = 1, + PROTOTYPE = 2, + }; + enum ErrorCode : uint8_t { + UNKNOWN_ERROR = 0, + PROTOCOL_DETECTION_FAILED, + MEASUREMENT_INIT_FAILED, + }; + bool detect_protocol_(); + ProtocolVersion protocol_version_{UNKNOWN}; + ErrorCode error_code_{UNKNOWN_ERROR}; + char device_marking_[11]{}; + bool initialized_{false}; + bool wait_for_ready_{true}; + uint64_t serial_number_{0}; + + sensor::Sensor *formaldehyde_sensor_{nullptr}; + sensor::Sensor *temperature_sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; +}; + +} // namespace esphome::sfa40 diff --git a/tests/components/sfa40/common.yaml b/tests/components/sfa40/common.yaml new file mode 100644 index 0000000000..d883e02cb6 --- /dev/null +++ b/tests/components/sfa40/common.yaml @@ -0,0 +1,12 @@ +sensor: + - platform: sfa40 + i2c_id: i2c_bus + wait_for_ready: false + formaldehyde: + name: SFA40 formaldehyde + temperature: + name: SFA40 temperature + humidity: + name: SFA40 humidity + address: 0x5D + update_interval: 30s diff --git a/tests/components/sfa40/test.esp32-idf.yaml b/tests/components/sfa40/test.esp32-idf.yaml new file mode 100644 index 0000000000..ff65829346 --- /dev/null +++ b/tests/components/sfa40/test.esp32-idf.yaml @@ -0,0 +1,3 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml + sfa40: !include common.yaml diff --git a/tests/components/sfa40/test.esp8266-ard.yaml b/tests/components/sfa40/test.esp8266-ard.yaml new file mode 100644 index 0000000000..11c0bcbfc7 --- /dev/null +++ b/tests/components/sfa40/test.esp8266-ard.yaml @@ -0,0 +1,3 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp8266-ard.yaml + sfa40: !include common.yaml diff --git a/tests/components/sfa40/test.rp2040-ard.yaml b/tests/components/sfa40/test.rp2040-ard.yaml new file mode 100644 index 0000000000..7c7f8acdf6 --- /dev/null +++ b/tests/components/sfa40/test.rp2040-ard.yaml @@ -0,0 +1,3 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/rp2040-ard.yaml + sfa40: !include common.yaml diff --git a/tests/components/sfa40/validate.esp32-idf.yaml b/tests/components/sfa40/validate.esp32-idf.yaml new file mode 100644 index 0000000000..6f65ac4bdb --- /dev/null +++ b/tests/components/sfa40/validate.esp32-idf.yaml @@ -0,0 +1,9 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml + +sensor: + - platform: sfa40 + i2c_id: i2c_bus + wait_for_ready: true + formaldehyde: + name: SFA40 formaldehyde From 4eba9ee41afe4da53c11b51925278da0be20ad74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Metrich?= <45318189+FredM67@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:36:00 +0200 Subject: [PATCH 31/65] [core] Fix false-positive readability-non-const-parameter in string_ref.h (#18758) --- esphome/core/string_ref.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/core/string_ref.h b/esphome/core/string_ref.h index 33459f48af..2c7ec914c7 100644 --- a/esphome/core/string_ref.h +++ b/esphome/core/string_ref.h @@ -239,7 +239,9 @@ template inline R parse_number(const StringRef &str, siz } // NOLINTEND(google-runtime-int) } // namespace internal -// NOLINTBEGIN(readability-identifier-naming,google-runtime-int) +// readability-non-const-parameter: `pos` is written through by internal::parse_number, one call +// frame away; the check only inspects these bodies, so it wrongly proposes `const size_t *`. +// NOLINTBEGIN(readability-identifier-naming,google-runtime-int,readability-non-const-parameter) inline int stoi(const StringRef &str, size_t *pos = nullptr, int base = 10) { return static_cast(internal::parse_number(str, pos, base, std::strtol)); } @@ -252,7 +254,7 @@ inline float stof(const StringRef &str, size_t *pos = nullptr) { inline double stod(const StringRef &str, size_t *pos = nullptr) { return internal::parse_number(str, pos, std::strtod); } -// NOLINTEND(readability-identifier-naming,google-runtime-int) +// NOLINTEND(readability-identifier-naming,google-runtime-int,readability-non-const-parameter) #ifdef USE_JSON // NOLINTNEXTLINE(readability-identifier-naming) From 6a86ef0ee02dfb2001239bf8185109ed194928a2 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 26 Aug 2026 03:45:38 +1200 Subject: [PATCH 32/65] [ethernet] Add spi_id option to attach to an existing SPI bus (#18530) --- esphome/components/ethernet/__init__.py | 83 ++++++- .../components/ethernet/ethernet_component.h | 11 + .../ethernet/ethernet_component_esp32.cpp | 74 ++++-- esphome/components/spi/spi.h | 2 + tests/component_tests/conftest.py | 8 + .../ethernet/config/spi_id_shared_bus.yaml | 21 ++ .../ethernet/config/spi_own_bus.yaml | 16 ++ .../component_tests/ethernet/test_ethernet.py | 8 - tests/component_tests/ethernet/test_spi_id.py | 235 ++++++++++++++++++ .../component_tests/network/test_priority.py | 4 +- .../ethernet/common-w5500-spi-id.yaml | 17 ++ .../ethernet/test-w5500-spi-id.esp32-idf.yaml | 3 + 12 files changed, 437 insertions(+), 45 deletions(-) create mode 100644 tests/component_tests/ethernet/config/spi_id_shared_bus.yaml create mode 100644 tests/component_tests/ethernet/config/spi_own_bus.yaml create mode 100644 tests/component_tests/ethernet/test_spi_id.py create mode 100644 tests/components/ethernet/common-w5500-spi-id.yaml create mode 100644 tests/components/ethernet/test-w5500-spi-id.esp32-idf.yaml diff --git a/esphome/components/ethernet/__init__.py b/esphome/components/ethernet/__init__.py index a44a609d3c..0454440f14 100644 --- a/esphome/components/ethernet/__init__.py +++ b/esphome/components/ethernet/__init__.py @@ -4,6 +4,7 @@ import logging from esphome import automation, pins from esphome.automation import Condition import esphome.codegen as cg +from esphome.components import spi from esphome.components.network import ( add_use_address, get_network_priority, @@ -39,6 +40,7 @@ from esphome.const import ( CONF_POLLING_INTERVAL, CONF_RESET_PIN, CONF_SPI, + CONF_SPI_ID, CONF_STATIC_IP, CONF_SUBNET, CONF_TYPE, @@ -263,10 +265,42 @@ def _is_framework_spi_polling_mode_supported() -> bool: return False +# Options that come from the referenced spi bus when spi_id is set +_SPI_BUS_PROVIDED_OPTIONS = ( + CONF_CLK_PIN, + CONF_MOSI_PIN, + CONF_MISO_PIN, + CONF_INTERFACE, +) + + +def _validate_spi_bus(config: ConfigType) -> ConfigType: + """Cross-validate spi_id against the options the referenced bus provides.""" + if CONF_SPI_ID in config: + for key in _SPI_BUS_PROVIDED_OPTIONS: + if key in config: + raise cv.Invalid( + f"'{key}' cannot be used together with '{CONF_SPI_ID}'; " + f"it comes from the referenced 'spi:' bus.", + path=[key], + ) + else: + for key in (CONF_CLK_PIN, CONF_MOSI_PIN, CONF_MISO_PIN): + if key not in config: + raise cv.Invalid( + f"'{key}' is a required option when '{CONF_SPI_ID}' is not set.", + path=[key], + ) + return config + + def _validate_spi_interface(config: ConfigType) -> ConfigType: """Set default SPI interface or validate user choice against the variant.""" if not CORE.is_esp32: return config + if CONF_SPI_ID in config: + # The interface comes from the referenced spi bus; don't set a default. + return config from esphome.components.esp32 import VARIANT_ESP32, get_esp32_variant from esphome.components.spi import get_hw_interface_list @@ -451,9 +485,14 @@ def _spi_schema(default_clock: str = "26.67MHz", max_clock: int = int(80e6)) -> BASE_SCHEMA.extend( cv.Schema( { - cv.Required(CONF_CLK_PIN): pins.internal_gpio_output_pin_number, - cv.Required(CONF_MISO_PIN): pins.internal_gpio_input_pin_number, - cv.Required(CONF_MOSI_PIN): pins.internal_gpio_output_pin_number, + # clk/mosi/miso are required unless spi_id is set; enforced + # by _validate_spi_bus below. + cv.Optional(CONF_CLK_PIN): pins.internal_gpio_output_pin_number, + cv.Optional(CONF_MISO_PIN): pins.internal_gpio_input_pin_number, + cv.Optional(CONF_MOSI_PIN): pins.internal_gpio_output_pin_number, + cv.Optional(CONF_SPI_ID): cv.All( + cv.only_on_esp32, cv.use_id(spi.SPIComponent) + ), cv.Required(CONF_CS_PIN): pins.internal_gpio_output_pin_number, cv.Optional( CONF_INTERRUPT_PIN @@ -478,6 +517,7 @@ def _spi_schema(default_clock: str = "26.67MHz", max_clock: int = int(80e6)) -> ), ), cv.only_on([Platform.ESP32, Platform.RP2]), + _validate_spi_bus, _validate_spi_interface, ) @@ -529,6 +569,30 @@ def _final_validate_spi(config: ConfigType) -> None: return from esphome.components.spi import CONF_INTERFACE_INDEX, get_spi_interface + if CONF_SPI_ID in config: + # Sharing the bus: the standard spi device schema enforces that the + # referenced bus declares both data lines. The IDF ethernet drivers + # additionally need a hardware host, which shows as an interface index + # on the validated bus config. + spi.final_validate_device_schema( + "ethernet", require_mosi=True, require_miso=True + )(config) + cv.Schema( + { + cv.Required(CONF_SPI_ID): fv.id_declaration_match_schema( + { + cv.Required( + CONF_INTERFACE_INDEX, + msg="Component ethernet requires this spi bus to use " + "a hardware interface", + ): cv.valid + } + ) + }, + extra=cv.ALLOW_EXTRA, + )(config) + return + if spi_configs := fv.full_config.get().get(CONF_SPI): # get_spi_interface() returns strings like "SPI2_HOST" spi_host = f"{config[CONF_INTERFACE].upper()}_HOST" @@ -625,9 +689,15 @@ async def _to_code_esp32(var: cg.MockObj, config: ConfigType) -> None: ) if config[CONF_TYPE] in SPI_ETHERNET_TYPES: - cg.add(var.set_clk_pin(config[CONF_CLK_PIN])) - cg.add(var.set_miso_pin(config[CONF_MISO_PIN])) - cg.add(var.set_mosi_pin(config[CONF_MOSI_PIN])) + if (spi_id := config.get(CONF_SPI_ID)) is not None: + # Pins and host come from the shared spi bus. + spi_parent = await cg.get_variable(spi_id) + cg.add(var.set_spi_parent(spi_parent)) + else: + cg.add(var.set_clk_pin(config[CONF_CLK_PIN])) + cg.add(var.set_miso_pin(config[CONF_MISO_PIN])) + cg.add(var.set_mosi_pin(config[CONF_MOSI_PIN])) + cg.add(var.set_interface(SPI_INTERFACE_MAP[config[CONF_INTERFACE]])) cg.add(var.set_cs_pin(config[CONF_CS_PIN])) if CONF_INTERRUPT_PIN in config: cg.add(var.set_interrupt_pin(config[CONF_INTERRUPT_PIN])) @@ -641,7 +711,6 @@ async def _to_code_esp32(var: cg.MockObj, config: ConfigType) -> None: cg.add_define("USE_ETHERNET_SPI") - cg.add(var.set_interface(SPI_INTERFACE_MAP[config[CONF_INTERFACE]])) add_idf_sdkconfig_option("CONFIG_ETH_USE_SPI_ETHERNET", True) # CONFIG_ETH_SPI_ETHERNET_{TYPE} Kconfig options were removed in IDF 6.0 # Types that are never built into IDF ship no Kconfig option at all diff --git a/esphome/components/ethernet/ethernet_component.h b/esphome/components/ethernet/ethernet_component.h index 1482e7a828..2b67b9093b 100644 --- a/esphome/components/ethernet/ethernet_component.h +++ b/esphome/components/ethernet/ethernet_component.h @@ -13,6 +13,9 @@ #include "esp_eth.h" #ifdef USE_ETHERNET_SPI #include "hal/spi_types.h" +#ifdef USE_SPI +#include "esphome/components/spi/spi.h" +#endif #endif #include "esp_eth_mac.h" #include "esp_eth_mac_esp.h" @@ -176,6 +179,9 @@ class EthernetComponent final : public Component { void set_reset_pin(uint8_t reset_pin) { this->reset_pin_ = reset_pin; } void set_clock_speed(int clock_speed) { this->clock_speed_ = clock_speed; } void set_interface(spi_host_device_t interface) { this->interface_ = interface; } +#ifdef USE_SPI + void set_spi_parent(spi::SPIComponent *parent) { this->spi_parent_ = parent; } +#endif #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT void set_polling_interval(uint32_t polling_interval) { this->polling_interval_ = polling_interval; } #endif @@ -258,6 +264,11 @@ class EthernetComponent final : public Component { int phy_addr_spi_{-1}; int clock_speed_; spi_host_device_t interface_{SPI2_HOST}; +#ifdef USE_SPI + // When set, the SPI bus is owned and initialized by this spi component + // and the ethernet chip only adds a device to it. + spi::SPIComponent *spi_parent_{nullptr}; +#endif #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT uint32_t polling_interval_{0}; #endif diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index 069478e70c..1d9903271e 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -59,6 +59,9 @@ #ifdef USE_ETHERNET_SPI #include #include +#ifdef USE_SPI +#include "esphome/components/spi/spi.h" +#endif #endif namespace esphome::ethernet { @@ -168,25 +171,34 @@ void EthernetComponent::ethernet_lazy_init_() { // Install GPIO ISR handler to be able to service SPI Eth modules interrupts gpio_install_isr_service(0); - spi_bus_config_t buscfg = { - .mosi_io_num = this->mosi_pin_, - .miso_io_num = this->miso_pin_, - .sclk_io_num = this->clk_pin_, - .quadwp_io_num = -1, - .quadhd_io_num = -1, - .data4_io_num = -1, - .data5_io_num = -1, - .data6_io_num = -1, - .data7_io_num = -1, - .max_transfer_sz = 0, - .flags = 0, - .intr_flags = 0, - }; + spi_host_device_t host; +#ifdef USE_SPI + if (this->spi_parent_ != nullptr) { + // The bus is owned and already initialized by the spi component; share its host. + host = this->spi_parent_->get_interface(); + } else +#endif + { + spi_bus_config_t buscfg = { + .mosi_io_num = this->mosi_pin_, + .miso_io_num = this->miso_pin_, + .sclk_io_num = this->clk_pin_, + .quadwp_io_num = -1, + .quadhd_io_num = -1, + .data4_io_num = -1, + .data5_io_num = -1, + .data6_io_num = -1, + .data7_io_num = -1, + .max_transfer_sz = 0, + .flags = 0, + .intr_flags = 0, + }; - auto host = this->interface_; + host = this->interface_; - err = spi_bus_initialize(host, &buscfg, SPI_DMA_CH_AUTO); - ESPHL_ERROR_CHECK(err, "SPI bus initialize error"); + err = spi_bus_initialize(host, &buscfg, SPI_DMA_CH_AUTO); + ESPHL_ERROR_CHECK(err, "SPI bus initialize error"); + } #endif // Network interface setup handled by network component @@ -575,17 +587,25 @@ void EthernetComponent::dump_config() { YESNO(this->is_connected())); this->dump_connect_params_(); #ifdef USE_ETHERNET_SPI - ESP_LOGCONFIG(TAG, - " CLK Pin: %u\n" - " MISO Pin: %u\n" - " MOSI Pin: %u\n" - " CS Pin: %u", - this->clk_pin_, this->miso_pin_, this->mosi_pin_, this->cs_pin_); - const char *spi_interface = "spi3"; - if (this->interface_ == SPI2_HOST) { - spi_interface = "spi2"; +#ifdef USE_SPI + if (this->spi_parent_ != nullptr) { + // Pins and interface come from the shared spi bus; only CS is ours. + ESP_LOGCONFIG(TAG, " CS Pin: %u", this->cs_pin_); + } else +#endif + { + ESP_LOGCONFIG(TAG, + " CLK Pin: %u\n" + " MISO Pin: %u\n" + " MOSI Pin: %u\n" + " CS Pin: %u", + this->clk_pin_, this->miso_pin_, this->mosi_pin_, this->cs_pin_); + const char *spi_interface = "spi3"; + if (this->interface_ == SPI2_HOST) { + spi_interface = "spi2"; + } + ESP_LOGCONFIG(TAG, " Interface: %s", spi_interface); } - ESP_LOGCONFIG(TAG, " Interface: %s", spi_interface); #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT if (this->polling_interval_ != 0) { ESP_LOGCONFIG(TAG, " Polling Interval: %" PRIu32 " ms", this->polling_interval_); diff --git a/esphome/components/spi/spi.h b/esphome/components/spi/spi.h index 0358ed278f..17c59c895a 100644 --- a/esphome/components/spi/spi.h +++ b/esphome/components/spi/spi.h @@ -352,6 +352,8 @@ class SPIComponent final : public Component { this->using_hw_ = true; } + SPIInterface get_interface() const { return this->interface_; } + void set_interface_name(const char *name) { this->interface_name_ = name; } float get_setup_priority() const override { return setup_priority::BUS; } diff --git a/tests/component_tests/conftest.py b/tests/component_tests/conftest.py index 3730978ec3..4f0b786cc2 100644 --- a/tests/component_tests/conftest.py +++ b/tests/component_tests/conftest.py @@ -57,6 +57,14 @@ def reset_core() -> Generator[None]: CORE.reset() +@pytest.fixture(autouse=True) +def reset_full_config() -> Generator[None]: + """Give each test a clean final-validate config and restore it after.""" + token = final_validate.full_config.set({}) + yield + final_validate.full_config.reset(token) + + @pytest.fixture def set_core_config() -> Generator[SetCoreConfigCallable]: """Fixture to set up the core configuration for tests.""" diff --git a/tests/component_tests/ethernet/config/spi_id_shared_bus.yaml b/tests/component_tests/ethernet/config/spi_id_shared_bus.yaml new file mode 100644 index 0000000000..ffcd6a59f3 --- /dev/null +++ b/tests/component_tests/ethernet/config/spi_id_shared_bus.yaml @@ -0,0 +1,21 @@ +esphome: + name: test + +esp32: + board: esp32dev + +spi: + - id: spi_bus + interface: spi2 + clk_pin: GPIO18 + mosi_pin: GPIO23 + miso_pin: GPIO19 + +ethernet: + id: eth_component + type: W5500 + spi_id: spi_bus + cs_pin: GPIO5 + interrupt_pin: GPIO36 + reset_pin: GPIO22 + clock_speed: 20MHz diff --git a/tests/component_tests/ethernet/config/spi_own_bus.yaml b/tests/component_tests/ethernet/config/spi_own_bus.yaml new file mode 100644 index 0000000000..3169e6a39a --- /dev/null +++ b/tests/component_tests/ethernet/config/spi_own_bus.yaml @@ -0,0 +1,16 @@ +esphome: + name: test + +esp32: + board: esp32dev + +ethernet: + id: eth_component + type: W5500 + clk_pin: GPIO18 + mosi_pin: GPIO23 + miso_pin: GPIO19 + cs_pin: GPIO5 + interrupt_pin: GPIO36 + reset_pin: GPIO22 + clock_speed: 20MHz diff --git a/tests/component_tests/ethernet/test_ethernet.py b/tests/component_tests/ethernet/test_ethernet.py index 9308d0b099..522357f972 100644 --- a/tests/component_tests/ethernet/test_ethernet.py +++ b/tests/component_tests/ethernet/test_ethernet.py @@ -27,14 +27,6 @@ _CH390_CONFIG = { } -@pytest.fixture(autouse=True) -def _reset_full_config(): - """Reset fv.full_config so each test starts with a clean slate.""" - token = fv.full_config.set({}) - yield - fv.full_config.reset(token) - - def test_rejects_wifi_and_ethernet_without_priority() -> None: """Wi-Fi + ethernet without a network: priority: list must be rejected.""" fv.full_config.set({"wifi": {}, "ethernet": {}}) diff --git a/tests/component_tests/ethernet/test_spi_id.py b/tests/component_tests/ethernet/test_spi_id.py new file mode 100644 index 0000000000..e6dc9249e6 --- /dev/null +++ b/tests/component_tests/ethernet/test_spi_id.py @@ -0,0 +1,235 @@ +"""Tests for the ethernet `spi_id:` option (attach to a shared spi bus).""" + +from collections.abc import Callable +from pathlib import Path + +import pytest +from voluptuous import Invalid + +from esphome import config_validation as cv +from esphome.components.esp32 import ( + KEY_BOARD, + KEY_IDF_VERSION, + KEY_VARIANT, + VARIANT_ESP32S3, +) +from esphome.components.ethernet import CONF_INTERFACE, CONFIG_SCHEMA, _final_validate +from esphome.components.rp2.const import KEY_BOARD as RP2_KEY_BOARD + +# Registers the rp2 pin schema so RP2 configs can validate pins. +import esphome.components.rp2.gpio # noqa: F401 +from esphome.components.spi import CONF_INTERFACE_INDEX +from esphome.const import ( + CONF_CLK_PIN, + CONF_ID, + CONF_MISO_PIN, + CONF_MOSI_PIN, + CONF_SPI, + CONF_SPI_ID, + CONF_TYPE, + PlatformFramework, +) +from esphome.core import CORE, ID +import esphome.final_validate as fv + +from ..types import SetCoreConfigCallable + +_W5500_PIN_CONFIG = { + "type": "W5500", + "clk_pin": 47, + "mosi_pin": 48, + "miso_pin": 14, + "cs_pin": 21, +} + +_W5500_SPI_ID_CONFIG = { + "type": "W5500", + "spi_id": "spi_bus", + "cs_pin": 21, +} + + +def _set_esp32_s3(set_core_config: SetCoreConfigCallable) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={ + KEY_BOARD: "esp32-s3-devkitc-1", + KEY_VARIANT: VARIANT_ESP32S3, + KEY_IDF_VERSION: cv.Version(5, 3, 2), + }, + ) + # _validate derives use_address from the node name, which has no default here. + CORE.name = "spi-id-test" + + +def test_spi_id_accepted_without_pins_or_interface( + set_core_config: SetCoreConfigCallable, +) -> None: + """With spi_id set, the pin options are not required and no interface is defaulted.""" + _set_esp32_s3(set_core_config) + config = CONFIG_SCHEMA(dict(_W5500_SPI_ID_CONFIG)) + assert config[CONF_SPI_ID] == ID("spi_bus") + # The interface comes from the referenced bus; no default may be injected. + assert CONF_INTERFACE not in config + + +@pytest.mark.parametrize( + ("key", "value"), + [ + (CONF_CLK_PIN, 47), + (CONF_MOSI_PIN, 48), + (CONF_MISO_PIN, 14), + (CONF_INTERFACE, "spi2"), + ], +) +def test_spi_id_rejects_bus_options( + set_core_config: SetCoreConfigCallable, key: str, value: int | str +) -> None: + """Options provided by the referenced bus must be rejected alongside spi_id.""" + _set_esp32_s3(set_core_config) + with pytest.raises(Invalid, match=f"'{key}' cannot be used together with 'spi_id'"): + CONFIG_SCHEMA({**_W5500_SPI_ID_CONFIG, key: value}) + + +@pytest.mark.parametrize("key", [CONF_CLK_PIN, CONF_MOSI_PIN, CONF_MISO_PIN]) +def test_bus_pins_still_required_without_spi_id( + set_core_config: SetCoreConfigCallable, key: str +) -> None: + """Without spi_id, the bus pin options stay required.""" + _set_esp32_s3(set_core_config) + config = {k: v for k, v in _W5500_PIN_CONFIG.items() if k != key} + with pytest.raises( + Invalid, match=f"'{key}' is a required option when 'spi_id' is not set" + ): + CONFIG_SCHEMA(config) + + +def test_spi_id_rejected_on_rp2(set_core_config: SetCoreConfigCallable) -> None: + """spi_id is ESP32-only; the RP2 path is unchanged.""" + set_core_config( + PlatformFramework.RP2_ARDUINO, platform_data={RP2_KEY_BOARD: "rpipicow"} + ) + CORE.name = "spi-id-test" + config = { + "type": "W5500", + "spi_id": "spi_bus", + "clk_pin": 18, + "mosi_pin": 19, + "miso_pin": 16, + "cs_pin": 17, + } + with pytest.raises(Invalid, match="only available on"): + CONFIG_SCHEMA(config) + + +def _eth_spi_id_final_config() -> dict: + return {CONF_TYPE: "W5500", CONF_SPI_ID: ID("spi_bus")} + + +class _FakeFinalConfig(dict): + """Dict-backed FinalValidateConfig with just enough ID resolution for + fv.id_declaration_match_schema to find an spi bus fragment.""" + + def get_path_for_id(self, id: ID) -> list: + for index, conf in enumerate(self[CONF_SPI]): + if conf[CONF_ID] == id: + return [CONF_SPI, index, CONF_ID] + raise KeyError(id) + + def get_config_for_path(self, path: list) -> dict: + return self[path[0]][path[1]] + + +def _set_spi_buses(*buses: dict) -> None: + fv.full_config.set(_FakeFinalConfig({CONF_SPI: list(buses)})) + + +_SHAREABLE_BUS = { + CONF_ID: ID("spi_bus"), + CONF_INTERFACE_INDEX: 0, + CONF_MISO_PIN: {}, + CONF_MOSI_PIN: {}, +} + + +def test_final_validate_accepts_hardware_bus_with_data_pins( + set_core_config: SetCoreConfigCallable, +) -> None: + """A hardware spi bus that declares miso_pin and mosi_pin may be shared.""" + _set_esp32_s3(set_core_config) + # An unrelated bus first: the ID lookup must skip past it. + _set_spi_buses({CONF_ID: ID("other_bus"), CONF_INTERFACE_INDEX: 1}, _SHAREABLE_BUS) + _final_validate(_eth_spi_id_final_config()) + + +def test_final_validate_rejects_software_bus( + set_core_config: SetCoreConfigCallable, +) -> None: + """A software spi bus (no hardware interface index) cannot be shared.""" + _set_esp32_s3(set_core_config) + bus = {k: v for k, v in _SHAREABLE_BUS.items() if k != CONF_INTERFACE_INDEX} + _set_spi_buses(bus) + with pytest.raises(Invalid, match="requires this spi bus to use a hardware"): + _final_validate(_eth_spi_id_final_config()) + + +@pytest.mark.parametrize("pin_key", [CONF_MISO_PIN, CONF_MOSI_PIN]) +def test_final_validate_rejects_bus_without_data_pin( + set_core_config: SetCoreConfigCallable, pin_key: str +) -> None: + """The shared bus must declare both data pins to drive the ethernet chip.""" + _set_esp32_s3(set_core_config) + bus = {k: v for k, v in _SHAREABLE_BUS.items() if k != pin_key} + _set_spi_buses(bus) + with pytest.raises(Invalid, match=f"requires this spi bus to declare a {pin_key}"): + _final_validate(_eth_spi_id_final_config()) + + +def test_final_validate_rejects_colliding_host_without_spi_id( + set_core_config: SetCoreConfigCallable, +) -> None: + """Without spi_id, claiming the same host as an spi bus stays an error.""" + _set_esp32_s3(set_core_config) + fv.full_config.set({CONF_SPI: [{CONF_ID: ID("spi_bus"), CONF_INTERFACE_INDEX: 0}]}) + config = {CONF_TYPE: "W5500", CONF_INTERFACE: "spi2"} + with pytest.raises(Invalid, match="both using interface 'SPI2_HOST'"): + _final_validate(config) + + +def test_final_validate_accepts_distinct_host_without_spi_id( + set_core_config: SetCoreConfigCallable, +) -> None: + """Without spi_id, a different host than the spi bus is accepted.""" + _set_esp32_s3(set_core_config) + fv.full_config.set({CONF_SPI: [{CONF_ID: ID("spi_bus"), CONF_INTERFACE_INDEX: 0}]}) + _final_validate({CONF_TYPE: "W5500", CONF_INTERFACE: "spi3"}) + + +def test_generated_code_uses_spi_parent( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """With spi_id, codegen wires the spi parent and skips the bus options.""" + main_cpp = generate_main(component_config_path("spi_id_shared_bus.yaml")) + + assert "eth_component->set_spi_parent(spi_bus);" in main_cpp + assert "eth_component->set_cs_pin(5);" in main_cpp + assert "eth_component->set_clk_pin(" not in main_cpp + assert "eth_component->set_miso_pin(" not in main_cpp + assert "eth_component->set_mosi_pin(" not in main_cpp + assert "eth_component->set_interface(" not in main_cpp + + +def test_generated_code_without_spi_id_initializes_own_bus( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Without spi_id, codegen still emits the pin and interface setters.""" + main_cpp = generate_main(component_config_path("spi_own_bus.yaml")) + + assert "eth_component->set_spi_parent(" not in main_cpp + assert "eth_component->set_clk_pin(18);" in main_cpp + assert "eth_component->set_miso_pin(19);" in main_cpp + assert "eth_component->set_mosi_pin(23);" in main_cpp + assert "eth_component->set_cs_pin(5);" in main_cpp + assert "eth_component->set_interface(::SPI3_HOST);" in main_cpp diff --git a/tests/component_tests/network/test_priority.py b/tests/component_tests/network/test_priority.py index 017f0711a3..041b358dda 100644 --- a/tests/component_tests/network/test_priority.py +++ b/tests/component_tests/network/test_priority.py @@ -24,11 +24,9 @@ from tests.component_tests.types import SetCoreConfigCallable @pytest.fixture(autouse=True) def _clear_core_data(): - """Wipe CORE.data and reset fv.full_config so each test starts clean.""" + """Wipe CORE.data so each test starts clean.""" CORE.data.clear() - token = fv.full_config.set({}) yield - fv.full_config.reset(token) CORE.data.clear() diff --git a/tests/components/ethernet/common-w5500-spi-id.yaml b/tests/components/ethernet/common-w5500-spi-id.yaml new file mode 100644 index 0000000000..14c01842f0 --- /dev/null +++ b/tests/components/ethernet/common-w5500-spi-id.yaml @@ -0,0 +1,17 @@ +ethernet: + type: W5500 + spi_id: spi_bus + cs_pin: 5 + interrupt_pin: 36 + reset_pin: 22 + clock_speed: 10Mhz + manual_ip: + static_ip: 192.168.178.56 + gateway: 192.168.178.1 + subnet: 255.255.255.0 + domain: .local + mac_address: "02:AA:BB:CC:DD:01" + on_connect: + - logger.log: "Ethernet connected!" + on_disconnect: + - logger.log: "Ethernet disconnected!" diff --git a/tests/components/ethernet/test-w5500-spi-id.esp32-idf.yaml b/tests/components/ethernet/test-w5500-spi-id.esp32-idf.yaml new file mode 100644 index 0000000000..16a25ced2a --- /dev/null +++ b/tests/components/ethernet/test-w5500-spi-id.esp32-idf.yaml @@ -0,0 +1,3 @@ +packages: + spi: !include ../../test_build_components/common/spi/esp32-idf.yaml + ethernet: !include common-w5500-spi-id.yaml From 0b78800e3d2e03e5e4ea398ed2f63b6ebc3de174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Metrich?= <45318189+FredM67@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:07:18 +0200 Subject: [PATCH 33/65] [emontx] Add apparent power (AP) and frequency (F) sensor support (#18586) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/emontx/sensor/__init__.py | 98 +++++++++++++--- .../emontx/test_sensor_defaults.py | 109 ++++++++++++++++++ .../components/emontx/validate.esp32-idf.yaml | 15 +++ 3 files changed, 207 insertions(+), 15 deletions(-) diff --git a/esphome/components/emontx/sensor/__init__.py b/esphome/components/emontx/sensor/__init__.py index 967bc4e699..56a7fb8b55 100644 --- a/esphome/components/emontx/sensor/__init__.py +++ b/esphome/components/emontx/sensor/__init__.py @@ -7,8 +7,10 @@ from esphome.const import ( CONF_ID, CONF_STATE_CLASS, CONF_UNIT_OF_MEASUREMENT, + DEVICE_CLASS_APPARENT_POWER, DEVICE_CLASS_CURRENT, DEVICE_CLASS_ENERGY, + DEVICE_CLASS_FREQUENCY, DEVICE_CLASS_POWER, DEVICE_CLASS_POWER_FACTOR, DEVICE_CLASS_TEMPERATURE, @@ -18,8 +20,10 @@ from esphome.const import ( UNIT_AMPERE, UNIT_CELSIUS, UNIT_EMPTY, + UNIT_HERTZ, UNIT_PULSES, UNIT_VOLT, + UNIT_VOLT_AMPS, UNIT_WATT, UNIT_WATT_HOURS, ) @@ -29,6 +33,32 @@ from .. import CONF_EMONTX_ID, CONF_TAG_NAME, EmonTx, emontx_ns EmonTxSensor = emontx_ns.class_("EmonTxSensor", sensor.Sensor, cg.Component) +# Known emonTx/avrdb JSON tag conventions, gathered from real firmware +# (see https://github.com/openenergymonitor/avrdb_firmware), used to decide +# whether each tag below requires a numeric index or may also appear bare: +# +# Tag family Bare (no index) Numeric-indexed +# ----------- ----------------------- ---------------------------------- +# P (power) no P1, P2, ... (multi-channel boards) +# E (energy) no E1, E2, ... +# V (voltage) Vrms (NOT matched here, V1, V2, V3 (per-phase boards) +# doesn't fit "V"+digits) +# I (current) no I1, I2, ... +# T (temp.) no T1, T2, ... +# F (frequency) F (single mains freq.) not seen indexed +# PULSE pulse (single-CT boards) PULSE1, PULSE2, ... (other variants) +# PF (power not seen bare PF1, PF2, ... (currently unused/ +# factor) commented out in avrdb firmware) +# AP (apparent not seen bare AP1, AP2, ... (not an avrdb tag at +# power) all; avrdb uses "VA"+index instead, +# itself currently unused/commented +# out; "AP" is kept here for other +# firmware/integrations using it) +# +# This is why a bare "PULSE" resolves to proper defaults below, but bare +# "PF"/"AP" fall back to generic defaults instead: only PULSE has a +# confirmed bare-tag use in real, currently-shipping firmware. + # Define sensor type configurations by prefix SENSOR_CONFIGS = { "P": { @@ -63,7 +93,25 @@ SENSOR_CONFIGS = { }, } -# Pattern-based configurations +# Tags reported once, without a numeric index (e.g. "F"), matched exactly +# rather than by prefix. +EXACT_TAG_CONFIGS = { + "F": { + CONF_UNIT_OF_MEASUREMENT: UNIT_HERTZ, + CONF_DEVICE_CLASS: DEVICE_CLASS_FREQUENCY, + CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT, + CONF_ACCURACY_DECIMALS: 2, + }, +} + +# Pattern-based configurations. The remainder after the prefix must be a +# non-empty numeric index (like V1/I1/E1), so e.g. "APPLE" doesn't collide +# with the "AP" prefix and a bare "PF"/"AP" (no index) doesn't match. +# "PULSE" is the exception: some emonTx firmware (e.g. avrdb-based single-CT +# variants) reports a single pulse counter as a bare "pulse" tag with no +# numeric index at all, so that pattern also accepts an empty suffix. +PATTERNS_ALLOWING_BARE_TAG = {"PULSE"} + PATTERN_CONFIGS = { "PULSE": { CONF_UNIT_OF_MEASUREMENT: UNIT_PULSES, @@ -77,14 +125,21 @@ PATTERN_CONFIGS = { CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT, CONF_ACCURACY_DECIMALS: 2, }, + "AP": { + CONF_UNIT_OF_MEASUREMENT: UNIT_VOLT_AMPS, + CONF_DEVICE_CLASS: DEVICE_CLASS_APPARENT_POWER, + CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT, + CONF_ACCURACY_DECIMALS: 2, + }, } # BASE_SCHEMA intentionally omits state_class and accuracy_decimals defaults. # Passing them to sensor_schema() would register them via cv.Optional(key, default=...), # making them always present in the validated config dict and preventing # apply_tag_defaults from overriding them with the correct per-prefix values. -# They are injected by apply_tag_defaults below, after running through -# sensor.validate_state_class() so the value is code-generation-ready. +# They are injected by apply_tag_defaults below, after running through the +# same validators sensor_schema() would use (see _DEFAULT_VALIDATORS) so the +# values are code-generation-ready. BASE_SCHEMA = sensor.sensor_schema(EmonTxSensor).extend( { cv.GenerateID(CONF_EMONTX_ID): cv.use_id(EmonTx), @@ -93,30 +148,43 @@ BASE_SCHEMA = sensor.sensor_schema(EmonTxSensor).extend( ) +_DEFAULT_VALIDATORS = { + CONF_STATE_CLASS: sensor.validate_state_class, + CONF_DEVICE_CLASS: sensor.validate_device_class, + CONF_UNIT_OF_MEASUREMENT: sensor.validate_unit_of_measurement, +} + + def _apply_defaults(config: ConfigType, defaults: dict) -> None: """Inject defaults into config, skipping keys already set by the user. - state_class values are run through validate_state_class so they are - code-generation-ready, matching what sensor_schema() would normally do.""" + Values are run through the same validators sensor_schema() would use, so + they are code-generation-ready and a typo'd constant fails validation + instead of shipping silently.""" for key, value in defaults.items(): if key not in config: - if key == CONF_STATE_CLASS: - value = sensor.validate_state_class(value) + if key in _DEFAULT_VALIDATORS: + value = _DEFAULT_VALIDATORS[key](value) config[key] = value def apply_tag_defaults(config: ConfigType) -> ConfigType: """Apply defaults based on tag prefix if applicable, but don't restrict any tags.""" tag = config[CONF_TAG_NAME] + tag_upper = tag.upper() + if (exact_config := EXACT_TAG_CONFIGS.get(tag_upper)) is not None: + _apply_defaults(config, exact_config) + return config + + for pattern, pattern_config in PATTERN_CONFIGS.items(): + suffix = tag_upper[len(pattern) :] + bare_ok = not suffix and pattern in PATTERNS_ALLOWING_BARE_TAG + if tag_upper.startswith(pattern) and (suffix.isdigit() or bare_ok): + _apply_defaults(config, pattern_config) + return config + + # Only apply defaults for known prefixes with numeric indices (e.g. E1, V2, T3) if len(tag) >= 2: - tag_upper = tag.upper() - - for pattern, pattern_config in PATTERN_CONFIGS.items(): - if tag_upper.startswith(pattern): - _apply_defaults(config, pattern_config) - return config - - # Only apply defaults for known prefixes with numeric indices (e.g. E1, V2, T3) prefix = tag_upper[0] if prefix in SENSOR_CONFIGS and tag[1:].isdigit(): _apply_defaults(config, SENSOR_CONFIGS[prefix]) diff --git a/tests/component_tests/emontx/test_sensor_defaults.py b/tests/component_tests/emontx/test_sensor_defaults.py index 00d24d282e..5c6a8e4108 100644 --- a/tests/component_tests/emontx/test_sensor_defaults.py +++ b/tests/component_tests/emontx/test_sensor_defaults.py @@ -6,9 +6,28 @@ from esphome.components import sensor from esphome.components.emontx.sensor import CONFIG_SCHEMA, apply_tag_defaults from esphome.const import ( CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, CONF_STATE_CLASS, + CONF_UNIT_OF_MEASUREMENT, + DEVICE_CLASS_APPARENT_POWER, + DEVICE_CLASS_CURRENT, + DEVICE_CLASS_ENERGY, + DEVICE_CLASS_FREQUENCY, + DEVICE_CLASS_POWER, + DEVICE_CLASS_POWER_FACTOR, + DEVICE_CLASS_TEMPERATURE, + DEVICE_CLASS_VOLTAGE, STATE_CLASS_MEASUREMENT, STATE_CLASS_TOTAL_INCREASING, + UNIT_AMPERE, + UNIT_CELSIUS, + UNIT_EMPTY, + UNIT_HERTZ, + UNIT_PULSES, + UNIT_VOLT, + UNIT_VOLT_AMPS, + UNIT_WATT, + UNIT_WATT_HOURS, ) @@ -61,9 +80,25 @@ def _make_config(tag: str) -> dict: ("PULSE1", STATE_CLASS_TOTAL_INCREASING, 0), ("PULSE12", STATE_CLASS_TOTAL_INCREASING, 0), ("PF1", STATE_CLASS_MEASUREMENT, 2), + ("AP1", STATE_CLASS_MEASUREMENT, 2), + ("AP12", STATE_CLASS_MEASUREMENT, 2), + # Frequency: reported as a single, un-numbered tag + ("F", STATE_CLASS_MEASUREMENT, 2), # Unknown / free-form tags fall back to generic defaults ("CUSTOM1", STATE_CLASS_MEASUREMENT, 0), ("X", STATE_CLASS_MEASUREMENT, 0), + # "F1" is not the exact "F" tag, so it falls back to generic defaults + ("F1", STATE_CLASS_MEASUREMENT, 0), + # "PULSE" (no index) is how some real emonTx firmware reports a + # single pulse counter, so it still resolves to the PULSE defaults + ("PULSE", STATE_CLASS_TOTAL_INCREASING, 0), + # Real firmware sends this lowercase; tag_upper's case-folding must + # still match it against the PULSE pattern + ("pulse", STATE_CLASS_TOTAL_INCREASING, 0), + # PF/AP require a numeric index; the bare prefix alone (no index) + # falls back to generic defaults + ("PF", STATE_CLASS_MEASUREMENT, 0), + ("AP", STATE_CLASS_MEASUREMENT, 0), ], ) def test_apply_tag_defaults(tag, expected_state_class, expected_decimals): @@ -76,6 +111,80 @@ def test_apply_tag_defaults(tag, expected_state_class, expected_decimals): assert result[CONF_ACCURACY_DECIMALS] == expected_decimals +@pytest.mark.parametrize( + ("tag", "expected_unit", "expected_device_class"), + [ + # Known numeric-index prefixes + ("E1", UNIT_WATT_HOURS, DEVICE_CLASS_ENERGY), + ("E12", UNIT_WATT_HOURS, DEVICE_CLASS_ENERGY), + ("P1", UNIT_WATT, DEVICE_CLASS_POWER), + ("V1", UNIT_VOLT, DEVICE_CLASS_VOLTAGE), + ("I1", UNIT_AMPERE, DEVICE_CLASS_CURRENT), + ("T1", UNIT_CELSIUS, DEVICE_CLASS_TEMPERATURE), + # Known patterns + ("PULSE1", UNIT_PULSES, DEVICE_CLASS_ENERGY), + ("PULSE12", UNIT_PULSES, DEVICE_CLASS_ENERGY), + # Bare "PULSE" (no index), as reported by some real emonTx firmware + ("PULSE", UNIT_PULSES, DEVICE_CLASS_ENERGY), + # Real firmware sends this lowercase; tag_upper's case-folding must + # still match it against the PULSE pattern + ("pulse", UNIT_PULSES, DEVICE_CLASS_ENERGY), + ("PF1", UNIT_EMPTY, DEVICE_CLASS_POWER_FACTOR), + ("AP1", UNIT_VOLT_AMPS, DEVICE_CLASS_APPARENT_POWER), + ("AP12", UNIT_VOLT_AMPS, DEVICE_CLASS_APPARENT_POWER), + # Frequency: reported as a single, un-numbered tag + ("F", UNIT_HERTZ, DEVICE_CLASS_FREQUENCY), + ], +) +def test_apply_tag_defaults_unit_and_device_class( + tag, expected_unit, expected_device_class +): + """apply_tag_defaults must inject the correct, validated unit_of_measurement + and device_class for each tag type when no user overrides are present.""" + config = _make_config(tag) + result = apply_tag_defaults(config) + + assert result[CONF_UNIT_OF_MEASUREMENT] == sensor.validate_unit_of_measurement( + expected_unit + ) + assert result[CONF_DEVICE_CLASS] == sensor.validate_device_class( + expected_device_class + ) + + +@pytest.mark.parametrize( + "tag", + [ + "CUSTOM1", + "X", + # Non-numeric suffixes must not collide with a PATTERN_CONFIGS prefix + # (e.g. "APPLE" starting with "AP", "PFX" starting with "PF"). + "APPLE", + "PFX", + "PULSE_A", + # "F1" is not the exact "F" tag + "F1", + # Bare "PF"/"AP" (no numeric index) don't match; unlike "PULSE", + # real firmware never reports these without an index + "PF", + "AP", + ], +) +def test_apply_tag_defaults_unknown_tag_has_no_unit_or_device_class(tag): + """Unknown / free-form tags only get generic state_class and + accuracy_decimals defaults; unit_of_measurement and device_class are left + for the user to set explicitly.""" + config = _make_config(tag) + result = apply_tag_defaults(config) + + assert CONF_UNIT_OF_MEASUREMENT not in result + assert CONF_DEVICE_CLASS not in result + assert result[CONF_STATE_CLASS] == sensor.validate_state_class( + STATE_CLASS_MEASUREMENT + ) + assert result[CONF_ACCURACY_DECIMALS] == 0 + + @pytest.mark.parametrize( ("tag", "user_state_class", "user_decimals"), [ diff --git a/tests/components/emontx/validate.esp32-idf.yaml b/tests/components/emontx/validate.esp32-idf.yaml index 7caee78a07..882ee26fcb 100644 --- a/tests/components/emontx/validate.esp32-idf.yaml +++ b/tests/components/emontx/validate.esp32-idf.yaml @@ -57,6 +57,21 @@ sensor: name: Power Factor 1 emontx_id: test_emontx + # Apparent power sensor (AP pattern): expects state_class=measurement, + # unit=VA, device_class=apparent_power, accuracy_decimals=2 + - platform: emontx + tag_name: AP1 + name: Apparent Power 1 + emontx_id: test_emontx + + # Frequency sensor (F, matched exactly, not as a prefix): expects + # state_class=measurement, unit=Hz, device_class=frequency, + # accuracy_decimals=2 + - platform: emontx + tag_name: F + name: Frequency + emontx_id: test_emontx + # Unknown tag: no prefix match, falls back to state_class=measurement, # accuracy_decimals=0 - platform: emontx From 19b434f99c6db5123263ff07f53fa507b025ff30 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 12:36:57 -0500 Subject: [PATCH 34/65] [espidf] Cache the discovered component list to skip the discovery configure (#18752) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/build_gen/espidf.py | 71 ++++--- esphome/espidf/toolchain.py | 129 ++++++++++-- tests/unit_tests/build_gen/test_espidf.py | 91 +++++++- tests/unit_tests/test_espidf_toolchain.py | 242 +++++++++++++++++++++- 4 files changed, 473 insertions(+), 60 deletions(-) diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 5d4e6b8401..2ef89cf595 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -1,6 +1,7 @@ """ESP-IDF direct build generator for ESPHome.""" import json +import logging from pathlib import Path from esphome.components.esp32 import ( @@ -11,6 +12,7 @@ from esphome.components.esp32 import ( ) import esphome.config_validation as cv from esphome.core import CORE +from esphome.espidf import variant_to_idf_target from esphome.framework_helpers import ( get_project_compile_flags, get_project_cxx_compile_flags, @@ -18,6 +20,8 @@ from esphome.framework_helpers import ( ) from esphome.helpers import mkdir_p, write_file_if_changed +_LOGGER = logging.getLogger(__name__) + # Replaces the IDF default C++ standard (-std=gnu++2b appended to # CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via # cg.set_cpp_standard(). Emitted between include(project.cmake) and project(), @@ -31,11 +35,12 @@ idf_build_set_property(CXX_COMPILE_OPTIONS "${{esphome_cxx_compile_options}}")"" def get_available_components() -> list[str] | None: - """Get list of built-in ESP-IDF components from project_description.json. + """List the built-in ESP-IDF components from ``project_description.json``. - Excludes ``src``, IDF-managed components (``managed_components/``), and - converted PIO libs (``pio_components/``). Returns ``None`` if the build - dir or ``project_description.json`` isn't ready yet. + Only components below its ``idf_path/components`` count, which leaves out + ``src``, IDF-managed components, converted PIO libs and project local + ones such as the Arduino ``component_stubs``. Returns ``None`` if the + build dir or ``project_description.json`` isn't ready yet. """ if CORE.build_path is None: return None @@ -46,30 +51,24 @@ def get_available_components() -> list[str] | None: try: with project_desc.open(encoding="utf-8") as f: data = json.load(f) - - component_info = data.get("build_component_info", {}) - - result = [] - for name, info in component_info.items(): - # Exclude our own src component - if name == "src": - continue - - # Exclude IDF-managed and converted-PIO components (external). - comp_dir = info.get("dir", "") - if "managed_components" in comp_dir or "pio_components" in comp_dir: - continue - - result.append(name) - - return result - except (json.JSONDecodeError, OSError): + root = (Path(data["idf_path"]) / "components").resolve() + result = [ + name + for name, info in data.get("build_component_info", {}).items() + if (comp_dir := info.get("dir")) + and Path(comp_dir).resolve().is_relative_to(root) + ] + except (json.JSONDecodeError, KeyError, OSError) as err: + _LOGGER.debug("Could not read %s: %s", project_desc, err) return None + if not result: + _LOGGER.warning("No ESP-IDF components found under %s", root) + return result def has_discovered_components() -> bool: - """Check if we have discovered components from a previous configure.""" - return get_available_components() is not None + """Check if a previous configure discovered any built-in components.""" + return bool(get_available_components()) def _cmake_quote(value: str) -> str: @@ -79,15 +78,17 @@ def _cmake_quote(value: str) -> str: return f'"{escaped}"' -def get_project_cmakelists(minimal: bool = False) -> str: +def get_project_cmakelists( + minimal: bool = False, builtin_components: list[str] | None = None +) -> str: """Generate the top-level CMakeLists.txt for ESP-IDF project. When ``minimal`` is true, omit ``ESPHOME_PROJECT_BUILTIN_COMPONENTS`` since ``project_description.json`` may be stale on the first write. + ``builtin_components`` supplies the discovered list (from the cache) + instead of reading it from ``project_description.json``. """ - # Get IDF target from ESP32 variant (e.g., ESP32S3 -> esp32s3) - variant = get_esp32_variant() - idf_target = variant.lower().replace("-", "") + idf_target = variant_to_idf_target(get_esp32_variant()) # esp_idf_size 2.x (bundled with IDF >=6.0) made NG the default and # removed the --ng flag; on 1.x (IDF 5.5) --ng is required to get @@ -162,9 +163,11 @@ def get_project_cmakelists(minimal: bool = False) -> str: else "\n".join( f"idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS {name} APPEND)" for name in sorted( - set(get_available_components() or []).difference( - CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";") - ) + set( + builtin_components + if builtin_components is not None + else get_available_components() or [] + ).difference(CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";")) ) ) ) @@ -279,7 +282,9 @@ target_link_options(${{COMPONENT_LIB}} PUBLIC """ -def write_project(minimal: bool = False) -> None: +def write_project( + minimal: bool = False, builtin_components: list[str] | None = None +) -> None: """Write ESP-IDF project files.""" mkdir_p(CORE.build_path) mkdir_p(CORE.relative_src_path()) @@ -287,7 +292,7 @@ def write_project(minimal: bool = False) -> None: # Write top-level CMakeLists.txt write_file_if_changed( CORE.relative_build_path("CMakeLists.txt"), - get_project_cmakelists(minimal=minimal), + get_project_cmakelists(minimal=minimal, builtin_components=builtin_components), ) # Write component CMakeLists.txt in src/ diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index baf316a4ee..986f9dfb8b 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -1,6 +1,7 @@ """ESP-IDF direct build API for ESPHome.""" from dataclasses import dataclass, field +import hashlib import json import logging import os @@ -23,7 +24,7 @@ from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary -from esphome.helpers import add_git_ceiling_directory +from esphome.helpers import add_git_ceiling_directory, write_file _LOGGER = logging.getLogger(__name__) @@ -256,6 +257,106 @@ def run_reconfigure() -> int: return run_idf_py(*_get_sdkconfig_args(), "reconfigure") +def _builtin_component_cache_path() -> Path | None: + """Cache file for this build's built-in component list. + + The file lives inside the extracted framework directory so it is + discarded together with that exact checkout (re-extract, source + override, clean-all); the target and the EXCLUDE_COMPONENTS set name it. + The sdkconfig is not part of the key: IDF components register regardless + of CONFIG_* options and only gate their sources on them. A checkout + supplied through IDF_PATH is not managed by ESPHome and is never cached. + """ + if "IDF_PATH" in os.environ: + return None + target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT]) + excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "") + excluded_key = hashlib.sha256(excluded.encode()).hexdigest()[:12] + return ( + _get_idf_path() / ".esphome_component_lists" / f"{target}-{excluded_key}.json" + ) + + +def load_cached_builtin_components() -> list[str] | None: + """Return the cached built-in component list for this build, if valid. + + Every name must still exist under ``$IDF_PATH/components`` so a stale + entry is treated as a miss instead of failing the configure. + """ + if (path := _builtin_component_cache_path()) is None: + return None + try: + components = json.loads(path.read_text(encoding="utf-8")) + present = { + entry.name + for entry in (path.parents[1] / "components").iterdir() + if entry.is_dir() + } + except (OSError, ValueError): + return None + if ( + isinstance(components, list) + and all(isinstance(c, str) for c in components) + and present.issuperset(components) + ): + return components + return None + + +def save_cached_builtin_components(components: list[str]) -> None: + """Store a built-in component list that just configured successfully.""" + if not components or (path := _builtin_component_cache_path()) is None: + return + try: + write_file(path, json.dumps(components, separators=(",", ":"))) + except EsphomeError as err: + _LOGGER.warning("Could not write component list cache %s: %s", path, err) + + +def _write_project_and_reconfigure(builtin_components: list[str] | None) -> int: + """Write the full CMakeLists.txt and run the configure for it.""" + from esphome.build_gen.espidf import write_project + + _LOGGER.info("Writing CMakeLists.txt with the built-in component list...") + write_project(minimal=False, builtin_components=builtin_components) + # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt + # is strictly newer than build.ninja, which fails on coarse-mtime + # filesystems (#18682). Also keeps idf.py from regenerating memory.ld + # in testing mode. + return run_reconfigure() + + +def _configure_project() -> int: + """Configure the project, discovering the built-in components if needed. + + A cached component list skips the discovery configure. If the configure + with a cached list fails the entry is dropped and discovery runs once; a + list is only cached after it configured successfully. + """ + from esphome.build_gen.espidf import get_available_components, write_project + + if (cached := load_cached_builtin_components()) is not None: + _LOGGER.info("Using cached ESP-IDF component list") + if _write_project_and_reconfigure(cached) == 0: + return 0 + _LOGGER.warning("Cached component list failed; rediscovering") + _builtin_component_cache_path().unlink(missing_ok=True) + _LOGGER.info("Discovering available ESP-IDF components...") + write_project(minimal=True) + if (rc := run_reconfigure()) != 0: + _LOGGER.error("Component discovery failed") + return rc + discovered = get_available_components() + if not discovered: + _LOGGER.error("Component discovery found no built-in ESP-IDF components") + return 1 + if (rc := _write_project_and_reconfigure(discovered)) != 0: + _LOGGER.error("Reconfigure with discovered components failed") + return rc + save_cached_builtin_components(discovered) + return 0 + + def has_outdated_files(): """Check if the build configuration is stale. @@ -382,29 +483,17 @@ def run_compile(config, verbose: bool) -> int: """Compile the ESP-IDF project. Uses two-phase configure to auto-discover available components: - 1. If no previous build, configure with minimal REQUIRES to discover components + 1. If no previous build, configure with minimal REQUIRES to discover + components (skipped when a cached list for this IDF/target/exclusion + set exists) 2. Regenerate CMakeLists.txt with discovered components 3. Run full build """ - from esphome.build_gen.espidf import write_project - # Check if we need to do discovery phase - if need_reconfigure(): - _LOGGER.info("Discovering available ESP-IDF components...") - write_project(minimal=True) - rc = run_reconfigure() - if rc != 0: - _LOGGER.error("Component discovery failed") - return rc - _LOGGER.info("Regenerating CMakeLists.txt with discovered components...") - write_project(minimal=False) - # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt - # is strictly newer than build.ninja, which fails on coarse-mtime - # filesystems (#18682). Also keeps idf.py from regenerating memory.ld - # in testing mode. - rc = run_reconfigure() - if rc != 0: - _LOGGER.error("Reconfigure with discovered components failed") + if not need_reconfigure(): + _LOGGER.info("Build configuration is up to date") + else: + if (rc := _configure_project()) != 0: return rc # cmake does not rewrite CMakeCache.txt when only properties change, # so restamp it or every build repeats discovery. Only after success, diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 29010bcf0e..079f10ddb9 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging from pathlib import Path from unittest.mock import patch @@ -35,22 +36,25 @@ def _reset_core(tmp_path: Path) -> None: } -def _write_project_description(tmp_path: Path, components: dict[str, str]) -> None: +def _write_project_description( + tmp_path: Path, components: dict[str, str], idf_path: str = "/idf" +) -> None: """Stub a project_description.json with the given component_name -> dir map.""" build_dir = tmp_path / "build" build_dir.mkdir(exist_ok=True) (build_dir / "project_description.json").write_text( json.dumps( { + "idf_path": idf_path, "build_component_info": { name: {"dir": dir_} for name, dir_ in components.items() - } + }, } ) ) -def _render(minimal: bool = False) -> str: +def _render(minimal: bool = False, builtin_components: list[str] | None = None) -> str: """Render the top-level CMakeLists with the standard variant/name patches.""" with ( patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"), @@ -58,7 +62,9 @@ def _render(minimal: bool = False) -> str: ): from esphome.build_gen.espidf import get_project_cmakelists - return get_project_cmakelists(minimal=minimal) + return get_project_cmakelists( + minimal=minimal, builtin_components=builtin_components + ) def test_get_available_components_returns_none_without_build_path() -> None: @@ -77,8 +83,11 @@ def test_get_available_components_returns_none_without_project_description( assert get_available_components() is None -def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> None: - """Built-ins are returned; src/, managed_components/, pio_components/ skipped.""" +def test_get_available_components_keeps_only_idf_tree_components( + tmp_path: Path, +) -> None: + """Only components under idf_path/components are built-ins: src, managed, + converted PIO libs and Arduino component_stubs are all left out.""" _write_project_description( tmp_path, { @@ -86,6 +95,7 @@ def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> "esp_lcd": "/idf/components/esp_lcd", "espressif__arduino-esp32": f"{tmp_path}/managed_components/arduino", "JPEGDEC": f"{tmp_path}/pio_components/arduino/abc/bitbank2/JPEGDEC", + "cbor": f"{tmp_path}/component_stubs/cbor", "freertos": "/idf/components/freertos", }, ) @@ -94,6 +104,75 @@ def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> assert sorted(get_available_components()) == ["esp_lcd", "freertos"] +def test_codegen_and_configure_writes_render_the_same_cmakelists( + tmp_path: Path, +) -> None: + """write_project() at codegen time (no list) and the configure-time write + (discovered list) must agree, or ninja re-runs cmake on every build.""" + _write_project_description( + tmp_path, + { + "lwip": "/idf/components/lwip", + "cbor": f"{tmp_path}/component_stubs/cbor", + }, + ) + from esphome.build_gen.espidf import get_available_components + + assert _render() == _render(builtin_components=get_available_components()) + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS cbor" not in _render() + + +def test_get_available_components_warns_when_nothing_is_under_idf_path( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_project_description(tmp_path, {"cbor": f"{tmp_path}/component_stubs/cbor"}) + from esphome.build_gen.espidf import ( + get_available_components, + has_discovered_components, + ) + + assert get_available_components() == [] + assert "No ESP-IDF components found under" in caplog.text + # An empty discovery must not count as configured, or it would be latched in. + assert not has_discovered_components() + + +def test_get_available_components_ignores_corrupt_or_unexpected_file( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + build_dir = tmp_path / "build" + build_dir.mkdir() + from esphome.build_gen.espidf import ( + get_available_components, + has_discovered_components, + ) + + (build_dir / "project_description.json").write_text("{not json") + assert get_available_components() is None + assert not has_discovered_components() + (build_dir / "project_description.json").write_text('{"build_component_info": {}}') + with caplog.at_level(logging.DEBUG, logger="esphome.build_gen.espidf"): + assert get_available_components() is None + assert "Could not read" in caplog.text + + +def test_has_discovered_components_after_configure(tmp_path: Path) -> None: + _write_project_description(tmp_path, {"lwip": "/idf/components/lwip"}) + from esphome.build_gen.espidf import has_discovered_components + + assert has_discovered_components() + + +def test_get_project_cmakelists_uses_supplied_builtin_components() -> None: + """A cached list replaces project_description.json and is still filtered + by EXCLUDE_COMPONENTS.""" + with patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": "fatfs;unity"}): + content = _render(builtin_components=["lwip", "fatfs", "esp_timer"]) + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_timer APPEND" in content + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS lwip APPEND" in content + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS fatfs APPEND" not in content + + def test_get_project_cmakelists_minimal_omits_builtin_components_property( tmp_path: Path, ) -> None: diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index c54daec6a4..9deb27d83c 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -2,6 +2,8 @@ # pylint: disable=protected-access +from collections.abc import Iterator +from contextlib import contextmanager import json import os from pathlib import Path @@ -309,6 +311,11 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No with ( patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=None), + patch.object(toolchain, "save_cached_builtin_components"), + patch( + "esphome.build_gen.espidf.get_available_components", return_value=["lwip"] + ), patch("esphome.build_gen.espidf.write_project"), patch.object(toolchain, "run_reconfigure", return_value=0), patch.object(toolchain, "run_idf_py", return_value=0), @@ -329,6 +336,11 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: with ( patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=None), + patch.object(toolchain, "save_cached_builtin_components"), + patch( + "esphome.build_gen.espidf.get_available_components", return_value=["lwip"] + ), patch("esphome.build_gen.espidf.write_project"), patch.object(toolchain, "run_reconfigure", return_value=0), patch.object(toolchain, "run_idf_py", return_value=0), @@ -354,7 +366,7 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( calls: list[tuple] = [] reconfigures = 0 - def record_write(minimal: bool = False) -> None: + def record_write(minimal: bool = False, builtin_components=None) -> None: calls.append(("write_project", minimal)) def record_reconfigure() -> int: @@ -365,6 +377,11 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( with ( patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=None), + patch.object(toolchain, "save_cached_builtin_components"), + patch( + "esphome.build_gen.espidf.get_available_components", return_value=["lwip"] + ), patch("esphome.build_gen.espidf.write_project", side_effect=record_write), patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure), patch.object(toolchain, "run_idf_py", return_value=0) as mock_build, @@ -383,6 +400,229 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( assert cmakecache.stat().st_mtime == old +def _record_compile_calls( + cached: list[str] | None, + saved: list[str] | None = None, + reconfigure_rcs: tuple[int, ...] = (), + cache_file: Path | None = None, +) -> tuple[int, list[tuple]]: + """Run run_compile with a stubbed cache and return (rc, call log). + + ``reconfigure_rcs`` overrides the exit codes of the first reconfigures; + later ones succeed. + """ + calls: list[tuple] = [] + rcs = iter(reconfigure_rcs) + + def record_reconfigure() -> int: + calls.append(("run_reconfigure",)) + return next(rcs, 0) + + def record_write(minimal: bool = False, builtin_components=None) -> None: + calls.append(("write_project", minimal, builtin_components)) + + def record_save(components: list[str]) -> None: + calls.append(("save", components)) + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=cached), + patch.object( + toolchain, "save_cached_builtin_components", side_effect=record_save + ), + patch("esphome.build_gen.espidf.get_available_components", return_value=saved), + patch("esphome.build_gen.espidf.write_project", side_effect=record_write), + patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure), + patch.object( + toolchain, "_builtin_component_cache_path", return_value=cache_file + ), + patch.object( + toolchain, + "run_idf_py", + side_effect=lambda *a, **kw: calls.append(("build",)) or 0, + ), + patch.object(toolchain, "print_summary"), + ): + rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) + return rc, calls + + +def test_run_compile_poisoned_cache_is_dropped_and_rediscovered( + setup_core: Path, tmp_path: Path +) -> None: + """A cached list that fails the configure is deleted and discovery runs + once more instead of every later build failing the same way.""" + _setup_build(setup_core) + cache_file = tmp_path / "esp32-abc.json" + cache_file.write_text("[]") + rc, calls = _record_compile_calls( + ["stale"], saved=["lwip"], reconfigure_rcs=(1,), cache_file=cache_file + ) + assert rc == 0 + assert not cache_file.exists() + assert calls == [ + ("write_project", False, ["stale"]), + ("run_reconfigure",), + ("write_project", True, None), + ("run_reconfigure",), + ("write_project", False, ["lwip"]), + ("run_reconfigure",), + ("save", ["lwip"]), + ("build",), + ] + + +def test_run_compile_cache_miss_discovers_and_saves(setup_core: Path) -> None: + """Without a cached list the discovery configure runs, the discovered list + feeds the full write and is cached only after that configure succeeds.""" + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, saved=["lwip"]) + assert rc == 0 + assert calls == [ + ("write_project", True, None), + ("run_reconfigure",), + ("write_project", False, ["lwip"]), + ("run_reconfigure",), + ("save", ["lwip"]), + ("build",), + ] + + +def test_run_compile_discovery_failure_stops_before_full_write( + setup_core: Path, +) -> None: + """A failed discovery configure returns its exit code and never writes + the full CMakeLists, a cache entry or a build.""" + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, reconfigure_rcs=(2,)) + assert rc == 2 + assert calls == [("write_project", True, None), ("run_reconfigure",)] + + +@pytest.mark.parametrize("discovered", [None, []], ids=["no_manifest", "empty"]) +def test_run_compile_fails_when_discovery_finds_nothing( + setup_core: Path, + caplog: pytest.LogCaptureFixture, + discovered: list[str] | None, +) -> None: + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, saved=discovered) + assert rc == 1 + assert calls == [("write_project", True, None), ("run_reconfigure",)] + assert "found no built-in ESP-IDF components" in caplog.text + + +def test_run_compile_does_not_cache_a_list_that_failed_to_configure( + setup_core: Path, +) -> None: + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, saved=["lwip"], reconfigure_rcs=(0, 3)) + assert rc == 3 + assert ("save", ["lwip"]) not in calls + assert ("build",) not in calls + + +def test_run_compile_cache_hit_skips_discovery(setup_core: Path) -> None: + """A cached list goes straight to the full write; the explicit reconfigure + after it (#18730) still runs.""" + _setup_build(setup_core) + rc, calls = _record_compile_calls(["esp_timer", "lwip"]) + assert rc == 0 + assert calls == [ + ("write_project", False, ["esp_timer", "lwip"]), + ("run_reconfigure",), + ("build",), + ] + + +@contextmanager +def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]: + """Patch everything the cache key derives from onto a temp IDF tree and + yield that tree's path.""" + idf_path = tmp_path / "idf" + (idf_path / "components").mkdir(parents=True, exist_ok=True) + with ( + patch.object(toolchain, "_get_idf_path", return_value=idf_path), + patch.dict(CORE.data, {KEY_ESP32: {KEY_VARIANT: "ESP32"}}), + patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": excluded}), + ): + yield idf_path + + +def test_component_cache_round_trip(setup_core: Path, tmp_path: Path) -> None: + """A saved list is read back until it is dropped.""" + _setup_build(setup_core) + with _cache_env(tmp_path, "fatfs") as idf_path: + for name in ("lwip", "esp_timer"): + (idf_path / "components" / name).mkdir() + assert toolchain.load_cached_builtin_components() is None + toolchain.save_cached_builtin_components(["esp_timer", "lwip"]) + assert toolchain.load_cached_builtin_components() == ["esp_timer", "lwip"] + toolchain._builtin_component_cache_path().unlink() + assert toolchain.load_cached_builtin_components() is None + + +def test_component_cache_misses_on_key_change_or_missing_component( + setup_core: Path, tmp_path: Path +) -> None: + """A different exclusion set uses another entry, an entry naming a + component that no longer exists is ignored, and a custom IDF_PATH is + never cached.""" + _setup_build(setup_core) + with _cache_env(tmp_path, "fatfs") as idf_path: + (idf_path / "components" / "lwip").mkdir() + toolchain.save_cached_builtin_components(["lwip"]) + path = toolchain._builtin_component_cache_path() + assert path.parent == idf_path / ".esphome_component_lists" + assert path.name.startswith("esp32-") + assert toolchain.load_cached_builtin_components() == ["lwip"] + with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}): + assert toolchain.load_cached_builtin_components() is None + with _cache_env(tmp_path, "fatfs;unity"): + assert toolchain.load_cached_builtin_components() is None + with _cache_env(tmp_path, "fatfs") as idf_path: + path.write_text(json.dumps(["lwip", "gone"])) + assert toolchain.load_cached_builtin_components() is None + # A plain file with the right name is not a component directory. + (idf_path / "components" / "gone").write_text("not a directory") + assert toolchain.load_cached_builtin_components() is None + + +def test_component_cache_save_skips_empty_list_or_custom_idf_path( + setup_core: Path, tmp_path: Path +) -> None: + _setup_build(setup_core) + with _cache_env(tmp_path, "") as idf_path: + toolchain.save_cached_builtin_components([]) + with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}): + toolchain.save_cached_builtin_components(["lwip"]) + assert not (idf_path / ".esphome_component_lists").exists() + + +def test_component_cache_write_failure_is_logged( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _setup_build(setup_core) + with ( + _cache_env(tmp_path, ""), + patch.object(toolchain, "write_file", side_effect=EsphomeError("disk full")), + ): + toolchain.save_cached_builtin_components(["lwip"]) + assert toolchain.load_cached_builtin_components() is None + assert "Could not write component list cache" in caplog.text + + +def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) -> None: + _setup_build(setup_core) + with _cache_env(tmp_path, ""): + path = toolchain._builtin_component_cache_path() + path.parent.mkdir(parents=True) + path.write_text("{not json") + assert toolchain.load_cached_builtin_components() is None + path.write_text(json.dumps({"components": ["lwip"]})) + assert toolchain.load_cached_builtin_components() is None + + def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None: """compile_process_limit is forwarded to run_idf_py as the job limit.""" _setup_build(setup_core) From 688e833413323fe0fad5a4c06af60a72681e917a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 12:38:20 -0500 Subject: [PATCH 35/65] [core] Consolidate HTTP download paths (#18742) --- .../components/dashboard_import/__init__.py | 19 +- esphome/components/nrf52/framework.py | 59 ++--- esphome/espidf/framework.py | 23 +- esphome/external_files.py | 14 +- esphome/framework_helpers.py | 220 ++++++---------- esphome/happy_eyeballs.py | 18 +- esphome/net_retry.py | 39 ++- esphome/platformio/library.py | 29 ++- tests/unit_tests/test_dashboard_import.py | 55 ++++ tests/unit_tests/test_espidf_framework.py | 19 +- tests/unit_tests/test_framework_helpers.py | 246 +++++++----------- tests/unit_tests/test_happy_eyeballs.py | 37 +++ tests/unit_tests/test_net_retry.py | 43 ++- tests/unit_tests/test_nrf52_framework.py | 6 +- tests/unit_tests/test_platformio_library.py | 31 ++- 15 files changed, 469 insertions(+), 389 deletions(-) diff --git a/esphome/components/dashboard_import/__init__.py b/esphome/components/dashboard_import/__init__.py index c27669d77e..906e9762bf 100644 --- a/esphome/components/dashboard_import/__init__.py +++ b/esphome/components/dashboard_import/__init__.py @@ -4,7 +4,6 @@ import re import secrets from typing import Any -import requests from ruamel.yaml import YAML from esphome import git @@ -13,7 +12,7 @@ from esphome.components.packages import validate_source_shorthand import esphome.config_validation as cv from esphome.const import CONF_ESPHOME, CONF_PROJECT, CONF_REF, CONF_WIFI import esphome.final_validate as fv -from esphome.happy_eyeballs import ensure_happy_eyeballs +from esphome.net_retry import fetch_with_retry, http_request from esphome.types import ConfigType from esphome.yaml_util import dump @@ -111,14 +110,20 @@ def import_config( if git_file.query and "full_config" in git_file.query: url = git_file.raw_url - try: - ensure_happy_eyeballs() - req = requests.get(url, timeout=30) + + # Deferred so config-time imports of this component stay light; + # http_request does the lazy import for the request itself. + import requests + + def _fetch() -> str: + req = http_request("GET", url, timeout=30) req.raise_for_status() + return req.text + + try: + contents = fetch_with_retry(url, _fetch, what="Import") except requests.exceptions.RequestException as e: raise ValueError(f"Error while fetching {url}: {e}") from e - - contents = req.text yaml = YAML() loaded_yaml = yaml.load(contents) if ( diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index d487820440..e24569e322 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -5,7 +5,6 @@ from pathlib import Path import platform import shutil import sys -import tempfile import platformdirs @@ -13,9 +12,8 @@ import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError from esphome.framework_helpers import ( - archive_extract_all, create_venv, - download_from_mirrors, + download_and_extract, get_python_env_executable_path, rmdir, run_command_ok, @@ -346,34 +344,37 @@ def check_and_install() -> None: if not sentinel.exists(): rmdir(toolchains_dir, msg=f"Clean up {TOOLCHAIN_VERSION} toolchain environment") sysname, machine, extension = _get_toolchain_platform_info() - with tempfile.NamedTemporaryFile() as tmp: - _LOGGER.info("Downloading Zephyr SDK %s minimal ...", TOOLCHAIN_VERSION) - download_from_mirrors( - SDK_NG_MINIMAL_MIRRORS, - { - "VERSION": TOOLCHAIN_VERSION, - "sysname": sysname, - "machine": machine, - "extension": extension, - }, - tmp.file, - ) - archive_extract_all(tmp.file, toolchains_dir, progress_header="Extracting") - with tempfile.NamedTemporaryFile() as tmp: - _LOGGER.info("Downloading %s toolchain ...", TOOLCHAIN_VERSION) - download_from_mirrors( + substitutions = { + "VERSION": TOOLCHAIN_VERSION, + "sysname": sysname, + "machine": machine, + "extension": extension, + } + # Downloaded next to the destination (not a temp file) so an + # interrupted download's .part file resumes on the next run. + for mirrors, extract_dir, what, slug in ( + (SDK_NG_MINIMAL_MIRRORS, toolchains_dir, "Zephyr SDK minimal", "minimal"), + ( SDK_NG_TOOLCHAIN_MIRRORS, - { - "VERSION": TOOLCHAIN_VERSION, - "sysname": sysname, - "machine": machine, - "extension": extension, - }, - tmp.file, - ) - archive_extract_all( - tmp.file, toolchains_dir / "arm-zephyr-eabi", + "toolchain", + "toolchain", + ), + ): + _LOGGER.info("Downloading %s %s ...", TOOLCHAIN_VERSION, what) + download_and_extract( + mirrors, + substitutions, + toolchains_dir.with_name(f"{toolchains_dir.name}.{slug}.archive"), + extract_dir, progress_header="Extracting", ) + # Best-effort prune of resume leftovers, including a previous + # TOOLCHAIN_VERSION's orphans; the SDK archives are hundreds of MB. + # A locked file must not discard the just-completed install. + for leftover in toolchains_dir.parent.glob("*.archive.part*"): + try: + leftover.unlink() + except OSError as err: + _LOGGER.debug("Could not remove %s: %s", leftover, err) sentinel.touch() diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 179346e072..c2e1e00830 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -17,8 +17,8 @@ import platformdirs from esphome.core import CORE, Version from esphome.framework_helpers import ( PathType, - archive_extract_all, create_venv, + download_and_extract, download_from_mirrors, download_with_resume, failure_reason, @@ -909,20 +909,13 @@ def _check_esphome_idf_framework_install( # a temp file) so an interrupted download resumes on the next # run; the cache is pruned after a successful install anyway. tarball_path = get_idf_tools_path() / "dist" / f"esp-idf-{version}.tar.xz" - download_from_mirrors(mirrors, substitutions, tarball_path) - - _LOGGER.info("Extracting ESP-IDF %s framework ...", version) - try: - with tarball_path.open("rb") as tarball: - archive_extract_all( - tarball, framework_path, progress_header="Extracting" - ) - finally: - # Success: drop the archive rather than caching ~70MB twice. - # Failure: a corrupt archive (e.g. torn by an unclean - # shutdown) must not be reused — without a checksum only a - # failed extraction can expose it, so force a re-download. - tarball_path.unlink(missing_ok=True) + download_and_extract( + mirrors, + substitutions, + tarball_path, + framework_path, + progress_header="Extracting", + ) extracted_marker.touch() # Idempotent post-extract patch: written every invocation so a build diff --git a/esphome/external_files.py b/esphome/external_files.py index 58be4a7c26..ff3b5baf7a 100644 --- a/esphome/external_files.py +++ b/esphome/external_files.py @@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor import contextlib from dataclasses import dataclass, field from datetime import UTC, datetime +from functools import partial import hashlib import logging import os @@ -14,9 +15,8 @@ import time import esphome.config_validation as cv from esphome.const import CONF_FILE, CONF_TYPE, CONF_URL, __version__ from esphome.core import CORE, EsphomeError, TimePeriodSeconds -from esphome.happy_eyeballs import ensure_happy_eyeballs from esphome.helpers import write_file -from esphome.net_retry import fetch_with_retry +from esphome.net_retry import fetch_with_retry, http_request from esphome.types import ConfigType _LOGGER = logging.getLogger(__name__) @@ -143,7 +143,6 @@ def has_remote_file_changed( # Deferred so configs with no remote files skip the heavy import. import requests - ensure_happy_eyeballs() if local_file_path.exists(): _LOGGER.debug("has_remote_file_changed: File exists at %s", local_file_path) try: @@ -165,9 +164,7 @@ def has_remote_file_changed( # the GET's own retry. response = fetch_with_retry( url, - lambda: requests.head( - url, headers=headers, timeout=timeout, allow_redirects=True - ), + partial(http_request, "HEAD", url, headers=headers, timeout=timeout), what="Revalidation", ) @@ -282,7 +279,6 @@ def download_content( ) from failure.cause # The file appeared since the failure; revalidate normally. del run_data.failed_paths[path] - ensure_happy_eyeballs() if CORE.skip_external_update and path.exists(): _LOGGER.debug("Skipping update for %s (refresh disabled)", url) run_data.unchecked_paths.add(path) @@ -304,7 +300,8 @@ def download_content( _LOGGER.debug("Saving to %s", path) def _fetch() -> tuple[requests.Response, bytes]: - req = requests.get( + req = http_request( + "GET", url, timeout=timeout, headers={"User-agent": f"ESPHome/{__version__} (https://esphome.io)"}, @@ -371,7 +368,6 @@ def download_content_many( unique = list(seen.values()) if not unique: return - ensure_happy_eyeballs() _LOGGER.info("Checking %d %s for updates", len(unique), description) def _download_one(file: RemoteFile) -> None: diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 2a2ce6dacf..031db85a65 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -15,9 +15,12 @@ import threading import time from typing import IO, TYPE_CHECKING -from esphome.happy_eyeballs import ensure_happy_eyeballs from esphome.helpers import ProgressBar, rmtree -from esphome.net_retry import NETWORK_MAX_ATTEMPTS, is_transient_download_error +from esphome.net_retry import ( + NETWORK_MAX_ATTEMPTS, + http_request, + is_transient_download_error, +) if TYPE_CHECKING: import requests @@ -600,12 +603,10 @@ def _open_ranged( Raises on connect errors and HTTP error statuses; the response is closed on failure. """ - import requests - headers = {"Range": f"bytes={offset}-"} if offset else {} if offset and validator: headers["If-Range"] = validator - resp = requests.get(url, stream=True, timeout=timeout, headers=headers) + resp = http_request("GET", url, stream=True, timeout=timeout, headers=headers) if offset and resp.status_code == 416: resp.close() return None, offset @@ -941,8 +942,6 @@ def download_with_resume( from esphome.core import EsphomeError - ensure_happy_eyeballs() - dest = Path(dest) part = _part_path(dest) meta = part.with_name(part.name + ".meta") @@ -1078,20 +1077,9 @@ def failure_reason(e: BaseException) -> str: return str(e).split(" for url: ", maxsplit=1)[0] or repr(e) -def _spent_attempts_error(e: Exception, attempts: int) -> Exception: - """Wrap a failure whose mirror already consumed download attempts, so - the sweep classifies it as permanent.""" - from esphome.core import EsphomeError - - err = EsphomeError(f"failed after {attempts} attempts: {failure_reason(e)}") - err.__cause__ = e - return err - - def _try_mirrors_once( urls: list[str], - path_target: Path | None, - f: IO[bytes] | None, + path_target: Path, timeout: int, failures: list[tuple[str, Exception]], progress: Callable[[int], None] | None = None, @@ -1110,109 +1098,73 @@ def _try_mirrors_once( for url in urls: _LOGGER.debug("Trying to download from %s", url) - # Path targets delegate to download_with_resume so a partial - # download persists (and resumes) across esphome runs. - if path_target is not None: - try: - download_with_resume( - url, - path_target, - attempts=_MIRROR_ATTEMPTS, - timeout=timeout, - # Pre-body failures (connect/HTTP errors) fall to the - # next mirror immediately; only mid-stream drops - # retry-with-resume on the same URL. - retry_connect_errors=False, - progress=progress, - ) - return url - except (requests.RequestException, OSError, EsphomeError) as e: - # Everything download_with_resume classifies as a download - # failure; programming errors propagate. - _LOGGER.debug("Failed to download %s: %s", url, str(e)) - failures.append((url, e)) - continue - - # File-like targets download here; mid-stream failures retry the - # same mirror with resume (see download_with_resume) instead of - # starting over. There is no checksum to verify a resumed file - # against, so a stitch is only trusted when the server proves - # consistency: the If-Range validator guarantees 206 only for - # unchanged content, and the expected total length (when the first - # response carried one) guards against short or shifted bodies. - # Without a validator the retry restarts from zero. - offset = 0 - expected_total = 0 - validator = None - for attempt in range(_MIRROR_ATTEMPTS): - try: - resp, offset = _open_ranged(url, offset, timeout, validator) - except (requests.RequestException, OSError) as e: - # Connect/HTTP error, no bytes flowed — next mirror. Wrap - # when earlier attempts were already spent on this mirror. - _LOGGER.debug("Failed to download %s: %s", url, str(e)) - failures.append( - (url, _spent_attempts_error(e, attempt + 1) if attempt else e) - ) - break - - try: - # A None response means HTTP 416: the file already holds - # every byte the server has (a drop after the last byte); - # only the length check below remains. - if resp is not None: - with resp: - if offset == 0: - validator = _response_validator(resp) - expected_total = _content_length(resp) - _stream_response_to_file(resp, f, offset, progress=progress) - - if expected_total and f.tell() != expected_total: - raise EsphomeError( - f"size mismatch: expected {expected_total}, got {f.tell()}" - ) - if not expected_total: - # Same trust decision as download_with_resume's - # unverifiable promotion; surface it at the same level. - _LOGGER.debug( - "Downloaded %s without any way to verify completeness", - url, - ) - - _LOGGER.debug("Downloaded successfully from: %s", url) - - # Reset file pointer and return - f.seek(0) - return url - - except (requests.RequestException, OSError, EsphomeError) as e: - # Mid-stream drop: keep the received bytes and retry this - # mirror from the current position — but only when the - # server gave a validator to resume against safely AND a - # total length to prove the stitched file complete (the - # length check above is the only verification here). - _LOGGER.debug("Failed to download %s: %s", url, str(e)) - if validator and expected_total: - offset = f.tell() - else: - _LOGGER.debug( - "Restarting %s from zero: cannot prove a " - "resumed file complete (validator=%s, total=%s)", - url, - validator is not None, - expected_total, - ) - offset = 0 - if attempt == _MIRROR_ATTEMPTS - 1: - failures.append((url, _spent_attempts_error(e, _MIRROR_ATTEMPTS))) + # Delegate to download_with_resume so a partial download persists + # (and resumes) across esphome runs. + try: + download_with_resume( + url, + path_target, + attempts=_MIRROR_ATTEMPTS, + timeout=timeout, + # Pre-body failures (connect/HTTP errors) fall to the + # next mirror immediately; only mid-stream drops + # retry-with-resume on the same URL. + retry_connect_errors=False, + progress=progress, + ) + return url + except (requests.RequestException, OSError, EsphomeError) as e: + # Everything download_with_resume classifies as a download + # failure; programming errors propagate. + _LOGGER.debug("Failed to download %s: %s", url, str(e)) + failures.append((url, e)) return None +def download_and_extract( + mirrors: list[str], + substitutions: dict[str, str], + archive_path: PathType, + extract_dir: PathType, + timeout: int = 30, + progress_header: str | None = None, + progress: Callable[[int], None] | None = None, +) -> str: + """Download an archive from ``mirrors`` to ``archive_path``, extract it + into ``extract_dir``, and delete the archive. + + The archive should live next to its destination (not in a temp dir) so + an interrupted download's ``.part`` file resumes on the next run. The + archive is deleted whether extraction succeeds or fails: a + complete-but-corrupt file (e.g. torn by an unclean shutdown) must not + poison the next run, and without a checksum only a failed extraction + can expose it. + + Returns the source URL the download came from. + """ + archive_path = Path(archive_path) + url = download_from_mirrors( + mirrors, substitutions, archive_path, timeout=timeout, progress=progress + ) + try: + archive_extract_all(archive_path, extract_dir, progress_header=progress_header) + finally: + # Best-effort: an AV handle on the just-written archive (Windows) + # must not replace the real extraction error or fail a successful + # extraction. A surviving archive is harmless; download_with_resume + # re-verifies or re-downloads it next run. + try: + archive_path.unlink(missing_ok=True) + except OSError as err: + _LOGGER.debug("Could not remove archive %s: %s", archive_path, err) + return url + + def download_from_mirrors( mirrors: list[str], substitutions: dict[str, str], - target: io.RawIOBase | IO[bytes] | PathType, + target: PathType, timeout: int = 30, progress: Callable[[int], None] | None = None, ) -> str: @@ -1222,7 +1174,7 @@ def download_from_mirrors( Args: mirrors: list of mirror URLs substitutions: Dictionary of substitutions to apply to URLs - target: Target file path or file-like object + target: Target file path timeout: Download timeout in seconds progress: Passed through to the download (see ``download_with_resume``); replaces the built-in per-file bar @@ -1234,9 +1186,8 @@ def download_from_mirrors( ``substitutions`` are skipped, so callers can offer templates that only apply to some downloads. - A path target downloads through ``download_with_resume``, so an - interrupted download resumes on the next esphome run; a file-like target - only resumes mid-stream drops within this call. + The target downloads through ``download_with_resume``, so an + interrupted download resumes on the next esphome run. When every mirror fails and at least one failure is transient (dropped connection, timeout, HTTP 429/5xx), the whole list is retried with a @@ -1250,21 +1201,11 @@ def download_from_mirrors( """ from esphome.core import EsphomeError - ensure_happy_eyeballs() + if not isinstance(target, (str, os.PathLike)): + raise TypeError(f"target must be a str or Path: {type(target)}") + path_target = Path(target) - # 1. Classify the target: filesystem path or open file object - path_target: Path | None = None - f: IO[bytes] | None = None - if isinstance(target, (str, os.PathLike)): - path_target = Path(target) - elif isinstance(target, (io.RawIOBase, io.IOBase)): - f = target - else: - raise TypeError( - f"target must be str, Path, or file-like object: {type(target)}" - ) - - # 2. Resolve the mirror templates (invariant across retry sweeps) + # 1. Resolve the mirror templates (invariant across retry sweeps) urls: list[str] = [] skipped: list[tuple[str, str]] = [] for mirror in mirrors: @@ -1283,7 +1224,7 @@ def download_from_mirrors( _LOGGER.warning("Skipping malformed mirror URL template %s: %r", mirror, e) skipped.append((mirror, f"skipped ({e!r})")) - # 3. Sweep the mirror list, retrying transient failures with backoff: + # 2. Sweep the mirror list, retrying transient failures with backoff: # a single pass keeps mirror failover fast, re-sweeping keeps one # network blip from failing the build when only one mirror applies. failures: list[tuple[str, Exception]] = [] @@ -1291,7 +1232,7 @@ def download_from_mirrors( sweep_failures: list[tuple[str, Exception]] = [] if ( url := _try_mirrors_once( - urls, path_target, f, timeout, sweep_failures, progress + urls, path_target, timeout, sweep_failures, progress ) ) is not None: return url @@ -1318,14 +1259,11 @@ def download_from_mirrors( # steady during the backoff instead of rewinding to zero done = 0 if progress is not None: - if f is not None: - done = f.tell() - else: - part = _part_path(path_target) - done = part.stat().st_size if part.is_file() else 0 + part = _part_path(path_target) + done = part.stat().st_size if part.is_file() else 0 _cancellable_sleep(delay, progress, done) - # 4. Report every attempted URL if all mirrors failed. failures spans + # 3. Report every attempted URL if all mirrors failed. failures spans # all sweeps (deduplicated by URL and reason), so neither an early # mirror's failure nor an earlier sweep's failure mode is hidden. if failures: diff --git a/esphome/happy_eyeballs.py b/esphome/happy_eyeballs.py index ebfb94f1f9..35092e7daa 100644 --- a/esphome/happy_eyeballs.py +++ b/esphome/happy_eyeballs.py @@ -12,6 +12,7 @@ from __future__ import annotations import logging import socket +import threading from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -27,20 +28,27 @@ HAPPY_EYEBALLS_DELAY = 0.25 _THREAD_WAIT_BUFFER = 5.0 +# Serialises the check-then-patch so concurrent first calls (download worker +# threads fanning out) build the replacement exactly once. +_PATCH_LOCK = threading.Lock() + + def ensure_happy_eyeballs() -> None: """Make urllib3 (and therefore requests) connect with Happy Eyeballs. - Idempotent; call before performing requests-based downloads. + Idempotent and thread-safe; call before performing requests-based + downloads. """ stock: Callable[..., socket.socket] | None = None try: import urllib3.util.connection - stock = urllib3.util.connection.create_connection - if getattr(stock, "_esphome_patched", False): - return + with _PATCH_LOCK: + stock = urllib3.util.connection.create_connection + if getattr(stock, "_esphome_patched", False): + return - urllib3.util.connection.create_connection = _make_create_connection() + urllib3.util.connection.create_connection = _make_create_connection() except (ImportError, AttributeError) as err: # urllib3 internals moved # WARNING: degraded mode brings back the stalls this module prevents. _LOGGER.warning( diff --git a/esphome/net_retry.py b/esphome/net_retry.py index f7e6e601ea..b91b333114 100644 --- a/esphome/net_retry.py +++ b/esphome/net_retry.py @@ -1,4 +1,4 @@ -"""Retry policy for HTTP downloads. +"""Retry policy and raw HTTP entry point for downloads. Kept import-light on purpose: this module is imported at config time, so it must not pull in requests (a heavy import, ~85ms) at module scope. @@ -9,6 +9,12 @@ from __future__ import annotations from collections.abc import Callable import logging import time +from typing import TYPE_CHECKING, Literal + +from esphome.happy_eyeballs import ensure_happy_eyeballs + +if TYPE_CHECKING: + import requests _LOGGER = logging.getLogger(__name__) @@ -112,3 +118,34 @@ def fetch_with_retry[T](url: str, fetch: Callable[[], T], what: str = "Download" ) time.sleep(delay) return fetch() + + +def http_request( + method: Literal["GET", "HEAD"], + url: str, + *, + timeout: float | tuple[float, float], + stream: bool = False, + headers: dict[str, str] | None = None, + allow_redirects: bool = True, +) -> requests.Response: + """Perform one HTTP request with the Happy Eyeballs patch in place. + + Every ESPHome file download funnels through here so the urllib3 patch + and the lazy requests import live in exactly one place. Status handling, + retries and streaming stay with the caller. The web server OTA and log + clients bypass this on purpose: they iterate already-resolved device + addresses themselves, so the patch buys them nothing. + """ + import requests + + ensure_happy_eyeballs() + # Dispatched through requests.get/head/... (not requests.request) so + # tests patching those entry points keep working. + return getattr(requests, method.lower())( + url, + timeout=timeout, + stream=stream, + headers=headers or {}, + allow_redirects=allow_redirects, + ) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 50e1408b13..1792647d6b 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -24,7 +24,6 @@ import logging import os from pathlib import Path, PurePosixPath import re -import tempfile from typing import Any from urllib.parse import urlsplit, urlunsplit from urllib.request import url2pathname @@ -32,8 +31,7 @@ from urllib.request import url2pathname from esphome import git from esphome.core import CORE, EsphomeError, Library from esphome.framework_helpers import ( - archive_extract_all, - download_from_mirrors, + download_and_extract, failure_reason, rmdir, run_batch_downloads, @@ -147,18 +145,21 @@ class URLSource(Source): if not extracted_marker.is_file() or force: rmdir(path, msg=f"Clean up library directory {path}") - # Download in temporary file - with tempfile.NamedTemporaryFile() as tmp: - if progress is None: - # A batch caller draws one combined bar and logs the list - _LOGGER.info("Downloading %s ...", self.url) - _LOGGER.debug("Location: %s", path) + if progress is None: + # A batch caller draws one combined bar and logs the list + _LOGGER.info("Downloading %s ...", self.url) + _LOGGER.debug("Location: %s", path) - download_from_mirrors([self.url], {}, tmp.file, progress=progress) - - _LOGGER.debug("Extracting archive to %s ...", path) - archive_extract_all(tmp.file, path) - extracted_marker.touch() + # The sibling archive path lets an interrupted download's .part + # file survive and resume on the next esphome run. + download_and_extract( + [self.url], + {}, + path.with_name(f"{path.name}.archive"), + path, + progress=progress, + ) + extracted_marker.touch() return path def __str__(self): diff --git a/tests/unit_tests/test_dashboard_import.py b/tests/unit_tests/test_dashboard_import.py index 427bee0f86..46a2fa5db0 100644 --- a/tests/unit_tests/test_dashboard_import.py +++ b/tests/unit_tests/test_dashboard_import.py @@ -10,8 +10,10 @@ during the adoption flow and depend on the output's ``esphome.name`` from __future__ import annotations from pathlib import Path +from unittest.mock import MagicMock, patch import pytest +import requests as req import yaml as pyyaml from esphome.components.dashboard_import import import_config @@ -201,3 +203,56 @@ def test_import_refuses_to_overwrite_existing_yaml(tmp_path: Path) -> None: ) # Original content survives unchanged. assert yaml_path.read_text() == "# user's hand-edited config\n" + + +def _full_config_kwargs(yaml_path: Path) -> dict: + return { + "path": str(yaml_path), + "name": "kitchen", + "friendly_name": None, + "project_name": "acme.kitchen-light", + "import_url": "github://acme/firmware/kitchen.yaml@main?full_config", + } + + +def test_full_config_import_fetches_and_writes_contents(tmp_path: Path) -> None: + yaml_path = tmp_path / "kitchen.yaml" + resp = MagicMock(text="esphome:\n name: orig\n") + with patch( + "esphome.components.dashboard_import.http_request", return_value=resp + ) as mock_req: + import_config(**_full_config_kwargs(yaml_path)) + assert yaml_path.read_text() == "esphome:\n name: orig\n" + assert mock_req.call_args[0][0] == "GET" + + +def test_full_config_import_retries_transient_errors(tmp_path: Path) -> None: + """The fetch goes through the shared retry policy: a transient network + error is retried instead of failing the adoption immediately.""" + yaml_path = tmp_path / "kitchen.yaml" + resp = MagicMock(text="esphome:\n name: orig\n") + with ( + patch( + "esphome.components.dashboard_import.http_request", + side_effect=[req.ConnectionError("reset"), resp], + ), + patch("esphome.net_retry.time.sleep") as mock_sleep, + ): + import_config(**_full_config_kwargs(yaml_path)) + assert yaml_path.exists() + mock_sleep.assert_called_once_with(2) + + +def test_full_config_import_wraps_permanent_errors_in_value_error( + tmp_path: Path, +) -> None: + """device-builder depends on the ValueError contract for fetch failures.""" + resp = MagicMock() + resp.raise_for_status.side_effect = req.HTTPError( + "404", response=MagicMock(status_code=404) + ) + with ( + patch("esphome.components.dashboard_import.http_request", return_value=resp), + pytest.raises(ValueError, match="Error while fetching"), + ): + import_config(**_full_config_kwargs(tmp_path / "kitchen.yaml")) diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index 6288933a6a..1bef743f4c 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -371,10 +371,9 @@ def _fake_download_from_mirrors( ) -> str: """Stand-in for download_from_mirrors that creates path targets, since the framework code opens the downloaded tarball afterwards.""" - if isinstance(target, (str, os.PathLike)): - path = Path(target) - path.parent.mkdir(parents=True, exist_ok=True) - path.touch() + path = Path(target) + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() return "https://example.com/idf.tar.xz" @@ -384,13 +383,15 @@ def espidf_mocks(setup_core: Path): # archive_extract_all is mocked, so pre-create the framework dir that the # extracted-marker touch writes into. _get_framework_path(_IDF_VERSION).mkdir(parents=True, exist_ok=True) + # One mock covers the tarball (via framework_helpers.download_and_extract) + # and the constraints file (espidf-bound download_from_mirrors), so call + # counts and ordering assertions span the two. + download = MagicMock(side_effect=_fake_download_from_mirrors) with ( patch("esphome.espidf.framework.rmdir") as rmdir_mock, - patch( - "esphome.espidf.framework.download_from_mirrors", - side_effect=_fake_download_from_mirrors, - ) as download, - patch("esphome.espidf.framework.archive_extract_all") as extract, + patch("esphome.framework_helpers.download_from_mirrors", download), + patch("esphome.espidf.framework.download_from_mirrors", download), + patch("esphome.framework_helpers.archive_extract_all") as extract, patch("esphome.espidf.framework.create_venv") as venv, patch("esphome.espidf.framework.run_command_ok", return_value=True) as run_ok, patch( diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 5916a2fd60..f001bd6c37 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2,6 +2,7 @@ # pylint: disable=protected-access +import gzip import hashlib import importlib.util import io @@ -31,6 +32,7 @@ from esphome.framework_helpers import ( _zip_extract_all, archive_extract_all, create_venv, + download_and_extract, download_from_mirrors, download_with_resume, get_project_compile_flags, @@ -1339,20 +1341,20 @@ class TestDownloadFromMirrors: assert url == "https://example.com/f" assert target.read_bytes() == b"filedata" - def test_file_object_target_reports_progress(self) -> None: - """The library prefetch's production path: a file-object target - streams through the mirror fallback and ticks the tracker.""" - buf = io.BytesIO() + def test_progress_callback_reports_bytes(self, tmp_path: Path) -> None: + """The library prefetch's production path: the mirror download ticks + the caller's tracker instead of drawing its own bar.""" + target = tmp_path / "f.bin" ticks: list[int] = [] with patch( "requests.get", return_value=_mock_response(b"filedata"), ): url = download_from_mirrors( - ["https://example.com/f"], {}, buf, progress=ticks.append + ["https://example.com/f"], {}, target, progress=ticks.append ) assert url == "https://example.com/f" - assert buf.getvalue() == b"filedata" + assert target.read_bytes() == b"filedata" assert ticks and ticks[-1] == len(b"filedata") def test_substitutions_applied_to_url(self, tmp_path: Path) -> None: @@ -1460,8 +1462,8 @@ class TestDownloadFromMirrors: ei.value ) - def test_falls_back_to_second_mirror(self) -> None: - buf = io.BytesIO() + def test_falls_back_to_second_mirror(self, tmp_path: Path) -> None: + target = tmp_path / "f.bin" with patch( "requests.get", side_effect=[_mock_response(b"", ok=False), _mock_response(b"second")], @@ -1469,18 +1471,18 @@ class TestDownloadFromMirrors: url = download_from_mirrors( ["https://mirror1.com/f", "https://mirror2.com/f"], {}, - buf, + target, ) assert url == "https://mirror2.com/f" - assert buf.getvalue() == b"second" + assert target.read_bytes() == b"second" - def test_mid_stream_drop_resumes_same_mirror(self) -> None: + def test_mid_stream_drop_resumes_same_mirror(self, tmp_path: Path) -> None: """A mid-stream failure retries the same mirror with Range and If-Range headers, keeping the bytes already received, before falling to the next.""" first = _interrupted_response(b"1234", etag='"v1"') first.headers = {**first.headers, "content-length": "8"} - buf = io.BytesIO() + target = tmp_path / "f.bin" with patch( "requests.get", side_effect=[first, _resumed_response(b"5678")], @@ -1488,10 +1490,10 @@ class TestDownloadFromMirrors: url = download_from_mirrors( ["https://mirror1.com/f", "https://mirror2.com/f"], {}, - buf, + target, ) assert url == "https://mirror1.com/f" - assert buf.getvalue() == b"12345678" + assert target.read_bytes() == b"12345678" assert mock_get.call_count == 2 assert mock_get.call_args_list[1][0][0] == "https://mirror1.com/f" # the resume is conditional on the content being unchanged @@ -1500,48 +1502,6 @@ class TestDownloadFromMirrors: "If-Range": '"v1"', } - def test_mid_stream_drop_without_validator_restarts(self) -> None: - """A server offering no ETag/Last-Modified cannot be resumed safely; - the retry restarts from zero instead of stitching unverified bytes.""" - buf = io.BytesIO() - with patch( - "requests.get", - side_effect=[_interrupted_response(b"1234"), _mock_response(b"full")], - ) as mock_get: - download_from_mirrors(["https://mirror1.com/f"], {}, buf) - assert buf.getvalue() == b"full" - assert "Range" not in mock_get.call_args_list[1][1]["headers"] - - def test_drop_after_last_byte_recovers_via_416(self) -> None: - """A connection drop after the final body byte leaves a complete file; - the retry's 416 answer plus the length check turn it into success - instead of a wasted refetch.""" - first = _interrupted_response(b"1234", etag='"v1"') - first.headers = {**first.headers, "content-length": "4"} - r416 = _mock_response(b"", ok=False) - r416.status_code = 416 - buf = io.BytesIO() - with patch("requests.get", side_effect=[first, r416]) as mock_get: - url = download_from_mirrors(["https://mirror1.com/f"], {}, buf) - assert url == "https://mirror1.com/f" - assert buf.getvalue() == b"1234" - assert mock_get.call_count == 2 - - def test_mirror_drop_without_length_restarts(self) -> None: - """With no content-length there is no way to prove a stitched file - complete, so the retry restarts even though a validator exists.""" - buf = io.BytesIO() - with patch( - "requests.get", - side_effect=[ - _interrupted_response(b"1234", etag='"v1"'), - _mock_response(b"full"), - ], - ) as mock_get: - download_from_mirrors(["https://mirror1.com/f"], {}, buf) - assert buf.getvalue() == b"full" - assert "Range" not in mock_get.call_args_list[1][1]["headers"] - def test_path_target_resumes_across_runs(self, tmp_path: Path) -> None: """A path target routes through download_with_resume: a part file and metadata from a previous run resume instead of restarting.""" @@ -1573,32 +1533,14 @@ class TestDownloadFromMirrors: assert url == "https://mirror2.com/f" assert dest.read_bytes() == b"data" - def test_resumed_short_body_fails_length_check(self) -> None: - """A stitched file whose final length disagrees with the advertised - total is rejected instead of reported as success.""" - first = _interrupted_response(b"1234", etag='"v1"') - first.headers = {**first.headers, "content-length": "8"} - # the resume ends early (5 of 8 bytes); the poisoned part is then - # discarded and the fresh retry also delivers a short body - short_resume = _resumed_response(b"5") - short_fresh = _mock_response(b"56") - short_fresh.headers = {**short_fresh.headers, "content-length": "8"} - buf = io.BytesIO() - with ( - patch("requests.get", side_effect=[first, short_resume, short_fresh]), - pytest.raises(EsphomeError, match="all mirrors"), - ): - download_from_mirrors(["https://mirror1.com/f"], {}, buf) - - def test_failed_mirror_leftovers_not_kept_for_next_mirror(self) -> None: - """Bytes from a mirror that failed all attempts must not leak into the - next mirror's download (no bogus Range request, fresh content).""" - exhausted = [_interrupted_response(b"AAAA", etag='"a1"')] - for _ in range(2): - r = _interrupted_response(b"BB") - r.status_code = 206 - exhausted.append(r) - buf = io.BytesIO() + def test_failed_mirror_leftovers_not_resumed_on_next_mirror( + self, tmp_path: Path + ) -> None: + """A part file left by a mirror that failed all attempts must not be + stitched onto the next mirror's download (its meta names the other + URL, so the retry restarts from zero without a Range request).""" + exhausted = [_interrupted_response(b"AAAA") for _ in range(3)] + target = tmp_path / "f.bin" with patch( "requests.get", side_effect=exhausted + [_mock_response(b"clean")], @@ -1606,15 +1548,17 @@ class TestDownloadFromMirrors: url = download_from_mirrors( ["https://mirror1.com/f", "https://mirror2.com/f"], {}, - buf, + target, ) assert url == "https://mirror2.com/f" - assert buf.getvalue() == b"clean" + assert target.read_bytes() == b"clean" # the second mirror starts fresh, without a Range header assert mock_get.call_args_list[3][0][0] == "https://mirror2.com/f" assert "Range" not in mock_get.call_args_list[3][1]["headers"] - def test_all_mirrors_fail_raises_error_listing_every_attempt(self) -> None: + def test_all_mirrors_fail_raises_error_listing_every_attempt( + self, tmp_path: Path + ) -> None: with ( patch( "requests.get", @@ -1625,7 +1569,7 @@ class TestDownloadFromMirrors: download_from_mirrors( ["https://mirror1.com/f", "https://mirror2.com/f"], {}, - io.BytesIO(), + tmp_path / "out.bin", ) # Every attempted URL appears in the message, and the first mirror's # exception (the primary URL, usually the one that matters) is chained. @@ -1641,16 +1585,6 @@ class TestDownloadFromMirrors: with pytest.raises(TypeError, match="target must be"): download_from_mirrors(["https://example.com/f"], {}, 42) # type: ignore[arg-type] - def test_file_like_target_written(self) -> None: - buf = io.BytesIO() - with patch( - "requests.get", - return_value=_mock_response(b"bytes"), - ): - download_from_mirrors(["https://example.com/f"], {}, buf) - buf.seek(0) - assert buf.read() == b"bytes" - def test_progress_bar_shown_when_content_length_known(self, tmp_path: Path) -> None: r = _mock_response(b"1234567890") r.headers = {"content-length": "10"} @@ -1676,13 +1610,10 @@ class TestDownloadFromMirrors: assert target.exists() assert target.read_bytes() == b"" - @pytest.mark.parametrize("target_kind", ["path", "file-like"]) - def test_transient_failure_retries_mirror_sweep( - self, tmp_path: Path, target_kind: str - ) -> None: + def test_transient_failure_retries_mirror_sweep(self, tmp_path: Path) -> None: """A transient connect error on the only applicable mirror retries the whole mirror list with backoff instead of failing the build.""" - target = tmp_path / "idf.tar.xz" if target_kind == "path" else io.BytesIO() + target = tmp_path / "idf.tar.xz" with ( patch( "requests.get", @@ -1695,33 +1626,10 @@ class TestDownloadFromMirrors: ): url = download_from_mirrors(["https://mirror1.com/f"], {}, target) assert url == "https://mirror1.com/f" - data = target.read_bytes() if target_kind == "path" else target.getvalue() - assert data == b"data" + assert target.read_bytes() == b"data" assert mock_get.call_count == 2 mock_sleep.assert_called_once_with(2) - def test_backoff_tick_reports_filelike_bytes(self) -> None: - """For a file-like target the backoff tick carries f.tell(), so the - combined bar holds steady through the sweep retry.""" - target = io.BytesIO() - ticks: list[int] = [] - with ( - patch( - "requests.get", - side_effect=[ - req.ConnectionError("down"), - _mock_response(b"data"), - ], - ), - patch("esphome.framework_helpers._cancellable_sleep") as mock_sleep, - ): - download_from_mirrors( - ["https://mirror1.com/f"], {}, target, progress=ticks.append - ) - # No bytes had streamed at backoff time, so the tick carries 0 - assert mock_sleep.call_args == call(2, ticks.append, 0) - assert target.getvalue() == b"data" - def test_backoff_tick_reports_partial_bytes(self, tmp_path: Path) -> None: """The backoff tick carries the bytes already in the part file, so a combined bar holds steady instead of rewinding to zero.""" @@ -1831,41 +1739,83 @@ class TestDownloadFromMirrors: assert isinstance(ei.value.__cause__, req.ConnectionError) mock_sleep.assert_called_once_with(2) - def test_exhausted_mid_stream_attempts_not_swept(self) -> None: - """A file-like mirror that spent all its mid-stream attempts is not - retried again at the sweep level (unlike a path target, it has no - part file to resume from on a later sweep).""" - buf = io.BytesIO() + def test_exhausted_mid_stream_attempts_not_swept(self, tmp_path: Path) -> None: + """A mirror that spent all its mid-stream attempts fails permanently + instead of re-arming the sweep, and its part file survives so the + next esphome run resumes it.""" with ( patch( "requests.get", side_effect=[_interrupted_response(b"1234") for _ in range(3)], ) as mock_get, patch("esphome.framework_helpers.time.sleep") as mock_sleep, - pytest.raises(EsphomeError, match="failed after 3 attempts"), + pytest.raises(EsphomeError, match="after 3 attempts"), ): - download_from_mirrors(["https://mirror1.com/f"], {}, buf) + download_from_mirrors(["https://mirror1.com/f"], {}, tmp_path / "out.bin") assert mock_get.call_count == 3 mock_sleep.assert_not_called() + assert (tmp_path / "out.bin.part").exists() + + +class TestDownloadAndExtract: + def test_downloads_extracts_and_deletes_archive(self, tmp_path: Path) -> None: + content = gzip.compress( + _make_tar([_reg("file.txt")], {"file.txt": b"data"}).getvalue() + ) + dest = tmp_path / "out" + with patch("requests.get", return_value=_mock_response(content)): + url = download_and_extract( + ["https://example.com/lib.tar.gz"], + {}, + tmp_path / "lib.archive", + dest, + ) + assert url == "https://example.com/lib.tar.gz" + assert (dest / "file.txt").read_bytes() == b"data" + # the archive is consumed; only the extraction remains + assert not (tmp_path / "lib.archive").exists() + + def test_locked_archive_does_not_mask_result(self, tmp_path: Path) -> None: + """A cleanup unlink blocked by e.g. an AV handle (Windows) must not + replace the extraction result; the archive simply survives.""" + content = gzip.compress( + _make_tar([_reg("file.txt")], {"file.txt": b"data"}).getvalue() + ) + real_unlink = Path.unlink + + def locked_unlink(self: Path, missing_ok: bool = False) -> None: + if self.name.endswith(".archive"): + raise PermissionError("held by antivirus") + real_unlink(self, missing_ok=missing_ok) - def test_mid_stream_drop_then_connect_error_not_swept(self) -> None: - """A connect error on a later attempt (after a mid-stream drop spent - one) also counts as spent budget and does not re-arm the sweep.""" - buf = io.BytesIO() with ( - patch( - "requests.get", - side_effect=[ - _interrupted_response(b"1234"), - req.ConnectionError("down"), - ], - ) as mock_get, - patch("esphome.framework_helpers.time.sleep") as mock_sleep, - pytest.raises(EsphomeError, match="failed after 2 attempts"), + patch("requests.get", return_value=_mock_response(content)), + patch("pathlib.Path.unlink", locked_unlink), ): - download_from_mirrors(["https://mirror1.com/f"], {}, buf) - assert mock_get.call_count == 2 - mock_sleep.assert_not_called() + url = download_and_extract( + ["https://example.com/lib.tar.gz"], + {}, + tmp_path / "lib.archive", + tmp_path / "out", + ) + assert url == "https://example.com/lib.tar.gz" + assert (tmp_path / "out" / "file.txt").read_bytes() == b"data" + assert (tmp_path / "lib.archive").exists() # left behind, harmless + + def test_corrupt_archive_deleted_on_extract_failure(self, tmp_path: Path) -> None: + """A complete-but-corrupt archive must not survive to poison the next + run; without a checksum only a failed extraction can expose it.""" + with ( + patch("requests.get", return_value=_mock_response(b"not an archive")), + pytest.raises(ValueError, match="Unsupported archive format"), + ): + download_and_extract( + ["https://example.com/lib.tar.gz"], + {}, + tmp_path / "lib.archive", + tmp_path / "out", + ) + assert not (tmp_path / "lib.archive").exists() def test_importing_framework_helpers_does_not_import_requests() -> None: diff --git a/tests/unit_tests/test_happy_eyeballs.py b/tests/unit_tests/test_happy_eyeballs.py index 3335a8a3e3..ccb7aa3c67 100644 --- a/tests/unit_tests/test_happy_eyeballs.py +++ b/tests/unit_tests/test_happy_eyeballs.py @@ -4,7 +4,9 @@ from __future__ import annotations import asyncio from collections.abc import Generator +from concurrent.futures import ThreadPoolExecutor import socket +import threading from typing import Any from unittest.mock import Mock, patch @@ -61,6 +63,41 @@ def test_ensure_happy_eyeballs_patches_and_is_idempotent( assert urllib3.util.connection.create_connection is patched +def test_ensure_happy_eyeballs_concurrent_first_calls_patch_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Worker threads fanning out (download_content_many, run_batch_downloads) + may race the first call; the replacement is built exactly once.""" + import urllib3.util.connection + + from esphome import happy_eyeballs + + def stock(*args: Any, **kwargs: Any) -> None: + pass + + monkeypatch.setattr(urllib3.util.connection, "create_connection", stock) + + barrier = threading.Barrier(8) + builds: list[int] = [] + real_make = happy_eyeballs._make_create_connection + + def counting_make() -> Any: + builds.append(1) + return real_make() + + monkeypatch.setattr(happy_eyeballs, "_make_create_connection", counting_make) + + def racer() -> None: + barrier.wait(timeout=10) + ensure_happy_eyeballs() + + with ThreadPoolExecutor(max_workers=8) as ex: + list(ex.map(lambda _: racer(), range(8))) + + assert builds == [1] + assert urllib3.util.connection.create_connection._esphome_patched + + def test_connects_and_restores_socket_state( create_connection: Any, listener: tuple[str, int], mock_gai: Any ) -> None: diff --git a/tests/unit_tests/test_net_retry.py b/tests/unit_tests/test_net_retry.py index c22bda5ee5..6c2a1ee05f 100644 --- a/tests/unit_tests/test_net_retry.py +++ b/tests/unit_tests/test_net_retry.py @@ -7,7 +7,11 @@ import pytest import requests as req from esphome.core import EsphomeError -from esphome.net_retry import fetch_with_retry, is_transient_download_error +from esphome.net_retry import ( + fetch_with_retry, + http_request, + is_transient_download_error, +) def _http_error(status: int) -> req.HTTPError: @@ -141,3 +145,40 @@ class TestFetchWithRetry: assert mock_sleep.call_args_list == [call(2), call(4)] assert "(attempt 2/3)" in caplog.text assert "(attempt 3/3)" in caplog.text + + +class TestHttpRequest: + def test_applies_happy_eyeballs_and_forwards_arguments(self) -> None: + with ( + patch("esphome.net_retry.ensure_happy_eyeballs") as mock_he, + patch("requests.get", return_value=MagicMock()) as mock_get, + ): + resp = http_request( + "GET", + "https://example.com/f", + timeout=30, + stream=True, + headers={"Range": "bytes=4-"}, + ) + mock_he.assert_called_once_with() + assert resp is mock_get.return_value + assert mock_get.call_args == call( + "https://example.com/f", + timeout=30, + stream=True, + headers={"Range": "bytes=4-"}, + allow_redirects=True, + ) + + def test_dispatches_head_through_requests_head(self) -> None: + """Dispatch goes through requests.get/head so tests patching those + entry points keep working.""" + with patch("requests.head", return_value=MagicMock()) as mock_head: + http_request("HEAD", "https://example.com/f", timeout=(5, 30)) + assert mock_head.call_args[1]["timeout"] == (5, 30) + + def test_no_status_handling(self) -> None: + """Error statuses are the caller's problem; nothing raises here.""" + resp = MagicMock(status_code=404) + with patch("requests.get", return_value=resp): + assert http_request("GET", "https://example.com/f", timeout=1) is resp diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index c2ee0c2a75..7b83a1edc7 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -104,11 +104,13 @@ def mock_nrf52_ops(): patch( "esphome.components.nrf52.framework.run_command_ok", return_value=True ) as mock_run_cmd, + # download_and_extract resolves its internals in framework_helpers, + # so the download/extract seams are patched there. patch( - "esphome.components.nrf52.framework.download_from_mirrors", + "esphome.framework_helpers.download_from_mirrors", return_value="https://example.com/tc.tar.xz", ) as mock_download, - patch("esphome.components.nrf52.framework.archive_extract_all") as mock_extract, + patch("esphome.framework_helpers.archive_extract_all") as mock_extract, ): yield SimpleNamespace( rmdir=mock_rmdir, diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index ef24f99953..0c873dc3fe 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -158,16 +158,12 @@ def test_urlsource_download_extracts_then_reuses_marker( ): monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None) dl_calls: list[list[str]] = [] - monkeypatch.setattr( - lib, - "download_from_mirrors", - lambda urls, headers, f, progress=None: dl_calls.append(urls), - ) - def fake_extract(fileobj, path): - Path(path).mkdir(parents=True, exist_ok=True) + def fake_download_and_extract(urls, subs, archive_path, extract_dir, **kwargs): + dl_calls.append(urls) + Path(extract_dir).mkdir(parents=True, exist_ok=True) - monkeypatch.setattr(lib, "archive_extract_all", fake_extract) + monkeypatch.setattr(lib, "download_and_extract", fake_download_and_extract) src = URLSource("http://example.test/lib.tar.gz") out = src.download("mylib") @@ -187,6 +183,25 @@ def test_urlsource_download_extracts_then_reuses_marker( assert "Downloading" not in caplog.text +def test_urlsource_downloads_to_sibling_archive_path(setup_core, monkeypatch): + """The archive downloads to a deterministic path next to the cache dir + (not a random temp file), so an interrupted download's .part file + resumes on the next run.""" + monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None) + targets: list[Path] = [] + + def fake_download_and_extract(urls, subs, archive_path, extract_dir, **kwargs): + targets.append(Path(archive_path)) + Path(extract_dir).mkdir(parents=True, exist_ok=True) + + monkeypatch.setattr(lib, "download_and_extract", fake_download_and_extract) + + src = URLSource("http://example.test/lib.tar.gz") + out = src.download("mylib") + + assert targets == [out.with_name(f"{out.name}.archive")] + + def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): registry = lib._make_registry_client() monkeypatch.setattr( From 8749c3d18d29ba1c63b7790e0f3f7a783d7c9c84 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 12:40:57 -0500 Subject: [PATCH 36/65] [esp32_ble_tracker] Warn when the scan window is above 600ms with wifi (#18725) --- .../components/esp32_ble_tracker/__init__.py | 47 +++++++++++++++++ .../test_scan_window_default.py | 52 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index c6e34f37ca..906144e5fd 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -143,6 +143,13 @@ def validate_max_connections_deprecated(config: ConfigType) -> ConfigType: # BLE uses the airtime wifi does not claim. IDF_SCAN_WINDOW_FIX_VERSION = cv.Version(5, 5, 5) +# Above this the scanner holds the shared radio long enough that wifi drops +# packets and connections on some access points (others cope fine, which is +# why this is a warning and not an error); old proxy configs with 1100 ms +# windows are a recurring cause of instability (esphome/esphome#18655). Only +# wifi shares the radio; long windows are fine on ethernet builds. +MAX_RECOMMENDED_WIFI_SCAN_WINDOW = TimePeriod(milliseconds=600) + @dataclass class TrackerData: @@ -209,6 +216,45 @@ def _raise_defaulted_scan_window(config: ConfigType) -> ConfigType: return config +def _warn_long_scan_window_with_wifi(config: ConfigType) -> ConfigType: + """Warn when the scan window is long enough to starve wifi. + + Runs after _raise_defaulted_scan_window so it sees the final window. + software_coexistence is only present when wifi is configured, so ethernet + builds never warn: BLE has the radio to itself there. Presence is what + matters, not the value; with the arbiter disabled a long window starves + wifi outright. + """ + params = config[CONF_SCAN_PARAMETERS] + window = params[CONF_WINDOW] + if CONF_SOFTWARE_COEXISTENCE not in config: + return config + if window <= MAX_RECOMMENDED_WIFI_SCAN_WINDOW: + return config + if _get_data().scan_window_defaulted: + # The window was raised to match the interval, so point at the key the + # user actually set. + _LOGGER.warning( + "BLE scan interval of %s sets the scan window to the same value, " + "which starves wifi on the same radio and can cause wifi disconnects " + "depending on the access point; keep the interval at or below %s " + "(for example interval: 320ms). Long windows are only a problem with " + "wifi, they are fine on ethernet", + params[CONF_INTERVAL], + MAX_RECOMMENDED_WIFI_SCAN_WINDOW, + ) + return config + _LOGGER.warning( + "BLE scan window of %s with wifi on the same radio starves wifi and " + "can cause wifi disconnects depending on the access point; keep the " + "window at or below %s (for example interval: 320ms, window: 300ms). " + "Long windows are only a problem with wifi, they are fine on ethernet", + window, + MAX_RECOMMENDED_WIFI_SCAN_WINDOW, + ) + return config + + # 320 ms is the ESP-IDF reference scan interval; the shared schema also # tightens validation to the controller's 2.5 ms .. 10240 ms range and rejects # window/interval pairs that collapse to the same 0.625 ms unit count. @@ -271,6 +317,7 @@ CONFIG_SCHEMA = cv.All( ).extend(cv.COMPONENT_SCHEMA), validate_max_connections_deprecated, _raise_defaulted_scan_window, + _warn_long_scan_window_with_wifi, ) diff --git a/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py b/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py index 8612ac6732..1381aaf4c2 100644 --- a/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py +++ b/tests/component_tests/esp32_ble_tracker/test_scan_window_default.py @@ -12,6 +12,7 @@ arbiter a full-duty scan would starve wifi, so the 30 ms default is kept. from __future__ import annotations from collections.abc import Callable +import logging from pathlib import Path import pytest @@ -221,3 +222,54 @@ def test_connection_scan_window_codegen( assert window_call in main_cpp assert ("set_connection_scan_window(48)" in main_cpp) == connection_call assert ("'connection_scan_window' has no effect" in caplog.text) == warns + + +@pytest.mark.parametrize( + ("wifi", "params", "expect_warning"), + [ + (True, {"interval": "1100ms", "window": "1100ms"}, True), + (True, {"interval": "1100ms", "window": "601ms"}, True), + (True, {"interval": "1100ms", "window": "600ms"}, False), + (False, {"interval": "1100ms", "window": "1100ms"}, False), + ], +) +def test_long_window_with_wifi_warns( + stage_esp32: Callable[..., None], + caplog: pytest.LogCaptureFixture, + wifi: bool, + params: ConfigType, + expect_warning: bool, +) -> None: + """A scan window above 600 ms warns only when wifi shares the radio.""" + stage_esp32("5.5.5", wifi=wifi) + with caplog.at_level(logging.WARNING): + _scan_params({"scan_parameters": params}) + assert ("starves wifi" in caplog.text) is expect_warning + + +def test_long_window_warns_with_coexistence_disabled( + stage_esp32: Callable[..., None], + caplog: pytest.LogCaptureFixture, +) -> None: + """Disabling the arbiter is the worst case for a long window, so it still warns.""" + stage_esp32("5.5.5", wifi=True) + with caplog.at_level(logging.WARNING): + _scan_params( + { + CONF_SOFTWARE_COEXISTENCE: False, + "scan_parameters": {"interval": "1100ms", "window": "1100ms"}, + } + ) + assert "BLE scan window of 1100ms" in caplog.text + + +def test_raised_window_warning_points_at_interval( + stage_esp32: Callable[..., None], + caplog: pytest.LogCaptureFixture, +) -> None: + """When the window was raised to a long interval, the warning names the interval.""" + stage_esp32("5.5.5", wifi=True) + with caplog.at_level(logging.WARNING): + _scan_params({"scan_parameters": {"interval": "1s"}}) + assert "BLE scan interval of 1s" in caplog.text + assert "BLE scan window of" not in caplog.text From 145373ed134c1a40c8d9e9f53d6d3cff44cda99c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 13:04:41 -0500 Subject: [PATCH 37/65] [api] Widen message type storage to uint16_t (#18526) --- esphome/components/api/api_connection.cpp | 16 +- esphome/components/api/api_connection.h | 37 +-- esphome/components/api/api_frame_helper.h | 18 +- .../components/api/api_frame_helper_noise.cpp | 4 +- .../components/api/api_frame_helper_noise.h | 4 +- .../api/api_frame_helper_plaintext.cpp | 24 +- .../api/api_frame_helper_plaintext.h | 5 +- esphome/components/api/api_pb2.h | 284 +++++++++--------- esphome/components/api/proto.h | 5 - script/api_protobuf/api_protobuf.py | 29 +- .../api/test_api_protobuf_generator.py | 18 +- 11 files changed, 238 insertions(+), 206 deletions(-) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 05abbf0b75..9b1026d2a9 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1790,10 +1790,12 @@ void APIConnection::complete_authentication_() { bool APIConnection::send_hello_response_(const HelloRequest &msg) { // Copy client name with truncation if needed (set_client_name handles truncation) this->helper_->set_client_name(msg.client_info.c_str(), msg.client_info.size()); - this->client_api_version_major_ = msg.api_version_major; - this->client_api_version_minor_ = msg.api_version_minor; + this->client_api_version_major_ = + static_cast(std::min(msg.api_version_major, std::numeric_limits::max())); + this->client_api_version_minor_ = + static_cast(std::min(msg.api_version_minor, std::numeric_limits::max())); char peername[socket::SOCKADDR_STR_LEN]; - ESP_LOGV(TAG, "Hello from client: '%s' | %s | API Version %" PRIu16 ".%" PRIu16, this->helper_->get_client_name(), + ESP_LOGV(TAG, "Hello from client: '%s' | %s | API Version %u.%u", this->helper_->get_client_name(), this->helper_->get_peername_to(peername), this->client_api_version_major_, this->client_api_version_minor_); HelloResponse resp; @@ -2224,7 +2226,7 @@ bool APIConnection::try_to_clear_buffer_slow_(bool log_out_of_space) { } return false; } -bool APIConnection::send_message_(uint32_t payload_size, uint8_t message_type, MessageEncodeFn encode_fn, +bool APIConnection::send_message_(uint32_t payload_size, uint16_t message_type, MessageEncodeFn encode_fn, const void *msg) { #ifdef HAS_PROTO_MESSAGE_DUMP // Skip dump for log messages (recursive logging risk) and camera frames (high-frequency noise) @@ -2253,7 +2255,7 @@ uint16_t APIConnection::encode_to_buffer_slow(uint32_t calculated_size, MessageE APIConnection *conn, uint32_t remaining_size) { return encode_to_buffer(calculated_size, encode_fn, msg, conn, remaining_size); } -bool APIConnection::send_buffer(ProtoWriteBuffer buffer, uint8_t message_type) { +bool APIConnection::send_buffer(ProtoWriteBuffer buffer, uint16_t message_type) { const bool is_log_message = (message_type == SubscribeLogsResponse::MESSAGE_TYPE); if (!this->try_to_clear_buffer(!is_log_message)) { @@ -2283,12 +2285,12 @@ void APIConnection::on_fatal_error() { this->flags_.remove = true; } -bool APIConnection::schedule_message_front_(EntityBase *entity, uint8_t message_type, uint8_t estimated_size) { +bool APIConnection::schedule_message_front_(EntityBase *entity, uint16_t message_type, uint8_t estimated_size) { this->deferred_batch_.add_item_front(entity, message_type, estimated_size); return this->schedule_batch_(); } -bool APIConnection::send_message_smart_(EntityBase *entity, uint8_t message_type, uint8_t estimated_size, +bool APIConnection::send_message_smart_(EntityBase *entity, uint16_t message_type, uint8_t estimated_size, uint8_t aux_data_index) { if (this->should_send_immediately_(message_type) && this->helper_->can_write_without_blocking()) { auto &shared_buf = this->parent_->get_shared_buffer_ref(); diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 1b47c23cfe..5a554f4857 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -326,8 +326,10 @@ class APIConnection final : public APIServerConnectionBase { bool is_marked_for_removal() const { return this->flags_.remove; } uint8_t get_log_subscription_level() const { return this->flags_.log_subscription; } - // Get client API version for feature detection - bool client_supports_api_version(uint16_t major, uint16_t minor) const { + // Get client API version for feature detection. + // Stored versions saturate at 255 (see send_hello_response_), so requesting + // a minimum above that can never match. + bool client_supports_api_version(uint8_t major, uint8_t minor) const { return this->client_api_version_major_ > major || (this->client_api_version_major_ == major && this->client_api_version_minor_ >= minor); } @@ -374,7 +376,7 @@ class APIConnection final : public APIServerConnectionBase { return true; return this->try_to_clear_buffer_slow_(log_out_of_space); } - bool send_buffer(ProtoWriteBuffer buffer, uint8_t message_type); + bool send_buffer(ProtoWriteBuffer buffer, uint16_t message_type); const char *get_name() const { return this->helper_->get_client_name(); } /// Get peer name (IP address) into caller-provided buffer, returns buf for convenience @@ -423,7 +425,7 @@ class APIConnection final : public APIServerConnectionBase { } // Non-template buffer management for send_message - bool send_message_(uint32_t payload_size, uint8_t message_type, MessageEncodeFn encode_fn, const void *msg); + bool send_message_(uint32_t payload_size, uint16_t message_type, MessageEncodeFn encode_fn, const void *msg); // Core batch encoding logic. ALWAYS_INLINE so encode_fn devirtualizes at hot call sites. // Defined in api_connection_buffer.h (needs APIServer complete). @@ -664,10 +666,9 @@ class APIConnection final : public APIServerConnectionBase { struct BatchItem { EntityBase *entity; // 4 bytes - Entity pointer - uint8_t message_type; // 1 byte - Message type for protocol and dispatch + uint16_t message_type; // 2 bytes - Message type for protocol and dispatch uint8_t estimated_size; // 1 byte - Estimated message size (max 255 bytes) uint8_t aux_data_index{AUX_DATA_UNUSED}; // 1 byte - For events: index into entity's event_types - // 1 byte padding }; std::vector items; @@ -677,7 +678,7 @@ class APIConnection final : public APIServerConnectionBase { // connections that do, buffers are released after initial sync anyway // Add item to the batch (with deduplication) - void add_item(EntityBase *entity, uint8_t message_type, uint8_t estimated_size, + void add_item(EntityBase *entity, uint16_t message_type, uint8_t estimated_size, uint8_t aux_data_index = AUX_DATA_UNUSED) { // Dedup: O(n) scan but optimized for RAM over performance // Skip deduplication for events - they are edge-triggered, every occurrence matters @@ -693,7 +694,7 @@ class APIConnection final : public APIServerConnectionBase { this->items.push_back({entity, message_type, estimated_size, aux_data_index}); } // Add item to the front of the batch (for high priority messages like ping) - void add_item_front(EntityBase *entity, uint8_t message_type, uint8_t estimated_size) { + void add_item_front(EntityBase *entity, uint16_t message_type, uint8_t estimated_size) { // Swap to front avoids expensive vector::insert which shifts all elements this->items.push_back({entity, message_type, estimated_size, AUX_DATA_UNUSED}); if (this->items.size() > 1) { @@ -758,13 +759,15 @@ class APIConnection final : public APIServerConnectionBase { #endif } flags_{}; // 2 bytes total - // 2-byte types immediately after flags_ (no padding between them) - uint16_t client_api_version_major_{0}; - uint16_t client_api_version_minor_{0}; + // 2-byte type immediately after flags_ (no padding between them) + uint16_t batch_message_type_{0}; // Current message type during batch encoding // 1-byte types to fill remaining space before next 4-byte boundary + // Client API versions are clamped to 255 on receive (see send_hello_response_) + uint8_t client_api_version_major_{0}; + uint8_t client_api_version_minor_{0}; ActiveIterator active_iterator_{ActiveIterator::NONE}; - uint8_t batch_message_type_{0}; // Current message type during batch encoding - // Total: 2 (flags) + 2 + 2 + 1 + 1 = 8 bytes, aligned to 4-byte boundary + // Total: 2 (flags) + 2 + 1 + 1 + 1 + 1 (batch_header_size_ below) = 8 bytes, + // aligned to 4-byte boundary // Actual header size used by encode_to_buffer for the current message. // Read by process_batch_multi_ to pass into MessageInfo. @@ -813,7 +816,7 @@ class APIConnection final : public APIServerConnectionBase { // 2. It's an EventResponse (events are edge-triggered - every occurrence matters) // 3. OR: User has opted into immediate sending (should_try_send_immediately = true // AND batch_delay = 0) - inline bool should_send_immediately_(uint8_t message_type) const { + inline bool should_send_immediately_(uint16_t message_type) const { return ( #ifdef USE_UPDATE message_type == UpdateStateResponse::MESSAGE_TYPE || @@ -827,11 +830,11 @@ class APIConnection final : public APIServerConnectionBase { // Helper method to send a message either immediately or via batching // Tries immediate send if should_send_immediately_() returns true and buffer has space // Falls back to batching if immediate send fails or isn't applicable - bool send_message_smart_(EntityBase *entity, uint8_t message_type, uint8_t estimated_size, + bool send_message_smart_(EntityBase *entity, uint16_t message_type, uint8_t estimated_size, uint8_t aux_data_index = DeferredBatch::AUX_DATA_UNUSED); // Helper function to schedule a deferred message with known message type - bool schedule_message_(EntityBase *entity, uint8_t message_type, uint8_t estimated_size, + bool schedule_message_(EntityBase *entity, uint16_t message_type, uint8_t estimated_size, uint8_t aux_data_index = DeferredBatch::AUX_DATA_UNUSED) { this->deferred_batch_.add_item(entity, message_type, estimated_size, aux_data_index); return this->schedule_batch_(); @@ -839,7 +842,7 @@ class APIConnection final : public APIServerConnectionBase { // Helper function to schedule a high priority message at the front of the batch // Out-of-line: callers (on_shutdown, check_keepalive_) are cold paths - bool schedule_message_front_(EntityBase *entity, uint8_t message_type, uint8_t estimated_size); + bool schedule_message_front_(EntityBase *entity, uint16_t message_type, uint8_t estimated_size); // Helper function to log client messages with name and peername void log_client_(int level, const LogString *message); diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h index 1c60bb87a5..ff8aa7834c 100644 --- a/esphome/components/api/api_frame_helper.h +++ b/esphome/components/api/api_frame_helper.h @@ -49,16 +49,16 @@ struct ReadPacketBuffer { }; // Packed message info structure to minimize memory usage -// Note: message_type is uint8_t — all current protobuf message types fit in 8 bits. -// The noise wire format encodes types as 16-bit, but the high byte is always 0. -// If message types ever exceed 255, this and encrypt_noise_message_ must be updated. +// message_type matches the wire formats: noise carries a fixed 16-bit type +// field, plaintext a type varint. The proto codegen caps message IDs at 16383 +// so the plaintext type varint fits the 2 bytes budgeted in HEADER_PADDING. struct MessageInfo { uint16_t offset; // Offset in buffer where message starts uint16_t payload_size; // Size of the message payload - uint8_t message_type; // Message type (0-255) + uint16_t message_type; // Message type (0-16383) uint8_t header_size; // Actual header size used (avoids recomputation in write path) - MessageInfo(uint8_t type, uint16_t off, uint16_t size, uint8_t hdr) + MessageInfo(uint16_t type, uint16_t off, uint16_t size, uint8_t hdr) : offset(off), payload_size(size), message_type(type), header_size(hdr) {} }; @@ -173,7 +173,7 @@ class APIFrameHelper { } // Write a single protobuf message - the hot path (87-100% of all writes). // Caller must ensure state is DATA before calling. - virtual APIError write_protobuf_packet(uint8_t type, ProtoWriteBuffer buffer) = 0; + virtual APIError write_protobuf_packet(uint16_t type, ProtoWriteBuffer buffer) = 0; // Write multiple protobuf messages in a single batched operation. // Caller must ensure state is DATA and messages is not empty. // messages contains (message_type, offset, length) for each message in the buffer. @@ -187,15 +187,15 @@ class APIFrameHelper { // Distinguishes protocols via frame_footer_size_ (noise always has a non-zero MAC // footer, plaintext has footer=0). If a protocol with a plaintext footer is ever // added, this should become a virtual method. - uint8_t frame_header_size(uint16_t payload_size, uint8_t message_type) const { + uint8_t frame_header_size(uint16_t payload_size, uint16_t message_type) const { #if defined(USE_API_NOISE) && defined(USE_API_PLAINTEXT) return this->frame_footer_size_ ? this->frame_header_padding_ - : static_cast(1 + ProtoSize::varint16(payload_size) + ProtoSize::varint8(message_type)); + : static_cast(1 + ProtoSize::varint16(payload_size) + ProtoSize::varint16(message_type)); #elif defined(USE_API_NOISE) return this->frame_header_padding_; #else // USE_API_PLAINTEXT only - return static_cast(1 + ProtoSize::varint16(payload_size) + ProtoSize::varint8(message_type)); + return static_cast(1 + ProtoSize::varint16(payload_size) + ProtoSize::varint16(message_type)); #endif } // Get the frame footer size required by this protocol diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index d7554e62c5..9c4cc2aa78 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -442,7 +442,7 @@ APIError APINoiseFrameHelper::read_packet(ReadPacketBuffer *buffer) { } // Encrypt a single noise message in place and return the encrypted frame length. // Returns APIError::OK on success. -APIError APINoiseFrameHelper::encrypt_noise_message_(uint8_t *buf_start, uint16_t payload_size, uint8_t message_type, +APIError APINoiseFrameHelper::encrypt_noise_message_(uint8_t *buf_start, uint16_t payload_size, uint16_t message_type, uint16_t &encrypted_len_out) { // The noise frame header is written after encryption, when the size is known @@ -472,7 +472,7 @@ APIError APINoiseFrameHelper::encrypt_noise_message_(uint8_t *buf_start, uint16_ return APIError::OK; } -APIError APINoiseFrameHelper::write_protobuf_packet(uint8_t type, ProtoWriteBuffer buffer) { +APIError APINoiseFrameHelper::write_protobuf_packet(uint16_t type, ProtoWriteBuffer buffer) { #ifdef ESPHOME_DEBUG_API assert(this->state_ == State::DATA); #endif diff --git a/esphome/components/api/api_frame_helper_noise.h b/esphome/components/api/api_frame_helper_noise.h index 05060c77de..366751738e 100644 --- a/esphome/components/api/api_frame_helper_noise.h +++ b/esphome/components/api/api_frame_helper_noise.h @@ -31,7 +31,7 @@ class APINoiseFrameHelper final : public APIFrameHelper { #endif APIError loop() override; APIError read_packet(ReadPacketBuffer *buffer) override; - APIError write_protobuf_packet(uint8_t type, ProtoWriteBuffer buffer) override; + APIError write_protobuf_packet(uint16_t type, ProtoWriteBuffer buffer) override; APIError write_protobuf_messages(ProtoWriteBuffer buffer, std::span messages) override; protected: @@ -44,7 +44,7 @@ class APINoiseFrameHelper final : public APIFrameHelper { APIError state_action_handshake_write_(); APIError try_read_frame_(); APIError write_frame_(const uint8_t *data, uint16_t len); - APIError encrypt_noise_message_(uint8_t *buf_start, uint16_t payload_size, uint8_t message_type, + APIError encrypt_noise_message_(uint8_t *buf_start, uint16_t payload_size, uint16_t message_type, uint16_t &encrypted_len_out); APIError init_handshake_(); APIError check_handshake_finished_(); diff --git a/esphome/components/api/api_frame_helper_plaintext.cpp b/esphome/components/api/api_frame_helper_plaintext.cpp index 9359f568fb..09ace7294a 100644 --- a/esphome/components/api/api_frame_helper_plaintext.cpp +++ b/esphome/components/api/api_frame_helper_plaintext.cpp @@ -5,6 +5,7 @@ #include "esphome/core/hal.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include "api_pb2.h" #include "proto.h" #include #include @@ -252,24 +253,21 @@ ESPHOME_ALWAYS_INLINE static inline void encode_varint_16(uint16_t value, uint8_ *p = static_cast(value); } -// Encode an 8-bit varint (1-2 bytes) using pre-computed length. -ESPHOME_ALWAYS_INLINE static inline void encode_varint_8(uint8_t value, uint8_t varint_len, uint8_t *p) { - if (varint_len == 2) { - *p++ = static_cast(value | 0x80); - *p = static_cast(value >> 7); - } else { - *p = value; - } -} +// The generator rejects message IDs above MAX_MESSAGE_TYPE, so the type varint +// can never outgrow the 2 bytes HEADER_PADDING budgets for it. Without this +// bound, write_plaintext_header's header_offset would underflow for the first +// message in a batch and the header write would land outside the buffer. +static_assert(1 + 3 + ProtoSize::varint16(MAX_MESSAGE_TYPE) <= APIPlaintextFrameHelper::HEADER_PADDING, + "HEADER_PADDING cannot fit the type varint of the largest message ID"); // Write plaintext header into pre-allocated padding before payload. // padding_size: bytes reserved before payload (HEADER_PADDING for first/single msg, // actual header size for contiguous batch messages). // Returns the total header length (indicator + varints). ESPHOME_ALWAYS_INLINE static inline uint8_t write_plaintext_header(uint8_t *buf_start, uint16_t payload_size, - uint8_t message_type, uint8_t padding_size) { + uint16_t message_type, uint8_t padding_size) { uint8_t size_varint_len = ProtoSize::varint16(payload_size); - uint8_t type_varint_len = ProtoSize::varint8(message_type); + uint8_t type_varint_len = ProtoSize::varint16(message_type); uint8_t total_header_len = 1 + size_varint_len + type_varint_len; // The header is right-justified within the padding so it sits immediately before payload. @@ -292,12 +290,12 @@ ESPHOME_ALWAYS_INLINE static inline uint8_t write_plaintext_header(uint8_t *buf_ // Encode varints directly into buffer using pre-computed lengths encode_varint_16(payload_size, size_varint_len, buf_start + header_offset + 1); - encode_varint_8(message_type, type_varint_len, buf_start + header_offset + 1 + size_varint_len); + encode_varint_16(message_type, type_varint_len, buf_start + header_offset + 1 + size_varint_len); return total_header_len; } -APIError APIPlaintextFrameHelper::write_protobuf_packet(uint8_t type, ProtoWriteBuffer buffer) { +APIError APIPlaintextFrameHelper::write_protobuf_packet(uint16_t type, ProtoWriteBuffer buffer) { #ifdef ESPHOME_DEBUG_API assert(this->state_ == State::DATA); #endif diff --git a/esphome/components/api/api_frame_helper_plaintext.h b/esphome/components/api/api_frame_helper_plaintext.h index ea3f6d7280..00e7c7b1bc 100644 --- a/esphome/components/api/api_frame_helper_plaintext.h +++ b/esphome/components/api/api_frame_helper_plaintext.h @@ -10,7 +10,8 @@ class APIPlaintextFrameHelper final : public APIFrameHelper { // Plaintext header structure (worst case): // Pos 0: indicator (0x00) // Pos 1-3: payload size varint (up to 3 bytes) - // Pos 4-5: message type varint (up to 2 bytes) + // Pos 4-5: message type varint (up to 2 bytes; covers message IDs up to + // 16383, enforced by the proto codegen) // Pos 6+: actual payload data static constexpr uint8_t HEADER_PADDING = 1 + 3 + 2; // indicator + size varint + type varint @@ -21,7 +22,7 @@ class APIPlaintextFrameHelper final : public APIFrameHelper { APIError init() override; APIError loop() override; APIError read_packet(ReadPacketBuffer *buffer) override; - APIError write_protobuf_packet(uint8_t type, ProtoWriteBuffer buffer) override; + APIError write_protobuf_packet(uint16_t type, ProtoWriteBuffer buffer) override; APIError write_protobuf_messages(ProtoWriteBuffer buffer, std::span messages) override; #ifdef USE_API_NOISE // After try_read_frame_ returned PROTOCOL_SWITCH_TO_NOISE: copy out the diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index cd2f32deaf..48e277fce1 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -9,6 +9,10 @@ namespace esphome::api { +// Upper bound on message IDs, enforced by the code generator: the plaintext +// frame header budgets 2 varint bytes for the type (HEADER_PADDING). +static constexpr uint16_t MAX_MESSAGE_TYPE = 16383; + namespace enums { enum DisconnectReason : uint32_t { @@ -407,7 +411,7 @@ class CommandProtoMessage : public ProtoDecodableMessage { }; class HelloRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 1; + static constexpr uint16_t MESSAGE_TYPE = 1; static constexpr uint8_t ESTIMATED_SIZE = 17; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("hello_request"); } @@ -425,7 +429,7 @@ class HelloRequest final : public ProtoDecodableMessage { }; class HelloResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 2; + static constexpr uint16_t MESSAGE_TYPE = 2; static constexpr uint8_t ESTIMATED_SIZE = 26; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("hello_response"); } @@ -444,7 +448,7 @@ class HelloResponse final : public ProtoMessage { }; class DisconnectRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 5; + static constexpr uint16_t MESSAGE_TYPE = 5; static constexpr uint8_t ESTIMATED_SIZE = 2; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("disconnect_request"); } @@ -461,7 +465,7 @@ class DisconnectRequest final : public ProtoDecodableMessage { }; class DisconnectResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 6; + static constexpr uint16_t MESSAGE_TYPE = 6; static constexpr uint8_t ESTIMATED_SIZE = 0; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("disconnect_response"); } @@ -474,7 +478,7 @@ class DisconnectResponse final : public ProtoMessage { }; class PingRequest final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 7; + static constexpr uint16_t MESSAGE_TYPE = 7; static constexpr uint8_t ESTIMATED_SIZE = 0; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("ping_request"); } @@ -487,7 +491,7 @@ class PingRequest final : public ProtoMessage { }; class PingResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 8; + static constexpr uint16_t MESSAGE_TYPE = 8; static constexpr uint8_t ESTIMATED_SIZE = 0; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("ping_response"); } @@ -544,7 +548,7 @@ class SerialProxyInfo final : public ProtoMessage { #endif class DeviceInfoResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 10; + static constexpr uint16_t MESSAGE_TYPE = 10; static constexpr uint16_t ESTIMATED_SIZE = 312; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("device_info_response"); } @@ -655,7 +659,7 @@ class ZWaveProxyCapabilities final : public ProtoMessage { #endif class DeviceCapabilitiesResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 150; + static constexpr uint16_t MESSAGE_TYPE = 150; static constexpr uint8_t ESTIMATED_SIZE = 102; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("device_capabilities_response"); } @@ -682,7 +686,7 @@ class DeviceCapabilitiesResponse final : public ProtoMessage { }; class ListEntitiesDoneResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 19; + static constexpr uint16_t MESSAGE_TYPE = 19; static constexpr uint8_t ESTIMATED_SIZE = 0; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_done_response"); } @@ -696,7 +700,7 @@ class ListEntitiesDoneResponse final : public ProtoMessage { #ifdef USE_BINARY_SENSOR class ListEntitiesBinarySensorResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 12; + static constexpr uint16_t MESSAGE_TYPE = 12; static constexpr uint8_t ESTIMATED_SIZE = 51; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_binary_sensor_response"); } @@ -713,7 +717,7 @@ class ListEntitiesBinarySensorResponse final : public InfoResponseProtoMessage { }; class BinarySensorStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 21; + static constexpr uint16_t MESSAGE_TYPE = 21; static constexpr uint8_t ESTIMATED_SIZE = 13; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("binary_sensor_state_response"); } @@ -732,7 +736,7 @@ class BinarySensorStateResponse final : public StateResponseProtoMessage { #ifdef USE_COVER class ListEntitiesCoverResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 13; + static constexpr uint16_t MESSAGE_TYPE = 13; static constexpr uint8_t ESTIMATED_SIZE = 57; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_cover_response"); } @@ -752,7 +756,7 @@ class ListEntitiesCoverResponse final : public InfoResponseProtoMessage { }; class CoverStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 22; + static constexpr uint16_t MESSAGE_TYPE = 22; static constexpr uint8_t ESTIMATED_SIZE = 21; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("cover_state_response"); } @@ -770,7 +774,7 @@ class CoverStateResponse final : public StateResponseProtoMessage { }; class CoverCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 30; + static constexpr uint16_t MESSAGE_TYPE = 30; static constexpr uint8_t ESTIMATED_SIZE = 25; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("cover_command_request"); } @@ -792,7 +796,7 @@ class CoverCommandRequest final : public CommandProtoMessage { #ifdef USE_FAN class ListEntitiesFanResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 14; + static constexpr uint16_t MESSAGE_TYPE = 14; static constexpr uint8_t ESTIMATED_SIZE = 68; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_fan_response"); } @@ -812,7 +816,7 @@ class ListEntitiesFanResponse final : public InfoResponseProtoMessage { }; class FanStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 23; + static constexpr uint16_t MESSAGE_TYPE = 23; static constexpr uint8_t ESTIMATED_SIZE = 28; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("fan_state_response"); } @@ -832,7 +836,7 @@ class FanStateResponse final : public StateResponseProtoMessage { }; class FanCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 31; + static constexpr uint16_t MESSAGE_TYPE = 31; static constexpr uint8_t ESTIMATED_SIZE = 38; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("fan_command_request"); } @@ -860,7 +864,7 @@ class FanCommandRequest final : public CommandProtoMessage { #ifdef USE_LIGHT class ListEntitiesLightResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 15; + static constexpr uint16_t MESSAGE_TYPE = 15; static constexpr uint8_t ESTIMATED_SIZE = 73; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_light_response"); } @@ -879,7 +883,7 @@ class ListEntitiesLightResponse final : public InfoResponseProtoMessage { }; class LightStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 24; + static constexpr uint16_t MESSAGE_TYPE = 24; static constexpr uint8_t ESTIMATED_SIZE = 67; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("light_state_response"); } @@ -906,7 +910,7 @@ class LightStateResponse final : public StateResponseProtoMessage { }; class LightCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 32; + static constexpr uint16_t MESSAGE_TYPE = 32; static constexpr uint8_t ESTIMATED_SIZE = 112; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("light_command_request"); } @@ -950,7 +954,7 @@ class LightCommandRequest final : public CommandProtoMessage { #ifdef USE_SENSOR class ListEntitiesSensorResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 16; + static constexpr uint16_t MESSAGE_TYPE = 16; static constexpr uint8_t ESTIMATED_SIZE = 66; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_sensor_response"); } @@ -970,7 +974,7 @@ class ListEntitiesSensorResponse final : public InfoResponseProtoMessage { }; class SensorStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 25; + static constexpr uint16_t MESSAGE_TYPE = 25; static constexpr uint8_t ESTIMATED_SIZE = 16; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("sensor_state_response"); } @@ -989,7 +993,7 @@ class SensorStateResponse final : public StateResponseProtoMessage { #ifdef USE_SWITCH class ListEntitiesSwitchResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 17; + static constexpr uint16_t MESSAGE_TYPE = 17; static constexpr uint8_t ESTIMATED_SIZE = 51; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_switch_response"); } @@ -1006,7 +1010,7 @@ class ListEntitiesSwitchResponse final : public InfoResponseProtoMessage { }; class SwitchStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 26; + static constexpr uint16_t MESSAGE_TYPE = 26; static constexpr uint8_t ESTIMATED_SIZE = 11; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("switch_state_response"); } @@ -1022,7 +1026,7 @@ class SwitchStateResponse final : public StateResponseProtoMessage { }; class SwitchCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 33; + static constexpr uint16_t MESSAGE_TYPE = 33; static constexpr uint8_t ESTIMATED_SIZE = 11; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("switch_command_request"); } @@ -1040,7 +1044,7 @@ class SwitchCommandRequest final : public CommandProtoMessage { #ifdef USE_TEXT_SENSOR class ListEntitiesTextSensorResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 18; + static constexpr uint16_t MESSAGE_TYPE = 18; static constexpr uint8_t ESTIMATED_SIZE = 49; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_text_sensor_response"); } @@ -1056,7 +1060,7 @@ class ListEntitiesTextSensorResponse final : public InfoResponseProtoMessage { }; class TextSensorStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 27; + static constexpr uint16_t MESSAGE_TYPE = 27; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("text_sensor_state_response"); } @@ -1074,7 +1078,7 @@ class TextSensorStateResponse final : public StateResponseProtoMessage { #endif class SubscribeLogsRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 28; + static constexpr uint16_t MESSAGE_TYPE = 28; static constexpr uint8_t ESTIMATED_SIZE = 4; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("subscribe_logs_request"); } @@ -1090,7 +1094,7 @@ class SubscribeLogsRequest final : public ProtoDecodableMessage { }; class SubscribeLogsResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 29; + static constexpr uint16_t MESSAGE_TYPE = 29; static constexpr uint8_t ESTIMATED_SIZE = 21; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("subscribe_logs_response"); } @@ -1113,7 +1117,7 @@ class SubscribeLogsResponse final : public ProtoMessage { #ifdef USE_API_NOISE class NoiseEncryptionSetKeyRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 124; + static constexpr uint16_t MESSAGE_TYPE = 124; static constexpr uint8_t ESTIMATED_SIZE = 19; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("noise_encryption_set_key_request"); } @@ -1129,7 +1133,7 @@ class NoiseEncryptionSetKeyRequest final : public ProtoDecodableMessage { }; class NoiseEncryptionSetKeyResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 125; + static constexpr uint16_t MESSAGE_TYPE = 125; static constexpr uint8_t ESTIMATED_SIZE = 2; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("noise_encryption_set_key_response"); } @@ -1159,7 +1163,7 @@ class HomeassistantServiceMap final : public ProtoMessage { }; class HomeassistantActionRequest final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 35; + static constexpr uint16_t MESSAGE_TYPE = 35; static constexpr uint8_t ESTIMATED_SIZE = 128; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("homeassistant_action_request"); } @@ -1190,7 +1194,7 @@ class HomeassistantActionRequest final : public ProtoMessage { #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES class HomeassistantActionResponse final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 130; + static constexpr uint16_t MESSAGE_TYPE = 130; static constexpr uint8_t ESTIMATED_SIZE = 34; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("homeassistant_action_response"); } @@ -1214,7 +1218,7 @@ class HomeassistantActionResponse final : public ProtoDecodableMessage { #ifdef USE_API_HOMEASSISTANT_STATES class SubscribeHomeAssistantStateResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 39; + static constexpr uint16_t MESSAGE_TYPE = 39; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("subscribe_home_assistant_state_response"); } @@ -1232,7 +1236,7 @@ class SubscribeHomeAssistantStateResponse final : public ProtoMessage { }; class HomeAssistantStateResponse final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 40; + static constexpr uint16_t MESSAGE_TYPE = 40; static constexpr uint8_t ESTIMATED_SIZE = 27; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("home_assistant_state_response"); } @@ -1250,7 +1254,7 @@ class HomeAssistantStateResponse final : public ProtoDecodableMessage { #endif class GetTimeRequest final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 36; + static constexpr uint16_t MESSAGE_TYPE = 36; static constexpr uint8_t ESTIMATED_SIZE = 0; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("get_time_request"); } @@ -1292,7 +1296,7 @@ class ParsedTimezone final : public ProtoDecodableMessage { }; class GetTimeResponse final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 37; + static constexpr uint16_t MESSAGE_TYPE = 37; static constexpr uint8_t ESTIMATED_SIZE = 22; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("get_time_response"); } @@ -1323,7 +1327,7 @@ class ListEntitiesServicesArgument final : public ProtoMessage { }; class ListEntitiesServicesResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 41; + static constexpr uint16_t MESSAGE_TYPE = 41; static constexpr uint8_t ESTIMATED_SIZE = 50; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_services_response"); } @@ -1363,7 +1367,7 @@ class ExecuteServiceArgument final : public ProtoDecodableMessage { }; class ExecuteServiceRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 42; + static constexpr uint16_t MESSAGE_TYPE = 42; static constexpr uint8_t ESTIMATED_SIZE = 45; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("execute_service_request"); } @@ -1390,7 +1394,7 @@ class ExecuteServiceRequest final : public ProtoDecodableMessage { #ifdef USE_API_USER_DEFINED_ACTION_RESPONSES class ExecuteServiceResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 131; + static constexpr uint16_t MESSAGE_TYPE = 131; static constexpr uint8_t ESTIMATED_SIZE = 34; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("execute_service_response"); } @@ -1414,7 +1418,7 @@ class ExecuteServiceResponse final : public ProtoMessage { #ifdef USE_CAMERA class ListEntitiesCameraResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 43; + static constexpr uint16_t MESSAGE_TYPE = 43; static constexpr uint8_t ESTIMATED_SIZE = 40; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_camera_response"); } @@ -1429,7 +1433,7 @@ class ListEntitiesCameraResponse final : public InfoResponseProtoMessage { }; class CameraImageResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 44; + static constexpr uint16_t MESSAGE_TYPE = 44; static constexpr uint8_t ESTIMATED_SIZE = 30; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("camera_image_response"); } @@ -1451,7 +1455,7 @@ class CameraImageResponse final : public StateResponseProtoMessage { }; class CameraImageRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 45; + static constexpr uint16_t MESSAGE_TYPE = 45; static constexpr uint8_t ESTIMATED_SIZE = 4; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("camera_image_request"); } @@ -1469,7 +1473,7 @@ class CameraImageRequest final : public ProtoDecodableMessage { #ifdef USE_CLIMATE class ListEntitiesClimateResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 46; + static constexpr uint16_t MESSAGE_TYPE = 46; static constexpr uint8_t ESTIMATED_SIZE = 153; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_climate_response"); } @@ -1503,7 +1507,7 @@ class ListEntitiesClimateResponse final : public InfoResponseProtoMessage { }; class ClimateStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 47; + static constexpr uint16_t MESSAGE_TYPE = 47; static constexpr uint8_t ESTIMATED_SIZE = 68; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("climate_state_response"); } @@ -1531,7 +1535,7 @@ class ClimateStateResponse final : public StateResponseProtoMessage { }; class ClimateCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 48; + static constexpr uint16_t MESSAGE_TYPE = 48; static constexpr uint8_t ESTIMATED_SIZE = 84; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("climate_command_request"); } @@ -1569,7 +1573,7 @@ class ClimateCommandRequest final : public CommandProtoMessage { #ifdef USE_WATER_HEATER class ListEntitiesWaterHeaterResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 132; + static constexpr uint16_t MESSAGE_TYPE = 132; static constexpr uint8_t ESTIMATED_SIZE = 65; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_water_heater_response"); } @@ -1590,7 +1594,7 @@ class ListEntitiesWaterHeaterResponse final : public InfoResponseProtoMessage { }; class WaterHeaterStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 133; + static constexpr uint16_t MESSAGE_TYPE = 133; static constexpr uint8_t ESTIMATED_SIZE = 35; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("water_heater_state_response"); } @@ -1611,7 +1615,7 @@ class WaterHeaterStateResponse final : public StateResponseProtoMessage { }; class WaterHeaterCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 134; + static constexpr uint16_t MESSAGE_TYPE = 134; static constexpr uint8_t ESTIMATED_SIZE = 34; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("water_heater_command_request"); } @@ -1634,7 +1638,7 @@ class WaterHeaterCommandRequest final : public CommandProtoMessage { #ifdef USE_NUMBER class ListEntitiesNumberResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 49; + static constexpr uint16_t MESSAGE_TYPE = 49; static constexpr uint8_t ESTIMATED_SIZE = 75; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_number_response"); } @@ -1655,7 +1659,7 @@ class ListEntitiesNumberResponse final : public InfoResponseProtoMessage { }; class NumberStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 50; + static constexpr uint16_t MESSAGE_TYPE = 50; static constexpr uint8_t ESTIMATED_SIZE = 16; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("number_state_response"); } @@ -1672,7 +1676,7 @@ class NumberStateResponse final : public StateResponseProtoMessage { }; class NumberCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 51; + static constexpr uint16_t MESSAGE_TYPE = 51; static constexpr uint8_t ESTIMATED_SIZE = 14; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("number_command_request"); } @@ -1690,7 +1694,7 @@ class NumberCommandRequest final : public CommandProtoMessage { #ifdef USE_SELECT class ListEntitiesSelectResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 52; + static constexpr uint16_t MESSAGE_TYPE = 52; static constexpr uint8_t ESTIMATED_SIZE = 58; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_select_response"); } @@ -1706,7 +1710,7 @@ class ListEntitiesSelectResponse final : public InfoResponseProtoMessage { }; class SelectStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 53; + static constexpr uint16_t MESSAGE_TYPE = 53; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("select_state_response"); } @@ -1723,7 +1727,7 @@ class SelectStateResponse final : public StateResponseProtoMessage { }; class SelectCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 54; + static constexpr uint16_t MESSAGE_TYPE = 54; static constexpr uint8_t ESTIMATED_SIZE = 18; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("select_command_request"); } @@ -1742,7 +1746,7 @@ class SelectCommandRequest final : public CommandProtoMessage { #ifdef USE_SIREN class ListEntitiesSirenResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 55; + static constexpr uint16_t MESSAGE_TYPE = 55; static constexpr uint8_t ESTIMATED_SIZE = 62; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_siren_response"); } @@ -1760,7 +1764,7 @@ class ListEntitiesSirenResponse final : public InfoResponseProtoMessage { }; class SirenStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 56; + static constexpr uint16_t MESSAGE_TYPE = 56; static constexpr uint8_t ESTIMATED_SIZE = 11; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("siren_state_response"); } @@ -1776,7 +1780,7 @@ class SirenStateResponse final : public StateResponseProtoMessage { }; class SirenCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 57; + static constexpr uint16_t MESSAGE_TYPE = 57; static constexpr uint8_t ESTIMATED_SIZE = 37; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("siren_command_request"); } @@ -1802,7 +1806,7 @@ class SirenCommandRequest final : public CommandProtoMessage { #ifdef USE_LOCK class ListEntitiesLockResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 58; + static constexpr uint16_t MESSAGE_TYPE = 58; static constexpr uint8_t ESTIMATED_SIZE = 55; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_lock_response"); } @@ -1821,7 +1825,7 @@ class ListEntitiesLockResponse final : public InfoResponseProtoMessage { }; class LockStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 59; + static constexpr uint16_t MESSAGE_TYPE = 59; static constexpr uint8_t ESTIMATED_SIZE = 11; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("lock_state_response"); } @@ -1837,7 +1841,7 @@ class LockStateResponse final : public StateResponseProtoMessage { }; class LockCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 60; + static constexpr uint16_t MESSAGE_TYPE = 60; static constexpr uint8_t ESTIMATED_SIZE = 22; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("lock_command_request"); } @@ -1858,7 +1862,7 @@ class LockCommandRequest final : public CommandProtoMessage { #ifdef USE_BUTTON class ListEntitiesButtonResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 61; + static constexpr uint16_t MESSAGE_TYPE = 61; static constexpr uint8_t ESTIMATED_SIZE = 49; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_button_response"); } @@ -1874,7 +1878,7 @@ class ListEntitiesButtonResponse final : public InfoResponseProtoMessage { }; class ButtonCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 62; + static constexpr uint16_t MESSAGE_TYPE = 62; static constexpr uint8_t ESTIMATED_SIZE = 9; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("button_command_request"); } @@ -1906,7 +1910,7 @@ class MediaPlayerSupportedFormat final : public ProtoMessage { }; class ListEntitiesMediaPlayerResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 63; + static constexpr uint16_t MESSAGE_TYPE = 63; static constexpr uint8_t ESTIMATED_SIZE = 80; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_media_player_response"); } @@ -1924,7 +1928,7 @@ class ListEntitiesMediaPlayerResponse final : public InfoResponseProtoMessage { }; class MediaPlayerStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 64; + static constexpr uint16_t MESSAGE_TYPE = 64; static constexpr uint8_t ESTIMATED_SIZE = 18; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("media_player_state_response"); } @@ -1942,7 +1946,7 @@ class MediaPlayerStateResponse final : public StateResponseProtoMessage { }; class MediaPlayerCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 65; + static constexpr uint16_t MESSAGE_TYPE = 65; static constexpr uint8_t ESTIMATED_SIZE = 35; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("media_player_command_request"); } @@ -1968,7 +1972,7 @@ class MediaPlayerCommandRequest final : public CommandProtoMessage { #ifdef USE_BLUETOOTH_PROXY class SubscribeBluetoothLEAdvertisementsRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 66; + static constexpr uint16_t MESSAGE_TYPE = 66; static constexpr uint8_t ESTIMATED_SIZE = 4; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("subscribe_bluetooth_le_advertisements_request"); } @@ -1996,7 +2000,7 @@ class BluetoothLERawAdvertisement final : public ProtoMessage { }; class BluetoothLERawAdvertisementsResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 93; + static constexpr uint16_t MESSAGE_TYPE = 93; static constexpr uint8_t ESTIMATED_SIZE = 136; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_le_raw_advertisements_response"); } @@ -2015,7 +2019,7 @@ class BluetoothLERawAdvertisementsResponse final : public ProtoMessage { #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS class BluetoothDeviceRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 68; + static constexpr uint16_t MESSAGE_TYPE = 68; static constexpr uint8_t ESTIMATED_SIZE = 12; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_device_request"); } @@ -2033,7 +2037,7 @@ class BluetoothDeviceRequest final : public ProtoDecodableMessage { }; class BluetoothDeviceConnectionResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 69; + static constexpr uint16_t MESSAGE_TYPE = 69; static constexpr uint8_t ESTIMATED_SIZE = 14; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_device_connection_response"); } @@ -2052,7 +2056,7 @@ class BluetoothDeviceConnectionResponse final : public ProtoMessage { }; class BluetoothGATTGetServicesRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 70; + static constexpr uint16_t MESSAGE_TYPE = 70; static constexpr uint8_t ESTIMATED_SIZE = 4; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_get_services_request"); } @@ -2109,7 +2113,7 @@ class BluetoothGATTService final : public ProtoMessage { }; class BluetoothGATTGetServicesResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 71; + static constexpr uint16_t MESSAGE_TYPE = 71; static constexpr uint8_t ESTIMATED_SIZE = 38; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_get_services_response"); } @@ -2126,7 +2130,7 @@ class BluetoothGATTGetServicesResponse final : public ProtoMessage { }; class BluetoothGATTGetServicesDoneResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 72; + static constexpr uint16_t MESSAGE_TYPE = 72; static constexpr uint8_t ESTIMATED_SIZE = 4; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_get_services_done_response"); } @@ -2142,7 +2146,7 @@ class BluetoothGATTGetServicesDoneResponse final : public ProtoMessage { }; class BluetoothGATTReadRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 73; + static constexpr uint16_t MESSAGE_TYPE = 73; static constexpr uint8_t ESTIMATED_SIZE = 8; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_read_request"); } @@ -2158,7 +2162,7 @@ class BluetoothGATTReadRequest final : public ProtoDecodableMessage { }; class BluetoothGATTReadResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 74; + static constexpr uint16_t MESSAGE_TYPE = 74; static constexpr uint8_t ESTIMATED_SIZE = 27; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_read_response"); } @@ -2181,7 +2185,7 @@ class BluetoothGATTReadResponse final : public ProtoMessage { }; class BluetoothGATTWriteRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 75; + static constexpr uint16_t MESSAGE_TYPE = 75; static constexpr uint8_t ESTIMATED_SIZE = 29; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_write_request"); } @@ -2201,7 +2205,7 @@ class BluetoothGATTWriteRequest final : public ProtoDecodableMessage { }; class BluetoothGATTReadDescriptorRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 76; + static constexpr uint16_t MESSAGE_TYPE = 76; static constexpr uint8_t ESTIMATED_SIZE = 8; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_read_descriptor_request"); } @@ -2217,7 +2221,7 @@ class BluetoothGATTReadDescriptorRequest final : public ProtoDecodableMessage { }; class BluetoothGATTWriteDescriptorRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 77; + static constexpr uint16_t MESSAGE_TYPE = 77; static constexpr uint8_t ESTIMATED_SIZE = 27; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_write_descriptor_request"); } @@ -2236,7 +2240,7 @@ class BluetoothGATTWriteDescriptorRequest final : public ProtoDecodableMessage { }; class BluetoothGATTNotifyRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 78; + static constexpr uint16_t MESSAGE_TYPE = 78; static constexpr uint8_t ESTIMATED_SIZE = 10; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_notify_request"); } @@ -2253,7 +2257,7 @@ class BluetoothGATTNotifyRequest final : public ProtoDecodableMessage { }; class BluetoothGATTNotifyDataResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 79; + static constexpr uint16_t MESSAGE_TYPE = 79; static constexpr uint8_t ESTIMATED_SIZE = 27; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_notify_data_response"); } @@ -2276,7 +2280,7 @@ class BluetoothGATTNotifyDataResponse final : public ProtoMessage { }; class BluetoothConnectionsFreeResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 81; + static constexpr uint16_t MESSAGE_TYPE = 81; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_connections_free_response"); } @@ -2294,7 +2298,7 @@ class BluetoothConnectionsFreeResponse final : public ProtoMessage { }; class BluetoothGATTErrorResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 82; + static constexpr uint16_t MESSAGE_TYPE = 82; static constexpr uint8_t ESTIMATED_SIZE = 12; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_error_response"); } @@ -2312,7 +2316,7 @@ class BluetoothGATTErrorResponse final : public ProtoMessage { }; class BluetoothGATTWriteResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 83; + static constexpr uint16_t MESSAGE_TYPE = 83; static constexpr uint8_t ESTIMATED_SIZE = 8; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_write_response"); } @@ -2329,7 +2333,7 @@ class BluetoothGATTWriteResponse final : public ProtoMessage { }; class BluetoothGATTNotifyResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 84; + static constexpr uint16_t MESSAGE_TYPE = 84; static constexpr uint8_t ESTIMATED_SIZE = 8; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_gatt_notify_response"); } @@ -2346,7 +2350,7 @@ class BluetoothGATTNotifyResponse final : public ProtoMessage { }; class BluetoothDevicePairingResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 85; + static constexpr uint16_t MESSAGE_TYPE = 85; static constexpr uint8_t ESTIMATED_SIZE = 10; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_device_pairing_response"); } @@ -2364,7 +2368,7 @@ class BluetoothDevicePairingResponse final : public ProtoMessage { }; class BluetoothDeviceUnpairingResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 86; + static constexpr uint16_t MESSAGE_TYPE = 86; static constexpr uint8_t ESTIMATED_SIZE = 10; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_device_unpairing_response"); } @@ -2382,7 +2386,7 @@ class BluetoothDeviceUnpairingResponse final : public ProtoMessage { }; class BluetoothDeviceClearCacheResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 88; + static constexpr uint16_t MESSAGE_TYPE = 88; static constexpr uint8_t ESTIMATED_SIZE = 10; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_device_clear_cache_response"); } @@ -2402,7 +2406,7 @@ class BluetoothDeviceClearCacheResponse final : public ProtoMessage { #ifdef USE_BLUETOOTH_PROXY class BluetoothScannerStateResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 126; + static constexpr uint16_t MESSAGE_TYPE = 126; static constexpr uint8_t ESTIMATED_SIZE = 6; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_scanner_state_response"); } @@ -2420,7 +2424,7 @@ class BluetoothScannerStateResponse final : public ProtoMessage { }; class BluetoothScannerSetModeRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 127; + static constexpr uint16_t MESSAGE_TYPE = 127; static constexpr uint8_t ESTIMATED_SIZE = 2; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_scanner_set_mode_request"); } @@ -2437,7 +2441,7 @@ class BluetoothScannerSetModeRequest final : public ProtoDecodableMessage { #ifdef USE_VOICE_ASSISTANT class SubscribeVoiceAssistantRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 89; + static constexpr uint16_t MESSAGE_TYPE = 89; static constexpr uint8_t ESTIMATED_SIZE = 6; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("subscribe_voice_assistant_request"); } @@ -2466,7 +2470,7 @@ class VoiceAssistantAudioSettings final : public ProtoMessage { }; class VoiceAssistantRequest final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 90; + static constexpr uint16_t MESSAGE_TYPE = 90; static constexpr uint8_t ESTIMATED_SIZE = 41; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_request"); } @@ -2486,7 +2490,7 @@ class VoiceAssistantRequest final : public ProtoMessage { }; class VoiceAssistantResponse final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 91; + static constexpr uint16_t MESSAGE_TYPE = 91; static constexpr uint8_t ESTIMATED_SIZE = 6; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_response"); } @@ -2513,7 +2517,7 @@ class VoiceAssistantEventData final : public ProtoDecodableMessage { }; class VoiceAssistantEventResponse final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 92; + static constexpr uint16_t MESSAGE_TYPE = 92; static constexpr uint8_t ESTIMATED_SIZE = 36; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_event_response"); } @@ -2530,7 +2534,7 @@ class VoiceAssistantEventResponse final : public ProtoDecodableMessage { }; class VoiceAssistantAudio final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 106; + static constexpr uint16_t MESSAGE_TYPE = 106; static constexpr uint8_t ESTIMATED_SIZE = 40; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_audio"); } @@ -2552,7 +2556,7 @@ class VoiceAssistantAudio final : public ProtoDecodableMessage { }; class VoiceAssistantTimerEventResponse final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 115; + static constexpr uint16_t MESSAGE_TYPE = 115; static constexpr uint8_t ESTIMATED_SIZE = 30; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_timer_event_response"); } @@ -2573,7 +2577,7 @@ class VoiceAssistantTimerEventResponse final : public ProtoDecodableMessage { }; class VoiceAssistantAnnounceRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 119; + static constexpr uint16_t MESSAGE_TYPE = 119; static constexpr uint8_t ESTIMATED_SIZE = 29; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_announce_request"); } @@ -2592,7 +2596,7 @@ class VoiceAssistantAnnounceRequest final : public ProtoDecodableMessage { }; class VoiceAssistantAnnounceFinished final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 120; + static constexpr uint16_t MESSAGE_TYPE = 120; static constexpr uint8_t ESTIMATED_SIZE = 2; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_announce_finished"); } @@ -2638,7 +2642,7 @@ class VoiceAssistantExternalWakeWord final : public ProtoDecodableMessage { }; class VoiceAssistantConfigurationRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 121; + static constexpr uint16_t MESSAGE_TYPE = 121; static constexpr uint8_t ESTIMATED_SIZE = 34; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_configuration_request"); } @@ -2653,7 +2657,7 @@ class VoiceAssistantConfigurationRequest final : public ProtoDecodableMessage { }; class VoiceAssistantConfigurationResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 122; + static constexpr uint16_t MESSAGE_TYPE = 122; static constexpr uint8_t ESTIMATED_SIZE = 56; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_configuration_response"); } @@ -2671,7 +2675,7 @@ class VoiceAssistantConfigurationResponse final : public ProtoMessage { }; class VoiceAssistantSetConfiguration final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 123; + static constexpr uint16_t MESSAGE_TYPE = 123; static constexpr uint8_t ESTIMATED_SIZE = 18; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("voice_assistant_set_configuration"); } @@ -2688,7 +2692,7 @@ class VoiceAssistantSetConfiguration final : public ProtoDecodableMessage { #ifdef USE_ALARM_CONTROL_PANEL class ListEntitiesAlarmControlPanelResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 94; + static constexpr uint16_t MESSAGE_TYPE = 94; static constexpr uint8_t ESTIMATED_SIZE = 48; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_alarm_control_panel_response"); } @@ -2706,7 +2710,7 @@ class ListEntitiesAlarmControlPanelResponse final : public InfoResponseProtoMess }; class AlarmControlPanelStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 95; + static constexpr uint16_t MESSAGE_TYPE = 95; static constexpr uint8_t ESTIMATED_SIZE = 11; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("alarm_control_panel_state_response"); } @@ -2722,7 +2726,7 @@ class AlarmControlPanelStateResponse final : public StateResponseProtoMessage { }; class AlarmControlPanelCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 96; + static constexpr uint16_t MESSAGE_TYPE = 96; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("alarm_control_panel_command_request"); } @@ -2742,7 +2746,7 @@ class AlarmControlPanelCommandRequest final : public CommandProtoMessage { #ifdef USE_TEXT class ListEntitiesTextResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 97; + static constexpr uint16_t MESSAGE_TYPE = 97; static constexpr uint8_t ESTIMATED_SIZE = 59; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_text_response"); } @@ -2761,7 +2765,7 @@ class ListEntitiesTextResponse final : public InfoResponseProtoMessage { }; class TextStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 98; + static constexpr uint16_t MESSAGE_TYPE = 98; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("text_state_response"); } @@ -2778,7 +2782,7 @@ class TextStateResponse final : public StateResponseProtoMessage { }; class TextCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 99; + static constexpr uint16_t MESSAGE_TYPE = 99; static constexpr uint8_t ESTIMATED_SIZE = 18; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("text_command_request"); } @@ -2797,7 +2801,7 @@ class TextCommandRequest final : public CommandProtoMessage { #ifdef USE_DATETIME_DATE class ListEntitiesDateResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 100; + static constexpr uint16_t MESSAGE_TYPE = 100; static constexpr uint8_t ESTIMATED_SIZE = 40; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_date_response"); } @@ -2812,7 +2816,7 @@ class ListEntitiesDateResponse final : public InfoResponseProtoMessage { }; class DateStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 101; + static constexpr uint16_t MESSAGE_TYPE = 101; static constexpr uint8_t ESTIMATED_SIZE = 23; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("date_state_response"); } @@ -2831,7 +2835,7 @@ class DateStateResponse final : public StateResponseProtoMessage { }; class DateCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 102; + static constexpr uint16_t MESSAGE_TYPE = 102; static constexpr uint8_t ESTIMATED_SIZE = 21; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("date_command_request"); } @@ -2851,7 +2855,7 @@ class DateCommandRequest final : public CommandProtoMessage { #ifdef USE_DATETIME_TIME class ListEntitiesTimeResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 103; + static constexpr uint16_t MESSAGE_TYPE = 103; static constexpr uint8_t ESTIMATED_SIZE = 40; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_time_response"); } @@ -2866,7 +2870,7 @@ class ListEntitiesTimeResponse final : public InfoResponseProtoMessage { }; class TimeStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 104; + static constexpr uint16_t MESSAGE_TYPE = 104; static constexpr uint8_t ESTIMATED_SIZE = 23; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("time_state_response"); } @@ -2885,7 +2889,7 @@ class TimeStateResponse final : public StateResponseProtoMessage { }; class TimeCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 105; + static constexpr uint16_t MESSAGE_TYPE = 105; static constexpr uint8_t ESTIMATED_SIZE = 21; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("time_command_request"); } @@ -2905,7 +2909,7 @@ class TimeCommandRequest final : public CommandProtoMessage { #ifdef USE_EVENT class ListEntitiesEventResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 107; + static constexpr uint16_t MESSAGE_TYPE = 107; static constexpr uint8_t ESTIMATED_SIZE = 67; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_event_response"); } @@ -2922,7 +2926,7 @@ class ListEntitiesEventResponse final : public InfoResponseProtoMessage { }; class EventResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 108; + static constexpr uint16_t MESSAGE_TYPE = 108; static constexpr uint8_t ESTIMATED_SIZE = 18; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("event_response"); } @@ -2940,7 +2944,7 @@ class EventResponse final : public StateResponseProtoMessage { #ifdef USE_VALVE class ListEntitiesValveResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 109; + static constexpr uint16_t MESSAGE_TYPE = 109; static constexpr uint8_t ESTIMATED_SIZE = 55; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_valve_response"); } @@ -2959,7 +2963,7 @@ class ListEntitiesValveResponse final : public InfoResponseProtoMessage { }; class ValveStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 110; + static constexpr uint16_t MESSAGE_TYPE = 110; static constexpr uint8_t ESTIMATED_SIZE = 16; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("valve_state_response"); } @@ -2976,7 +2980,7 @@ class ValveStateResponse final : public StateResponseProtoMessage { }; class ValveCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 111; + static constexpr uint16_t MESSAGE_TYPE = 111; static constexpr uint8_t ESTIMATED_SIZE = 18; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("valve_command_request"); } @@ -2996,7 +3000,7 @@ class ValveCommandRequest final : public CommandProtoMessage { #ifdef USE_DATETIME_DATETIME class ListEntitiesDateTimeResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 112; + static constexpr uint16_t MESSAGE_TYPE = 112; static constexpr uint8_t ESTIMATED_SIZE = 40; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_date_time_response"); } @@ -3011,7 +3015,7 @@ class ListEntitiesDateTimeResponse final : public InfoResponseProtoMessage { }; class DateTimeStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 113; + static constexpr uint16_t MESSAGE_TYPE = 113; static constexpr uint8_t ESTIMATED_SIZE = 16; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("date_time_state_response"); } @@ -3028,7 +3032,7 @@ class DateTimeStateResponse final : public StateResponseProtoMessage { }; class DateTimeCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 114; + static constexpr uint16_t MESSAGE_TYPE = 114; static constexpr uint8_t ESTIMATED_SIZE = 14; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("date_time_command_request"); } @@ -3046,7 +3050,7 @@ class DateTimeCommandRequest final : public CommandProtoMessage { #ifdef USE_UPDATE class ListEntitiesUpdateResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 116; + static constexpr uint16_t MESSAGE_TYPE = 116; static constexpr uint8_t ESTIMATED_SIZE = 49; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_update_response"); } @@ -3062,7 +3066,7 @@ class ListEntitiesUpdateResponse final : public InfoResponseProtoMessage { }; class UpdateStateResponse final : public StateResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 117; + static constexpr uint16_t MESSAGE_TYPE = 117; static constexpr uint8_t ESTIMATED_SIZE = 65; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("update_state_response"); } @@ -3086,7 +3090,7 @@ class UpdateStateResponse final : public StateResponseProtoMessage { }; class UpdateCommandRequest final : public CommandProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 118; + static constexpr uint16_t MESSAGE_TYPE = 118; static constexpr uint8_t ESTIMATED_SIZE = 11; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("update_command_request"); } @@ -3104,7 +3108,7 @@ class UpdateCommandRequest final : public CommandProtoMessage { #ifdef USE_ZWAVE_PROXY class ZWaveProxyFrame final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 128; + static constexpr uint16_t MESSAGE_TYPE = 128; static constexpr uint8_t ESTIMATED_SIZE = 19; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("z_wave_proxy_frame"); } @@ -3122,7 +3126,7 @@ class ZWaveProxyFrame final : public ProtoDecodableMessage { }; class ZWaveProxyRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 129; + static constexpr uint16_t MESSAGE_TYPE = 129; static constexpr uint8_t ESTIMATED_SIZE = 21; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("z_wave_proxy_request"); } @@ -3142,7 +3146,7 @@ class ZWaveProxyRequest final : public ProtoDecodableMessage { }; class ZWaveProxyRequestResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 151; + static constexpr uint16_t MESSAGE_TYPE = 151; static constexpr uint8_t ESTIMATED_SIZE = 4; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("z_wave_proxy_request_response"); } @@ -3161,7 +3165,7 @@ class ZWaveProxyRequestResponse final : public ProtoMessage { #ifdef USE_INFRARED class ListEntitiesInfraredResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 135; + static constexpr uint16_t MESSAGE_TYPE = 135; static constexpr uint8_t ESTIMATED_SIZE = 48; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_infrared_response"); } @@ -3180,7 +3184,7 @@ class ListEntitiesInfraredResponse final : public InfoResponseProtoMessage { #if defined(USE_IR_RF) || defined(USE_RADIO_FREQUENCY) class InfraredRFTransmitRawTimingsRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 136; + static constexpr uint16_t MESSAGE_TYPE = 136; static constexpr uint8_t ESTIMATED_SIZE = 224; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("infrared_rf_transmit_raw_timings_request"); } @@ -3206,7 +3210,7 @@ class InfraredRFTransmitRawTimingsRequest final : public ProtoDecodableMessage { }; class InfraredRFReceiveEvent final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 137; + static constexpr uint16_t MESSAGE_TYPE = 137; static constexpr uint8_t ESTIMATED_SIZE = 17; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("infrared_rf_receive_event"); } @@ -3228,7 +3232,7 @@ class InfraredRFReceiveEvent final : public ProtoMessage { #ifdef USE_RADIO_FREQUENCY class ListEntitiesRadioFrequencyResponse final : public InfoResponseProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 148; + static constexpr uint16_t MESSAGE_TYPE = 148; static constexpr uint8_t ESTIMATED_SIZE = 56; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_radio_frequency_response"); } @@ -3249,7 +3253,7 @@ class ListEntitiesRadioFrequencyResponse final : public InfoResponseProtoMessage #ifdef USE_SERIAL_PROXY class SerialProxyConfigureRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 138; + static constexpr uint16_t MESSAGE_TYPE = 138; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_configure_request"); } @@ -3269,7 +3273,7 @@ class SerialProxyConfigureRequest final : public ProtoDecodableMessage { }; class SerialProxyDataReceived final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 139; + static constexpr uint16_t MESSAGE_TYPE = 139; static constexpr uint8_t ESTIMATED_SIZE = 23; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_data_received"); } @@ -3291,7 +3295,7 @@ class SerialProxyDataReceived final : public ProtoMessage { }; class SerialProxyWriteRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 140; + static constexpr uint16_t MESSAGE_TYPE = 140; static constexpr uint8_t ESTIMATED_SIZE = 23; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_write_request"); } @@ -3309,7 +3313,7 @@ class SerialProxyWriteRequest final : public ProtoDecodableMessage { }; class SerialProxySetModemPinsRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 141; + static constexpr uint16_t MESSAGE_TYPE = 141; static constexpr uint8_t ESTIMATED_SIZE = 8; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_set_modem_pins_request"); } @@ -3325,7 +3329,7 @@ class SerialProxySetModemPinsRequest final : public ProtoDecodableMessage { }; class SerialProxyGetModemPinsRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 142; + static constexpr uint16_t MESSAGE_TYPE = 142; static constexpr uint8_t ESTIMATED_SIZE = 4; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_get_modem_pins_request"); } @@ -3340,7 +3344,7 @@ class SerialProxyGetModemPinsRequest final : public ProtoDecodableMessage { }; class SerialProxyGetModemPinsResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 143; + static constexpr uint16_t MESSAGE_TYPE = 143; static constexpr uint8_t ESTIMATED_SIZE = 10; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_get_modem_pins_response"); } @@ -3358,7 +3362,7 @@ class SerialProxyGetModemPinsResponse final : public ProtoMessage { }; class SerialProxyRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 144; + static constexpr uint16_t MESSAGE_TYPE = 144; static constexpr uint8_t ESTIMATED_SIZE = 6; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_request"); } @@ -3374,7 +3378,7 @@ class SerialProxyRequest final : public ProtoDecodableMessage { }; class SerialProxyRequestResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 147; + static constexpr uint16_t MESSAGE_TYPE = 147; static constexpr uint8_t ESTIMATED_SIZE = 17; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("serial_proxy_request_response"); } @@ -3395,7 +3399,7 @@ class SerialProxyRequestResponse final : public ProtoMessage { #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS class BluetoothSetConnectionParamsRequest final : public ProtoDecodableMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 145; + static constexpr uint16_t MESSAGE_TYPE = 145; static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_set_connection_params_request"); } @@ -3414,7 +3418,7 @@ class BluetoothSetConnectionParamsRequest final : public ProtoDecodableMessage { }; class BluetoothSetConnectionParamsResponse final : public ProtoMessage { public: - static constexpr uint8_t MESSAGE_TYPE = 146; + static constexpr uint16_t MESSAGE_TYPE = 146; static constexpr uint8_t ESTIMATED_SIZE = 8; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("bluetooth_set_connection_params_response"); } diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index f058f6af22..a226e080e8 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -684,11 +684,6 @@ class ProtoSize { return value < VARINT_THRESHOLD_1_BYTE ? 1 : (value < VARINT_THRESHOLD_2_BYTE ? 2 : 3); } - // Varint encoded length for an 8-bit value (1 or 2 bytes). - static constexpr inline uint8_t ESPHOME_ALWAYS_INLINE varint8(uint8_t value) { - return value < VARINT_THRESHOLD_1_BYTE ? 1 : 2; - } - /** * @brief Calculates the size in bytes needed to encode a uint32_t value as a varint * diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index dc3dd4b868..ca1c5736c8 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -475,6 +475,19 @@ TYPE_INFO: dict[int, TypeInfo] = {} # TYPE_DOUBLE = 1, TYPE_FIXED64 = 6, TYPE_SFIXED64 = 16, TYPE_SINT64 = 18 UNSUPPORTED_TYPES = {1: "double", 6: "fixed64", 16: "sfixed64", 18: "sint64"} +# The plaintext frame header budgets 2 varint bytes for the message type +# (APIPlaintextFrameHelper::HEADER_PADDING), which caps message IDs at 16383. +MAX_MESSAGE_ID = 16383 + + +def validate_message_id(message_id: int, message_name: str) -> None: + """Reject message IDs whose plaintext type varint would not fit in 2 bytes.""" + if message_id > MAX_MESSAGE_ID: + raise ValueError( + f"Message ID {message_id} for {message_name} exceeds the plaintext " + f"2-byte type varint maximum ({MAX_MESSAGE_ID})" + ) + def validate_field_type(field_type: int, field_name: str = "") -> None: """Validate that the field type is supported by ESPHome API. @@ -2549,14 +2562,10 @@ def build_message_type( # Add MESSAGE_TYPE method if this is a service message if message_id is not None: - # Validate that message_id fits in uint8_t - if message_id > 255: - raise ValueError( - f"Message ID {message_id} for {desc.name} exceeds uint8_t maximum (255)" - ) + validate_message_id(message_id, desc.name) # Add static constexpr for message type - public_content.append(f"static constexpr uint8_t MESSAGE_TYPE = {message_id};") + public_content.append(f"static constexpr uint16_t MESSAGE_TYPE = {message_id};") # Add estimated size constant estimated_size = calculate_message_estimated_size(desc) @@ -3212,8 +3221,12 @@ def main() -> None: #include "api_pb2_includes.h" """ - content += """ -namespace esphome::api { + content += f""" +namespace esphome::api {{ + +// Upper bound on message IDs, enforced by the code generator: the plaintext +// frame header budgets 2 varint bytes for the type (HEADER_PADDING). +static constexpr uint16_t MAX_MESSAGE_TYPE = {MAX_MESSAGE_ID}; """ diff --git a/tests/unit_tests/components/api/test_api_protobuf_generator.py b/tests/unit_tests/components/api/test_api_protobuf_generator.py index 2a07cbd49c..797125ba8f 100644 --- a/tests/unit_tests/components/api/test_api_protobuf_generator.py +++ b/tests/unit_tests/components/api/test_api_protobuf_generator.py @@ -15,7 +15,12 @@ import pytest sys.path.insert(0, str(Path(__file__).parents[4] / "script" / "api_protobuf")) -from api_protobuf import _make_ifdef_line, get_varint64_ifdef # noqa: E402 +from api_protobuf import ( # noqa: E402 + MAX_MESSAGE_ID, + _make_ifdef_line, + get_varint64_ifdef, + validate_message_id, +) from google.protobuf import descriptor_pb2 # noqa: E402 @@ -91,3 +96,14 @@ def test_make_ifdef_line_conjunction_and_negation() -> None: assert ( _make_ifdef_line("USE_X && !USE_Y") == "#if defined(USE_X) && !defined(USE_Y)" ) + + +def test_message_id_at_maximum_is_accepted() -> None: + # 16383 is the largest ID whose plaintext type varint fits the 2 bytes + # budgeted in HEADER_PADDING. + validate_message_id(MAX_MESSAGE_ID, "MaxMessage") + + +def test_message_id_above_maximum_is_rejected() -> None: + with pytest.raises(ValueError, match="exceeds the plaintext"): + validate_message_id(MAX_MESSAGE_ID + 1, "TooBigMessage") From cf5ec2d27722ea19dbe88ea052b5aa02259c7c3a Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:42:32 -0400 Subject: [PATCH 38/65] [ci] Remove the remaining max-parallel caps (#18762) --- .github/workflows/ci-docker.yml | 2 -- .github/workflows/ci.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index f3f7cb30eb..42be51cdd9 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -182,8 +182,6 @@ jobs: contents: read # actions/checkout to load the test configs strategy: fail-fast: false - # Modest cap so this smoke test leaves room on the shared runner pool. - max-parallel: 8 matrix: # One entry per distinct toolchain. ESP32 variants (c3/c6/s2/s3/p4) # share a toolchain bundle, so esp32 is exercised on the base variant diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9da0937555..a2762faa4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -946,7 +946,6 @@ jobs: ESPHOME_SDK_NRF_PREFIX: ~/.esphome-sdk-nrf strategy: fail-fast: false - max-parallel: ${{ needs.determine-jobs.outputs.release-pr == 'true' && 32 || 16 }} matrix: batch: ${{ fromJson(needs.determine-jobs.outputs.component-test-batches) }} steps: From 150f75d8f6f88f158f7df80349f943e4607da0fb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 17:29:57 -0500 Subject: [PATCH 39/65] [esp8266] Add linker-script surgery and board build metadata for the native toolchain (#18555) --- esphome/components/esp8266/__init__.py | 40 +++-- esphome/components/esp8266/boards.py | 136 +++++++++++++++- esphome/components/esp8266/build_surgery.py | 123 +++++++++++++++ esphome/components/esp8266/const.py | 5 + .../components/esp8266/test_boards.py | 33 ++++ .../components/esp8266/test_build_surgery.py | 145 ++++++++++++++++++ 6 files changed, 469 insertions(+), 13 deletions(-) create mode 100644 esphome/components/esp8266/build_surgery.py create mode 100644 tests/unit_tests/components/esp8266/test_boards.py create mode 100644 tests/unit_tests/components/esp8266/test_build_surgery.py diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 3dd9750c6f..75483c5293 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -35,7 +35,7 @@ from esphome.platformio.toolchain import copy_ccache_script from esphome.storage_json import StorageJSON from esphome.types import ConfigType -from .boards import BOARDS, ESP8266_LD_SCRIPTS +from .boards import BOARDS, ESP8266_LD_SCRIPTS, board_ld_script from .const import ( CONF_EARLY_PIN_INIT, CONF_ENABLE_SERIAL, @@ -44,6 +44,7 @@ from .const import ( KEY_BOARD, KEY_ESP8266, KEY_FLASH_SIZE, + KEY_LDSCRIPT, KEY_PIN_INITIAL_STATES, KEY_SERIAL1_REQUIRED, KEY_SERIAL_REQUIRED, @@ -276,6 +277,31 @@ def check_rosetta() -> None: ) +def _choose_ld_script(board: str, ver: cv.Version) -> str | None: + """The flash ld to pin for this board and core, or None for cores + without ld-script support.""" + board_data = BOARDS[board] + ld_scripts = ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]] + if ver <= cv.Version(2, 3, 0): + # No ld script support + return None + if ver <= cv.Version(2, 4, 2): + # Old ld script path; the modern per-board override names do not + # exist in this core's SDK, so the override cannot be honored. + # Substituting the size default would move _FS_end and the + # preferences sector, wiping flash-backed state on flash. + if KEY_LDSCRIPT in board_data: + raise EsphomeError( + f"Board {board} requires its {board_data[KEY_LDSCRIPT]} " + f"flash layout, which Arduino core {ver} cannot honor; " + "use a core newer than 2.4.2" + ) + return ld_scripts[0] + # A per-board override preserves a layout the board shipped with + # (see d1_wroom_02 in boards.py) + return board_ld_script(board_data) + + @coroutine_with_priority(CoroPriority.PLATFORM) async def to_code(config: ConfigType) -> None: cg.add(esp8266_ns.setup_preferences()) @@ -397,17 +423,7 @@ async def to_code(config: ConfigType) -> None: ) if config[CONF_BOARD] in BOARDS: - flash_size = BOARDS[config[CONF_BOARD]][KEY_FLASH_SIZE] - ld_scripts = ESP8266_LD_SCRIPTS[flash_size] - - if ver <= cv.Version(2, 3, 0): - # No ld script support - ld_script = None - elif ver <= cv.Version(2, 4, 2): - # Old ld script path - ld_script = ld_scripts[0] - else: - ld_script = ld_scripts[1] + ld_script = _choose_ld_script(config[CONF_BOARD], ver) if ld_script is not None: cg.add_platformio_option("board_build.ldscript", ld_script) diff --git a/esphome/components/esp8266/boards.py b/esphome/components/esp8266/boards.py index 02bfa9e662..268c6b50aa 100644 --- a/esphome/components/esp8266/boards.py +++ b/esphome/components/esp8266/boards.py @@ -1,3 +1,5 @@ +from .const import KEY_FLASH_SIZE, KEY_LDSCRIPT + FLASH_SIZE_1_MB = 2**20 FLASH_SIZE_512_KB = FLASH_SIZE_1_MB // 2 FLASH_SIZE_2_MB = 2 * FLASH_SIZE_1_MB @@ -164,7 +166,8 @@ ESP8266_BOARD_PINS = { } """ -BOARDS generate with: +BOARDS generate with (preserve per-board KEY_LDSCRIPT overrides such as +d1_wroom_02; the recipe emits only name/flash_size): git clone https://github.com/platformio/platform-espressif8266 for x in platform-espressif8266/boards/*.json; do @@ -182,6 +185,19 @@ for x in platform-espressif8266/boards/*.json; do done | sort """ + +def board_ld_script(board_data: dict) -> str: + """The modern (core > 2.4.2) flash linker script for a board: its + shipped-layout override, else the size default (the no-FS layout). + + Single source of truth for the PlatformIO pinning in __init__ and the + native generator's fallback, so the per-board rule cannot drift. + """ + return board_data.get( + KEY_LDSCRIPT, ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]][1] + ) + + BOARDS = { "agruminolemon": { "name": "Lifely Agrumino Lemon v4", @@ -199,6 +215,15 @@ BOARDS = { "name": "WeMos D1 mini Pro", "flash_size": FLASH_SIZE_16_MB, }, + "d1_wroom_02": { + "name": "WeMos D1 ESP-WROOM-02", + "flash_size": FLASH_SIZE_2_MB, + # This board joined BOARDS after shipping with the manifest default + # (64 KB filesystem region); the flash-size default (2m.ld) would + # move _FS_end and with it the preferences sector, wiping existing + # devices' flash-backed state on update. + KEY_LDSCRIPT: "eagle.flash.2m64.ld", + }, "d1": { "name": "WEMOS D1 R1", "flash_size": FLASH_SIZE_4_MB, @@ -360,3 +385,112 @@ BOARDS = { "flash_size": FLASH_SIZE_4_MB, }, } + + +# Per-board variant dir + identity defines from platform-espressif8266 4.x +# build.extra_flags; the shared -DESP8266/-DARDUINO_ARCH_ESP8266 are added +# by the generator. +# +# Regenerate ESP8266_BOARD_BUILD with (v4.2.1 is the platform version the +# native toolchain mirrors; regenerate against the tag when bumping it): +# +# git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266 +# python3 - <<'EOF' +# import json, glob, os +# for f in sorted(glob.glob("platform-espressif8266/boards/*.json")): +# b = json.load(open(f))["build"] +# extra = b["extra_flags"] +# extra = extra.split() if isinstance(extra, str) else extra +# defines = [ +# e[2:] for e in extra if e not in ("-DESP8266", "-DARDUINO_ARCH_ESP8266") +# ] +# entries = ", ".join(f'"{d}"' for d in defines) + ("," if len(defines) == 1 else "") +# board = os.path.splitext(os.path.basename(f))[0] +# print(f' "{board}": {{"variant": "{b["variant"]}", "defines": ({entries})}},') +# EOF +ESP8266_BOARD_BUILD = { + "agruminolemon": { + "variant": "agruminolemonv4", + "defines": ("ARDUINO_ESP8266_AGRUMINO_LEMON_V4",), + }, + "d1": {"variant": "d1", "defines": ("ARDUINO_ESP8266_WEMOS_D1R1",)}, + "d1_mini": {"variant": "d1_mini", "defines": ("ARDUINO_ESP8266_WEMOS_D1MINI",)}, + "d1_mini_lite": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1MINILITE",), + }, + "d1_mini_pro": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1MINIPRO",), + }, + "d1_wroom_02": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1WROOM02",), + }, + "eduinowifi": { + "variant": "eduinowifi", + "defines": ("ARDUINO_ESP8266_SCHIRMILABS_EDUINO_WIFI",), + }, + "esp01": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp01_1m": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp07": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP07",)}, + "esp07s": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP07",)}, + "esp12e": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP12",)}, + "esp210": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP210",)}, + "esp8285": {"variant": "esp8285", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp_wroom_02": { + "variant": "nodemcu", + "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",), + }, + "espduino": {"variant": "ESPDuino", "defines": ("ARDUINO_ESP8266_ESP13",)}, + "espectro": {"variant": "espectro", "defines": ("ARDUINO_ESP8266_ESPECTRO_CORE",)}, + "espino": {"variant": "espino", "defines": ("ARDUINO_ESP8266_ESP12",)}, + "espinotee": {"variant": "espinotee", "defines": ("ARDUINO_ESP8266_ESP13",)}, + "espmxdevkit": { + "variant": "esp8285", + "defines": ("ARDUINO_ESP8266_ESP01", "LED_BUILTIN=16"), + }, + "espresso_lite_v1": { + "variant": "espresso_lite_v1", + "defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V1",), + }, + "espresso_lite_v2": { + "variant": "espresso_lite_v2", + "defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V2",), + }, + "gen4iod": {"variant": "generic", "defines": ("ARDUINO_GEN4_IOD",)}, + "heltec_wifi_kit_8": { + "variant": "wifi_kit_8", + "defines": ("ARDUINO_wifi_kit_8",), + }, + "huzzah": {"variant": "adafruit", "defines": ("ARDUINO_ESP8266_ADAFRUIT_HUZZAH",)}, + "inventone": {"variant": "inventone", "defines": ("ARDUINO_ESP8266_INVENT_ONE",)}, + "modwifi": {"variant": "generic", "defines": ("ARDUINO_MOD_WIFI_ESP8266",)}, + "nodemcu": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU",)}, + "nodemcuv2": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU_ESP12E",)}, + "oak": {"variant": "oak", "defines": ("ARDUINO_ESP8266_OAK",)}, + "phoenix_v1": { + "variant": "phoenix_v1", + "defines": ("ARDUINO_ESP8266_PHOENIX_V1",), + }, + "phoenix_v2": { + "variant": "phoenix_v2", + "defines": ("ARDUINO_ESP8266_PHOENIX_V2",), + }, + "sonoff_basic": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_BASIC",)}, + "sonoff_s20": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_S20",)}, + "sonoff_sv": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_SV",)}, + "sonoff_th": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_TH",)}, + "sparkfunBlynk": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)}, + "thing": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)}, + "thingdev": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING_DEV",)}, + "wifi_slot": {"variant": "wifi_slot", "defines": ("ARDUINO_AMPERKA_WIFI_SLOT",)}, + "wifiduino": {"variant": "wifiduino", "defines": ("ARDUINO_WIFIDUINO_ESP8266",)}, + "wifinfo": {"variant": "wifinfo", "defines": ("ARDUINO_WIFINFO",)}, + "wio_link": {"variant": "wiolink", "defines": ("ARDUINO_ESP8266_WIO_LINK",)}, + "wio_node": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",)}, + "xinabox_cw01": { + "variant": "xinabox", + "defines": ("ARDUINO_ESP8266_XINABOX_CW01",), + }, +} diff --git a/esphome/components/esp8266/build_surgery.py b/esphome/components/esp8266/build_surgery.py new file mode 100644 index 0000000000..eb6ed1b91b --- /dev/null +++ b/esphome/components/esp8266/build_surgery.py @@ -0,0 +1,123 @@ +"""Linker-script surgery shared with the native (PlatformIO-free) toolchain. + +These mirror the PlatformIO extra scripts in this directory +(``relocate_ratetable.py.script`` and ``testing_mode.py.script``), which run +inside SCons and must stay self-contained. The native build generator applies +the same patches to the linker scripts it generates, so the logic lives here +as plain functions. Keep both in sync when changing either. +``segment_length`` is native-toolchain-only and has no script twin. +""" + +from __future__ import annotations + +from collections.abc import Collection +import hashlib +import re + +# Move the NONOS SDK wifi rate tables from flash to DRAM; see +# relocate_ratetable.py.script for the full background (NONOS SDK issue 320). +RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)" +_RATETABLE_COMMENT = ( + "/* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" +) +# Match the whole line: "_data_start" is also a substring of the +# "_dport0_data_start" line in the earlier .dport0.data section +_RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE) + +# Memory sizes for testing mode (allow larger builds for CI component grouping) +TESTING_IRAM_SIZE = "0x200000" # 2MB +TESTING_DRAM_SIZE = "0x200000" # 2MB +TESTING_FLASH_SIZE = "0x2000000" # 32MB + + +def relocate_ratetable(content: str) -> str: + """Insert the rate-table DRAM rule into a generated common linker script.""" + if RATETABLE_RULE in content: + return content + match = _RATETABLE_ANCHOR.search(content) + if match is None: + raise RuntimeError( + "'_data_start' anchor not found in the generated linker script; " + "cannot apply wifi rate table DRAM relocation " + "(has the Arduino core linker script changed?)" + ) + insert_pos = match.end() + return ( + content[:insert_pos] + + f"\n {_RATETABLE_COMMENT}" + + f"\n {RATETABLE_RULE}" + + content[insert_pos:] + ) + + +_TESTING_SEGMENT_SIZES = { + "iram1_0_seg": TESTING_IRAM_SIZE, + "dram0_0_seg": TESTING_DRAM_SIZE, + "irom0_0_seg": TESTING_FLASH_SIZE, +} + + +def _segment_line_re(segment_name: str) -> re.Pattern[str]: + """The MEMORY line for one segment: `` : org = 0x..., len = 0x...``. + + Anchored to the start of the line so a name never matches inside a + longer one (``ram0_0_seg`` must not read ``dram0_0_seg``). The size + group stops at the hex digits, leaving any ``ul`` suffix (from the + preprocessed ``MMU_IRAM_SIZE``) in place. + """ + return re.compile( + rf"(^[ \t]*{re.escape(segment_name)}" + r"\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)" + r"(0x[0-9a-fA-F]+)", + re.MULTILINE, + ) + + +def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str: + """Enlarge the named memory segments so grouped CI test builds can link. + + Each caller passes the segments its linker script defines: the + generated common ld carries ``iram1_0_seg``; the flash ld carries + ``dram0_0_seg`` and ``irom0_0_seg``. A segment that fails to match + raises, since a silently kept real memory limit would fail grouped + builds far from the cause. + """ + for segment in _TESTING_SEGMENT_SIZES: + if segment not in segments and _segment_line_re(segment).search(content): + raise RuntimeError( + f"Testing-mode segment {segment} is present in the linker " + "script but was not selected for patching" + ) + for segment in segments: + if segment not in _TESTING_SEGMENT_SIZES: + raise RuntimeError(f"Unknown testing-mode segment {segment!r}") + content, count = _segment_line_re(segment).subn( + rf"\g<1>{_TESTING_SEGMENT_SIZES[segment]}", content + ) + if count == 0: + raise RuntimeError( + f"Testing-mode memory patch failed: segment {segment} " + "not found (has the Arduino core linker script changed?)" + ) + return content + + +def segment_length(content: str, segment_name: str) -> int | None: + """Read a memory segment's length from linker script content. + + Returns None for an absent segment OR an unparsable line; callers must + treat None as "no usable budget" and warn (as the Flash summary does), + never as "no limit". + """ + match = _segment_line_re(segment_name).search(content) + return int(match.group(2), 16) if match else None + + +def surgery_fingerprint() -> str: + """Hash of this module's source; linker-script caches include it so an + edit here invalidates them.""" + import inspect + import sys + + source = inspect.getsource(sys.modules[__name__]) + return hashlib.sha256(source.encode()).hexdigest() diff --git a/esphome/components/esp8266/const.py b/esphome/components/esp8266/const.py index 3e89ab989f..50f103ed2d 100644 --- a/esphome/components/esp8266/const.py +++ b/esphome/components/esp8266/const.py @@ -15,6 +15,11 @@ CONF_ENABLE_SERIAL1 = "enable_serial1" KEY_WAVEFORM_REQUIRED = "waveform_required" KEY_SERIAL_REQUIRED = "serial_required" KEY_SERIAL1_REQUIRED = "serial1_required" +# Set for the native (non-PlatformIO) toolchain's build generator +KEY_FLASH_MODE = "flash_mode" +KEY_SCANF_FLOAT = "scanf_float" +# Per-board flash-layout override consumed by board_ld_script() +KEY_LDSCRIPT = "ldscript" # esp8266 namespace is already defined by arduino, manually prefix esphome esp8266_ns = cg.global_ns.namespace("esphome").namespace("esp8266") diff --git a/tests/unit_tests/components/esp8266/test_boards.py b/tests/unit_tests/components/esp8266/test_boards.py new file mode 100644 index 0000000000..df0e536d42 --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_boards.py @@ -0,0 +1,33 @@ +"""Tests for the per-board linker-script rule.""" + +import pytest + +from esphome.components.esp8266 import _choose_ld_script +from esphome.components.esp8266.boards import BOARDS, board_ld_script +import esphome.config_validation as cv +from esphome.core import EsphomeError + + +def test_d1_wroom_02_keeps_its_shipped_layout() -> None: + """The override must survive a BOARDS regeneration or key typo: the + 2m.ld default moves _FS_end and the preferences sector on deployed + devices.""" + assert board_ld_script(BOARDS["d1_wroom_02"]) == "eagle.flash.2m64.ld" + + +def test_default_boards_use_the_flash_size_layout() -> None: + assert board_ld_script(BOARDS["d1_mini"]) == "eagle.flash.4m.ld" + assert board_ld_script(BOARDS["esp01_1m"]) == "eagle.flash.1m.ld" + + +def test_choose_ld_script_paths() -> None: + """Old cores get the size default, overriding boards hard-error there + (a substituted layout would wipe flash-backed state), modern cores + honor the override.""" + assert _choose_ld_script("nodemcuv2", cv.Version(2, 3, 0)) is None + assert _choose_ld_script("nodemcuv2", cv.Version(2, 4, 2)) == "eagle.flash.4m.ld" + assert _choose_ld_script("d1_wroom_02", cv.Version(2, 7, 4)) == ( + "eagle.flash.2m64.ld" + ) + with pytest.raises(EsphomeError, match="cannot honor"): + _choose_ld_script("d1_wroom_02", cv.Version(2, 4, 2)) diff --git a/tests/unit_tests/components/esp8266/test_build_surgery.py b/tests/unit_tests/components/esp8266/test_build_surgery.py new file mode 100644 index 0000000000..411a35eb96 --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_build_surgery.py @@ -0,0 +1,145 @@ +"""Tests for the linker-script surgery shared with the native toolchain.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + +import pytest + +from esphome.components.esp8266 import build_surgery +from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD +from esphome.components.esp8266.build_surgery import ( + RATETABLE_RULE, + apply_testing_memory_patches, + relocate_ratetable, + segment_length, +) + +_COMMON_LD_SNIPPET = """\ + .dport0.data : ALIGN(4) + { + _dport0_data_start = ABSOLUTE(.); + } >dport0_0_seg :dport0_0_phdr + .data : ALIGN(4) + { + _data_start = ABSOLUTE(.); + *(.data) + } >dram0_0_seg :dram0_0_phdr +""" + +# Shaped like the real SDK flash ld scripts: no iram1_0_seg (that lives in +# the generated common ld only) +_FLASH_LD_SNIPPET = """\ +MEMORY +{ + dport0_0_seg : org = 0x3FF00000, len = 0x10 + dram0_0_seg : org = 0x3FFE8000, len = 0x14000 + irom0_0_seg : org = 0x40201010, len = 0xfeff0 +} +""" + +# Shaped like the preprocessed common ld: MMU_IRAM_SIZE expands with a ul +# suffix the patcher must leave in place +_COMMON_LD_MEMORY_SNIPPET = """\ +MEMORY +{ + iram1_0_seg : org = 0x40100000, len = 0x8000ul +} +""" + + +def test_relocate_ratetable_inserts_after_data_start() -> None: + patched = relocate_ratetable(_COMMON_LD_SNIPPET) + assert RATETABLE_RULE in patched + # Inserted after the .data section's anchor, not the .dport0.data one + # (whose closing brace bounds the decoy block) + assert RATETABLE_RULE not in patched[: patched.index("} >dport0_0_seg")] + assert patched.index(RATETABLE_RULE) < patched.index("*(.data)") + # Idempotent on an already-patched script + assert relocate_ratetable(patched) == patched + + +def test_relocate_ratetable_requires_anchor() -> None: + with pytest.raises(RuntimeError, match="_data_start"): + relocate_ratetable("SECTIONS { }") + + +def test_testing_memory_patches_enlarge_segments() -> None: + patched = apply_testing_memory_patches( + _FLASH_LD_SNIPPET, ("dram0_0_seg", "irom0_0_seg") + ) + assert segment_length(patched, "dram0_0_seg") == 0x200000 + assert segment_length(patched, "irom0_0_seg") == 0x2000000 + # Untouched segments keep their sizes + assert segment_length(patched, "dport0_0_seg") == 0x10 + + +def test_testing_memory_patches_keep_ul_suffix() -> None: + """The common ld's preprocessed sizes carry a ul suffix; the patch must + replace only the hex digits, as testing_mode.py.script does.""" + patched = apply_testing_memory_patches(_COMMON_LD_MEMORY_SNIPPET, ("iram1_0_seg",)) + assert "len = 0x200000ul" in patched + assert segment_length(patched, "iram1_0_seg") == 0x200000 + + +def test_segment_length_requires_whole_name() -> None: + """A name must match its own line, never inside a longer segment name.""" + assert segment_length(_FLASH_LD_SNIPPET, "ram0_0_seg") is None + + +def test_testing_memory_patches_unknown_segment_raises() -> None: + with pytest.raises(RuntimeError, match="Unknown testing-mode segment"): + apply_testing_memory_patches("MEMORY { }", ("bogus_seg",)) + + +def test_segment_length() -> None: + assert segment_length(_FLASH_LD_SNIPPET, "irom0_0_seg") == 0xFEFF0 + assert segment_length(_FLASH_LD_SNIPPET, "missing_seg") is None + + +def test_testing_memory_patches_missing_segment_raises() -> None: + """A named segment the patch could not find raises instead of silently + keeping the real memory limits.""" + with pytest.raises(RuntimeError, match="dram0_0_seg"): + apply_testing_memory_patches("MEMORY { }", ("dram0_0_seg",)) + + +def test_board_build_covers_every_board() -> None: + """Every supported board has native build metadata (the table may carry + extras that BOARDS does not expose).""" + assert set(BOARDS) <= set(ESP8266_BOARD_BUILD) + + +def test_surgery_fingerprint_is_stable_and_sensitive(tmp_path) -> None: + """The properties the linker-script cache depends on: the fingerprint is + stable across calls and changes when the module's source changes.""" + + first = build_surgery.surgery_fingerprint() + assert first == build_surgery.surgery_fingerprint() + assert len(first) == 64 + int(first, 16) # sha256 hex digest + + # A modified copy of the module must fingerprint differently + copy = tmp_path / "build_surgery_variant.py" + copy.write_text( + Path(build_surgery.__file__).read_text(encoding="utf-8") + + "\nEXTRA_BEHAVIORAL_INPUT = 1\n", + encoding="utf-8", + ) + spec = importlib.util.spec_from_file_location("build_surgery_variant", copy) + variant = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = variant + try: + spec.loader.exec_module(variant) + assert variant.surgery_fingerprint() != first + finally: + del sys.modules[spec.name] + + +def test_testing_memory_patches_present_but_unselected_raises() -> None: + """A known segment left off the caller's list must fail, not silently + keep its real memory limit.""" + with pytest.raises(RuntimeError, match="not selected"): + apply_testing_memory_patches(_FLASH_LD_SNIPPET, ("dram0_0_seg",)) From c272c4c1a64d08547e510c3e9e6b90ccaadfda0d Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:44:42 +1000 Subject: [PATCH 40/65] [lvgl] Add table widget (#18422) Co-authored-by: Claude Sonnet 5 Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- esphome/components/lvgl/lvgl_esphome.cpp | 46 +++ esphome/components/lvgl/lvgl_esphome.h | 25 ++ esphome/components/lvgl/widgets/table.py | 280 ++++++++++++++++++ tests/components/lvgl/lvgl-package.yaml | 32 ++ tests/unit_tests/components/lvgl/__init__.py | 0 .../components/lvgl/test_table_codegen.py | 206 +++++++++++++ .../components/lvgl/test_table_config.py | 142 +++++++++ 7 files changed, 731 insertions(+) create mode 100644 esphome/components/lvgl/widgets/table.py create mode 100644 tests/unit_tests/components/lvgl/__init__.py create mode 100644 tests/unit_tests/components/lvgl/test_table_codegen.py create mode 100644 tests/unit_tests/components/lvgl/test_table_config.py diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index 22fccdd92a..684f472ebd 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -525,6 +525,52 @@ void IndicatorLine::update_length_() { } #endif +#ifdef USE_LVGL_TABLE +uint32_t lv_table_get_selected_row(lv_obj_t *obj) { + uint32_t row; + uint32_t column; + lv_table_get_selected_cell(obj, &row, &column); + return row; +} + +uint32_t lv_table_get_selected_column(lv_obj_t *obj) { + uint32_t row; + uint32_t column; + lv_table_get_selected_cell(obj, &row, &column); + return column; +} + +void LvTableType::set_obj(lv_obj_t *lv_obj) { + LvCompound::set_obj(lv_obj); + lv_obj_add_event_cb( + lv_obj, + [](lv_event_t *e) { + auto *table = static_cast(lv_event_get_user_data(e)); + table->update_column_widths_(); + }, + LV_EVENT_SIZE_CHANGED, this); +} + +void LvTableType::add_column_width_pct(uint32_t col, uint8_t pct) { + for (auto &i : this->column_pct_) { + if (i.col == col) { + i.pct = pct; + this->update_column_widths_(); + return; + } + } + this->column_pct_.push_back({col, pct}); + this->update_column_widths_(); +} + +void LvTableType::update_column_widths_() { + auto content_width = lv_obj_get_content_width(this->obj); + for (const auto &col : this->column_pct_) { + lv_table_set_column_width(this->obj, col.col, content_width * col.pct / 100); + } +} +#endif // USE_LVGL_TABLE + #ifdef USE_LVGL_KEY_LISTENER LVEncoderListener::LVEncoderListener(lv_indev_type_t type, uint16_t long_press_time, uint16_t long_press_repeat_time) { this->drv_ = lv_indev_create(); diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index 98b97e26d7..ceba786e43 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -58,6 +58,10 @@ lv_obj_t *lv_container_create(lv_obj_t *parent); void lv_scale_draw_event_cb(lv_event_t *e, int16_t range_start, int16_t range_end, lv_color_t color_start, lv_color_t color_end, int width, bool local); #endif +#ifdef USE_LVGL_TABLE +uint32_t lv_table_get_selected_row(lv_obj_t *obj); +uint32_t lv_table_get_selected_column(lv_obj_t *obj); +#endif #if LV_COLOR_DEPTH == 16 static const display::ColorBitness LV_BITNESS = display::ColorBitness::COLOR_BITNESS_565; #elif LV_COLOR_DEPTH == 32 @@ -511,6 +515,27 @@ class LvLineType : public LvCompound { FixedVector points_{}; }; #endif +#ifdef USE_LVGL_TABLE +// Unlike most size properties, lv_table_set_column_width() only accepts a literal pixel +// count, so percentage column widths must be recomputed by hand whenever the table's own +// content width changes. +class LvTableType : public LvCompound { + public: + void set_obj(lv_obj_t *lv_obj) override; + // count is the number of percentage-width columns, known at code-generation time. + void init_column_pct(size_t count) { this->column_pct_.init(count); } + void add_column_width_pct(uint32_t col, uint8_t pct); + + protected: + void update_column_widths_(); + + struct ColumnPct { + uint32_t col; + uint8_t pct; + }; + FixedVector column_pct_{}; +}; +#endif // USE_LVGL_TABLE #if defined(USE_LVGL_DROPDOWN) || defined(LV_USE_ROLLER) class LvSelectable : public LvCompound { public: diff --git a/esphome/components/lvgl/widgets/table.py b/esphome/components/lvgl/widgets/table.py new file mode 100644 index 0000000000..efae2be2be --- /dev/null +++ b/esphome/components/lvgl/widgets/table.py @@ -0,0 +1,280 @@ +from contextlib import ExitStack + +from esphome import automation +import esphome.codegen as cg +from esphome.components.const import CONF_ROWS +import esphome.config_validation as cv +from esphome.const import CONF_ID, CONF_ITEMS, CONF_ROW, CONF_TEXT, CONF_WIDTH +from esphome.core import ID +from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.schema_extractors import SCHEMA_EXTRACT +from esphome.types import ConfigFragmentType, ConfigType, SafeExpType + +from ..automation import action_to_code +from ..defines import CONF_COLUMN, CONF_MAIN, LValidator, literal +from ..lv_validation import lv_int, lv_text, pixels_or_percent, pixels_validator +from ..lvcode import LocalVariable, lv, lv_add, lv_expr +from ..types import LvCompound, LvType, ObjUpdateAction, lv_coord_t +from . import Widget, WidgetType, get_widgets +from .label import CONF_LABEL + +CONF_TABLE = "table" +CONF_CELLS = "cells" +CONF_COLUMNS = "columns" +CONF_ROW_COUNT = "row_count" +CONF_COLUMN_COUNT = "column_count" +CONF_MERGE_RIGHT = "merge_right" +CONF_TEXT_CROP = "text_crop" +CONF_SELECTED_ROW = "selected_row" +CONF_SELECTED_COLUMN = "selected_column" + +CELL_SCHEMA = cv.Schema( + { + cv.Optional(CONF_TEXT, default=""): lv_text, + # Not templatable: the value selects between two different LVGL calls + # (set/clear cell ctrl), so a runtime lambda can't be mapped to a single call. + cv.Optional(CONF_MERGE_RIGHT): cv.boolean, + cv.Optional(CONF_TEXT_CROP): cv.boolean, + } +) + +# A cell can be given as a bare piece of text, or a dict for more control +TABLE_CELL_SCHEMA = cv.maybe_simple_value(CELL_SCHEMA, key=CONF_TEXT) + +# A row can be given as a bare list of cells, or a dict for future extension +ROW_SCHEMA = cv.maybe_simple_value( + cv.Schema({cv.Required(CONF_CELLS): cv.ensure_list(TABLE_CELL_SCHEMA)}), + key=CONF_CELLS, +) + + +def _column_width_validator(value: ConfigFragmentType) -> int | float | list[str]: + """Like pixels_or_percent, but rejects negative widths, which would + defeat the 100%-total check and wrap around in the generated uint8_t pct.""" + if value == SCHEMA_EXTRACT: + return ["pixels", "..%"] + return cv.Any(pixels_validator, cv.percentage)(value) + + +column_width = LValidator( + _column_width_validator, + lv_coord_t, + retmapper=pixels_or_percent.retmapper, + animatable=True, +) + +COLUMN_SCHEMA = cv.Schema( + { + cv.Optional(CONF_WIDTH): column_width, + } +) + + +def _validate_table(config: ConfigType) -> ConfigType: + rows = config.get(CONF_ROWS) + min_row_count = len(rows) if rows else 0 + min_column_count = max(len(row[CONF_CELLS]) for row in rows) if rows else 0 + row_count = config.get(CONF_ROW_COUNT) + if row_count is not None and row_count < min_row_count: + raise cv.Invalid( + f"{CONF_ROW_COUNT} must be at least {min_row_count} to hold all the given rows", + path=[CONF_ROW_COUNT], + ) + column_count = config.get(CONF_COLUMN_COUNT) + if column_count is not None and column_count < min_column_count: + raise cv.Invalid( + f"{CONF_COLUMN_COUNT} must be at least {min_column_count} to hold all the cells in a row", + path=[CONF_COLUMN_COUNT], + ) + column_count = column_count if column_count is not None else min_column_count + columns = config.get(CONF_COLUMNS) + if columns and column_count and len(columns) > column_count: + raise cv.Invalid( + f"{CONF_COLUMNS} defines {len(columns)} columns, but the table has only {column_count}", + path=[CONF_COLUMNS], + ) + total_pct = sum( + width + for column in columns or () + if isinstance((width := column.get(CONF_WIDTH)), float) + ) + if total_pct > 1.0: + raise cv.Invalid( + f"{CONF_COLUMNS} percentage widths add up to {total_pct * 100:.0f}%, which exceeds 100%", + path=[CONF_COLUMNS], + ) + return config + + +TABLE_SCHEMA = cv.Schema( + { + cv.Optional(CONF_ROWS): cv.ensure_list(ROW_SCHEMA), + cv.Optional(CONF_ROW_COUNT): cv.positive_int, + cv.Optional(CONF_COLUMN_COUNT): cv.positive_int, + cv.Optional(CONF_COLUMNS): cv.ensure_list(COLUMN_SCHEMA), + cv.Optional(CONF_SELECTED_ROW): lv_int, + cv.Optional(CONF_SELECTED_COLUMN): lv_int, + } +).add_extra(_validate_table) + +lv_table_t = LvType( + "LvTableType", + parents=(LvCompound,), + largs=[(cg.uint32, "row"), (cg.uint32, "column")], + lvalue=lambda w: [ + lv_expr.table_get_selected_row(w.obj), + lv_expr.table_get_selected_column(w.obj), + ], + has_on_value=True, +) + + +async def set_cell_ctrl( + w: Widget, row: SafeExpType, column: SafeExpType, cell: ConfigType +) -> None: + for key, ctrl in ( + (CONF_MERGE_RIGHT, "LV_TABLE_CELL_CTRL_MERGE_RIGHT"), + (CONF_TEXT_CROP, "LV_TABLE_CELL_CTRL_TEXT_CROP"), + ): + if key not in cell: + continue + if cell[key]: + lv.table_set_cell_ctrl(w.obj, row, column, literal(ctrl)) + else: + lv.table_clear_cell_ctrl(w.obj, row, column, literal(ctrl)) + + +async def set_selected_cell(w: Widget, config: ConfigType) -> None: + selected_row = config.get(CONF_SELECTED_ROW) + selected_column = config.get(CONF_SELECTED_COLUMN) + if selected_row is None and selected_column is None: + return + # LV_TABLE_CELL_NONE selects the whole column/row when only one index is given + row_value = ( + await lv_int.process(selected_row) + if selected_row is not None + else literal("LV_TABLE_CELL_NONE") + ) + column_value = ( + await lv_int.process(selected_column) + if selected_column is not None + else literal("LV_TABLE_CELL_NONE") + ) + lv.table_set_selected_cell(w.obj, row_value, column_value) + + +TABLE_MODIFY_SCHEMA = cv.Schema( + { + cv.Optional(CONF_SELECTED_ROW): lv_int, + cv.Optional(CONF_SELECTED_COLUMN): lv_int, + } +) + + +class TableType(WidgetType): + def __init__(self): + super().__init__( + CONF_TABLE, + lv_table_t, + (CONF_MAIN, CONF_ITEMS), + TABLE_SCHEMA, + modify_schema=TABLE_MODIFY_SCHEMA, + ) + + def get_uses(self) -> tuple[str]: + return (CONF_LABEL,) + + async def to_code(self, w: Widget, config: dict) -> None: + rows = config.get(CONF_ROWS) + row_count = config.get(CONF_ROW_COUNT) + column_count = config.get(CONF_COLUMN_COUNT) + if rows is not None: + if row_count is None: + row_count = len(rows) + if column_count is None: + column_count = max((len(row[CONF_CELLS]) for row in rows), default=0) + if row_count is not None: + lv.table_set_row_count(w.obj, row_count) + if column_count is not None: + lv.table_set_column_count(w.obj, column_count) + columns = config.get(CONF_COLUMNS, ()) + pct_column_count = sum( + 1 for column in columns if isinstance(column.get(CONF_WIDTH), float) + ) + if pct_column_count: + lv_add(w.var.init_column_pct(pct_column_count)) + for index, column in enumerate(columns): + if (width := column.get(CONF_WIDTH)) is None: + continue + if isinstance(width, float): + # A percentage: column_width validation leaves it as a 0.0-1.0 + # fraction. LVGL's table widget only accepts a literal pixel width, so + # the actual width is recomputed at runtime from the table's own size. + lv_add(w.var.add_column_width_pct(index, round(width * 100))) + else: + lv.table_set_column_width( + w.obj, index, await column_width.process(width) + ) + for row_index, row in enumerate(rows or ()): + for column_index, cell in enumerate(row[CONF_CELLS]): + lv.table_set_cell_value( + w.obj, + row_index, + column_index, + await lv_text.process(cell[CONF_TEXT]), + ) + await set_cell_ctrl(w, row_index, column_index, cell) + await set_selected_cell(w, config) + + +table_spec = TableType() + + +@automation.register_action( + "lvgl.table.cell.update", + ObjUpdateAction, + cv.Schema( + { + cv.Required(CONF_ID): cv.use_id(lv_table_t), + cv.Required(CONF_ROW): lv_int, + cv.Required(CONF_COLUMN): lv_int, + cv.Optional(CONF_TEXT): lv_text, + cv.Optional(CONF_MERGE_RIGHT): cv.boolean, + cv.Optional(CONF_TEXT_CROP): cv.boolean, + } + ).add_extra(cv.has_at_least_one_key(CONF_TEXT, CONF_MERGE_RIGHT, CONF_TEXT_CROP)), + synchronous=True, +) +async def table_cell_update_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + widgets = await get_widgets(config) + + async def do_update(w: Widget): + row = await lv_int.process(config[CONF_ROW]) + column = await lv_int.process(config[CONF_COLUMN]) + fields_set = sum( + key in config for key in (CONF_TEXT, CONF_MERGE_RIGHT, CONF_TEXT_CROP) + ) + with ExitStack() as stack: + if fields_set > 1: + # row/column feed more than one generated call below: cache them in + # local variables so a !lambda value is only evaluated once. + row = stack.enter_context( + LocalVariable("row", cg.int_, row, modifier="") + ) + column = stack.enter_context( + LocalVariable("column", cg.int_, column, modifier="") + ) + if CONF_TEXT in config: + lv.table_set_cell_value( + w.obj, row, column, await lv_text.process(config[CONF_TEXT]) + ) + await set_cell_ctrl(w, row, column, config) + + return await action_to_code( + widgets, do_update, action_id, template_arg, args, config + ) diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index c78e910bc8..57be4e9043 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -1181,6 +1181,38 @@ lvgl: - logger.log: format: "bar value %f" args: [x] + - table: + id: table_id + align: top_mid + y: 60 + columns: + - width: 40% + - width: 80 + rows: + - ["Name", "Value"] + - cells: + - text: "Temp" + merge_right: true + - text: "22.5" + text_crop: true + selected_row: 0 + on_value: + then: + - logger.log: + format: "table selected row %u col %u" + args: [row, column] + on_click: + then: + - lvgl.table.cell.update: + id: table_id + row: 1 + column: 1 + text: !lambda return str_sprintf("%.1f", (float) rand() / RAND_MAX * 100); + merge_right: false + - lvgl.table.update: + id: table_id + selected_row: !lambda return (int) ((float) rand() / RAND_MAX * 2); + selected_column: 0 - line: id: lv_line_id align: center diff --git a/tests/unit_tests/components/lvgl/__init__.py b/tests/unit_tests/components/lvgl/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/lvgl/test_table_codegen.py b/tests/unit_tests/components/lvgl/test_table_codegen.py new file mode 100644 index 0000000000..390f67dffc --- /dev/null +++ b/tests/unit_tests/components/lvgl/test_table_codegen.py @@ -0,0 +1,206 @@ +"""Tests for the LVGL table widget's C++ code generation.""" + +from __future__ import annotations + +import pytest + +from esphome.automation import ACTION_REGISTRY +from esphome.components.lvgl.defines import set_widgets_completed +from esphome.components.lvgl.lvcode import LvContext +from esphome.components.lvgl.schemas import container_schema +from esphome.components.lvgl.trigger import generate_triggers +from esphome.components.lvgl.widgets import Widget, widget_to_code +from esphome.components.lvgl.widgets.table import table_spec +from esphome.const import ( + CONF_AUTOMATION_ID, + CONF_ON_VALUE, + CONF_THEN, + CONF_TRIGGER_ID, + CONF_TYPE_ID, +) +from esphome.core import CORE, ID +from esphome.cpp_generator import MockObj, TemplateArguments +from esphome.yaml_util import make_data_base + + +async def _create_table(raw_config: dict) -> Widget: + """Validate `raw_config` as a table widget and generate its creation code.""" + config = container_schema(table_spec)(raw_config) + parent = MockObj("parent_obj") + async with LvContext(): + return await widget_to_code(config, table_spec, parent) + + +def _statements() -> list[str]: + return [str(s) for s in CORE.main_statements] + + +@pytest.mark.asyncio +async def test_create_table_sets_row_and_column_count(setup_core) -> None: + await _create_table( + {"id": "table_counts", "rows": [["Name", "Value"], ["Temp", "22.5"]]} + ) + statements = _statements() + assert any("lv_table_set_row_count(table_counts->obj, 2)" in s for s in statements) + assert any( + "lv_table_set_column_count(table_counts->obj, 2)" in s for s in statements + ) + + +@pytest.mark.asyncio +async def test_create_table_writes_cell_values(setup_core) -> None: + await _create_table({"id": "table_cells", "rows": [["Name", "Value"]]}) + statements = _statements() + assert any( + 'lv_table_set_cell_value(table_cells->obj, 0, 0, "Name")' in s + for s in statements + ) + assert any( + 'lv_table_set_cell_value(table_cells->obj, 0, 1, "Value")' in s + for s in statements + ) + + +@pytest.mark.asyncio +async def test_create_table_sets_cell_control_flags(setup_core) -> None: + await _create_table( + { + "id": "table_ctrl", + "rows": [ + { + "cells": [ + {"text": "wide", "merge_right": True}, + {"text": "cropped", "text_crop": True}, + ] + } + ], + } + ) + statements = _statements() + assert any( + "lv_table_set_cell_ctrl(table_ctrl->obj, 0, 0, LV_TABLE_CELL_CTRL_MERGE_RIGHT)" + in s + for s in statements + ) + assert any( + "lv_table_set_cell_ctrl(table_ctrl->obj, 0, 1, LV_TABLE_CELL_CTRL_TEXT_CROP)" + in s + for s in statements + ) + # text_crop omitted for cell 0: no clear_cell_ctrl() should be emitted. + assert not any( + "table_ctrl->obj, 0, 0, LV_TABLE_CELL_CTRL_TEXT_CROP" in s for s in statements + ) + + +@pytest.mark.asyncio +async def test_pixel_column_width_calls_lvgl_directly(setup_core) -> None: + await _create_table({"id": "table_px", "columns": [{"width": 96}]}) + statements = _statements() + assert any( + "lv_table_set_column_width(table_px->obj, 0, 96)" in s for s in statements + ) + + +@pytest.mark.asyncio +async def test_percent_column_width_uses_the_dynamic_helper(setup_core) -> None: + """Regression test: lv_table_set_column_width() only accepts a literal + pixel count, so a percentage width must not be passed to it directly - + it has to go through the LvTableType helper that recomputes it at + runtime from the table's actual content width. + """ + await _create_table({"id": "table_pct", "columns": [{"width": "40%"}]}) + statements = _statements() + assert any("table_pct->init_column_pct(1)" in s for s in statements) + assert any("table_pct->add_column_width_pct(0, 40)" in s for s in statements) + assert not any( + "lv_table_set_column_width(table_pct->obj, 0" in s for s in statements + ) + + +@pytest.mark.asyncio +async def test_selected_cell_with_both_indices(setup_core) -> None: + await _create_table( + {"id": "table_sel_both", "selected_row": 1, "selected_column": 2} + ) + statements = _statements() + assert any( + "lv_table_set_selected_cell(table_sel_both->obj, 1, 2)" in s for s in statements + ) + + +@pytest.mark.asyncio +async def test_selected_cell_with_only_row_selects_whole_row(setup_core) -> None: + await _create_table({"id": "table_sel_row", "selected_row": 1}) + statements = _statements() + assert any( + "lv_table_set_selected_cell(table_sel_row->obj, 1, LV_TABLE_CELL_NONE)" in s + for s in statements + ) + + +@pytest.mark.asyncio +async def test_selected_cell_omitted_entirely_when_not_configured( + setup_core, +) -> None: + await _create_table({"id": "table_no_selection", "rows": [["a"]]}) + statements = _statements() + assert not any("lv_table_set_selected_cell" in s for s in statements) + + +@pytest.mark.asyncio +async def test_cell_update_action_writes_only_the_given_fields(setup_core) -> None: + await _create_table({"id": "table_update", "rows": [["a", "b"], ["c", "d"]]}) + set_widgets_completed(True) + # Only inspect statements emitted by the action below, not by creation. + before = len(_statements()) + + entry = ACTION_REGISTRY["lvgl.table.cell.update"] + config = entry.schema( + {"id": "table_update", "row": 1, "column": 1, "text": "new value"} + ) + action_id = ID("test_cell_update_action", is_declaration=True, type=entry.type_id) + await entry.coroutine_fun(config, action_id, TemplateArguments(), []) + + statements = _statements()[before:] + assert any( + 'lv_table_set_cell_value(table_update->obj, 1, 1, "new value")' in s + for s in statements + ) + # Neither control flag was specified, so neither call should be emitted. + assert not any("LV_TABLE_CELL_CTRL" in s for s in statements) + + +@pytest.mark.asyncio +async def test_on_value_registers_a_value_changed_event_callback(setup_core) -> None: + config = container_schema(table_spec)( + { + "id": "table_on_value", + "rows": [["a"]], + "on_value": [ + {"lambda": make_data_base("id(table_on_value).get_selected_row();")} + ], + } + ) + # Auto-generated IDs (trigger/automation/action) are normally resolved to + # unique names by esphome's full config pass before code generation; do + # that by hand here since this test only exercises the widget/trigger + # codegen slice in isolation. + automation_conf = config[CONF_ON_VALUE][0] + automation_conf[CONF_TRIGGER_ID].resolve([]) + automation_conf[CONF_AUTOMATION_ID].resolve([]) + automation_conf[CONF_THEN][0][CONF_TYPE_ID].resolve([]) + + parent = MockObj("parent_obj") + async with LvContext(): + await widget_to_code(config, table_spec, parent) + set_widgets_completed(True) + await generate_triggers() + + statements = _statements() + assert any( + "table_on_value->obj" in s + and "add_event_cb" in s + and "LV_EVENT_VALUE_CHANGED" in s + for s in statements + ) diff --git a/tests/unit_tests/components/lvgl/test_table_config.py b/tests/unit_tests/components/lvgl/test_table_config.py new file mode 100644 index 0000000000..047d1781ae --- /dev/null +++ b/tests/unit_tests/components/lvgl/test_table_config.py @@ -0,0 +1,142 @@ +"""Tests for the LVGL table widget's configuration validation.""" + +from __future__ import annotations + +import pytest + +from esphome import config_validation as cv +from esphome.automation import ACTION_REGISTRY +from esphome.components.lvgl.widgets.table import ( + CONF_MERGE_RIGHT, + CONF_TEXT_CROP, + TABLE_SCHEMA, +) + + +def test_minimal_config_is_valid() -> None: + assert TABLE_SCHEMA({}) == {} + + +def test_row_shorthand_expands_to_plain_cells() -> None: + config = TABLE_SCHEMA({"rows": [["Name", "Value"]]}) + [row] = config["rows"] + assert row["cells"] == [{"text": "Name"}, {"text": "Value"}] + + +def test_row_dict_form_with_cell_overrides() -> None: + config = TABLE_SCHEMA( + { + "rows": [ + { + "cells": [ + "Temp", + {"text": "22.5", "text_crop": True, "merge_right": True}, + ] + } + ] + } + ) + [row] = config["rows"] + assert row["cells"][0] == {"text": "Temp"} + assert row["cells"][1] == { + "text": "22.5", + "merge_right": True, + "text_crop": True, + } + + +def test_row_count_defaults_are_not_injected_by_the_schema() -> None: + # Inference of row/column counts from `rows` happens at code generation + # time, not during validation - the schema should leave them unset. + config = TABLE_SCHEMA({"rows": [["a", "b"], ["c"]]}) + assert "row_count" not in config + assert "column_count" not in config + + +def test_explicit_row_and_column_count_are_kept() -> None: + config = TABLE_SCHEMA({"row_count": 5, "column_count": 3}) + assert config["row_count"] == 5 + assert config["column_count"] == 3 + + +def test_row_count_too_small_for_given_rows_raises() -> None: + with pytest.raises(cv.Invalid, match="row_count"): + TABLE_SCHEMA({"rows": [["a"], ["b"], ["c"]], "row_count": 2}) + + +def test_column_count_too_small_for_given_cells_raises() -> None: + with pytest.raises(cv.Invalid, match="column_count"): + TABLE_SCHEMA({"rows": [["a", "b", "c"]], "column_count": 2}) + + +def test_columns_list_longer_than_column_count_raises() -> None: + with pytest.raises(cv.Invalid, match="columns"): + TABLE_SCHEMA( + { + "column_count": 1, + "columns": [{"width": 10}, {"width": 20}], + } + ) + + +def test_columns_list_matching_inferred_column_count_is_valid() -> None: + config = TABLE_SCHEMA( + { + "rows": [["a", "b"]], + "columns": [{"width": 10}, {"width": 20}], + } + ) + assert [c["width"] for c in config["columns"]] == [10, 20] + + +@pytest.mark.parametrize( + ("width", "expected"), + [ + (100, 100), + ("50%", 0.5), + ("32px", 32), + ], +) +def test_column_width_accepts_pixels_and_percent(width, expected) -> None: + config = TABLE_SCHEMA({"columns": [{"width": width}]}) + assert config["columns"][0]["width"] == expected + + +def test_columns_percent_widths_summing_over_100_percent_raises() -> None: + with pytest.raises(cv.Invalid, match="columns"): + TABLE_SCHEMA({"columns": [{"width": "60%"}, {"width": "50%"}]}) + + +def test_columns_percent_widths_summing_to_100_percent_is_valid() -> None: + config = TABLE_SCHEMA({"columns": [{"width": "60%"}, {"width": "40%"}]}) + assert [c["width"] for c in config["columns"]] == [0.6, 0.4] + + +def test_columns_mixed_pixel_and_percent_widths_ignore_pixels_in_the_total() -> None: + # Pixel widths aren't part of the percentage budget, so they shouldn't + # count towards the 100% limit. + config = TABLE_SCHEMA( + {"columns": [{"width": 200}, {"width": "80%"}, {"width": "20%"}]} + ) + assert [c["width"] for c in config["columns"]] == [200, 0.8, 0.2] + + +def test_selected_row_and_selected_column_are_independently_optional() -> None: + config = TABLE_SCHEMA({"selected_row": 1}) + assert config["selected_row"] == 1 + assert "selected_column" not in config + + +def test_cell_update_action_requires_at_least_one_field() -> None: + entry = ACTION_REGISTRY["lvgl.table.cell.update"] + with pytest.raises(cv.Invalid): + entry.schema({"id": "some_table", "row": 0, "column": 0}) + + +def test_cell_update_action_accepts_a_single_field() -> None: + entry = ACTION_REGISTRY["lvgl.table.cell.update"] + config = entry.schema( + {"id": "some_table", "row": 0, "column": 0, "merge_right": True} + ) + assert config[CONF_MERGE_RIGHT] is True + assert CONF_TEXT_CROP not in config From 370e8fffed7a72e1d804827bdf8bebea4d6ddfc6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" <3060199+jesserockz@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:16:02 -0500 Subject: [PATCH 41/65] [core] Add the arduino toolchain seam and validate --toolchain on every platform (#18556) --- esphome/__main__.py | 17 +++--- esphome/compiled_config.py | 15 ++++++ esphome/components/esp32/__init__.py | 18 ++----- esphome/components/esp8266/__init__.py | 3 ++ esphome/components/host/__init__.py | 1 + esphome/components/libretiny/__init__.py | 3 +- esphome/components/nrf52/__init__.py | 11 ++-- esphome/components/rp2/__init__.py | 1 + esphome/config_validation.py | 62 ++++++++++++++++++++++ esphome/const.py | 8 +++ esphome/core/__init__.py | 16 ++++++ esphome/core/config.py | 50 +++++++++++++---- tests/component_tests/esp32/test_esp32.py | 14 +++++ tests/unit_tests/core/test_config.py | 49 +++++++++++++++++ tests/unit_tests/test_compiled_config.py | 31 +++++++++++ tests/unit_tests/test_config_validation.py | 45 ++++++++++++++++ tests/unit_tests/test_core.py | 18 +++++++ tests/unit_tests/test_main.py | 36 +++++++++++++ tests/unit_tests/test_nrf52_framework.py | 15 ++++-- 19 files changed, 370 insertions(+), 43 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 0da86b3ec0..632d2ba3d0 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2734,10 +2734,14 @@ def run_esphome(argv): # Skipped when -s overrides are passed, since the cache was written # against the previous substitution set. config: ConfigType | None = None - cache_eligible = ( + cache_write_eligible = ( args.command in ("upload", "logs") and not command_line_substitutions ) - if cache_eligible: + # An explicit --toolchain must re-run the per-platform validators, so + # gate only the cache read; the refresh below saves the result unless + # the sidecar records a different toolchain. + cache_read_eligible = cache_write_eligible and args.toolchain is None + if cache_read_eligible: from esphome.compiled_config import load_compiled_config config = load_compiled_config(conf_path) @@ -2761,17 +2765,14 @@ def run_esphome(argv): return 2 CORE.config = config - # Fallback for platforms whose validators didn't set the toolchain - # (only the esp32 component reads esp32.framework.toolchain). All - # other platforms only support PlatformIO today. Must run before the - # cache refresh below so its sidecar records the same toolchain a - # compile would. + # The cache fast path skips validation, and legacy sidecars lack the + # toolchain field. Must run before the cache refresh below. if CORE.toolchain is None: CORE.toolchain = Toolchain.PLATFORMIO # Refresh the cache so the next upload/logs hits the fast path # instead of re-running read_config. - if cache_eligible and cache_missed: + if cache_write_eligible and cache_missed: from esphome.compiled_config import save_compiled_config_and_sidecar save_compiled_config_and_sidecar(config) diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py index be03eea965..0d855d71db 100644 --- a/esphome/compiled_config.py +++ b/esphome/compiled_config.py @@ -100,6 +100,21 @@ def _refresh_sidecar() -> bool: ) return False if old is not None and old.can_apply_to_core(): + if ( + old.toolchain is not None + and CORE.toolchain is not None + and old.toolchain != CORE.toolchain.value + ): + # Platforms normalize toolchain-sensitive keys differently; + # never cache a config validated under a different toolchain + # than the compile's + _LOGGER.debug( + "Not caching: config validated with toolchain %r but the " + "last compile used %r", + CORE.toolchain.value, + old.toolchain, + ) + return False # Compile-written; nothing to refresh. return True if CORE.build_path is not None and CORE.build_path.exists(): diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 073d87402a..bc91f29a42 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1105,19 +1105,11 @@ def _check_esp_idf_versions(config: ConfigType) -> ConfigType: return config -def _validate_toolchain(value) -> Toolchain: - return Toolchain( - cv.one_of(Toolchain.PLATFORMIO, Toolchain.ESP_IDF, lower=True)(value) - ) - - -def _resolve_toolchain(value: ConfigType) -> ConfigType: - # Resolve toolchain: CLI (already on CORE.toolchain) > YAML > default. - # Runs before _detect_variant so downstream validators can rely on - # CORE.toolchain instead of re-resolving it from the config dict. - if CORE.toolchain is None: - CORE.toolchain = value.get(CONF_TOOLCHAIN, Toolchain.ESP_IDF) - return value +_TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.ESP_IDF) +_validate_toolchain = cv.toolchain_enum(_TOOLCHAINS) +# Runs before _detect_variant so downstream validators can rely on +# CORE.toolchain instead of re-resolving it from the config dict. +_resolve_toolchain = cv.resolve_toolchain("ESP32", _TOOLCHAINS, Toolchain.ESP_IDF) def _check_versions(config: ConfigType) -> ConfigType: diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 75483c5293..6f29cd7774 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -247,6 +247,9 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_SCANF_FLOAT): cv.boolean, } ), + # Until the native toolchain lands, PlatformIO is the only backend; + # reject a --toolchain this platform cannot serve yet. + cv.require_platformio_toolchain("ESP8266"), set_core_data, ) diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index c5846f5406..401bba5118 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -37,6 +37,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_MAC_ADDRESS, default="98:35:69:ab:f6:79"): cv.mac_address, } ), + cv.require_platformio_toolchain("host"), set_core_data, ) diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index c56cc48055..50dc787799 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -300,7 +300,7 @@ FRAMEWORK_SCHEMA = cv.All( _check_debug_order, ) -CONFIG_SCHEMA = cv.All(_notify_old_style) +CONFIG_SCHEMA = cv.All(_notify_old_style, cv.require_platformio_toolchain("LibreTiny")) BASE_SCHEMA = cv.Schema( { @@ -314,6 +314,7 @@ BASE_SCHEMA = cv.Schema( ) BASE_SCHEMA.add_extra(_detect_variant) +BASE_SCHEMA.add_extra(cv.require_platformio_toolchain("LibreTiny")) BASE_SCHEMA.add_extra(_update_core_data) diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 2d25558254..aeeaba0c11 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -125,10 +125,8 @@ def set_core_data(config: ConfigType) -> ConfigType: return config -def _resolve_toolchain(config: ConfigType) -> ConfigType: - if CORE.toolchain is None: - CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.SDK_NRF) - return config +_TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.SDK_NRF) +_resolve_toolchain = cv.resolve_toolchain("nRF52", _TOOLCHAINS, Toolchain.SDK_NRF) def set_framework(config: ConfigType) -> ConfigType: @@ -170,10 +168,7 @@ BOOTLOADERS = [ ] -def _validate_toolchain(value) -> Toolchain: - return Toolchain( - cv.one_of(Toolchain.PLATFORMIO, Toolchain.SDK_NRF, lower=True)(value) - ) +_validate_toolchain = cv.toolchain_enum(_TOOLCHAINS) def _detect_bootloader(config: ConfigType) -> ConfigType: diff --git a/esphome/components/rp2/__init__.py b/esphome/components/rp2/__init__.py index ed975ec01a..dae7df26c3 100644 --- a/esphome/components/rp2/__init__.py +++ b/esphome/components/rp2/__init__.py @@ -312,6 +312,7 @@ CONFIG_SCHEMA = cv.All( ), cv.has_at_least_one_key(CONF_BOARD, CONF_VARIANT), _detect_variant, + cv.require_platformio_toolchain("RP2"), set_core_data, ) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index f455c7b8bf..98001d5d5b 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -53,6 +53,7 @@ from esphome.const import ( CONF_SETUP_PRIORITY, CONF_STATE_TOPIC, CONF_SUBSCRIBE_QOS, + CONF_TOOLCHAIN, CONF_TOPIC, CONF_TYPE, CONF_TYPE_ID, @@ -75,6 +76,7 @@ from esphome.const import ( TYPE_GIT, TYPE_LOCAL, Framework, + Toolchain, __version__ as ESPHOME_VERSION, ) from esphome.core import ( @@ -106,6 +108,9 @@ from esphome.util import parse_esphome_version # noqa: F401 from esphome.voluptuous_schema import _Schema from esphome.yaml_util import SensitiveStr, make_data_base +if typing.TYPE_CHECKING: + from esphome.types import ConfigType + _LOGGER = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -2532,6 +2537,63 @@ def platformio_version_constraint(value): return constraints +def _check_supported_toolchain( + platform_name: str, supported: tuple[Toolchain, ...] +) -> None: + """Raise when the resolved ``CORE.toolchain`` is not in ``supported`` + (one message shape for every platform).""" + toolchain = CORE.toolchain + if toolchain is None: + # A caller ran the check before resolving; an ordering bug, not a + # user error + raise Invalid(f"Toolchain was not resolved before {platform_name} validation") + if toolchain not in supported: + names = ", ".join(f"'{tc.value}'" for tc in supported) + raise Invalid( + f"Unsupported toolchain " + f"'{toolchain.value}' for " + f"{platform_name}. Supported: {names}." + ) + + +def toolchain_enum(supported: tuple[Toolchain, ...]) -> Callable[[str], Toolchain]: + """Schema validator for a platform's ``toolchain`` config key.""" + + def validator(value: str) -> Toolchain: + return Toolchain(one_of(*supported, lower=True)(value)) + + return validator + + +def resolve_toolchain( + platform_name: str, supported: tuple[Toolchain, ...], default: Toolchain +) -> Callable[[ConfigType], ConfigType]: + """Resolve ``CORE.toolchain`` (CLI > YAML > default) and reject one the + platform cannot serve. + + Add to the platform's validation chain before anything that reads + ``CORE.toolchain``. + """ + + def validator(config: ConfigType) -> ConfigType: + if CORE.toolchain is None: + CORE.toolchain = config.get(CONF_TOOLCHAIN, default) + _check_supported_toolchain(platform_name, supported) + return config + + return validator + + +def require_platformio_toolchain( + platform_name: str, +) -> Callable[[ConfigType], ConfigType]: + """Reject a CLI-selected toolchain other than PlatformIO, for platforms + with only the PlatformIO backend.""" + return resolve_toolchain( + platform_name, (Toolchain.PLATFORMIO,), Toolchain.PLATFORMIO + ) + + def require_framework_version( *, max_version=False, diff --git a/esphome/const.py b/esphome/const.py index 0dd948544f..6f83f0c937 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -21,6 +21,14 @@ class Toolchain(StrEnum): PLATFORMIO = "platformio" ESP_IDF = "esp-idf" SDK_NRF = "sdk-nrf" + # ESP8266: the Arduino core built directly (no PlatformIO) + ARDUINO = "arduino" + + +# Toolchains that drive their build natively and never read platformio.ini. +# SDK_NRF is absent on purpose: the zephyr backend keeps consuming +# platformio_options. +NATIVE_TOOLCHAINS = frozenset({Toolchain.ESP_IDF, Toolchain.ARDUINO}) class Platform(StrEnum): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 0f1ac9213e..2ec2a08e83 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -21,6 +21,7 @@ from esphome.const import ( KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + NATIVE_TOOLCHAINS, PLATFORM_BK72XX, PLATFORM_ESP32, PLATFORM_ESP8266, @@ -982,6 +983,19 @@ class EsphomeCore: def using_toolchain_sdk_nrf(self): return self.toolchain == Toolchain.SDK_NRF + @property + def using_toolchain_arduino(self): + """The native ESP8266 Arduino build toolchain (unlike + ``using_arduino``, which is the target framework).""" + return self.toolchain == Toolchain.ARDUINO + + @property + def using_native_toolchain(self): + """Whether the selected toolchain builds natively, without reading + ``platformio.ini`` (see ``NATIVE_TOOLCHAINS`` in ``esphome.const``; + keep its membership in sync with ``write_cpp_file``'s dispatch).""" + return self.toolchain in NATIVE_TOOLCHAINS + @property def using_zephyr(self): return self.target_framework == "zephyr" @@ -1095,6 +1109,8 @@ class EsphomeCore: return build_flag def add_build_unflag(self, build_unflag: str) -> None: + # No warning for using_toolchain_arduino: the native ESP8266 build + # honors build_unflags (token-level, matching PlatformIO). if self.using_toolchain_esp_idf: # The native ESP-IDF build generator does not consume build_unflags _LOGGER.warning( diff --git a/esphome/core/config.py b/esphome/core/config.py index 1095a4886e..472ca64c9a 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -555,12 +555,24 @@ def _add_library_str(lib: str) -> None: cg.add_library(lib, None) +# platformio_options keys the native ESP8266 Arduino generator (a later PR +# in this chain) will honor; its ignored-option warning will consume the same +# list so the two cannot drift +NATIVE_ARDUINO_PIO_OPTIONS = frozenset({"board_build.f_cpu", "board_build.ldscript"}) +# The full set that survives into CORE.platformio_options under the native +# arduino toolchain: lib_ignore is the only specially-translated key below +# that is stored rather than translated away. Consumed by the esp8266 native +# backend (later in this chain) for its ignored-option warning; defined here +# so it stays adjacent to the routing. +NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS = NATIVE_ARDUINO_PIO_OPTIONS | {"lib_ignore"} + + @coroutine_with_priority(CoroPriority.FINAL) async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> None: - if CORE.using_toolchain_esp_idf: - # The native ESP-IDF build doesn't read platformio.ini; honor the - # options with a native equivalent and warn about the rest, which - # would otherwise be silently ignored. + if CORE.using_native_toolchain: + # The native builds don't read platformio.ini; honor the options + # with a native equivalent and warn about the rest, which would + # otherwise be silently ignored. for key, val in pio_options.items(): vals = [val] if isinstance(val, str) else val if key == CONF_BUILD_FLAGS: @@ -573,23 +585,41 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No ) for flag in vals: cg.add_build_flag(flag) + elif key == "build_unflags": + # Native equivalent: add_build_unflag (honored token-level by + # the arduino generator; the IDF generator warns there) + for flag in vals: + CORE.add_build_unflag(flag) elif key == "lib_deps": - # Routed through the regular library mechanism so the libraries - # are converted to IDF components like any other PIO library + # Routed through the regular library mechanism so the + # libraries reach the native backend's converter (IDF + # components, or the ESP8266 native library resolution) for lib in vals: _add_library_str(lib) elif key == "lib_ignore": - # Read by the PIO-library-to-IDF-component conversion - # (generate_idf_components); filters both top-level libraries - # and dependencies discovered during conversion + # Read by the shared library conversion (lib_ignore_set in + # platformio/library.py); filters top-level libraries and + # discovered dependencies cg.add_platformio_option(key, vals) + elif ( + key in NATIVE_ARDUINO_PIO_OPTIONS + and CORE.using_toolchain_arduino + and vals + ): + # The esp8266 native generator reads these as scalars; the + # schema also permits the list form, where the last value + # wins like a later platformio.ini line (an empty list falls + # through to the ignored-option warning). Other native + # toolchains have no equivalent and fall through too. + cg.add_platformio_option(key, vals[-1]) elif key != "upload_speed": # upload_speed needs no handling: it is read from the raw # config at upload time (upload_using_esptool) _LOGGER.warning( "esphome->platformio_options->%s is ignored when building with " - "the native ESP-IDF toolchain", + "the native '%s' toolchain", key, + CORE.toolchain.value, ) return # Add includes at the very end, so that they override everything diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 0ffbe16a17..297844b4e6 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -132,6 +132,20 @@ def test_esp32_rejects_unsupported_toolchains( CONFIG_SCHEMA({"variant": VARIANT_ESP32, "toolchain": config_toolchain}) +def test_esp32_rejects_unsupported_cli_toolchain( + set_core_config: SetCoreConfigCallable, +) -> None: + """A --toolchain the platform cannot serve fails instead of silently + building with PlatformIO (the CLI path bypasses the YAML validator).""" + set_core_config(PlatformFramework.ESP32_IDF) + + from esphome.components.esp32 import CONFIG_SCHEMA + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): + CONFIG_SCHEMA({"variant": VARIANT_ESP32}) + + @pytest.mark.parametrize( ("config", "error_match"), [ diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index e09edd7f26..e620f8ec7f 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1285,6 +1285,7 @@ async def test_add_platformio_options_native_idf( await config._add_platformio_options( { "build_flags": "-DSINGLE_FLAG", # string and list forms both valid + "build_unflags": ["-Os"], "lib_deps": ["bblanchon/ArduinoJson@7.4.2"], "lib_ignore": "libsodium", "upload_speed": "115200", @@ -1294,6 +1295,7 @@ async def test_add_platformio_options_native_idf( assert "-DSINGLE_FLAG" in CORE.build_flags assert "ArduinoJson" in CORE.platformio_libraries + assert "-Os" in CORE.build_unflags # lib_ignore is stored (listified) for generate_idf_components to read; # nothing else lands in platformio_options on the native toolchain. assert CORE.platformio_options == {"lib_ignore": ["libsodium"]} @@ -1389,3 +1391,50 @@ def test_esphome_build_internals_are_yaml_only() -> None: assert markers[field].visibility is cv.Visibility.ADVANCED, field # A regular device-config field stays on the main form. assert markers[CONF_NAME_ADD_MAC_SUFFIX].visibility is None + + +@pytest.mark.asyncio +async def test_add_platformio_options_native_arduino( + caplog: pytest.LogCaptureFixture, +) -> None: + """The native ESP8266 Arduino toolchain honors board_build.f_cpu (a + real-world overclock knob) and warns about the rest like native IDF.""" + CORE.toolchain = Toolchain.ARDUINO + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp8266", + KEY_TARGET_FRAMEWORK: "arduino", + } + + await config._add_platformio_options( + { + "board_build.f_cpu": "160000000L", + # The schema also permits the list form; the last value wins + # and reaches the generator as a scalar + "board_build.ldscript": ["eagle.flash.2m.ld", "eagle.flash.4m2m.ld"], + "board_build.filesystem": "littlefs", + "upload_speed": "115200", + } + ) + + assert CORE.platformio_options["board_build.f_cpu"] == "160000000L" + assert CORE.platformio_options["board_build.ldscript"] == "eagle.flash.4m2m.ld" + assert "board_build.f_cpu is ignored" not in caplog.text + assert "board_build.ldscript is ignored" not in caplog.text + assert ( + "esphome->platformio_options->board_build.filesystem is ignored" in caplog.text + ) + # An empty list for an honored key is not a scalar; it falls through + # to the ignored-option warning instead of an IndexError + await config._add_platformio_options({"board_build.ldscript": []}) + assert "board_build.ldscript is ignored" in caplog.text + assert "'arduino' toolchain" in caplog.text + assert "upload_speed" not in caplog.text + + +def test_esp8266_rejects_unsupported_cli_toolchain() -> None: + """Until the native backend lands, ESP8266 serves only PlatformIO.""" + from esphome.components.esp8266 import CONFIG_SCHEMA + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): + CONFIG_SCHEMA({"board": "nodemcuv2"}) diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 77690a6897..4333420a9e 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -68,6 +68,7 @@ def _write_storage( esp_platform: str | None = "ESP32", core_platform: str | None = "esp32", build_path: str | None = "/build/lite_test", + toolchain: str | None = None, ) -> None: """Write a vanilla StorageJSON sidecar for the cache tests.""" storage_path.parent.mkdir(parents=True, exist_ok=True) @@ -88,6 +89,7 @@ def _write_storage( "no_mdns": False, "framework": "arduino", "core_platform": core_platform, + "toolchain": toolchain, } storage_path.write_text(json.dumps(data), encoding="utf-8") @@ -629,6 +631,35 @@ def test_save_compiled_config_and_sidecar_builds_real_sidecar(tmp_path: Path) -> assert load_compiled_config(yaml_path) is not None +@pytest.mark.parametrize( + ("sidecar_toolchain", "saved"), + [ + ("esp-idf", False), + ("platformio", True), + (None, True), # legacy sidecar without the field: guard is inert + ], +) +def test_save_compiled_config_and_sidecar_toolchain_mismatch( + tmp_path: Path, sidecar_toolchain: str | None, saved: bool +) -> None: + """A config validated under a different toolchain than the compile's + must not overwrite the cache.""" + yaml_path = _bare_yaml(tmp_path) + _prime_core(tmp_path) + CORE.config = {CONF_ESPHOME: {CONF_NAME: "lite_test"}} + CORE.toolchain = Toolchain.PLATFORMIO + _write_storage( + tmp_path / ".esphome" / "storage" / "lite_test.yaml.json", + toolchain=sidecar_toolchain, + ) + + save_compiled_config_and_sidecar(CORE.config) + + cache = tmp_path / ".esphome" / "storage" / "lite_test.yaml.validated.json" + assert cache.exists() is saved + assert (load_compiled_config(yaml_path) is not None) is saved + + @pytest.mark.parametrize("command", ["upload", "logs"]) def test_run_esphome_upload_and_logs_refresh_cache_on_fallback( tmp_path: Path, command: str diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 971c4e462d..0f927a6513 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -1,3 +1,4 @@ +import importlib import json import logging from pathlib import Path @@ -48,6 +49,7 @@ from esphome.const import ( TYPE_GIT, TYPE_LOCAL, Framework, + Toolchain, ) from esphome.core import ( CORE, @@ -3165,3 +3167,46 @@ def test_file__remapped_path_is_directory_raises(setup_core: Path) -> None: with pytest.raises(Invalid, match="is not a file"): cv.file_("/original/config/headers") + + +def test_require_platformio_toolchain() -> None: + """Platforms with only the PlatformIO backend reject other toolchains.""" + validator = cv.require_platformio_toolchain("RP2") + CORE.toolchain = None + config: dict = {} + assert validator(config) is config + assert CORE.toolchain == Toolchain.PLATFORMIO + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(Invalid, match="Unsupported toolchain 'arduino' for RP2"): + validator(config) + + +def test_check_supported_toolchain_unresolved_is_an_ordering_bug() -> None: + """Calling the check before resolution fails naming the ordering bug, + not a user-facing unsupported-toolchain error.""" + CORE.toolchain = None + with pytest.raises(Invalid, match="not resolved before RP2 validation"): + cv._check_supported_toolchain("RP2", (Toolchain.PLATFORMIO,)) + + +@pytest.mark.parametrize( + ("platform", "minimal_config"), + [ + ("host", {}), + ("rp2", {"board": "rpipicow"}), + ("bk72xx", {"board": "generic-bk7231n-qfn32-tuya"}), + ("rtl87xx", {"board": "generic-rtl8710bn-2mb-788k"}), + ("ln882x", {"board": "generic-ln882h"}), + # The legacy stub platform must reject too, not just the chip families + ("libretiny", {}), + ], +) +def test_every_platformio_only_platform_rejects_arduino_toolchain( + platform: str, minimal_config: dict +) -> None: + """A platform that cannot serve a CLI toolchain rejects it at validation.""" + module = importlib.import_module(f"esphome.components.{platform}") + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(Invalid, match="Unsupported toolchain 'arduino'"): + module.CONFIG_SCHEMA(dict(minimal_config)) diff --git a/tests/unit_tests/test_core.py b/tests/unit_tests/test_core.py index 7adf955217..0c96f8c8c9 100644 --- a/tests/unit_tests/test_core.py +++ b/tests/unit_tests/test_core.py @@ -958,6 +958,24 @@ class TestEsphomeCore: target.toolchain = const.Toolchain.ESP_IDF assert target.using_toolchain_sdk_nrf is False + def test_using_toolchain_arduino(self, target): + """A toolchain choice, distinct from the arduino target framework.""" + target.toolchain = const.Toolchain.ARDUINO + assert target.using_toolchain_arduino is True + target.toolchain = const.Toolchain.PLATFORMIO + assert target.using_toolchain_arduino is False + + def test_using_native_toolchain(self, target): + """True exactly for the toolchains that never read platformio.ini.""" + target.toolchain = const.Toolchain.ESP_IDF + assert target.using_native_toolchain is True + target.toolchain = const.Toolchain.ARDUINO + assert target.using_native_toolchain is True + target.toolchain = const.Toolchain.PLATFORMIO + assert target.using_native_toolchain is False + target.toolchain = const.Toolchain.SDK_NRF + assert target.using_native_toolchain is False + def test_add_library__extracts_short_name_from_path(self, target): """Test add_library extracts short name from library paths like owner/lib.""" target.data[const.KEY_CORE] = { diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index c7a5c85638..08c99e2119 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -7195,6 +7195,42 @@ def test_compile_program_espidf_idedata_none_warns( assert "No idedata was generated" in caplog.text +def test_cli_toolchain_skips_the_validated_config_cache(tmp_path: Path) -> None: + """An explicit --toolchain must run the per-platform validators, so the + upload/logs fast path becomes a cache miss.""" + conf = tmp_path / "device.yaml" + conf.write_text("esphome:\n name: t\n") + argv = ["esphome", "--toolchain", "arduino", "logs", str(conf)] + with ( + patch("esphome.compiled_config.load_compiled_config") as mock_cache, + patch("esphome.config.read_config", return_value=None) as mock_read, + ): + assert run_esphome(argv) == 2 + mock_cache.assert_not_called() + mock_read.assert_called_once() + + +def test_cli_toolchain_still_refreshes_the_validated_config_cache( + tmp_path: Path, +) -> None: + """An explicit --toolchain gates only the cache read; with a matching + sidecar the freshly validated config is still saved.""" + conf = tmp_path / "device.yaml" + conf.write_text("esphome:\n name: t\n") + argv = ["esphome", "--toolchain", "platformio", "logs", str(conf)] + with ( + patch("esphome.compiled_config.load_compiled_config") as mock_load, + patch("esphome.config.read_config", return_value={CONF_ESPHOME: {}}), + patch("esphome.compiled_config.save_compiled_config_and_sidecar") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", {"logs": Mock(return_value=0)} + ), + ): + assert run_esphome(argv) == 0 + mock_load.assert_not_called() + mock_save.assert_called_once() + + @pytest.mark.asyncio async def test_wrap_to_code_comment_is_insertion_order_independent() -> None: """The config comment dumps with sorted keys: voluptuous fills schema diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index 7b83a1edc7..b78a94a2e7 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -7,8 +7,10 @@ import sys from types import SimpleNamespace from unittest.mock import patch +import platformdirs import pytest +from esphome.components.nrf52 import _resolve_toolchain from esphome.components.nrf52.framework import ( _PLATFORMIO_PENV_REQUIREMENTS, _REQUIREMENTS, @@ -22,8 +24,9 @@ from esphome.components.nrf52.framework import ( get_sdk_nrf_tools_path, setup_platformio_python_env, ) +import esphome.config_validation as cv from esphome.config_validation import Version -from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION +from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain from esphome.core import CORE, EsphomeError from esphome.framework_helpers import get_python_env_executable_path @@ -560,7 +563,6 @@ def testget_tools_path_blank_env_falls_back_to_default( Path("") would resolve to the working directory, which clean-all could then delete by accident. """ - import platformdirs monkeypatch.setenv("ESPHOME_SDK_NRF_PREFIX", value) expected = ( @@ -572,7 +574,6 @@ def testget_tools_path_blank_env_falls_back_to_default( def testget_tools_path_default_is_global_cache( monkeypatch: pytest.MonkeyPatch, ) -> None: - import platformdirs monkeypatch.delenv("ESPHOME_SDK_NRF_PREFIX", raising=False) expected = ( @@ -621,3 +622,11 @@ def test_needs_venv_rebuild_on_dangling_interpreter_symlink(tmp_path: Path) -> N assert not python.exists() assert _needs_venv_rebuild(python, sentinel, "abc123") + + +def test_resolve_toolchain_rejects_unsupported() -> None: + """A --toolchain nRF52 cannot serve fails instead of degrading silently.""" + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): + _resolve_toolchain({}) From 7ff56c62f9c89b14d1d22d14614a3506003c8a22 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" <3060199+jesserockz@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:23:16 -0500 Subject: [PATCH 42/65] [core] Add shared registry, ninja, and cache infrastructure for native toolchains (#18570) --- esphome/build_helpers/ccache.py | 92 +++ esphome/build_helpers/ninja.py | 92 +++ esphome/build_helpers/tools_cache.py | 36 + esphome/components/nrf52/framework.py | 16 +- esphome/config_validation.py | 12 +- esphome/espidf/framework.py | 102 +-- esphome/framework_helpers.py | 58 ++ esphome/helpers.py | 3 +- esphome/platformio/registry.py | 311 ++++++++ esphome/platformio/toolchain.py | 88 +-- esphome/writer.py | 11 +- requirements.txt | 1 + tests/unit_tests/build_helpers/test_ccache.py | 122 +++ tests/unit_tests/build_helpers/test_ninja.py | 143 ++++ tests/unit_tests/test_espidf_framework.py | 111 ++- tests/unit_tests/test_framework_helpers.py | 34 + tests/unit_tests/test_platformio_registry.py | 725 ++++++++++++++++++ tests/unit_tests/test_platformio_toolchain.py | 89 +-- tests/unit_tests/test_writer.py | 16 +- 19 files changed, 1830 insertions(+), 232 deletions(-) create mode 100644 esphome/build_helpers/ccache.py create mode 100644 esphome/build_helpers/ninja.py create mode 100644 esphome/build_helpers/tools_cache.py create mode 100644 esphome/platformio/registry.py create mode 100644 tests/unit_tests/build_helpers/test_ccache.py create mode 100644 tests/unit_tests/build_helpers/test_ninja.py create mode 100644 tests/unit_tests/test_platformio_registry.py diff --git a/esphome/build_helpers/ccache.py b/esphome/build_helpers/ccache.py new file mode 100644 index 0000000000..5b5c7f247f --- /dev/null +++ b/esphome/build_helpers/ccache.py @@ -0,0 +1,92 @@ +"""Shared ccache policy for build backends: env-knob parsing, binary +resolution, and default ``CCACHE_*`` values.""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +from esphome.framework_helpers import strip_win_long_path_prefix, tool_version_runs +from esphome.helpers import FALSY_ENV_STRINGS, TRUTHY_ENV_STRINGS + +_LOGGER = logging.getLogger(__name__) + + +def _ccache_runs(ccache: str) -> bool: + """Return True when the ``ccache`` found on PATH actually runs.""" + return tool_version_runs( + ccache, + "Ignoring ccache at %s because it failed to run; compiling without ccache", + ) + + +def parse_enable_env(name: str) -> bool | None: + """Strictly parse an on/off environment knob; None when unset or invalid. + + ``bool(str)`` truthiness would flip ``no``/``off`` to enabled, so only + 1/true/yes/on and 0/false/no/off count; anything else warns and reads + as unset so the caller's default policy applies. + """ + raw = os.environ.get(name) + if raw is None: + return None + lowered = raw.strip().lower() + if not lowered: + # ENV KNOB= (Docker/CI) has always read as a disable + return False + if lowered in TRUTHY_ENV_STRINGS: + return True + if lowered in FALSY_ENV_STRINGS: + return False + _LOGGER.warning("Ignoring unrecognized %s=%r; use 1 or 0", name, raw) + return None + + +def resolve_ccache_path() -> str | None: + """The ccache binary to wrap compiles with, or None when disabled. + + An explicit ``ESPHOME_CCACHE_ENABLE=1`` skips the runnability probe; the + Windows extended-length prefix is stripped before probing (#18399). + """ + import shutil + + explicit = parse_enable_env("ESPHOME_CCACHE_ENABLE") + if explicit is False: + return None + ccache = shutil.which("ccache") + if ccache is None: + if explicit: + _LOGGER.warning( + "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " + "compiling without ccache" + ) + return None + ccache = strip_win_long_path_prefix(ccache) + if not explicit and not _ccache_runs(ccache): + return None + return ccache + + +def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: + """Default ``CCACHE_*`` values for a build subprocess (not os.environ). + + Values the user already set in the environment are respected. Depend + mode is on: both native backends emit depfiles (-MMD / CMake), which + keeps cache-miss overhead low. + """ + from esphome.core import CORE + + # An unset build_path means the env was built before preload; fail loudly + # rather than silently drop CCACHE_BASEDIR. + if CORE.build_path is None: + raise ValueError( + "CORE.build_path must be set before constructing the build environment" + ) + defaults = { + "CCACHE_DIR": str(cache_dir), + "CCACHE_NOHASHDIR": "true", + "CCACHE_DEPEND": "1", + "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), + } + return {k: v for k, v in defaults.items() if k not in os.environ} diff --git a/esphome/build_helpers/ninja.py b/esphome/build_helpers/ninja.py new file mode 100644 index 0000000000..8c25bc9513 --- /dev/null +++ b/esphome/build_helpers/ninja.py @@ -0,0 +1,92 @@ +"""Platform-neutral helpers for ninja-driven native builds.""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +import re +import shutil + +from esphome.core import EsphomeError +from esphome.framework_helpers import strip_win_long_path_prefix, tool_version_runs + +_LOGGER = logging.getLogger(__name__) + + +def _ninja_runs(binary: str) -> bool: + """Whether the ninja found on PATH actually runs (see tool_version_runs).""" + return tool_version_runs( + binary, + "Ignoring ninja at %s because it failed to run; " + "falling back to the bundled wheel", + ) + + +def find_ninja() -> Path: + """Locate the ninja binary: a runnable PATH hit first, else the ninja + PyPI wheel.""" + if binary := shutil.which("ninja"): + binary = strip_win_long_path_prefix(binary) + if _ninja_runs(binary): + return Path(binary) + import_error: ImportError | None = None + try: + import ninja + except ImportError as err: + import_error = err + wheel_binary = None + else: + wheel_binary = Path(ninja.BIN_DIR) / ( + "ninja.exe" if os.name == "nt" else "ninja" + ) + if wheel_binary is None or not wheel_binary.is_file(): + raise EsphomeError( + "ninja not found on PATH or in the ninja package; reinstall the " + "esphome Python environment" + ) from import_error + return wheel_binary + + +def escape(value: Path | str) -> str: + """Escape a path or token for a ninja file.""" + return str(value).replace("$", "$$").replace(":", "$:").replace(" ", "$ ") + + +def quote_arg(tok: str) -> str: + """Quote with the CreateProcess argv rule (as ``subprocess.list2cmdline``): + backslash runs double only before a quote. Windows-only; ``$`` must + already be doubled for ninja. + """ + quoted = re.sub(r'(\\*)"', lambda m: m.group(1) * 2 + '\\"', tok) + quoted = re.sub(r"(\\+)\Z", lambda m: m.group(1) * 2, quoted) + return f'"{quoted}"' + + +# Force-quote any token containing a character outside the shlex.quote-style +# safe set: ninja hands POSIX commands to /bin/sh -c, so bare (, ;, <, *, ` +# and friends would be re-parsed as shell syntax. +_NEEDS_QUOTE = re.compile(r"[^\w@%+=:,./-]") + + +def shell_token(tok: str, force: bool = False) -> str: + """Re-quote a lexed token for the platform shell; ``force`` always quotes. + + Single quotes on POSIX (/bin/sh), the argv rule on Windows + (CreateProcess). ``$`` is doubled first because ninja expands it before + the command reaches the shell. + """ + tok = tok.replace("$", "$$") # ninja would expand a bare $ to nothing + if not (force or not tok or _NEEDS_QUOTE.search(tok)): + return tok + # An empty token must become '' / "" or it vanishes from the argv + if os.name == "nt": + return quote_arg(tok) + # shlex.quote's rule; inlined because the $-doubled token must not be + # re-examined for safe characters + return "'" + tok.replace("'", "'\"'\"'") + "'" + + +def quote_path(value: Path | str) -> str: + """Force-quote a path for the ninja command line (shell/CreateProcess).""" + return shell_token(str(value), force=True) diff --git a/esphome/build_helpers/tools_cache.py b/esphome/build_helpers/tools_cache.py new file mode 100644 index 0000000000..e7193a8e2a --- /dev/null +++ b/esphome/build_helpers/tools_cache.py @@ -0,0 +1,36 @@ +"""Machine-global tools cache location shared by the native backends.""" + +from __future__ import annotations + +from pathlib import Path + + +def tools_cache_path(env_var: str, subdir: str) -> Path: + """A backend's machine-global tools directory, with an env override. + + A blank/whitespace override is treated as unset: ``Path("")`` resolves + to the CWD, which ``clean-all`` would then delete. + """ + import platformdirs + + from esphome.helpers import get_str_env + + if prefix := get_str_env(env_var, "").strip(): + # resolve(): symlinked prefixes otherwise trip idf.py's + # venv-mismatch warning on every build + return Path(prefix).expanduser().resolve() + # appauthor=False keeps the Windows path short (no vendor segment); + # deep IDF trees run into MAX_PATH otherwise + return ( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir + ).resolve() + + +# (env override, cache subdir) per native backend. writer.clean_all wipes +# every entry via tools_cache_path, so listing a cache here is the single +# step that registers it for removal; the backends' own path getters use +# the same named pairs so the two cannot drift. +IDF_TOOLS_CACHE = ("ESPHOME_ESP_IDF_PREFIX", "idf") +SDK_NRF_TOOLS_CACHE = ("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf") +ARDUINO8266_TOOLS_CACHE = ("ESPHOME_ARDUINO8266_PREFIX", "arduino8266") +TOOLS_CACHE_SPECS = (IDF_TOOLS_CACHE, SDK_NRF_TOOLS_CACHE, ARDUINO8266_TOOLS_CACHE) diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index e24569e322..5e2cf197fb 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -6,8 +6,7 @@ import platform import shutil import sys -import platformdirs - +from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError @@ -19,7 +18,6 @@ from esphome.framework_helpers import ( run_command_ok, str_to_lst_of_str, ) -from esphome.helpers import get_str_env _LOGGER = logging.getLogger(__name__) @@ -49,15 +47,9 @@ SDK_NG_MINIMAL_MIRRORS = str_to_lst_of_str( def get_sdk_nrf_tools_path() -> Path: - # A blank ESPHOME_SDK_NRF_PREFIX must be treated as unset: Path("") - # resolves to the CWD, which clean-all would then delete. - if prefix := get_str_env("ESPHOME_SDK_NRF_PREFIX", "").strip(): - path = Path(prefix).expanduser() - else: - # Machine-global (OS user cache dir) so all projects share one install; - # see espidf.framework.get_idf_tools_path for the location rationale. - path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "sdk-nrf" - return path.resolve() + # Machine-global (OS user cache dir) so all projects share one install; + # see espidf.framework.get_idf_tools_path for the location rationale. + return tools_cache_path(*SDK_NRF_TOOLS_CACHE) def _needs_venv_rebuild( diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 98001d5d5b..09962e8c95 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -93,7 +93,13 @@ from esphome.core import ( ) from esphome.enum import StrEnum from esphome.expression import SUBSTITUTION_VARIABLE_PROG as VARIABLE_PROG -from esphome.helpers import add_class_to_obj, docs_url, list_starts_with +from esphome.helpers import ( + FALSY_BOOL_STRINGS, + TRUTHY_BOOL_STRINGS, + add_class_to_obj, + docs_url, + list_starts_with, +) from esphome.schema_extractors import ( SCHEMA_EXTRACT, schema_extractor, @@ -581,9 +587,9 @@ def boolean(value): return value if isinstance(value, str): value = value.lower() - if value in ("true", "yes", "on", "enable"): + if value in TRUTHY_BOOL_STRINGS: return True - if value in ("false", "no", "off", "disable"): + if value in FALSY_BOOL_STRINGS: return False raise Invalid( f"Expected boolean value, but cannot convert {value} to a boolean. Please use 'true' or 'false'" diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index c2e1e00830..239d874dbd 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -12,9 +12,13 @@ import re import shutil from typing import Any, NoReturn -import platformdirs - -from esphome.core import CORE, Version +from esphome.build_helpers.ccache import ( + ccache_defaults_env, + parse_enable_env, + resolve_ccache_path, +) +from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path +from esphome.core import Version from esphome.framework_helpers import ( PathType, create_venv, @@ -29,8 +33,9 @@ from esphome.framework_helpers import ( run_command, run_command_ok, str_to_lst_of_str, + tool_version_runs, ) -from esphome.helpers import get_bool_env, get_str_env, write_file_if_changed +from esphome.helpers import write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -91,22 +96,10 @@ def get_idf_tools_path() -> Path: Returns: Path object pointing to the ESP-IDF tools directory """ - # Treat an empty/whitespace ESPHOME_ESP_IDF_PREFIX as unset: Path("") - # resolves to the CWD, which would install into (and let clean-all delete) - # the working directory by accident. - if prefix := get_str_env("ESPHOME_ESP_IDF_PREFIX", "").strip(): - path = Path(prefix).expanduser() - else: - # Machine-global so all projects share the multi-GB install instead of - # a per-config-directory copy. The user cache dir (not ~/.esphome) - # avoids colliding with data_dir when configs live in the home dir. - # appauthor=False drops the redundant \ segment on Windows - # (which otherwise repeats "esphome\esphome\") to keep the path short. - path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf" - # Resolve so an unnormalized config path (e.g. compiling ``../config/x.yaml``) - # doesn't leave ``..`` segments in the IDF_TOOLS_PATH handed to idf.py, which - # otherwise warns that the venv interpreter path doesn't match the install. - return path.resolve() + # Machine-global so all projects share the multi-GB install instead of + # a per-config-directory copy; see build_helpers.tools_cache.tools_cache_path + # for the env-override and normalization rules. + return tools_cache_path(*IDF_TOOLS_CACHE) # Windows' default MAX_PATH is 260 characters. ESP-IDF toolchains nest deeply @@ -1190,8 +1183,10 @@ def check_esp_idf_install( def _ccache_env() -> dict[str, str]: """Return ccache settings for ESP-IDF compiles. - Enabled by default whenever the ``ccache`` binary is on PATH; set - ``IDF_CCACHE_ENABLE=0`` in the environment to opt out. The cache lives under + Enabled by default whenever a runnable ``ccache`` binary is on PATH. + ``IDF_CCACHE_ENABLE=0`` opts out and ``=1`` forces it on; when that knob + is unset the shared ``ESPHOME_CCACHE_ENABLE`` applies (same 0/1 forms, + unrecognized values warn and count as unset). The cache lives under the IDF tools path (the machine-global cache dir, or ``ESPHOME_ESP_IDF_PREFIX``), so it is shared across all projects and removed by ``esphome clean-all`` along with the framework. @@ -1206,33 +1201,44 @@ def _ccache_env() -> dict[str, str]: Only values the user has not already set in the environment are returned, so a custom ``CCACHE_DIR`` / ``CCACHE_MAXSIZE`` / etc. is respected. """ - # Honor an explicit choice already in the environment (opt-out or opt-in). - if "IDF_CCACHE_ENABLE" in os.environ: - if not get_bool_env("IDF_CCACHE_ENABLE"): - return {} - elif shutil.which("ccache") is None: - # ESP-IDF silently skips ccache without the binary; don't enable it. - return {} + # IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared + # ESPHOME_CCACHE_ENABLE. + idf_knob = parse_enable_env("IDF_CCACHE_ENABLE") + if idf_knob is False: + # The raw value (e.g. "disable") is still inherited by idf.py via + # os.environ, where a non-false-constant string reads as truthy; + # export the canonical off spelling instead + return {"IDF_CCACHE_ENABLE": "0"} + if idf_knob is True: + # Forced on ignores the runnability verdict, but the outcome is + # worth saying out loud. Probed directly (not via the resolver, + # whose failure message says "compiling without ccache" -- exactly + # what forced-on does NOT do): only the truly-missing case means + # idf.py compiles without ccache; a broken binary is still used, + # since idf.py does its own PATH lookup. + if (ccache := shutil.which("ccache")) is None: + _LOGGER.warning( + "IDF_CCACHE_ENABLE=1 but no ccache binary is on PATH; " + "idf.py will compile without ccache" + ) + else: + # The probe warns with this message iff the binary fails + tool_version_runs( + ccache, + "IDF_CCACHE_ENABLE=1 forces on the ccache at %s even though " + "it failed to run; idf.py will use it anyway", + ) + elif resolve_ccache_path() is None: + # ESP-IDF silently skips ccache without the binary; export the + # canonical off spelling so an unparsable inherited value (or a + # probe-rejected ccache idf.py would still find) cannot enable it + return {"IDF_CCACHE_ENABLE": "0"} - # ccache is enabled past here. build_path is set during preload for every - # config-loading command, so it being unset means a caller built the IDF env - # too early -- fail loudly rather than silently drop CCACHE_BASEDIR (which - # would quietly cost cross-device cache hits). - if CORE.build_path is None: - raise ValueError( - "CORE.build_path must be set before constructing the ESP-IDF build " - "environment" - ) - - defaults = { - "IDF_CCACHE_ENABLE": "1", - "CCACHE_DIR": str(get_idf_tools_path() / "ccache"), - "CCACHE_NOHASHDIR": "true", - "CCACHE_DEPEND": "1", - "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), - } - # Don't override CCACHE_* values the user already set in their environment. - return {k: v for k, v in defaults.items() if k not in os.environ} + env = ccache_defaults_env(get_idf_tools_path() / "ccache") + # Exactly one canonical spelling ever reaches idf.py, whatever the + # accepted input spelling was ("enable", "yes", ...) + env["IDF_CCACHE_ENABLE"] = "1" + return env def get_framework_env( diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 031db85a65..aab7acc0e8 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -204,6 +204,30 @@ def run_command( return False, None, None +def tool_version_runs(binary: str, warning: str) -> bool: + """Probe ``binary --version``; on failure warn with ``warning`` % binary. + + ``shutil.which`` proves existence, not runnability (Windows .bat/.cmd + shims, stale package-manager shims). + """ + try: + subprocess.run( + [binary, "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=15, + # Repo-wide convention (posix_spawn fast path) + close_fds=False, + ) + except (OSError, subprocess.SubprocessError) as err: + # The cause (permission denied, missing DLL, timeout) is the one + # detail the user needs to fix it + _LOGGER.warning("%s (%s)", warning % binary, err) + return False + return True + + def run_command_ok(*args, **kwargs) -> bool: """ Execute a command and return only the success status. @@ -1284,3 +1308,37 @@ def download_from_mirrors( f"No mirror URL template matched the provided substitutions:{details}" ) raise ValueError("download_from_mirrors called with an empty mirrors list") + + +def strip_win_long_path_prefix(path: str) -> str: + r"""Strip the Windows extended-length path prefix from ``path``. + + Handles both forms documented at + https://learn.microsoft.com/windows/win32/fileio/naming-a-file: + + * ``\\?\C:\path\to\file`` -> ``C:\path\to\file`` + * ``\\?\UNC\server\share\path`` -> ``\\server\share\path`` + + The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with + ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates + into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from + the environment, falling back to ``os.path.normpath(sys.executable)``) + and ends up baked into SCons-emitted command lines for build steps such + as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand + the ``\\?\`` prefix, so the build fails with + "The system cannot find the path specified." Stripping the prefix early + keeps the path shell-quotable. + + Also applied to the ccache path exported by the ccache helpers, which + ``shutil.which`` can return with the same prefix. + + No-op on non-Windows platforms. + """ + if sys.platform != "win32": + return path + if path.startswith("\\\\?\\UNC\\"): + # \\?\UNC\server\share\... -> \\server\share\... + return "\\\\" + path[len("\\\\?\\UNC\\") :] + if path.startswith("\\\\?\\"): + return path[len("\\\\?\\") :] + return path diff --git a/esphome/helpers.py b/esphome/helpers.py index d30e9b16a2..4397111c2e 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -31,7 +31,8 @@ SockAddr = IPv4SockAddr | IPv6SockAddr _LOGGER = logging.getLogger(__name__) -# cv.boolean's closed spelling tables, shared with the env-knob parsing below +# cv.boolean's closed spelling tables, shared with the strict env-knob +# parser (build_helpers.ccache.parse_enable_env) TRUTHY_BOOL_STRINGS = frozenset({"true", "yes", "on", "enable"}) FALSY_BOOL_STRINGS = frozenset({"false", "no", "off", "disable"}) # cv.boolean's spelling tables plus the 1/0 env convention diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py new file mode 100644 index 0000000000..9538a28ff4 --- /dev/null +++ b/esphome/platformio/registry.py @@ -0,0 +1,311 @@ +"""Install packages from the PlatformIO registry without importing the +platformio package (identical bits, esphome's own download machinery).""" + +from __future__ import annotations + +from collections.abc import Callable, Collection +from functools import cache, partial +import json +import logging +import os +from pathlib import Path +import platform +from typing import NamedTuple + +from esphome.core import EsphomeError +from esphome.framework_helpers import ( + archive_extract_all, + download_from_mirrors, + download_with_resume, + rmdir, + run_batch_downloads, +) +from esphome.net_retry import fetch_with_retry, http_request + +_LOGGER = logging.getLogger(__name__) + +_REGISTRY_URL = ( + "https://api.registry.platformio.org/v3/packages/platformio/tool/{package}" +) + + +def get_systype() -> str: + """The registry system tag for the current host. + + Transliterates ``platformio.util.get_systype()`` (same + ``PLATFORMIO_SYSTEM_TYPE`` override). Deviation: windows-arm64 maps to + ``windows_amd64`` (no arm64 toolchains; x86 emulation). + """ + if systype := os.environ.get("PLATFORMIO_SYSTEM_TYPE"): + return systype + system = platform.system().lower() + arch = platform.machine().lower() + if system == "windows": + if not arch: # same fallback as upstream (platformio issue #4353) + arch = "x86_" + platform.architecture()[0] + if "x86" in arch: + arch = "amd64" if "64" in arch else "x86" + elif arch == "arm64": + arch = "amd64" + if arch == "aarch64" and platform.architecture()[0] == "32bit": + # 64-bit kernel with a 32-bit userland (e.g. 32-bit Raspberry Pi OS) + arch = "armv7l" + return f"{system}_{arch}" if arch else system + + +@cache +def registry_download(package: str, version: str) -> tuple[str, str, int | None]: + """Resolve a package's download URL, sha256, and size via the registry. + + The metadata fetch goes through ``http_request``/``fetch_with_retry`` + (the consolidated HTTP path) so it shares the Happy Eyeballs patch and + transient-retry policy of every other small fetch. Cached per process + so the prefetch and the install resolve each package once (failures + are not cached; the install retries them). + """ + url = _REGISTRY_URL.format(package=package) + + def _fetch() -> str: + resp = http_request("GET", url, timeout=30) + resp.raise_for_status() + return resp.text + + import requests + + try: + body = fetch_with_retry(url, _fetch, what="Registry lookup") + except requests.exceptions.RequestException as err: + raise EsphomeError( + f"Could not fetch registry metadata for {package}: {err}" + ) from err + try: + data = json.loads(body) + except ValueError as err: + raise EsphomeError( + f"The package registry returned invalid JSON for {package}: {err}" + ) from err + if not isinstance(data, dict): + raise EsphomeError( + f"Unexpected package registry response for {package}: {str(data)[:200]}" + ) + systype = get_systype() + versions = data.get("versions") + if not isinstance(versions, list): + # A schema change or an error/captive-portal payload must not be + # reported as "version not found" + raise EsphomeError( + f"Unexpected package registry response for {package}: {str(data)[:200]}" + ) + for ver in versions: + if not isinstance(ver, dict): + raise EsphomeError( + f"Unexpected package registry response for {package}: {str(data)[:200]}" + ) + if ver.get("name") != version: + continue + files = ver.get("files") + if not isinstance(files, list): + raise EsphomeError( + f"Unexpected package registry response for {package}: {str(ver)[:200]}" + ) + for file in files: + if not isinstance(file, dict): + raise EsphomeError( + f"Unexpected package registry response for {package}: " + f"{str(ver)[:200]}" + ) + # Only a missing key means "any system"; an empty list must not + # match, and a bare string would make ``in`` a substring test. + systems = file.get("system") + if systems is None: + systems = ["*"] + elif isinstance(systems, str): + systems = [systems] + elif not isinstance(systems, list): + # An int would make ``in`` a TypeError and a dict a key test + raise EsphomeError( + f"Unexpected package registry response for {package}: " + f"{str(file)[:200]}" + ) + if "*" in systems or systype in systems: + sha256 = (file.get("checksum") or {}).get("sha256") + if not sha256: + # Never extract an unverified archive; the registry + # publishes a checksum for every package file. + raise EsphomeError( + f"The package registry returned no sha256 for " + f"{package} {version}; refusing the unverified download" + ) + url = file.get("download_url") + if not url: + raise EsphomeError( + f"The package registry returned no download URL for " + f"{package} {version}" + ) + return (url, sha256, file.get("size")) + raise EsphomeError( + f"No {package} {version} build for this platform ({systype})" + ) + raise EsphomeError(f"{package} {version} not found in the package registry") + + +def _check_layout(name: str, dest: Path, expect: Collection[str]) -> None: + """Raise when an install tree is missing an expected directory (runs on + fresh extracts and on marker hits).""" + for rel in expect: + if not (dest / rel).is_dir(): + raise EsphomeError( + f"{name} at {dest} is missing the expected {rel} " + "directory; run 'esphome clean-all' and retry" + ) + + +class _PendingArchive(NamedTuple): + name: str + version: str + dest: Path + url: str + sha256: str + size: int + + +def _already_installed(dest: Path) -> bool: + """Whether ``dest`` holds a completed install (extraction marker).""" + return (dest / ".esphome_extracted").is_file() + + +def prefetch_packages( + packages: list[tuple[str, str, Path, list[str]]], downloads_dir: Path +) -> None: + """Download pending package archives in parallel under one combined bar. + + ``packages`` holds ``(name, version, dest, mirrors)`` per package. Purely + an optimization: ``install_package`` verifies every archive and + re-downloads anything this pass left unfinished. Mirror overrides and + registry entries without a size stay on the sequential path so its + per-file bars remain trustworthy. Each fetch holds the same per-dest + lock as ``install_package``: the archive's ``.part`` file is shared, and + two concurrent writers would truncate each other's bytes. + """ + from filelock import FileLock + + pending: list[_PendingArchive] = [] + seen: set[str] = set() + for name, version, dest, mirrors in packages: + if mirrors or (dest / ".esphome_extracted").is_file(): + continue + archive_name = f"{name}-{version}" + if archive_name in seen: + # A duplicate entry would race itself between two workers + continue + seen.add(archive_name) + try: + url, sha256, size = registry_download(name, version) + except EsphomeError as err: + # The sequential install reports the real failure with context + _LOGGER.debug("Prefetch resolve for %s failed: %s", name, err) + continue + if not size: + continue + archive = downloads_dir / archive_name + if archive.is_file() and archive.stat().st_size == size: + continue + pending.append(_PendingArchive(name, version, dest, url, sha256, size)) + if len(pending) < 2: + return + downloads_dir.mkdir(parents=True, exist_ok=True) + _LOGGER.info( + "Downloading %d package archive(s): %s", + len(pending), + ", ".join(entry.name for entry in pending), + ) + + def _fetch(entry: _PendingArchive, tracker: Callable[[int], None]) -> None: + entry.dest.parent.mkdir(parents=True, exist_ok=True) + with FileLock(f"{entry.dest}.lock", fallback_to_soft=False): + # Marker re-check: a concurrent build may have installed (and + # deleted the archive of) this package while we waited; + # re-downloading would orphan a fresh copy in downloads_dir + # no branch: the thread tracer misses the skip edge; both + # arms of _already_installed are pinned directly + if not _already_installed(entry.dest): # pragma: no branch + download_with_resume( + entry.url, + downloads_dir / f"{entry.name}-{entry.version}", + sha256=entry.sha256, + size=entry.size, + progress=tracker, + ) + + failures = run_batch_downloads( + "Downloading packages", + [(entry.name, entry.size, partial(_fetch, entry)) for entry in pending], + ) + for name, err in failures: + if isinstance(err, (EsphomeError, OSError)): + # Expected download failures: install_package retries this one + # itself, with a visible bar + _LOGGER.debug("Prefetch of %s failed: %s", name, err) + else: + # Anything else is a programming error that would otherwise + # become a permanent silent no-op + _LOGGER.warning("Prefetch of %s failed: %r", name, err, exc_info=err) + + +def install_package( + name: str, + version: str, + dest: Path, + mirrors: list[str], + downloads_dir: Path, + expect: Collection[str], +) -> None: + """Download, verify, and extract one package if not already installed. + + The registry path is integrity-checked against the sha256 the registry + publishes; a mirror override (URL templates with ``{VERSION}``/``{SYSTEM}`` + substitution) is trusted as configured. ``downloads_dir`` holds the + archive between runs so an interrupted download resumes. + """ + if not expect: + # Layout validation before marker.touch() is the only guard against + # caching a truncated mirror archive as a good install + raise ValueError("install_package requires a non-empty expect") + marker = dest / ".esphome_extracted" + if marker.is_file(): + _check_layout(name, dest, expect) + return + from filelock import FileLock + + # Serialize concurrent cold builds (same filelock pattern as git.py). + dest.parent.mkdir(parents=True, exist_ok=True) + # A soft-lock fallback would turn a hard-killed run into a permanent + # hang (see git.py). + with FileLock(f"{dest}.lock", fallback_to_soft=False): + if marker.is_file(): + # Another process finished the install while we waited + return + rmdir(dest, msg=f"Clean up incomplete {name} install") + # Persistent location so an interrupted download resumes across runs. + downloads_dir.mkdir(parents=True, exist_ok=True) + archive = downloads_dir / f"{name}-{version}" + _LOGGER.info("Downloading %s %s ...", name, version) + if mirrors: + _LOGGER.warning( + "Downloading %s from a mirror override; checksum verification " + "is skipped for mirrors", + name, + ) + download_from_mirrors( + mirrors, {"VERSION": version, "SYSTEM": get_systype()}, archive + ) + else: + url, sha256, size = registry_download(name, version) + download_with_resume(url, archive, sha256=sha256, size=size) + _LOGGER.info("Extracting %s ...", name) + archive_extract_all(archive, dest, progress_header="Extracting") + # Validate the layout before recording success, so an unexpected + # package is never cached as a working install. + _check_layout(name, dest, expect) + marker.touch() + archive.unlink(missing_ok=True) diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index a98ef3e9fe..cf2094dfe0 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -4,19 +4,18 @@ import logging import os from pathlib import Path import re -import shutil -import subprocess import sys from typing import TYPE_CHECKING, Any import platformdirs +from esphome.build_helpers.ccache import resolve_ccache_path from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE from esphome.core import CORE, EsphomeError +from esphome.framework_helpers import strip_win_long_path_prefix from esphome.helpers import ( add_git_ceiling_directory, copy_file_if_changed, - get_bool_env, rmtree, write_file, ) @@ -41,40 +40,6 @@ _PIO_PYTHON_STAMP_LOCK = ".esphome.pio.stamp.lock" _PIO_PYTHON_STAMP_SCHEMA = "0" -def _strip_win_long_path_prefix(path: str) -> str: - r"""Strip the Windows extended-length path prefix from ``path``. - - Handles both forms documented at - https://learn.microsoft.com/windows/win32/fileio/naming-a-file: - - * ``\\?\C:\path\to\file`` -> ``C:\path\to\file`` - * ``\\?\UNC\server\share\path`` -> ``\\server\share\path`` - - The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with - ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates - into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from - the environment, falling back to ``os.path.normpath(sys.executable)``) - and ends up baked into SCons-emitted command lines for build steps such - as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand - the ``\\?\`` prefix, so the build fails with - "The system cannot find the path specified." Stripping the prefix early - keeps the path shell-quotable. - - Also applied to the ccache path exported by ``_ccache_env()``, which - ``shutil.which`` can return with the same prefix. - - No-op on non-Windows platforms. - """ - if sys.platform != "win32": - return path - if path.startswith("\\\\?\\UNC\\"): - # \\?\UNC\server\share\... -> \\server\share\... - return "\\\\" + path[len("\\\\?\\UNC\\") :] - if path.startswith("\\\\?\\"): - return path[len("\\\\?\\") :] - return path - - def get_platformio_config() -> "ProjectConfig | None": """Return PlatformIO's ``ProjectConfig``, or None when PlatformIO is absent.""" try: @@ -238,35 +203,6 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) -def _ccache_runs(ccache: str) -> bool: - """Return True when the ``ccache`` found on PATH actually runs. - - ``shutil.which`` proves existence, not runnability: on Windows it also - matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose - target is gone. Wrapping compiles around such a find fails every compile - step with an opaque OS error, so probe once and fall back to compiling - without ccache when the probe fails. - """ - try: - subprocess.run( - [ccache, "--version"], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=15, - # Repo-wide convention (posix_spawn fast path); see the - # close_fds=False call sites across esphome/ and script/helpers.py - close_fds=False, - ) - except (OSError, subprocess.SubprocessError): - _LOGGER.warning( - "Ignoring ccache at %s because it failed to run; compiling without ccache", - ccache, - ) - return False - return True - - def _ccache_env() -> dict[str, str]: r"""Return ccache settings for PlatformIO builds. @@ -285,7 +221,7 @@ def _ccache_env() -> dict[str, str]: runs fine through ``CreateProcess``, which is how ESP-IDF invokes it, but SCons runs every compile through ``cmd.exe``, which fails on it with "The system cannot find the path specified." (#18399), so the prefix is - stripped here with ``_strip_win_long_path_prefix()`` before the + stripped here with ``strip_win_long_path_prefix()`` before the runnability probe, which therefore validates the exact string the build will execute. ``ESPHOME_CCACHE_PATH`` is an internal channel, not a user setting: the @@ -311,22 +247,8 @@ def _ccache_env() -> dict[str, str]: build dir. The other ``CCACHE_*`` values the user already set in the environment are respected. """ - explicit = "ESPHOME_CCACHE_ENABLE" in os.environ - if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): - return {"ESPHOME_CCACHE_ENABLE": "0"} - ccache_path = shutil.which("ccache") + ccache_path = resolve_ccache_path() if ccache_path is None: - if explicit: - _LOGGER.warning( - "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " - "compiling without ccache" - ) - return {"ESPHOME_CCACHE_ENABLE": "0"} - # Strip before probing so the probe validates (and the failure warning - # names) the exact string the build will execute through cmd.exe. - ccache_path = _strip_win_long_path_prefix(ccache_path) - # An explicit opt-in skips the runnability probe. - if not explicit and not _ccache_runs(ccache_path): return {"ESPHOME_CCACHE_ENABLE": "0"} env = { "ESPHOME_CCACHE_ENABLE": "1", @@ -388,7 +310,7 @@ def run_platformio_cli(*args, **kwargs) -> str | int: # Strip the Windows extended-length path prefix from sys.executable so it # doesn't propagate into PlatformIO's $PYTHONEXE and break SCons-emitted # command lines run through cmd.exe. - python_exe = _strip_win_long_path_prefix(sys.executable) + python_exe = strip_win_long_path_prefix(sys.executable) if python_exe != sys.executable: # Only override PYTHONEXEPATH when we actually stripped a prefix. # PlatformIO's get_pythonexe_path() reads this and falls back to diff --git a/esphome/writer.py b/esphome/writer.py index 85c0642774..0b9e7669ef 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -706,14 +706,17 @@ def clean_all(configuration: list[str]): # the per-config loop above can't reach. Wipe the default cache root # (also catches leftovers from older install layouts), then the resolved # install paths for the ESPHOME_*_PREFIX overrides (docker/add-on/CI) - # that live outside it. + # that live outside it. Every backend's cache is listed in + # TOOLS_CACHE_SPECS, so registering one there is the only step. import platformdirs - from esphome.components.nrf52.framework import get_sdk_nrf_tools_path - from esphome.espidf.framework import get_idf_tools_path + from esphome.build_helpers.tools_cache import TOOLS_CACHE_SPECS, tools_cache_path cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve() - for install_path in (cache_root, get_idf_tools_path(), get_sdk_nrf_tools_path()): + install_paths = [cache_root] + [ + tools_cache_path(*spec) for spec in TOOLS_CACHE_SPECS + ] + for install_path in install_paths: if install_path.is_dir(): _LOGGER.info("Deleting %s", install_path) rmtree(install_path) diff --git a/requirements.txt b/requirements.txt index 4f6bdbad4c..3d4439bf10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.3 # native esp-idf toolchain global cache dir +ninja==1.13.0 # native esp8266 arduino toolchain build driver filelock==3.32.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this diff --git a/tests/unit_tests/build_helpers/test_ccache.py b/tests/unit_tests/build_helpers/test_ccache.py new file mode 100644 index 0000000000..0237db4081 --- /dev/null +++ b/tests/unit_tests/build_helpers/test_ccache.py @@ -0,0 +1,122 @@ +"""Tests for the shared ccache policy in esphome.build_helpers.ccache.""" + +from __future__ import annotations + +import os +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from esphome.build_helpers import ccache + + +def test_resolve_opt_out() -> None: + with patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}): + assert ccache.resolve_ccache_path() is None + + +def test_resolve_no_binary(caplog: pytest.LogCaptureFixture) -> None: + with ( + patch.dict(os.environ, {}, clear=True), + patch("shutil.which", return_value=None), + ): + assert ccache.resolve_ccache_path() is None + assert "no ccache binary" not in caplog.text + + +def test_resolve_probe_failure() -> None: + with ( + patch.dict(os.environ, {}, clear=True), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run", side_effect=OSError("boom")), + ): + assert ccache.resolve_ccache_path() is None + + +def test_resolve_explicit_skips_probe_and_warns_missing( + caplog: pytest.LogCaptureFixture, +) -> None: + with ( + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch.object(ccache, "_ccache_runs", side_effect=AssertionError), + ): + assert ccache.resolve_ccache_path() == "/usr/bin/ccache" + with ( + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), + patch("shutil.which", return_value=None), + ): + assert ccache.resolve_ccache_path() is None + assert "no ccache binary is on PATH" in caplog.text + + +def test_probe_spawns_with_close_fds_false() -> None: + with patch("esphome.framework_helpers.subprocess.run") as mock_run: + assert ccache._ccache_runs("/usr/bin/ccache") is True + assert mock_run.call_args.kwargs["close_fds"] is False + + +def test_defaults_env(tmp_path: Path) -> None: + with ( + patch("esphome.core.CORE", SimpleNamespace(build_path=tmp_path / "b")), + patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True), + ): + env = ccache.ccache_defaults_env(tmp_path / "cache") + assert env["CCACHE_DIR"] == str(tmp_path / "cache") + assert env["CCACHE_DEPEND"] == "1" + assert "CCACHE_NOHASHDIR" not in env # user value respected + + +def test_defaults_env_requires_build_path() -> None: + with ( + patch("esphome.core.CORE", SimpleNamespace(build_path=None)), + pytest.raises(ValueError, match="build_path"), + ): + ccache.ccache_defaults_env(Path("/x")) + + +@pytest.mark.parametrize("value", ["no", "off", "false", "0"]) +def test_resolve_opt_out_synonyms(value: str) -> None: + """Every recognized falsy spelling disables ccache.""" + with patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": value}): + assert ccache.resolve_ccache_path() is None + + +def test_resolve_unrecognized_value_warns_and_probes( + caplog: pytest.LogCaptureFixture, +) -> None: + """An unparsable ESPHOME_CCACHE_ENABLE is treated as unset: it must not + silently enable ccache or skip the runnability probe.""" + with ( + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "enabled"}), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch.object(ccache, "_ccache_runs", return_value=False) as mock_probe, + ): + assert ccache.resolve_ccache_path() is None + mock_probe.assert_called_once() + assert "unrecognized ESPHOME_CCACHE_ENABLE" in caplog.text + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("1", True), + ("enable", True), + ("ON", True), + ("0", False), + ("disable", False), + ("Off", False), + ("maybe", None), + # ENV KNOB= (Docker/CI) has always read as a disable + ("", False), + (" ", False), + ], +) +def test_parse_enable_env_spelling_tables( + monkeypatch: pytest.MonkeyPatch, raw: str, expected: bool | None +) -> None: + """cv.boolean's spelling tables plus the 1/0 env convention.""" + monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", raw) + assert ccache.parse_enable_env("ESPHOME_CCACHE_ENABLE") is expected diff --git a/tests/unit_tests/build_helpers/test_ninja.py b/tests/unit_tests/build_helpers/test_ninja.py new file mode 100644 index 0000000000..6f0bbda0b9 --- /dev/null +++ b/tests/unit_tests/build_helpers/test_ninja.py @@ -0,0 +1,143 @@ +"""Tests for esphome.build_helpers.ninja.""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.build_helpers import ninja as ninja_helper +from esphome.core import EsphomeError + + +def test_find_ninja_prefers_path(tmp_path: Path) -> None: + with ( + patch("shutil.which", return_value=str(tmp_path / "ninja")), + patch.object(ninja_helper, "_ninja_runs", return_value=True), + ): + assert ninja_helper.find_ninja() == tmp_path / "ninja" + + +def test_find_ninja_falls_back_to_wheel(tmp_path: Path) -> None: + """Without a PATH entry, the ninja PyPI wheel's binary is used.""" + binary_name = "ninja.exe" if os.name == "nt" else "ninja" + (tmp_path / binary_name).touch() + wheel = MagicMock(BIN_DIR=str(tmp_path)) + with ( + patch("shutil.which", return_value=None), + patch.dict(sys.modules, {"ninja": wheel}), + ): + assert ninja_helper.find_ninja() == tmp_path / binary_name + + +def test_find_ninja_package_not_installed() -> None: + """A missing ninja package raises the actionable message, not ImportError.""" + with ( + patch("shutil.which", return_value=None), + patch.dict(sys.modules, {"ninja": None}), + pytest.raises(EsphomeError, match="ninja not found"), + ): + ninja_helper.find_ninja() + + +def test_find_ninja_missing_everywhere(tmp_path: Path) -> None: + wheel = MagicMock(BIN_DIR=str(tmp_path)) + with ( + patch("shutil.which", return_value=None), + patch.dict(sys.modules, {"ninja": wheel}), + pytest.raises(EsphomeError, match="ninja not found"), + ): + ninja_helper.find_ninja() + + +def test_escape_ninja_specials() -> None: + assert ninja_helper.escape("a b:c$d") == "a$ b$:c$$d" + + +def _q(tok: str) -> str: + """The platform's shell_token quote wrapper (argv rule on Windows).""" + return f'"{tok}"' if os.name == "nt" else f"'{tok}'" + + +def test_quote_arg_windows_argv_rule() -> None: + # Backslash runs double only before a quote (subprocess.list2cmdline rule) + assert ninja_helper.quote_arg('-DX=a\\"b c') == '"-DX=a\\\\\\"b c"' + assert ninja_helper.quote_arg("a b\\") == '"a b\\\\"' + + +def test_shell_token_quotes_only_when_needed() -> None: + assert ninja_helper.shell_token("-Os") == "-Os" + assert ninja_helper.shell_token("-DP=C:\\x y") == _q("-DP=C:\\x y") + assert ninja_helper.shell_token("plain", force=True) == _q("plain") + + +def test_shell_token_quotes_shell_metacharacters() -> None: + """Tokens like -DMASK=(1<<3) must not reach /bin/sh -c bare.""" + assert ninja_helper.shell_token("-DMASK=(1<<3)") == _q("-DMASK=(1<<3)") + assert ninja_helper.shell_token("-DX=a;b") == _q("-DX=a;b") + assert ninja_helper.shell_token("-DX=$HOME") == _q("-DX=$$HOME") + + +def test_shell_token_posix_roundtrips_through_sh() -> None: + """Backslash runs, $, backticks, and quotes must reach the compiler + exactly as lexed once ninja un-doubles $$ and /bin/sh strips quotes.""" + + if sys.platform == "win32": + pytest.skip("POSIX sh quoting") + for tok in ("-DP=a\\\\b", "-DX=$VAR", "-DY=`date`", "-DZ=it's", '-DC="q"'): + quoted = ninja_helper.shell_token(tok).replace("$$", "$") + out = subprocess.run( + ["/bin/sh", "-c", f'printf "%s" {quoted}'], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout == tok + + +def test_quote_path_force_quotes() -> None: + assert ninja_helper.quote_path(Path("a b")) == _q("a b") + assert ninja_helper.quote_path("simple") == _q("simple") + + +def test_shell_token_empty_token_is_quoted() -> None: + """An empty argv element must survive as an explicit pair of quotes.""" + assert ninja_helper.shell_token("") == _q("") + + +def test_find_ninja_probes_path_hit(tmp_path: Path) -> None: + """A broken PATH shim falls back to the wheel instead of failing every + build later.""" + binary_name = "ninja.exe" if os.name == "nt" else "ninja" + (tmp_path / binary_name).touch() + wheel = MagicMock(BIN_DIR=str(tmp_path)) + with ( + patch("shutil.which", return_value="/broken/ninja"), + patch.object(ninja_helper, "_ninja_runs", return_value=False), + patch.dict(sys.modules, {"ninja": wheel}), + ): + assert ninja_helper.find_ninja() == tmp_path / binary_name + + +def test_ninja_probe_failure_warns(caplog: pytest.LogCaptureFixture) -> None: + with patch("esphome.framework_helpers.subprocess.run", side_effect=OSError("boom")): + assert ninja_helper._ninja_runs("/broken/ninja") is False + assert "failed to run" in caplog.text + + +def test_ninja_probe_success() -> None: + with patch("esphome.framework_helpers.subprocess.run") as mock_run: + assert ninja_helper._ninja_runs("/usr/bin/ninja") is True + assert mock_run.call_args.kwargs["close_fds"] is False + + +def test_shell_token_windows_branch_uses_argv_rule() -> None: + """The nt branch quotes with the CreateProcess argv rule (the ubuntu + coverage run never takes it naturally).""" + with patch.object(os, "name", "nt"): + assert ninja_helper.shell_token("a b") == '"a b"' + assert ninja_helper.shell_token("", force=True) == '""' diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index 1bef743f4c..45a971ca01 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -1560,13 +1560,14 @@ def test_get_framework_env_without_python_env_uses_os_path(tmp_path: Path) -> No def _ccache_patches(tmp_path: Path, which: str | None, build_path: Path | None): return ( - patch("esphome.espidf.framework.shutil.which", return_value=which), + patch("esphome.espidf.framework.resolve_ccache_path", return_value=which), patch( "esphome.espidf.framework.get_idf_tools_path", return_value=tmp_path / "tools", ), + # ccache_defaults_env (build_helpers.ccache) reads CORE at call time patch( - "esphome.espidf.framework.CORE", + "esphome.core.CORE", SimpleNamespace(build_path=build_path), ), ) @@ -1587,7 +1588,8 @@ def test_ccache_env_disabled_when_binary_missing(tmp_path: Path) -> None: # build_path is None here too: a disabled cache must not require it. p1, p2, p3 = _ccache_patches(tmp_path, None, None) with patch.dict("os.environ", {}, clear=True), p1, p2, p3: - assert _ccache_env() == {} + # Canonical off, so an inherited/unparsable value cannot enable it + assert _ccache_env() == {"IDF_CCACHE_ENABLE": "0"} def test_ccache_env_opt_out_via_env(tmp_path: Path) -> None: @@ -1595,18 +1597,111 @@ def test_ccache_env_opt_out_via_env(tmp_path: Path) -> None: # short-circuits before build_path is needed. p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", None) with patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "0"}, clear=True), p1, p2, p3: - assert _ccache_env() == {} + # The canonical off spelling is exported: the raw value is inherited + # by idf.py, where a spelling like "disable" would read as truthy + assert _ccache_env() == {"IDF_CCACHE_ENABLE": "0"} -def test_ccache_env_opt_in_without_binary(tmp_path: Path) -> None: - # Explicit IDF_CCACHE_ENABLE=1 forces it on without probing PATH. It's - # already in the environment, so it isn't re-emitted, but the rest is. +def test_ccache_env_opt_in_without_binary( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # Explicit IDF_CCACHE_ENABLE=1 forces it on; without a usable binary + # idf.py silently skips ccache, so this branch must say so out loud. p1, p2, p3 = _ccache_patches(tmp_path, None, tmp_path / "build") with patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "1"}, clear=True), p1, p2, p3: env = _ccache_env() - assert "IDF_CCACHE_ENABLE" not in env + assert env["IDF_CCACHE_ENABLE"] == "1" assert env["CCACHE_DIR"] == str(tmp_path / "tools" / "ccache") assert env["CCACHE_DEPEND"] == "1" + assert "no ccache binary is on PATH" in caplog.text + + +def test_ccache_env_opt_in_with_working_binary( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # Forced on with a working binary: no warning fires at all. + ccache = tmp_path / "ccache" + ccache.touch() + p1, p2, p3 = _ccache_patches(tmp_path, str(ccache), tmp_path / "build") + with ( + patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "1"}, clear=True), + patch("esphome.espidf.framework.shutil.which", return_value=str(ccache)), + patch("esphome.espidf.framework.tool_version_runs", return_value=True), + p1, + p2, + p3, + ): + env = _ccache_env() + assert env["IDF_CCACHE_ENABLE"] == "1" + assert not [r for r in caplog.records if r.levelno >= logging.WARNING] + + +def test_ccache_env_opt_in_with_rejected_binary( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # Forced on with a present-but-rejected binary: idf.py does its own + # PATH lookup and uses it anyway; the warning must say so, not claim + # the build runs without ccache. + # A present but non-executable file: the real probe fails and logs + # the forced-on message (patching the probe would silence it) + broken = tmp_path / "broken-ccache" + broken.touch() + p1, p2, p3 = _ccache_patches(tmp_path, None, tmp_path / "build") + with ( + patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "1"}, clear=True), + patch("esphome.espidf.framework.shutil.which", return_value=str(broken)), + p1, + p2, + p3, + ): + env = _ccache_env() + assert env["IDF_CCACHE_ENABLE"] == "1" + assert "idf.py will use it anyway" in caplog.text + # Exactly one story: the resolver's contradictory "compiling without + # ccache" must not precede it + assert "compiling without ccache" not in caplog.text + + +def test_ccache_env_honors_shared_esphome_opt_out(tmp_path: Path) -> None: + """ESPHOME_CCACHE_ENABLE=0 disables ccache here too; the shared policy + must not apply to every backend except this one.""" + _p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", tmp_path / "build") + env_vars = {"ESPHOME_CCACHE_ENABLE": "0", "PATH": "/usr/bin"} + with patch.dict("os.environ", env_vars, clear=True), p2, p3: + # The real resolver runs so the opt-out parse is exercised + assert _ccache_env() == {"IDF_CCACHE_ENABLE": "0"} + + +@pytest.mark.parametrize("value", ["off", "no"]) +def test_ccache_env_idf_knob_parses_strictly(tmp_path: Path, value: str) -> None: + """IDF_CCACHE_ENABLE uses the same strict table as the shared knob, so + "off" disables instead of reading as truthy.""" + p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", tmp_path / "build") + with patch.dict("os.environ", {"IDF_CCACHE_ENABLE": value}, clear=True), p1, p2, p3: + assert _ccache_env() == {"IDF_CCACHE_ENABLE": "0"} + + +def test_ccache_env_idf_knob_unrecognized_warns_and_defers( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unparsable IDF_CCACHE_ENABLE warns, defers to the shared resolver, + and is not forwarded to idf.py as truthy.""" + p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", tmp_path / "build") + env_vars = {"IDF_CCACHE_ENABLE": "enabled"} + with patch.dict("os.environ", env_vars, clear=True), p1, p2, p3: + env = _ccache_env() + assert "unrecognized IDF_CCACHE_ENABLE" in caplog.text + assert env["IDF_CCACHE_ENABLE"] == "1" + + +def test_ccache_env_idf_knob_wins_over_shared_opt_out(tmp_path: Path) -> None: + """IDF_CCACHE_ENABLE=1 takes precedence over ESPHOME_CCACHE_ENABLE=0.""" + p1, p2, p3 = _ccache_patches(tmp_path, None, tmp_path / "build") + env_vars = {"IDF_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_ENABLE": "0"} + with patch.dict("os.environ", env_vars, clear=True), p1, p2, p3: + env = _ccache_env() + assert env["CCACHE_DIR"] == str(tmp_path / "tools" / "ccache") + assert env["IDF_CCACHE_ENABLE"] == "1" def test_ccache_env_preserves_user_overrides(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index f001bd6c37..8844212600 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2278,3 +2278,37 @@ class TestGetProjectCxxCompileFlags: def test_empty_flags(self) -> None: with patch("esphome.core.CORE", _make_core_cxx(set())): assert get_project_cxx_compile_flags() == [] + + +@pytest.mark.parametrize( + ("platform", "input_path", "expected"), + [ + # win32: drive-letter extended-length prefix is stripped + ( + "win32", + "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + ), + # win32: UNC extended-length prefix is translated to a regular UNC path + ( + "win32", + "\\\\?\\UNC\\server\\share\\python.exe", + "\\\\server\\share\\python.exe", + ), + # win32: paths without the prefix are returned unchanged + ( + "win32", + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + ), + # non-win32: prefix is left alone (no-op) + ("linux", "\\\\?\\C:\\python.exe", "\\\\?\\C:\\python.exe"), + ("darwin", "/usr/bin/python3", "/usr/bin/python3"), + ], +) +def test_strip_win_long_path_prefix( + platform: str, input_path: str, expected: str +) -> None: + r"""``\\?\`` and ``\\?\UNC\`` prefixes are stripped only on win32.""" + with patch("esphome.framework_helpers.sys.platform", platform): + assert framework_helpers.strip_win_long_path_prefix(input_path) == expected diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py new file mode 100644 index 0000000000..6ba8691c4e --- /dev/null +++ b/tests/unit_tests/test_platformio_registry.py @@ -0,0 +1,725 @@ +"""Tests for esphome.platformio.registry (PIO-registry package installs).""" + +from __future__ import annotations + +from contextlib import contextmanager +import json +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.core import EsphomeError +from esphome.platformio import registry + + +def test_registry_download_resolves_once_per_process() -> None: + """The prefetch and the install share one metadata resolve per package.""" + calls: list[dict] = [] + payload = { + "versions": [ + { + "name": "1.0.0", + "files": [ + { + "download_url": "http://x/pkg.tar.gz", + "checksum": {"sha256": "ab" * 32}, + "size": 5, + } + ], + } + ] + } + + def fake_request(method, url, **kwargs): + calls.append(url) + return _http_response(json.dumps(payload)) + + with patch.object(registry, "http_request", side_effect=fake_request): + first = registry.registry_download("o/pkg", "1.0.0") + second = registry.registry_download("o/pkg", "1.0.0") + assert first == second + assert len(calls) == 1 + + +@pytest.fixture(autouse=True) +def _fresh_registry_cache(): + # registry_download memoizes per process; tests reuse package names + registry.registry_download.cache_clear() + yield + registry.registry_download.cache_clear() + + +@pytest.mark.parametrize( + ("system", "machine", "expected"), + [ + ("Darwin", "arm64", "darwin_arm64"), + ("Darwin", "x86_64", "darwin_x86_64"), + ("Windows", "AMD64", "windows_amd64"), + # Deviation from upstream: auto-mapped to the emulated-x86 packages + ("Windows", "ARM64", "windows_amd64"), + ("Windows", "x86", "windows_x86"), + ("Linux", "x86_64", "linux_x86_64"), + ("Linux", "aarch64", "linux_aarch64"), + ("Linux", "i686", "linux_i686"), + ("Linux", "armv7l", "linux_armv7l"), + # Unknown hosts pass through like upstream; the registry lookup + # then fails naming the tag + ("FreeBSD", "amd64", "freebsd_amd64"), + ], +) +def test_get_systype(system: str, machine: str, expected: str) -> None: + with ( + patch("platform.system", return_value=system), + patch("platform.machine", return_value=machine), + patch("platform.architecture", return_value=("64bit", "")), + ): + assert registry.get_systype() == expected + + +def test_get_systype_env_override() -> None: + """PLATFORMIO_SYSTEM_TYPE wins, exactly as in upstream get_systype().""" + with patch.dict(os.environ, {"PLATFORMIO_SYSTEM_TYPE": "windows_amd64"}): + assert registry.get_systype() == "windows_amd64" + + +def test_get_systype_aarch64_32bit_userland() -> None: + """A 32-bit userland on a 64-bit arm kernel gets armv7l binaries.""" + with ( + patch("platform.system", return_value="Linux"), + patch("platform.machine", return_value="aarch64"), + patch("platform.architecture", return_value=("32bit", "")), + ): + assert registry.get_systype() == "linux_armv7l" + + +def test_get_systype_windows_empty_machine() -> None: + """An empty machine string falls back to the architecture bits.""" + with ( + patch("platform.system", return_value="Windows"), + patch("platform.machine", return_value=""), + patch("platform.architecture", return_value=("64bit", "")), + ): + assert registry.get_systype() == "windows_amd64" + + +def _http_response(text: str) -> MagicMock: + resp = MagicMock() + resp.text = text + resp.raise_for_status.return_value = None + return resp + + +def _registry_response(files: list[dict]): + """Patch the consolidated HTTP path to serve a canned registry response.""" + payload = {"versions": [{"name": "1.0.0", "files": files}]} + return patch.object( + registry, "http_request", return_value=_http_response(json.dumps(payload)) + ) + + +def test_registry_download_uses_shared_http_path() -> None: + """The metadata fetch delegates to the consolidated http_request path; + request failures surface as a named EsphomeError.""" + import requests as req + + with ( + patch.object( + registry, + "http_request", + side_effect=req.exceptions.ConnectionError("registry down"), + ) as mock_request, + pytest.raises(EsphomeError, match="Could not fetch registry metadata"), + ): + registry.registry_download("pkg", "1.0.0") + (method, url), _ = mock_request.call_args + assert method == "GET" + assert url == registry._REGISTRY_URL.format(package="pkg") + + +def test_registry_download_invalid_json_is_clean() -> None: + with ( + patch.object( + registry, + "http_request", + return_value=_http_response("not json"), + ), + pytest.raises(EsphomeError, match="invalid JSON"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_matches_system() -> None: + with ( + _registry_response( + [ + {"system": ["windows_amd64"], "download_url": "http://x/win"}, + { + "system": ["linux_x86_64"], + "download_url": "http://x/linux", + "checksum": {"sha256": "abc123"}, + "size": 42, + }, + ] + ), + patch.object(registry, "get_systype", return_value="linux_x86_64"), + ): + assert registry.registry_download("pkg", "1.0.0") == ( + "http://x/linux", + "abc123", + 42, + ) + + +def test_registry_download_bare_string_system() -> None: + """A bare-string system tag is an exact match, not a substring test.""" + with ( + _registry_response( + [ + {"system": "linux_x86", "download_url": "http://x/x86"}, + { + "system": "linux_x86_64", + "download_url": "http://x/x86_64", + "checksum": {"sha256": "abc"}, + }, + ] + ), + patch.object(registry, "get_systype", return_value="linux_x86_64"), + ): + assert registry.registry_download("pkg", "1.0.0")[0] == "http://x/x86_64" + + +def test_registry_download_wildcard_system() -> None: + with _registry_response( + [ + { + "system": "*", + "download_url": "http://x/any", + "checksum": {"sha256": "abc"}, + "size": 7, + } + ] + ): + assert registry.registry_download("pkg", "1.0.0") == ( + "http://x/any", + "abc", + 7, + ) + + +def test_registry_download_missing_checksum_raises() -> None: + """An unverifiable archive is refused, never silently extracted.""" + with ( + _registry_response([{"system": "*", "download_url": "http://x/any"}]), + pytest.raises(EsphomeError, match="no sha256"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_no_system_match() -> None: + with ( + _registry_response( + [{"system": ["windows_amd64"], "download_url": "http://x/win"}] + ), + patch.object(registry, "get_systype", return_value="linux_x86_64"), + pytest.raises(EsphomeError, match="No pkg 1.0.0 build"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_version_not_found() -> None: + with ( + patch.object( + registry, + "http_request", + return_value=_http_response( + json.dumps({"versions": [{"name": "2.0.0", "files": []}]}) + ), + ), + pytest.raises(EsphomeError, match="not found"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_install_package_skips_when_marker_exists(tmp_path: Path) -> None: + dest = tmp_path / "pkg" + (dest / "payload").mkdir(parents=True) + (dest / ".esphome_extracted").touch() + with patch.object(registry, "download_from_mirrors") as mock_download: + registry.install_package( + "pkg", "1.0.0", dest, [], tmp_path / "dl", expect=("payload",) + ) + mock_download.assert_not_called() + + +def test_install_package_marker_hit_rechecks_layout(tmp_path: Path) -> None: + """A marked install that later lost files fails by name instead of + surfacing as an opaque toolchain error.""" + dest = tmp_path / "pkg" + dest.mkdir() + (dest / ".esphome_extracted").touch() + with pytest.raises(EsphomeError, match="missing the expected payload"): + registry.install_package( + "pkg", "1.0.0", dest, [], tmp_path / "dl", expect=("payload",) + ) + + +def test_install_package_downloads_via_mirrors(tmp_path: Path) -> None: + dest = tmp_path / "pkg" + mirrors = ["http://mirror/{VERSION}/{SYSTEM}.tar.gz"] + with ( + patch.object(registry, "download_from_mirrors") as mock_download, + patch.object(registry, "archive_extract_all") as mock_extract, + patch.object(registry, "get_systype", return_value="linux_x86_64"), + ): + # Extraction is expected to create the directory + mock_extract.side_effect = lambda *_a, **_kw: (dest / "payload").mkdir( + parents=True + ) + registry.install_package( + "pkg", "1.0.0", dest, mirrors, tmp_path / "dl", expect=("payload",) + ) + assert mock_download.call_args[0][0] is mirrors + assert mock_download.call_args[0][1] == { + "VERSION": "1.0.0", + "SYSTEM": "linux_x86_64", + } + assert (dest / ".esphome_extracted").is_file() + + +def test_install_package_downloads_via_registry(tmp_path: Path) -> None: + """The registry path downloads with the registry's sha256 and size.""" + dest = tmp_path / "pkg" + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object(registry, "archive_extract_all") as mock_extract, + patch.object( + registry, + "registry_download", + return_value=("http://x/pkg.tar.gz", "abc123", 42), + ), + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "payload").mkdir( + parents=True + ) + registry.install_package( + "pkg", "1.0.0", dest, [], tmp_path / "dl", expect=("payload",) + ) + assert mock_download.call_args[0][0] == "http://x/pkg.tar.gz" + assert mock_download.call_args[1] == {"sha256": "abc123", "size": 42} + + +def test_install_package_validates_expected_layout(tmp_path: Path) -> None: + """The success marker is only written when the extracted tree is usable.""" + dest = tmp_path / "pkg" + with ( + patch.object(registry, "download_from_mirrors"), + patch.object(registry, "archive_extract_all") as mock_extract, + patch.object(registry, "get_systype", return_value="linux_x86_64"), + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "bin").mkdir(parents=True) + registry.install_package( + "pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl", expect=("bin",) + ) + assert (dest / ".esphome_extracted").is_file() + + +def test_install_package_unexpected_layout_raises(tmp_path: Path) -> None: + dest = tmp_path / "pkg" + with ( + patch.object(registry, "download_from_mirrors"), + patch.object(registry, "archive_extract_all") as mock_extract, + patch.object(registry, "get_systype", return_value="linux_x86_64"), + pytest.raises(EsphomeError, match="missing the expected bin"), + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "payload").mkdir( + parents=True + ) + registry.install_package( + "pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl", expect=("bin",) + ) + assert not (dest / ".esphome_extracted").exists() + + +def test_install_package_marker_rechecked_under_lock(tmp_path: Path) -> None: + """A concurrent install finishing while we wait for the lock is detected.""" + dest = tmp_path / "pkg" + marker = dest / ".esphome_extracted" + + @contextmanager + def _fake_lock(*_a, **_kw): + dest.mkdir(parents=True, exist_ok=True) + marker.touch() + yield + + with ( + patch("filelock.FileLock", _fake_lock), + patch.object(registry, "download_from_mirrors") as mock_download, + patch.object(registry, "rmdir") as mock_rmdir, + ): + registry.install_package( + "pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl", expect=("payload",) + ) + mock_download.assert_not_called() + mock_rmdir.assert_not_called() + + +def test_install_package_uses_hard_lock(tmp_path: Path) -> None: + """The install lock must never degrade to a soft (existence) lock.""" + dest = tmp_path / "pkg" + with ( + patch("filelock.FileLock") as mock_lock, + patch.object(registry, "download_from_mirrors"), + patch.object(registry, "archive_extract_all") as mock_extract, + patch.object(registry, "get_systype", return_value="linux_x86_64"), + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "payload").mkdir( + parents=True, exist_ok=True + ) + registry.install_package( + "pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl", expect=("payload",) + ) + assert mock_lock.call_args.kwargs["fallback_to_soft"] is False + + +def test_registry_download_empty_system_list_does_not_match() -> None: + """An explicitly empty system list must not act as a wildcard.""" + with ( + _registry_response([{"system": [], "download_url": "http://x/any"}]), + patch.object(registry, "get_systype", return_value="linux_x86_64"), + pytest.raises(EsphomeError, match="No pkg 1.0.0 build"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_unexpected_payload_is_named() -> None: + """An error envelope without a versions list is not 'version not found'.""" + + with ( + patch.object( + registry, + "http_request", + return_value=_http_response(json.dumps({"message": "rate limited"})), + ), + pytest.raises(EsphomeError, match="Unexpected package registry response"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_missing_system_key_matches_any() -> None: + """A file with no system key at all serves every host.""" + with _registry_response( + [{"download_url": "http://x/any", "checksum": {"sha256": "abc"}, "size": 1}] + ): + assert registry.registry_download("pkg", "1.0.0") == ("http://x/any", "abc", 1) + + +def test_registry_download_missing_files_list_is_named() -> None: + """A version entry without a files list is an unexpected payload, not a + missing platform build.""" + with ( + _registry_response(None), + pytest.raises(EsphomeError, match="Unexpected package registry response"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_missing_download_url_is_named() -> None: + with ( + _registry_response([{"system": "*", "checksum": {"sha256": "abc"}, "size": 1}]), + pytest.raises(EsphomeError, match="no download URL"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_install_package_empty_expect_rejected(tmp_path: Path) -> None: + """Layout validation is the only guard before marker.touch(), so an + empty expect is a caller bug, not a lenient install.""" + with pytest.raises(ValueError, match="non-empty expect"): + registry.install_package( + "pkg", "1.0.0", tmp_path / "pkg", [], tmp_path / "dl", expect=() + ) + + +def test_registry_download_non_dict_version_entry_is_named() -> None: + """A versions list of bare strings is an unexpected payload, not an + AttributeError traceback.""" + + with ( + patch.object( + registry, + "http_request", + return_value=_http_response(json.dumps({"versions": ["1.0.0", "2.0.0"]})), + ), + pytest.raises(EsphomeError, match="Unexpected package registry response"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_non_dict_file_entry_is_named() -> None: + with ( + patch.object( + registry, + "http_request", + return_value=_http_response( + json.dumps({"versions": [{"name": "1.0.0", "files": ["a.tar.gz"]}]}) + ), + ), + pytest.raises(EsphomeError, match="Unexpected package registry response"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_non_dict_payload_is_named() -> None: + """A JSON array answer is an unexpected payload at the outermost level.""" + + with ( + patch.object( + registry, + "http_request", + return_value=_http_response(json.dumps(["1.0.0"])), + ), + pytest.raises(EsphomeError, match="Unexpected package registry response"), + ): + registry.registry_download("pkg", "1.0.0") + + +def test_registry_download_non_list_system_is_named() -> None: + """A system field that is neither missing, str, nor list is an + unexpected payload, not a TypeError from the ``in`` test.""" + with ( + _registry_response([{"system": 5, "checksum": {"sha256": "abc"}, "size": 1}]), + pytest.raises(EsphomeError, match="Unexpected package registry response"), + ): + registry.registry_download("pkg", "1.0.0") + + +def _resolve_for(sizes: dict[str, int | None]): + def resolve(name: str, version: str): + size = sizes[name] + if size == -1: + raise EsphomeError("registry down") + return (f"http://x/{name}.tar.gz", "abc123", size) + + return resolve + + +def test_prefetch_packages_downloads_pending_in_parallel(tmp_path: Path) -> None: + """Two uninstalled packages download together under one combined bar, + with the registry's sha256 and size and a batch progress tracker.""" + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 20}) + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", []), + ("b", "2.0", tmp_path / "b", []), + ], + tmp_path / "dl", + ) + assert mock_download.call_count == 2 + # Locking makes worker completion order nondeterministic + calls = sorted(mock_download.call_args_list, key=lambda c: c[0][0]) + for call, (name, version, size) in zip( + calls, [("a", "1.0", 10), ("b", "2.0", 20)], strict=True + ): + assert call[0][0] == f"http://x/{name}.tar.gz" + assert call[0][1] == tmp_path / "dl" / f"{name}-{version}" + assert call[1]["sha256"] == "abc123" + assert call[1]["size"] == size + assert callable(call[1]["progress"]) + + +def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: + """A dest whose marker appeared while the worker waited on the lock is + already installed; re-downloading would orphan an archive copy.""" + dest = tmp_path / "a" + dest.mkdir() + + from contextlib import contextmanager + + @contextmanager + def marker_appears_under_lock(path, **kwargs): + # Simulates the concurrent build finishing while we waited + (dest / ".esphome_extracted").touch() + yield + + with ( + patch("filelock.FileLock", side_effect=marker_appears_under_lock), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10}) + ), + ): + registry.prefetch_packages([("a", "1.0", dest, [])], tmp_path / "dl") + mock_download.assert_not_called() + + +def test_already_installed_probe(tmp_path: Path) -> None: + """Both arms of the marker probe the prefetch worker keys on.""" + dest = tmp_path / "pkg" + dest.mkdir() + assert registry._already_installed(dest) is False + (dest / ".esphome_extracted").touch() + assert registry._already_installed(dest) is True + + +def test_prefetch_packages_dedupes_duplicate_entries(tmp_path: Path) -> None: + """Duplicate (name, version) entries would race each other between two + workers; only one survives (and one is too few to parallelize).""" + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10}) + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", []), + ("a", "1.0", tmp_path / "a", []), + ], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + +def test_prefetch_packages_single_pending_skips(tmp_path: Path) -> None: + """One pending package has nothing to parallelize; the sequential + install keeps its own bar.""" + marker_dest = tmp_path / "a" + marker_dest.mkdir() + (marker_dest / ".esphome_extracted").touch() + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"b": 20}) + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", marker_dest, []), + ("b", "2.0", tmp_path / "b", []), + ], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + +def test_prefetch_packages_mirror_and_sizeless_stay_sequential( + tmp_path: Path, +) -> None: + """Mirror overrides and size-less registry entries are left to the + sequential path so its per-file bars stay trustworthy.""" + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, + "registry_download", + side_effect=_resolve_for({"b": None, "c": 30}), + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", ["http://mirror/{VERSION}"]), + ("b", "2.0", tmp_path / "b", []), + ("c", "3.0", tmp_path / "c", []), + ], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + +def test_prefetch_packages_resolve_failure_defers_to_install( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A registry failure only skips the prefetch; install_package reports + the real error with context.""" + caplog.set_level("DEBUG") + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": -1, "b": 20}) + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", []), + ("b", "2.0", tmp_path / "b", []), + ], + tmp_path / "dl", + ) + mock_download.assert_not_called() + assert "Prefetch resolve for a failed" in caplog.text + + +def test_prefetch_packages_complete_archive_skipped(tmp_path: Path) -> None: + """An archive already fully downloaded is not re-fetched.""" + dl = tmp_path / "dl" + dl.mkdir() + (dl / "a-1.0").write_bytes(b"x" * 10) + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 20}) + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", []), + ("b", "2.0", tmp_path / "b", []), + ], + dl, + ) + mock_download.assert_not_called() + + +def test_prefetch_packages_download_failure_is_debug( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failed prefetch download is logged and left for install_package.""" + caplog.set_level("DEBUG") + with ( + patch.object( + registry, "download_with_resume", side_effect=OSError("boom") + ) as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 20}) + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", []), + ("b", "2.0", tmp_path / "b", []), + ], + tmp_path / "dl", + ) + assert mock_download.call_count == 2 + assert "Prefetch of a failed" in caplog.text + assert "Prefetch of b failed" in caplog.text + + +def test_prefetch_packages_unexpected_failure_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A programming error (not a download failure) surfaces at WARNING + instead of becoming a permanent silent no-op.""" + with ( + patch.object( + registry, "download_with_resume", side_effect=TypeError("bad call") + ), + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 20}) + ), + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", []), + ("b", "2.0", tmp_path / "b", []), + ], + tmp_path / "dl", + ) + assert "TypeError" in caplog.text diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 28304270a4..63c40f3609 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -431,8 +431,8 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None: with ( patch.dict(os.environ, {}, clear=True), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), - patch.object(toolchain.subprocess, "run"), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run"), ): env = toolchain._ccache_env() @@ -469,7 +469,7 @@ def test_ccache_env_disabled_without_binary( with ( patch.dict(os.environ, env_vars, clear=True), - patch.object(toolchain.shutil, "which", return_value=None), + patch("shutil.which", return_value=None), caplog.at_level("WARNING"), ): env = toolchain._ccache_env() @@ -494,8 +494,8 @@ def test_ccache_env_disabled_when_probe_fails( with ( patch.dict(os.environ, {}, clear=True), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), - patch.object(toolchain.subprocess, "run", side_effect=probe_error), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run", side_effect=probe_error), ): env = toolchain._ccache_env() @@ -508,8 +508,8 @@ def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), - patch.object(toolchain.subprocess, "run") as mock_probe, + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run") as mock_probe, ): env = toolchain._ccache_env() @@ -537,9 +537,9 @@ def test_ccache_env_strips_win_long_path_prefix(setup_core: Path) -> None: patch.dict(os.environ, {}, clear=True), # shutil.which is patched, so the win32 code path of the real # implementation (which crashes on a POSIX host) is never reached. - patch("esphome.platformio.toolchain.sys.platform", "win32"), - patch.object(toolchain.shutil, "which", return_value=prefixed), - patch.object(toolchain.subprocess, "run") as mock_probe, + patch("esphome.framework_helpers.sys.platform", "win32"), + patch("shutil.which", return_value=prefixed), + patch("esphome.framework_helpers.subprocess.run") as mock_probe, ): env = toolchain._ccache_env() @@ -555,7 +555,7 @@ def test_ccache_env_opt_out(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}, clear=True), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch("shutil.which", return_value="/usr/bin/ccache"), ): env = toolchain._ccache_env() @@ -568,7 +568,7 @@ def test_ccache_env_normalizes_enable_value(setup_core: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "yes"}, clear=True), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + patch("shutil.which", return_value="/usr/bin/ccache"), ): env = toolchain._ccache_env() @@ -587,8 +587,8 @@ def test_ccache_env_respects_user_values_and_refreshes_basedir( with ( patch.dict(os.environ, user_env, clear=True), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), - patch.object(toolchain.subprocess, "run"), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run"), ): env = toolchain._ccache_env() @@ -606,8 +606,8 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only( with ( patch.dict(os.environ, {}, clear=False), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), - patch.object(toolchain.subprocess, "run"), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run"), ): os.environ.pop("ESPHOME_CCACHE_ENABLE", None) mock_run_external_process.return_value = 0 @@ -628,8 +628,8 @@ def test_ccache_env_requires_build_path(setup_core: Path) -> None: with ( patch.dict(os.environ, {}, clear=True), - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), - patch.object(toolchain.subprocess, "run"), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run"), pytest.raises(ValueError, match="CORE.build_path must be set"), ): toolchain._ccache_env() @@ -642,8 +642,8 @@ def test_run_platformio_cli_merges_caller_env( CORE.build_path = str(setup_core / "build" / "test") with ( - patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), - patch.object(toolchain.subprocess, "run"), + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.framework_helpers.subprocess.run"), ): mock_run_external_process.return_value = 0 toolchain.run_platformio_cli( @@ -800,9 +800,7 @@ def test_ccache_env_real_probe_runs_stripped_path(setup_core: Path) -> None: with ( patch.dict(os.environ, {}, clear=False), - patch.object( - toolchain.shutil, "which", return_value="\\\\?\\" + sys.executable - ), + patch("shutil.which", return_value="\\\\?\\" + sys.executable), ): os.environ.pop("ESPHOME_CCACHE_ENABLE", None) env = toolchain._ccache_env() @@ -843,40 +841,6 @@ def test_ccache_wrapper_through_cmd_exe( assert marker.read_text() == "compiled" -@pytest.mark.parametrize( - ("platform", "input_path", "expected"), - [ - # win32: drive-letter extended-length prefix is stripped - ( - "win32", - "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", - "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", - ), - # win32: UNC extended-length prefix is translated to a regular UNC path - ( - "win32", - "\\\\?\\UNC\\server\\share\\python.exe", - "\\\\server\\share\\python.exe", - ), - # win32: paths without the prefix are returned unchanged - ( - "win32", - "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", - "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", - ), - # non-win32: prefix is left alone (no-op) - ("linux", "\\\\?\\C:\\python.exe", "\\\\?\\C:\\python.exe"), - ("darwin", "/usr/bin/python3", "/usr/bin/python3"), - ], -) -def test_strip_win_long_path_prefix( - platform: str, input_path: str, expected: str -) -> None: - r"""``\\?\`` and ``\\?\UNC\`` prefixes are stripped only on win32.""" - with patch("esphome.platformio.toolchain.sys.platform", platform): - assert toolchain._strip_win_long_path_prefix(input_path) == expected - - def test_run_platformio_cli_strips_win_long_path_prefix( setup_core: Path, mock_run_external_process: Mock ) -> None: @@ -900,7 +864,7 @@ def test_run_platformio_cli_strips_win_long_path_prefix( # so the stdlib sees it too) would send shutil.which down the Windows # code path, which crashes on a POSIX host. patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}, clear=False), - patch("esphome.platformio.toolchain.sys.platform", "win32"), + patch("esphome.framework_helpers.sys.platform", "win32"), patch("esphome.platformio.toolchain.sys.executable", prefixed_exe), ): # Pop any pre-existing PYTHONEXEPATH so the assertion below reflects @@ -932,7 +896,7 @@ def test_run_platformio_cli_does_not_set_pythonexepath_without_strip( with ( patch.dict(os.environ, {}, clear=False), - patch("esphome.platformio.toolchain.sys.platform", "linux"), + patch("esphome.framework_helpers.sys.platform", "linux"), patch("esphome.platformio.toolchain.sys.executable", plain_exe), ): os.environ.pop("PYTHONEXEPATH", None) @@ -1977,10 +1941,3 @@ def test_run_platformio_cli_invokes_heal( with patch.object(toolchain, "heal_platformio_python_env") as mock_heal: toolchain.run_platformio_cli("test") mock_heal.assert_called_once() - - -def test_ccache_probe_spawns_with_close_fds_false() -> None: - """The probe follows the repo-wide posix_spawn convention.""" - with patch("subprocess.run") as mock_run: - assert toolchain._ccache_runs("/usr/bin/ccache") is True - assert mock_run.call_args.kwargs["close_fds"] is False diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 0a53dba9c2..9c20ee10d2 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -14,6 +14,7 @@ from unittest.mock import MagicMock, patch import pytest +from esphome.build_helpers.tools_cache import TOOLS_CACHE_SPECS from esphome.const import ( PLATFORM_BK72XX, PLATFORM_ESP32, @@ -68,15 +69,12 @@ def _isolate_platformio_paths(tmp_path_factory: pytest.TempPathFactory) -> Any: test_clean_all_partial_exists) install their own inner patch which stacks on top of this one and wins for the duration of their block. - Also pin ``ESPHOME_ESP_IDF_PREFIX`` and ``ESPHOME_SDK_NRF_PREFIX`` to - nonexistent tmp dirs, and patch ``platformdirs.user_cache_dir``, for the - same reason: ``clean_all`` removes the machine-global toolchain installs + Also pin every ``TOOLS_CACHE_SPECS`` env override to a nonexistent tmp + dir, and patch ``platformdirs.user_cache_dir``, for the same reason: ``clean_all`` removes the machine-global toolchain installs and their default cache root, which otherwise resolve to the real ``~/.cache/esphome``. """ pio_root = tmp_path_factory.mktemp("isolated_pio") / "nonexistent" - idf_root = tmp_path_factory.mktemp("isolated_idf") / "nonexistent" - sdk_nrf_root = tmp_path_factory.mktemp("isolated_sdk_nrf") / "nonexistent" cache_root = tmp_path_factory.mktemp("isolated_cache") / "nonexistent" mock_cfg = MagicMock() mock_cfg.get.side_effect = lambda section, option: ( @@ -90,8 +88,12 @@ def _isolate_platformio_paths(tmp_path_factory: pytest.TempPathFactory) -> Any: patch.dict( "os.environ", { - "ESPHOME_ESP_IDF_PREFIX": str(idf_root), - "ESPHOME_SDK_NRF_PREFIX": str(sdk_nrf_root), + # Derived from the registry so a new backend's cache can + # never drift out of the sandbox and hit a real toolchain + env_var: str( + tmp_path_factory.mktemp(f"isolated_{subdir}") / "nonexistent" + ) + for env_var, subdir in TOOLS_CACHE_SPECS }, ), patch("platformdirs.user_cache_dir", return_value=str(cache_root)), From 9e4c52989c6015be0a95ff0761c4c9a935b44ef1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 20:24:47 -0500 Subject: [PATCH 43/65] [esp8266] Add the native framework and toolchain installer (#18557) --- esphome/arduino8266/__init__.py | 9 + esphome/arduino8266/framework.py | 164 +++++++++++++++++ esphome/components/esp8266/__init__.py | 11 +- .../unit_tests/test_arduino8266_framework.py | 170 ++++++++++++++++++ tests/unit_tests/test_writer.py | 22 +++ 5 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 esphome/arduino8266/__init__.py create mode 100644 esphome/arduino8266/framework.py create mode 100644 tests/unit_tests/test_arduino8266_framework.py diff --git a/esphome/arduino8266/__init__.py b/esphome/arduino8266/__init__.py new file mode 100644 index 0000000000..8f403a8553 --- /dev/null +++ b/esphome/arduino8266/__init__.py @@ -0,0 +1,9 @@ +"""Native (PlatformIO-free) build support for the ESP8266 Arduino core. + +This package downloads the Arduino ESP8266 core and the xtensa-lx106 +toolchain, generates a ninja build for them plus the ESPHome sources, and +drives the build directly — the ESP8266 equivalent of ``esphome.espidf``. + +Deliberately importable without the esp8266 component to avoid circular +imports; the component wires these modules in via lazy imports. +""" diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py new file mode 100644 index 0000000000..1edbe4b36f --- /dev/null +++ b/esphome/arduino8266/framework.py @@ -0,0 +1,164 @@ +"""Download and install the Arduino ESP8266 core, toolchain, and ninja. + +Artifacts land in a machine-global cache (shared across projects, like the +ESP-IDF install in ``esphome.espidf.framework``): + + /arduino8266/frameworks// framework-arduinoespressif8266 + /arduino8266/toolchains// toolchain-xtensa (gcc 10.3) + +Packages come from the PlatformIO registry (identical bits to the PlatformIO +backend); ``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes +from PATH or the ninja PyPI wheel. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import NamedTuple + +from esphome.build_helpers.ccache import ccache_defaults_env +from esphome.build_helpers.ninja import find_ninja +from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path +from esphome.core import EsphomeError, Version +from esphome.framework_helpers import str_to_lst_of_str +from esphome.platformio.registry import install_package, prefetch_packages + +FRAMEWORK_PACKAGE = "framework-arduinoespressif8266" +TOOLCHAIN_PACKAGE = "toolchain-xtensa" +# gcc 10.3, the toolchain Arduino core 3.x builds with; the build +# generator's compile flags are tuned to it. +TOOLCHAIN_VERSION = "2.100300.220621" + +ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "") +) +ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", "") +) + + +def get_arduino8266_tools_path() -> Path: + # Machine-global so all projects share one install; see + # espidf.framework.get_idf_tools_path for the location rationale. + return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) + + +# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the +# encoder below cannot name 3.0.0/3.0.1 either (see its docstring) +MIN_FRAMEWORK_VERSION = Version(3, 1, 1) + + +def framework_package_version(ver: Version) -> str: + """Map an Arduino core version to its registry package version (3.1.2 -> + 3.30102.0; the leading 3 is the package major). + + Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor + at MIN_FRAMEWORK_VERSION. + """ + if ver.major > 3: + raise EsphomeError( + f"Arduino core {ver} is not supported yet; " + "the newest known core series is 3.x" + ) + if ver <= Version(2, 6, 2): + # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same + # boundary as _format_framework_arduino_version's era guard) + raise EsphomeError( + f"Arduino core {ver} uses an older package encoding than this " + "helper implements (newer than 2.6.2)" + ) + return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + + +def get_framework_path(package_version: str) -> Path: + return get_arduino8266_tools_path() / "frameworks" / package_version + + +def get_toolchain_path() -> Path: + return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION + + +class InstalledPaths(NamedTuple): + """Locations of the installed framework, toolchain, and ninja binary.""" + + framework: Path + toolchain: Path + ninja: Path + + +def check_and_install(framework_version: Version) -> InstalledPaths: + """Ensure framework, toolchain, and ninja are installed; return their paths.""" + if framework_version < MIN_FRAMEWORK_VERSION: + # Config validation enforces this too; keep the module honest when + # called directly. + raise EsphomeError( + f"The native toolchain requires the Arduino core " + f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}" + ) + # Probe the cheap local dependency before ~110 MB of downloads + ninja_path = find_ninja() + package_version = framework_package_version(framework_version) + framework_path = get_framework_path(package_version) + downloads_dir = get_arduino8266_tools_path() / "downloads" + toolchain_path = get_toolchain_path() + # One spec per package: the prefetch and the installs must agree + specs = ( + ( + FRAMEWORK_PACKAGE, + package_version, + framework_path, + ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + ("cores/esp8266", "tools/sdk", "libraries"), + ), + ( + TOOLCHAIN_PACKAGE, + TOOLCHAIN_VERSION, + toolchain_path, + ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + # xtensa-lx106-elf pins the target: every gcc package has a bin/ + ("bin", "xtensa-lx106-elf"), + ), + ) + # Fetch both archives at once; the installs below verify and extract + prefetch_packages([spec[:4] for spec in specs], downloads_dir) + for name, version, dest, mirrors, expect in specs: + install_package(name, version, dest, mirrors, downloads_dir, expect=expect) + return InstalledPaths( + framework=framework_path, toolchain=toolchain_path, ninja=ninja_path + ) + + +def toolchain_tool(toolchain_path: Path, name: str) -> Path: + """Path to one toolchain tool (gcc, g++, ar, size, addr2line, ...). + + The single owner of the ``bin/xtensa-lx106-elf-`` layout and the + Windows suffix, so a toolchain package bump touches one spot. + """ + suffix = ".exe" if os.name == "nt" else "" + return toolchain_path / "bin" / f"xtensa-lx106-elf-{name}{suffix}" + + +def get_build_env(toolchain_path: Path, ccache: str | None) -> dict[str, str]: + env = os.environ.copy() + # Drop empty entries: a trailing separator from an absent PATH would + # make the shell search the current directory for tools + parts = [ + str(toolchain_path / "bin"), + *filter(None, env.get("PATH", "").split(os.pathsep)), + ] + env["PATH"] = os.pathsep.join(parts) + env.update(ccache_env(ccache)) + return env + + +def ccache_env(ccache: str | None) -> dict[str, str]: + """Return ccache settings for the build subprocess (not os.environ). + + ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None + when disabled. Values the user already set in the environment are + respected. + """ + if ccache is None: + return {} + return ccache_defaults_env(get_arduino8266_tools_path() / "ccache") diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 6f29cd7774..63665e7681 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -137,7 +137,16 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" if ver <= cv.Version(2, 6, 2): return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + # Same encoding the native toolchain uses for its package download, so a + # version bump cannot drift between the two paths. + from esphome.arduino8266.framework import framework_package_version + + try: + return f"~{framework_package_version(ver)}" + except EsphomeError as err: + # Anchor the 4.x rejection to the framework version line instead of + # aborting with a bare traceback-level error + raise cv.Invalid(str(err), path=[CONF_VERSION]) from err # NOTE: Keep this in mind when updating the recommended version: diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py new file mode 100644 index 0000000000..bd0a620e10 --- /dev/null +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -0,0 +1,170 @@ +"""Tests for esphome.arduino8266.framework (downloads and environment).""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.arduino8266 import framework +import esphome.config_validation as cv +from esphome.core import CORE, EsphomeError + + +@pytest.fixture(autouse=True) +def _build_path(tmp_path: Path) -> None: + CORE.build_path = tmp_path + + +def test_framework_package_version() -> None: + assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" + assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" + # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path) + assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0" + # A future major bump needs its own encoding, not a doomed registry lookup + with pytest.raises(EsphomeError, match="not supported yet"): + framework.framework_package_version(cv.Version(4, 0, 0)) + # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release + # keeps this encoding + with pytest.raises(EsphomeError, match="older package encoding"): + framework.framework_package_version(cv.Version(2, 6, 2)) + assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0" + assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0" + + +def test_format_framework_arduino_version_pins_all_series() -> None: + """The esp8266 component's PIO source formatter across every encoding + era, including the 4.x rejection it now shares with the installer.""" + from esphome.components.esp8266 import _format_framework_arduino_version as fmt + + assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0" + assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0" + assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0" + assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" + # Anchored to the framework version line, not a bare EsphomeError + with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: + fmt(cv.Version(4, 0, 0)) + assert excinfo.value.path == ["version"] + + +def test_tools_path_default_and_prefix(tmp_path: Path) -> None: + with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}): + assert framework.get_arduino8266_tools_path() == tmp_path.resolve() + # A blank prefix must be treated as unset, not as the CWD + with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": " "}): + path = framework.get_arduino8266_tools_path() + assert path.name == "arduino8266" + assert path != Path.cwd() + + +def test_check_and_install_returns_paths(tmp_path: Path) -> None: + with ( + patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "install_package") as mock_install, + patch.object(framework, "prefetch_packages") as mock_prefetch, + patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), + ): + paths = framework.check_and_install(cv.Version(3, 1, 2)) + assert paths.framework == tmp_path / "frameworks" / "3.30102.0" + assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION + assert paths.ninja == tmp_path / "ninja" + assert mock_install.call_count == 2 + # Full argument pinning: a copy-paste swap between the two near-identical + # calls (mirrors, destination) must not stay green + fw_call, tc_call = mock_install.call_args_list + assert fw_call.args == ( + framework.FRAMEWORK_PACKAGE, + "3.30102.0", + tmp_path / "frameworks" / "3.30102.0", + framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + tmp_path / "downloads", + ) + assert fw_call.kwargs["expect"] == ("cores/esp8266", "tools/sdk", "libraries") + assert tc_call.args == ( + framework.TOOLCHAIN_PACKAGE, + framework.TOOLCHAIN_VERSION, + tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION, + framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + tmp_path / "downloads", + ) + assert tc_call.kwargs["expect"] == ("bin", "xtensa-lx106-elf") + # The prefetch sees the same package specs as the installs + assert mock_prefetch.call_args.args == ( + [ + ( + framework.FRAMEWORK_PACKAGE, + "3.30102.0", + tmp_path / "frameworks" / "3.30102.0", + framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + ), + ( + framework.TOOLCHAIN_PACKAGE, + framework.TOOLCHAIN_VERSION, + tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION, + framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + ), + ], + tmp_path / "downloads", + ) + + +def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: + with patch.object(framework, "ccache_env", return_value={"CCACHE_DIR": "x"}): + env = framework.get_build_env(tmp_path, None) + assert env["PATH"].startswith(str(tmp_path / "bin") + os.pathsep) + assert env["CCACHE_DIR"] == "x" + + +def test_ccache_env(tmp_path: Path) -> None: + assert framework.ccache_env(None) == {} + with patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True): + env = framework.ccache_env("/usr/bin/ccache") + # User-set values are respected; the rest get defaults + assert "CCACHE_NOHASHDIR" not in env + assert env["CCACHE_DEPEND"] == "1" + assert env["CCACHE_BASEDIR"] == str(Path(CORE.build_path).resolve()) + assert env["CCACHE_DIR"].endswith("ccache") + + +def test_check_and_install_rejects_old_core(tmp_path: Path) -> None: + """Calling the installer below the floor fails before any download.""" + with pytest.raises(EsphomeError, match=">= 3.1.1"): + framework.check_and_install(cv.Version(3, 0, 2)) + + +def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: + """An absent PATH must not leave a trailing separator (an empty entry + means the current directory to the shell).""" + with ( + patch.dict(os.environ, {}, clear=True), + patch.object(framework, "ccache_env", return_value={}), + ): + env = framework.get_build_env(tmp_path, None) + assert env["PATH"] == str(tmp_path / "bin") + with ( + patch.dict( + os.environ, {"PATH": f"/usr/bin{os.pathsep}{os.pathsep}/bin"}, clear=True + ), + patch.object(framework, "ccache_env", return_value={}), + ): + env = framework.get_build_env(tmp_path, None) + assert env["PATH"].split(os.pathsep) == [str(tmp_path / "bin"), "/usr/bin", "/bin"] + + +def test_ccache_env_accepts_a_preresolved_path() -> None: + """The caller resolves ccache once and threads it through; None means + resolved-and-disabled.""" + with patch.dict(os.environ, {}, clear=True): + assert framework.ccache_env(None) == {} + env = framework.ccache_env("/usr/bin/ccache") + assert env["CCACHE_DIR"].endswith("ccache") + + +def test_toolchain_tool_layout(tmp_path: Path) -> None: + """One owner for the bin/xtensa-lx106-elf- layout.""" + tool = framework.toolchain_tool(tmp_path, "addr2line") + assert tool.parent == tmp_path / "bin" + assert tool.name.startswith("xtensa-lx106-elf-addr2line") + assert (tool.suffix == ".exe") is (os.name == "nt") diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 9c20ee10d2..47feae3e3c 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -1059,6 +1059,28 @@ def test_clean_all_removes_global_sdk_nrf_install( assert str(sdk_nrf_install.resolve()) in caplog.text +@patch("esphome.writer.CORE") +def test_clean_all_removes_global_arduino8266_install( + mock_core: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """clean_all removes the machine-global native arduino8266 install dir.""" + arduino8266_install = tmp_path / "arduino8266_install" + (arduino8266_install / "frameworks").mkdir(parents=True) + monkeypatch.setenv("ESPHOME_ARDUINO8266_PREFIX", str(arduino8266_install)) + + config_dir = tmp_path / "config" + config_dir.mkdir() + + with caplog.at_level("INFO"): + clean_all([str(config_dir)]) + + assert not arduino8266_install.exists() + assert str(arduino8266_install.resolve()) in caplog.text + + @patch("esphome.writer.CORE") def test_clean_all_removes_default_cache_root( mock_core: MagicMock, From 8b8de0c9c65923a845981ad317dd3ad919697040 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:33:39 +1000 Subject: [PATCH 44/65] [lvgl] Fix on_value/on_update triggers for LVGL select entities (#18778) Co-authored-by: Claude Sonnet 5 --- esphome/components/lvgl/lvgl_esphome.cpp | 8 ++-- esphome/components/lvgl/lvgl_esphome.h | 6 +-- esphome/components/lvgl/select/lvgl_select.h | 15 ++----- esphome/components/lvgl/types.py | 3 ++ .../dropdown_update_fires_event_test.yaml | 36 ++++++++++++++++ .../lvgl/test_dropdown_update_fires_event.py | 41 +++++++++++++++++++ 6 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml create mode 100644 tests/component_tests/lvgl/test_dropdown_update_fires_event.py diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index 684f472ebd..a10fdb0582 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -597,21 +597,21 @@ std::string LvSelectable::get_selected_text() { return this->options_[selected]; } -static std::string join_string(std::vector options) { +static std::string join_string(const FixedVector &options) { return std::accumulate( options.begin(), options.end(), std::string(), - [](const std::string &a, const std::string &b) -> std::string { return a + (!a.empty() ? "\n" : "") + b; }); + [](const std::string &a, const char *b) -> std::string { return a + (!a.empty() ? "\n" : "") + b; }); } void LvSelectable::set_selected_text(const std::string &text, lv_anim_enable_t anim) { - auto index = std::find(this->options_.begin(), this->options_.end(), text); + auto *index = std::find(this->options_.begin(), this->options_.end(), text); if (index != this->options_.end()) { this->set_selected_index(index - this->options_.begin(), anim); lv_obj_send_event(this->obj, lv_update_event, nullptr); } } -void LvSelectable::set_options(std::vector options) { +void LvSelectable::set_options(FixedVector options) { auto index = this->get_selected_index(); if (index >= options.size()) index = options.size() - 1; diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index ceba786e43..8b7397c4cd 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -543,12 +543,12 @@ class LvSelectable : public LvCompound { virtual void set_selected_index(size_t index, lv_anim_enable_t anim) = 0; void set_selected_text(const std::string &text, lv_anim_enable_t anim); std::string get_selected_text(); - const std::vector &get_options() { return this->options_; } - void set_options(std::vector options); + const FixedVector &get_options() { return this->options_; } + void set_options(FixedVector options); protected: virtual void set_option_string(const char *options) = 0; - std::vector options_{}; + FixedVector options_{}; }; #ifdef USE_LVGL_DROPDOWN diff --git a/esphome/components/lvgl/select/lvgl_select.h b/esphome/components/lvgl/select/lvgl_select.h index e36357328c..dafdd91eb5 100644 --- a/esphome/components/lvgl/select/lvgl_select.h +++ b/esphome/components/lvgl/select/lvgl_select.h @@ -50,19 +50,10 @@ class LVGLSelect final : public select::Select, public Component { protected: void control(size_t index) override { this->widget_->set_selected_index(index, this->anim_); - this->publish(); - } - void set_options_() { - // Widget uses std::vector, SelectTraits uses FixedVector - // Convert by extracting c_str() pointers - const auto &opts = this->widget_->get_options(); - FixedVector opt_ptrs; - opt_ptrs.init(opts.size()); - for (const auto &opt : opts) { - opt_ptrs.push_back(opt.c_str()); - } - this->traits.set_options(opt_ptrs); + // The update event fires the widget's on_value/on_update triggers + lv_obj_send_event(this->widget_->obj, lv_update_event, nullptr); } + void set_options_() { this->traits.set_options(this->widget_->get_options()); } LvSelectable *widget_; lv_anim_enable_t anim_; diff --git a/esphome/components/lvgl/types.py b/esphome/components/lvgl/types.py index 61efe385e6..cc8d9438a9 100644 --- a/esphome/components/lvgl/types.py +++ b/esphome/components/lvgl/types.py @@ -3,6 +3,8 @@ from esphome.const import CONF_TEXT, CONF_VALUE from esphome.cpp_generator import MockObj from esphome.cpp_types import Component, esphome_ns +from .defines import CONF_SELECTED_INDEX + class LvType(cg.MockObjClass): def __init__(self, *args, **kwargs): @@ -112,3 +114,4 @@ class LvSelect(LvType): parents=parens, **kwargs, ) + self.value_property = CONF_SELECTED_INDEX diff --git a/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml b/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml new file mode 100644 index 0000000000..2fe59b2f1a --- /dev/null +++ b/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml @@ -0,0 +1,36 @@ +esphome: + name: test-dropdown-update-event + on_boot: + - lvgl.dropdown.update: + id: test_dropdown + selected_index: 2 + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +lvgl: + widgets: + - dropdown: + id: test_dropdown + options: + - First + - Second + - Third + on_update: + - lambda: |- + ESP_LOGD("test", "dropdown updated"); diff --git a/tests/component_tests/lvgl/test_dropdown_update_fires_event.py b/tests/component_tests/lvgl/test_dropdown_update_fires_event.py new file mode 100644 index 0000000000..1e034ad6eb --- /dev/null +++ b/tests/component_tests/lvgl/test_dropdown_update_fires_event.py @@ -0,0 +1,41 @@ +"""Regression test: lvgl.dropdown.update with selected_index must fire on_value/on_update. + +LvSelect (backing both dropdown and roller) did not set `value_property`, so the generic +update-action machinery in automation.py never sent the synthetic update event for a +`selected_index:` change made via `lvgl.dropdown.update`/`lvgl.roller.update`, unlike `value:` +on number widgets or `text:` on text widgets. Fixed by setting `LvSelect.value_property` to +`CONF_SELECTED_INDEX`. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + config_path = ( + Path(request.fspath).parent / "config" / "dropdown_update_fires_event_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_dropdown_update_sends_update_event(main_cpp: str) -> None: + assert ( + "lv_obj_send_event(test_dropdown->obj, lvgl::lv_update_event, nullptr)" + in main_cpp + ) From cb981c2930f2efe813b59f7dc5beedbe82cda55b Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 25 Aug 2026 22:19:54 -0500 Subject: [PATCH 45/65] [remote_transmitter] ISR-driven transmission and non_blocking support on RTL8720C (#18648) --- .../components/remote_transmitter/__init__.py | 16 +- .../remote_transmitter/remote_transmitter.h | 31 +- .../remote_transmitter_rtl87xx.cpp | 285 ++++++++++++++++-- .../remote_transmitter/__init__.py | 0 .../test_non_blocking_gate.py | 42 +++ .../remote_transmitter/test.rtl87xx-ard.yaml | 1 + 6 files changed, 350 insertions(+), 25 deletions(-) create mode 100644 tests/component_tests/remote_transmitter/__init__.py create mode 100644 tests/component_tests/remote_transmitter/test_non_blocking_gate.py diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index 9d8761ea90..8ae51829e7 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -3,6 +3,8 @@ import logging from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base +from esphome.components.libretiny import get_libretiny_family +from esphome.components.libretiny.const import FAMILY_RTL8720C from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -43,6 +45,16 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) +def _validate_non_blocking_platform(value: bool) -> bool: + # non_blocking requires hardware transmission: RMT on ESP32, the gtimer + # envelope chain on RTL8720C. Reject everywhere else at config time. + if CORE.is_esp32: + return cv.boolean(value) + if CORE.is_libretiny and get_libretiny_family() == FAMILY_RTL8720C: + return cv.boolean(value) + raise cv.Invalid("non_blocking is only supported on ESP32 and RTL8720C") + + MULTI_CONF = True CONFIG_SCHEMA = ( cv.Schema( @@ -76,7 +88,7 @@ CONFIG_SCHEMA = ( esp32_s2=64, esp32_s3=48, ): cv.All(cv.only_on_esp32, cv.int_range(min=2)), - cv.Optional(CONF_NON_BLOCKING): cv.All(cv.only_on_esp32, cv.boolean), + cv.Optional(CONF_NON_BLOCKING): _validate_non_blocking_platform, cv.Optional(CONF_ON_TRANSMIT): automation.validate_automation(single=True), cv.Optional(CONF_ON_COMPLETE): automation.validate_automation(single=True), } @@ -164,6 +176,8 @@ async def to_code(config: ConfigType) -> None: ) else: var = cg.new_Pvariable(config[CONF_ID], pin) + if (non_blocking := config.get(CONF_NON_BLOCKING)) is not None: + cg.add(var.set_non_blocking(non_blocking)) await cg.register_component(var, config) cg.add(var.set_carrier_duty_percent(config[CONF_CARRIER_DUTY_PERCENT])) diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index 94bcb74b09..ef9a80f668 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -56,15 +56,23 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa #if defined(USE_ESP32) && SOC_RMT_SUPPORTED void set_with_dma(bool with_dma) { this->with_dma_ = with_dma; } void set_eot_level(bool eot_level) { this->eot_level_ = eot_level; } +#endif +#if (defined(USE_ESP32) && SOC_RMT_SUPPORTED) || defined(USE_LIBRETINY_VARIANT_RTL8720C) void set_non_blocking(bool non_blocking) { this->non_blocking_ = non_blocking; } #endif +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + void loop() override; + // called from the envelope timer ISR trampoline; not part of the public API + void advance_envelope_isr(); +#endif Trigger<> *get_transmit_trigger() { return &this->transmit_trigger_; } Trigger<> *get_complete_trigger() { return &this->complete_trigger_; } protected: void send_internal(uint32_t send_times, uint32_t send_wait) override; -#if defined(USE_ESP8266) || defined(USE_LIBRETINY) || defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) +#if defined(USE_ESP8266) || (defined(USE_LIBRETINY) && !defined(USE_LIBRETINY_VARIANT_RTL8720C)) || \ + defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) void await_target_time_(); uint32_t target_time_{0}; #endif @@ -81,6 +89,27 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa uint32_t current_carrier_frequency_{0}; void *pwm_{nullptr}; // pwmout_t*, opaque here to keep the SDK header out of this shared header #endif +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + void start_isr_item_(size_t index); + void arm_envelope_timer_(uint32_t duration_us); + void abort_stalled_chain_(); + void deliver_completion_(); + void wait_until_idle_(); + void arm_chain_(uint32_t send_times, uint32_t send_wait); + void update_carrier_(uint32_t carrier_frequency); + std::vector isr_data_; // owned copy of the frame; temp_ may be re-encoded mid-flight + float isr_mark_duty_{0.0f}; + float isr_space_duty_{0.0f}; + volatile size_t isr_index_{0}; + volatile uint32_t isr_repeats_left_{0}; + uint32_t isr_send_wait_{0}; + volatile uint32_t isr_wait_remaining_{0}; // remainder of a duration chained across one-shots + volatile bool isr_in_gap_{false}; + volatile bool transmitting_{false}; + bool non_blocking_{false}; + bool complete_pending_{false}; + bool stall_aborted_{false}; // this transmission ended via abort; blocks warning clear +#endif #if defined(USE_ESP32) && SOC_RMT_SUPPORTED void configure_rmt_(); diff --git a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp index b7078b9d69..9f629168f2 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp @@ -5,31 +5,49 @@ // clang-tidy cannot parse the Realtek SDK headers pulled in via ArduinoPrivate.h #if defined(USE_RTL87XX) && !defined(CLANG_TIDY) -// ArduinoPrivate.h = Arduino.h + the SDK's mbed HAL (pwmout etc.) with the core's fixes for +// ArduinoPrivate.h = Arduino.h + the SDK's mbed HAL (pwmout, gtimer) with the core's fixes for // type-name collisions between the two (e.g. PinMode) #include +#ifndef USE_LIBRETINY_VARIANT_RTL8720C #include #include +#endif namespace esphome::remote_transmitter { static const char *const TAG = "remote_transmitter"; -// The carrier is generated by the PWM peripheral instead of bit-banging the pin: software carrier -// generation requires disabling interrupts for the whole frame, but this core's micros() is derived -// from the FreeRTOS tick and freezes while interrupts are off, so the timing loop never advances and -// the watchdog resets the chip. With hardware PWM, software only times the mark/space envelope and -// interrupts can stay enabled. -// -// The PWM is driven through the SDK's pwmout HAL directly rather than the Arduino wiring layer: -// changing the carrier frequency via the wiring requires a GPIO/PWM pin mode round-trip, which -// use-after-frees the core's per-pin state (pinRemoveMode() frees without nulling) and corrupts the -// heap. pwmout_period_us() changes the frequency with no mode transitions. +// PWM peripheral carrier, envelope paced by a gtimer interrupt chain. Bit-banging would need +// interrupts disabled for the whole frame, but this core's micros() derives from the FreeRTOS +// tick and freezes then. The SDK pwmout HAL is driven directly: the Arduino wiring layer's +// GPIO/PWM mode round-trip use-after-frees LibreTiny's per-pin state. + +#ifdef USE_LIBRETINY_VARIANT_RTL8720C +static constexpr uint32_t ENVELOPE_TIMER_ID = TIMER6; // GTimer7 +// Margin past a transmission's expected duration before the chain is declared stalled +static constexpr uint32_t STALL_MARGIN_MS = 1000; +// Longest single one-shot armed; longer durations are chained (ROM us->tick headroom unverified) +static constexpr uint32_t MAX_ONE_SHOT_US = 50000; + +// Shared envelope timer: a second gtimer_init on the same id fails silently, so all +// instances serialize on s_active_transmitter +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) +static uint8_t s_pwm_tick_sources[] = {GTimer1, GTimer2, GTimer3, GTimer4, GTimer5, GTimer6, 0xff}; +static gtimer_t s_envelope_timer; +static bool s_envelope_timer_ready = false; +static RemoteTransmitterComponent *volatile s_active_transmitter = nullptr; +// Deadline for the in-flight transmission (millis-based); only touched from the main task +static uint32_t s_expected_end_ms = 0; +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) + +static void IRAM_ATTR envelope_timer_isr(uint32_t arg) { + reinterpret_cast(arg)->advance_envelope_isr(); +} +#endif // USE_LIBRETINY_VARIANT_RTL8720C void RemoteTransmitterComponent::setup() { - // Deliberately no pin_->setup(): registering the pin as GPIO claims it in the SDK's pin - // management, and the pad is then never handed over to the PWM peripheral -- pwmout_init() - // must own the pin from the start. + // no pin_->setup(): a GPIO claim in the SDK's pin management blocks pwmout_init from + // owning the pad PinInfo *info = pinInfo(this->pin_->get_pin()); if (info == nullptr || !pinSupported(info, PIN_PWM)) { // checked here because the AmebaZ (RTL8710B) SDK does not report PWM init failure @@ -40,7 +58,7 @@ void RemoteTransmitterComponent::setup() { auto *pwm = new pwmout_t(); this->pwm_ = pwm; pwmout_init(pwm, static_cast(info->gpio)); -#if LT_RTL8720C +#ifdef USE_LIBRETINY_VARIANT_RTL8720C // only the AmebaZ2 SDK's pwmout_s reports init success if (!pwm->is_init) { ESP_LOGE(TAG, "PWM init failed on pin %u", this->pin_->get_pin()); @@ -49,9 +67,19 @@ void RemoteTransmitterComponent::setup() { this->mark_failed(); return; } + // Shrink the PWM tick-source pool before the period claim below so GTimer7 stays free + // for the envelope; pwmout_init just registered the full pool. + hal_pwm_comm_tick_source_list(s_pwm_tick_sources); #endif pwmout_period_us(pwm, 26); // placeholder; the real carrier period is set per transmission pwmout_write(pwm, this->pin_->is_inverted() ? 1.0f : 0.0f); +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + if (!s_envelope_timer_ready) { + gtimer_init(&s_envelope_timer, ENVELOPE_TIMER_ID); + s_envelope_timer_ready = true; + } + this->disable_loop(); // loop() is only needed while a non-blocking completion is pending +#endif } void RemoteTransmitterComponent::dump_config() { @@ -59,9 +87,224 @@ void RemoteTransmitterComponent::dump_config() { "Remote Transmitter:\n" " Carrier Duty: %u%%", this->carrier_duty_percent_); +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + ESP_LOGCONFIG(TAG, " Non-blocking: %s", YESNO(this->non_blocking_)); +#endif LOG_PIN(" Pin: ", this->pin_); } +void RemoteTransmitterComponent::digital_write(bool value) { + if (this->pwm_ == nullptr) + return; +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + // serialize behind an in-flight chain, matching the ESP32/RMT non-blocking behavior + this->wait_until_idle_(); +#endif + pwmout_write(static_cast(this->pwm_), (value != this->pin_->is_inverted()) ? 1.0f : 0.0f); +} + +#ifdef USE_LIBRETINY_VARIANT_RTL8720C +// Arms the shared envelope timer, chaining durations longer than MAX_ONE_SHOT_US. ISR-safe. +void IRAM_ATTR RemoteTransmitterComponent::arm_envelope_timer_(uint32_t duration_us) { + // clamp to 1us (a zero-length one-shot never fires); the remainder must not underflow + const uint32_t chunk = std::max(uint32_t(1), std::min(duration_us, MAX_ONE_SHOT_US)); + this->isr_wait_remaining_ = duration_us > chunk ? duration_us - chunk : 0; + gtimer_start_one_shout(&s_envelope_timer, chunk, (void *) envelope_timer_isr, (uint32_t) this); +} + +// Aborts a chain that stopped advancing: stop the timer, idle the pin, release the token. +// Every step is a no-op if the chain completed meanwhile. Task context only. +void RemoteTransmitterComponent::abort_stalled_chain_() { + // cleared first so a straggler one-shot bails at the ISR entry check + this->transmitting_ = false; + gtimer_stop(&s_envelope_timer); + pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); + s_active_transmitter = nullptr; + this->stall_aborted_ = true; + this->status_set_warning("envelope timer stalled"); + ESP_LOGE(TAG, "Envelope timer stalled; transmission aborted"); + delay(1); // let any already-latched interrupt land while the chain state is safe +} + +// Delivers one deferred completion with its status bookkeeping +void RemoteTransmitterComponent::deliver_completion_() { + if (!this->stall_aborted_) + this->status_clear_warning(); + this->complete_pending_ = false; + this->complete_trigger_.trigger(); +} + +// Writes the duty for one envelope item and arms the timer for its duration. +// Runs in ISR context (and once from send_internal to kick the chain): no logging, no allocation. +void IRAM_ATTR RemoteTransmitterComponent::start_isr_item_(size_t index) { + const int32_t item = this->isr_data_[index]; + pwmout_write(static_cast(this->pwm_), item > 0 ? this->isr_mark_duty_ : this->isr_space_duty_); + this->arm_envelope_timer_(uint32_t(item > 0 ? item : -item)); +} + +void IRAM_ATTR RemoteTransmitterComponent::advance_envelope_isr() { + if (!this->transmitting_) + return; // chain was aborted; this is a stale one-shot that was already latched + if (this->isr_wait_remaining_ > 0) { + // continue a duration longer than one hardware one-shot + this->arm_envelope_timer_(this->isr_wait_remaining_); + return; + } + if (this->isr_in_gap_) { + // inter-repeat gap elapsed; restart the item chain + this->isr_in_gap_ = false; + this->isr_index_ = 0; + this->start_isr_item_(0); + return; + } + this->isr_index_++; + if (this->isr_index_ < this->isr_data_.size()) { + this->start_isr_item_(this->isr_index_); + return; + } + // end of one repetition + pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); + if (this->isr_repeats_left_ > 1) { + this->isr_repeats_left_--; + this->isr_index_ = 0; + if (this->isr_send_wait_ > 0) { + this->isr_in_gap_ = true; + this->arm_envelope_timer_(this->isr_send_wait_); + } else { + this->start_isr_item_(0); + } + return; + } + this->transmitting_ = false; + s_active_transmitter = nullptr; +} + +// Waits until no chain is in flight, delivering any deferred completions; a completion +// automation may start a new send, so repeat until truly idle. Bounded by the stall deadline. +void RemoteTransmitterComponent::wait_until_idle_() { + while (true) { + while (true) { + // snapshot: the final ISR can clear the volatile pointer between a check and a use + auto *active = s_active_transmitter; + if (active == nullptr) + break; + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + active->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + if (!this->complete_pending_) + break; + this->deliver_completion_(); + } +} + +// Retunes the PWM period when the carrier changes; the ISR sets duty per item +void RemoteTransmitterComponent::update_carrier_(uint32_t carrier_frequency) { + if (carrier_frequency == 0 || carrier_frequency == this->current_carrier_frequency_) + return; + // round(1000000/freq), clamped so a bad lambda can't hand the SDK a zero period + const uint32_t period = std::max(uint32_t(1), (1000000UL + carrier_frequency / 2) / carrier_frequency); + pwmout_period_us(static_cast(this->pwm_), period); + this->current_carrier_frequency_ = carrier_frequency; +} + +// Stages the repeat schedule and stall deadline, then starts the interrupt chain +void RemoteTransmitterComponent::arm_chain_(uint32_t send_times, uint32_t send_wait) { + this->isr_repeats_left_ = send_times; + this->isr_send_wait_ = send_wait; + this->isr_index_ = 0; + this->isr_in_gap_ = false; + this->stall_aborted_ = false; + uint64_t frame_us = 0; + for (int32_t item : this->isr_data_) + frame_us += uint32_t(item > 0 ? item : -item); + const uint64_t total_us = frame_us * send_times + uint64_t(send_wait) * (send_times - 1); + s_expected_end_ms = millis() + uint32_t(total_us / 1000) + STALL_MARGIN_MS; + this->transmitting_ = true; + s_active_transmitter = this; + this->start_isr_item_(0); +} + +void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { + if (this->pwm_ == nullptr) { + ESP_LOGW(TAG, "Cannot send: PWM not initialized"); + return; + } + this->wait_until_idle_(); + if (send_times == 0) { + // parity with the loop-based implementations: transmit nothing, but both triggers + // still fire so an on_complete-sequenced automation does not stall + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + ESP_LOGD(TAG, "Sending remote code"); + const uint32_t carrier_frequency = this->temp_.get_carrier_frequency(); + // unmodulated protocols (no carrier or 100% duty) drive the pin constantly during marks + float mark_duty = + (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) ? this->carrier_duty_percent_ / 100.0f : 1.0f; + float space_duty = 0.0f; + if (this->pin_->is_inverted()) { + mark_duty = 1.0f - mark_duty; + space_duty = 1.0f; + } + this->update_carrier_(carrier_frequency); + // own copy: with non_blocking the caller may re-encode temp_ while this frame is in flight + this->isr_data_.assign(this->temp_.get_data().begin(), this->temp_.get_data().end()); + if (this->isr_data_.empty()) { + ESP_LOGW(TAG, "Empty data"); + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + this->isr_mark_duty_ = mark_duty; + this->isr_space_duty_ = space_duty; + // trigger first: the deadline computed in arm_chain_ must not be charged for user code + this->transmit_trigger_.trigger(); + // the automation may have started a send on another instance; let it finish before + // claiming the shared timer (a same-instance send remains unsupported here) + this->wait_until_idle_(); + this->arm_chain_(send_times, send_wait); + if (this->non_blocking_) { + this->complete_pending_ = true; + this->enable_loop(); + return; + } + // blocking mode: wait out the chain, bounded by the stall deadline + while (this->transmitting_) { + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + this->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + this->deliver_completion_(); +} + +void RemoteTransmitterComponent::loop() { + if (!this->complete_pending_) { + this->disable_loop(); + return; + } + if (this->transmitting_) { + // non-blocking stall recovery: without this, a dead chain would leave the carrier + // driven and on_complete unfired until the next send happened to abort it + if ((int32_t) (millis() - s_expected_end_ms) <= 0) + return; + this->abort_stalled_chain_(); + } + // release the loop before user code runs: the automation may start a new non-blocking + // send, and its enable_loop() must be the last writer or its completion would strand + this->disable_loop(); + this->deliver_completion_(); +} + +#else // !USE_LIBRETINY_VARIANT_RTL8720C -- AmebaZ (RTL8710B): spin-based envelope, per-frame priority boost + void RemoteTransmitterComponent::await_target_time_() { const uint32_t current_time = micros(); if (this->target_time_ == 0) { @@ -72,15 +315,8 @@ void RemoteTransmitterComponent::await_target_time_() { } } -void RemoteTransmitterComponent::digital_write(bool value) { - if (this->pwm_ == nullptr) - return; - pwmout_write(static_cast(this->pwm_), (value != this->pin_->is_inverted()) ? 1.0f : 0.0f); -} - void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { - auto *pwm = static_cast(this->pwm_); - if (pwm == nullptr) { + if (this->pwm_ == nullptr) { ESP_LOGW(TAG, "Cannot send: PWM not initialized"); return; } @@ -94,6 +330,7 @@ void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t sen mark_duty = 1.0f - mark_duty; space_duty = 1.0f; } + auto *pwm = static_cast(this->pwm_); if (carrier_frequency > 0 && carrier_frequency != this->current_carrier_frequency_) { // round(1000000/freq), clamped like the bit-bang path so a bad lambda can't hand the SDK a zero period const uint32_t period = std::max(uint32_t(1), (1000000UL + carrier_frequency / 2) / carrier_frequency); @@ -132,6 +369,8 @@ void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t sen this->complete_trigger_.trigger(); } +#endif // USE_LIBRETINY_VARIANT_RTL8720C + } // namespace esphome::remote_transmitter #endif // USE_RTL87XX && !CLANG_TIDY diff --git a/tests/component_tests/remote_transmitter/__init__.py b/tests/component_tests/remote_transmitter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py new file mode 100644 index 0000000000..f843f1e84f --- /dev/null +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -0,0 +1,42 @@ +"""non_blocking is family-gated at config validation; the CI build boards never compile +the ISR paths, so this gate is the only CI-reachable coverage for the platform matrix.""" + +import pytest + +from esphome.components.libretiny.const import ( + FAMILY_RTL8710B, + FAMILY_RTL8720C, + KEY_FAMILY, + KEY_LIBRETINY, +) +from esphome.components.remote_transmitter import _validate_non_blocking_platform +import esphome.config_validation as cv +from esphome.const import PlatformFramework +from esphome.core import CORE + +from ..types import SetCoreConfigCallable + + +@pytest.mark.parametrize( + ("platform_framework", "family", "accepted"), + [ + (PlatformFramework.ESP32_IDF, None, True), + (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), + (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), + (PlatformFramework.ESP8266_ARDUINO, None, False), + ], +) +def test_non_blocking_platform_gate( + set_core_config: SetCoreConfigCallable, + platform_framework: PlatformFramework, + family: str | None, + accepted: bool, +) -> None: + set_core_config(platform_framework) + if family is not None: + CORE.data[KEY_LIBRETINY] = {KEY_FAMILY: family} + if accepted: + assert _validate_non_blocking_platform(True) is True + else: + with pytest.raises(cv.Invalid, match="non_blocking is only supported on"): + _validate_non_blocking_platform(True) diff --git a/tests/components/remote_transmitter/test.rtl87xx-ard.yaml b/tests/components/remote_transmitter/test.rtl87xx-ard.yaml index 769adbdf5c..74caa24cdd 100644 --- a/tests/components/remote_transmitter/test.rtl87xx-ard.yaml +++ b/tests/components/remote_transmitter/test.rtl87xx-ard.yaml @@ -2,6 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO12 carrier_duty_percent: 50% + # non_blocking is rtl8720c-only; the CI board is an RTL8710B packages: buttons: !include common-buttons.yaml From 612d58ec37e7fc565bf6fef3214247a4ff0f5366 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 23:11:26 -0500 Subject: [PATCH 46/65] [http_request] Default watchdog_timeout from timeout on ESP32 (#18732) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/components/http_request/__init__.py | 34 ++++++++++++++- .../http_request/http_request_idf.cpp | 3 +- .../component_tests/http_request/__init__.py | 0 .../config/test_esp32_default.yaml | 12 ++++++ .../config/test_esp32_explicit.yaml | 13 ++++++ .../config/test_esp32_platform_wider.yaml | 13 ++++++ .../http_request/config/test_esp32_stock.yaml | 11 +++++ .../http_request/config/test_esp8266.yaml | 13 ++++++ .../http_request/config/test_rp2040.yaml | 13 ++++++ .../component_tests/http_request/test_init.py | 42 +++++++++++++++++++ 10 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 tests/component_tests/http_request/__init__.py create mode 100644 tests/component_tests/http_request/config/test_esp32_default.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_explicit.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_platform_wider.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_stock.yaml create mode 100644 tests/component_tests/http_request/config/test_esp8266.yaml create mode 100644 tests/component_tests/http_request/config/test_rp2040.yaml create mode 100644 tests/component_tests/http_request/test_init.py diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 2abf097aec..de35d52a40 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -17,12 +17,14 @@ from esphome.const import ( CONF_TIMEOUT, CONF_URL, CONF_WATCHDOG_TIMEOUT, + PLATFORM_ESP32, PLATFORM_HOST, PlatformFramework, __version__, ) -from esphome.core import CORE, ID, Lambda +from esphome.core import CORE, ID, Lambda, TimePeriodMilliseconds from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.helpers import IS_MACOS from esphome.types import ConfigType @@ -94,6 +96,34 @@ def validate_ssl_verification(config: ConfigType) -> ConfigType: return config +# esp_http_client_open() runs DNS, TCP connect and the TLS handshake with no +# watchdog feed in between; each can take up to `timeout` on ESP-IDF. +WATCHDOG_TIMEOUT_MULTIPLIER = 3 +# Headroom over the exact worst case so a fully stalled open does not land on +# the watchdog deadline. +WATCHDOG_TIMEOUT_MARGIN_MS = 1000 + + +def default_watchdog_timeout(config: ConfigType) -> None: + """Arm the request watchdog on ESP32 when the user did not set it. + + The default never goes below the platform task watchdog, so a user who + widened `esp32.watchdog_timeout` keeps that window during requests. + """ + if not CORE.is_esp32 or CONF_WATCHDOG_TIMEOUT in config: + return + derived_ms = ( + config[CONF_TIMEOUT].total_milliseconds * WATCHDOG_TIMEOUT_MULTIPLIER + + WATCHDOG_TIMEOUT_MARGIN_MS + ) + platform_ms = fv.full_config.get()[PLATFORM_ESP32][ + CONF_WATCHDOG_TIMEOUT + ].total_milliseconds + config[CONF_WATCHDOG_TIMEOUT] = TimePeriodMilliseconds( + milliseconds=max(derived_ms, platform_ms) + ) + + def _declare_request_class(value: Any) -> ID: if CORE.is_host: return cv.declare_id(HttpRequestHost)(value) @@ -153,6 +183,8 @@ CONFIG_SCHEMA = cv.All( validate_ssl_verification, ) +FINAL_VALIDATE_SCHEMA = default_watchdog_timeout + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/http_request/http_request_idf.cpp b/esphome/components/http_request/http_request_idf.cpp index 470ed332f1..10313be89d 100644 --- a/esphome/components/http_request/http_request_idf.cpp +++ b/esphome/components/http_request/http_request_idf.cpp @@ -142,12 +142,13 @@ std::shared_ptr HttpRequestIDF::perform(const std::string &url, c const char *buf = body.c_str(); while (write_left > 0) { int written = esp_http_client_write(client, buf + write_index, write_left); - if (written < 0) { + if (written <= 0) { err = ESP_FAIL; break; } write_left -= written; write_index += written; + container->feed_wdt(); } } diff --git a/tests/component_tests/http_request/__init__.py b/tests/component_tests/http_request/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/http_request/config/test_esp32_default.yaml b/tests/component_tests/http_request/config/test_esp32_default.yaml new file mode 100644 index 0000000000..86744dcb11 --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_default.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s diff --git a/tests/component_tests/http_request/config/test_esp32_explicit.yaml b/tests/component_tests/http_request/config/test_esp32_explicit.yaml new file mode 100644 index 0000000000..e0d0074caa --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_explicit.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + watchdog_timeout: 20s diff --git a/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml b/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml new file mode 100644 index 0000000000..77a85da2ff --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + watchdog_timeout: 60s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s diff --git a/tests/component_tests/http_request/config/test_esp32_stock.yaml b/tests/component_tests/http_request/config/test_esp32_stock.yaml new file mode 100644 index 0000000000..70d2701466 --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_stock.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: diff --git a/tests/component_tests/http_request/config/test_esp8266.yaml b/tests/component_tests/http_request/config/test_esp8266.yaml new file mode 100644 index 0000000000..d0698dc57e --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp8266.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + verify_ssl: false diff --git a/tests/component_tests/http_request/config/test_rp2040.yaml b/tests/component_tests/http_request/config/test_rp2040.yaml new file mode 100644 index 0000000000..030736c30d --- /dev/null +++ b/tests/component_tests/http_request/config/test_rp2040.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +rp2: + board: rpipicow + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + verify_ssl: false diff --git a/tests/component_tests/http_request/test_init.py b/tests/component_tests/http_request/test_init.py new file mode 100644 index 0000000000..446c4acbd0 --- /dev/null +++ b/tests/component_tests/http_request/test_init.py @@ -0,0 +1,42 @@ +"""Tests for the http_request watchdog timeout default.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.config import read_config +from esphome.const import CONF_WATCHDOG_TIMEOUT +from esphome.core import CORE, TimePeriodMilliseconds + + +@pytest.mark.parametrize( + ("yaml_file", "expected_ms"), + [ + # stock 4.5s timeout: 3 x 4.5s plus 1s margin + ("test_esp32_stock.yaml", 14500), + # 3 x 10s plus 1s margin + ("test_esp32_default.yaml", 31000), + # esp32.watchdog_timeout: 60s is wider than the derived value and wins + ("test_esp32_platform_wider.yaml", 60000), + # explicit value is kept as is + ("test_esp32_explicit.yaml", 20000), + ], +) +def test_esp32_watchdog_timeout( + component_config_path: Callable[[str], Path], yaml_file: str, expected_ms: int +) -> None: + CORE.config_path = component_config_path(yaml_file) + config = read_config({}) + assert config["http_request"][CONF_WATCHDOG_TIMEOUT] == TimePeriodMilliseconds( + milliseconds=expected_ms + ) + + +@pytest.mark.parametrize("yaml_file", ["test_esp8266.yaml", "test_rp2040.yaml"]) +def test_other_platforms_leave_watchdog_unset( + component_config_path: Callable[[str], Path], yaml_file: str +) -> None: + CORE.config_path = component_config_path(yaml_file) + config = read_config({}) + assert CONF_WATCHDOG_TIMEOUT not in config["http_request"] From 9baff7652074031d27ba4c547bfee6250e36151a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 23:29:14 -0500 Subject: [PATCH 47/65] [api] Treat homeassistant.event variables as lambdas (#18759) --- esphome/components/api/__init__.py | 45 +++++++++++-- esphome/config_validation.py | 35 ++++++++++- esphome/core/__init__.py | 3 +- .../api/test_homeassistant_variables.py | 63 +++++++++++++++++++ .../api/test_homeassistant_variables.yaml | 32 ++++++++++ tests/components/api/common-base.yaml | 8 +++ tests/components/homeassistant/common.yaml | 4 +- tests/unit_tests/test_config_validation.py | 46 ++++++++++++++ 8 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 tests/component_tests/api/test_homeassistant_variables.py create mode 100644 tests/component_tests/api/test_homeassistant_variables.yaml diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index a10bfd3418..2e891a9663 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -1,4 +1,5 @@ import logging +import re from typing import Any from esphome import automation @@ -499,6 +500,40 @@ async def to_code(config: ConfigType) -> None: KEY_VALUE_SCHEMA = cv.Schema({cv.string: cv.templatable(cv.string_strict)}) +_ID_CALL_PROG = re.compile(r"\bid\s*\(") + + +# Remove before 2027.3.0: untagged strings that look like lambda source keep +# being compiled as lambdas during the deprecation window +def _coerce_implicit_lambda(value: Any) -> Any: + if not isinstance(value, str): + return value + if cv.looks_like_returning_lambda(value): + _LOGGER.warning( + "[api] The 'variables' value '%s' looks like a lambda but is " + "missing the !lambda tag. It is compiled as a lambda for now but " + "will be sent as literal text from 2027.3.0. Add !lambda to keep " + "it evaluated; literal text belongs under 'data:'.", + value, + ) + # cv.templatable runs returning_lambda on the coerced Lambda + return cv.lambda_(value) + if _ID_CALL_PROG.search(value): + # lambda source without a return: issue 5394's mistake class + _LOGGER.warning( + "[api] The 'variables' value '%s' is sent as literal text; wrap " + "it in !lambda 'return ...;' to evaluate it instead.", + value, + ) + return value + + +# Static strings or !lambda values. cv.templatable stays introspectable for +# schema tooling; removing the shim leaves KEY_VALUE_SCHEMA. +VARIABLES_SCHEMA = cv.Schema( + {cv.string: cv.All(_coerce_implicit_lambda, cv.templatable(cv.string_strict))} +) + def _validate_response_config(config: ConfigType) -> ConfigType: # Validate dependencies: @@ -535,9 +570,7 @@ HOMEASSISTANT_ACTION_ACTION_SCHEMA = cv.All( ), cv.Optional(CONF_DATA, default={}): KEY_VALUE_SCHEMA, cv.Optional(CONF_DATA_TEMPLATE, default={}): KEY_VALUE_SCHEMA, - cv.Optional(CONF_VARIABLES, default={}): cv.Schema( - {cv.string: cv.returning_lambda} - ), + cv.Optional(CONF_VARIABLES, default={}): VARIABLES_SCHEMA, cv.Optional(CONF_RESPONSE_TEMPLATE): cv.templatable(cv.string), cv.Optional(CONF_CAPTURE_RESPONSE, default=False): cv.boolean, cv.Optional(CONF_ON_SUCCESS): automation.validate_automation(single=True), @@ -598,6 +631,8 @@ async def homeassistant_service_to_code( cg.add(var.init_variables(len(config[CONF_VARIABLES]))) for key, value in config[CONF_VARIABLES].items(): templ = await cg.templatable(value, args, None) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) cg.add(var.add_variable(cg.FlashStringLiteral(key), templ)) if on_error := config.get(CONF_ON_ERROR): @@ -652,7 +687,7 @@ HOMEASSISTANT_EVENT_ACTION_SCHEMA = cv.Schema( cv.Required(CONF_EVENT): validate_homeassistant_event, cv.Optional(CONF_DATA, default={}): KEY_VALUE_SCHEMA, cv.Optional(CONF_DATA_TEMPLATE, default={}): KEY_VALUE_SCHEMA, - cv.Optional(CONF_VARIABLES, default={}): KEY_VALUE_SCHEMA, + cv.Optional(CONF_VARIABLES, default={}): VARIABLES_SCHEMA, } ) @@ -698,6 +733,8 @@ async def homeassistant_event_to_code( cg.add(var.init_variables(len(config[CONF_VARIABLES]))) for key, value in config[CONF_VARIABLES].items(): templ = await cg.templatable(value, args, None) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) cg.add(var.add_variable(cg.FlashStringLiteral(key), templ)) return var diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 09962e8c95..904cbd1919 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1882,13 +1882,46 @@ def lambda_(value): return value +# 'return' at a statement boundary; only consulted when the source has no +# semicolon, so ';' is not a boundary. Migration use only, see +# looks_like_returning_lambda. +LAMBDA_RETURN_STATEMENT_PROG = re.compile(r"(?:^|[:{})\n])\s*return\b") +LAMBDA_RETURN_KEYWORD_PROG = re.compile(r"\breturn\b") +# RESERVED_IDS subset that can begin a return expression; 'this'/'true' would +# promote prose and infix 'and'/'or' cannot start an expression. +_CPP_LEADING_WORD_OPERATORS = "not|new|sizeof|delete" +# Two or more plain words: prose, not C++. A single word is indistinguishable +# from 'return x'. Migration use only, see looks_like_returning_lambda. +LAMBDA_PROSE_TAIL_PROG = re.compile( + rf"(?!(?:{_CPP_LEADING_WORD_OPERATORS})\b)[A-Za-z']+(?:,?\s+[A-Za-z']+)+[.!?]?" +) + + +def looks_like_returning_lambda(value: str) -> bool: + """Check whether a string looks like C++ lambda source: a semicolon means + code, so any return keyword counts; without one, a boundary return whose + tail does not read as prose is a return statement missing its semicolon. + + For migrating deprecated implicit lambdas only; new validators must + require an explicit !lambda tag instead of guessing. + """ + src = Lambda.comment_remover(value) + if ";" in src: + return LAMBDA_RETURN_KEYWORD_PROG.search(src) is not None + for match in LAMBDA_RETURN_STATEMENT_PROG.finditer(src): + tail = src[match.end() :].split("\n", 1)[0].strip() + if not LAMBDA_PROSE_TAIL_PROG.fullmatch(tail): + return True + return False + + def returning_lambda(value): """Coerce this configuration option to a lambda. Additionally, make sure the lambda returns something. """ value = lambda_(value) - if "return" not in value.value: + if LAMBDA_RETURN_KEYWORD_PROG.search(Lambda.comment_remover(value.value)) is None: raise Invalid( "Lambda doesn't contain a 'return' statement, but the lambda " "is expected to return a value. \n" diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 2ec2a08e83..77efc91bef 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -339,7 +339,8 @@ class Lambda: self._requires_ids = None # https://stackoverflow.com/a/241506/229052 - def comment_remover(self, text): + @staticmethod + def comment_remover(text): def replacer(match): s = match.group(0) if s.startswith("/"): diff --git a/tests/component_tests/api/test_homeassistant_variables.py b/tests/component_tests/api/test_homeassistant_variables.py new file mode 100644 index 0000000000..48e53d8f4c --- /dev/null +++ b/tests/component_tests/api/test_homeassistant_variables.py @@ -0,0 +1,63 @@ +"""Tests for variables handling in homeassistant.event and homeassistant.action.""" + +from collections.abc import Callable +import logging +from pathlib import Path + +import pytest + +CONFIG = "tests/component_tests/api/test_homeassistant_variables.yaml" + + +def test_plain_string_with_return_is_compiled_as_lambda_with_warning( + generate_main: Callable[[str | Path], str], + caplog: pytest.LogCaptureFixture, +) -> None: + """A plain string with a return statement compiles as a lambda and warns.""" + with caplog.at_level(logging.WARNING): + main_cpp = generate_main(CONFIG) + + assert main_cpp.count('add_variable(ESPHOME_F("lambda_var"), []() {') == 2 + assert "return millis();" in main_cpp + # The source text must not be sent as a static string value. + assert '"return millis();"' not in main_cpp + assert "missing the !lambda tag" in caplog.text + + +def test_static_string_is_kept_as_static_value( + generate_main: Callable[[str | Path], str], + caplog: pytest.LogCaptureFixture, +) -> None: + """A static string stays static, PROGMEM wrapped, with no warning.""" + with caplog.at_level(logging.WARNING): + main_cpp = generate_main(CONFIG) + + assert ( + main_cpp.count( + 'add_variable(ESPHOME_F("static_var"), ESPHOME_F("static value"));' + ) + == 2 + ) + assert "static value" not in caplog.text + + +def test_static_id_value_stays_literal_with_hint( + generate_main: Callable[[str | Path], str], + caplog: pytest.LogCaptureFixture, +) -> None: + """Lambda source without a return stays literal text but warns.""" + with caplog.at_level(logging.WARNING): + main_cpp = generate_main(CONFIG) + + assert 'ESPHOME_F("id(test_sensor).state")' in main_cpp + assert "sent as literal text" in caplog.text + + +def test_explicit_lambda_tag_is_compiled_as_lambda( + generate_main: Callable[[str | Path], str], +) -> None: + """A !lambda value keeps working unchanged.""" + main_cpp = generate_main(CONFIG) + + assert 'add_variable(ESPHOME_F("tagged_var"), []() {' in main_cpp + assert "return App.get_name();" in main_cpp diff --git a/tests/component_tests/api/test_homeassistant_variables.yaml b/tests/component_tests/api/test_homeassistant_variables.yaml new file mode 100644 index 0000000000..e1ec07cc74 --- /dev/null +++ b/tests/component_tests/api/test_homeassistant_variables.yaml @@ -0,0 +1,32 @@ +esphome: + name: test + on_boot: + then: + # Plain strings with a return statement compile as lambdas + - homeassistant.event: + event: esphome.test_event + data_template: + message: "{{ lambda_var }} {{ static_var }} {{ tagged_var }}" + variables: + lambda_var: |- + return millis(); + static_var: static value + tagged_var: !lambda return App.get_name(); + hint_var: id(test_sensor).state + - homeassistant.action: + action: notify.notify + data_template: + message: "{{ lambda_var }} {{ static_var }}" + variables: + lambda_var: |- + return millis(); + static_var: static value + +esp32: + board: esp32dev + +wifi: + ssid: SomeNetwork + password: SomePassword + +api: diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index d7470ee4b3..c9eb200471 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -9,6 +9,14 @@ esphome: event: esphome.button_pressed data: message: Button was pressed + - homeassistant.event: + event: esphome.button_pressed_with_variables + data_template: + message: Button {{ button_name }} ({{ button_index }}) was pressed from {{ button_source }} + variables: + button_name: !lambda 'return std::string("test_button");' + button_index: !lambda 'return 1;' + button_source: static_value - homeassistant.action: action: notify.html5 data: diff --git a/tests/components/homeassistant/common.yaml b/tests/components/homeassistant/common.yaml index 71a7ac65c2..1099f7ea85 100644 --- a/tests/components/homeassistant/common.yaml +++ b/tests/components/homeassistant/common.yaml @@ -12,7 +12,7 @@ esphome: data_template: message: The humidity is {{ my_variable }}%. variables: - my_variable: "return id(ha_hello_world_temperature).state;" + my_variable: !lambda "return id(ha_hello_world_temperature).state;" - homeassistant.action: action: notify.html5 data: @@ -24,7 +24,7 @@ esphome: data_template: message: The humidity is {{ my_variable }}%. variables: - my_variable: "return id(ha_hello_world_temperature).state;" + my_variable: !lambda "return id(ha_hello_world_temperature).state;" wifi: ssid: MySSID diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 0f927a6513..457b9d017b 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -2565,6 +2565,52 @@ def test_returning_lambda_no_return() -> None: cv.returning_lambda(Lambda("int x = 5;")) +def test_returning_lambda_return_only_in_comment() -> None: + with pytest.raises(Invalid, match="return statement"): + cv.returning_lambda(Lambda("// return 5;\nint x = 5;")) + + +def test_returning_lambda_missing_semicolon_is_accepted() -> None: + """A forgotten semicolon is left for the C++ compiler to report.""" + assert isinstance(cv.returning_lambda(Lambda("return x")), Lambda) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("return 5;", True), + ("if (x) { return x; } return 0;", True), + ("if (x) return 1; else return 0;", True), + ("switch (x) { case 0: return 1; }", True), + # a semicolon means code: any return keyword counts + ("return not x;", True), + ("return a and b;", True), + ("please return the sensor; then wait", True), + # a forgotten semicolon is still lambda source; the compiler reports it + ("return id(x).state", True), + ("return x", True), + ("return 5", True), + ("return not x", True), + # accepted: a one-word tail is indistinguishable from 'return x' + ("return soon", True), + ("Alert: return home", True), + ("static value", False), + ("no returns here", False), + ("the_return_value", False), + # without a semicolon, prose is not lambda source + ("please return the item", False), + ("return to sender", False), + ("return a and b", False), + # return only inside a comment is not a return statement + ("// return 5;\nint x = 5;", False), + ("/* return 5; */ int x = 5;", False), + ("return 5; // done", True), + ], +) +def test_looks_like_returning_lambda(value: str, expected: bool) -> None: + assert cv.looks_like_returning_lambda(value) is expected + + # --------------------------------------------------------------------------- # dimensions # --------------------------------------------------------------------------- From e37a540fb729edfd15fbd32281b7ccf53ecd74cb Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 25 Aug 2026 23:59:37 -0500 Subject: [PATCH 48/65] [esp32] Apply custom eFuse MAC as base MAC for all interfaces (#18452) --- esphome/components/esp32/core.cpp | 8 ++++++ esphome/components/esp32/helpers.cpp | 25 +++++++++++++------ .../wifi/wifi_component_esp_idf.cpp | 5 ---- esphome/core/helpers.h | 5 ++++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/esphome/components/esp32/core.cpp b/esphome/components/esp32/core.cpp index 098a59937a..a6916fe739 100644 --- a/esphome/components/esp32/core.cpp +++ b/esphome/components/esp32/core.cpp @@ -2,6 +2,7 @@ #include "esphome/core/application.h" #include "esphome/core/defines.h" +#include "esphome/core/helpers.h" #include "preferences.h" #include #include @@ -29,6 +30,13 @@ void loop_task(void *pv_params) { } extern "C" void app_main() { + // Apply the custom eFuse MAC (if burned and valid) as the base MAC before any + // interface (Wi-Fi, Ethernet, Bluetooth, 802.15.4) derives its address from it. + // The logger does not exist yet, so only log-free helpers may be used here. + uint8_t mac[MAC_ADDRESS_SIZE]; + if (get_custom_mac_address(mac)) { + set_mac_address(mac); + } initArduino(); esp32::setup_preferences(); #if CONFIG_FREERTOS_UNICORE diff --git a/esphome/components/esp32/helpers.cpp b/esphome/components/esp32/helpers.cpp index c2ff6cf34d..91b4241211 100644 --- a/esphome/components/esp32/helpers.cpp +++ b/esphome/components/esp32/helpers.cpp @@ -71,23 +71,32 @@ static bool read_valid_mac(uint8_t *mac, esp_err_t err) { return err == ESP_OK & static constexpr size_t MAC_ADDRESS_SIZE_BITS = MAC_ADDRESS_SIZE * 8; // 48 bits +// Must not use the ESPHome logger (may run before it exists, e.g. from app_main()). +bool get_custom_mac_address(uint8_t *mac) { + // has_custom_mac_address() checks the raw eFuse field, while the reads below select their + // method differently and may still fail (CRC), so the result must be validated again. + if (!has_custom_mac_address()) + return false; +#if defined(CONFIG_SOC_IEEE802154_SUPPORTED) + return read_valid_mac(mac, esp_efuse_read_field_blob(ESP_EFUSE_MAC_CUSTOM, mac, MAC_ADDRESS_SIZE_BITS)); +#else + return read_valid_mac(mac, esp_efuse_mac_get_custom(mac)); +#endif +} + void get_mac_address_raw(uint8_t *mac) { // NOLINT(readability-non-const-parameter) + if (get_custom_mac_address(mac)) { + return; + } #if defined(CONFIG_SOC_IEEE802154_SUPPORTED) // When CONFIG_SOC_IEEE802154_SUPPORTED is defined, esp_efuse_mac_get_default // returns the 802.15.4 EUI-64 address, so we read directly from eFuse instead. - // Both paths already read raw eFuse bytes, so there is no CRC-bypass fallback + // This already reads raw eFuse bytes, so there is no CRC-bypass fallback // (unlike the non-IEEE802154 path where esp_efuse_mac_get_default does CRC checks). - if (has_custom_mac_address() && - read_valid_mac(mac, esp_efuse_read_field_blob(ESP_EFUSE_MAC_CUSTOM, mac, MAC_ADDRESS_SIZE_BITS))) { - return; - } if (read_valid_mac(mac, esp_efuse_read_field_blob(ESP_EFUSE_MAC_FACTORY, mac, MAC_ADDRESS_SIZE_BITS))) { return; } #else - if (has_custom_mac_address() && read_valid_mac(mac, esp_efuse_mac_get_custom(mac))) { - return; - } if (read_valid_mac(mac, esp_efuse_mac_get_default(mac))) { return; } diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 32d46887b6..06f0981020 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -140,11 +140,6 @@ void event_handler(void *arg, esp_event_base_t event_base, int32_t event_id, voi } void WiFiComponent::wifi_pre_setup_() { - uint8_t mac[MAC_ADDRESS_SIZE]; - if (has_custom_mac_address()) { - get_mac_address_raw(mac); - set_mac_address(mac); - } // Network interface setup handled by network component s_wifi_event_group = xEventGroupCreate(); if (s_wifi_event_group == nullptr) { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index e60316d4ee..9fdc088ecb 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -2089,6 +2089,11 @@ const char *get_mac_address_pretty_into_buffer(std::span Date: Wed, 26 Aug 2026 07:26:50 +0200 Subject: [PATCH 49/65] [climate_ir_lg] advanced vert. swing, setting temp. in heat/cool mode, jet mode decoding, other fixes + refactor (#10875) --- esphome/components/climate_ir_lg/climate.py | 3 + .../climate_ir_lg/climate_ir_lg.cpp | 330 ++++++++++++++---- .../components/climate_ir_lg/climate_ir_lg.h | 6 +- tests/components/climate_ir_lg/common.yaml | 3 + 4 files changed, 264 insertions(+), 78 deletions(-) diff --git a/esphome/components/climate_ir_lg/climate.py b/esphome/components/climate_ir_lg/climate.py index 48fd373b78..255fca9ad1 100644 --- a/esphome/components/climate_ir_lg/climate.py +++ b/esphome/components/climate_ir_lg/climate.py @@ -13,9 +13,11 @@ CONF_HEADER_LOW = "header_low" CONF_BIT_HIGH = "bit_high" CONF_BIT_ONE_LOW = "bit_one_low" CONF_BIT_ZERO_LOW = "bit_zero_low" +CONF_ADVANCED_COMMANDS_SUPPORT = "advanced_commands_support" CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(LgIrClimate).extend( { + cv.Optional(CONF_ADVANCED_COMMANDS_SUPPORT, default=False): cv.boolean, cv.Optional( CONF_HEADER_HIGH, default="8000us" ): cv.positive_time_period_microseconds, @@ -38,6 +40,7 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(LgIrClimate).extend( async def to_code(config: ConfigType) -> None: var = await climate_ir.new_climate_ir(config) + cg.add(var.set_advanced_commands_support(config[CONF_ADVANCED_COMMANDS_SUPPORT])) cg.add(var.set_header_high(config[CONF_HEADER_HIGH])) cg.add(var.set_header_low(config[CONF_HEADER_LOW])) cg.add(var.set_bit_high(config[CONF_BIT_HIGH])) diff --git a/esphome/components/climate_ir_lg/climate_ir_lg.cpp b/esphome/components/climate_ir_lg/climate_ir_lg.cpp index 588566dd9d..bb612eda7b 100644 --- a/esphome/components/climate_ir_lg/climate_ir_lg.cpp +++ b/esphome/components/climate_ir_lg/climate_ir_lg.cpp @@ -5,11 +5,85 @@ namespace esphome::climate_ir_lg { static const char *const TAG = "climate.climate_ir_lg"; -// Commands -const uint32_t COMMAND_MASK = 0xFF000; -const uint32_t COMMAND_OFF = 0xC0000; -const uint32_t COMMAND_SWING = 0x10000; +// All codes provided here are missing the checksum (last 4 bits) +// this checksum needs to be calculated before sending (look at `calc_checksum_()`) +const uint32_t LG_HEADER = 0x8800000; + +// Commands +const uint32_t COMMAND_HEADER_MASK = 0xFF000; +const uint32_t COMMAND_DATA_MASK = 0x00FF0; +const uint32_t CHECKSUM_MASK = 0xF; + +enum CommandBasic : uint32_t { + HEADER_BASIC = 0x10000, + BASIC_SWING_TOGGLE = 0x000, + + // JET MODE (only for cooling/drying/heating modes) + // For 30 minutes: max airflow (stronger than F5 aka FAN_MAX) + PO (min/min/max temperature respectively) + // After 30 minutes: F5 aka FAN_MAX + min/min/max temperature respectively + BASIC_JET = 0x080, +}; + +enum CommandSys : uint32_t { + HEADER_SYS = 0xC0000, + + COMMAND_OFF = 0x050, + + // Also known as 'auto-dry' + AUTO_CLEAN_ON = 0x0B0, + AUTO_CLEAN_OFF = 0x0C0, + + PURIFY_ON = 0x000, // From either OFF or Mode -> Purify + PURIFY_OFF = 0x080, // From Mode + Purify -> Mode + + QUIET_OUTDOOR_ON = 0xA60, + QUIET_OUTDOOR_OFF = 0xA70, + + // ENERGY CTRL (only in Cooling mode) + COOL_ENERG_CTRL_80 = 0x7D0, // 80% + COOL_ENERG_CTRL_60 = 0x7E0, // 60% + COOL_ENERG_CTRL_40 = 0x800, // 40% + COOL_ENERG_CTRL_OFF = 0x7F0, // OFF + + DISPLAY_KW = 0x460, + LIGHT_ON_OFF = 0x0A0, + + TEMP_UNIT_F = 0x170, + TEMP_UNIT_C = 0x160, +}; + +enum CommandAdvSwing : uint32_t { + HEADER_ADV_SWING = 0x13000, + + // Only 5 bits are relevant, I got 0x13952 once - not sure what is the 8th bit so ignoring that. + ADV_SWING_DATA_MASK = 0x1F0, + + // Commands for Advanced Vertical Control: Swing + 6 fixed positions + VERT_FIX_1 = 0x040, // Down + VERT_FIX_2 = 0x050, + VERT_FIX_3 = 0x060, + VERT_FIX_4 = 0x070, + VERT_FIX_5 = 0x080, + VERT_FIX_6 = 0x090, // Up + VERT_SWING_ON = 0x140, // Swing between 1 and 6 + VERT_SWING_OFF = 0x150, // Stops immediately + + // Commands for Advanced Horizontal Control: Swing (3 modes) + 5 fixed positions + HORI_FIX_1 = 0x0B0, // Left + HORI_FIX_2 = 0x0C0, + HORI_FIX_3 = 0x0D0, + HORI_FIX_4 = 0x0E0, + HORI_FIX_5 = 0x0F0, // Right + HORI_SWING_ON_LEFT = 0x100, // Swing between 1 and 3 + HORI_SWING_ON_RIGHT = 0x110, // Swing between 3 and 5 + HORI_SWING_ON_FULL = 0x160, // Swing between 1 and 5 + HORI_SWING_OFF = 0x170, // Stops immediately +}; + +// Following commands contain mode, fan speed and temperature + +// Modes const uint32_t COMMAND_ON_COOL = 0x00000; const uint32_t COMMAND_ON_DRY = 0x01000; const uint32_t COMMAND_ON_FAN_ONLY = 0x02000; @@ -23,11 +97,13 @@ const uint32_t COMMAND_AI = 0x0B000; const uint32_t COMMAND_HEAT = 0x0C000; // Fan speed -const uint32_t FAN_MASK = 0xF0; +const uint32_t FAN_SPEED_MASK = 0xF0; const uint32_t FAN_AUTO = 0x50; -const uint32_t FAN_MIN = 0x00; -const uint32_t FAN_MED = 0x20; -const uint32_t FAN_MAX = 0x40; +const uint32_t FAN_MIN = 0x00; // AKA F1 +const uint32_t FAN_F2 = 0x90; +const uint32_t FAN_MED = 0x20; // AKA F3 +const uint32_t FAN_F4 = 0xA0; +const uint32_t FAN_MAX = 0x40; // AKA F5 // Temperature const uint8_t TEMP_RANGE = TEMP_MAX - TEMP_MIN + 1; @@ -37,16 +113,37 @@ const uint32_t TEMP_SHIFT = 8; const uint16_t BITS = 28; void LgIrClimate::transmit_state() { - uint32_t remote_state = 0x8800000; + uint32_t remote_state = LG_HEADER; - // ESP_LOGD(TAG, "climate_lg_ir mode_before_ code: 0x%02X", modeBefore_); + // ESP_LOGD(TAG, "climate_lg_ir mode_before_ code: 0x%02X", this->modeBefore_); // Set command if (this->send_swing_cmd_) { this->send_swing_cmd_ = false; - remote_state |= COMMAND_SWING; - } else { - bool climate_is_off = (this->mode_before_ == climate::CLIMATE_MODE_OFF); + if (this->advanced_commands_support_) { + switch (this->swing_mode) { + case climate::CLIMATE_SWING_VERTICAL: + ESP_LOGD(TAG, "setting swing vertical"); + remote_state |= CommandAdvSwing::HEADER_ADV_SWING; + remote_state |= CommandAdvSwing::VERT_SWING_ON; + break; + case climate::CLIMATE_SWING_OFF: + ESP_LOGD(TAG, "setting swing off"); + remote_state |= CommandAdvSwing::HEADER_ADV_SWING; + remote_state |= CommandAdvSwing::VERT_SWING_OFF; + break; + default: + return; + } + this->transmit_(remote_state); + this->publish_state(); + return; + } else { // just toggle swing when advanced_commands_support is not set + remote_state |= HEADER_BASIC; + remote_state |= BASIC_SWING_TOGGLE; + } + } else { // Mode commands + const bool climate_is_off = (this->mode_before_ == climate::CLIMATE_MODE_OFF); switch (this->mode) { case climate::CLIMATE_MODE_COOL: remote_state |= climate_is_off ? COMMAND_ON_COOL : COMMAND_COOL; @@ -65,8 +162,8 @@ void LgIrClimate::transmit_state() { break; case climate::CLIMATE_MODE_OFF: default: - remote_state |= COMMAND_OFF; - break; + remote_state |= CommandSys::HEADER_SYS; + remote_state |= CommandSys::COMMAND_OFF; } } @@ -75,9 +172,8 @@ void LgIrClimate::transmit_state() { ESP_LOGD(TAG, "climate_lg_ir mode code: 0x%02X", this->mode); // Set fan speed - if (this->mode == climate::CLIMATE_MODE_OFF) { - remote_state |= FAN_AUTO; - } else { + if (this->mode != + climate::CLIMATE_MODE_OFF) { // https://github.com/esphome/esphome/pull/10875#issuecomment-5042765948 switch (this->fan_mode.value_or(climate::CLIMATE_FAN_ON)) { case climate::CLIMATE_FAN_HIGH: remote_state |= FAN_MAX; @@ -95,10 +191,20 @@ void LgIrClimate::transmit_state() { } } - // Set temperature - if (this->mode == climate::CLIMATE_MODE_COOL || this->mode == climate::CLIMATE_MODE_HEAT) { - auto temp = (uint8_t) roundf(clamp(this->target_temperature, TEMP_MIN, TEMP_MAX)); - remote_state |= ((temp - 15) << TEMP_SHIFT); + uint8_t temp; + switch (this->mode) { + case climate::CLIMATE_MODE_HEAT_COOL: + if (!this->advanced_commands_support_) { // Keep previous behavior + break; + } + [[fallthrough]]; + case climate::CLIMATE_MODE_COOL: + case climate::CLIMATE_MODE_HEAT: + temp = static_cast(roundf(clamp(this->target_temperature, TEMP_MIN, TEMP_MAX))); + remote_state |= (temp - 15) << TEMP_SHIFT; + break; + default: + break; } this->transmit_(remote_state); @@ -124,62 +230,134 @@ bool LgIrClimate::on_receive(remote_base::RemoteReceiveData data) { } } - ESP_LOGD(TAG, "Decoded 0x%02" PRIX32, remote_state); - if ((remote_state & 0xFF00000) != 0x8800000) + ESP_LOGD(TAG, "Received 0x%02" PRIX32, remote_state); + if ((remote_state & 0xFF00000) != LG_HEADER) return false; - // Get command - if ((remote_state & COMMAND_MASK) == COMMAND_OFF) { - this->mode = climate::CLIMATE_MODE_OFF; - } else if ((remote_state & COMMAND_MASK) == COMMAND_SWING) { - this->swing_mode = - this->swing_mode == climate::CLIMATE_SWING_OFF ? climate::CLIMATE_SWING_VERTICAL : climate::CLIMATE_SWING_OFF; - } else { - switch (remote_state & COMMAND_MASK) { - case COMMAND_DRY: - case COMMAND_ON_DRY: - this->mode = climate::CLIMATE_MODE_DRY; - break; - case COMMAND_FAN_ONLY: - case COMMAND_ON_FAN_ONLY: - this->mode = climate::CLIMATE_MODE_FAN_ONLY; - break; - case COMMAND_AI: - case COMMAND_ON_AI: - this->mode = climate::CLIMATE_MODE_HEAT_COOL; - break; - case COMMAND_HEAT: - case COMMAND_ON_HEAT: - this->mode = climate::CLIMATE_MODE_HEAT; - break; - case COMMAND_COOL: - case COMMAND_ON_COOL: - default: - this->mode = climate::CLIMATE_MODE_COOL; - break; - } - - // Get fan speed - if (this->mode == climate::CLIMATE_MODE_HEAT_COOL) { - this->fan_mode = climate::CLIMATE_FAN_AUTO; - } else if (this->mode == climate::CLIMATE_MODE_COOL || this->mode == climate::CLIMATE_MODE_DRY || - this->mode == climate::CLIMATE_MODE_FAN_ONLY || this->mode == climate::CLIMATE_MODE_HEAT) { - if ((remote_state & FAN_MASK) == FAN_AUTO) { - this->fan_mode = climate::CLIMATE_FAN_AUTO; - } else if ((remote_state & FAN_MASK) == FAN_MIN) { - this->fan_mode = climate::CLIMATE_FAN_LOW; - } else if ((remote_state & FAN_MASK) == FAN_MED) { - this->fan_mode = climate::CLIMATE_FAN_MEDIUM; - } else if ((remote_state & FAN_MASK) == FAN_MAX) { - this->fan_mode = climate::CLIMATE_FAN_HIGH; + // Decode commands + switch (remote_state & COMMAND_HEADER_MASK) { + case CommandSys::HEADER_SYS: + ESP_LOGD(TAG, "Got system command! With data: 0x%02" PRIX32, remote_state & COMMAND_DATA_MASK); + if ((remote_state & COMMAND_DATA_MASK) == CommandSys::COMMAND_OFF) { + this->mode = climate::CLIMATE_MODE_OFF; + } else { + return false; + } + break; + case CommandAdvSwing::HEADER_ADV_SWING: + ESP_LOGD(TAG, "Got advanced swing command! With data: 0x%02" PRIX32, + remote_state & CommandAdvSwing::ADV_SWING_DATA_MASK); + switch (remote_state & CommandAdvSwing::ADV_SWING_DATA_MASK) { + case CommandAdvSwing::VERT_SWING_ON: + this->swing_mode = climate::CLIMATE_SWING_VERTICAL; + break; + case CommandAdvSwing::VERT_SWING_OFF: + case CommandAdvSwing::VERT_FIX_1: + case CommandAdvSwing::VERT_FIX_2: + case CommandAdvSwing::VERT_FIX_3: + case CommandAdvSwing::VERT_FIX_4: + case CommandAdvSwing::VERT_FIX_5: + case CommandAdvSwing::VERT_FIX_6: + this->swing_mode = climate::CLIMATE_SWING_OFF; + break; + default: + return false; // Ignore all other (horizontal) swing commands } - } - // Get temperature - if (this->mode == climate::CLIMATE_MODE_COOL || this->mode == climate::CLIMATE_MODE_HEAT) { - this->target_temperature = ((remote_state & TEMP_MASK) >> TEMP_SHIFT) + 15; - } + this->publish_state(); + return true; + + case HEADER_BASIC: + if ((remote_state & COMMAND_DATA_MASK) == BASIC_JET) { + switch (this->mode) { + case climate::CLIMATE_MODE_COOL: + case climate::CLIMATE_MODE_HEAT: + case climate::CLIMATE_MODE_DRY: + this->target_temperature = + this->mode == climate::CLIMATE_MODE_HEAT ? this->maximum_temperature_ : this->minimum_temperature_; + this->fan_mode = climate::CLIMATE_FAN_HIGH; + // When enabling PO(WER) also known as JET mode, swing is set to VERT_3, but after 30 mins it will switch + // back to what it was before, so let's just not change it here it at all + this->publish_state(); + return true; + default: + ESP_LOGD(TAG, "Got jet command, but current mode does not support it! Ignoring."); + return false; + } + } + + // Keep previous behavior in case of other BASIC command + if (this->swing_mode == climate::CLIMATE_SWING_OFF) { // Just flip between vertical and off + this->swing_mode = climate::CLIMATE_SWING_VERTICAL; + } else { + this->swing_mode = climate::CLIMATE_SWING_OFF; + } + this->publish_state(); + return true; + // Following commands also contain fan speed and temperature, so no 'return' in these cases + case COMMAND_DRY: + case COMMAND_ON_DRY: + this->mode = climate::CLIMATE_MODE_DRY; + break; + case COMMAND_FAN_ONLY: + case COMMAND_ON_FAN_ONLY: + this->mode = climate::CLIMATE_MODE_FAN_ONLY; + break; + case COMMAND_AI: + case COMMAND_ON_AI: + this->mode = climate::CLIMATE_MODE_HEAT_COOL; + break; + case COMMAND_HEAT: + case COMMAND_ON_HEAT: + this->mode = climate::CLIMATE_MODE_HEAT; + break; + case COMMAND_COOL: + case COMMAND_ON_COOL: + this->mode = climate::CLIMATE_MODE_COOL; + break; + default: + ESP_LOGD(TAG, "Got unknown command! Ignoring!"); + return false; } + + // Decode fan speed + switch (remote_state & FAN_SPEED_MASK) { + case FAN_AUTO: + this->fan_mode = climate::CLIMATE_FAN_AUTO; + break; + case FAN_MIN: + case FAN_F2: + this->fan_mode = climate::CLIMATE_FAN_LOW; + break; + case FAN_MED: + case FAN_F4: + this->fan_mode = climate::CLIMATE_FAN_MEDIUM; + break; + case FAN_MAX: + this->fan_mode = climate::CLIMATE_FAN_HIGH; + break; + default: + ESP_LOGD(TAG, "Got unknown fan speed! Ignoring!"); + return false; + } + + // Keep previous behavior + if (this->mode == climate::CLIMATE_MODE_HEAT_COOL && !(this->advanced_commands_support_)) { + this->fan_mode = climate::CLIMATE_FAN_AUTO; + } + + // Decode temperature for modes that support it + switch (this->mode) { + case climate::CLIMATE_MODE_HEAT_COOL: + case climate::CLIMATE_MODE_COOL: + case climate::CLIMATE_MODE_HEAT: + this->target_temperature = ((remote_state & TEMP_MASK) >> TEMP_SHIFT) + 15; + break; + default: + break; + } + + this->mode_before_ = this->mode; this->publish_state(); return true; @@ -207,14 +385,14 @@ void LgIrClimate::transmit_(uint32_t value) { data->mark(this->bit_high_); transmit.perform(); } + void LgIrClimate::calc_checksum_(uint32_t &value) { - uint32_t mask = 0xF; uint32_t sum = 0; for (uint8_t i = 1; i < 8; i++) { - sum += (value & (mask << (i * 4))) >> (i * 4); + sum += (value & (CHECKSUM_MASK << (i * 4))) >> (i * 4); } - value |= (sum & mask); + value |= (sum & CHECKSUM_MASK); } } // namespace esphome::climate_ir_lg diff --git a/esphome/components/climate_ir_lg/climate_ir_lg.h b/esphome/components/climate_ir_lg/climate_ir_lg.h index 341f0a4ef1..c9c0c0c005 100644 --- a/esphome/components/climate_ir_lg/climate_ir_lg.h +++ b/esphome/components/climate_ir_lg/climate_ir_lg.h @@ -21,12 +21,13 @@ class LgIrClimate final : public climate_ir::ClimateIR { /// Override control to change settings of the climate device. void control(const climate::ClimateCall &call) override { this->send_swing_cmd_ = call.get_swing_mode().has_value(); - // swing resets after unit powered off + // swing resets after unit powered off, except when advanced_commands_support_ is set auto mode = call.get_mode(); - if (mode.has_value() && *mode == climate::CLIMATE_MODE_OFF) + if (mode.has_value() && *mode == climate::CLIMATE_MODE_OFF && !(this->advanced_commands_support_)) this->swing_mode = climate::CLIMATE_SWING_OFF; climate_ir::ClimateIR::control(call); } + void set_advanced_commands_support(bool value) { this->advanced_commands_support_ = value; } void set_header_high(uint32_t header_high) { this->header_high_ = header_high; } void set_header_low(uint32_t header_low) { this->header_low_ = header_low; } void set_bit_high(uint32_t bit_high) { this->bit_high_ = bit_high; } @@ -44,6 +45,7 @@ class LgIrClimate final : public climate_ir::ClimateIR { void calc_checksum_(uint32_t &value); void transmit_(uint32_t value); + bool advanced_commands_support_{false}; uint32_t header_high_; uint32_t header_low_; uint32_t bit_high_; diff --git a/tests/components/climate_ir_lg/common.yaml b/tests/components/climate_ir_lg/common.yaml index e0bc185d2c..5536c36742 100644 --- a/tests/components/climate_ir_lg/common.yaml +++ b/tests/components/climate_ir_lg/common.yaml @@ -12,5 +12,8 @@ climate: - platform: climate_ir_lg name: LG Climate transmitter_id: xmitr + header_high: 3300us + header_low: 9840us + advanced_commands_support: true sensor: climate_ir_lg_temp_sensor humidity_sensor: humidity_sensor From 1e7c48e2cfc6eac01ec515b34be57cde768d67d4 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Wed, 26 Aug 2026 06:35:46 -0700 Subject: [PATCH 50/65] [modbus_controller] Add integration tests for register offset, response size and write buffer (#18741) --- ...t_mock_modbus_deprecated_write_buffer.yaml | 106 ++++++++++++++ .../uart_mock_modbus_register_offset.yaml | 138 ++++++++++++++++++ tests/integration/test_uart_mock_modbus.py | 119 ++++++++++++++- 3 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml create mode 100644 tests/integration/fixtures/uart_mock_modbus_register_offset.yaml diff --git a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml b/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml new file mode 100644 index 0000000000..f378e3de43 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml @@ -0,0 +1,106 @@ +esphome: + name: uart-mock-modbus-dep-buffer + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg10 + type: uint16_t + initial_value: "0" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x10 + value_type: U_WORD + read_lambda: return id(reg10); + write_lambda: |- + id(reg10) = x; + return true; + +# A number whose write_lambda uses the DEPRECATED buffer parameter (fills `payload` with a legacy raw +# frame as words: device address + function code + data) instead of the new item->write_* API. The write +# must still land with its legacy semantics, and the one-time deprecation warning must fire only once per +# entity no matter how many writes happen. +number: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "buf_number" + id: buf_number + address: 0x10 + register_type: holding + value_type: U_WORD + min_value: 0 + max_value: 1000 + step: 1 + write_lambda: |- + // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0010, value. + payload.push_back(0x0106); + payload.push_back(0x0010); + payload.push_back((uint16_t) x); + return {}; + +# Reports the server-side register so the test can observe that the deprecated buffer write landed. +sensor: + - platform: template + name: "written_value" + id: written_value + update_interval: 0.5s + lambda: "return id(reg10);" + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # The test drives the writes via number_command; the mock is autostart. diff --git a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml b/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml new file mode 100644 index 0000000000..e93e78d5a3 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml @@ -0,0 +1,138 @@ +esphome: + name: uart-mock-modbus-reg-offset + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg10 + type: uint16_t + initial_value: "100" + - id: reg11 + type: uint16_t + initial_value: "200" + - id: reg12 + type: uint16_t + initial_value: "300" + - id: reg13 + type: uint16_t + initial_value: "0xABCD" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x10 + value_type: U_WORD + read_lambda: return id(reg10); + write_lambda: id(reg10) = x; return true; + - address: 0x11 + value_type: U_WORD + read_lambda: return id(reg11); + write_lambda: id(reg11) = x; return true; + - address: 0x12 + value_type: U_WORD + read_lambda: return id(reg12); + write_lambda: id(reg12) = x; return true; + - address: 0x13 + value_type: U_WORD + read_lambda: return id(reg13); + write_lambda: id(reg13) = x; return true; + +# A holding-register switch at 0x10 with a 2-BYTE offset. offset is byte-based, so the write must target +# register 0x10 + 2/2 = 0x11. The old (pre-fix) behavior folded offset into the address as a register +# count, hitting 0x12 instead. assumed_state keeps the switch write-only so it does not read any register. +switch: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "offset_switch" + register_type: holding + address: 0x10 + offset: 2 + assumed_state: true + # A holding-register switch that READS its state. Byte offset 6 -> register 0x10 + 6/2 = 0x13. Post-fix + # the switch itself resolves to 0x13 (whole registers fold into the address, residual byte stays) and + # joins the 0x10..0x13 range, so no separate 0x13 sensor is needed. Pre-fix the whole byte offset folds + # into the address (0x16), where the server answers ILLEGAL_DATA_ADDRESS and the switch never publishes. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "read_offset_switch" + register_type: holding + address: 0x10 + offset: 6 + bitmask: 0x1 + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_10" + address: 0x10 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_11" + address: 0x11 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_12" + address: 0x12 + register_type: holding + value_type: U_WORD + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index c84fb34e70..707637cfc2 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -24,7 +24,7 @@ from dataclasses import dataclass from aioesphomeapi import ButtonInfo, NumberInfo, SwitchInfo import pytest -from .state_utils import SensorTracker, find_entity +from .state_utils import SensorTracker, find_entity, wait_for_state from .types import APIClientConnectedFactory, RunCompiledFunction @@ -965,3 +965,120 @@ async def test_uart_mock_modbus_client_read_write( await tracker.setup_and_start_scenario(client) await tracker.await_all(futures) _assert_no_modbus_errors(error_log_lines, warning_log_lines) + + +@pytest.mark.xfail( + strict=True, + reason="Byte-accurate register-offset writes require the modbus_controller " + "entity-device change; on dev the byte offset is folded into the address " + "(writes 0x12 instead of 0x11). The write and read assertions both flip via " + "the same switch-constructor fold. Remove this marker when that change merges.", +) +@pytest.mark.asyncio +async def test_uart_mock_modbus_register_offset( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that a byte offset on a holding-register write is byte-accurate. + + `offset` is a byte offset, so a holding-register write at address 0x10 with offset: 2 must target + register 0x10 + 2/2 = 0x11. The pre-fix behavior folded the byte offset into the address as a register + count (0x10 + 2 = 0x12). The switch is assumed_state (write-only), so reg_11 turning 0xFFFF pins the + fix; had the write landed on 0x12 the wait would time out and reg_12 would change instead. + """ + + tracker = SensorTracker(["reg_10", "reg_11", "reg_12"]) + initial = tracker.expect_all({"reg_10": 100, "reg_11": 200, "reg_12": 300}) + wrote_11 = tracker.expect("reg_11", 65535) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + await tracker.await_all(initial, timeout=4.0) + + switch = find_entity(entities, "offset_switch", SwitchInfo) + assert switch is not None, "offset_switch not found" + client.switch_command(switch.key, True) + + # reg_11 (0x10 + offset 2/2) must receive the write; if the write went to 0x12 this times out. + await tracker.await_change(wrote_11, "reg_11", timeout=4.0) + # And 0x12 (the pre-fix register-offset target) must be untouched. + assert tracker.sensor_states["reg_12"][-1] == 300, ( + "reg_12 (0x12) should be untouched - offset is byte-based, so the write targets 0x11; " + f"got {tracker.sensor_states['reg_12']}" + ) + + # Read path: read_offset_switch has byte offset 6. Post-fix the switch folds the whole registers + # into its address (0x10 + 6/2 = 0x13, residual byte 0) and joins the 0x10..0x13 range, so the + # read lands in-bounds on 0xABCD (bit 0 set) -> ON. Pre-fix the whole byte offset folded into the + # address (0x16); the server answers ILLEGAL_DATA_ADDRESS there and the switch never publishes. + read_switch = find_entity(entities, "read_offset_switch", SwitchInfo) + assert read_switch is not None, "read_offset_switch not found" + # The ON transition happened at the first poll and switch states are deduped, so this relies on + # wait_for_state's fresh subscribe_states re-dumping every entity's current state. + await wait_for_state( + client, + lambda s: ( + getattr(s, "key", None) == read_switch.key + and getattr(s, "state", None) is True + ), + timeout=6.0, + ) + + +@pytest.mark.xfail( + strict=True, + reason="The deprecated write buffer requires the modbus_controller " + "entity-device change; on dev a nullopt-returning write_lambda early-returns " + "before the buffer is used, so the write never happens. The warn-once " + "assertion matches the log substring 'write_lambda buffer'. Remove this " + "marker when that change merges.", +) +@pytest.mark.asyncio +async def test_uart_mock_modbus_deprecated_write_buffer( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test the deprecated write_lambda buffer path still works, and warns once per entity. + + buf_number's write_lambda fills the old `payload` buffer with a legacy raw frame as words (device + address + function code + data) instead of calling item->write_*. Two writes must both land with the + legacy raw-frame semantics, and the one-time deprecation warning must fire exactly once per entity + regardless of how many writes happen. + """ + + warn_count = 0 + + def line_callback(line: str) -> None: + nonlocal warn_count + if "write_lambda buffer" in line: + warn_count += 1 + + tracker = SensorTracker(["written_value"]) + + async with ( + run_compiled(yaml_config, line_callback=line_callback), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + number = find_entity(entities, "buf_number", NumberInfo) + assert number is not None, "buf_number not found" + + # First write via the deprecated buffer path. + client.number_command(number.key, 111) + await tracker.await_change( + tracker.expect("written_value", 111), "written_value", timeout=4.0 + ) + # Second write: lands too, but must not warn again (warn-once per entity). + client.number_command(number.key, 222) + await tracker.await_change( + tracker.expect("written_value", 222), "written_value", timeout=4.0 + ) + + assert warn_count == 1, ( + f"deprecation warning should fire exactly once per entity, got {warn_count}" + ) From 4eb85a24c20f5eab14ccefb487e196832fafe890 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:04:57 +1000 Subject: [PATCH 51/65] [mipi_spi] Fix dimensions for jc3636518v2 (#18786) --- esphome/components/mipi_spi/models/jc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/esphome/components/mipi_spi/models/jc.py b/esphome/components/mipi_spi/models/jc.py index ca9adb4a72..8d2591aefe 100644 --- a/esphome/components/mipi_spi/models/jc.py +++ b/esphome/components/mipi_spi/models/jc.py @@ -266,8 +266,6 @@ DriverChip( "JC3636W518V2", height=360, width=360, - offset_height=1, - draw_rounding=1, cs_pin=10, reset_pin=47, invert_colors=True, From 7b4894da03677670e5f2dc712599e1a280801e9c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 10:16:44 -0500 Subject: [PATCH 52/65] [mdns] Skip MDNS.update() while the ESP8266 radio cannot transmit (#18785) --- esphome/components/mdns/mdns_esp8266.cpp | 14 +++++++++++++- esphome/components/wifi/wifi_component.cpp | 6 +++--- esphome/components/wifi/wifi_component.h | 7 +++++++ esphome/components/wifi/wifi_component_esp8266.cpp | 2 +- esphome/components/wifi/wifi_component_esp_idf.cpp | 2 +- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index f6d5786675..1f0b3c9519 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -41,7 +41,19 @@ static void register_esp8266(MDNSComponent *, StaticVectorset_interval(MDNS_POLL_ID, MDNS_UPDATE_INTERVAL_MS, []() { MDNS.update(); }); + this->set_interval(MDNS_POLL_ID, MDNS_UPDATE_INTERVAL_MS, []() { +#ifdef USE_MDNS_WIFI_LISTENER + // MDNS.update() can suspend the loop in UdpContext::sendTimeout() while a send is + // failing (radio off-channel during a roam scan, or mid reconnect); an incoming + // packet then re-enters LEAmDNS from lwIP and corrupts shared UdpContext state. + // Skip the tick while the radio cannot transmit (#18760), but keep polling while + // the AP is serving clients (AP-only or fallback AP with the STA down). + auto *wifi = wifi::global_wifi_component; + if (wifi->is_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) + return; +#endif + MDNS.update(); + }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } #endif diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index b8a31f97a3..d82929e5cb 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -530,7 +530,7 @@ void WiFiComponent::log_discarded_scan_result_(const char *ssid, const uint8_t * #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE // Skip logging during roaming scans to avoid log buffer overflow // (roaming scans typically find many networks but only care about same-SSID APs) - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { return; } char bssid_s[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; @@ -833,7 +833,7 @@ void WiFiComponent::loop() { // Post-connect roaming: check for better AP if (this->post_connect_roaming_) { - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { if (this->scan_done_) { this->process_roaming_scan_(); } @@ -2144,7 +2144,7 @@ void WiFiComponent::retry_connect() { // Roam connection failed - transition to reconnecting ESP_LOGD(TAG, "Roam failed, reconnecting (attempt %u/%u)", this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); this->roaming_state_ = RoamingState::RECONNECTING; - } else if (this->roaming_state_ == RoamingState::SCANNING) { + } else if (this->is_roaming_scan_active()) { // Disconnected during roam scan - transition to RECONNECTING so the attempts // counter is preserved when reconnection succeeds (IDLE would reset it) ESP_LOGD(TAG, "Disconnected during roam scan (attempt %u/%u)", this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 382d3d5932..cfdbc1a968 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -478,6 +478,13 @@ class WiFiComponent final : public Component { bool is_connected() const { return this->connected_; } + /// True while a post-connect roaming scan holds the radio off-channel. + bool is_roaming_scan_active() const { return this->roaming_state_ == RoamingState::SCANNING; } + + /// True while a post-connect roam is in progress (scanning off-channel, reassociating, + /// or recovering from a failed roam). + bool is_roaming() const { return this->roaming_state_ != RoamingState::IDLE; } + #ifdef USE_ESP32 /// esp_netif handle of the station interface, used by network for default-route /// arbitration. nullptr until wifi_lazy_init_() has run. diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 005d655d88..b4a91fb3cd 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -717,7 +717,7 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { static constexpr uint32_t SCAN_ACTIVE_MAX_DEFAULT_MS = 500; static constexpr uint32_t SCAN_ACTIVE_MIN_ROAMING_MS = 100; static constexpr uint32_t SCAN_ACTIVE_MAX_ROAMING_MS = 300; - bool roaming = this->roaming_state_ == RoamingState::SCANNING; + bool roaming = this->is_roaming_scan_active(); if (passive) { config.scan_time.passive = roaming ? SCAN_PASSIVE_ROAMING_MS : SCAN_PASSIVE_DEFAULT_MS; } else { diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 06f0981020..ce75d21330 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -1059,7 +1059,7 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { // When scanning while connected (roaming), return to home channel between // each scanned channel to maintain the connection (helps with BLE/WiFi coexistence) #ifdef CONFIG_SOC_WIFI_SUPPORTED - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { config.coex_background_scan = true; } #endif From 5c79c92c0657ef8ae9cee1745bddbe3e7e0fc439 Mon Sep 17 00:00:00 2001 From: guillempages Date: Wed, 26 Aug 2026 18:26:39 +0200 Subject: [PATCH 53/65] [runtime_image] Add FILTER_SOURCE_FILES (#18768) --- esphome/components/runtime_image/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/esphome/components/runtime_image/__init__.py b/esphome/components/runtime_image/__init__.py index 9277c214ff..a220503045 100644 --- a/esphome/components/runtime_image/__init__.py +++ b/esphome/components/runtime_image/__init__.py @@ -10,6 +10,7 @@ from esphome.components.image import ( validate_transparency, validate_type, ) +from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv from esphome.const import CONF_FORMAT, CONF_ID, CONF_RESIZE, CONF_TYPE from esphome.core import CORE @@ -124,6 +125,15 @@ IMAGE_FORMATS = { "PNG": PNGFormat(), } +FILTER_SOURCE_FILES = filter_source_files_from_defines( + { + "bmp_decoder.cpp": "USE_RUNTIME_IMAGE_BMP", + "jpeg_decoder.cpp": "USE_RUNTIME_IMAGE_JPEG", + "png_decoder.cpp": "USE_RUNTIME_IMAGE_PNG", + "qoi_decoder.cpp": "USE_RUNTIME_IMAGE_QOI", + } +) + AUTO_FORMAT = AUTOFormat() From 1cbe3a49b2bf9375f0c778f9d44084c12ec3eaea Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Wed, 26 Aug 2026 11:30:07 -0500 Subject: [PATCH 54/65] [remote_transmitter] ISR-driven transmission and non_blocking support on BK7231N/BK7238 (#18660) --- .../components/remote_transmitter/__init__.py | 26 +- .../remote_transmitter/remote_transmitter.cpp | 4 +- .../remote_transmitter/remote_transmitter.h | 44 +++- .../remote_transmitter_bk72xx.cpp | 187 +++++++++++++++ .../remote_transmitter_libretiny_isr.cpp | 224 ++++++++++++++++++ .../remote_transmitter_rtl87xx.cpp | 212 ++--------------- .../test_non_blocking_gate.py | 6 + .../remote_transmitter/test.bk72xx-ard.yaml | 1 + 8 files changed, 497 insertions(+), 207 deletions(-) create mode 100644 esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp create mode 100644 esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index 8ae51829e7..cb2aebec91 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -4,7 +4,11 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base from esphome.components.libretiny import get_libretiny_family -from esphome.components.libretiny.const import FAMILY_RTL8720C +from esphome.components.libretiny.const import ( + FAMILY_BK7231N, + FAMILY_BK7238, + FAMILY_RTL8720C, +) from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -45,14 +49,19 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) +_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7231N, FAMILY_BK7238) + + def _validate_non_blocking_platform(value: bool) -> bool: - # non_blocking requires hardware transmission: RMT on ESP32, the gtimer - # envelope chain on RTL8720C. Reject everywhere else at config time. + # non_blocking requires hardware transmission: RMT on ESP32, a hardware timer + # envelope chain on the listed LibreTiny families. Reject elsewhere at config time. if CORE.is_esp32: return cv.boolean(value) - if CORE.is_libretiny and get_libretiny_family() == FAMILY_RTL8720C: + if CORE.is_libretiny and get_libretiny_family() in _NON_BLOCKING_LIBRETINY_FAMILIES: return cv.boolean(value) - raise cv.Invalid("non_blocking is only supported on ESP32 and RTL8720C") + raise cv.Invalid( + "non_blocking is only supported on ESP32, RTL8720C, BK7231N and BK7238" + ) MULTI_CONF = True @@ -202,6 +211,13 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( "remote_transmitter_rtl87xx.cpp": { PlatformFramework.RTL87XX_ARDUINO, }, + "remote_transmitter_bk72xx.cpp": { + PlatformFramework.BK72XX_ARDUINO, + }, + "remote_transmitter_libretiny_isr.cpp": { + PlatformFramework.RTL87XX_ARDUINO, + PlatformFramework.BK72XX_ARDUINO, + }, "remote_transmitter.cpp": { PlatformFramework.ESP32_ARDUINO, PlatformFramework.ESP32_IDF, diff --git a/esphome/components/remote_transmitter/remote_transmitter.cpp b/esphome/components/remote_transmitter/remote_transmitter.cpp index 67341e936f..5e82213a48 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter.cpp @@ -2,8 +2,8 @@ #include "esphome/core/log.h" #include "esphome/core/application.h" -#if (defined(USE_LIBRETINY) && !defined(USE_RTL87XX)) || defined(USE_ESP8266) || defined(USE_RP2) || \ - (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) +#if (defined(USE_LIBRETINY) && !defined(USE_RTL87XX) && !defined(REMOTE_TRANSMITTER_BK_PWM)) || \ + defined(USE_ESP8266) || defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) namespace esphome::remote_transmitter { diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index ef9a80f668..313b26364d 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -12,6 +12,13 @@ #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 +// The BK7231N-style PWM block (hardware shadow-load duty updates) enables the ISR-driven +// transmitter on these families; family-level proxy for the SDK's CFG_SOC_NAME gate. +// See remote_transmitter_bk72xx.cpp. +#if defined(USE_LIBRETINY_VARIANT_BK7231N) || defined(USE_LIBRETINY_VARIANT_BK7238) +#define REMOTE_TRANSMITTER_BK_PWM +#endif + namespace esphome::remote_transmitter { #if defined(USE_ESP32) && SOC_RMT_SUPPORTED @@ -57,13 +64,16 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa void set_with_dma(bool with_dma) { this->with_dma_ = with_dma; } void set_eot_level(bool eot_level) { this->eot_level_ = eot_level; } #endif -#if (defined(USE_ESP32) && SOC_RMT_SUPPORTED) || defined(USE_LIBRETINY_VARIANT_RTL8720C) +#if (defined(USE_ESP32) && SOC_RMT_SUPPORTED) || defined(USE_LIBRETINY_VARIANT_RTL8720C) || \ + defined(REMOTE_TRANSMITTER_BK_PWM) void set_non_blocking(bool non_blocking) { this->non_blocking_ = non_blocking; } #endif -#ifdef USE_LIBRETINY_VARIANT_RTL8720C +#if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) void loop() override; // called from the envelope timer ISR trampoline; not part of the public API void advance_envelope_isr(); + // same, for trampolines whose SDK callback carries no user argument + static void advance_active_isr(); #endif Trigger<> *get_transmit_trigger() { return &this->transmit_trigger_; } @@ -71,12 +81,14 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa protected: void send_internal(uint32_t send_times, uint32_t send_wait) override; -#if defined(USE_ESP8266) || (defined(USE_LIBRETINY) && !defined(USE_LIBRETINY_VARIANT_RTL8720C)) || \ +#if defined(USE_ESP8266) || \ + (defined(USE_LIBRETINY) && !defined(USE_LIBRETINY_VARIANT_RTL8720C) && !defined(REMOTE_TRANSMITTER_BK_PWM)) || \ defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) void await_target_time_(); uint32_t target_time_{0}; #endif -#if defined(USE_ESP8266) || (defined(USE_LIBRETINY) && !defined(USE_RTL87XX)) || defined(USE_RP2) || \ +#if defined(USE_ESP8266) || \ + (defined(USE_LIBRETINY) && !defined(USE_RTL87XX) && !defined(REMOTE_TRANSMITTER_BK_PWM)) || defined(USE_RP2) || \ (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) void calculate_on_off_time_(uint32_t carrier_frequency, uint32_t *on_time_period, uint32_t *off_time_period); @@ -89,17 +101,22 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa uint32_t current_carrier_frequency_{0}; void *pwm_{nullptr}; // pwmout_t*, opaque here to keep the SDK header out of this shared header #endif -#ifdef USE_LIBRETINY_VARIANT_RTL8720C +#if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) + // Envelope chain, shared by every family that paces transmission from a hardware timer + // (remote_transmitter_libretiny_isr.cpp) void start_isr_item_(size_t index); void arm_envelope_timer_(uint32_t duration_us); void abort_stalled_chain_(); void deliver_completion_(); void wait_until_idle_(); void arm_chain_(uint32_t send_times, uint32_t send_wait); - void update_carrier_(uint32_t carrier_frequency); + // Hooks implemented per family: everything the chain needs from the hardware + bool envelope_ready_() const; // PWM claimed successfully in setup() + void prepare_carrier_(uint32_t carrier_frequency); // retune period, stage mark/space levels + void write_envelope_level_(bool mark); // drive carrier (mark) or idle (space) + void arm_one_shot_(uint32_t duration_us); // fire advance_envelope_isr after duration_us + void stop_envelope_timer_(); std::vector isr_data_; // owned copy of the frame; temp_ may be re-encoded mid-flight - float isr_mark_duty_{0.0f}; - float isr_space_duty_{0.0f}; volatile size_t isr_index_{0}; volatile uint32_t isr_repeats_left_{0}; uint32_t isr_send_wait_{0}; @@ -110,6 +127,17 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa bool complete_pending_{false}; bool stall_aborted_{false}; // this transmission ended via abort; blocks warning clear #endif +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + float isr_mark_duty_{0.0f}; + float isr_space_duty_{0.0f}; +#endif +#ifdef REMOTE_TRANSMITTER_BK_PWM + void write_pwm_t1_(uint32_t t1_counts); + uint32_t isr_mark_t1_{0}; + uint32_t isr_space_t1_{0}; + uint32_t isr_period_t4_{684}; // 26MHz counts; ~38kHz default until a send sets the real carrier + int8_t pwm_channel_{-1}; +#endif #if defined(USE_ESP32) && SOC_RMT_SUPPORTED void configure_rmt_(); diff --git a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp new file mode 100644 index 0000000000..0081ae47b3 --- /dev/null +++ b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp @@ -0,0 +1,187 @@ +#include "remote_transmitter.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +// clang-tidy cannot parse the Beken SDK headers pulled in via ArduinoPrivate.h +#if defined(USE_BK72XX) && !defined(CLANG_TIDY) + +// ArduinoPrivate.h = Arduino.h + the BDK SDK headers (pwm_pub.h, bk_timer_pub.h, icu_pub.h) +// with the core's fixes for type-name collisions between the two +#include + +// Only the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) +// supports glitch-free per-edge duty updates; older SoCs compile the generic bit-bang +// implementation (remote_transmitter.cpp) instead, and this file compiles to nothing. +// REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h. + +namespace esphome::remote_transmitter { + +static const char *const TAG = "remote_transmitter"; + +#ifdef REMOTE_TRANSMITTER_BK_PWM + +// PWM peripheral carrier (26MHz block), envelope paced by a BKTIMER1 interrupt chain: each +// interrupt writes the next duty through the shadow registers (T1..T4 + CFG_UPDATA hardware +// load, glitch-free at the next carrier period). Direct register writes beat the driver's +// pwm_update_param() (~19us vs ~26us edge error) and have no shared state to race against. +// BKTIMER1 is the only free channel: TIMER0 = FreeRTOS tick, TIMER2 = SDK cal, TIMER4 = wdt. + +static constexpr uint32_t REG_PWM_BASE = 0x00802B00UL; +static constexpr uint32_t REG_PWM_GROUP_STRIDE = 0x40; // one register group per channel pair +static constexpr uint32_t REG_PWM_T_REGS[2] = {0x04, 0x14}; // T1..T4 offsets within a group +static constexpr uint32_t PWM_INT_STATUS_MASK = 3UL << 30; // write-1-clear -- always write as zero +static constexpr uint8_t ENVELOPE_TIMER = BKTIMER1; + +// The bk_timer handler receives only the channel number, so the chain resolves the instance +// that owns the timer. No IRAM_ATTR: hal.h makes it a no-op on BK72xx (the SDK masks IRQs +// around flash writes). +static void envelope_timer_isr(UINT8 channel) { RemoteTransmitterComponent::advance_active_isr(); } + +// Channel <-> pin comes from the board variant's own PIN_PWMn defines rather than a +// family-wide assumption, so an unusual pinout maps correctly instead of silently +// driving another pad +struct PwmPinChannel { + uint8_t pin; + int8_t channel; +}; +static constexpr PwmPinChannel PWM_PIN_CHANNELS[] = { +#ifdef PIN_PWM0 + {PIN_PWM0, 0}, +#endif +#ifdef PIN_PWM1 + {PIN_PWM1, 1}, +#endif +#ifdef PIN_PWM2 + {PIN_PWM2, 2}, +#endif +#ifdef PIN_PWM3 + {PIN_PWM3, 3}, +#endif +#ifdef PIN_PWM4 + {PIN_PWM4, 4}, +#endif +#ifdef PIN_PWM5 + {PIN_PWM5, 5}, +#endif +}; + +static int8_t pwm_channel_for_pin(uint8_t pin) { + for (const auto &entry : PWM_PIN_CHANNELS) { + if (entry.pin == pin) + return entry.channel; + } + return -1; +} + +void RemoteTransmitterComponent::setup() { + // Deliberately no pin_->setup(): the pin must belong to the PWM function, not GPIO + const int8_t channel = pwm_channel_for_pin(this->pin_->get_pin()); + if (channel < 0) { + ESP_LOGE(TAG, "Pin %u is not PWM-capable", this->pin_->get_pin()); + this->mark_failed(); + return; + } + this->pwm_channel_ = channel; + const uint32_t idle_t1 = this->pin_->is_inverted() ? this->isr_period_t4_ : 0; + pwm_param_st param{}; + param.chan = channel; + param.t1 = idle_t1; + param.t4 = this->isr_period_t4_; + param.init_level = idle_t1 ? 1 : 0; + if (pwm_init_param(¶m) != 0 || pwm_start(channel) != 0) { + ESP_LOGE(TAG, "PWM init failed on pin %u", this->pin_->get_pin()); + this->pwm_channel_ = -1; + this->mark_failed(); + return; + } + this->disable_loop(); // loop() is only needed while a non-blocking completion is pending +} + +void RemoteTransmitterComponent::dump_config() { + ESP_LOGCONFIG(TAG, + "Remote Transmitter:\n" + " Carrier Duty: %u%%\n" + " Non-blocking: %s", + this->carrier_duty_percent_, YESNO(this->non_blocking_)); + LOG_PIN(" Pin: ", this->pin_); +} + +// Writes the duty compare registers and sets the hardware CFG_UPDATA shadow-load bit; +// the new duty latches glitch-free at the next carrier period. ISR-safe: registers only. +// The group control word is shared with the paired channel, but every SDK write to it runs +// under GLOBAL_INT_DISABLE (bk_pwm), so it cannot be torn by this interrupt. +void RemoteTransmitterComponent::write_pwm_t1_(uint32_t t1_counts) { + const uint32_t group = this->pwm_channel_ / 2; + const uint32_t post = this->pwm_channel_ % 2; + const uint32_t group_base = REG_PWM_BASE + REG_PWM_GROUP_STRIDE * group; + auto *t_regs = (volatile uint32_t *) (group_base + REG_PWM_T_REGS[post]); + auto *ctrl = (volatile uint32_t *) group_base; + const uint32_t init_level_bit = 1UL << (8 * post + 6); // output level while the counter is stopped + const uint32_t cfg_updata_bit = 1UL << (8 * post + 7); // 0->1 latches T1..T4 at the next period + t_regs[0] = t1_counts; // T1: high time + t_regs[1] = 0; // T2 + t_regs[2] = 0; // T3 + t_regs[3] = this->isr_period_t4_; // T4: period + uint32_t cfg = *ctrl; + cfg &= ~(PWM_INT_STATUS_MASK | init_level_bit | cfg_updata_bit); + if (t1_counts != 0) + cfg |= init_level_bit; + *ctrl = cfg; + *ctrl = cfg | cfg_updata_bit; +} + +// --- envelope chain hooks (see remote_transmitter_libretiny_isr.cpp) --- + +bool RemoteTransmitterComponent::envelope_ready_() const { return this->pwm_channel_ >= 0; } + +// Recomputes the carrier period in 26MHz counts and stages the per-item duties; +// unmodulated protocols drive the pin constantly during marks +void RemoteTransmitterComponent::prepare_carrier_(uint32_t carrier_frequency) { + if (carrier_frequency > 0) { + this->isr_period_t4_ = std::max(uint32_t(2), (26000000UL + carrier_frequency / 2) / carrier_frequency); + } + uint32_t mark_t1 = (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) + ? std::max(uint32_t(1), this->isr_period_t4_ * this->carrier_duty_percent_ / 100) + : this->isr_period_t4_; + uint32_t space_t1 = 0; + if (this->pin_->is_inverted()) { + mark_t1 = this->isr_period_t4_ - mark_t1; + space_t1 = this->isr_period_t4_; + } + this->isr_mark_t1_ = mark_t1; + this->isr_space_t1_ = space_t1; +} + +void RemoteTransmitterComponent::write_envelope_level_(bool mark) { + this->write_pwm_t1_(mark ? this->isr_mark_t1_ : this->isr_space_t1_); +} + +// The driver's microsecond init path is register writes under a nested interrupt guard, +// so it is safe to call from the chain's own interrupt +void RemoteTransmitterComponent::arm_one_shot_(uint32_t duration_us) { + timer_param_t param{}; + param.channel = ENVELOPE_TIMER; + param.div = 1; + param.period = duration_us; + param.t_Int_Handler = envelope_timer_isr; + sddev_control((char *) TIMER_DEV_NAME, CMD_TIMER_INIT_PARAM_US, ¶m); +} + +void RemoteTransmitterComponent::stop_envelope_timer_() { + UINT32 channel = ENVELOPE_TIMER; + sddev_control((char *) TIMER_DEV_NAME, CMD_TIMER_UNIT_DISABLE, &channel); +} + +void RemoteTransmitterComponent::digital_write(bool value) { + if (this->pwm_channel_ < 0) + return; + // serialize behind an in-flight chain, matching the ESP32/RMT non-blocking behavior + this->wait_until_idle_(); + this->write_pwm_t1_((value != this->pin_->is_inverted()) ? this->isr_period_t4_ : 0); +} + +#endif // REMOTE_TRANSMITTER_BK_PWM + +} // namespace esphome::remote_transmitter + +#endif // USE_BK72XX && !CLANG_TIDY diff --git a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp new file mode 100644 index 0000000000..003cdfa986 --- /dev/null +++ b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp @@ -0,0 +1,224 @@ +#include "remote_transmitter.h" +#include "esphome/core/application.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +// Envelope chain shared by the LibreTiny families that pace transmission from a hardware +// timer interrupt: RTL8720C (gtimer) and the BK7231N-style PWM block (BKTIMER1). Everything +// platform-specific sits behind five hooks implemented in the per-family files -- carrier +// setup, duty writes, one-shot arming and timer stop. Families without a usable timer keep +// the generic bit-bang implementation and compile none of this. +#if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) + +namespace esphome::remote_transmitter { + +static const char *const TAG = "remote_transmitter"; + +// Margin past a transmission's expected duration before the chain is declared stalled +static constexpr uint32_t STALL_MARGIN_MS = 1000; +// Longest single one-shot armed; longer durations are chained. Both families need the cap: +// the Beken driver computes period_us * 26 in 32 bits (overflows past ~165s) and the Realtek +// us->tick conversion lives in mask ROM with unverified headroom. +static constexpr uint32_t MAX_ONE_SHOT_US = 50000; + +// One hardware timer is shared by all instances (MULTI_CONF), so they serialize on this +// token; the deadline always describes whichever chain currently owns it. +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) +static RemoteTransmitterComponent *volatile s_active_transmitter = nullptr; +static uint32_t s_expected_end_ms = 0; +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) + +// Entry point for trampolines whose SDK callback carries no user argument +void IRAM_ATTR RemoteTransmitterComponent::advance_active_isr() { + auto *transmitter = s_active_transmitter; + if (transmitter != nullptr) + transmitter->advance_envelope_isr(); +} + +// Arms the envelope timer, chaining durations longer than MAX_ONE_SHOT_US. ISR-safe. +void IRAM_ATTR RemoteTransmitterComponent::arm_envelope_timer_(uint32_t duration_us) { + // clamp to 1us (a zero-length one-shot never fires); the remainder must not underflow + const uint32_t chunk = std::max(uint32_t(1), std::min(duration_us, MAX_ONE_SHOT_US)); + this->isr_wait_remaining_ = duration_us > chunk ? duration_us - chunk : 0; + this->arm_one_shot_(chunk); +} + +// Writes the level for one envelope item and arms the timer for its duration. +// Runs in ISR context (and once from arm_chain_ to kick the chain): no logging, no allocation. +void IRAM_ATTR RemoteTransmitterComponent::start_isr_item_(size_t index) { + const int32_t item = this->isr_data_[index]; + this->write_envelope_level_(item > 0); + this->arm_envelope_timer_(uint32_t(item > 0 ? item : -item)); +} + +void IRAM_ATTR RemoteTransmitterComponent::advance_envelope_isr() { + if (!this->transmitting_) + return; // chain was aborted; this is a stale one-shot that was already latched + if (this->isr_wait_remaining_ > 0) { + // continue a duration longer than one hardware one-shot + this->arm_envelope_timer_(this->isr_wait_remaining_); + return; + } + if (this->isr_in_gap_) { + // inter-repeat gap elapsed; restart the item chain + this->isr_in_gap_ = false; + this->isr_index_ = 0; + this->start_isr_item_(0); + return; + } + this->isr_index_ = this->isr_index_ + 1; + if (this->isr_index_ < this->isr_data_.size()) { + this->start_isr_item_(this->isr_index_); + return; + } + // end of one repetition + this->write_envelope_level_(false); + if (this->isr_repeats_left_ > 1) { + this->isr_repeats_left_ = this->isr_repeats_left_ - 1; + this->isr_index_ = 0; + if (this->isr_send_wait_ > 0) { + this->isr_in_gap_ = true; + this->arm_envelope_timer_(this->isr_send_wait_); + } else { + this->start_isr_item_(0); + } + return; + } + // required on Beken (its timer reloads); on Realtek this only clears the enable bit of a + // one-shot that has already fired + this->stop_envelope_timer_(); + this->transmitting_ = false; + s_active_transmitter = nullptr; +} + +// Aborts a chain that stopped advancing: stop the timer, idle the pin, release the token. +// Every step is a no-op if the chain completed meanwhile. Task context only. +void RemoteTransmitterComponent::abort_stalled_chain_() { + // cleared first so a straggler one-shot bails at the ISR entry check + this->transmitting_ = false; + this->stop_envelope_timer_(); + this->write_envelope_level_(false); + s_active_transmitter = nullptr; + this->stall_aborted_ = true; + this->status_set_warning("envelope timer stalled"); + ESP_LOGE(TAG, "Envelope timer stalled; transmission aborted"); + delay(1); // let any already-latched interrupt land while the chain state is safe +} + +// Delivers one deferred completion with its status bookkeeping +void RemoteTransmitterComponent::deliver_completion_() { + if (!this->stall_aborted_) + this->status_clear_warning(); + this->complete_pending_ = false; + this->complete_trigger_.trigger(); +} + +// Waits until no chain is in flight, delivering any deferred completions; a completion +// automation may start a new send, so repeat until truly idle. Bounded by the stall deadline. +void RemoteTransmitterComponent::wait_until_idle_() { + while (true) { + while (true) { + // snapshot: the final ISR can clear the volatile pointer between a check and a use + auto *active = s_active_transmitter; + if (active == nullptr) + break; + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + active->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + if (!this->complete_pending_) + break; + this->deliver_completion_(); + } +} + +// Stages the repeat schedule and stall deadline, then starts the interrupt chain +void RemoteTransmitterComponent::arm_chain_(uint32_t send_times, uint32_t send_wait) { + this->isr_repeats_left_ = send_times; + this->isr_send_wait_ = send_wait; + this->isr_index_ = 0; + this->isr_in_gap_ = false; + this->stall_aborted_ = false; + uint64_t frame_us = 0; + for (int32_t item : this->isr_data_) + frame_us += uint32_t(item > 0 ? item : -item); + const uint64_t total_us = frame_us * send_times + uint64_t(send_wait) * (send_times - 1); + s_expected_end_ms = millis() + uint32_t(total_us / 1000) + STALL_MARGIN_MS; + this->transmitting_ = true; + s_active_transmitter = this; + this->start_isr_item_(0); +} + +void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { + if (!this->envelope_ready_()) { + // both triggers still fire, so an on_complete-sequenced automation does not stall + ESP_LOGW(TAG, "Cannot send: PWM not initialized"); + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + this->wait_until_idle_(); + if (send_times == 0) { + // parity with the loop-based implementations: transmit nothing, but both triggers + // still fire so an on_complete-sequenced automation does not stall + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + ESP_LOGD(TAG, "Sending remote code"); + this->prepare_carrier_(this->temp_.get_carrier_frequency()); + // own copy: with non_blocking the caller may re-encode temp_ while this frame is in flight + this->isr_data_.assign(this->temp_.get_data().begin(), this->temp_.get_data().end()); + if (this->isr_data_.empty()) { + ESP_LOGW(TAG, "Empty data"); + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + // trigger first: the deadline computed in arm_chain_ must not be charged for user code + this->transmit_trigger_.trigger(); + // the automation may have started a send on another instance; let it finish before + // claiming the shared timer (a same-instance send remains unsupported here) + this->wait_until_idle_(); + this->arm_chain_(send_times, send_wait); + if (this->non_blocking_) { + this->complete_pending_ = true; + this->enable_loop(); + return; + } + // blocking mode: wait out the chain, bounded by the stall deadline + while (this->transmitting_) { + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + this->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + this->deliver_completion_(); +} + +void RemoteTransmitterComponent::loop() { + if (!this->complete_pending_) { + this->disable_loop(); + return; + } + if (this->transmitting_) { + // non-blocking stall recovery: without this, a dead chain would leave the carrier + // driven and on_complete unfired until the next send happened to abort it + if ((int32_t) (millis() - s_expected_end_ms) <= 0) + return; + this->abort_stalled_chain_(); + } + // release the loop before user code runs: the automation may start a new non-blocking + // send, and its enable_loop() must be the last writer or its completion would strand + this->disable_loop(); + this->deliver_completion_(); +} + +} // namespace esphome::remote_transmitter + +#endif // USE_LIBRETINY_VARIANT_RTL8720C || REMOTE_TRANSMITTER_BK_PWM diff --git a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp index 9f629168f2..6db9faac36 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp @@ -24,20 +24,13 @@ static const char *const TAG = "remote_transmitter"; #ifdef USE_LIBRETINY_VARIANT_RTL8720C static constexpr uint32_t ENVELOPE_TIMER_ID = TIMER6; // GTimer7 -// Margin past a transmission's expected duration before the chain is declared stalled -static constexpr uint32_t STALL_MARGIN_MS = 1000; -// Longest single one-shot armed; longer durations are chained (ROM us->tick headroom unverified) -static constexpr uint32_t MAX_ONE_SHOT_US = 50000; -// Shared envelope timer: a second gtimer_init on the same id fails silently, so all -// instances serialize on s_active_transmitter +// One envelope timer for all instances: a second gtimer_init on the same id fails silently, +// so the chain serializes them (remote_transmitter_libretiny_isr.cpp) // NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) static uint8_t s_pwm_tick_sources[] = {GTimer1, GTimer2, GTimer3, GTimer4, GTimer5, GTimer6, 0xff}; static gtimer_t s_envelope_timer; static bool s_envelope_timer_ready = false; -static RemoteTransmitterComponent *volatile s_active_transmitter = nullptr; -// Deadline for the in-flight transmission (millis-based); only touched from the main task -static uint32_t s_expected_end_ms = 0; // NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) static void IRAM_ATTR envelope_timer_isr(uint32_t arg) { @@ -104,105 +97,22 @@ void RemoteTransmitterComponent::digital_write(bool value) { } #ifdef USE_LIBRETINY_VARIANT_RTL8720C -// Arms the shared envelope timer, chaining durations longer than MAX_ONE_SHOT_US. ISR-safe. -void IRAM_ATTR RemoteTransmitterComponent::arm_envelope_timer_(uint32_t duration_us) { - // clamp to 1us (a zero-length one-shot never fires); the remainder must not underflow - const uint32_t chunk = std::max(uint32_t(1), std::min(duration_us, MAX_ONE_SHOT_US)); - this->isr_wait_remaining_ = duration_us > chunk ? duration_us - chunk : 0; - gtimer_start_one_shout(&s_envelope_timer, chunk, (void *) envelope_timer_isr, (uint32_t) this); -} +// --- envelope chain hooks (see remote_transmitter_libretiny_isr.cpp) --- -// Aborts a chain that stopped advancing: stop the timer, idle the pin, release the token. -// Every step is a no-op if the chain completed meanwhile. Task context only. -void RemoteTransmitterComponent::abort_stalled_chain_() { - // cleared first so a straggler one-shot bails at the ISR entry check - this->transmitting_ = false; - gtimer_stop(&s_envelope_timer); - pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); - s_active_transmitter = nullptr; - this->stall_aborted_ = true; - this->status_set_warning("envelope timer stalled"); - ESP_LOGE(TAG, "Envelope timer stalled; transmission aborted"); - delay(1); // let any already-latched interrupt land while the chain state is safe -} +bool RemoteTransmitterComponent::envelope_ready_() const { return this->pwm_ != nullptr; } -// Delivers one deferred completion with its status bookkeeping -void RemoteTransmitterComponent::deliver_completion_() { - if (!this->stall_aborted_) - this->status_clear_warning(); - this->complete_pending_ = false; - this->complete_trigger_.trigger(); -} - -// Writes the duty for one envelope item and arms the timer for its duration. -// Runs in ISR context (and once from send_internal to kick the chain): no logging, no allocation. -void IRAM_ATTR RemoteTransmitterComponent::start_isr_item_(size_t index) { - const int32_t item = this->isr_data_[index]; - pwmout_write(static_cast(this->pwm_), item > 0 ? this->isr_mark_duty_ : this->isr_space_duty_); - this->arm_envelope_timer_(uint32_t(item > 0 ? item : -item)); -} - -void IRAM_ATTR RemoteTransmitterComponent::advance_envelope_isr() { - if (!this->transmitting_) - return; // chain was aborted; this is a stale one-shot that was already latched - if (this->isr_wait_remaining_ > 0) { - // continue a duration longer than one hardware one-shot - this->arm_envelope_timer_(this->isr_wait_remaining_); - return; +// Retunes the PWM period when the carrier changes and stages the per-item duties; +// unmodulated protocols (no carrier or 100% duty) drive the pin constantly during marks +void RemoteTransmitterComponent::prepare_carrier_(uint32_t carrier_frequency) { + float mark_duty = + (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) ? this->carrier_duty_percent_ / 100.0f : 1.0f; + float space_duty = 0.0f; + if (this->pin_->is_inverted()) { + mark_duty = 1.0f - mark_duty; + space_duty = 1.0f; } - if (this->isr_in_gap_) { - // inter-repeat gap elapsed; restart the item chain - this->isr_in_gap_ = false; - this->isr_index_ = 0; - this->start_isr_item_(0); - return; - } - this->isr_index_++; - if (this->isr_index_ < this->isr_data_.size()) { - this->start_isr_item_(this->isr_index_); - return; - } - // end of one repetition - pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); - if (this->isr_repeats_left_ > 1) { - this->isr_repeats_left_--; - this->isr_index_ = 0; - if (this->isr_send_wait_ > 0) { - this->isr_in_gap_ = true; - this->arm_envelope_timer_(this->isr_send_wait_); - } else { - this->start_isr_item_(0); - } - return; - } - this->transmitting_ = false; - s_active_transmitter = nullptr; -} - -// Waits until no chain is in flight, delivering any deferred completions; a completion -// automation may start a new send, so repeat until truly idle. Bounded by the stall deadline. -void RemoteTransmitterComponent::wait_until_idle_() { - while (true) { - while (true) { - // snapshot: the final ISR can clear the volatile pointer between a check and a use - auto *active = s_active_transmitter; - if (active == nullptr) - break; - if ((int32_t) (millis() - s_expected_end_ms) > 0) { - active->abort_stalled_chain_(); - break; - } - App.feed_wdt(); - delay(1); - } - if (!this->complete_pending_) - break; - this->deliver_completion_(); - } -} - -// Retunes the PWM period when the carrier changes; the ISR sets duty per item -void RemoteTransmitterComponent::update_carrier_(uint32_t carrier_frequency) { + this->isr_mark_duty_ = mark_duty; + this->isr_space_duty_ = space_duty; if (carrier_frequency == 0 || carrier_frequency == this->current_carrier_frequency_) return; // round(1000000/freq), clamped so a bad lambda can't hand the SDK a zero period @@ -211,97 +121,15 @@ void RemoteTransmitterComponent::update_carrier_(uint32_t carrier_frequency) { this->current_carrier_frequency_ = carrier_frequency; } -// Stages the repeat schedule and stall deadline, then starts the interrupt chain -void RemoteTransmitterComponent::arm_chain_(uint32_t send_times, uint32_t send_wait) { - this->isr_repeats_left_ = send_times; - this->isr_send_wait_ = send_wait; - this->isr_index_ = 0; - this->isr_in_gap_ = false; - this->stall_aborted_ = false; - uint64_t frame_us = 0; - for (int32_t item : this->isr_data_) - frame_us += uint32_t(item > 0 ? item : -item); - const uint64_t total_us = frame_us * send_times + uint64_t(send_wait) * (send_times - 1); - s_expected_end_ms = millis() + uint32_t(total_us / 1000) + STALL_MARGIN_MS; - this->transmitting_ = true; - s_active_transmitter = this; - this->start_isr_item_(0); +void IRAM_ATTR RemoteTransmitterComponent::write_envelope_level_(bool mark) { + pwmout_write(static_cast(this->pwm_), mark ? this->isr_mark_duty_ : this->isr_space_duty_); } -void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { - if (this->pwm_ == nullptr) { - ESP_LOGW(TAG, "Cannot send: PWM not initialized"); - return; - } - this->wait_until_idle_(); - if (send_times == 0) { - // parity with the loop-based implementations: transmit nothing, but both triggers - // still fire so an on_complete-sequenced automation does not stall - this->transmit_trigger_.trigger(); - this->deliver_completion_(); - return; - } - ESP_LOGD(TAG, "Sending remote code"); - const uint32_t carrier_frequency = this->temp_.get_carrier_frequency(); - // unmodulated protocols (no carrier or 100% duty) drive the pin constantly during marks - float mark_duty = - (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) ? this->carrier_duty_percent_ / 100.0f : 1.0f; - float space_duty = 0.0f; - if (this->pin_->is_inverted()) { - mark_duty = 1.0f - mark_duty; - space_duty = 1.0f; - } - this->update_carrier_(carrier_frequency); - // own copy: with non_blocking the caller may re-encode temp_ while this frame is in flight - this->isr_data_.assign(this->temp_.get_data().begin(), this->temp_.get_data().end()); - if (this->isr_data_.empty()) { - ESP_LOGW(TAG, "Empty data"); - this->transmit_trigger_.trigger(); - this->deliver_completion_(); - return; - } - this->isr_mark_duty_ = mark_duty; - this->isr_space_duty_ = space_duty; - // trigger first: the deadline computed in arm_chain_ must not be charged for user code - this->transmit_trigger_.trigger(); - // the automation may have started a send on another instance; let it finish before - // claiming the shared timer (a same-instance send remains unsupported here) - this->wait_until_idle_(); - this->arm_chain_(send_times, send_wait); - if (this->non_blocking_) { - this->complete_pending_ = true; - this->enable_loop(); - return; - } - // blocking mode: wait out the chain, bounded by the stall deadline - while (this->transmitting_) { - if ((int32_t) (millis() - s_expected_end_ms) > 0) { - this->abort_stalled_chain_(); - break; - } - App.feed_wdt(); - delay(1); - } - this->deliver_completion_(); +void IRAM_ATTR RemoteTransmitterComponent::arm_one_shot_(uint32_t duration_us) { + gtimer_start_one_shout(&s_envelope_timer, duration_us, (void *) envelope_timer_isr, (uint32_t) this); } -void RemoteTransmitterComponent::loop() { - if (!this->complete_pending_) { - this->disable_loop(); - return; - } - if (this->transmitting_) { - // non-blocking stall recovery: without this, a dead chain would leave the carrier - // driven and on_complete unfired until the next send happened to abort it - if ((int32_t) (millis() - s_expected_end_ms) <= 0) - return; - this->abort_stalled_chain_(); - } - // release the loop before user code runs: the automation may start a new non-blocking - // send, and its enable_loop() must be the last writer or its completion would strand - this->disable_loop(); - this->deliver_completion_(); -} +void IRAM_ATTR RemoteTransmitterComponent::stop_envelope_timer_() { gtimer_stop(&s_envelope_timer); } #else // !USE_LIBRETINY_VARIANT_RTL8720C -- AmebaZ (RTL8710B): spin-based envelope, per-frame priority boost diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py index f843f1e84f..ee2769e177 100644 --- a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -4,6 +4,9 @@ the ISR paths, so this gate is the only CI-reachable coverage for the platform m import pytest from esphome.components.libretiny.const import ( + FAMILY_BK7231N, + FAMILY_BK7231T, + FAMILY_BK7238, FAMILY_RTL8710B, FAMILY_RTL8720C, KEY_FAMILY, @@ -23,6 +26,9 @@ from ..types import SetCoreConfigCallable (PlatformFramework.ESP32_IDF, None, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7238, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231T, False), (PlatformFramework.ESP8266_ARDUINO, None, False), ], ) diff --git a/tests/components/remote_transmitter/test.bk72xx-ard.yaml b/tests/components/remote_transmitter/test.bk72xx-ard.yaml index 2a5cceddec..ea2feafda9 100644 --- a/tests/components/remote_transmitter/test.bk72xx-ard.yaml +++ b/tests/components/remote_transmitter/test.bk72xx-ard.yaml @@ -2,6 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO26 carrier_duty_percent: 50% + # non_blocking is bk7231n/bk7238-only; the CI board is a BK7252 packages: buttons: !include common-buttons.yaml From 5328813814b52178e296a8cb8122825e98a0deaf Mon Sep 17 00:00:00 2001 From: MakerYuichi <106516578+MakerYuichi@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:13:55 +0530 Subject: [PATCH 55/65] [time] Silence compiler warning by initializing transit variables (#18715) (#18723) Co-authored-by: doraemon2200 <106516578+doraemon2200@users.noreply.github.com> --- esphome/components/time/posix_tz.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/time/posix_tz.cpp b/esphome/components/time/posix_tz.cpp index 188df599f6..002aadfec3 100644 --- a/esphome/components/time/posix_tz.cpp +++ b/esphome/components/time/posix_tz.cpp @@ -178,7 +178,8 @@ static int __attribute__((noinline)) days_from_year_start(int year, int month, i } time_t __attribute__((noinline)) calculate_dst_transition(int year, const DSTRule &rule, int32_t base_offset_seconds) { - int month, day; + int month = 1; + int day = 1; switch (rule.type) { case DSTRuleType::MONTH_WEEK_DAY: { From 8a89f3075f31b3bf0db104551fb566312528b872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Metrich?= <45318189+FredM67@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:03:10 +0200 Subject: [PATCH 56/65] [emontx] Fix sensor storage initialization ordering (#18771) --- esphome/components/emontx/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/esphome/components/emontx/__init__.py b/esphome/components/emontx/__init__.py index 7dde794f0b..3821f3e10e 100644 --- a/esphome/components/emontx/__init__.py +++ b/esphome/components/emontx/__init__.py @@ -116,14 +116,16 @@ _CALLBACK_AUTOMATIONS = ( async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) - await cg.register_component(var, config) - await uart.register_uart_device(var, config) - # Initialize sensor storage with count from final_validate + # Initialize sensor storage with count from final_validate before any + # await, so platform to_code() calls always see it initialized + # regardless of YAML key order. sensor_count = _get_data().sensor_counts.get(str(config[CONF_ID]), 0) if sensor_count > 0: cg.add(var.init_sensors(sensor_count)) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) From 34e74536774b0b031f1f69bc166fb86ffc6c7746 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 26 Aug 2026 15:26:31 -0300 Subject: [PATCH 57/65] [climate] Don't restore a saved mode the device no longer supports (#18296) --- esphome/components/climate/climate.cpp | 9 ++- tests/components/climate/climate_test.cpp | 73 +++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/components/climate/climate_test.cpp diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 0f01443bd0..6ca9e394f7 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -551,7 +551,14 @@ ClimateCall ClimateDeviceRestoreState::to_call(Climate *climate) { void ClimateDeviceRestoreState::apply(Climate *climate) { auto traits = climate->get_traits(); - climate->mode = this->mode; + // A saved mode the device no longer offers cannot be selected again, so skip it and leave the + // entity on the mode it already has. The other saved fields are still restored. + if (traits.supports_mode(this->mode)) { + climate->mode = this->mode; + } else { + ESP_LOGW(TAG, "'%s' - Saved mode %s is no longer supported, keeping %s", climate->get_name().c_str(), + LOG_STR_ARG(climate_mode_to_string(this->mode)), LOG_STR_ARG(climate_mode_to_string(climate->mode))); + } if (traits.has_feature_flags(CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE | CLIMATE_REQUIRES_TWO_POINT_TARGET_TEMPERATURE)) { climate->target_temperature_low = this->target_temperature_low; diff --git a/tests/components/climate/climate_test.cpp b/tests/components/climate/climate_test.cpp new file mode 100644 index 0000000000..bda014b87a --- /dev/null +++ b/tests/components/climate/climate_test.cpp @@ -0,0 +1,73 @@ +#include +#include "esphome/components/climate/climate.h" + +namespace esphome::climate::testing { + +// Minimal concrete Climate that offers a fixed set of modes, so the restore path can be exercised +// without any hardware or platform component. +class TestClimate : public Climate { + public: + ClimateTraits traits() override { + auto traits = ClimateTraits(); + traits.set_supported_modes({CLIMATE_MODE_OFF, CLIMATE_MODE_COOL}); + traits.set_supported_fan_modes({CLIMATE_FAN_LOW, CLIMATE_FAN_HIGH}); + return traits; + } + + protected: + void control(const ClimateCall &call) override {} +}; + +TEST(ClimateRestoreStateTest, RestoresASupportedMode) { + TestClimate climate; + // Value-initialized: several members (mode, swing_mode, the temperature union) have no default + // member initializer, so leaving the {} off would read indeterminate values. + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_COOL; + + state.apply(&climate); + + EXPECT_EQ(climate.mode, CLIMATE_MODE_COOL); +} + +TEST(ClimateRestoreStateTest, DoesNotRestoreAnUnsupportedMode) { + TestClimate climate; + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_HEAT; + + state.apply(&climate); + + // The device never advertised HEAT, so the mode stays where it was. + EXPECT_EQ(climate.mode, CLIMATE_MODE_OFF); +} + +TEST(ClimateRestoreStateTest, LeavesTheCurrentModeAloneRatherThanForcingOff) { + TestClimate climate; + // apply() is public and nothing restricts it to setup(), so the entity is not necessarily off + // when an unsupported mode is dropped. It keeps what it had rather than being forced to OFF. + climate.mode = CLIMATE_MODE_COOL; + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_HEAT; + + state.apply(&climate); + + EXPECT_EQ(climate.mode, CLIMATE_MODE_COOL); +} + +TEST(ClimateRestoreStateTest, KeepsRestoringTheOtherFieldsWhenTheModeIsDropped) { + TestClimate climate; + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_HEAT; + state.target_temperature = 21.0f; + state.uses_custom_fan_mode = false; + state.fan_mode = CLIMATE_FAN_HIGH; + + state.apply(&climate); + + EXPECT_EQ(climate.mode, CLIMATE_MODE_OFF); + EXPECT_FLOAT_EQ(climate.target_temperature, 21.0f); + // Compared as an optional: this asserts both that the fan mode was restored and what it holds. + EXPECT_EQ(climate.fan_mode, CLIMATE_FAN_HIGH); +} + +} // namespace esphome::climate::testing From f17ef133f34be727df30f6cebf3e80f2af515ff3 Mon Sep 17 00:00:00 2001 From: Josef Zweck Date: Wed, 26 Aug 2026 20:27:16 +0200 Subject: [PATCH 58/65] [hoermann_hcp] Add buttons to hoermann_hcp (#18544) --- .../hoermann_hcp/button/__init__.py | 43 +++++++++++ .../hoermann_hcp/button/hoermann_hcp_button.h | 34 +++++++++ .../components/hoermann_hcp/hoermann_hcp.cpp | 5 ++ .../components/hoermann_hcp/hoermann_hcp.h | 6 +- .../button/hoermann_hcp_button_test.cpp | 72 +++++++++++++++++++ tests/components/hoermann_hcp/common.yaml | 7 ++ 6 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 esphome/components/hoermann_hcp/button/__init__.py create mode 100644 esphome/components/hoermann_hcp/button/hoermann_hcp_button.h create mode 100644 tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp diff --git a/esphome/components/hoermann_hcp/button/__init__.py b/esphome/components/hoermann_hcp/button/__init__.py new file mode 100644 index 0000000000..dc2efcec44 --- /dev/null +++ b/esphome/components/hoermann_hcp/button/__init__.py @@ -0,0 +1,43 @@ +import esphome.codegen as cg +from esphome.components import button +import esphome.config_validation as cv +from esphome.const import ICON_AIR_FILTER +from esphome.types import ConfigType + +from .. import CONF_HOERMANN_HCP_ID, HoermannHcp, hoermann_hcp_ns + +DEPENDENCIES = ["hoermann_hcp"] + +CONF_HALF_OPEN = "half_open" +CONF_VENT = "vent" + +ICON_GARAGE_OPEN_VARIANT = "mdi:garage-open-variant" + +HoermannHcpVentButton = hoermann_hcp_ns.class_("HoermannHcpVentButton", button.Button) +HoermannHcpHalfOpenButton = hoermann_hcp_ns.class_( + "HoermannHcpHalfOpenButton", button.Button +) + +BUTTON_KEYS = (CONF_VENT, CONF_HALF_OPEN) + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(CONF_HOERMANN_HCP_ID): cv.use_id(HoermannHcp), + cv.Optional(CONF_VENT): button.button_schema( + HoermannHcpVentButton, icon=ICON_AIR_FILTER + ), + cv.Optional(CONF_HALF_OPEN): button.button_schema( + HoermannHcpHalfOpenButton, icon=ICON_GARAGE_OPEN_VARIANT + ), + } + ), + cv.has_at_least_one_key(*BUTTON_KEYS), +) + + +async def to_code(config: ConfigType) -> None: + parent = await cg.get_variable(config[CONF_HOERMANN_HCP_ID]) + for key in BUTTON_KEYS: + if (conf := config.get(key)) is not None: + await button.new_button(conf, parent) diff --git a/esphome/components/hoermann_hcp/button/hoermann_hcp_button.h b/esphome/components/hoermann_hcp/button/hoermann_hcp_button.h new file mode 100644 index 0000000000..e9ebceee88 --- /dev/null +++ b/esphome/components/hoermann_hcp/button/hoermann_hcp_button.h @@ -0,0 +1,34 @@ +#pragma once + +#include "esphome/components/button/button.h" +#include "../hoermann_hcp.h" + +namespace esphome::hoermann_hcp { + +// The door commands the cover has no equivalent for. A refused command is already reported by the hub and +// leaves nothing to correct here, because a button carries no state of its own. +class HoermannHcpButton : public button::Button { + public: + explicit HoermannHcpButton(HoermannHcp *parent) : parent_(parent) {} + + protected: + HoermannHcp *const parent_; +}; + +class HoermannHcpVentButton final : public HoermannHcpButton { + public: + using HoermannHcpButton::HoermannHcpButton; + + protected: + void press_action() override { this->parent_->vent_door(); } +}; + +class HoermannHcpHalfOpenButton final : public HoermannHcpButton { + public: + using HoermannHcpButton::HoermannHcpButton; + + protected: + void press_action() override { this->parent_->half_open_door(); } +}; + +} // namespace esphome::hoermann_hcp diff --git a/esphome/components/hoermann_hcp/hoermann_hcp.cpp b/esphome/components/hoermann_hcp/hoermann_hcp.cpp index a780854831..17df927eb7 100644 --- a/esphome/components/hoermann_hcp/hoermann_hcp.cpp +++ b/esphome/components/hoermann_hcp/hoermann_hcp.cpp @@ -22,6 +22,9 @@ static constexpr uint8_t MAX_LIGHT_TOGGLES_IN_FLIGHT = 4; static constexpr HoermannHcpCommand COMMAND_OPEN{"open", 0x0210, 0x0110}; static constexpr HoermannHcpCommand COMMAND_CLOSE{"close", 0x0220, 0x0120}; static constexpr HoermannHcpCommand COMMAND_IMPULSE{"impulse", 0x0240, 0x0140}; +// The intermediate positions are named in the second register, so the first only carries the phase. +static constexpr HoermannHcpCommand COMMAND_VENT{"vent", 0x0200, 0x0100, 0x4000, 0x4000}; +static constexpr HoermannHcpCommand COMMAND_HALF_OPEN{"half open", 0x0200, 0x0100, 0x0400, 0x0400}; // The lamp is named in the second register, but its phase bytes follow no scheme the door commands share. static constexpr HoermannHcpCommand COMMAND_TOGGLE_LAMP{"toggle light", 0x0100, 0x0800, 0x0200, 0x0200, false}; @@ -286,6 +289,8 @@ bool HoermannHcp::queue_command_(const HoermannHcpCommand &command) { bool HoermannHcp::open_door() { return this->queue_command_(COMMAND_OPEN); } bool HoermannHcp::close_door() { return this->queue_command_(COMMAND_CLOSE); } bool HoermannHcp::impulse_door() { return this->queue_command_(COMMAND_IMPULSE); } +bool HoermannHcp::vent_door() { return this->queue_command_(COMMAND_VENT); } +bool HoermannHcp::half_open_door() { return this->queue_command_(COMMAND_HALF_OPEN); } bool HoermannHcp::toggle_light() { if (this->light_toggles_in_flight_ >= MAX_LIGHT_TOGGLES_IN_FLIGHT) { ESP_LOGW(TAG, "Too many lamp toggles are still waiting to be confirmed, dropping this one"); diff --git a/esphome/components/hoermann_hcp/hoermann_hcp.h b/esphome/components/hoermann_hcp/hoermann_hcp.h index 41fd7617e4..83be385c7b 100644 --- a/esphome/components/hoermann_hcp/hoermann_hcp.h +++ b/esphome/components/hoermann_hcp/hoermann_hcp.h @@ -22,7 +22,8 @@ enum class DoorState : uint8_t { }; // A HCP command is a simulated key press: the pressed value is presented to the bus controller, then after a -// short delay the released value. Each half also carries a second register, which only the lamp command uses. +// short delay the released value. Each half also carries a second register, which names the buttons that do +// not fit into the first. struct HoermannHcpCommand { const char *name; uint16_t pressed_value; @@ -54,6 +55,9 @@ class HoermannHcp : public PollingComponent, public modbus::ModbusServerDevice { bool open_door(); bool close_door(); bool impulse_door(); + // The door drives to these intermediate positions on its own, so neither takes a target to be stopped at. + bool vent_door(); + bool half_open_door(); bool stop_door(); bool set_position(float position); bool toggle_light(); diff --git a/tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp b/tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp new file mode 100644 index 0000000000..9c38bc1708 --- /dev/null +++ b/tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp @@ -0,0 +1,72 @@ +#include + +#include "esphome/components/hoermann_hcp/button/hoermann_hcp_button.h" + +#include "../common.h" + +namespace esphome::hoermann_hcp::testing { + +// The intermediate positions are named in the second register, which repeats that name on release. +TEST(HoermannHcpButtonTest, VentButtonSendsTheVentCommand) { + TestableHoermannHcp door; + HoermannHcpVentButton vent(&door); + connect_controller(door); + + vent.press(); + + auto [pressed, pressed_2] = poll_command(door); + EXPECT_EQ(pressed, 0x0200); + EXPECT_EQ(pressed_2, 0x4000); + std::this_thread::sleep_for(KEY_PRESS_ELAPSED); + auto [released, released_2] = poll_command(door); + EXPECT_EQ(released, 0x0100); + EXPECT_EQ(released_2, 0x4000); +} + +TEST(HoermannHcpButtonTest, HalfOpenButtonSendsTheHalfOpenCommand) { + TestableHoermannHcp door; + HoermannHcpHalfOpenButton half_open(&door); + connect_controller(door); + + half_open.press(); + + auto [pressed, pressed_2] = poll_command(door); + EXPECT_EQ(pressed, 0x0200); + EXPECT_EQ(pressed_2, 0x0400); + std::this_thread::sleep_for(KEY_PRESS_ELAPSED); + auto [released, released_2] = poll_command(door); + EXPECT_EQ(released, 0x0100); + EXPECT_EQ(released_2, 0x0400); +} + +// The door drives to the vent position on its own, so a position the cover was still travelling to must not +// stop it on the way there. +TEST(HoermannHcpButtonTest, VentAbandonsAnArmedTarget) { + TestableHoermannHcp door; // starts out fully closed + HoermannHcpVentButton vent(&door); + connect_controller(door); + door.set_position(0.5f); + consume_command(door); + door.on_write_registers(BROADCAST_REG, make_registers({0x0000, 0x0014, 0x0100})); + ASSERT_EQ(door.get_door_state(), DoorState::OPENING); + + vent.press(); + consume_command(door); + + // Position 120/200 = 0.6 is past the abandoned target, which must no longer stop the door. + door.on_write_registers(BROADCAST_REG, make_registers({0x0000, 0x0078, 0x0100})); + EXPECT_EQ(poll_command(door).first, 0x0000); +} + +// A button carries no state, so a refused press is simply dropped rather than fired once the controller +// turns up, which could be much later. +TEST(HoermannHcpButtonTest, PressWithoutABusControllerSendsNothing) { + HoermannHcp door; // never contacted by a bus controller + HoermannHcpVentButton vent(&door); + + vent.press(); + + EXPECT_EQ(poll_command(door).first, 0x0000); +} + +} // namespace esphome::hoermann_hcp::testing diff --git a/tests/components/hoermann_hcp/common.yaml b/tests/components/hoermann_hcp/common.yaml index 552b1cb0fd..618a8181bf 100644 --- a/tests/components/hoermann_hcp/common.yaml +++ b/tests/components/hoermann_hcp/common.yaml @@ -12,6 +12,13 @@ binary_sensor: is_connected: name: Garage Connected +button: + - platform: hoermann_hcp + vent: + name: Garage Vent + half_open: + name: Garage Half Open + light: - platform: hoermann_hcp name: Garage Light From d28b17c619937f13353fb1857f52239a89eb8735 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:26:45 +0000 Subject: [PATCH 59/65] Bump filelock from 3.32.3 to 3.32.4 (#18799) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d4439bf10..1a98c2a8e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.3 # native esp-idf toolchain global cache dir ninja==1.13.0 # native esp8266 arduino toolchain build driver -filelock==3.32.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.2 From fedd46e648a3099915476e4d47c357e946e3eafd Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:55:29 +0000 Subject: [PATCH 60/65] Bump aioesphomeapi from 46.2.0 to 46.2.1 (#18804) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1a98c2a8e4..de00f07836 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==46.2.0 +aioesphomeapi==46.2.1 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 5878406918eda313684cca4c9f2e0fa737358a47 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:33:52 -0500 Subject: [PATCH 61/65] Bump bundled esphome-device-builder to 1.13.1 (#18807) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d46f01838e..0da8048c57 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 RUN \ platformio settings set enable_telemetry No \ From 950816579764d38865153c1738f473800df64882 Mon Sep 17 00:00:00 2001 From: guillempages Date: Wed, 26 Aug 2026 23:36:47 +0200 Subject: [PATCH 62/65] [runtime_image] Add check for dimensions in BMP (#18800) --- esphome/components/runtime_image/bmp_decoder.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esphome/components/runtime_image/bmp_decoder.cpp b/esphome/components/runtime_image/bmp_decoder.cpp index 5d45621fb7..204d6cc14b 100644 --- a/esphome/components/runtime_image/bmp_decoder.cpp +++ b/esphome/components/runtime_image/bmp_decoder.cpp @@ -80,6 +80,10 @@ int HOT BmpDecoder::decode(uint8_t *buffer, size_t size) { this->width_ = encode_uint32(buffer[21], buffer[20], buffer[19], buffer[18]); this->height_ = encode_uint32(buffer[25], buffer[24], buffer[23], buffer[22]); + if (this->width_ <= 0 || this->height_ <= 0) { + ESP_LOGE(TAG, "Invalid image dimensions: (%zdx%zd)", this->width_, this->height_); + return DECODE_ERROR_UNSUPPORTED_FORMAT; + } this->bits_per_pixel_ = encode_uint16(buffer[29], buffer[28]); this->compression_method_ = encode_uint32(buffer[33], buffer[32], buffer[31], buffer[30]); this->image_data_size_ = encode_uint32(buffer[37], buffer[36], buffer[35], buffer[34]); From 5df1c7f1d3e2df2c5d4355c1cde8f9882c6b8b25 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Wed, 26 Aug 2026 16:24:09 -0700 Subject: [PATCH 63/65] [modbus_controller] Writer entities as their own hub device; heap-free, byte-accurate write path (#18082) Co-authored-by: J. Nick Koston --- esphome/components/modbus/helpers.py | 3 + .../components/modbus_controller/__init__.py | 30 +++-- .../modbus_controller/modbus_controller.cpp | 67 ++++++++++ .../modbus_controller/modbus_controller.h | 118 +++++++++++++++++- .../modbus_controller/number/__init__.py | 10 +- .../number/modbus_number.cpp | 98 ++++++++------- .../modbus_controller/number/modbus_number.h | 7 +- .../modbus_controller/output/__init__.py | 13 +- .../output/modbus_output.cpp | 115 +++++++++-------- .../modbus_controller/output/modbus_output.h | 24 ++-- .../modbus_controller/select/__init__.py | 20 ++- .../select/modbus_select.cpp | 55 ++++---- .../modbus_controller/select/modbus_select.h | 7 +- .../modbus_controller/switch/__init__.py | 7 +- .../switch/modbus_switch.cpp | 93 +++++++------- .../modbus_controller/switch/modbus_switch.h | 7 +- .../command_payload_test.cpp | 4 +- .../uart_mock_modbus_lambda_write.yaml | 97 ++++++++++++++ tests/integration/test_uart_mock_modbus.py | 58 ++++++--- 19 files changed, 598 insertions(+), 235 deletions(-) create mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml diff --git a/esphome/components/modbus/helpers.py b/esphome/components/modbus/helpers.py index e7eaacee0c..ec95b82045 100644 --- a/esphome/components/modbus/helpers.py +++ b/esphome/components/modbus/helpers.py @@ -3,6 +3,9 @@ import esphome.codegen as cg modbus_ns = cg.esphome_ns.namespace("modbus") modbus_helpers_ns = modbus_ns.namespace("helpers") +RegisterValues = modbus_ns.class_("RegisterValues") +PduBuffer = modbus_helpers_ns.class_("PduBuffer") + FunctionCode_ns = modbus_ns.namespace("FunctionCode") FunctionCode = FunctionCode_ns.enum("FunctionCode") diff --git a/esphome/components/modbus_controller/__init__.py b/esphome/components/modbus_controller/__init__.py index 924a260d37..e87eccb32c 100644 --- a/esphome/components/modbus_controller/__init__.py +++ b/esphome/components/modbus_controller/__init__.py @@ -191,7 +191,7 @@ ModbusItemBaseSchema = cv.Schema( ) -def validate_modbus_register(config): +def validate_modbus_register(config: ConfigType) -> ConfigType: # custom_command is the deprecated alias for custom_pdu (migrated later in final validate); treat # either as "a custom frame is configured" so the address/register_type rules match. has_custom = CONF_CUSTOM_PDU in config or CONF_CUSTOM_COMMAND in config @@ -278,7 +278,7 @@ def _final_validate(config: ConfigType) -> None: FINAL_VALIDATE_SCHEMA = _final_validate -def modbus_calc_properties(config): +def modbus_calc_properties(config: ConfigType) -> tuple[int, int]: byte_offset = 0 reg_count = 0 if CONF_OFFSET in config: @@ -307,8 +307,12 @@ def modbus_calc_properties(config): async def add_modbus_base_properties( - var, config, sensor_type, lambda_param_type=cg.float_, lambda_return_type=float -): + var: cg.MockObj, + config: ConfigType, + sensor_type: cg.MockObjClass, + lambda_param_type: cg.MockObj = cg.float_, + lambda_return_type: Any = float, +) -> None: if CONF_CUSTOM_PDU in config: cg.add(var.set_custom_pdu(config[CONF_CUSTOM_PDU])) @@ -347,8 +351,11 @@ _CALLBACK_AUTOMATIONS = ( ) -async def to_code(config): - var = cg.new_Pvariable(config[CONF_ID]) +async def to_code(config: ConfigType) -> None: + # Await the hub first, so no entity can bind to a controller that doesn't have one yet. + hub = await cg.get_variable(config[modbus.CONF_MODBUS_ID]) + var = cg.new_Pvariable(config[CONF_ID], hub, config[CONF_ADDRESS]) + await cg.register_component(var, config) cg.add(var.set_max_cmd_retries(config[CONF_MAX_CMD_RETRIES])) cg.add(var.set_offline_skip_updates(config[CONF_OFFLINE_SKIP_UPDATES])) cg.add( @@ -356,17 +363,22 @@ async def to_code(config): modbus.command_options_expression(config, direction="read") ) ) - await register_modbus_device(var, config) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) -async def register_modbus_device(var, config): +async def register_modbus_device(var: cg.MockObj, config: ConfigType) -> cg.MockObj: + # Remove before 2027.3.0 + _LOGGER.warning( + "'modbus_controller.register_modbus_device' is deprecated, use " + "'modbus.register_modbus_client_device' and set the address on your own " + "class instead. Will be removed in 2027.3.0" + ) cg.add(var.set_address(config[CONF_ADDRESS])) await cg.register_component(var, config) return await modbus.register_modbus_client_device(var, config) -def function_code_to_register(function_code): +def function_code_to_register(function_code: str) -> cg.MockObj: FUNCTION_CODE_TYPE_MAP = { "read_coils": EntityType.COIL, "read_discrete_inputs": EntityType.DISCRETE_INPUT, diff --git a/esphome/components/modbus_controller/modbus_controller.cpp b/esphome/components/modbus_controller/modbus_controller.cpp index 20b8f516e9..9d7b719e15 100644 --- a/esphome/components/modbus_controller/modbus_controller.cpp +++ b/esphome/components/modbus_controller/modbus_controller.cpp @@ -10,6 +10,73 @@ static const char *const TAG = "modbus_controller"; void ModbusController::setup() { this->create_polling_commands_(); } +void WriterDevice::warn_write_buffer_deprecated(const LogString *platform, uint16_t address) { + if (this->write_buffer_deprecated_warned_) + return; + this->write_buffer_deprecated_warned_ = true; + ESP_LOGW(TAG, + "Modbus %s (address 0x%X): filling the write_lambda buffer parameter is deprecated; call a write helper / " + "queue_pdu() on the entity (item) instead. The buffer parameter is removed in 2027.3.0", + LOG_STR_ARG(platform), address); +} + +bool WriterDevice::send_raw_frame_deprecated(std::span frame) { + if (frame.empty()) + return false; + this->dispatched_ = true; + return this->parent_->queue_pdu(frame[0], frame.subspan(1), this); +} + +void WriterDevice::set_controller(ModbusController *controller) { + this->controller_ = controller; + this->set_parent(controller->hub()); + this->set_address(controller->device_address()); +} + +void WriterDevice::notify_online_(std::span request_pdu) { + if (this->controller_ != nullptr) + this->controller_->set_online(true, fc_of(request_pdu), addr_of(request_pdu)); +} + +void WriterDevice::on_response(std::span request_pdu, std::span response_pdu) { + this->notify_online_(request_pdu); + this->dispatch_response_(request_pdu, response_pdu, std::nullopt); +} + +void WriterDevice::on_error(std::span request_pdu, modbus::ExceptionCode exception_code) { + ESP_LOGW(TAG, "Modbus error function code: 0x%X register 0x%X exception: %d", fc_of(request_pdu), + addr_of(request_pdu), static_cast(exception_code)); + this->notify_online_(request_pdu); // an exception is still a legitimate reply -> device is online + this->dispatch_response_(request_pdu, {}, exception_code); +} + +// Fired once per wire transmission (including hub re-queues from a retry), so the on_command_sent trigger +// reflects when the frame actually went out, not when it was queued. +void WriterDevice::on_sent(std::span request_pdu) { + if (this->controller_ != nullptr) + this->controller_->command_sent(fc_of(request_pdu), addr_of(request_pdu)); +} + +void WriterDevice::on_not_sent(std::span request_pdu) { + // Only the offline teardown reaches this (a supersede retires silently), so the frame is genuinely + // lost; a dropped write was already published optimistically, so surface it. + if (modbus::helpers::is_function_code_write(fc_of(request_pdu))) { + ESP_LOGW(TAG, "Write not sent: function 0x%X register 0x%X", fc_of(request_pdu), addr_of(request_pdu)); + } else { + ESP_LOGD(TAG, "Request not sent: function 0x%X register 0x%X", fc_of(request_pdu), addr_of(request_pdu)); + } +} + +bool WriterDevice::on_no_response(std::span request_pdu) { + if (this->controller_ == nullptr) + return false; + this->controller_->increment_non_response_count(); + if (this->controller_->can_send()) + return true; // the hub re-queues the frame it is holding; on_sent fires again on the retry + this->controller_->set_online(false, fc_of(request_pdu), addr_of(request_pdu)); + return false; +} + ModbusCommandItem::ModbusCommandItem(ModbusController &controller, modbus::ModbusClientHub *parent, uint8_t address, RegisterRange &&range) : modbus::ModbusClientDevice(parent, address), diff --git a/esphome/components/modbus_controller/modbus_controller.h b/esphome/components/modbus_controller/modbus_controller.h index 1db07f1ee8..1f36d5a7c8 100644 --- a/esphome/components/modbus_controller/modbus_controller.h +++ b/esphome/components/modbus_controller/modbus_controller.h @@ -232,6 +232,115 @@ struct RegisterRange { SensorSet sensors; // all sensors of this range }; +/// A hub device owned by a writer entity (switch/number/select/output) through WriterEntity. +/// Centralises the feedback to the controller - online/offline tracking, retry counting and the +/// on_command_sent trigger - and records every dispatch, so a write lambda can tell "I sent it myself" +/// from "use the default write". The hub base is inherited protected, so the public members below are +/// the entity's whole request API and nothing can bypass the recording or re-target the device. +class WriterDevice final : protected modbus::ModbusClientDevice { + protected: + void on_response(std::span request_pdu, std::span response_pdu) override; + void on_error(std::span request_pdu, modbus::ExceptionCode exception_code) override; + void on_sent(std::span request_pdu) override; + void on_not_sent(std::span request_pdu) override; + bool on_no_response(std::span request_pdu) override; + + void notify_online_(std::span request_pdu); + /// Function code / register address decoded from a request PDU ([fc, addr_hi, addr_lo, ...]). + static int fc_of(std::span pdu) { return pdu.empty() ? 0 : (pdu[0] & modbus::FUNCTION_CODE_MASK); } + static int addr_of(std::span pdu) { + return pdu.size() >= 3 ? modbus::helpers::get_data(pdu.data(), 1) : 0; + } + + /// Declared before controller_ so they land in the padding after ModbusClientDevice::custom_response_warned_ + /// instead of adding a word to every entity that owns a device. + /// dispatched_: a frame was queued since the last clear_dispatched_(). + /// write_buffer_deprecated_warned_: warn-once for the legacy write_lambda buffer parameter. + bool dispatched_{false}; + bool write_buffer_deprecated_warned_{false}; + ModbusController *controller_{nullptr}; + + public: + /// Whether a frame was queued to the hub since the last clear_dispatched_(). + bool dispatched() const { return this->dispatched_; } + + bool write_single_register(uint16_t address, uint16_t value) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_single_register(address, value); + } + bool write_single_coil(uint16_t address, bool value) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_single_coil(address, value); + } + bool write_multiple_registers(uint16_t address, std::span values) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_multiple_registers(address, values); + } + bool write_multiple_coils(uint16_t address, std::span values) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_multiple_coils(address, values); + } + bool write_multiple_coils(uint16_t address, modbus::PackedBits bits) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_multiple_coils(address, bits); + } + bool queue_pdu(std::span pdu, modbus::CommandOptions options = {}) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::queue_pdu(pdu, options); + } + /// Send a legacy raw frame (address + function code + data) to the frame's own address. + /// Serves only the deprecated write_lambda buffer path. Remove before 2027.3.0. + bool send_raw_frame_deprecated(std::span frame); + + void clear_tx_queue_for_device() { modbus::ModbusClientDevice::clear_tx_queue_for_device(); } + + // Entity plumbing, public because the owning WriterEntity holds the only reachable instance (device_ is + // protected there and the hub sees just the masked base) - reachability is the access gate, not a friend. + void set_controller(ModbusController *controller); + void clear_dispatched() { this->dispatched_ = false; } + /// Warn once per entity that filling the write_lambda buffer parameter is deprecated (the entity is now the + /// command - call a write helper / queue_pdu() on `item` instead). The buffer parameter is removed in 2027.3.0. + void warn_write_buffer_deprecated(const LogString *platform, uint16_t address); +}; + +/// Gives a writer entity the write API of the WriterDevice it owns. The device is a member, not a base: +/// the mixin declares no virtual function, so an entity mixing it in gains no second vtable and all the +/// writer platforms share the single WriterDevice vtable instead of each emitting its own copy. +/// The forwarders keep `item->write_*()` working unchanged inside a write_lambda. +class WriterEntity { + public: + bool dispatched() const { return this->device_.dispatched(); } + bool write_single_register(uint16_t address, uint16_t value) { + return this->device_.write_single_register(address, value); + } + bool write_single_coil(uint16_t address, bool value) { return this->device_.write_single_coil(address, value); } + bool write_multiple_registers(uint16_t address, std::span values) { + return this->device_.write_multiple_registers(address, values); + } + bool write_multiple_coils(uint16_t address, std::span values) { + return this->device_.write_multiple_coils(address, values); + } + bool write_multiple_coils(uint16_t address, modbus::PackedBits bits) { + return this->device_.write_multiple_coils(address, bits); + } + bool queue_pdu(std::span pdu, modbus::CommandOptions options = {}) { + return this->device_.queue_pdu(pdu, options); + } + void clear_tx_queue_for_device() { this->device_.clear_tx_queue_for_device(); } + + protected: + bool send_raw_frame_deprecated_(std::span frame) { + return this->device_.send_raw_frame_deprecated(frame); + } + void set_controller_(ModbusController *controller) { this->device_.set_controller(controller); } + void clear_dispatched_() { this->device_.clear_dispatched(); } + void warn_write_buffer_deprecated_(const LogString *platform, uint16_t address) { + this->device_.warn_write_buffer_deprecated(platform, address); + } + + WriterDevice device_; +}; + /// A single modbus command. Each command is its own ModbusClientDevice: it sends its frame to the hub /// and the hub routes the response back to this object's on_modbus_* callbacks, so the controller no /// longer has to match responses to a FIFO queue. @@ -398,17 +507,16 @@ inline bool offline_retry_due(uint16_t update_counter, uint16_t module_offline_a class ModbusController final : public PollingComponent { public: + // The controller is not itself a modbus device - its commands and writer entities send as their own + // devices, built against this hub + address. + ModbusController(modbus::ModbusClientHub *hub, uint8_t address) : hub_(hub), address_(address) {} + void dump_config() override; // No loop() override: the hub owns transmit/receive timing and each command routes its own // response, so the controller never joins the looping components at all. void setup() override; void update() override; - // The controller is not itself a modbus device - its commands and writer entities send as their own - // devices. It only owns the hub + address so those senders can be built against them. - void set_parent(modbus::ModbusClientHub *hub) { this->hub_ = hub; } - void set_address(uint8_t address) { this->address_ = address; } - /// The hub and modbus address this controller talks to. Used to build commands/entities that send as /// their own device. modbus::ModbusClientHub *hub() const { return this->hub_; } diff --git a/esphome/components/modbus_controller/number/__init__.py b/esphome/components/modbus_controller/number/__init__.py index a43e10a51e..6a5b7041b8 100644 --- a/esphome/components/modbus_controller/number/__init__.py +++ b/esphome/components/modbus_controller/number/__init__.py @@ -3,6 +3,7 @@ from esphome.components import number from esphome.components.modbus.helpers import ( MODBUS_WRITE_REGISTER_TYPE, SENSOR_VALUE_TYPE, + RegisterValues, ) import esphome.config_validation as cv from esphome.const import ( @@ -13,6 +14,7 @@ from esphome.const import ( CONF_MULTIPLY, CONF_STEP, ) +from esphome.types import ConfigType from .. import ( ModbusItemBaseSchema, @@ -43,7 +45,7 @@ ModbusNumber = modbus_controller_ns.class_( ) -def validate_min_max(config): +def validate_min_max(config: ConfigType) -> ConfigType: if config[CONF_MAX_VALUE] <= config[CONF_MIN_VALUE]: raise cv.Invalid("max_value must be greater than min_value") if config[CONF_MIN_VALUE] < -16777215: @@ -53,7 +55,7 @@ def validate_min_max(config): return config -def validate_modbus_number(config): +def validate_modbus_number(config: ConfigType) -> ConfigType: # custom_command is the deprecated alias for custom_pdu (migrated later in final validate). has_custom = CONF_CUSTOM_PDU in config or CONF_CUSTOM_COMMAND in config if not has_custom and CONF_ADDRESS not in config: @@ -89,7 +91,7 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item -async def to_code(config): +async def to_code(config: ConfigType) -> None: byte_offset, reg_count = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], @@ -124,7 +126,7 @@ async def to_code(config): [ (ModbusNumber.operator("ptr"), "item"), (cg.float_, "x"), - (cg.std_vector.template(cg.uint16).operator("ref"), "payload"), + (RegisterValues.operator("ref"), "payload"), ], return_type=cg.optional.template(float), ) diff --git a/esphome/components/modbus_controller/number/modbus_number.cpp b/esphome/components/modbus_controller/number/modbus_number.cpp index 7903b2e317..e890a2a9ac 100644 --- a/esphome/components/modbus_controller/number/modbus_number.cpp +++ b/esphome/components/modbus_controller/number/modbus_number.cpp @@ -1,4 +1,3 @@ -#include #include "modbus_number.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -29,62 +28,73 @@ void ModbusNumber::parse_and_publish(std::span data) { } void ModbusNumber::control(float value) { - optional write_cmd; - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::RegisterValues data; float write_value = value; - // Is there are lambda configured? if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*(), override the value (return a value), or + // (deprecated) fill `data` with the register words to write. auto val = (*this->write_transform_func_)(this, value, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - write_value = val.value(); - } else { + if (this->dispatched()) { + this->publish_state(value); + return; + } + if (!data.empty()) { + // Deprecated buffer path (frozen): the lambda filled a legacy raw frame as words; pack it big-endian. + this->warn_write_buffer_deprecated_(LOG_STR("number"), this->start_address); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE + char hex_buf[format_hex_pretty_uint16_size(MODBUS_NUMBER_MAX_LOG_REGISTERS)]; +#endif + ESP_LOGV(TAG, "Modbus Number write raw: %s", + format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); + // Sized to hold RegisterValues at capacity, so a full buffer can never truncate into a valid frame. + StaticVector bytes; + for (uint16_t word : data) { + const auto word_bytes = decode_value(word); + bytes.push_back(word_bytes[0]); + bytes.push_back(word_bytes[1]); + } + if (!this->send_raw_frame_deprecated_(std::span(bytes.data(), bytes.size()))) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } + this->publish_state(value); + return; + } + if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; } + ESP_LOGV(TAG, "Value overwritten by lambda"); + write_value = val.value(); } else { write_value = this->multiply_by_ * write_value; } - if (!data.empty()) { -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_uint16_size(MODBUS_NUMBER_MAX_LOG_REGISTERS)]; -#endif - ESP_LOGV(TAG, "Modbus Number write raw: %s", - format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - write_cmd.emplace(ModbusCommandItem::create_custom_command( - this->parent_, data, - [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { - this->parent_->on_write_register_response(register_type, this->start_address, data); - })); - } else { - std::vector payload; - modbus::helpers::float_to_payload(payload, write_value, this->sensor_value_type); + modbus::helpers::float_to_payload(data, write_value, this->sensor_value_type); + // float_to_payload() appends nothing for RAW, so an empty payload must be caught before data[0] below. + if (data.empty()) { + ESP_LOGW(TAG, "No payload was created for updating number"); + return; + } - ESP_LOGD(TAG, - "Updating register: connected Sensor=%s start address=0x%X register count=%d new value=%.02f (val=%.02f)", - this->get_name().c_str(), this->start_address, this->register_count, value, write_value); + ESP_LOGD(TAG, + "Updating register: connected Sensor=%s start address=0x%X register count=%d new value=%.02f (val=%.02f)", + this->get_name().c_str(), this->start_address, this->register_count, value, write_value); - // Create and send the write command - if (this->register_count == 1 && !this->use_write_multiple_) { - write_cmd.emplace( - ModbusCommandItem::create_write_single_command(this->parent_, this->write_address(), payload[0])); - } else { - write_cmd.emplace(ModbusCommandItem::create_write_multiple_command(this->parent_, this->write_address(), - this->register_count, payload)); - } - // publish new value - write_cmd->on_data_func = [this, value](modbus::EntityType register_type, uint16_t start_address, - std::span data) { - // gets called when the write command is ack'd from the device - this->parent_->on_write_register_response(register_type, start_address, data); - this->publish_state(value); - }; + bool queued; + if (this->register_count == 1 && !this->use_write_multiple_) { + queued = this->write_single_register(this->write_address(), data[0]); + } else { + queued = this->write_multiple_registers(this->write_address(), data); + } + if (!queued) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; } - this->parent_->queue_command(std::move(*write_cmd)); this->publish_state(value); } void ModbusNumber::dump_config() { LOG_NUMBER(TAG, "Modbus Number", this); } diff --git a/esphome/components/modbus_controller/number/modbus_number.h b/esphome/components/modbus_controller/number/modbus_number.h index 538a982f80..59c76e18f2 100644 --- a/esphome/components/modbus_controller/number/modbus_number.h +++ b/esphome/components/modbus_controller/number/modbus_number.h @@ -10,7 +10,7 @@ namespace esphome::modbus_controller { using value_to_data_t = std::function(float); -class ModbusNumber final : public number::Number, public Component, public SensorItem { +class ModbusNumber final : public number::Number, public Component, public SensorItem, public WriterEntity { public: ModbusNumber(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, SensorValueType value_type, int register_count, bool force_new_range) { @@ -26,11 +26,11 @@ class ModbusNumber final : public number::Number, public Component, public Senso void dump_config() override; void parse_and_publish(std::span data) override; float get_setup_priority() const override { return setup_priority::HARDWARE; } - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } void set_write_multiply(float factor) { this->multiply_by_ = factor; } using transform_func_t = optional (*)(ModbusNumber *, float, std::span); - using write_transform_func_t = optional (*)(ModbusNumber *, float, std::vector &); + using write_transform_func_t = optional (*)(ModbusNumber *, float, modbus::RegisterValues &); void set_template(transform_func_t f) { this->transform_func_ = f; } void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } @@ -39,7 +39,6 @@ class ModbusNumber final : public number::Number, public Component, public Senso void control(float value) override; optional transform_func_{nullopt}; optional write_transform_func_{nullopt}; - ModbusController *parent_{nullptr}; float multiply_by_{1.0}; bool use_write_multiple_{false}; }; diff --git a/esphome/components/modbus_controller/output/__init__.py b/esphome/components/modbus_controller/output/__init__.py index 178c99caa1..34a0f488ec 100644 --- a/esphome/components/modbus_controller/output/__init__.py +++ b/esphome/components/modbus_controller/output/__init__.py @@ -1,8 +1,13 @@ import esphome.codegen as cg from esphome.components import output -from esphome.components.modbus.helpers import SENSOR_VALUE_TYPE +from esphome.components.modbus.helpers import ( + SENSOR_VALUE_TYPE, + PduBuffer, + RegisterValues, +) import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID, CONF_MULTIPLY +from esphome.types import ConfigType from .. import ( ModbusItemBaseSchema, @@ -73,7 +78,7 @@ CONFIG_SCHEMA = cv.typed_schema( ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: byte_offset, reg_count = modbus_calc_properties(config) # Binary Output write_template = None @@ -89,7 +94,7 @@ async def to_code(config): [ (ModbusBinaryOutput.operator("ptr"), "item"), (cg.bool_, "x"), - (cg.std_vector.template(cg.uint8).operator("ref"), "payload"), + (PduBuffer.operator("ref"), "payload"), ], return_type=cg.optional.template(bool), ) @@ -109,7 +114,7 @@ async def to_code(config): [ (ModbusFloatOutput.operator("ptr"), "item"), (cg.float_, "x"), - (cg.std_vector.template(cg.uint16).operator("ref"), "payload"), + (RegisterValues.operator("ref"), "payload"), ], return_type=cg.optional.template(float), ) diff --git a/esphome/components/modbus_controller/output/modbus_output.cpp b/esphome/components/modbus_controller/output/modbus_output.cpp index 48249f4387..b05d3889fd 100644 --- a/esphome/components/modbus_controller/output/modbus_output.cpp +++ b/esphome/components/modbus_controller/output/modbus_output.cpp @@ -2,6 +2,8 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include + namespace esphome::modbus_controller { static const char *const TAG = "modbus_controller.output"; @@ -13,25 +15,33 @@ static constexpr size_t MODBUS_OUTPUT_MAX_LOG_BYTES = 64; * */ void ModbusFloatOutput::write_state(float value) { - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::RegisterValues data; auto original_value = value; - // Is there are lambda configured? if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*(), override the value (return a value), or + // (deprecated) fill `data` with the register words to write. auto val = (*this->write_transform_func_)(this, value, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - value = val.value(); - } else { + if (this->dispatched()) { + return; + } + if (!data.empty()) { + // Deprecated buffer path (frozen): the lambda supplied the register words for the shared write below. + this->warn_write_buffer_deprecated_(LOG_STR("float output"), this->start_address); + } else if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; + } else { + ESP_LOGV(TAG, "Value overwritten by lambda"); + value = val.value(); } } else { value = this->multiply_by_ * value; } - // lambda didn't set payload + if (data.empty()) { modbus::helpers::float_to_payload(data, value, this->sensor_value_type); } @@ -57,16 +67,15 @@ void ModbusFloatOutput::write_state(float value) { return; } - // Create and send the write command - optional write_cmd; + bool queued; if (this->register_count == 1 && !this->use_write_multiple_) { - write_cmd.emplace( - ModbusCommandItem::create_write_single_command(this->parent_, this->start_address + this->offset, data[0])); + queued = this->write_single_register(this->write_address(), data[0]); } else { - write_cmd.emplace(ModbusCommandItem::create_write_multiple_command( - this->parent_, this->start_address + this->offset, data.size(), data)); + queued = this->write_multiple_registers(this->write_address(), data); + } + if (!queued) { + ESP_LOGW(TAG, "Modbus output write (address 0x%X) was refused by the hub", this->write_address()); } - this->parent_->queue_command(std::move(*write_cmd)); } void ModbusFloatOutput::dump_config() { @@ -81,50 +90,52 @@ void ModbusFloatOutput::dump_config() { // ModbusBinaryOutput void ModbusBinaryOutput::write_state(bool state) { - // This will be called every time the user requests a state change. - optional cmd; - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::helpers::PduBuffer data; - // Is there are lambda configured? if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*/queue_pdu(), override the value (return a value), + // or (deprecated) fill `data` with a custom PDU. auto val = (*this->write_transform_func_)(this, state, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - state = val.value(); - } else { + if (this->dispatched()) { + return; + } + if (!data.empty()) { + this->warn_write_buffer_deprecated_(LOG_STR("binary output"), this->start_address); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE + char hex_buf[format_hex_pretty_size(MODBUS_OUTPUT_MAX_LOG_BYTES)]; +#endif + ESP_LOGV(TAG, "Modbus binary output write raw: %s", + format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); + // The lambda filled a legacy raw frame (device address + function code + data). + if (!this->send_raw_frame_deprecated_(data)) { + ESP_LOGW(TAG, "Modbus output write (address 0x%X) was refused by the hub", this->write_address()); + } + return; + } + if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; } + ESP_LOGV(TAG, "Value overwritten by lambda"); + state = val.value(); } - if (!data.empty()) { -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_size(MODBUS_OUTPUT_MAX_LOG_BYTES)]; -#endif - ESP_LOGV(TAG, "Modbus binary output write raw: %s", - format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - cmd.emplace(ModbusCommandItem::create_custom_command( - this->parent_, data, - [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { - this->parent_->on_write_register_response(register_type, this->start_address, data); - })); + ESP_LOGV(TAG, "Write new state: value is %s, type is %d address = %X, offset = %x", ONOFF(state), + (int) this->register_type, this->start_address, this->offset); + // offset for coil and discrete inputs is the coil/register number not bytes + bool queued; + if (this->use_write_multiple_) { + std::array states{state}; + queued = this->write_multiple_coils(this->write_address(), states); } else { - ESP_LOGV(TAG, "Write new state: value is %s, type is %d address = %X, offset = %x", ONOFF(state), - (int) this->register_type, this->start_address, this->offset); - - // offset for coil and discrete inputs is the coil/register number not bytes - if (this->use_write_multiple_) { - std::vector states{state}; - cmd.emplace( - ModbusCommandItem::create_write_multiple_coils(this->parent_, this->start_address + this->offset, states)); - } else { - cmd.emplace( - ModbusCommandItem::create_write_single_coil(this->parent_, this->start_address + this->offset, state)); - } + queued = this->write_single_coil(this->write_address(), state); + } + if (!queued) { + ESP_LOGW(TAG, "Modbus output write (address 0x%X) was refused by the hub", this->write_address()); } - this->parent_->queue_command(std::move(*cmd)); } void ModbusBinaryOutput::dump_config() { diff --git a/esphome/components/modbus_controller/output/modbus_output.h b/esphome/components/modbus_controller/output/modbus_output.h index e79c442aa4..b942dcea62 100644 --- a/esphome/components/modbus_controller/output/modbus_output.h +++ b/esphome/components/modbus_controller/output/modbus_output.h @@ -8,26 +8,24 @@ namespace esphome::modbus_controller { -class ModbusFloatOutput final : public output::FloatOutput, public Component, public SensorItem { +class ModbusFloatOutput final : public output::FloatOutput, public Component, public SensorItem, public WriterEntity { public: ModbusFloatOutput(uint16_t start_address, uint8_t offset, SensorValueType value_type, int register_count) { this->register_type = modbus::EntityType::HOLDING; - this->set_address(start_address); - this->set_offset_from_start_address(offset); + this->set_address(start_address + offset); + this->set_offset_from_start_address(0); this->bitmask = 0xFFFFFFFF; this->register_count = register_count; this->sensor_value_type = value_type; - this->set_address(this->start_address + offset); - this->set_offset_from_start_address(0); } void dump_config() override; - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } void set_write_multiply(float factor) { this->multiply_by_ = factor; } // Do nothing void parse_and_publish(std::span data) override{}; - using write_transform_func_t = optional (*)(ModbusFloatOutput *, float, std::vector &); + using write_transform_func_t = optional (*)(ModbusFloatOutput *, float, modbus::RegisterValues &); void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } @@ -35,29 +33,28 @@ class ModbusFloatOutput final : public output::FloatOutput, public Component, pu void write_state(float value) override; optional write_transform_func_{nullopt}; - ModbusController *parent_{nullptr}; float multiply_by_{1.0}; bool use_write_multiple_{false}; }; -class ModbusBinaryOutput final : public output::BinaryOutput, public Component, public SensorItem { +class ModbusBinaryOutput final : public output::BinaryOutput, public Component, public SensorItem, public WriterEntity { public: ModbusBinaryOutput(uint16_t start_address, uint8_t offset) { this->register_type = modbus::EntityType::COIL; - this->set_address(start_address); + // A coil offset is a coil count; fold it into the address. + this->set_address(start_address + offset); this->bitmask = 0xFFFFFFFF; this->sensor_value_type = SensorValueType::BIT; this->register_count = 1; - this->set_address(this->start_address + offset); this->set_offset_from_start_address(0); } void dump_config() override; - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } // Do nothing void parse_and_publish(std::span data) override{}; - using write_transform_func_t = optional (*)(ModbusBinaryOutput *, bool, std::vector &); + using write_transform_func_t = optional (*)(ModbusBinaryOutput *, bool, modbus::helpers::PduBuffer &); void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } @@ -65,7 +62,6 @@ class ModbusBinaryOutput final : public output::BinaryOutput, public Component, void write_state(bool state) override; optional write_transform_func_{nullopt}; - ModbusController *parent_{nullptr}; bool use_write_multiple_{false}; }; diff --git a/esphome/components/modbus_controller/select/__init__.py b/esphome/components/modbus_controller/select/__init__.py index 1d77f9235d..07893e3303 100644 --- a/esphome/components/modbus_controller/select/__init__.py +++ b/esphome/components/modbus_controller/select/__init__.py @@ -1,8 +1,16 @@ +from collections.abc import Callable +from typing import Any + import esphome.codegen as cg from esphome.components import select -from esphome.components.modbus.helpers import SENSOR_VALUE_TYPE, TYPE_REGISTER_MAP +from esphome.components.modbus.helpers import ( + SENSOR_VALUE_TYPE, + TYPE_REGISTER_MAP, + RegisterValues, +) import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID, CONF_LAMBDA, CONF_OPTIMISTIC +from esphome.types import ConfigType from .. import ( ModbusController, @@ -29,8 +37,8 @@ ModbusSelect = modbus_controller_ns.class_( ) -def ensure_option_map(): - def validator(value): +def ensure_option_map() -> Callable[[Any], dict[str, int]]: + def validator(value: Any) -> dict[str, int]: cv.check_not_templatable(value) option = cv.All(cv.string_strict) mapping = cv.All(cv.int_range(-(2**63), 2**63 - 1)) @@ -47,7 +55,7 @@ def ensure_option_map(): return validator -def register_count_value_type_min(value): +def register_count_value_type_min(value: ConfigType) -> ConfigType: reg_count = value.get(CONF_REGISTER_COUNT) if reg_count is not None: value_type = value[CONF_VALUE_TYPE] @@ -87,7 +95,7 @@ CONFIG_SCHEMA = cv.All( ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: value_type = config[CONF_VALUE_TYPE] reg_count = config.get(CONF_REGISTER_COUNT) if reg_count is None: @@ -132,7 +140,7 @@ async def to_code(config): (ModbusSelect.operator("const_ptr"), "item"), (cg.std_string.operator("const").operator("ref"), "x"), (cg.int64, "value"), - (cg.std_vector.template(cg.uint16).operator("ref"), "payload"), + (RegisterValues.operator("ref"), "payload"), ], return_type=cg.optional.template(cg.int64), ) diff --git a/esphome/components/modbus_controller/select/modbus_select.cpp b/esphome/components/modbus_controller/select/modbus_select.cpp index 0a9383b1b0..c1cc241d6b 100644 --- a/esphome/components/modbus_controller/select/modbus_select.cpp +++ b/esphome/components/modbus_controller/select/modbus_select.cpp @@ -46,35 +46,43 @@ void ModbusSelect::control(size_t index) { const char *option = this->option_at(index); ESP_LOGD(TAG, "Found value %lld for option '%s'", *mapval, option); - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::RegisterValues data; if (this->write_transform_func_.has_value()) { - // Transform func requires string parameter for backward compatibility + // The lambda may drive the write itself via item->write_*(), override the mapping value (return a value), + // or (deprecated) fill `data` with the register words to write. Transform func requires string parameter + // for backward compatibility. auto val = (*this->write_transform_func_)(this, std::string(option), *mapval, data); - if (val.has_value()) { - mapval = val; - ESP_LOGV(TAG, "write_lambda returned mapping value %lld", *mapval); - } else { + if (this->dispatched()) { + if (this->optimistic_) + this->publish_state(index); + return; + } + if (!data.empty()) { + // Deprecated buffer path (frozen): the lambda supplied the register words for the shared write below. + this->warn_write_buffer_deprecated_(LOG_STR("select"), this->start_address); + } else if (!val.has_value()) { ESP_LOGD(TAG, "Communication handled by write_lambda - exiting control"); return; + } else { + mapval = val; + ESP_LOGV(TAG, "write_lambda returned mapping value %lld", *mapval); } } if (data.empty()) { modbus::helpers::number_to_payload(data, *mapval, this->sensor_value_type); - } else { - ESP_LOGV(TAG, "Using payload from write lambda"); + // number_to_payload() appends nothing for RAW. + if (data.empty()) { + ESP_LOGW(TAG, "No payload was created for updating select"); + return; + } } - if (data.empty()) { - ESP_LOGW(TAG, "No payload was created for updating select"); - return; - } - - // The command declares register_count registers, so the payload must be exactly that many words: - // a value type narrower than the declared width is zero-padded (the config deliberately allows - // register_count larger than the value type). Anything else would put a byte count on the wire - // that disagrees with the quantity field, which conformant devices reject. // register_count declares the READ range width - it may pull neighboring registers into one poll - // so a write covers exactly the registers the value occupies: the quantity comes from the payload, // never from register_count (padding to it would zero registers the user only declared for reading). @@ -86,16 +94,17 @@ void ModbusSelect::control(size_t index) { } const uint16_t write_address = this->write_address(); - optional write_cmd; + bool queued; if ((this->register_count == 1) && (!this->use_write_multiple_)) { - write_cmd.emplace(ModbusCommandItem::create_write_single_command(this->parent_, write_address, data[0])); + queued = this->write_single_register(write_address, data[0]); } else { - write_cmd.emplace( - ModbusCommandItem::create_write_multiple_command(this->parent_, write_address, data.size(), data)); + queued = this->write_multiple_registers(write_address, data); } - this->parent_->queue_command(std::move(*write_cmd)); - + if (!queued) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } if (this->optimistic_) this->publish_state(index); } diff --git a/esphome/components/modbus_controller/select/modbus_select.h b/esphome/components/modbus_controller/select/modbus_select.h index 41ebd4f658..c6ac76a45b 100644 --- a/esphome/components/modbus_controller/select/modbus_select.h +++ b/esphome/components/modbus_controller/select/modbus_select.h @@ -9,7 +9,7 @@ namespace esphome::modbus_controller { -class ModbusSelect final : public Component, public select::Select, public SensorItem { +class ModbusSelect final : public Component, public select::Select, public SensorItem, public WriterEntity { public: ModbusSelect(SensorValueType sensor_value_type, uint16_t start_address, uint8_t register_count, bool force_new_range, std::vector mapping) { @@ -26,9 +26,9 @@ class ModbusSelect final : public Component, public select::Select, public Senso using transform_func_t = optional (*)(ModbusSelect *const, int64_t, std::span); using write_transform_func_t = optional (*)(ModbusSelect *const, const std::string &, int64_t, - std::vector &); + modbus::RegisterValues &); - void set_parent(ModbusController *const parent) { this->parent_ = parent; } + void set_parent(ModbusController *const parent) { this->set_controller_(parent); } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } void set_template(transform_func_t f) { this->transform_func_ = f; } @@ -40,7 +40,6 @@ class ModbusSelect final : public Component, public select::Select, public Senso protected: std::vector mapping_{}; - ModbusController *parent_{nullptr}; bool use_write_multiple_{false}; bool optimistic_{false}; optional transform_func_{nullopt}; diff --git a/esphome/components/modbus_controller/switch/__init__.py b/esphome/components/modbus_controller/switch/__init__.py index dedd2ceedf..c52067f941 100644 --- a/esphome/components/modbus_controller/switch/__init__.py +++ b/esphome/components/modbus_controller/switch/__init__.py @@ -1,8 +1,9 @@ import esphome.codegen as cg from esphome.components import switch -from esphome.components.modbus.helpers import MODBUS_REGISTER_TYPE +from esphome.components.modbus.helpers import MODBUS_REGISTER_TYPE, PduBuffer import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ASSUMED_STATE, CONF_ID +from esphome.types import ConfigType from .. import ( ModbusItemBaseSchema, @@ -48,7 +49,7 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item -async def to_code(config): +async def to_code(config: ConfigType) -> None: byte_offset, _ = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], @@ -74,7 +75,7 @@ async def to_code(config): [ (ModbusSwitch.operator("ptr"), "item"), (cg.bool_, "x"), - (cg.std_vector.template(cg.uint8).operator("ref"), "payload"), + (PduBuffer.operator("ref"), "payload"), ], return_type=cg.optional.template(bool), ) diff --git a/esphome/components/modbus_controller/switch/modbus_switch.cpp b/esphome/components/modbus_controller/switch/modbus_switch.cpp index 810d904d85..c942ff1e6f 100644 --- a/esphome/components/modbus_controller/switch/modbus_switch.cpp +++ b/esphome/components/modbus_controller/switch/modbus_switch.cpp @@ -3,6 +3,8 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include + namespace esphome::modbus_controller { static const char *const TAG = "modbus_controller.switch"; @@ -58,57 +60,64 @@ void ModbusSwitch::parse_and_publish(std::span data) { } void ModbusSwitch::write_state(bool state) { - // This will be called every time the user requests a state change. - optional cmd; - std::vector data; - // Is there are lambda configured? + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::helpers::PduBuffer data; if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*/queue_pdu(), override the written value (return a + // value), or (deprecated) fill `data` with a custom PDU. auto val = (*this->write_transform_func_)(this, state, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - state = val.value(); - } else { + if (this->dispatched()) { + this->publish_state(state); + return; + } + if (!data.empty()) { + this->warn_write_buffer_deprecated_(LOG_STR("switch"), this->start_address); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE + char hex_buf[format_hex_pretty_size(MODBUS_SWITCH_MAX_LOG_BYTES)]; +#endif + ESP_LOGV(TAG, "Modbus Switch write raw: %s", + format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); + // The lambda filled a legacy raw frame (device address + function code + data). + if (!this->send_raw_frame_deprecated_(data)) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } + this->publish_state(state); + return; + } + if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; } + ESP_LOGV(TAG, "Value overwritten by lambda"); + state = val.value(); } - if (!data.empty()) { -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_size(MODBUS_SWITCH_MAX_LOG_BYTES)]; -#endif - ESP_LOGV(TAG, "Modbus Switch write raw: %s", - format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - cmd.emplace(ModbusCommandItem::create_custom_command( - this->parent_, data, - [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { - this->parent_->on_write_register_response(register_type, this->start_address, data); - })); - } else { - ESP_LOGV(TAG, "write_state '%s': new value = %s type = %d address = %X offset = %x", this->get_name().c_str(), - ONOFF(state), (int) this->register_type, this->start_address, this->offset); - if (this->register_type == modbus::EntityType::COIL) { - // offset for coil and discrete inputs is the coil/register number not bytes - if (this->use_write_multiple_) { - std::vector states{state}; - cmd.emplace(ModbusCommandItem::create_write_multiple_coils(this->parent_, this->write_address(), states)); - } else { - cmd.emplace(ModbusCommandItem::create_write_single_coil(this->parent_, this->write_address(), state)); - } + ESP_LOGV(TAG, "write_state '%s': new value = %s type = %d address = %X offset = %x", this->get_name().c_str(), + ONOFF(state), (int) this->register_type, this->start_address, this->offset); + bool queued; + if (this->register_type == EntityType::COIL) { + // offset for coil and discrete inputs is the coil/register number not bytes + if (this->use_write_multiple_) { + std::array states{state}; + queued = this->write_multiple_coils(this->write_address(), states); } else { - if (this->use_write_multiple_) { - std::vector bool_states(1, state ? (0xFFFF & this->bitmask) : 0); - cmd.emplace( - ModbusCommandItem::create_write_multiple_command(this->parent_, this->write_address(), 1, bool_states)); - } else { - cmd.emplace(ModbusCommandItem::create_write_single_command(this->parent_, this->write_address(), - state ? 0xFFFF & this->bitmask : 0u)); - } + queued = this->write_single_coil(this->write_address(), state); + } + } else { + if (this->use_write_multiple_) { + std::array states{static_cast(state ? (0xFFFF & this->bitmask) : 0)}; + queued = this->write_multiple_registers(this->write_address(), states); + } else { + queued = this->write_single_register(this->write_address(), state ? 0xFFFF & this->bitmask : 0u); } } - this->parent_->queue_command(std::move(*cmd)); + if (!queued) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } this->publish_state(state); } // ModbusSwitch end diff --git a/esphome/components/modbus_controller/switch/modbus_switch.h b/esphome/components/modbus_controller/switch/modbus_switch.h index c21a1939bc..1d3d03919f 100644 --- a/esphome/components/modbus_controller/switch/modbus_switch.h +++ b/esphome/components/modbus_controller/switch/modbus_switch.h @@ -8,7 +8,7 @@ namespace esphome::modbus_controller { -class ModbusSwitch final : public Component, public switch_::Switch, public SensorItem { +class ModbusSwitch final : public Component, public switch_::Switch, public SensorItem, public WriterEntity { public: ModbusSwitch(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, bool force_new_range) { @@ -30,17 +30,16 @@ class ModbusSwitch final : public Component, public switch_::Switch, public Sens void set_assumed_state(bool assumed_state); void set_state(bool state) { this->state = state; } void parse_and_publish(std::span data) override; - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } using transform_func_t = optional (*)(ModbusSwitch *, bool, std::span); - using write_transform_func_t = optional (*)(ModbusSwitch *, bool, std::vector &); + using write_transform_func_t = optional (*)(ModbusSwitch *, bool, modbus::helpers::PduBuffer &); void set_template(transform_func_t f) { this->publish_transform_func_ = f; } void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } protected: bool assumed_state() override; - ModbusController *parent_{nullptr}; bool use_write_multiple_{false}; optional publish_transform_func_{nullopt}; optional write_transform_func_{nullopt}; diff --git a/tests/components/modbus_controller/command_payload_test.cpp b/tests/components/modbus_controller/command_payload_test.cpp index c125a44da5..a0a59f5106 100644 --- a/tests/components/modbus_controller/command_payload_test.cpp +++ b/tests/components/modbus_controller/command_payload_test.cpp @@ -13,7 +13,7 @@ namespace esphome::modbus_controller::testing { // malformed. Built at its true byte count, the oversize frame is refused by the hub's size check with // a log instead. TEST(ModbusCommandPayload, CoilWritePayloadIsExactSizedNotTruncated) { - ModbusController controller; + ModbusController controller(nullptr, 1); std::vector coils(modbus::MAX_NUM_OF_COILS_TO_WRITE + 1, true); auto cmd = ModbusCommandItem::create_write_multiple_coils(&controller, 0x10, coils); EXPECT_EQ(cmd.payload.size(), modbus::packed_bit_bytes(coils.size())); @@ -21,7 +21,7 @@ TEST(ModbusCommandPayload, CoilWritePayloadIsExactSizedNotTruncated) { // LSB-first packing with zeroed pad bits, matching the wire layout the PDU builders produce. TEST(ModbusCommandPayload, CoilWritePacksLsbFirstWithZeroPad) { - ModbusController controller; + ModbusController controller(nullptr, 1); const std::vector coils{true, false, true, true}; auto cmd = ModbusCommandItem::create_write_multiple_coils(&controller, 0x10, coils); ASSERT_EQ(cmd.payload.size(), 1u); diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml new file mode 100644 index 0000000000..86e17ea0d7 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml @@ -0,0 +1,97 @@ +esphome: + name: uart-mock-modbus-lambda-write + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg30 + type: uint16_t + initial_value: "0" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x30 + value_type: U_WORD + read_lambda: return id(reg30); + write_lambda: id(reg30) = x; return true; + +# A COIL-type switch (assumed_state, write-only) whose write_lambda ignores its own coil type and instead +# drives a HOLDING-REGISTER write on the mock server through the entity itself: `item` IS the command, so +# item->write_single_register() sends a register write from a coil entity (cross-type). Returning nothing +# (an empty optional) tells the write path the lambda already dispatched the frame - no default coil write. +switch: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "cross_switch" + register_type: coil + address: 0x00 + assumed_state: true + write_lambda: |- + item->write_single_register(0x30, x ? 1234 : 0); + return {}; + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_30" + address: 0x30 + register_type: holding + value_type: U_WORD + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index 707637cfc2..3dfeda9b37 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -969,10 +969,10 @@ async def test_uart_mock_modbus_client_read_write( @pytest.mark.xfail( strict=True, - reason="Byte-accurate register-offset writes require the modbus_controller " - "entity-device change; on dev the byte offset is folded into the address " - "(writes 0x12 instead of 0x11). The write and read assertions both flip via " - "the same switch-constructor fold. Remove this marker when that change merges.", + reason="Byte-accurate register-offset writes land in the follow-up offset fix; " + "until then the byte offset is folded into the address (writes 0x12 instead of " + "0x11). The write and read assertions both flip via the same switch-constructor " + "fold. Remove this marker when that change merges.", ) @pytest.mark.asyncio async def test_uart_mock_modbus_register_offset( @@ -1029,14 +1029,42 @@ async def test_uart_mock_modbus_register_offset( ) -@pytest.mark.xfail( - strict=True, - reason="The deprecated write buffer requires the modbus_controller " - "entity-device change; on dev a nullopt-returning write_lambda early-returns " - "before the buffer is used, so the write never happens. The warn-once " - "assertion matches the log substring 'write_lambda buffer'. Remove this " - "marker when that change merges.", -) +@pytest.mark.asyncio +async def test_uart_mock_modbus_lambda_write( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test a write_lambda that drives the write through the entity itself (item is the command). + + `cross_switch` is a coil-type switch whose write_lambda ignores its own type and calls + item->write_single_register(0x30, ...) - a register write issued from a coil entity. The lambda + returns an empty optional, so the write path detects the lambda already dispatched a frame and does + not fall back to the default coil write. Success is reg_30 reading back the value the lambda wrote, + which proves both the new item->write_* path and cross-type flexibility. + """ + + tracker = SensorTracker(["reg_30"]) + initial = tracker.expect("reg_30", 0) + wrote_30 = tracker.expect("reg_30", 1234) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + await tracker.await_change(initial, "reg_30", timeout=4.0) + + switch = find_entity(entities, "cross_switch", SwitchInfo) + assert switch is not None, "cross_switch not found" + client.switch_command(switch.key, True) + + # The coil switch's lambda wrote register 0x30 via item->write_single_register(); reg_30 must + # read back 1234. If the entity-as-command dispatch were broken, no register write would go out + # and this would time out. + await tracker.await_change(wrote_30, "reg_30", timeout=4.0) + + @pytest.mark.asyncio async def test_uart_mock_modbus_deprecated_write_buffer( yaml_config: str, @@ -1046,9 +1074,9 @@ async def test_uart_mock_modbus_deprecated_write_buffer( """Test the deprecated write_lambda buffer path still works, and warns once per entity. buf_number's write_lambda fills the old `payload` buffer with a legacy raw frame as words (device - address + function code + data) instead of calling item->write_*. Two writes must both land with the - legacy raw-frame semantics, and the one-time deprecation warning must fire exactly once per entity - regardless of how many writes happen. + address + function code + data) and returns {} instead of calling item->write_*. Both writes must + land - a filled buffer is sent, as the docs have always described - and the one-time deprecation + warning must fire exactly once per entity regardless of how many writes happen. """ warn_count = 0 From 3361d031de3f47741503ecfcd49202f2027c0e13 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 19:46:13 -0500 Subject: [PATCH 64/65] [api] Drop connection instead of crashing when overflow buffer allocation fails (#18802) --- esphome/components/api/api_frame_helper.cpp | 2 +- esphome/components/api/api_overflow_buffer.cpp | 16 ++++++++++++++-- esphome/components/api/api_overflow_buffer.h | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/api_frame_helper.cpp b/esphome/components/api/api_frame_helper.cpp index 7425304766..38da444a18 100644 --- a/esphome/components/api/api_frame_helper.cpp +++ b/esphome/components/api/api_frame_helper.cpp @@ -172,7 +172,7 @@ APIError APIFrameHelper::write_raw_iov_(const struct iovec *iov, int iovcnt, uin // Queue unsent data into overflow buffer if (!this->overflow_buf_.enqueue_iov(iov, iovcnt, total_write_len, static_cast(sent))) { - HELPER_LOG("Overflow buffer full, dropping connection"); + HELPER_LOG("Overflow buffer full or out of memory, dropping connection"); this->state_ = State::FAILED; return APIError::SOCKET_WRITE_FAILED; } diff --git a/esphome/components/api/api_overflow_buffer.cpp b/esphome/components/api/api_overflow_buffer.cpp index a57a2fb1bb..48d8fe18ba 100644 --- a/esphome/components/api/api_overflow_buffer.cpp +++ b/esphome/components/api/api_overflow_buffer.cpp @@ -1,6 +1,7 @@ #include "api_overflow_buffer.h" #ifdef USE_API #include +#include namespace esphome::api { @@ -61,9 +62,18 @@ bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, uint16_ return false; uint16_t buffer_size = total_len - skip; + // nothrow: a failed allocation returns nullptr so the connection is dropped + // cleanly instead of plain new's crash or abort on OOM // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - auto *entry = new Entry{new uint8_t[buffer_size], buffer_size, 0}; - this->queue_[this->tail_] = entry; + auto *data = new (std::nothrow) uint8_t[buffer_size]; + if (data == nullptr) + return false; + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + auto *entry = new (std::nothrow) Entry{data, buffer_size, 0}; + if (entry == nullptr) { + delete[] data; + return false; + } uint16_t to_skip = skip; uint16_t write_pos = 0; @@ -80,6 +90,8 @@ bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, uint16_ } } + // Publish only after the copy completes so a half-built entry is never reachable + this->queue_[this->tail_] = entry; this->tail_ = (this->tail_ + 1) % API_MAX_SEND_QUEUE; this->count_++; return true; diff --git a/esphome/components/api/api_overflow_buffer.h b/esphome/components/api/api_overflow_buffer.h index 1227e83126..03a334b281 100644 --- a/esphome/components/api/api_overflow_buffer.h +++ b/esphome/components/api/api_overflow_buffer.h @@ -61,7 +61,7 @@ class APIOverflowBuffer { /// Enqueue unsent IOV data into the backlog. /// Copies iov data starting at byte offset `skip` into a new entry. - /// Returns false if the queue is full (caller should fail the connection). + /// Returns false if the queue is full or allocation fails (caller should fail the connection). bool enqueue_iov(const struct iovec *iov, int iovcnt, uint16_t total_len, uint16_t skip); protected: From 39177402ddabe5b474894a8f53860dea6a861f4f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 19:46:28 -0500 Subject: [PATCH 65/65] [api] Deprecate media player supports_pause field (#18801) --- esphome/components/api/api.proto | 3 ++- esphome/components/api/api_connection.cpp | 1 - esphome/components/api/api_pb2.cpp | 2 -- esphome/components/api/api_pb2.h | 3 +-- esphome/components/api/api_pb2_dump.cpp | 1 - 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 1942ff568b..c11700782e 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1654,7 +1654,8 @@ message ListEntitiesMediaPlayerResponse { bool disabled_by_default = 6; EntityCategory entity_category = 7; - bool supports_pause = 8; + // Deprecated in ESPHome 2026.9.0; use feature_flags instead. + bool supports_pause = 8 [deprecated = true]; repeated MediaPlayerSupportedFormat supported_formats = 9; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9b1026d2a9..7b0cb7069e 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1099,7 +1099,6 @@ uint16_t APIConnection::try_send_media_player_info(EntityBase *entity, APIConnec auto *media_player = static_cast(entity); ListEntitiesMediaPlayerResponse msg; auto traits = media_player->get_traits(); - msg.supports_pause = traits.get_supports_pause(); msg.feature_flags = traits.get_feature_flags(); for (auto &supported_format : traits.get_supported_formats()) { msg.supported_formats.emplace_back(); diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index b5062f9e9f..f56d791b67 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -2323,7 +2323,6 @@ uint8_t *ListEntitiesMediaPlayerResponse::encode(ProtoWriteBuffer &buffer PROTO_ #endif ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 6, this->disabled_by_default); ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 7, static_cast(this->entity_category)); - ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 8, this->supports_pause); for (auto &it : this->supported_formats) { ProtoEncode::encode_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 9, it); } @@ -2343,7 +2342,6 @@ uint32_t ListEntitiesMediaPlayerResponse::calculate_size() const { #endif size += ProtoSize::calc_bool(1, this->disabled_by_default); size += this->entity_category ? 2 : 0; - size += ProtoSize::calc_bool(1, this->supports_pause); if (!this->supported_formats.empty()) { for (const auto &it : this->supported_formats) { size += ProtoSize::calc_message_force(1, it.calculate_size()); diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 48e277fce1..bed28d2956 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1911,11 +1911,10 @@ class MediaPlayerSupportedFormat final : public ProtoMessage { class ListEntitiesMediaPlayerResponse final : public InfoResponseProtoMessage { public: static constexpr uint16_t MESSAGE_TYPE = 63; - static constexpr uint8_t ESTIMATED_SIZE = 80; + static constexpr uint8_t ESTIMATED_SIZE = 78; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_media_player_response"); } #endif - bool supports_pause{false}; std::vector supported_formats{}; uint32_t feature_flags{0}; uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 9f53438531..846c0ad652 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -1962,7 +1962,6 @@ const char *ListEntitiesMediaPlayerResponse::dump_to(DumpBuffer &out) const { #endif dump_field(out, ESPHOME_PSTR("disabled_by_default"), this->disabled_by_default); dump_field(out, ESPHOME_PSTR("entity_category"), static_cast(this->entity_category)); - dump_field(out, ESPHOME_PSTR("supports_pause"), this->supports_pause); for (const auto &it : this->supported_formats) { out.append(4, ' ').append_p(ESPHOME_PSTR("supported_formats")).append(": "); it.dump_to(out);