From 8b8de0c9c65923a845981ad317dd3ad919697040 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:33:39 +1000 Subject: [PATCH 01/30] [lvgl] Fix on_value/on_update triggers for LVGL select entities (#18778) Co-authored-by: Claude Sonnet 5 --- esphome/components/lvgl/lvgl_esphome.cpp | 8 ++-- esphome/components/lvgl/lvgl_esphome.h | 6 +-- esphome/components/lvgl/select/lvgl_select.h | 15 ++----- esphome/components/lvgl/types.py | 3 ++ .../dropdown_update_fires_event_test.yaml | 36 ++++++++++++++++ .../lvgl/test_dropdown_update_fires_event.py | 41 +++++++++++++++++++ 6 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml create mode 100644 tests/component_tests/lvgl/test_dropdown_update_fires_event.py diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index 684f472ebd..a10fdb0582 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -597,21 +597,21 @@ std::string LvSelectable::get_selected_text() { return this->options_[selected]; } -static std::string join_string(std::vector options) { +static std::string join_string(const FixedVector &options) { return std::accumulate( options.begin(), options.end(), std::string(), - [](const std::string &a, const std::string &b) -> std::string { return a + (!a.empty() ? "\n" : "") + b; }); + [](const std::string &a, const char *b) -> std::string { return a + (!a.empty() ? "\n" : "") + b; }); } void LvSelectable::set_selected_text(const std::string &text, lv_anim_enable_t anim) { - auto index = std::find(this->options_.begin(), this->options_.end(), text); + auto *index = std::find(this->options_.begin(), this->options_.end(), text); if (index != this->options_.end()) { this->set_selected_index(index - this->options_.begin(), anim); lv_obj_send_event(this->obj, lv_update_event, nullptr); } } -void LvSelectable::set_options(std::vector options) { +void LvSelectable::set_options(FixedVector options) { auto index = this->get_selected_index(); if (index >= options.size()) index = options.size() - 1; diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index ceba786e43..8b7397c4cd 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -543,12 +543,12 @@ class LvSelectable : public LvCompound { virtual void set_selected_index(size_t index, lv_anim_enable_t anim) = 0; void set_selected_text(const std::string &text, lv_anim_enable_t anim); std::string get_selected_text(); - const std::vector &get_options() { return this->options_; } - void set_options(std::vector options); + const FixedVector &get_options() { return this->options_; } + void set_options(FixedVector options); protected: virtual void set_option_string(const char *options) = 0; - std::vector options_{}; + FixedVector options_{}; }; #ifdef USE_LVGL_DROPDOWN diff --git a/esphome/components/lvgl/select/lvgl_select.h b/esphome/components/lvgl/select/lvgl_select.h index e36357328c..dafdd91eb5 100644 --- a/esphome/components/lvgl/select/lvgl_select.h +++ b/esphome/components/lvgl/select/lvgl_select.h @@ -50,19 +50,10 @@ class LVGLSelect final : public select::Select, public Component { protected: void control(size_t index) override { this->widget_->set_selected_index(index, this->anim_); - this->publish(); - } - void set_options_() { - // Widget uses std::vector, SelectTraits uses FixedVector - // Convert by extracting c_str() pointers - const auto &opts = this->widget_->get_options(); - FixedVector opt_ptrs; - opt_ptrs.init(opts.size()); - for (const auto &opt : opts) { - opt_ptrs.push_back(opt.c_str()); - } - this->traits.set_options(opt_ptrs); + // The update event fires the widget's on_value/on_update triggers + lv_obj_send_event(this->widget_->obj, lv_update_event, nullptr); } + void set_options_() { this->traits.set_options(this->widget_->get_options()); } LvSelectable *widget_; lv_anim_enable_t anim_; diff --git a/esphome/components/lvgl/types.py b/esphome/components/lvgl/types.py index 61efe385e6..cc8d9438a9 100644 --- a/esphome/components/lvgl/types.py +++ b/esphome/components/lvgl/types.py @@ -3,6 +3,8 @@ from esphome.const import CONF_TEXT, CONF_VALUE from esphome.cpp_generator import MockObj from esphome.cpp_types import Component, esphome_ns +from .defines import CONF_SELECTED_INDEX + class LvType(cg.MockObjClass): def __init__(self, *args, **kwargs): @@ -112,3 +114,4 @@ class LvSelect(LvType): parents=parens, **kwargs, ) + self.value_property = CONF_SELECTED_INDEX diff --git a/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml b/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml new file mode 100644 index 0000000000..2fe59b2f1a --- /dev/null +++ b/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml @@ -0,0 +1,36 @@ +esphome: + name: test-dropdown-update-event + on_boot: + - lvgl.dropdown.update: + id: test_dropdown + selected_index: 2 + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +lvgl: + widgets: + - dropdown: + id: test_dropdown + options: + - First + - Second + - Third + on_update: + - lambda: |- + ESP_LOGD("test", "dropdown updated"); diff --git a/tests/component_tests/lvgl/test_dropdown_update_fires_event.py b/tests/component_tests/lvgl/test_dropdown_update_fires_event.py new file mode 100644 index 0000000000..1e034ad6eb --- /dev/null +++ b/tests/component_tests/lvgl/test_dropdown_update_fires_event.py @@ -0,0 +1,41 @@ +"""Regression test: lvgl.dropdown.update with selected_index must fire on_value/on_update. + +LvSelect (backing both dropdown and roller) did not set `value_property`, so the generic +update-action machinery in automation.py never sent the synthetic update event for a +`selected_index:` change made via `lvgl.dropdown.update`/`lvgl.roller.update`, unlike `value:` +on number widgets or `text:` on text widgets. Fixed by setting `LvSelect.value_property` to +`CONF_SELECTED_INDEX`. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + config_path = ( + Path(request.fspath).parent / "config" / "dropdown_update_fires_event_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_dropdown_update_sends_update_event(main_cpp: str) -> None: + assert ( + "lv_obj_send_event(test_dropdown->obj, lvgl::lv_update_event, nullptr)" + in main_cpp + ) From cb981c2930f2efe813b59f7dc5beedbe82cda55b Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 25 Aug 2026 22:19:54 -0500 Subject: [PATCH 02/30] [remote_transmitter] ISR-driven transmission and non_blocking support on RTL8720C (#18648) --- .../components/remote_transmitter/__init__.py | 16 +- .../remote_transmitter/remote_transmitter.h | 31 +- .../remote_transmitter_rtl87xx.cpp | 285 ++++++++++++++++-- .../remote_transmitter/__init__.py | 0 .../test_non_blocking_gate.py | 42 +++ .../remote_transmitter/test.rtl87xx-ard.yaml | 1 + 6 files changed, 350 insertions(+), 25 deletions(-) create mode 100644 tests/component_tests/remote_transmitter/__init__.py create mode 100644 tests/component_tests/remote_transmitter/test_non_blocking_gate.py diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index 9d8761ea90..8ae51829e7 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -3,6 +3,8 @@ import logging from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base +from esphome.components.libretiny import get_libretiny_family +from esphome.components.libretiny.const import FAMILY_RTL8720C from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -43,6 +45,16 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) +def _validate_non_blocking_platform(value: bool) -> bool: + # non_blocking requires hardware transmission: RMT on ESP32, the gtimer + # envelope chain on RTL8720C. Reject everywhere else at config time. + if CORE.is_esp32: + return cv.boolean(value) + if CORE.is_libretiny and get_libretiny_family() == FAMILY_RTL8720C: + return cv.boolean(value) + raise cv.Invalid("non_blocking is only supported on ESP32 and RTL8720C") + + MULTI_CONF = True CONFIG_SCHEMA = ( cv.Schema( @@ -76,7 +88,7 @@ CONFIG_SCHEMA = ( esp32_s2=64, esp32_s3=48, ): cv.All(cv.only_on_esp32, cv.int_range(min=2)), - cv.Optional(CONF_NON_BLOCKING): cv.All(cv.only_on_esp32, cv.boolean), + cv.Optional(CONF_NON_BLOCKING): _validate_non_blocking_platform, cv.Optional(CONF_ON_TRANSMIT): automation.validate_automation(single=True), cv.Optional(CONF_ON_COMPLETE): automation.validate_automation(single=True), } @@ -164,6 +176,8 @@ async def to_code(config: ConfigType) -> None: ) else: var = cg.new_Pvariable(config[CONF_ID], pin) + if (non_blocking := config.get(CONF_NON_BLOCKING)) is not None: + cg.add(var.set_non_blocking(non_blocking)) await cg.register_component(var, config) cg.add(var.set_carrier_duty_percent(config[CONF_CARRIER_DUTY_PERCENT])) diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index 94bcb74b09..ef9a80f668 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -56,15 +56,23 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa #if defined(USE_ESP32) && SOC_RMT_SUPPORTED void set_with_dma(bool with_dma) { this->with_dma_ = with_dma; } void set_eot_level(bool eot_level) { this->eot_level_ = eot_level; } +#endif +#if (defined(USE_ESP32) && SOC_RMT_SUPPORTED) || defined(USE_LIBRETINY_VARIANT_RTL8720C) void set_non_blocking(bool non_blocking) { this->non_blocking_ = non_blocking; } #endif +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + void loop() override; + // called from the envelope timer ISR trampoline; not part of the public API + void advance_envelope_isr(); +#endif Trigger<> *get_transmit_trigger() { return &this->transmit_trigger_; } Trigger<> *get_complete_trigger() { return &this->complete_trigger_; } protected: void send_internal(uint32_t send_times, uint32_t send_wait) override; -#if defined(USE_ESP8266) || defined(USE_LIBRETINY) || defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) +#if defined(USE_ESP8266) || (defined(USE_LIBRETINY) && !defined(USE_LIBRETINY_VARIANT_RTL8720C)) || \ + defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) void await_target_time_(); uint32_t target_time_{0}; #endif @@ -81,6 +89,27 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa uint32_t current_carrier_frequency_{0}; void *pwm_{nullptr}; // pwmout_t*, opaque here to keep the SDK header out of this shared header #endif +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + void start_isr_item_(size_t index); + void arm_envelope_timer_(uint32_t duration_us); + void abort_stalled_chain_(); + void deliver_completion_(); + void wait_until_idle_(); + void arm_chain_(uint32_t send_times, uint32_t send_wait); + void update_carrier_(uint32_t carrier_frequency); + std::vector isr_data_; // owned copy of the frame; temp_ may be re-encoded mid-flight + float isr_mark_duty_{0.0f}; + float isr_space_duty_{0.0f}; + volatile size_t isr_index_{0}; + volatile uint32_t isr_repeats_left_{0}; + uint32_t isr_send_wait_{0}; + volatile uint32_t isr_wait_remaining_{0}; // remainder of a duration chained across one-shots + volatile bool isr_in_gap_{false}; + volatile bool transmitting_{false}; + bool non_blocking_{false}; + bool complete_pending_{false}; + bool stall_aborted_{false}; // this transmission ended via abort; blocks warning clear +#endif #if defined(USE_ESP32) && SOC_RMT_SUPPORTED void configure_rmt_(); diff --git a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp index b7078b9d69..9f629168f2 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp @@ -5,31 +5,49 @@ // clang-tidy cannot parse the Realtek SDK headers pulled in via ArduinoPrivate.h #if defined(USE_RTL87XX) && !defined(CLANG_TIDY) -// ArduinoPrivate.h = Arduino.h + the SDK's mbed HAL (pwmout etc.) with the core's fixes for +// ArduinoPrivate.h = Arduino.h + the SDK's mbed HAL (pwmout, gtimer) with the core's fixes for // type-name collisions between the two (e.g. PinMode) #include +#ifndef USE_LIBRETINY_VARIANT_RTL8720C #include #include +#endif namespace esphome::remote_transmitter { static const char *const TAG = "remote_transmitter"; -// The carrier is generated by the PWM peripheral instead of bit-banging the pin: software carrier -// generation requires disabling interrupts for the whole frame, but this core's micros() is derived -// from the FreeRTOS tick and freezes while interrupts are off, so the timing loop never advances and -// the watchdog resets the chip. With hardware PWM, software only times the mark/space envelope and -// interrupts can stay enabled. -// -// The PWM is driven through the SDK's pwmout HAL directly rather than the Arduino wiring layer: -// changing the carrier frequency via the wiring requires a GPIO/PWM pin mode round-trip, which -// use-after-frees the core's per-pin state (pinRemoveMode() frees without nulling) and corrupts the -// heap. pwmout_period_us() changes the frequency with no mode transitions. +// PWM peripheral carrier, envelope paced by a gtimer interrupt chain. Bit-banging would need +// interrupts disabled for the whole frame, but this core's micros() derives from the FreeRTOS +// tick and freezes then. The SDK pwmout HAL is driven directly: the Arduino wiring layer's +// GPIO/PWM mode round-trip use-after-frees LibreTiny's per-pin state. + +#ifdef USE_LIBRETINY_VARIANT_RTL8720C +static constexpr uint32_t ENVELOPE_TIMER_ID = TIMER6; // GTimer7 +// Margin past a transmission's expected duration before the chain is declared stalled +static constexpr uint32_t STALL_MARGIN_MS = 1000; +// Longest single one-shot armed; longer durations are chained (ROM us->tick headroom unverified) +static constexpr uint32_t MAX_ONE_SHOT_US = 50000; + +// Shared envelope timer: a second gtimer_init on the same id fails silently, so all +// instances serialize on s_active_transmitter +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) +static uint8_t s_pwm_tick_sources[] = {GTimer1, GTimer2, GTimer3, GTimer4, GTimer5, GTimer6, 0xff}; +static gtimer_t s_envelope_timer; +static bool s_envelope_timer_ready = false; +static RemoteTransmitterComponent *volatile s_active_transmitter = nullptr; +// Deadline for the in-flight transmission (millis-based); only touched from the main task +static uint32_t s_expected_end_ms = 0; +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) + +static void IRAM_ATTR envelope_timer_isr(uint32_t arg) { + reinterpret_cast(arg)->advance_envelope_isr(); +} +#endif // USE_LIBRETINY_VARIANT_RTL8720C void RemoteTransmitterComponent::setup() { - // Deliberately no pin_->setup(): registering the pin as GPIO claims it in the SDK's pin - // management, and the pad is then never handed over to the PWM peripheral -- pwmout_init() - // must own the pin from the start. + // no pin_->setup(): a GPIO claim in the SDK's pin management blocks pwmout_init from + // owning the pad PinInfo *info = pinInfo(this->pin_->get_pin()); if (info == nullptr || !pinSupported(info, PIN_PWM)) { // checked here because the AmebaZ (RTL8710B) SDK does not report PWM init failure @@ -40,7 +58,7 @@ void RemoteTransmitterComponent::setup() { auto *pwm = new pwmout_t(); this->pwm_ = pwm; pwmout_init(pwm, static_cast(info->gpio)); -#if LT_RTL8720C +#ifdef USE_LIBRETINY_VARIANT_RTL8720C // only the AmebaZ2 SDK's pwmout_s reports init success if (!pwm->is_init) { ESP_LOGE(TAG, "PWM init failed on pin %u", this->pin_->get_pin()); @@ -49,9 +67,19 @@ void RemoteTransmitterComponent::setup() { this->mark_failed(); return; } + // Shrink the PWM tick-source pool before the period claim below so GTimer7 stays free + // for the envelope; pwmout_init just registered the full pool. + hal_pwm_comm_tick_source_list(s_pwm_tick_sources); #endif pwmout_period_us(pwm, 26); // placeholder; the real carrier period is set per transmission pwmout_write(pwm, this->pin_->is_inverted() ? 1.0f : 0.0f); +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + if (!s_envelope_timer_ready) { + gtimer_init(&s_envelope_timer, ENVELOPE_TIMER_ID); + s_envelope_timer_ready = true; + } + this->disable_loop(); // loop() is only needed while a non-blocking completion is pending +#endif } void RemoteTransmitterComponent::dump_config() { @@ -59,9 +87,224 @@ void RemoteTransmitterComponent::dump_config() { "Remote Transmitter:\n" " Carrier Duty: %u%%", this->carrier_duty_percent_); +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + ESP_LOGCONFIG(TAG, " Non-blocking: %s", YESNO(this->non_blocking_)); +#endif LOG_PIN(" Pin: ", this->pin_); } +void RemoteTransmitterComponent::digital_write(bool value) { + if (this->pwm_ == nullptr) + return; +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + // serialize behind an in-flight chain, matching the ESP32/RMT non-blocking behavior + this->wait_until_idle_(); +#endif + pwmout_write(static_cast(this->pwm_), (value != this->pin_->is_inverted()) ? 1.0f : 0.0f); +} + +#ifdef USE_LIBRETINY_VARIANT_RTL8720C +// Arms the shared envelope timer, chaining durations longer than MAX_ONE_SHOT_US. ISR-safe. +void IRAM_ATTR RemoteTransmitterComponent::arm_envelope_timer_(uint32_t duration_us) { + // clamp to 1us (a zero-length one-shot never fires); the remainder must not underflow + const uint32_t chunk = std::max(uint32_t(1), std::min(duration_us, MAX_ONE_SHOT_US)); + this->isr_wait_remaining_ = duration_us > chunk ? duration_us - chunk : 0; + gtimer_start_one_shout(&s_envelope_timer, chunk, (void *) envelope_timer_isr, (uint32_t) this); +} + +// Aborts a chain that stopped advancing: stop the timer, idle the pin, release the token. +// Every step is a no-op if the chain completed meanwhile. Task context only. +void RemoteTransmitterComponent::abort_stalled_chain_() { + // cleared first so a straggler one-shot bails at the ISR entry check + this->transmitting_ = false; + gtimer_stop(&s_envelope_timer); + pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); + s_active_transmitter = nullptr; + this->stall_aborted_ = true; + this->status_set_warning("envelope timer stalled"); + ESP_LOGE(TAG, "Envelope timer stalled; transmission aborted"); + delay(1); // let any already-latched interrupt land while the chain state is safe +} + +// Delivers one deferred completion with its status bookkeeping +void RemoteTransmitterComponent::deliver_completion_() { + if (!this->stall_aborted_) + this->status_clear_warning(); + this->complete_pending_ = false; + this->complete_trigger_.trigger(); +} + +// Writes the duty for one envelope item and arms the timer for its duration. +// Runs in ISR context (and once from send_internal to kick the chain): no logging, no allocation. +void IRAM_ATTR RemoteTransmitterComponent::start_isr_item_(size_t index) { + const int32_t item = this->isr_data_[index]; + pwmout_write(static_cast(this->pwm_), item > 0 ? this->isr_mark_duty_ : this->isr_space_duty_); + this->arm_envelope_timer_(uint32_t(item > 0 ? item : -item)); +} + +void IRAM_ATTR RemoteTransmitterComponent::advance_envelope_isr() { + if (!this->transmitting_) + return; // chain was aborted; this is a stale one-shot that was already latched + if (this->isr_wait_remaining_ > 0) { + // continue a duration longer than one hardware one-shot + this->arm_envelope_timer_(this->isr_wait_remaining_); + return; + } + if (this->isr_in_gap_) { + // inter-repeat gap elapsed; restart the item chain + this->isr_in_gap_ = false; + this->isr_index_ = 0; + this->start_isr_item_(0); + return; + } + this->isr_index_++; + if (this->isr_index_ < this->isr_data_.size()) { + this->start_isr_item_(this->isr_index_); + return; + } + // end of one repetition + pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); + if (this->isr_repeats_left_ > 1) { + this->isr_repeats_left_--; + this->isr_index_ = 0; + if (this->isr_send_wait_ > 0) { + this->isr_in_gap_ = true; + this->arm_envelope_timer_(this->isr_send_wait_); + } else { + this->start_isr_item_(0); + } + return; + } + this->transmitting_ = false; + s_active_transmitter = nullptr; +} + +// Waits until no chain is in flight, delivering any deferred completions; a completion +// automation may start a new send, so repeat until truly idle. Bounded by the stall deadline. +void RemoteTransmitterComponent::wait_until_idle_() { + while (true) { + while (true) { + // snapshot: the final ISR can clear the volatile pointer between a check and a use + auto *active = s_active_transmitter; + if (active == nullptr) + break; + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + active->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + if (!this->complete_pending_) + break; + this->deliver_completion_(); + } +} + +// Retunes the PWM period when the carrier changes; the ISR sets duty per item +void RemoteTransmitterComponent::update_carrier_(uint32_t carrier_frequency) { + if (carrier_frequency == 0 || carrier_frequency == this->current_carrier_frequency_) + return; + // round(1000000/freq), clamped so a bad lambda can't hand the SDK a zero period + const uint32_t period = std::max(uint32_t(1), (1000000UL + carrier_frequency / 2) / carrier_frequency); + pwmout_period_us(static_cast(this->pwm_), period); + this->current_carrier_frequency_ = carrier_frequency; +} + +// Stages the repeat schedule and stall deadline, then starts the interrupt chain +void RemoteTransmitterComponent::arm_chain_(uint32_t send_times, uint32_t send_wait) { + this->isr_repeats_left_ = send_times; + this->isr_send_wait_ = send_wait; + this->isr_index_ = 0; + this->isr_in_gap_ = false; + this->stall_aborted_ = false; + uint64_t frame_us = 0; + for (int32_t item : this->isr_data_) + frame_us += uint32_t(item > 0 ? item : -item); + const uint64_t total_us = frame_us * send_times + uint64_t(send_wait) * (send_times - 1); + s_expected_end_ms = millis() + uint32_t(total_us / 1000) + STALL_MARGIN_MS; + this->transmitting_ = true; + s_active_transmitter = this; + this->start_isr_item_(0); +} + +void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { + if (this->pwm_ == nullptr) { + ESP_LOGW(TAG, "Cannot send: PWM not initialized"); + return; + } + this->wait_until_idle_(); + if (send_times == 0) { + // parity with the loop-based implementations: transmit nothing, but both triggers + // still fire so an on_complete-sequenced automation does not stall + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + ESP_LOGD(TAG, "Sending remote code"); + const uint32_t carrier_frequency = this->temp_.get_carrier_frequency(); + // unmodulated protocols (no carrier or 100% duty) drive the pin constantly during marks + float mark_duty = + (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) ? this->carrier_duty_percent_ / 100.0f : 1.0f; + float space_duty = 0.0f; + if (this->pin_->is_inverted()) { + mark_duty = 1.0f - mark_duty; + space_duty = 1.0f; + } + this->update_carrier_(carrier_frequency); + // own copy: with non_blocking the caller may re-encode temp_ while this frame is in flight + this->isr_data_.assign(this->temp_.get_data().begin(), this->temp_.get_data().end()); + if (this->isr_data_.empty()) { + ESP_LOGW(TAG, "Empty data"); + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + this->isr_mark_duty_ = mark_duty; + this->isr_space_duty_ = space_duty; + // trigger first: the deadline computed in arm_chain_ must not be charged for user code + this->transmit_trigger_.trigger(); + // the automation may have started a send on another instance; let it finish before + // claiming the shared timer (a same-instance send remains unsupported here) + this->wait_until_idle_(); + this->arm_chain_(send_times, send_wait); + if (this->non_blocking_) { + this->complete_pending_ = true; + this->enable_loop(); + return; + } + // blocking mode: wait out the chain, bounded by the stall deadline + while (this->transmitting_) { + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + this->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + this->deliver_completion_(); +} + +void RemoteTransmitterComponent::loop() { + if (!this->complete_pending_) { + this->disable_loop(); + return; + } + if (this->transmitting_) { + // non-blocking stall recovery: without this, a dead chain would leave the carrier + // driven and on_complete unfired until the next send happened to abort it + if ((int32_t) (millis() - s_expected_end_ms) <= 0) + return; + this->abort_stalled_chain_(); + } + // release the loop before user code runs: the automation may start a new non-blocking + // send, and its enable_loop() must be the last writer or its completion would strand + this->disable_loop(); + this->deliver_completion_(); +} + +#else // !USE_LIBRETINY_VARIANT_RTL8720C -- AmebaZ (RTL8710B): spin-based envelope, per-frame priority boost + void RemoteTransmitterComponent::await_target_time_() { const uint32_t current_time = micros(); if (this->target_time_ == 0) { @@ -72,15 +315,8 @@ void RemoteTransmitterComponent::await_target_time_() { } } -void RemoteTransmitterComponent::digital_write(bool value) { - if (this->pwm_ == nullptr) - return; - pwmout_write(static_cast(this->pwm_), (value != this->pin_->is_inverted()) ? 1.0f : 0.0f); -} - void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { - auto *pwm = static_cast(this->pwm_); - if (pwm == nullptr) { + if (this->pwm_ == nullptr) { ESP_LOGW(TAG, "Cannot send: PWM not initialized"); return; } @@ -94,6 +330,7 @@ void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t sen mark_duty = 1.0f - mark_duty; space_duty = 1.0f; } + auto *pwm = static_cast(this->pwm_); if (carrier_frequency > 0 && carrier_frequency != this->current_carrier_frequency_) { // round(1000000/freq), clamped like the bit-bang path so a bad lambda can't hand the SDK a zero period const uint32_t period = std::max(uint32_t(1), (1000000UL + carrier_frequency / 2) / carrier_frequency); @@ -132,6 +369,8 @@ void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t sen this->complete_trigger_.trigger(); } +#endif // USE_LIBRETINY_VARIANT_RTL8720C + } // namespace esphome::remote_transmitter #endif // USE_RTL87XX && !CLANG_TIDY diff --git a/tests/component_tests/remote_transmitter/__init__.py b/tests/component_tests/remote_transmitter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py new file mode 100644 index 0000000000..f843f1e84f --- /dev/null +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -0,0 +1,42 @@ +"""non_blocking is family-gated at config validation; the CI build boards never compile +the ISR paths, so this gate is the only CI-reachable coverage for the platform matrix.""" + +import pytest + +from esphome.components.libretiny.const import ( + FAMILY_RTL8710B, + FAMILY_RTL8720C, + KEY_FAMILY, + KEY_LIBRETINY, +) +from esphome.components.remote_transmitter import _validate_non_blocking_platform +import esphome.config_validation as cv +from esphome.const import PlatformFramework +from esphome.core import CORE + +from ..types import SetCoreConfigCallable + + +@pytest.mark.parametrize( + ("platform_framework", "family", "accepted"), + [ + (PlatformFramework.ESP32_IDF, None, True), + (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), + (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), + (PlatformFramework.ESP8266_ARDUINO, None, False), + ], +) +def test_non_blocking_platform_gate( + set_core_config: SetCoreConfigCallable, + platform_framework: PlatformFramework, + family: str | None, + accepted: bool, +) -> None: + set_core_config(platform_framework) + if family is not None: + CORE.data[KEY_LIBRETINY] = {KEY_FAMILY: family} + if accepted: + assert _validate_non_blocking_platform(True) is True + else: + with pytest.raises(cv.Invalid, match="non_blocking is only supported on"): + _validate_non_blocking_platform(True) diff --git a/tests/components/remote_transmitter/test.rtl87xx-ard.yaml b/tests/components/remote_transmitter/test.rtl87xx-ard.yaml index 769adbdf5c..74caa24cdd 100644 --- a/tests/components/remote_transmitter/test.rtl87xx-ard.yaml +++ b/tests/components/remote_transmitter/test.rtl87xx-ard.yaml @@ -2,6 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO12 carrier_duty_percent: 50% + # non_blocking is rtl8720c-only; the CI board is an RTL8710B packages: buttons: !include common-buttons.yaml From 612d58ec37e7fc565bf6fef3214247a4ff0f5366 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 23:11:26 -0500 Subject: [PATCH 03/30] [http_request] Default watchdog_timeout from timeout on ESP32 (#18732) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/components/http_request/__init__.py | 34 ++++++++++++++- .../http_request/http_request_idf.cpp | 3 +- .../component_tests/http_request/__init__.py | 0 .../config/test_esp32_default.yaml | 12 ++++++ .../config/test_esp32_explicit.yaml | 13 ++++++ .../config/test_esp32_platform_wider.yaml | 13 ++++++ .../http_request/config/test_esp32_stock.yaml | 11 +++++ .../http_request/config/test_esp8266.yaml | 13 ++++++ .../http_request/config/test_rp2040.yaml | 13 ++++++ .../component_tests/http_request/test_init.py | 42 +++++++++++++++++++ 10 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 tests/component_tests/http_request/__init__.py create mode 100644 tests/component_tests/http_request/config/test_esp32_default.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_explicit.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_platform_wider.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_stock.yaml create mode 100644 tests/component_tests/http_request/config/test_esp8266.yaml create mode 100644 tests/component_tests/http_request/config/test_rp2040.yaml create mode 100644 tests/component_tests/http_request/test_init.py diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 2abf097aec..de35d52a40 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -17,12 +17,14 @@ from esphome.const import ( CONF_TIMEOUT, CONF_URL, CONF_WATCHDOG_TIMEOUT, + PLATFORM_ESP32, PLATFORM_HOST, PlatformFramework, __version__, ) -from esphome.core import CORE, ID, Lambda +from esphome.core import CORE, ID, Lambda, TimePeriodMilliseconds from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.helpers import IS_MACOS from esphome.types import ConfigType @@ -94,6 +96,34 @@ def validate_ssl_verification(config: ConfigType) -> ConfigType: return config +# esp_http_client_open() runs DNS, TCP connect and the TLS handshake with no +# watchdog feed in between; each can take up to `timeout` on ESP-IDF. +WATCHDOG_TIMEOUT_MULTIPLIER = 3 +# Headroom over the exact worst case so a fully stalled open does not land on +# the watchdog deadline. +WATCHDOG_TIMEOUT_MARGIN_MS = 1000 + + +def default_watchdog_timeout(config: ConfigType) -> None: + """Arm the request watchdog on ESP32 when the user did not set it. + + The default never goes below the platform task watchdog, so a user who + widened `esp32.watchdog_timeout` keeps that window during requests. + """ + if not CORE.is_esp32 or CONF_WATCHDOG_TIMEOUT in config: + return + derived_ms = ( + config[CONF_TIMEOUT].total_milliseconds * WATCHDOG_TIMEOUT_MULTIPLIER + + WATCHDOG_TIMEOUT_MARGIN_MS + ) + platform_ms = fv.full_config.get()[PLATFORM_ESP32][ + CONF_WATCHDOG_TIMEOUT + ].total_milliseconds + config[CONF_WATCHDOG_TIMEOUT] = TimePeriodMilliseconds( + milliseconds=max(derived_ms, platform_ms) + ) + + def _declare_request_class(value: Any) -> ID: if CORE.is_host: return cv.declare_id(HttpRequestHost)(value) @@ -153,6 +183,8 @@ CONFIG_SCHEMA = cv.All( validate_ssl_verification, ) +FINAL_VALIDATE_SCHEMA = default_watchdog_timeout + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/http_request/http_request_idf.cpp b/esphome/components/http_request/http_request_idf.cpp index 470ed332f1..10313be89d 100644 --- a/esphome/components/http_request/http_request_idf.cpp +++ b/esphome/components/http_request/http_request_idf.cpp @@ -142,12 +142,13 @@ std::shared_ptr HttpRequestIDF::perform(const std::string &url, c const char *buf = body.c_str(); while (write_left > 0) { int written = esp_http_client_write(client, buf + write_index, write_left); - if (written < 0) { + if (written <= 0) { err = ESP_FAIL; break; } write_left -= written; write_index += written; + container->feed_wdt(); } } diff --git a/tests/component_tests/http_request/__init__.py b/tests/component_tests/http_request/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/http_request/config/test_esp32_default.yaml b/tests/component_tests/http_request/config/test_esp32_default.yaml new file mode 100644 index 0000000000..86744dcb11 --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_default.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s diff --git a/tests/component_tests/http_request/config/test_esp32_explicit.yaml b/tests/component_tests/http_request/config/test_esp32_explicit.yaml new file mode 100644 index 0000000000..e0d0074caa --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_explicit.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + watchdog_timeout: 20s diff --git a/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml b/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml new file mode 100644 index 0000000000..77a85da2ff --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + watchdog_timeout: 60s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s diff --git a/tests/component_tests/http_request/config/test_esp32_stock.yaml b/tests/component_tests/http_request/config/test_esp32_stock.yaml new file mode 100644 index 0000000000..70d2701466 --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_stock.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: diff --git a/tests/component_tests/http_request/config/test_esp8266.yaml b/tests/component_tests/http_request/config/test_esp8266.yaml new file mode 100644 index 0000000000..d0698dc57e --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp8266.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + verify_ssl: false diff --git a/tests/component_tests/http_request/config/test_rp2040.yaml b/tests/component_tests/http_request/config/test_rp2040.yaml new file mode 100644 index 0000000000..030736c30d --- /dev/null +++ b/tests/component_tests/http_request/config/test_rp2040.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +rp2: + board: rpipicow + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + verify_ssl: false diff --git a/tests/component_tests/http_request/test_init.py b/tests/component_tests/http_request/test_init.py new file mode 100644 index 0000000000..446c4acbd0 --- /dev/null +++ b/tests/component_tests/http_request/test_init.py @@ -0,0 +1,42 @@ +"""Tests for the http_request watchdog timeout default.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.config import read_config +from esphome.const import CONF_WATCHDOG_TIMEOUT +from esphome.core import CORE, TimePeriodMilliseconds + + +@pytest.mark.parametrize( + ("yaml_file", "expected_ms"), + [ + # stock 4.5s timeout: 3 x 4.5s plus 1s margin + ("test_esp32_stock.yaml", 14500), + # 3 x 10s plus 1s margin + ("test_esp32_default.yaml", 31000), + # esp32.watchdog_timeout: 60s is wider than the derived value and wins + ("test_esp32_platform_wider.yaml", 60000), + # explicit value is kept as is + ("test_esp32_explicit.yaml", 20000), + ], +) +def test_esp32_watchdog_timeout( + component_config_path: Callable[[str], Path], yaml_file: str, expected_ms: int +) -> None: + CORE.config_path = component_config_path(yaml_file) + config = read_config({}) + assert config["http_request"][CONF_WATCHDOG_TIMEOUT] == TimePeriodMilliseconds( + milliseconds=expected_ms + ) + + +@pytest.mark.parametrize("yaml_file", ["test_esp8266.yaml", "test_rp2040.yaml"]) +def test_other_platforms_leave_watchdog_unset( + component_config_path: Callable[[str], Path], yaml_file: str +) -> None: + CORE.config_path = component_config_path(yaml_file) + config = read_config({}) + assert CONF_WATCHDOG_TIMEOUT not in config["http_request"] From 9baff7652074031d27ba4c547bfee6250e36151a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 23:29:14 -0500 Subject: [PATCH 04/30] [api] Treat homeassistant.event variables as lambdas (#18759) --- esphome/components/api/__init__.py | 45 +++++++++++-- esphome/config_validation.py | 35 ++++++++++- esphome/core/__init__.py | 3 +- .../api/test_homeassistant_variables.py | 63 +++++++++++++++++++ .../api/test_homeassistant_variables.yaml | 32 ++++++++++ tests/components/api/common-base.yaml | 8 +++ tests/components/homeassistant/common.yaml | 4 +- tests/unit_tests/test_config_validation.py | 46 ++++++++++++++ 8 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 tests/component_tests/api/test_homeassistant_variables.py create mode 100644 tests/component_tests/api/test_homeassistant_variables.yaml diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index a10bfd3418..2e891a9663 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -1,4 +1,5 @@ import logging +import re from typing import Any from esphome import automation @@ -499,6 +500,40 @@ async def to_code(config: ConfigType) -> None: KEY_VALUE_SCHEMA = cv.Schema({cv.string: cv.templatable(cv.string_strict)}) +_ID_CALL_PROG = re.compile(r"\bid\s*\(") + + +# Remove before 2027.3.0: untagged strings that look like lambda source keep +# being compiled as lambdas during the deprecation window +def _coerce_implicit_lambda(value: Any) -> Any: + if not isinstance(value, str): + return value + if cv.looks_like_returning_lambda(value): + _LOGGER.warning( + "[api] The 'variables' value '%s' looks like a lambda but is " + "missing the !lambda tag. It is compiled as a lambda for now but " + "will be sent as literal text from 2027.3.0. Add !lambda to keep " + "it evaluated; literal text belongs under 'data:'.", + value, + ) + # cv.templatable runs returning_lambda on the coerced Lambda + return cv.lambda_(value) + if _ID_CALL_PROG.search(value): + # lambda source without a return: issue 5394's mistake class + _LOGGER.warning( + "[api] The 'variables' value '%s' is sent as literal text; wrap " + "it in !lambda 'return ...;' to evaluate it instead.", + value, + ) + return value + + +# Static strings or !lambda values. cv.templatable stays introspectable for +# schema tooling; removing the shim leaves KEY_VALUE_SCHEMA. +VARIABLES_SCHEMA = cv.Schema( + {cv.string: cv.All(_coerce_implicit_lambda, cv.templatable(cv.string_strict))} +) + def _validate_response_config(config: ConfigType) -> ConfigType: # Validate dependencies: @@ -535,9 +570,7 @@ HOMEASSISTANT_ACTION_ACTION_SCHEMA = cv.All( ), cv.Optional(CONF_DATA, default={}): KEY_VALUE_SCHEMA, cv.Optional(CONF_DATA_TEMPLATE, default={}): KEY_VALUE_SCHEMA, - cv.Optional(CONF_VARIABLES, default={}): cv.Schema( - {cv.string: cv.returning_lambda} - ), + cv.Optional(CONF_VARIABLES, default={}): VARIABLES_SCHEMA, cv.Optional(CONF_RESPONSE_TEMPLATE): cv.templatable(cv.string), cv.Optional(CONF_CAPTURE_RESPONSE, default=False): cv.boolean, cv.Optional(CONF_ON_SUCCESS): automation.validate_automation(single=True), @@ -598,6 +631,8 @@ async def homeassistant_service_to_code( cg.add(var.init_variables(len(config[CONF_VARIABLES]))) for key, value in config[CONF_VARIABLES].items(): templ = await cg.templatable(value, args, None) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) cg.add(var.add_variable(cg.FlashStringLiteral(key), templ)) if on_error := config.get(CONF_ON_ERROR): @@ -652,7 +687,7 @@ HOMEASSISTANT_EVENT_ACTION_SCHEMA = cv.Schema( cv.Required(CONF_EVENT): validate_homeassistant_event, cv.Optional(CONF_DATA, default={}): KEY_VALUE_SCHEMA, cv.Optional(CONF_DATA_TEMPLATE, default={}): KEY_VALUE_SCHEMA, - cv.Optional(CONF_VARIABLES, default={}): KEY_VALUE_SCHEMA, + cv.Optional(CONF_VARIABLES, default={}): VARIABLES_SCHEMA, } ) @@ -698,6 +733,8 @@ async def homeassistant_event_to_code( cg.add(var.init_variables(len(config[CONF_VARIABLES]))) for key, value in config[CONF_VARIABLES].items(): templ = await cg.templatable(value, args, None) + if isinstance(templ, str): + templ = cg.FlashStringLiteral(templ) cg.add(var.add_variable(cg.FlashStringLiteral(key), templ)) return var diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 09962e8c95..904cbd1919 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1882,13 +1882,46 @@ def lambda_(value): return value +# 'return' at a statement boundary; only consulted when the source has no +# semicolon, so ';' is not a boundary. Migration use only, see +# looks_like_returning_lambda. +LAMBDA_RETURN_STATEMENT_PROG = re.compile(r"(?:^|[:{})\n])\s*return\b") +LAMBDA_RETURN_KEYWORD_PROG = re.compile(r"\breturn\b") +# RESERVED_IDS subset that can begin a return expression; 'this'/'true' would +# promote prose and infix 'and'/'or' cannot start an expression. +_CPP_LEADING_WORD_OPERATORS = "not|new|sizeof|delete" +# Two or more plain words: prose, not C++. A single word is indistinguishable +# from 'return x'. Migration use only, see looks_like_returning_lambda. +LAMBDA_PROSE_TAIL_PROG = re.compile( + rf"(?!(?:{_CPP_LEADING_WORD_OPERATORS})\b)[A-Za-z']+(?:,?\s+[A-Za-z']+)+[.!?]?" +) + + +def looks_like_returning_lambda(value: str) -> bool: + """Check whether a string looks like C++ lambda source: a semicolon means + code, so any return keyword counts; without one, a boundary return whose + tail does not read as prose is a return statement missing its semicolon. + + For migrating deprecated implicit lambdas only; new validators must + require an explicit !lambda tag instead of guessing. + """ + src = Lambda.comment_remover(value) + if ";" in src: + return LAMBDA_RETURN_KEYWORD_PROG.search(src) is not None + for match in LAMBDA_RETURN_STATEMENT_PROG.finditer(src): + tail = src[match.end() :].split("\n", 1)[0].strip() + if not LAMBDA_PROSE_TAIL_PROG.fullmatch(tail): + return True + return False + + def returning_lambda(value): """Coerce this configuration option to a lambda. Additionally, make sure the lambda returns something. """ value = lambda_(value) - if "return" not in value.value: + if LAMBDA_RETURN_KEYWORD_PROG.search(Lambda.comment_remover(value.value)) is None: raise Invalid( "Lambda doesn't contain a 'return' statement, but the lambda " "is expected to return a value. \n" diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 2ec2a08e83..77efc91bef 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -339,7 +339,8 @@ class Lambda: self._requires_ids = None # https://stackoverflow.com/a/241506/229052 - def comment_remover(self, text): + @staticmethod + def comment_remover(text): def replacer(match): s = match.group(0) if s.startswith("/"): diff --git a/tests/component_tests/api/test_homeassistant_variables.py b/tests/component_tests/api/test_homeassistant_variables.py new file mode 100644 index 0000000000..48e53d8f4c --- /dev/null +++ b/tests/component_tests/api/test_homeassistant_variables.py @@ -0,0 +1,63 @@ +"""Tests for variables handling in homeassistant.event and homeassistant.action.""" + +from collections.abc import Callable +import logging +from pathlib import Path + +import pytest + +CONFIG = "tests/component_tests/api/test_homeassistant_variables.yaml" + + +def test_plain_string_with_return_is_compiled_as_lambda_with_warning( + generate_main: Callable[[str | Path], str], + caplog: pytest.LogCaptureFixture, +) -> None: + """A plain string with a return statement compiles as a lambda and warns.""" + with caplog.at_level(logging.WARNING): + main_cpp = generate_main(CONFIG) + + assert main_cpp.count('add_variable(ESPHOME_F("lambda_var"), []() {') == 2 + assert "return millis();" in main_cpp + # The source text must not be sent as a static string value. + assert '"return millis();"' not in main_cpp + assert "missing the !lambda tag" in caplog.text + + +def test_static_string_is_kept_as_static_value( + generate_main: Callable[[str | Path], str], + caplog: pytest.LogCaptureFixture, +) -> None: + """A static string stays static, PROGMEM wrapped, with no warning.""" + with caplog.at_level(logging.WARNING): + main_cpp = generate_main(CONFIG) + + assert ( + main_cpp.count( + 'add_variable(ESPHOME_F("static_var"), ESPHOME_F("static value"));' + ) + == 2 + ) + assert "static value" not in caplog.text + + +def test_static_id_value_stays_literal_with_hint( + generate_main: Callable[[str | Path], str], + caplog: pytest.LogCaptureFixture, +) -> None: + """Lambda source without a return stays literal text but warns.""" + with caplog.at_level(logging.WARNING): + main_cpp = generate_main(CONFIG) + + assert 'ESPHOME_F("id(test_sensor).state")' in main_cpp + assert "sent as literal text" in caplog.text + + +def test_explicit_lambda_tag_is_compiled_as_lambda( + generate_main: Callable[[str | Path], str], +) -> None: + """A !lambda value keeps working unchanged.""" + main_cpp = generate_main(CONFIG) + + assert 'add_variable(ESPHOME_F("tagged_var"), []() {' in main_cpp + assert "return App.get_name();" in main_cpp diff --git a/tests/component_tests/api/test_homeassistant_variables.yaml b/tests/component_tests/api/test_homeassistant_variables.yaml new file mode 100644 index 0000000000..e1ec07cc74 --- /dev/null +++ b/tests/component_tests/api/test_homeassistant_variables.yaml @@ -0,0 +1,32 @@ +esphome: + name: test + on_boot: + then: + # Plain strings with a return statement compile as lambdas + - homeassistant.event: + event: esphome.test_event + data_template: + message: "{{ lambda_var }} {{ static_var }} {{ tagged_var }}" + variables: + lambda_var: |- + return millis(); + static_var: static value + tagged_var: !lambda return App.get_name(); + hint_var: id(test_sensor).state + - homeassistant.action: + action: notify.notify + data_template: + message: "{{ lambda_var }} {{ static_var }}" + variables: + lambda_var: |- + return millis(); + static_var: static value + +esp32: + board: esp32dev + +wifi: + ssid: SomeNetwork + password: SomePassword + +api: diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index d7470ee4b3..c9eb200471 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -9,6 +9,14 @@ esphome: event: esphome.button_pressed data: message: Button was pressed + - homeassistant.event: + event: esphome.button_pressed_with_variables + data_template: + message: Button {{ button_name }} ({{ button_index }}) was pressed from {{ button_source }} + variables: + button_name: !lambda 'return std::string("test_button");' + button_index: !lambda 'return 1;' + button_source: static_value - homeassistant.action: action: notify.html5 data: diff --git a/tests/components/homeassistant/common.yaml b/tests/components/homeassistant/common.yaml index 71a7ac65c2..1099f7ea85 100644 --- a/tests/components/homeassistant/common.yaml +++ b/tests/components/homeassistant/common.yaml @@ -12,7 +12,7 @@ esphome: data_template: message: The humidity is {{ my_variable }}%. variables: - my_variable: "return id(ha_hello_world_temperature).state;" + my_variable: !lambda "return id(ha_hello_world_temperature).state;" - homeassistant.action: action: notify.html5 data: @@ -24,7 +24,7 @@ esphome: data_template: message: The humidity is {{ my_variable }}%. variables: - my_variable: "return id(ha_hello_world_temperature).state;" + my_variable: !lambda "return id(ha_hello_world_temperature).state;" wifi: ssid: MySSID diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 0f927a6513..457b9d017b 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -2565,6 +2565,52 @@ def test_returning_lambda_no_return() -> None: cv.returning_lambda(Lambda("int x = 5;")) +def test_returning_lambda_return_only_in_comment() -> None: + with pytest.raises(Invalid, match="return statement"): + cv.returning_lambda(Lambda("// return 5;\nint x = 5;")) + + +def test_returning_lambda_missing_semicolon_is_accepted() -> None: + """A forgotten semicolon is left for the C++ compiler to report.""" + assert isinstance(cv.returning_lambda(Lambda("return x")), Lambda) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("return 5;", True), + ("if (x) { return x; } return 0;", True), + ("if (x) return 1; else return 0;", True), + ("switch (x) { case 0: return 1; }", True), + # a semicolon means code: any return keyword counts + ("return not x;", True), + ("return a and b;", True), + ("please return the sensor; then wait", True), + # a forgotten semicolon is still lambda source; the compiler reports it + ("return id(x).state", True), + ("return x", True), + ("return 5", True), + ("return not x", True), + # accepted: a one-word tail is indistinguishable from 'return x' + ("return soon", True), + ("Alert: return home", True), + ("static value", False), + ("no returns here", False), + ("the_return_value", False), + # without a semicolon, prose is not lambda source + ("please return the item", False), + ("return to sender", False), + ("return a and b", False), + # return only inside a comment is not a return statement + ("// return 5;\nint x = 5;", False), + ("/* return 5; */ int x = 5;", False), + ("return 5; // done", True), + ], +) +def test_looks_like_returning_lambda(value: str, expected: bool) -> None: + assert cv.looks_like_returning_lambda(value) is expected + + # --------------------------------------------------------------------------- # dimensions # --------------------------------------------------------------------------- From e37a540fb729edfd15fbd32281b7ccf53ecd74cb Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 25 Aug 2026 23:59:37 -0500 Subject: [PATCH 05/30] [esp32] Apply custom eFuse MAC as base MAC for all interfaces (#18452) --- esphome/components/esp32/core.cpp | 8 ++++++ esphome/components/esp32/helpers.cpp | 25 +++++++++++++------ .../wifi/wifi_component_esp_idf.cpp | 5 ---- esphome/core/helpers.h | 5 ++++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/esphome/components/esp32/core.cpp b/esphome/components/esp32/core.cpp index 098a59937a..a6916fe739 100644 --- a/esphome/components/esp32/core.cpp +++ b/esphome/components/esp32/core.cpp @@ -2,6 +2,7 @@ #include "esphome/core/application.h" #include "esphome/core/defines.h" +#include "esphome/core/helpers.h" #include "preferences.h" #include #include @@ -29,6 +30,13 @@ void loop_task(void *pv_params) { } extern "C" void app_main() { + // Apply the custom eFuse MAC (if burned and valid) as the base MAC before any + // interface (Wi-Fi, Ethernet, Bluetooth, 802.15.4) derives its address from it. + // The logger does not exist yet, so only log-free helpers may be used here. + uint8_t mac[MAC_ADDRESS_SIZE]; + if (get_custom_mac_address(mac)) { + set_mac_address(mac); + } initArduino(); esp32::setup_preferences(); #if CONFIG_FREERTOS_UNICORE diff --git a/esphome/components/esp32/helpers.cpp b/esphome/components/esp32/helpers.cpp index c2ff6cf34d..91b4241211 100644 --- a/esphome/components/esp32/helpers.cpp +++ b/esphome/components/esp32/helpers.cpp @@ -71,23 +71,32 @@ static bool read_valid_mac(uint8_t *mac, esp_err_t err) { return err == ESP_OK & static constexpr size_t MAC_ADDRESS_SIZE_BITS = MAC_ADDRESS_SIZE * 8; // 48 bits +// Must not use the ESPHome logger (may run before it exists, e.g. from app_main()). +bool get_custom_mac_address(uint8_t *mac) { + // has_custom_mac_address() checks the raw eFuse field, while the reads below select their + // method differently and may still fail (CRC), so the result must be validated again. + if (!has_custom_mac_address()) + return false; +#if defined(CONFIG_SOC_IEEE802154_SUPPORTED) + return read_valid_mac(mac, esp_efuse_read_field_blob(ESP_EFUSE_MAC_CUSTOM, mac, MAC_ADDRESS_SIZE_BITS)); +#else + return read_valid_mac(mac, esp_efuse_mac_get_custom(mac)); +#endif +} + void get_mac_address_raw(uint8_t *mac) { // NOLINT(readability-non-const-parameter) + if (get_custom_mac_address(mac)) { + return; + } #if defined(CONFIG_SOC_IEEE802154_SUPPORTED) // When CONFIG_SOC_IEEE802154_SUPPORTED is defined, esp_efuse_mac_get_default // returns the 802.15.4 EUI-64 address, so we read directly from eFuse instead. - // Both paths already read raw eFuse bytes, so there is no CRC-bypass fallback + // This already reads raw eFuse bytes, so there is no CRC-bypass fallback // (unlike the non-IEEE802154 path where esp_efuse_mac_get_default does CRC checks). - if (has_custom_mac_address() && - read_valid_mac(mac, esp_efuse_read_field_blob(ESP_EFUSE_MAC_CUSTOM, mac, MAC_ADDRESS_SIZE_BITS))) { - return; - } if (read_valid_mac(mac, esp_efuse_read_field_blob(ESP_EFUSE_MAC_FACTORY, mac, MAC_ADDRESS_SIZE_BITS))) { return; } #else - if (has_custom_mac_address() && read_valid_mac(mac, esp_efuse_mac_get_custom(mac))) { - return; - } if (read_valid_mac(mac, esp_efuse_mac_get_default(mac))) { return; } diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 32d46887b6..06f0981020 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -140,11 +140,6 @@ void event_handler(void *arg, esp_event_base_t event_base, int32_t event_id, voi } void WiFiComponent::wifi_pre_setup_() { - uint8_t mac[MAC_ADDRESS_SIZE]; - if (has_custom_mac_address()) { - get_mac_address_raw(mac); - set_mac_address(mac); - } // Network interface setup handled by network component s_wifi_event_group = xEventGroupCreate(); if (s_wifi_event_group == nullptr) { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index e60316d4ee..9fdc088ecb 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -2089,6 +2089,11 @@ const char *get_mac_address_pretty_into_buffer(std::span Date: Wed, 26 Aug 2026 07:26:50 +0200 Subject: [PATCH 06/30] [climate_ir_lg] advanced vert. swing, setting temp. in heat/cool mode, jet mode decoding, other fixes + refactor (#10875) --- esphome/components/climate_ir_lg/climate.py | 3 + .../climate_ir_lg/climate_ir_lg.cpp | 330 ++++++++++++++---- .../components/climate_ir_lg/climate_ir_lg.h | 6 +- tests/components/climate_ir_lg/common.yaml | 3 + 4 files changed, 264 insertions(+), 78 deletions(-) diff --git a/esphome/components/climate_ir_lg/climate.py b/esphome/components/climate_ir_lg/climate.py index 48fd373b78..255fca9ad1 100644 --- a/esphome/components/climate_ir_lg/climate.py +++ b/esphome/components/climate_ir_lg/climate.py @@ -13,9 +13,11 @@ CONF_HEADER_LOW = "header_low" CONF_BIT_HIGH = "bit_high" CONF_BIT_ONE_LOW = "bit_one_low" CONF_BIT_ZERO_LOW = "bit_zero_low" +CONF_ADVANCED_COMMANDS_SUPPORT = "advanced_commands_support" CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(LgIrClimate).extend( { + cv.Optional(CONF_ADVANCED_COMMANDS_SUPPORT, default=False): cv.boolean, cv.Optional( CONF_HEADER_HIGH, default="8000us" ): cv.positive_time_period_microseconds, @@ -38,6 +40,7 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(LgIrClimate).extend( async def to_code(config: ConfigType) -> None: var = await climate_ir.new_climate_ir(config) + cg.add(var.set_advanced_commands_support(config[CONF_ADVANCED_COMMANDS_SUPPORT])) cg.add(var.set_header_high(config[CONF_HEADER_HIGH])) cg.add(var.set_header_low(config[CONF_HEADER_LOW])) cg.add(var.set_bit_high(config[CONF_BIT_HIGH])) diff --git a/esphome/components/climate_ir_lg/climate_ir_lg.cpp b/esphome/components/climate_ir_lg/climate_ir_lg.cpp index 588566dd9d..bb612eda7b 100644 --- a/esphome/components/climate_ir_lg/climate_ir_lg.cpp +++ b/esphome/components/climate_ir_lg/climate_ir_lg.cpp @@ -5,11 +5,85 @@ namespace esphome::climate_ir_lg { static const char *const TAG = "climate.climate_ir_lg"; -// Commands -const uint32_t COMMAND_MASK = 0xFF000; -const uint32_t COMMAND_OFF = 0xC0000; -const uint32_t COMMAND_SWING = 0x10000; +// All codes provided here are missing the checksum (last 4 bits) +// this checksum needs to be calculated before sending (look at `calc_checksum_()`) +const uint32_t LG_HEADER = 0x8800000; + +// Commands +const uint32_t COMMAND_HEADER_MASK = 0xFF000; +const uint32_t COMMAND_DATA_MASK = 0x00FF0; +const uint32_t CHECKSUM_MASK = 0xF; + +enum CommandBasic : uint32_t { + HEADER_BASIC = 0x10000, + BASIC_SWING_TOGGLE = 0x000, + + // JET MODE (only for cooling/drying/heating modes) + // For 30 minutes: max airflow (stronger than F5 aka FAN_MAX) + PO (min/min/max temperature respectively) + // After 30 minutes: F5 aka FAN_MAX + min/min/max temperature respectively + BASIC_JET = 0x080, +}; + +enum CommandSys : uint32_t { + HEADER_SYS = 0xC0000, + + COMMAND_OFF = 0x050, + + // Also known as 'auto-dry' + AUTO_CLEAN_ON = 0x0B0, + AUTO_CLEAN_OFF = 0x0C0, + + PURIFY_ON = 0x000, // From either OFF or Mode -> Purify + PURIFY_OFF = 0x080, // From Mode + Purify -> Mode + + QUIET_OUTDOOR_ON = 0xA60, + QUIET_OUTDOOR_OFF = 0xA70, + + // ENERGY CTRL (only in Cooling mode) + COOL_ENERG_CTRL_80 = 0x7D0, // 80% + COOL_ENERG_CTRL_60 = 0x7E0, // 60% + COOL_ENERG_CTRL_40 = 0x800, // 40% + COOL_ENERG_CTRL_OFF = 0x7F0, // OFF + + DISPLAY_KW = 0x460, + LIGHT_ON_OFF = 0x0A0, + + TEMP_UNIT_F = 0x170, + TEMP_UNIT_C = 0x160, +}; + +enum CommandAdvSwing : uint32_t { + HEADER_ADV_SWING = 0x13000, + + // Only 5 bits are relevant, I got 0x13952 once - not sure what is the 8th bit so ignoring that. + ADV_SWING_DATA_MASK = 0x1F0, + + // Commands for Advanced Vertical Control: Swing + 6 fixed positions + VERT_FIX_1 = 0x040, // Down + VERT_FIX_2 = 0x050, + VERT_FIX_3 = 0x060, + VERT_FIX_4 = 0x070, + VERT_FIX_5 = 0x080, + VERT_FIX_6 = 0x090, // Up + VERT_SWING_ON = 0x140, // Swing between 1 and 6 + VERT_SWING_OFF = 0x150, // Stops immediately + + // Commands for Advanced Horizontal Control: Swing (3 modes) + 5 fixed positions + HORI_FIX_1 = 0x0B0, // Left + HORI_FIX_2 = 0x0C0, + HORI_FIX_3 = 0x0D0, + HORI_FIX_4 = 0x0E0, + HORI_FIX_5 = 0x0F0, // Right + HORI_SWING_ON_LEFT = 0x100, // Swing between 1 and 3 + HORI_SWING_ON_RIGHT = 0x110, // Swing between 3 and 5 + HORI_SWING_ON_FULL = 0x160, // Swing between 1 and 5 + HORI_SWING_OFF = 0x170, // Stops immediately +}; + +// Following commands contain mode, fan speed and temperature + +// Modes const uint32_t COMMAND_ON_COOL = 0x00000; const uint32_t COMMAND_ON_DRY = 0x01000; const uint32_t COMMAND_ON_FAN_ONLY = 0x02000; @@ -23,11 +97,13 @@ const uint32_t COMMAND_AI = 0x0B000; const uint32_t COMMAND_HEAT = 0x0C000; // Fan speed -const uint32_t FAN_MASK = 0xF0; +const uint32_t FAN_SPEED_MASK = 0xF0; const uint32_t FAN_AUTO = 0x50; -const uint32_t FAN_MIN = 0x00; -const uint32_t FAN_MED = 0x20; -const uint32_t FAN_MAX = 0x40; +const uint32_t FAN_MIN = 0x00; // AKA F1 +const uint32_t FAN_F2 = 0x90; +const uint32_t FAN_MED = 0x20; // AKA F3 +const uint32_t FAN_F4 = 0xA0; +const uint32_t FAN_MAX = 0x40; // AKA F5 // Temperature const uint8_t TEMP_RANGE = TEMP_MAX - TEMP_MIN + 1; @@ -37,16 +113,37 @@ const uint32_t TEMP_SHIFT = 8; const uint16_t BITS = 28; void LgIrClimate::transmit_state() { - uint32_t remote_state = 0x8800000; + uint32_t remote_state = LG_HEADER; - // ESP_LOGD(TAG, "climate_lg_ir mode_before_ code: 0x%02X", modeBefore_); + // ESP_LOGD(TAG, "climate_lg_ir mode_before_ code: 0x%02X", this->modeBefore_); // Set command if (this->send_swing_cmd_) { this->send_swing_cmd_ = false; - remote_state |= COMMAND_SWING; - } else { - bool climate_is_off = (this->mode_before_ == climate::CLIMATE_MODE_OFF); + if (this->advanced_commands_support_) { + switch (this->swing_mode) { + case climate::CLIMATE_SWING_VERTICAL: + ESP_LOGD(TAG, "setting swing vertical"); + remote_state |= CommandAdvSwing::HEADER_ADV_SWING; + remote_state |= CommandAdvSwing::VERT_SWING_ON; + break; + case climate::CLIMATE_SWING_OFF: + ESP_LOGD(TAG, "setting swing off"); + remote_state |= CommandAdvSwing::HEADER_ADV_SWING; + remote_state |= CommandAdvSwing::VERT_SWING_OFF; + break; + default: + return; + } + this->transmit_(remote_state); + this->publish_state(); + return; + } else { // just toggle swing when advanced_commands_support is not set + remote_state |= HEADER_BASIC; + remote_state |= BASIC_SWING_TOGGLE; + } + } else { // Mode commands + const bool climate_is_off = (this->mode_before_ == climate::CLIMATE_MODE_OFF); switch (this->mode) { case climate::CLIMATE_MODE_COOL: remote_state |= climate_is_off ? COMMAND_ON_COOL : COMMAND_COOL; @@ -65,8 +162,8 @@ void LgIrClimate::transmit_state() { break; case climate::CLIMATE_MODE_OFF: default: - remote_state |= COMMAND_OFF; - break; + remote_state |= CommandSys::HEADER_SYS; + remote_state |= CommandSys::COMMAND_OFF; } } @@ -75,9 +172,8 @@ void LgIrClimate::transmit_state() { ESP_LOGD(TAG, "climate_lg_ir mode code: 0x%02X", this->mode); // Set fan speed - if (this->mode == climate::CLIMATE_MODE_OFF) { - remote_state |= FAN_AUTO; - } else { + if (this->mode != + climate::CLIMATE_MODE_OFF) { // https://github.com/esphome/esphome/pull/10875#issuecomment-5042765948 switch (this->fan_mode.value_or(climate::CLIMATE_FAN_ON)) { case climate::CLIMATE_FAN_HIGH: remote_state |= FAN_MAX; @@ -95,10 +191,20 @@ void LgIrClimate::transmit_state() { } } - // Set temperature - if (this->mode == climate::CLIMATE_MODE_COOL || this->mode == climate::CLIMATE_MODE_HEAT) { - auto temp = (uint8_t) roundf(clamp(this->target_temperature, TEMP_MIN, TEMP_MAX)); - remote_state |= ((temp - 15) << TEMP_SHIFT); + uint8_t temp; + switch (this->mode) { + case climate::CLIMATE_MODE_HEAT_COOL: + if (!this->advanced_commands_support_) { // Keep previous behavior + break; + } + [[fallthrough]]; + case climate::CLIMATE_MODE_COOL: + case climate::CLIMATE_MODE_HEAT: + temp = static_cast(roundf(clamp(this->target_temperature, TEMP_MIN, TEMP_MAX))); + remote_state |= (temp - 15) << TEMP_SHIFT; + break; + default: + break; } this->transmit_(remote_state); @@ -124,62 +230,134 @@ bool LgIrClimate::on_receive(remote_base::RemoteReceiveData data) { } } - ESP_LOGD(TAG, "Decoded 0x%02" PRIX32, remote_state); - if ((remote_state & 0xFF00000) != 0x8800000) + ESP_LOGD(TAG, "Received 0x%02" PRIX32, remote_state); + if ((remote_state & 0xFF00000) != LG_HEADER) return false; - // Get command - if ((remote_state & COMMAND_MASK) == COMMAND_OFF) { - this->mode = climate::CLIMATE_MODE_OFF; - } else if ((remote_state & COMMAND_MASK) == COMMAND_SWING) { - this->swing_mode = - this->swing_mode == climate::CLIMATE_SWING_OFF ? climate::CLIMATE_SWING_VERTICAL : climate::CLIMATE_SWING_OFF; - } else { - switch (remote_state & COMMAND_MASK) { - case COMMAND_DRY: - case COMMAND_ON_DRY: - this->mode = climate::CLIMATE_MODE_DRY; - break; - case COMMAND_FAN_ONLY: - case COMMAND_ON_FAN_ONLY: - this->mode = climate::CLIMATE_MODE_FAN_ONLY; - break; - case COMMAND_AI: - case COMMAND_ON_AI: - this->mode = climate::CLIMATE_MODE_HEAT_COOL; - break; - case COMMAND_HEAT: - case COMMAND_ON_HEAT: - this->mode = climate::CLIMATE_MODE_HEAT; - break; - case COMMAND_COOL: - case COMMAND_ON_COOL: - default: - this->mode = climate::CLIMATE_MODE_COOL; - break; - } - - // Get fan speed - if (this->mode == climate::CLIMATE_MODE_HEAT_COOL) { - this->fan_mode = climate::CLIMATE_FAN_AUTO; - } else if (this->mode == climate::CLIMATE_MODE_COOL || this->mode == climate::CLIMATE_MODE_DRY || - this->mode == climate::CLIMATE_MODE_FAN_ONLY || this->mode == climate::CLIMATE_MODE_HEAT) { - if ((remote_state & FAN_MASK) == FAN_AUTO) { - this->fan_mode = climate::CLIMATE_FAN_AUTO; - } else if ((remote_state & FAN_MASK) == FAN_MIN) { - this->fan_mode = climate::CLIMATE_FAN_LOW; - } else if ((remote_state & FAN_MASK) == FAN_MED) { - this->fan_mode = climate::CLIMATE_FAN_MEDIUM; - } else if ((remote_state & FAN_MASK) == FAN_MAX) { - this->fan_mode = climate::CLIMATE_FAN_HIGH; + // Decode commands + switch (remote_state & COMMAND_HEADER_MASK) { + case CommandSys::HEADER_SYS: + ESP_LOGD(TAG, "Got system command! With data: 0x%02" PRIX32, remote_state & COMMAND_DATA_MASK); + if ((remote_state & COMMAND_DATA_MASK) == CommandSys::COMMAND_OFF) { + this->mode = climate::CLIMATE_MODE_OFF; + } else { + return false; + } + break; + case CommandAdvSwing::HEADER_ADV_SWING: + ESP_LOGD(TAG, "Got advanced swing command! With data: 0x%02" PRIX32, + remote_state & CommandAdvSwing::ADV_SWING_DATA_MASK); + switch (remote_state & CommandAdvSwing::ADV_SWING_DATA_MASK) { + case CommandAdvSwing::VERT_SWING_ON: + this->swing_mode = climate::CLIMATE_SWING_VERTICAL; + break; + case CommandAdvSwing::VERT_SWING_OFF: + case CommandAdvSwing::VERT_FIX_1: + case CommandAdvSwing::VERT_FIX_2: + case CommandAdvSwing::VERT_FIX_3: + case CommandAdvSwing::VERT_FIX_4: + case CommandAdvSwing::VERT_FIX_5: + case CommandAdvSwing::VERT_FIX_6: + this->swing_mode = climate::CLIMATE_SWING_OFF; + break; + default: + return false; // Ignore all other (horizontal) swing commands } - } - // Get temperature - if (this->mode == climate::CLIMATE_MODE_COOL || this->mode == climate::CLIMATE_MODE_HEAT) { - this->target_temperature = ((remote_state & TEMP_MASK) >> TEMP_SHIFT) + 15; - } + this->publish_state(); + return true; + + case HEADER_BASIC: + if ((remote_state & COMMAND_DATA_MASK) == BASIC_JET) { + switch (this->mode) { + case climate::CLIMATE_MODE_COOL: + case climate::CLIMATE_MODE_HEAT: + case climate::CLIMATE_MODE_DRY: + this->target_temperature = + this->mode == climate::CLIMATE_MODE_HEAT ? this->maximum_temperature_ : this->minimum_temperature_; + this->fan_mode = climate::CLIMATE_FAN_HIGH; + // When enabling PO(WER) also known as JET mode, swing is set to VERT_3, but after 30 mins it will switch + // back to what it was before, so let's just not change it here it at all + this->publish_state(); + return true; + default: + ESP_LOGD(TAG, "Got jet command, but current mode does not support it! Ignoring."); + return false; + } + } + + // Keep previous behavior in case of other BASIC command + if (this->swing_mode == climate::CLIMATE_SWING_OFF) { // Just flip between vertical and off + this->swing_mode = climate::CLIMATE_SWING_VERTICAL; + } else { + this->swing_mode = climate::CLIMATE_SWING_OFF; + } + this->publish_state(); + return true; + // Following commands also contain fan speed and temperature, so no 'return' in these cases + case COMMAND_DRY: + case COMMAND_ON_DRY: + this->mode = climate::CLIMATE_MODE_DRY; + break; + case COMMAND_FAN_ONLY: + case COMMAND_ON_FAN_ONLY: + this->mode = climate::CLIMATE_MODE_FAN_ONLY; + break; + case COMMAND_AI: + case COMMAND_ON_AI: + this->mode = climate::CLIMATE_MODE_HEAT_COOL; + break; + case COMMAND_HEAT: + case COMMAND_ON_HEAT: + this->mode = climate::CLIMATE_MODE_HEAT; + break; + case COMMAND_COOL: + case COMMAND_ON_COOL: + this->mode = climate::CLIMATE_MODE_COOL; + break; + default: + ESP_LOGD(TAG, "Got unknown command! Ignoring!"); + return false; } + + // Decode fan speed + switch (remote_state & FAN_SPEED_MASK) { + case FAN_AUTO: + this->fan_mode = climate::CLIMATE_FAN_AUTO; + break; + case FAN_MIN: + case FAN_F2: + this->fan_mode = climate::CLIMATE_FAN_LOW; + break; + case FAN_MED: + case FAN_F4: + this->fan_mode = climate::CLIMATE_FAN_MEDIUM; + break; + case FAN_MAX: + this->fan_mode = climate::CLIMATE_FAN_HIGH; + break; + default: + ESP_LOGD(TAG, "Got unknown fan speed! Ignoring!"); + return false; + } + + // Keep previous behavior + if (this->mode == climate::CLIMATE_MODE_HEAT_COOL && !(this->advanced_commands_support_)) { + this->fan_mode = climate::CLIMATE_FAN_AUTO; + } + + // Decode temperature for modes that support it + switch (this->mode) { + case climate::CLIMATE_MODE_HEAT_COOL: + case climate::CLIMATE_MODE_COOL: + case climate::CLIMATE_MODE_HEAT: + this->target_temperature = ((remote_state & TEMP_MASK) >> TEMP_SHIFT) + 15; + break; + default: + break; + } + + this->mode_before_ = this->mode; this->publish_state(); return true; @@ -207,14 +385,14 @@ void LgIrClimate::transmit_(uint32_t value) { data->mark(this->bit_high_); transmit.perform(); } + void LgIrClimate::calc_checksum_(uint32_t &value) { - uint32_t mask = 0xF; uint32_t sum = 0; for (uint8_t i = 1; i < 8; i++) { - sum += (value & (mask << (i * 4))) >> (i * 4); + sum += (value & (CHECKSUM_MASK << (i * 4))) >> (i * 4); } - value |= (sum & mask); + value |= (sum & CHECKSUM_MASK); } } // namespace esphome::climate_ir_lg diff --git a/esphome/components/climate_ir_lg/climate_ir_lg.h b/esphome/components/climate_ir_lg/climate_ir_lg.h index 341f0a4ef1..c9c0c0c005 100644 --- a/esphome/components/climate_ir_lg/climate_ir_lg.h +++ b/esphome/components/climate_ir_lg/climate_ir_lg.h @@ -21,12 +21,13 @@ class LgIrClimate final : public climate_ir::ClimateIR { /// Override control to change settings of the climate device. void control(const climate::ClimateCall &call) override { this->send_swing_cmd_ = call.get_swing_mode().has_value(); - // swing resets after unit powered off + // swing resets after unit powered off, except when advanced_commands_support_ is set auto mode = call.get_mode(); - if (mode.has_value() && *mode == climate::CLIMATE_MODE_OFF) + if (mode.has_value() && *mode == climate::CLIMATE_MODE_OFF && !(this->advanced_commands_support_)) this->swing_mode = climate::CLIMATE_SWING_OFF; climate_ir::ClimateIR::control(call); } + void set_advanced_commands_support(bool value) { this->advanced_commands_support_ = value; } void set_header_high(uint32_t header_high) { this->header_high_ = header_high; } void set_header_low(uint32_t header_low) { this->header_low_ = header_low; } void set_bit_high(uint32_t bit_high) { this->bit_high_ = bit_high; } @@ -44,6 +45,7 @@ class LgIrClimate final : public climate_ir::ClimateIR { void calc_checksum_(uint32_t &value); void transmit_(uint32_t value); + bool advanced_commands_support_{false}; uint32_t header_high_; uint32_t header_low_; uint32_t bit_high_; diff --git a/tests/components/climate_ir_lg/common.yaml b/tests/components/climate_ir_lg/common.yaml index e0bc185d2c..5536c36742 100644 --- a/tests/components/climate_ir_lg/common.yaml +++ b/tests/components/climate_ir_lg/common.yaml @@ -12,5 +12,8 @@ climate: - platform: climate_ir_lg name: LG Climate transmitter_id: xmitr + header_high: 3300us + header_low: 9840us + advanced_commands_support: true sensor: climate_ir_lg_temp_sensor humidity_sensor: humidity_sensor From 1e7c48e2cfc6eac01ec515b34be57cde768d67d4 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Wed, 26 Aug 2026 06:35:46 -0700 Subject: [PATCH 07/30] [modbus_controller] Add integration tests for register offset, response size and write buffer (#18741) --- ...t_mock_modbus_deprecated_write_buffer.yaml | 106 ++++++++++++++ .../uart_mock_modbus_register_offset.yaml | 138 ++++++++++++++++++ tests/integration/test_uart_mock_modbus.py | 119 ++++++++++++++- 3 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml create mode 100644 tests/integration/fixtures/uart_mock_modbus_register_offset.yaml diff --git a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml b/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml new file mode 100644 index 0000000000..f378e3de43 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml @@ -0,0 +1,106 @@ +esphome: + name: uart-mock-modbus-dep-buffer + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg10 + type: uint16_t + initial_value: "0" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x10 + value_type: U_WORD + read_lambda: return id(reg10); + write_lambda: |- + id(reg10) = x; + return true; + +# A number whose write_lambda uses the DEPRECATED buffer parameter (fills `payload` with a legacy raw +# frame as words: device address + function code + data) instead of the new item->write_* API. The write +# must still land with its legacy semantics, and the one-time deprecation warning must fire only once per +# entity no matter how many writes happen. +number: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "buf_number" + id: buf_number + address: 0x10 + register_type: holding + value_type: U_WORD + min_value: 0 + max_value: 1000 + step: 1 + write_lambda: |- + // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0010, value. + payload.push_back(0x0106); + payload.push_back(0x0010); + payload.push_back((uint16_t) x); + return {}; + +# Reports the server-side register so the test can observe that the deprecated buffer write landed. +sensor: + - platform: template + name: "written_value" + id: written_value + update_interval: 0.5s + lambda: "return id(reg10);" + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # The test drives the writes via number_command; the mock is autostart. diff --git a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml b/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml new file mode 100644 index 0000000000..e93e78d5a3 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml @@ -0,0 +1,138 @@ +esphome: + name: uart-mock-modbus-reg-offset + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg10 + type: uint16_t + initial_value: "100" + - id: reg11 + type: uint16_t + initial_value: "200" + - id: reg12 + type: uint16_t + initial_value: "300" + - id: reg13 + type: uint16_t + initial_value: "0xABCD" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x10 + value_type: U_WORD + read_lambda: return id(reg10); + write_lambda: id(reg10) = x; return true; + - address: 0x11 + value_type: U_WORD + read_lambda: return id(reg11); + write_lambda: id(reg11) = x; return true; + - address: 0x12 + value_type: U_WORD + read_lambda: return id(reg12); + write_lambda: id(reg12) = x; return true; + - address: 0x13 + value_type: U_WORD + read_lambda: return id(reg13); + write_lambda: id(reg13) = x; return true; + +# A holding-register switch at 0x10 with a 2-BYTE offset. offset is byte-based, so the write must target +# register 0x10 + 2/2 = 0x11. The old (pre-fix) behavior folded offset into the address as a register +# count, hitting 0x12 instead. assumed_state keeps the switch write-only so it does not read any register. +switch: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "offset_switch" + register_type: holding + address: 0x10 + offset: 2 + assumed_state: true + # A holding-register switch that READS its state. Byte offset 6 -> register 0x10 + 6/2 = 0x13. Post-fix + # the switch itself resolves to 0x13 (whole registers fold into the address, residual byte stays) and + # joins the 0x10..0x13 range, so no separate 0x13 sensor is needed. Pre-fix the whole byte offset folds + # into the address (0x16), where the server answers ILLEGAL_DATA_ADDRESS and the switch never publishes. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "read_offset_switch" + register_type: holding + address: 0x10 + offset: 6 + bitmask: 0x1 + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_10" + address: 0x10 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_11" + address: 0x11 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_12" + address: 0x12 + register_type: holding + value_type: U_WORD + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index c84fb34e70..707637cfc2 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -24,7 +24,7 @@ from dataclasses import dataclass from aioesphomeapi import ButtonInfo, NumberInfo, SwitchInfo import pytest -from .state_utils import SensorTracker, find_entity +from .state_utils import SensorTracker, find_entity, wait_for_state from .types import APIClientConnectedFactory, RunCompiledFunction @@ -965,3 +965,120 @@ async def test_uart_mock_modbus_client_read_write( await tracker.setup_and_start_scenario(client) await tracker.await_all(futures) _assert_no_modbus_errors(error_log_lines, warning_log_lines) + + +@pytest.mark.xfail( + strict=True, + reason="Byte-accurate register-offset writes require the modbus_controller " + "entity-device change; on dev the byte offset is folded into the address " + "(writes 0x12 instead of 0x11). The write and read assertions both flip via " + "the same switch-constructor fold. Remove this marker when that change merges.", +) +@pytest.mark.asyncio +async def test_uart_mock_modbus_register_offset( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that a byte offset on a holding-register write is byte-accurate. + + `offset` is a byte offset, so a holding-register write at address 0x10 with offset: 2 must target + register 0x10 + 2/2 = 0x11. The pre-fix behavior folded the byte offset into the address as a register + count (0x10 + 2 = 0x12). The switch is assumed_state (write-only), so reg_11 turning 0xFFFF pins the + fix; had the write landed on 0x12 the wait would time out and reg_12 would change instead. + """ + + tracker = SensorTracker(["reg_10", "reg_11", "reg_12"]) + initial = tracker.expect_all({"reg_10": 100, "reg_11": 200, "reg_12": 300}) + wrote_11 = tracker.expect("reg_11", 65535) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + await tracker.await_all(initial, timeout=4.0) + + switch = find_entity(entities, "offset_switch", SwitchInfo) + assert switch is not None, "offset_switch not found" + client.switch_command(switch.key, True) + + # reg_11 (0x10 + offset 2/2) must receive the write; if the write went to 0x12 this times out. + await tracker.await_change(wrote_11, "reg_11", timeout=4.0) + # And 0x12 (the pre-fix register-offset target) must be untouched. + assert tracker.sensor_states["reg_12"][-1] == 300, ( + "reg_12 (0x12) should be untouched - offset is byte-based, so the write targets 0x11; " + f"got {tracker.sensor_states['reg_12']}" + ) + + # Read path: read_offset_switch has byte offset 6. Post-fix the switch folds the whole registers + # into its address (0x10 + 6/2 = 0x13, residual byte 0) and joins the 0x10..0x13 range, so the + # read lands in-bounds on 0xABCD (bit 0 set) -> ON. Pre-fix the whole byte offset folded into the + # address (0x16); the server answers ILLEGAL_DATA_ADDRESS there and the switch never publishes. + read_switch = find_entity(entities, "read_offset_switch", SwitchInfo) + assert read_switch is not None, "read_offset_switch not found" + # The ON transition happened at the first poll and switch states are deduped, so this relies on + # wait_for_state's fresh subscribe_states re-dumping every entity's current state. + await wait_for_state( + client, + lambda s: ( + getattr(s, "key", None) == read_switch.key + and getattr(s, "state", None) is True + ), + timeout=6.0, + ) + + +@pytest.mark.xfail( + strict=True, + reason="The deprecated write buffer requires the modbus_controller " + "entity-device change; on dev a nullopt-returning write_lambda early-returns " + "before the buffer is used, so the write never happens. The warn-once " + "assertion matches the log substring 'write_lambda buffer'. Remove this " + "marker when that change merges.", +) +@pytest.mark.asyncio +async def test_uart_mock_modbus_deprecated_write_buffer( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test the deprecated write_lambda buffer path still works, and warns once per entity. + + buf_number's write_lambda fills the old `payload` buffer with a legacy raw frame as words (device + address + function code + data) instead of calling item->write_*. Two writes must both land with the + legacy raw-frame semantics, and the one-time deprecation warning must fire exactly once per entity + regardless of how many writes happen. + """ + + warn_count = 0 + + def line_callback(line: str) -> None: + nonlocal warn_count + if "write_lambda buffer" in line: + warn_count += 1 + + tracker = SensorTracker(["written_value"]) + + async with ( + run_compiled(yaml_config, line_callback=line_callback), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + number = find_entity(entities, "buf_number", NumberInfo) + assert number is not None, "buf_number not found" + + # First write via the deprecated buffer path. + client.number_command(number.key, 111) + await tracker.await_change( + tracker.expect("written_value", 111), "written_value", timeout=4.0 + ) + # Second write: lands too, but must not warn again (warn-once per entity). + client.number_command(number.key, 222) + await tracker.await_change( + tracker.expect("written_value", 222), "written_value", timeout=4.0 + ) + + assert warn_count == 1, ( + f"deprecation warning should fire exactly once per entity, got {warn_count}" + ) From 4eb85a24c20f5eab14ccefb487e196832fafe890 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:04:57 +1000 Subject: [PATCH 08/30] [mipi_spi] Fix dimensions for jc3636518v2 (#18786) --- esphome/components/mipi_spi/models/jc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/esphome/components/mipi_spi/models/jc.py b/esphome/components/mipi_spi/models/jc.py index ca9adb4a72..8d2591aefe 100644 --- a/esphome/components/mipi_spi/models/jc.py +++ b/esphome/components/mipi_spi/models/jc.py @@ -266,8 +266,6 @@ DriverChip( "JC3636W518V2", height=360, width=360, - offset_height=1, - draw_rounding=1, cs_pin=10, reset_pin=47, invert_colors=True, From 7b4894da03677670e5f2dc712599e1a280801e9c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 10:16:44 -0500 Subject: [PATCH 09/30] [mdns] Skip MDNS.update() while the ESP8266 radio cannot transmit (#18785) --- esphome/components/mdns/mdns_esp8266.cpp | 14 +++++++++++++- esphome/components/wifi/wifi_component.cpp | 6 +++--- esphome/components/wifi/wifi_component.h | 7 +++++++ esphome/components/wifi/wifi_component_esp8266.cpp | 2 +- esphome/components/wifi/wifi_component_esp_idf.cpp | 2 +- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index f6d5786675..1f0b3c9519 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -41,7 +41,19 @@ static void register_esp8266(MDNSComponent *, StaticVectorset_interval(MDNS_POLL_ID, MDNS_UPDATE_INTERVAL_MS, []() { MDNS.update(); }); + this->set_interval(MDNS_POLL_ID, MDNS_UPDATE_INTERVAL_MS, []() { +#ifdef USE_MDNS_WIFI_LISTENER + // MDNS.update() can suspend the loop in UdpContext::sendTimeout() while a send is + // failing (radio off-channel during a roam scan, or mid reconnect); an incoming + // packet then re-enters LEAmDNS from lwIP and corrupts shared UdpContext state. + // Skip the tick while the radio cannot transmit (#18760), but keep polling while + // the AP is serving clients (AP-only or fallback AP with the STA down). + auto *wifi = wifi::global_wifi_component; + if (wifi->is_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) + return; +#endif + MDNS.update(); + }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } #endif diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index b8a31f97a3..d82929e5cb 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -530,7 +530,7 @@ void WiFiComponent::log_discarded_scan_result_(const char *ssid, const uint8_t * #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE // Skip logging during roaming scans to avoid log buffer overflow // (roaming scans typically find many networks but only care about same-SSID APs) - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { return; } char bssid_s[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; @@ -833,7 +833,7 @@ void WiFiComponent::loop() { // Post-connect roaming: check for better AP if (this->post_connect_roaming_) { - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { if (this->scan_done_) { this->process_roaming_scan_(); } @@ -2144,7 +2144,7 @@ void WiFiComponent::retry_connect() { // Roam connection failed - transition to reconnecting ESP_LOGD(TAG, "Roam failed, reconnecting (attempt %u/%u)", this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); this->roaming_state_ = RoamingState::RECONNECTING; - } else if (this->roaming_state_ == RoamingState::SCANNING) { + } else if (this->is_roaming_scan_active()) { // Disconnected during roam scan - transition to RECONNECTING so the attempts // counter is preserved when reconnection succeeds (IDLE would reset it) ESP_LOGD(TAG, "Disconnected during roam scan (attempt %u/%u)", this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 382d3d5932..cfdbc1a968 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -478,6 +478,13 @@ class WiFiComponent final : public Component { bool is_connected() const { return this->connected_; } + /// True while a post-connect roaming scan holds the radio off-channel. + bool is_roaming_scan_active() const { return this->roaming_state_ == RoamingState::SCANNING; } + + /// True while a post-connect roam is in progress (scanning off-channel, reassociating, + /// or recovering from a failed roam). + bool is_roaming() const { return this->roaming_state_ != RoamingState::IDLE; } + #ifdef USE_ESP32 /// esp_netif handle of the station interface, used by network for default-route /// arbitration. nullptr until wifi_lazy_init_() has run. diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 005d655d88..b4a91fb3cd 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -717,7 +717,7 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { static constexpr uint32_t SCAN_ACTIVE_MAX_DEFAULT_MS = 500; static constexpr uint32_t SCAN_ACTIVE_MIN_ROAMING_MS = 100; static constexpr uint32_t SCAN_ACTIVE_MAX_ROAMING_MS = 300; - bool roaming = this->roaming_state_ == RoamingState::SCANNING; + bool roaming = this->is_roaming_scan_active(); if (passive) { config.scan_time.passive = roaming ? SCAN_PASSIVE_ROAMING_MS : SCAN_PASSIVE_DEFAULT_MS; } else { diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 06f0981020..ce75d21330 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -1059,7 +1059,7 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { // When scanning while connected (roaming), return to home channel between // each scanned channel to maintain the connection (helps with BLE/WiFi coexistence) #ifdef CONFIG_SOC_WIFI_SUPPORTED - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { config.coex_background_scan = true; } #endif From 5c79c92c0657ef8ae9cee1745bddbe3e7e0fc439 Mon Sep 17 00:00:00 2001 From: guillempages Date: Wed, 26 Aug 2026 18:26:39 +0200 Subject: [PATCH 10/30] [runtime_image] Add FILTER_SOURCE_FILES (#18768) --- esphome/components/runtime_image/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/esphome/components/runtime_image/__init__.py b/esphome/components/runtime_image/__init__.py index 9277c214ff..a220503045 100644 --- a/esphome/components/runtime_image/__init__.py +++ b/esphome/components/runtime_image/__init__.py @@ -10,6 +10,7 @@ from esphome.components.image import ( validate_transparency, validate_type, ) +from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv from esphome.const import CONF_FORMAT, CONF_ID, CONF_RESIZE, CONF_TYPE from esphome.core import CORE @@ -124,6 +125,15 @@ IMAGE_FORMATS = { "PNG": PNGFormat(), } +FILTER_SOURCE_FILES = filter_source_files_from_defines( + { + "bmp_decoder.cpp": "USE_RUNTIME_IMAGE_BMP", + "jpeg_decoder.cpp": "USE_RUNTIME_IMAGE_JPEG", + "png_decoder.cpp": "USE_RUNTIME_IMAGE_PNG", + "qoi_decoder.cpp": "USE_RUNTIME_IMAGE_QOI", + } +) + AUTO_FORMAT = AUTOFormat() From 1cbe3a49b2bf9375f0c778f9d44084c12ec3eaea Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Wed, 26 Aug 2026 11:30:07 -0500 Subject: [PATCH 11/30] [remote_transmitter] ISR-driven transmission and non_blocking support on BK7231N/BK7238 (#18660) --- .../components/remote_transmitter/__init__.py | 26 +- .../remote_transmitter/remote_transmitter.cpp | 4 +- .../remote_transmitter/remote_transmitter.h | 44 +++- .../remote_transmitter_bk72xx.cpp | 187 +++++++++++++++ .../remote_transmitter_libretiny_isr.cpp | 224 ++++++++++++++++++ .../remote_transmitter_rtl87xx.cpp | 212 ++--------------- .../test_non_blocking_gate.py | 6 + .../remote_transmitter/test.bk72xx-ard.yaml | 1 + 8 files changed, 497 insertions(+), 207 deletions(-) create mode 100644 esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp create mode 100644 esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index 8ae51829e7..cb2aebec91 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -4,7 +4,11 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base from esphome.components.libretiny import get_libretiny_family -from esphome.components.libretiny.const import FAMILY_RTL8720C +from esphome.components.libretiny.const import ( + FAMILY_BK7231N, + FAMILY_BK7238, + FAMILY_RTL8720C, +) from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -45,14 +49,19 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) +_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7231N, FAMILY_BK7238) + + def _validate_non_blocking_platform(value: bool) -> bool: - # non_blocking requires hardware transmission: RMT on ESP32, the gtimer - # envelope chain on RTL8720C. Reject everywhere else at config time. + # non_blocking requires hardware transmission: RMT on ESP32, a hardware timer + # envelope chain on the listed LibreTiny families. Reject elsewhere at config time. if CORE.is_esp32: return cv.boolean(value) - if CORE.is_libretiny and get_libretiny_family() == FAMILY_RTL8720C: + if CORE.is_libretiny and get_libretiny_family() in _NON_BLOCKING_LIBRETINY_FAMILIES: return cv.boolean(value) - raise cv.Invalid("non_blocking is only supported on ESP32 and RTL8720C") + raise cv.Invalid( + "non_blocking is only supported on ESP32, RTL8720C, BK7231N and BK7238" + ) MULTI_CONF = True @@ -202,6 +211,13 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( "remote_transmitter_rtl87xx.cpp": { PlatformFramework.RTL87XX_ARDUINO, }, + "remote_transmitter_bk72xx.cpp": { + PlatformFramework.BK72XX_ARDUINO, + }, + "remote_transmitter_libretiny_isr.cpp": { + PlatformFramework.RTL87XX_ARDUINO, + PlatformFramework.BK72XX_ARDUINO, + }, "remote_transmitter.cpp": { PlatformFramework.ESP32_ARDUINO, PlatformFramework.ESP32_IDF, diff --git a/esphome/components/remote_transmitter/remote_transmitter.cpp b/esphome/components/remote_transmitter/remote_transmitter.cpp index 67341e936f..5e82213a48 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter.cpp @@ -2,8 +2,8 @@ #include "esphome/core/log.h" #include "esphome/core/application.h" -#if (defined(USE_LIBRETINY) && !defined(USE_RTL87XX)) || defined(USE_ESP8266) || defined(USE_RP2) || \ - (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) +#if (defined(USE_LIBRETINY) && !defined(USE_RTL87XX) && !defined(REMOTE_TRANSMITTER_BK_PWM)) || \ + defined(USE_ESP8266) || defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) namespace esphome::remote_transmitter { diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index ef9a80f668..313b26364d 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -12,6 +12,13 @@ #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 +// The BK7231N-style PWM block (hardware shadow-load duty updates) enables the ISR-driven +// transmitter on these families; family-level proxy for the SDK's CFG_SOC_NAME gate. +// See remote_transmitter_bk72xx.cpp. +#if defined(USE_LIBRETINY_VARIANT_BK7231N) || defined(USE_LIBRETINY_VARIANT_BK7238) +#define REMOTE_TRANSMITTER_BK_PWM +#endif + namespace esphome::remote_transmitter { #if defined(USE_ESP32) && SOC_RMT_SUPPORTED @@ -57,13 +64,16 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa void set_with_dma(bool with_dma) { this->with_dma_ = with_dma; } void set_eot_level(bool eot_level) { this->eot_level_ = eot_level; } #endif -#if (defined(USE_ESP32) && SOC_RMT_SUPPORTED) || defined(USE_LIBRETINY_VARIANT_RTL8720C) +#if (defined(USE_ESP32) && SOC_RMT_SUPPORTED) || defined(USE_LIBRETINY_VARIANT_RTL8720C) || \ + defined(REMOTE_TRANSMITTER_BK_PWM) void set_non_blocking(bool non_blocking) { this->non_blocking_ = non_blocking; } #endif -#ifdef USE_LIBRETINY_VARIANT_RTL8720C +#if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) void loop() override; // called from the envelope timer ISR trampoline; not part of the public API void advance_envelope_isr(); + // same, for trampolines whose SDK callback carries no user argument + static void advance_active_isr(); #endif Trigger<> *get_transmit_trigger() { return &this->transmit_trigger_; } @@ -71,12 +81,14 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa protected: void send_internal(uint32_t send_times, uint32_t send_wait) override; -#if defined(USE_ESP8266) || (defined(USE_LIBRETINY) && !defined(USE_LIBRETINY_VARIANT_RTL8720C)) || \ +#if defined(USE_ESP8266) || \ + (defined(USE_LIBRETINY) && !defined(USE_LIBRETINY_VARIANT_RTL8720C) && !defined(REMOTE_TRANSMITTER_BK_PWM)) || \ defined(USE_RP2) || (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) void await_target_time_(); uint32_t target_time_{0}; #endif -#if defined(USE_ESP8266) || (defined(USE_LIBRETINY) && !defined(USE_RTL87XX)) || defined(USE_RP2) || \ +#if defined(USE_ESP8266) || \ + (defined(USE_LIBRETINY) && !defined(USE_RTL87XX) && !defined(REMOTE_TRANSMITTER_BK_PWM)) || defined(USE_RP2) || \ (defined(USE_ESP32) && !SOC_RMT_SUPPORTED) void calculate_on_off_time_(uint32_t carrier_frequency, uint32_t *on_time_period, uint32_t *off_time_period); @@ -89,17 +101,22 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa uint32_t current_carrier_frequency_{0}; void *pwm_{nullptr}; // pwmout_t*, opaque here to keep the SDK header out of this shared header #endif -#ifdef USE_LIBRETINY_VARIANT_RTL8720C +#if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) + // Envelope chain, shared by every family that paces transmission from a hardware timer + // (remote_transmitter_libretiny_isr.cpp) void start_isr_item_(size_t index); void arm_envelope_timer_(uint32_t duration_us); void abort_stalled_chain_(); void deliver_completion_(); void wait_until_idle_(); void arm_chain_(uint32_t send_times, uint32_t send_wait); - void update_carrier_(uint32_t carrier_frequency); + // Hooks implemented per family: everything the chain needs from the hardware + bool envelope_ready_() const; // PWM claimed successfully in setup() + void prepare_carrier_(uint32_t carrier_frequency); // retune period, stage mark/space levels + void write_envelope_level_(bool mark); // drive carrier (mark) or idle (space) + void arm_one_shot_(uint32_t duration_us); // fire advance_envelope_isr after duration_us + void stop_envelope_timer_(); std::vector isr_data_; // owned copy of the frame; temp_ may be re-encoded mid-flight - float isr_mark_duty_{0.0f}; - float isr_space_duty_{0.0f}; volatile size_t isr_index_{0}; volatile uint32_t isr_repeats_left_{0}; uint32_t isr_send_wait_{0}; @@ -110,6 +127,17 @@ class RemoteTransmitterComponent final : public remote_base::RemoteTransmitterBa bool complete_pending_{false}; bool stall_aborted_{false}; // this transmission ended via abort; blocks warning clear #endif +#ifdef USE_LIBRETINY_VARIANT_RTL8720C + float isr_mark_duty_{0.0f}; + float isr_space_duty_{0.0f}; +#endif +#ifdef REMOTE_TRANSMITTER_BK_PWM + void write_pwm_t1_(uint32_t t1_counts); + uint32_t isr_mark_t1_{0}; + uint32_t isr_space_t1_{0}; + uint32_t isr_period_t4_{684}; // 26MHz counts; ~38kHz default until a send sets the real carrier + int8_t pwm_channel_{-1}; +#endif #if defined(USE_ESP32) && SOC_RMT_SUPPORTED void configure_rmt_(); diff --git a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp new file mode 100644 index 0000000000..0081ae47b3 --- /dev/null +++ b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp @@ -0,0 +1,187 @@ +#include "remote_transmitter.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +// clang-tidy cannot parse the Beken SDK headers pulled in via ArduinoPrivate.h +#if defined(USE_BK72XX) && !defined(CLANG_TIDY) + +// ArduinoPrivate.h = Arduino.h + the BDK SDK headers (pwm_pub.h, bk_timer_pub.h, icu_pub.h) +// with the core's fixes for type-name collisions between the two +#include + +// Only the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) +// supports glitch-free per-edge duty updates; older SoCs compile the generic bit-bang +// implementation (remote_transmitter.cpp) instead, and this file compiles to nothing. +// REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h. + +namespace esphome::remote_transmitter { + +static const char *const TAG = "remote_transmitter"; + +#ifdef REMOTE_TRANSMITTER_BK_PWM + +// PWM peripheral carrier (26MHz block), envelope paced by a BKTIMER1 interrupt chain: each +// interrupt writes the next duty through the shadow registers (T1..T4 + CFG_UPDATA hardware +// load, glitch-free at the next carrier period). Direct register writes beat the driver's +// pwm_update_param() (~19us vs ~26us edge error) and have no shared state to race against. +// BKTIMER1 is the only free channel: TIMER0 = FreeRTOS tick, TIMER2 = SDK cal, TIMER4 = wdt. + +static constexpr uint32_t REG_PWM_BASE = 0x00802B00UL; +static constexpr uint32_t REG_PWM_GROUP_STRIDE = 0x40; // one register group per channel pair +static constexpr uint32_t REG_PWM_T_REGS[2] = {0x04, 0x14}; // T1..T4 offsets within a group +static constexpr uint32_t PWM_INT_STATUS_MASK = 3UL << 30; // write-1-clear -- always write as zero +static constexpr uint8_t ENVELOPE_TIMER = BKTIMER1; + +// The bk_timer handler receives only the channel number, so the chain resolves the instance +// that owns the timer. No IRAM_ATTR: hal.h makes it a no-op on BK72xx (the SDK masks IRQs +// around flash writes). +static void envelope_timer_isr(UINT8 channel) { RemoteTransmitterComponent::advance_active_isr(); } + +// Channel <-> pin comes from the board variant's own PIN_PWMn defines rather than a +// family-wide assumption, so an unusual pinout maps correctly instead of silently +// driving another pad +struct PwmPinChannel { + uint8_t pin; + int8_t channel; +}; +static constexpr PwmPinChannel PWM_PIN_CHANNELS[] = { +#ifdef PIN_PWM0 + {PIN_PWM0, 0}, +#endif +#ifdef PIN_PWM1 + {PIN_PWM1, 1}, +#endif +#ifdef PIN_PWM2 + {PIN_PWM2, 2}, +#endif +#ifdef PIN_PWM3 + {PIN_PWM3, 3}, +#endif +#ifdef PIN_PWM4 + {PIN_PWM4, 4}, +#endif +#ifdef PIN_PWM5 + {PIN_PWM5, 5}, +#endif +}; + +static int8_t pwm_channel_for_pin(uint8_t pin) { + for (const auto &entry : PWM_PIN_CHANNELS) { + if (entry.pin == pin) + return entry.channel; + } + return -1; +} + +void RemoteTransmitterComponent::setup() { + // Deliberately no pin_->setup(): the pin must belong to the PWM function, not GPIO + const int8_t channel = pwm_channel_for_pin(this->pin_->get_pin()); + if (channel < 0) { + ESP_LOGE(TAG, "Pin %u is not PWM-capable", this->pin_->get_pin()); + this->mark_failed(); + return; + } + this->pwm_channel_ = channel; + const uint32_t idle_t1 = this->pin_->is_inverted() ? this->isr_period_t4_ : 0; + pwm_param_st param{}; + param.chan = channel; + param.t1 = idle_t1; + param.t4 = this->isr_period_t4_; + param.init_level = idle_t1 ? 1 : 0; + if (pwm_init_param(¶m) != 0 || pwm_start(channel) != 0) { + ESP_LOGE(TAG, "PWM init failed on pin %u", this->pin_->get_pin()); + this->pwm_channel_ = -1; + this->mark_failed(); + return; + } + this->disable_loop(); // loop() is only needed while a non-blocking completion is pending +} + +void RemoteTransmitterComponent::dump_config() { + ESP_LOGCONFIG(TAG, + "Remote Transmitter:\n" + " Carrier Duty: %u%%\n" + " Non-blocking: %s", + this->carrier_duty_percent_, YESNO(this->non_blocking_)); + LOG_PIN(" Pin: ", this->pin_); +} + +// Writes the duty compare registers and sets the hardware CFG_UPDATA shadow-load bit; +// the new duty latches glitch-free at the next carrier period. ISR-safe: registers only. +// The group control word is shared with the paired channel, but every SDK write to it runs +// under GLOBAL_INT_DISABLE (bk_pwm), so it cannot be torn by this interrupt. +void RemoteTransmitterComponent::write_pwm_t1_(uint32_t t1_counts) { + const uint32_t group = this->pwm_channel_ / 2; + const uint32_t post = this->pwm_channel_ % 2; + const uint32_t group_base = REG_PWM_BASE + REG_PWM_GROUP_STRIDE * group; + auto *t_regs = (volatile uint32_t *) (group_base + REG_PWM_T_REGS[post]); + auto *ctrl = (volatile uint32_t *) group_base; + const uint32_t init_level_bit = 1UL << (8 * post + 6); // output level while the counter is stopped + const uint32_t cfg_updata_bit = 1UL << (8 * post + 7); // 0->1 latches T1..T4 at the next period + t_regs[0] = t1_counts; // T1: high time + t_regs[1] = 0; // T2 + t_regs[2] = 0; // T3 + t_regs[3] = this->isr_period_t4_; // T4: period + uint32_t cfg = *ctrl; + cfg &= ~(PWM_INT_STATUS_MASK | init_level_bit | cfg_updata_bit); + if (t1_counts != 0) + cfg |= init_level_bit; + *ctrl = cfg; + *ctrl = cfg | cfg_updata_bit; +} + +// --- envelope chain hooks (see remote_transmitter_libretiny_isr.cpp) --- + +bool RemoteTransmitterComponent::envelope_ready_() const { return this->pwm_channel_ >= 0; } + +// Recomputes the carrier period in 26MHz counts and stages the per-item duties; +// unmodulated protocols drive the pin constantly during marks +void RemoteTransmitterComponent::prepare_carrier_(uint32_t carrier_frequency) { + if (carrier_frequency > 0) { + this->isr_period_t4_ = std::max(uint32_t(2), (26000000UL + carrier_frequency / 2) / carrier_frequency); + } + uint32_t mark_t1 = (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) + ? std::max(uint32_t(1), this->isr_period_t4_ * this->carrier_duty_percent_ / 100) + : this->isr_period_t4_; + uint32_t space_t1 = 0; + if (this->pin_->is_inverted()) { + mark_t1 = this->isr_period_t4_ - mark_t1; + space_t1 = this->isr_period_t4_; + } + this->isr_mark_t1_ = mark_t1; + this->isr_space_t1_ = space_t1; +} + +void RemoteTransmitterComponent::write_envelope_level_(bool mark) { + this->write_pwm_t1_(mark ? this->isr_mark_t1_ : this->isr_space_t1_); +} + +// The driver's microsecond init path is register writes under a nested interrupt guard, +// so it is safe to call from the chain's own interrupt +void RemoteTransmitterComponent::arm_one_shot_(uint32_t duration_us) { + timer_param_t param{}; + param.channel = ENVELOPE_TIMER; + param.div = 1; + param.period = duration_us; + param.t_Int_Handler = envelope_timer_isr; + sddev_control((char *) TIMER_DEV_NAME, CMD_TIMER_INIT_PARAM_US, ¶m); +} + +void RemoteTransmitterComponent::stop_envelope_timer_() { + UINT32 channel = ENVELOPE_TIMER; + sddev_control((char *) TIMER_DEV_NAME, CMD_TIMER_UNIT_DISABLE, &channel); +} + +void RemoteTransmitterComponent::digital_write(bool value) { + if (this->pwm_channel_ < 0) + return; + // serialize behind an in-flight chain, matching the ESP32/RMT non-blocking behavior + this->wait_until_idle_(); + this->write_pwm_t1_((value != this->pin_->is_inverted()) ? this->isr_period_t4_ : 0); +} + +#endif // REMOTE_TRANSMITTER_BK_PWM + +} // namespace esphome::remote_transmitter + +#endif // USE_BK72XX && !CLANG_TIDY diff --git a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp new file mode 100644 index 0000000000..003cdfa986 --- /dev/null +++ b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp @@ -0,0 +1,224 @@ +#include "remote_transmitter.h" +#include "esphome/core/application.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +// Envelope chain shared by the LibreTiny families that pace transmission from a hardware +// timer interrupt: RTL8720C (gtimer) and the BK7231N-style PWM block (BKTIMER1). Everything +// platform-specific sits behind five hooks implemented in the per-family files -- carrier +// setup, duty writes, one-shot arming and timer stop. Families without a usable timer keep +// the generic bit-bang implementation and compile none of this. +#if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) + +namespace esphome::remote_transmitter { + +static const char *const TAG = "remote_transmitter"; + +// Margin past a transmission's expected duration before the chain is declared stalled +static constexpr uint32_t STALL_MARGIN_MS = 1000; +// Longest single one-shot armed; longer durations are chained. Both families need the cap: +// the Beken driver computes period_us * 26 in 32 bits (overflows past ~165s) and the Realtek +// us->tick conversion lives in mask ROM with unverified headroom. +static constexpr uint32_t MAX_ONE_SHOT_US = 50000; + +// One hardware timer is shared by all instances (MULTI_CONF), so they serialize on this +// token; the deadline always describes whichever chain currently owns it. +// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) +static RemoteTransmitterComponent *volatile s_active_transmitter = nullptr; +static uint32_t s_expected_end_ms = 0; +// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) + +// Entry point for trampolines whose SDK callback carries no user argument +void IRAM_ATTR RemoteTransmitterComponent::advance_active_isr() { + auto *transmitter = s_active_transmitter; + if (transmitter != nullptr) + transmitter->advance_envelope_isr(); +} + +// Arms the envelope timer, chaining durations longer than MAX_ONE_SHOT_US. ISR-safe. +void IRAM_ATTR RemoteTransmitterComponent::arm_envelope_timer_(uint32_t duration_us) { + // clamp to 1us (a zero-length one-shot never fires); the remainder must not underflow + const uint32_t chunk = std::max(uint32_t(1), std::min(duration_us, MAX_ONE_SHOT_US)); + this->isr_wait_remaining_ = duration_us > chunk ? duration_us - chunk : 0; + this->arm_one_shot_(chunk); +} + +// Writes the level for one envelope item and arms the timer for its duration. +// Runs in ISR context (and once from arm_chain_ to kick the chain): no logging, no allocation. +void IRAM_ATTR RemoteTransmitterComponent::start_isr_item_(size_t index) { + const int32_t item = this->isr_data_[index]; + this->write_envelope_level_(item > 0); + this->arm_envelope_timer_(uint32_t(item > 0 ? item : -item)); +} + +void IRAM_ATTR RemoteTransmitterComponent::advance_envelope_isr() { + if (!this->transmitting_) + return; // chain was aborted; this is a stale one-shot that was already latched + if (this->isr_wait_remaining_ > 0) { + // continue a duration longer than one hardware one-shot + this->arm_envelope_timer_(this->isr_wait_remaining_); + return; + } + if (this->isr_in_gap_) { + // inter-repeat gap elapsed; restart the item chain + this->isr_in_gap_ = false; + this->isr_index_ = 0; + this->start_isr_item_(0); + return; + } + this->isr_index_ = this->isr_index_ + 1; + if (this->isr_index_ < this->isr_data_.size()) { + this->start_isr_item_(this->isr_index_); + return; + } + // end of one repetition + this->write_envelope_level_(false); + if (this->isr_repeats_left_ > 1) { + this->isr_repeats_left_ = this->isr_repeats_left_ - 1; + this->isr_index_ = 0; + if (this->isr_send_wait_ > 0) { + this->isr_in_gap_ = true; + this->arm_envelope_timer_(this->isr_send_wait_); + } else { + this->start_isr_item_(0); + } + return; + } + // required on Beken (its timer reloads); on Realtek this only clears the enable bit of a + // one-shot that has already fired + this->stop_envelope_timer_(); + this->transmitting_ = false; + s_active_transmitter = nullptr; +} + +// Aborts a chain that stopped advancing: stop the timer, idle the pin, release the token. +// Every step is a no-op if the chain completed meanwhile. Task context only. +void RemoteTransmitterComponent::abort_stalled_chain_() { + // cleared first so a straggler one-shot bails at the ISR entry check + this->transmitting_ = false; + this->stop_envelope_timer_(); + this->write_envelope_level_(false); + s_active_transmitter = nullptr; + this->stall_aborted_ = true; + this->status_set_warning("envelope timer stalled"); + ESP_LOGE(TAG, "Envelope timer stalled; transmission aborted"); + delay(1); // let any already-latched interrupt land while the chain state is safe +} + +// Delivers one deferred completion with its status bookkeeping +void RemoteTransmitterComponent::deliver_completion_() { + if (!this->stall_aborted_) + this->status_clear_warning(); + this->complete_pending_ = false; + this->complete_trigger_.trigger(); +} + +// Waits until no chain is in flight, delivering any deferred completions; a completion +// automation may start a new send, so repeat until truly idle. Bounded by the stall deadline. +void RemoteTransmitterComponent::wait_until_idle_() { + while (true) { + while (true) { + // snapshot: the final ISR can clear the volatile pointer between a check and a use + auto *active = s_active_transmitter; + if (active == nullptr) + break; + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + active->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + if (!this->complete_pending_) + break; + this->deliver_completion_(); + } +} + +// Stages the repeat schedule and stall deadline, then starts the interrupt chain +void RemoteTransmitterComponent::arm_chain_(uint32_t send_times, uint32_t send_wait) { + this->isr_repeats_left_ = send_times; + this->isr_send_wait_ = send_wait; + this->isr_index_ = 0; + this->isr_in_gap_ = false; + this->stall_aborted_ = false; + uint64_t frame_us = 0; + for (int32_t item : this->isr_data_) + frame_us += uint32_t(item > 0 ? item : -item); + const uint64_t total_us = frame_us * send_times + uint64_t(send_wait) * (send_times - 1); + s_expected_end_ms = millis() + uint32_t(total_us / 1000) + STALL_MARGIN_MS; + this->transmitting_ = true; + s_active_transmitter = this; + this->start_isr_item_(0); +} + +void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { + if (!this->envelope_ready_()) { + // both triggers still fire, so an on_complete-sequenced automation does not stall + ESP_LOGW(TAG, "Cannot send: PWM not initialized"); + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + this->wait_until_idle_(); + if (send_times == 0) { + // parity with the loop-based implementations: transmit nothing, but both triggers + // still fire so an on_complete-sequenced automation does not stall + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + ESP_LOGD(TAG, "Sending remote code"); + this->prepare_carrier_(this->temp_.get_carrier_frequency()); + // own copy: with non_blocking the caller may re-encode temp_ while this frame is in flight + this->isr_data_.assign(this->temp_.get_data().begin(), this->temp_.get_data().end()); + if (this->isr_data_.empty()) { + ESP_LOGW(TAG, "Empty data"); + this->transmit_trigger_.trigger(); + this->deliver_completion_(); + return; + } + // trigger first: the deadline computed in arm_chain_ must not be charged for user code + this->transmit_trigger_.trigger(); + // the automation may have started a send on another instance; let it finish before + // claiming the shared timer (a same-instance send remains unsupported here) + this->wait_until_idle_(); + this->arm_chain_(send_times, send_wait); + if (this->non_blocking_) { + this->complete_pending_ = true; + this->enable_loop(); + return; + } + // blocking mode: wait out the chain, bounded by the stall deadline + while (this->transmitting_) { + if ((int32_t) (millis() - s_expected_end_ms) > 0) { + this->abort_stalled_chain_(); + break; + } + App.feed_wdt(); + delay(1); + } + this->deliver_completion_(); +} + +void RemoteTransmitterComponent::loop() { + if (!this->complete_pending_) { + this->disable_loop(); + return; + } + if (this->transmitting_) { + // non-blocking stall recovery: without this, a dead chain would leave the carrier + // driven and on_complete unfired until the next send happened to abort it + if ((int32_t) (millis() - s_expected_end_ms) <= 0) + return; + this->abort_stalled_chain_(); + } + // release the loop before user code runs: the automation may start a new non-blocking + // send, and its enable_loop() must be the last writer or its completion would strand + this->disable_loop(); + this->deliver_completion_(); +} + +} // namespace esphome::remote_transmitter + +#endif // USE_LIBRETINY_VARIANT_RTL8720C || REMOTE_TRANSMITTER_BK_PWM diff --git a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp index 9f629168f2..6db9faac36 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_rtl87xx.cpp @@ -24,20 +24,13 @@ static const char *const TAG = "remote_transmitter"; #ifdef USE_LIBRETINY_VARIANT_RTL8720C static constexpr uint32_t ENVELOPE_TIMER_ID = TIMER6; // GTimer7 -// Margin past a transmission's expected duration before the chain is declared stalled -static constexpr uint32_t STALL_MARGIN_MS = 1000; -// Longest single one-shot armed; longer durations are chained (ROM us->tick headroom unverified) -static constexpr uint32_t MAX_ONE_SHOT_US = 50000; -// Shared envelope timer: a second gtimer_init on the same id fails silently, so all -// instances serialize on s_active_transmitter +// One envelope timer for all instances: a second gtimer_init on the same id fails silently, +// so the chain serializes them (remote_transmitter_libretiny_isr.cpp) // NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables) static uint8_t s_pwm_tick_sources[] = {GTimer1, GTimer2, GTimer3, GTimer4, GTimer5, GTimer6, 0xff}; static gtimer_t s_envelope_timer; static bool s_envelope_timer_ready = false; -static RemoteTransmitterComponent *volatile s_active_transmitter = nullptr; -// Deadline for the in-flight transmission (millis-based); only touched from the main task -static uint32_t s_expected_end_ms = 0; // NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables) static void IRAM_ATTR envelope_timer_isr(uint32_t arg) { @@ -104,105 +97,22 @@ void RemoteTransmitterComponent::digital_write(bool value) { } #ifdef USE_LIBRETINY_VARIANT_RTL8720C -// Arms the shared envelope timer, chaining durations longer than MAX_ONE_SHOT_US. ISR-safe. -void IRAM_ATTR RemoteTransmitterComponent::arm_envelope_timer_(uint32_t duration_us) { - // clamp to 1us (a zero-length one-shot never fires); the remainder must not underflow - const uint32_t chunk = std::max(uint32_t(1), std::min(duration_us, MAX_ONE_SHOT_US)); - this->isr_wait_remaining_ = duration_us > chunk ? duration_us - chunk : 0; - gtimer_start_one_shout(&s_envelope_timer, chunk, (void *) envelope_timer_isr, (uint32_t) this); -} +// --- envelope chain hooks (see remote_transmitter_libretiny_isr.cpp) --- -// Aborts a chain that stopped advancing: stop the timer, idle the pin, release the token. -// Every step is a no-op if the chain completed meanwhile. Task context only. -void RemoteTransmitterComponent::abort_stalled_chain_() { - // cleared first so a straggler one-shot bails at the ISR entry check - this->transmitting_ = false; - gtimer_stop(&s_envelope_timer); - pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); - s_active_transmitter = nullptr; - this->stall_aborted_ = true; - this->status_set_warning("envelope timer stalled"); - ESP_LOGE(TAG, "Envelope timer stalled; transmission aborted"); - delay(1); // let any already-latched interrupt land while the chain state is safe -} +bool RemoteTransmitterComponent::envelope_ready_() const { return this->pwm_ != nullptr; } -// Delivers one deferred completion with its status bookkeeping -void RemoteTransmitterComponent::deliver_completion_() { - if (!this->stall_aborted_) - this->status_clear_warning(); - this->complete_pending_ = false; - this->complete_trigger_.trigger(); -} - -// Writes the duty for one envelope item and arms the timer for its duration. -// Runs in ISR context (and once from send_internal to kick the chain): no logging, no allocation. -void IRAM_ATTR RemoteTransmitterComponent::start_isr_item_(size_t index) { - const int32_t item = this->isr_data_[index]; - pwmout_write(static_cast(this->pwm_), item > 0 ? this->isr_mark_duty_ : this->isr_space_duty_); - this->arm_envelope_timer_(uint32_t(item > 0 ? item : -item)); -} - -void IRAM_ATTR RemoteTransmitterComponent::advance_envelope_isr() { - if (!this->transmitting_) - return; // chain was aborted; this is a stale one-shot that was already latched - if (this->isr_wait_remaining_ > 0) { - // continue a duration longer than one hardware one-shot - this->arm_envelope_timer_(this->isr_wait_remaining_); - return; +// Retunes the PWM period when the carrier changes and stages the per-item duties; +// unmodulated protocols (no carrier or 100% duty) drive the pin constantly during marks +void RemoteTransmitterComponent::prepare_carrier_(uint32_t carrier_frequency) { + float mark_duty = + (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) ? this->carrier_duty_percent_ / 100.0f : 1.0f; + float space_duty = 0.0f; + if (this->pin_->is_inverted()) { + mark_duty = 1.0f - mark_duty; + space_duty = 1.0f; } - if (this->isr_in_gap_) { - // inter-repeat gap elapsed; restart the item chain - this->isr_in_gap_ = false; - this->isr_index_ = 0; - this->start_isr_item_(0); - return; - } - this->isr_index_++; - if (this->isr_index_ < this->isr_data_.size()) { - this->start_isr_item_(this->isr_index_); - return; - } - // end of one repetition - pwmout_write(static_cast(this->pwm_), this->isr_space_duty_); - if (this->isr_repeats_left_ > 1) { - this->isr_repeats_left_--; - this->isr_index_ = 0; - if (this->isr_send_wait_ > 0) { - this->isr_in_gap_ = true; - this->arm_envelope_timer_(this->isr_send_wait_); - } else { - this->start_isr_item_(0); - } - return; - } - this->transmitting_ = false; - s_active_transmitter = nullptr; -} - -// Waits until no chain is in flight, delivering any deferred completions; a completion -// automation may start a new send, so repeat until truly idle. Bounded by the stall deadline. -void RemoteTransmitterComponent::wait_until_idle_() { - while (true) { - while (true) { - // snapshot: the final ISR can clear the volatile pointer between a check and a use - auto *active = s_active_transmitter; - if (active == nullptr) - break; - if ((int32_t) (millis() - s_expected_end_ms) > 0) { - active->abort_stalled_chain_(); - break; - } - App.feed_wdt(); - delay(1); - } - if (!this->complete_pending_) - break; - this->deliver_completion_(); - } -} - -// Retunes the PWM period when the carrier changes; the ISR sets duty per item -void RemoteTransmitterComponent::update_carrier_(uint32_t carrier_frequency) { + this->isr_mark_duty_ = mark_duty; + this->isr_space_duty_ = space_duty; if (carrier_frequency == 0 || carrier_frequency == this->current_carrier_frequency_) return; // round(1000000/freq), clamped so a bad lambda can't hand the SDK a zero period @@ -211,97 +121,15 @@ void RemoteTransmitterComponent::update_carrier_(uint32_t carrier_frequency) { this->current_carrier_frequency_ = carrier_frequency; } -// Stages the repeat schedule and stall deadline, then starts the interrupt chain -void RemoteTransmitterComponent::arm_chain_(uint32_t send_times, uint32_t send_wait) { - this->isr_repeats_left_ = send_times; - this->isr_send_wait_ = send_wait; - this->isr_index_ = 0; - this->isr_in_gap_ = false; - this->stall_aborted_ = false; - uint64_t frame_us = 0; - for (int32_t item : this->isr_data_) - frame_us += uint32_t(item > 0 ? item : -item); - const uint64_t total_us = frame_us * send_times + uint64_t(send_wait) * (send_times - 1); - s_expected_end_ms = millis() + uint32_t(total_us / 1000) + STALL_MARGIN_MS; - this->transmitting_ = true; - s_active_transmitter = this; - this->start_isr_item_(0); +void IRAM_ATTR RemoteTransmitterComponent::write_envelope_level_(bool mark) { + pwmout_write(static_cast(this->pwm_), mark ? this->isr_mark_duty_ : this->isr_space_duty_); } -void RemoteTransmitterComponent::send_internal(uint32_t send_times, uint32_t send_wait) { - if (this->pwm_ == nullptr) { - ESP_LOGW(TAG, "Cannot send: PWM not initialized"); - return; - } - this->wait_until_idle_(); - if (send_times == 0) { - // parity with the loop-based implementations: transmit nothing, but both triggers - // still fire so an on_complete-sequenced automation does not stall - this->transmit_trigger_.trigger(); - this->deliver_completion_(); - return; - } - ESP_LOGD(TAG, "Sending remote code"); - const uint32_t carrier_frequency = this->temp_.get_carrier_frequency(); - // unmodulated protocols (no carrier or 100% duty) drive the pin constantly during marks - float mark_duty = - (carrier_frequency > 0 && this->carrier_duty_percent_ < 100) ? this->carrier_duty_percent_ / 100.0f : 1.0f; - float space_duty = 0.0f; - if (this->pin_->is_inverted()) { - mark_duty = 1.0f - mark_duty; - space_duty = 1.0f; - } - this->update_carrier_(carrier_frequency); - // own copy: with non_blocking the caller may re-encode temp_ while this frame is in flight - this->isr_data_.assign(this->temp_.get_data().begin(), this->temp_.get_data().end()); - if (this->isr_data_.empty()) { - ESP_LOGW(TAG, "Empty data"); - this->transmit_trigger_.trigger(); - this->deliver_completion_(); - return; - } - this->isr_mark_duty_ = mark_duty; - this->isr_space_duty_ = space_duty; - // trigger first: the deadline computed in arm_chain_ must not be charged for user code - this->transmit_trigger_.trigger(); - // the automation may have started a send on another instance; let it finish before - // claiming the shared timer (a same-instance send remains unsupported here) - this->wait_until_idle_(); - this->arm_chain_(send_times, send_wait); - if (this->non_blocking_) { - this->complete_pending_ = true; - this->enable_loop(); - return; - } - // blocking mode: wait out the chain, bounded by the stall deadline - while (this->transmitting_) { - if ((int32_t) (millis() - s_expected_end_ms) > 0) { - this->abort_stalled_chain_(); - break; - } - App.feed_wdt(); - delay(1); - } - this->deliver_completion_(); +void IRAM_ATTR RemoteTransmitterComponent::arm_one_shot_(uint32_t duration_us) { + gtimer_start_one_shout(&s_envelope_timer, duration_us, (void *) envelope_timer_isr, (uint32_t) this); } -void RemoteTransmitterComponent::loop() { - if (!this->complete_pending_) { - this->disable_loop(); - return; - } - if (this->transmitting_) { - // non-blocking stall recovery: without this, a dead chain would leave the carrier - // driven and on_complete unfired until the next send happened to abort it - if ((int32_t) (millis() - s_expected_end_ms) <= 0) - return; - this->abort_stalled_chain_(); - } - // release the loop before user code runs: the automation may start a new non-blocking - // send, and its enable_loop() must be the last writer or its completion would strand - this->disable_loop(); - this->deliver_completion_(); -} +void IRAM_ATTR RemoteTransmitterComponent::stop_envelope_timer_() { gtimer_stop(&s_envelope_timer); } #else // !USE_LIBRETINY_VARIANT_RTL8720C -- AmebaZ (RTL8710B): spin-based envelope, per-frame priority boost diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py index f843f1e84f..ee2769e177 100644 --- a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -4,6 +4,9 @@ the ISR paths, so this gate is the only CI-reachable coverage for the platform m import pytest from esphome.components.libretiny.const import ( + FAMILY_BK7231N, + FAMILY_BK7231T, + FAMILY_BK7238, FAMILY_RTL8710B, FAMILY_RTL8720C, KEY_FAMILY, @@ -23,6 +26,9 @@ from ..types import SetCoreConfigCallable (PlatformFramework.ESP32_IDF, None, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7238, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231T, False), (PlatformFramework.ESP8266_ARDUINO, None, False), ], ) diff --git a/tests/components/remote_transmitter/test.bk72xx-ard.yaml b/tests/components/remote_transmitter/test.bk72xx-ard.yaml index 2a5cceddec..ea2feafda9 100644 --- a/tests/components/remote_transmitter/test.bk72xx-ard.yaml +++ b/tests/components/remote_transmitter/test.bk72xx-ard.yaml @@ -2,6 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO26 carrier_duty_percent: 50% + # non_blocking is bk7231n/bk7238-only; the CI board is a BK7252 packages: buttons: !include common-buttons.yaml From 5328813814b52178e296a8cb8122825e98a0deaf Mon Sep 17 00:00:00 2001 From: MakerYuichi <106516578+MakerYuichi@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:13:55 +0530 Subject: [PATCH 12/30] [time] Silence compiler warning by initializing transit variables (#18715) (#18723) Co-authored-by: doraemon2200 <106516578+doraemon2200@users.noreply.github.com> --- esphome/components/time/posix_tz.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/time/posix_tz.cpp b/esphome/components/time/posix_tz.cpp index 188df599f6..002aadfec3 100644 --- a/esphome/components/time/posix_tz.cpp +++ b/esphome/components/time/posix_tz.cpp @@ -178,7 +178,8 @@ static int __attribute__((noinline)) days_from_year_start(int year, int month, i } time_t __attribute__((noinline)) calculate_dst_transition(int year, const DSTRule &rule, int32_t base_offset_seconds) { - int month, day; + int month = 1; + int day = 1; switch (rule.type) { case DSTRuleType::MONTH_WEEK_DAY: { From 8a89f3075f31b3bf0db104551fb566312528b872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Metrich?= <45318189+FredM67@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:03:10 +0200 Subject: [PATCH 13/30] [emontx] Fix sensor storage initialization ordering (#18771) --- esphome/components/emontx/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/esphome/components/emontx/__init__.py b/esphome/components/emontx/__init__.py index 7dde794f0b..3821f3e10e 100644 --- a/esphome/components/emontx/__init__.py +++ b/esphome/components/emontx/__init__.py @@ -116,14 +116,16 @@ _CALLBACK_AUTOMATIONS = ( async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) - await cg.register_component(var, config) - await uart.register_uart_device(var, config) - # Initialize sensor storage with count from final_validate + # Initialize sensor storage with count from final_validate before any + # await, so platform to_code() calls always see it initialized + # regardless of YAML key order. sensor_count = _get_data().sensor_counts.get(str(config[CONF_ID]), 0) if sensor_count > 0: cg.add(var.init_sensors(sensor_count)) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) From 34e74536774b0b031f1f69bc166fb86ffc6c7746 Mon Sep 17 00:00:00 2001 From: Leonardo Rivera Date: Wed, 26 Aug 2026 15:26:31 -0300 Subject: [PATCH 14/30] [climate] Don't restore a saved mode the device no longer supports (#18296) --- esphome/components/climate/climate.cpp | 9 ++- tests/components/climate/climate_test.cpp | 73 +++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/components/climate/climate_test.cpp diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 0f01443bd0..6ca9e394f7 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -551,7 +551,14 @@ ClimateCall ClimateDeviceRestoreState::to_call(Climate *climate) { void ClimateDeviceRestoreState::apply(Climate *climate) { auto traits = climate->get_traits(); - climate->mode = this->mode; + // A saved mode the device no longer offers cannot be selected again, so skip it and leave the + // entity on the mode it already has. The other saved fields are still restored. + if (traits.supports_mode(this->mode)) { + climate->mode = this->mode; + } else { + ESP_LOGW(TAG, "'%s' - Saved mode %s is no longer supported, keeping %s", climate->get_name().c_str(), + LOG_STR_ARG(climate_mode_to_string(this->mode)), LOG_STR_ARG(climate_mode_to_string(climate->mode))); + } if (traits.has_feature_flags(CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE | CLIMATE_REQUIRES_TWO_POINT_TARGET_TEMPERATURE)) { climate->target_temperature_low = this->target_temperature_low; diff --git a/tests/components/climate/climate_test.cpp b/tests/components/climate/climate_test.cpp new file mode 100644 index 0000000000..bda014b87a --- /dev/null +++ b/tests/components/climate/climate_test.cpp @@ -0,0 +1,73 @@ +#include +#include "esphome/components/climate/climate.h" + +namespace esphome::climate::testing { + +// Minimal concrete Climate that offers a fixed set of modes, so the restore path can be exercised +// without any hardware or platform component. +class TestClimate : public Climate { + public: + ClimateTraits traits() override { + auto traits = ClimateTraits(); + traits.set_supported_modes({CLIMATE_MODE_OFF, CLIMATE_MODE_COOL}); + traits.set_supported_fan_modes({CLIMATE_FAN_LOW, CLIMATE_FAN_HIGH}); + return traits; + } + + protected: + void control(const ClimateCall &call) override {} +}; + +TEST(ClimateRestoreStateTest, RestoresASupportedMode) { + TestClimate climate; + // Value-initialized: several members (mode, swing_mode, the temperature union) have no default + // member initializer, so leaving the {} off would read indeterminate values. + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_COOL; + + state.apply(&climate); + + EXPECT_EQ(climate.mode, CLIMATE_MODE_COOL); +} + +TEST(ClimateRestoreStateTest, DoesNotRestoreAnUnsupportedMode) { + TestClimate climate; + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_HEAT; + + state.apply(&climate); + + // The device never advertised HEAT, so the mode stays where it was. + EXPECT_EQ(climate.mode, CLIMATE_MODE_OFF); +} + +TEST(ClimateRestoreStateTest, LeavesTheCurrentModeAloneRatherThanForcingOff) { + TestClimate climate; + // apply() is public and nothing restricts it to setup(), so the entity is not necessarily off + // when an unsupported mode is dropped. It keeps what it had rather than being forced to OFF. + climate.mode = CLIMATE_MODE_COOL; + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_HEAT; + + state.apply(&climate); + + EXPECT_EQ(climate.mode, CLIMATE_MODE_COOL); +} + +TEST(ClimateRestoreStateTest, KeepsRestoringTheOtherFieldsWhenTheModeIsDropped) { + TestClimate climate; + ClimateDeviceRestoreState state{}; + state.mode = CLIMATE_MODE_HEAT; + state.target_temperature = 21.0f; + state.uses_custom_fan_mode = false; + state.fan_mode = CLIMATE_FAN_HIGH; + + state.apply(&climate); + + EXPECT_EQ(climate.mode, CLIMATE_MODE_OFF); + EXPECT_FLOAT_EQ(climate.target_temperature, 21.0f); + // Compared as an optional: this asserts both that the fan mode was restored and what it holds. + EXPECT_EQ(climate.fan_mode, CLIMATE_FAN_HIGH); +} + +} // namespace esphome::climate::testing From f17ef133f34be727df30f6cebf3e80f2af515ff3 Mon Sep 17 00:00:00 2001 From: Josef Zweck Date: Wed, 26 Aug 2026 20:27:16 +0200 Subject: [PATCH 15/30] [hoermann_hcp] Add buttons to hoermann_hcp (#18544) --- .../hoermann_hcp/button/__init__.py | 43 +++++++++++ .../hoermann_hcp/button/hoermann_hcp_button.h | 34 +++++++++ .../components/hoermann_hcp/hoermann_hcp.cpp | 5 ++ .../components/hoermann_hcp/hoermann_hcp.h | 6 +- .../button/hoermann_hcp_button_test.cpp | 72 +++++++++++++++++++ tests/components/hoermann_hcp/common.yaml | 7 ++ 6 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 esphome/components/hoermann_hcp/button/__init__.py create mode 100644 esphome/components/hoermann_hcp/button/hoermann_hcp_button.h create mode 100644 tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp diff --git a/esphome/components/hoermann_hcp/button/__init__.py b/esphome/components/hoermann_hcp/button/__init__.py new file mode 100644 index 0000000000..dc2efcec44 --- /dev/null +++ b/esphome/components/hoermann_hcp/button/__init__.py @@ -0,0 +1,43 @@ +import esphome.codegen as cg +from esphome.components import button +import esphome.config_validation as cv +from esphome.const import ICON_AIR_FILTER +from esphome.types import ConfigType + +from .. import CONF_HOERMANN_HCP_ID, HoermannHcp, hoermann_hcp_ns + +DEPENDENCIES = ["hoermann_hcp"] + +CONF_HALF_OPEN = "half_open" +CONF_VENT = "vent" + +ICON_GARAGE_OPEN_VARIANT = "mdi:garage-open-variant" + +HoermannHcpVentButton = hoermann_hcp_ns.class_("HoermannHcpVentButton", button.Button) +HoermannHcpHalfOpenButton = hoermann_hcp_ns.class_( + "HoermannHcpHalfOpenButton", button.Button +) + +BUTTON_KEYS = (CONF_VENT, CONF_HALF_OPEN) + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(CONF_HOERMANN_HCP_ID): cv.use_id(HoermannHcp), + cv.Optional(CONF_VENT): button.button_schema( + HoermannHcpVentButton, icon=ICON_AIR_FILTER + ), + cv.Optional(CONF_HALF_OPEN): button.button_schema( + HoermannHcpHalfOpenButton, icon=ICON_GARAGE_OPEN_VARIANT + ), + } + ), + cv.has_at_least_one_key(*BUTTON_KEYS), +) + + +async def to_code(config: ConfigType) -> None: + parent = await cg.get_variable(config[CONF_HOERMANN_HCP_ID]) + for key in BUTTON_KEYS: + if (conf := config.get(key)) is not None: + await button.new_button(conf, parent) diff --git a/esphome/components/hoermann_hcp/button/hoermann_hcp_button.h b/esphome/components/hoermann_hcp/button/hoermann_hcp_button.h new file mode 100644 index 0000000000..e9ebceee88 --- /dev/null +++ b/esphome/components/hoermann_hcp/button/hoermann_hcp_button.h @@ -0,0 +1,34 @@ +#pragma once + +#include "esphome/components/button/button.h" +#include "../hoermann_hcp.h" + +namespace esphome::hoermann_hcp { + +// The door commands the cover has no equivalent for. A refused command is already reported by the hub and +// leaves nothing to correct here, because a button carries no state of its own. +class HoermannHcpButton : public button::Button { + public: + explicit HoermannHcpButton(HoermannHcp *parent) : parent_(parent) {} + + protected: + HoermannHcp *const parent_; +}; + +class HoermannHcpVentButton final : public HoermannHcpButton { + public: + using HoermannHcpButton::HoermannHcpButton; + + protected: + void press_action() override { this->parent_->vent_door(); } +}; + +class HoermannHcpHalfOpenButton final : public HoermannHcpButton { + public: + using HoermannHcpButton::HoermannHcpButton; + + protected: + void press_action() override { this->parent_->half_open_door(); } +}; + +} // namespace esphome::hoermann_hcp diff --git a/esphome/components/hoermann_hcp/hoermann_hcp.cpp b/esphome/components/hoermann_hcp/hoermann_hcp.cpp index a780854831..17df927eb7 100644 --- a/esphome/components/hoermann_hcp/hoermann_hcp.cpp +++ b/esphome/components/hoermann_hcp/hoermann_hcp.cpp @@ -22,6 +22,9 @@ static constexpr uint8_t MAX_LIGHT_TOGGLES_IN_FLIGHT = 4; static constexpr HoermannHcpCommand COMMAND_OPEN{"open", 0x0210, 0x0110}; static constexpr HoermannHcpCommand COMMAND_CLOSE{"close", 0x0220, 0x0120}; static constexpr HoermannHcpCommand COMMAND_IMPULSE{"impulse", 0x0240, 0x0140}; +// The intermediate positions are named in the second register, so the first only carries the phase. +static constexpr HoermannHcpCommand COMMAND_VENT{"vent", 0x0200, 0x0100, 0x4000, 0x4000}; +static constexpr HoermannHcpCommand COMMAND_HALF_OPEN{"half open", 0x0200, 0x0100, 0x0400, 0x0400}; // The lamp is named in the second register, but its phase bytes follow no scheme the door commands share. static constexpr HoermannHcpCommand COMMAND_TOGGLE_LAMP{"toggle light", 0x0100, 0x0800, 0x0200, 0x0200, false}; @@ -286,6 +289,8 @@ bool HoermannHcp::queue_command_(const HoermannHcpCommand &command) { bool HoermannHcp::open_door() { return this->queue_command_(COMMAND_OPEN); } bool HoermannHcp::close_door() { return this->queue_command_(COMMAND_CLOSE); } bool HoermannHcp::impulse_door() { return this->queue_command_(COMMAND_IMPULSE); } +bool HoermannHcp::vent_door() { return this->queue_command_(COMMAND_VENT); } +bool HoermannHcp::half_open_door() { return this->queue_command_(COMMAND_HALF_OPEN); } bool HoermannHcp::toggle_light() { if (this->light_toggles_in_flight_ >= MAX_LIGHT_TOGGLES_IN_FLIGHT) { ESP_LOGW(TAG, "Too many lamp toggles are still waiting to be confirmed, dropping this one"); diff --git a/esphome/components/hoermann_hcp/hoermann_hcp.h b/esphome/components/hoermann_hcp/hoermann_hcp.h index 41fd7617e4..83be385c7b 100644 --- a/esphome/components/hoermann_hcp/hoermann_hcp.h +++ b/esphome/components/hoermann_hcp/hoermann_hcp.h @@ -22,7 +22,8 @@ enum class DoorState : uint8_t { }; // A HCP command is a simulated key press: the pressed value is presented to the bus controller, then after a -// short delay the released value. Each half also carries a second register, which only the lamp command uses. +// short delay the released value. Each half also carries a second register, which names the buttons that do +// not fit into the first. struct HoermannHcpCommand { const char *name; uint16_t pressed_value; @@ -54,6 +55,9 @@ class HoermannHcp : public PollingComponent, public modbus::ModbusServerDevice { bool open_door(); bool close_door(); bool impulse_door(); + // The door drives to these intermediate positions on its own, so neither takes a target to be stopped at. + bool vent_door(); + bool half_open_door(); bool stop_door(); bool set_position(float position); bool toggle_light(); diff --git a/tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp b/tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp new file mode 100644 index 0000000000..9c38bc1708 --- /dev/null +++ b/tests/components/hoermann_hcp/button/hoermann_hcp_button_test.cpp @@ -0,0 +1,72 @@ +#include + +#include "esphome/components/hoermann_hcp/button/hoermann_hcp_button.h" + +#include "../common.h" + +namespace esphome::hoermann_hcp::testing { + +// The intermediate positions are named in the second register, which repeats that name on release. +TEST(HoermannHcpButtonTest, VentButtonSendsTheVentCommand) { + TestableHoermannHcp door; + HoermannHcpVentButton vent(&door); + connect_controller(door); + + vent.press(); + + auto [pressed, pressed_2] = poll_command(door); + EXPECT_EQ(pressed, 0x0200); + EXPECT_EQ(pressed_2, 0x4000); + std::this_thread::sleep_for(KEY_PRESS_ELAPSED); + auto [released, released_2] = poll_command(door); + EXPECT_EQ(released, 0x0100); + EXPECT_EQ(released_2, 0x4000); +} + +TEST(HoermannHcpButtonTest, HalfOpenButtonSendsTheHalfOpenCommand) { + TestableHoermannHcp door; + HoermannHcpHalfOpenButton half_open(&door); + connect_controller(door); + + half_open.press(); + + auto [pressed, pressed_2] = poll_command(door); + EXPECT_EQ(pressed, 0x0200); + EXPECT_EQ(pressed_2, 0x0400); + std::this_thread::sleep_for(KEY_PRESS_ELAPSED); + auto [released, released_2] = poll_command(door); + EXPECT_EQ(released, 0x0100); + EXPECT_EQ(released_2, 0x0400); +} + +// The door drives to the vent position on its own, so a position the cover was still travelling to must not +// stop it on the way there. +TEST(HoermannHcpButtonTest, VentAbandonsAnArmedTarget) { + TestableHoermannHcp door; // starts out fully closed + HoermannHcpVentButton vent(&door); + connect_controller(door); + door.set_position(0.5f); + consume_command(door); + door.on_write_registers(BROADCAST_REG, make_registers({0x0000, 0x0014, 0x0100})); + ASSERT_EQ(door.get_door_state(), DoorState::OPENING); + + vent.press(); + consume_command(door); + + // Position 120/200 = 0.6 is past the abandoned target, which must no longer stop the door. + door.on_write_registers(BROADCAST_REG, make_registers({0x0000, 0x0078, 0x0100})); + EXPECT_EQ(poll_command(door).first, 0x0000); +} + +// A button carries no state, so a refused press is simply dropped rather than fired once the controller +// turns up, which could be much later. +TEST(HoermannHcpButtonTest, PressWithoutABusControllerSendsNothing) { + HoermannHcp door; // never contacted by a bus controller + HoermannHcpVentButton vent(&door); + + vent.press(); + + EXPECT_EQ(poll_command(door).first, 0x0000); +} + +} // namespace esphome::hoermann_hcp::testing diff --git a/tests/components/hoermann_hcp/common.yaml b/tests/components/hoermann_hcp/common.yaml index 552b1cb0fd..618a8181bf 100644 --- a/tests/components/hoermann_hcp/common.yaml +++ b/tests/components/hoermann_hcp/common.yaml @@ -12,6 +12,13 @@ binary_sensor: is_connected: name: Garage Connected +button: + - platform: hoermann_hcp + vent: + name: Garage Vent + half_open: + name: Garage Half Open + light: - platform: hoermann_hcp name: Garage Light From d28b17c619937f13353fb1857f52239a89eb8735 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:26:45 +0000 Subject: [PATCH 16/30] Bump filelock from 3.32.3 to 3.32.4 (#18799) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d4439bf10..1a98c2a8e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.3 # native esp-idf toolchain global cache dir ninja==1.13.0 # native esp8266 arduino toolchain build driver -filelock==3.32.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.2 From fedd46e648a3099915476e4d47c357e946e3eafd Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:55:29 +0000 Subject: [PATCH 17/30] Bump aioesphomeapi from 46.2.0 to 46.2.1 (#18804) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1a98c2a8e4..de00f07836 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==46.2.0 +aioesphomeapi==46.2.1 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From 5878406918eda313684cca4c9f2e0fa737358a47 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:33:52 -0500 Subject: [PATCH 18/30] Bump bundled esphome-device-builder to 1.13.1 (#18807) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d46f01838e..0da8048c57 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 RUN \ platformio settings set enable_telemetry No \ From 950816579764d38865153c1738f473800df64882 Mon Sep 17 00:00:00 2001 From: guillempages Date: Wed, 26 Aug 2026 23:36:47 +0200 Subject: [PATCH 19/30] [runtime_image] Add check for dimensions in BMP (#18800) --- esphome/components/runtime_image/bmp_decoder.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esphome/components/runtime_image/bmp_decoder.cpp b/esphome/components/runtime_image/bmp_decoder.cpp index 5d45621fb7..204d6cc14b 100644 --- a/esphome/components/runtime_image/bmp_decoder.cpp +++ b/esphome/components/runtime_image/bmp_decoder.cpp @@ -80,6 +80,10 @@ int HOT BmpDecoder::decode(uint8_t *buffer, size_t size) { this->width_ = encode_uint32(buffer[21], buffer[20], buffer[19], buffer[18]); this->height_ = encode_uint32(buffer[25], buffer[24], buffer[23], buffer[22]); + if (this->width_ <= 0 || this->height_ <= 0) { + ESP_LOGE(TAG, "Invalid image dimensions: (%zdx%zd)", this->width_, this->height_); + return DECODE_ERROR_UNSUPPORTED_FORMAT; + } this->bits_per_pixel_ = encode_uint16(buffer[29], buffer[28]); this->compression_method_ = encode_uint32(buffer[33], buffer[32], buffer[31], buffer[30]); this->image_data_size_ = encode_uint32(buffer[37], buffer[36], buffer[35], buffer[34]); From 5df1c7f1d3e2df2c5d4355c1cde8f9882c6b8b25 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Wed, 26 Aug 2026 16:24:09 -0700 Subject: [PATCH 20/30] [modbus_controller] Writer entities as their own hub device; heap-free, byte-accurate write path (#18082) Co-authored-by: J. Nick Koston --- esphome/components/modbus/helpers.py | 3 + .../components/modbus_controller/__init__.py | 30 +++-- .../modbus_controller/modbus_controller.cpp | 67 ++++++++++ .../modbus_controller/modbus_controller.h | 118 +++++++++++++++++- .../modbus_controller/number/__init__.py | 10 +- .../number/modbus_number.cpp | 98 ++++++++------- .../modbus_controller/number/modbus_number.h | 7 +- .../modbus_controller/output/__init__.py | 13 +- .../output/modbus_output.cpp | 115 +++++++++-------- .../modbus_controller/output/modbus_output.h | 24 ++-- .../modbus_controller/select/__init__.py | 20 ++- .../select/modbus_select.cpp | 55 ++++---- .../modbus_controller/select/modbus_select.h | 7 +- .../modbus_controller/switch/__init__.py | 7 +- .../switch/modbus_switch.cpp | 93 +++++++------- .../modbus_controller/switch/modbus_switch.h | 7 +- .../command_payload_test.cpp | 4 +- .../uart_mock_modbus_lambda_write.yaml | 97 ++++++++++++++ tests/integration/test_uart_mock_modbus.py | 58 ++++++--- 19 files changed, 598 insertions(+), 235 deletions(-) create mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml diff --git a/esphome/components/modbus/helpers.py b/esphome/components/modbus/helpers.py index e7eaacee0c..ec95b82045 100644 --- a/esphome/components/modbus/helpers.py +++ b/esphome/components/modbus/helpers.py @@ -3,6 +3,9 @@ import esphome.codegen as cg modbus_ns = cg.esphome_ns.namespace("modbus") modbus_helpers_ns = modbus_ns.namespace("helpers") +RegisterValues = modbus_ns.class_("RegisterValues") +PduBuffer = modbus_helpers_ns.class_("PduBuffer") + FunctionCode_ns = modbus_ns.namespace("FunctionCode") FunctionCode = FunctionCode_ns.enum("FunctionCode") diff --git a/esphome/components/modbus_controller/__init__.py b/esphome/components/modbus_controller/__init__.py index 924a260d37..e87eccb32c 100644 --- a/esphome/components/modbus_controller/__init__.py +++ b/esphome/components/modbus_controller/__init__.py @@ -191,7 +191,7 @@ ModbusItemBaseSchema = cv.Schema( ) -def validate_modbus_register(config): +def validate_modbus_register(config: ConfigType) -> ConfigType: # custom_command is the deprecated alias for custom_pdu (migrated later in final validate); treat # either as "a custom frame is configured" so the address/register_type rules match. has_custom = CONF_CUSTOM_PDU in config or CONF_CUSTOM_COMMAND in config @@ -278,7 +278,7 @@ def _final_validate(config: ConfigType) -> None: FINAL_VALIDATE_SCHEMA = _final_validate -def modbus_calc_properties(config): +def modbus_calc_properties(config: ConfigType) -> tuple[int, int]: byte_offset = 0 reg_count = 0 if CONF_OFFSET in config: @@ -307,8 +307,12 @@ def modbus_calc_properties(config): async def add_modbus_base_properties( - var, config, sensor_type, lambda_param_type=cg.float_, lambda_return_type=float -): + var: cg.MockObj, + config: ConfigType, + sensor_type: cg.MockObjClass, + lambda_param_type: cg.MockObj = cg.float_, + lambda_return_type: Any = float, +) -> None: if CONF_CUSTOM_PDU in config: cg.add(var.set_custom_pdu(config[CONF_CUSTOM_PDU])) @@ -347,8 +351,11 @@ _CALLBACK_AUTOMATIONS = ( ) -async def to_code(config): - var = cg.new_Pvariable(config[CONF_ID]) +async def to_code(config: ConfigType) -> None: + # Await the hub first, so no entity can bind to a controller that doesn't have one yet. + hub = await cg.get_variable(config[modbus.CONF_MODBUS_ID]) + var = cg.new_Pvariable(config[CONF_ID], hub, config[CONF_ADDRESS]) + await cg.register_component(var, config) cg.add(var.set_max_cmd_retries(config[CONF_MAX_CMD_RETRIES])) cg.add(var.set_offline_skip_updates(config[CONF_OFFLINE_SKIP_UPDATES])) cg.add( @@ -356,17 +363,22 @@ async def to_code(config): modbus.command_options_expression(config, direction="read") ) ) - await register_modbus_device(var, config) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) -async def register_modbus_device(var, config): +async def register_modbus_device(var: cg.MockObj, config: ConfigType) -> cg.MockObj: + # Remove before 2027.3.0 + _LOGGER.warning( + "'modbus_controller.register_modbus_device' is deprecated, use " + "'modbus.register_modbus_client_device' and set the address on your own " + "class instead. Will be removed in 2027.3.0" + ) cg.add(var.set_address(config[CONF_ADDRESS])) await cg.register_component(var, config) return await modbus.register_modbus_client_device(var, config) -def function_code_to_register(function_code): +def function_code_to_register(function_code: str) -> cg.MockObj: FUNCTION_CODE_TYPE_MAP = { "read_coils": EntityType.COIL, "read_discrete_inputs": EntityType.DISCRETE_INPUT, diff --git a/esphome/components/modbus_controller/modbus_controller.cpp b/esphome/components/modbus_controller/modbus_controller.cpp index 20b8f516e9..9d7b719e15 100644 --- a/esphome/components/modbus_controller/modbus_controller.cpp +++ b/esphome/components/modbus_controller/modbus_controller.cpp @@ -10,6 +10,73 @@ static const char *const TAG = "modbus_controller"; void ModbusController::setup() { this->create_polling_commands_(); } +void WriterDevice::warn_write_buffer_deprecated(const LogString *platform, uint16_t address) { + if (this->write_buffer_deprecated_warned_) + return; + this->write_buffer_deprecated_warned_ = true; + ESP_LOGW(TAG, + "Modbus %s (address 0x%X): filling the write_lambda buffer parameter is deprecated; call a write helper / " + "queue_pdu() on the entity (item) instead. The buffer parameter is removed in 2027.3.0", + LOG_STR_ARG(platform), address); +} + +bool WriterDevice::send_raw_frame_deprecated(std::span frame) { + if (frame.empty()) + return false; + this->dispatched_ = true; + return this->parent_->queue_pdu(frame[0], frame.subspan(1), this); +} + +void WriterDevice::set_controller(ModbusController *controller) { + this->controller_ = controller; + this->set_parent(controller->hub()); + this->set_address(controller->device_address()); +} + +void WriterDevice::notify_online_(std::span request_pdu) { + if (this->controller_ != nullptr) + this->controller_->set_online(true, fc_of(request_pdu), addr_of(request_pdu)); +} + +void WriterDevice::on_response(std::span request_pdu, std::span response_pdu) { + this->notify_online_(request_pdu); + this->dispatch_response_(request_pdu, response_pdu, std::nullopt); +} + +void WriterDevice::on_error(std::span request_pdu, modbus::ExceptionCode exception_code) { + ESP_LOGW(TAG, "Modbus error function code: 0x%X register 0x%X exception: %d", fc_of(request_pdu), + addr_of(request_pdu), static_cast(exception_code)); + this->notify_online_(request_pdu); // an exception is still a legitimate reply -> device is online + this->dispatch_response_(request_pdu, {}, exception_code); +} + +// Fired once per wire transmission (including hub re-queues from a retry), so the on_command_sent trigger +// reflects when the frame actually went out, not when it was queued. +void WriterDevice::on_sent(std::span request_pdu) { + if (this->controller_ != nullptr) + this->controller_->command_sent(fc_of(request_pdu), addr_of(request_pdu)); +} + +void WriterDevice::on_not_sent(std::span request_pdu) { + // Only the offline teardown reaches this (a supersede retires silently), so the frame is genuinely + // lost; a dropped write was already published optimistically, so surface it. + if (modbus::helpers::is_function_code_write(fc_of(request_pdu))) { + ESP_LOGW(TAG, "Write not sent: function 0x%X register 0x%X", fc_of(request_pdu), addr_of(request_pdu)); + } else { + ESP_LOGD(TAG, "Request not sent: function 0x%X register 0x%X", fc_of(request_pdu), addr_of(request_pdu)); + } +} + +bool WriterDevice::on_no_response(std::span request_pdu) { + if (this->controller_ == nullptr) + return false; + this->controller_->increment_non_response_count(); + if (this->controller_->can_send()) + return true; // the hub re-queues the frame it is holding; on_sent fires again on the retry + this->controller_->set_online(false, fc_of(request_pdu), addr_of(request_pdu)); + return false; +} + ModbusCommandItem::ModbusCommandItem(ModbusController &controller, modbus::ModbusClientHub *parent, uint8_t address, RegisterRange &&range) : modbus::ModbusClientDevice(parent, address), diff --git a/esphome/components/modbus_controller/modbus_controller.h b/esphome/components/modbus_controller/modbus_controller.h index 1db07f1ee8..1f36d5a7c8 100644 --- a/esphome/components/modbus_controller/modbus_controller.h +++ b/esphome/components/modbus_controller/modbus_controller.h @@ -232,6 +232,115 @@ struct RegisterRange { SensorSet sensors; // all sensors of this range }; +/// A hub device owned by a writer entity (switch/number/select/output) through WriterEntity. +/// Centralises the feedback to the controller - online/offline tracking, retry counting and the +/// on_command_sent trigger - and records every dispatch, so a write lambda can tell "I sent it myself" +/// from "use the default write". The hub base is inherited protected, so the public members below are +/// the entity's whole request API and nothing can bypass the recording or re-target the device. +class WriterDevice final : protected modbus::ModbusClientDevice { + protected: + void on_response(std::span request_pdu, std::span response_pdu) override; + void on_error(std::span request_pdu, modbus::ExceptionCode exception_code) override; + void on_sent(std::span request_pdu) override; + void on_not_sent(std::span request_pdu) override; + bool on_no_response(std::span request_pdu) override; + + void notify_online_(std::span request_pdu); + /// Function code / register address decoded from a request PDU ([fc, addr_hi, addr_lo, ...]). + static int fc_of(std::span pdu) { return pdu.empty() ? 0 : (pdu[0] & modbus::FUNCTION_CODE_MASK); } + static int addr_of(std::span pdu) { + return pdu.size() >= 3 ? modbus::helpers::get_data(pdu.data(), 1) : 0; + } + + /// Declared before controller_ so they land in the padding after ModbusClientDevice::custom_response_warned_ + /// instead of adding a word to every entity that owns a device. + /// dispatched_: a frame was queued since the last clear_dispatched_(). + /// write_buffer_deprecated_warned_: warn-once for the legacy write_lambda buffer parameter. + bool dispatched_{false}; + bool write_buffer_deprecated_warned_{false}; + ModbusController *controller_{nullptr}; + + public: + /// Whether a frame was queued to the hub since the last clear_dispatched_(). + bool dispatched() const { return this->dispatched_; } + + bool write_single_register(uint16_t address, uint16_t value) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_single_register(address, value); + } + bool write_single_coil(uint16_t address, bool value) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_single_coil(address, value); + } + bool write_multiple_registers(uint16_t address, std::span values) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_multiple_registers(address, values); + } + bool write_multiple_coils(uint16_t address, std::span values) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_multiple_coils(address, values); + } + bool write_multiple_coils(uint16_t address, modbus::PackedBits bits) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::write_multiple_coils(address, bits); + } + bool queue_pdu(std::span pdu, modbus::CommandOptions options = {}) { + this->dispatched_ = true; + return modbus::ModbusClientDevice::queue_pdu(pdu, options); + } + /// Send a legacy raw frame (address + function code + data) to the frame's own address. + /// Serves only the deprecated write_lambda buffer path. Remove before 2027.3.0. + bool send_raw_frame_deprecated(std::span frame); + + void clear_tx_queue_for_device() { modbus::ModbusClientDevice::clear_tx_queue_for_device(); } + + // Entity plumbing, public because the owning WriterEntity holds the only reachable instance (device_ is + // protected there and the hub sees just the masked base) - reachability is the access gate, not a friend. + void set_controller(ModbusController *controller); + void clear_dispatched() { this->dispatched_ = false; } + /// Warn once per entity that filling the write_lambda buffer parameter is deprecated (the entity is now the + /// command - call a write helper / queue_pdu() on `item` instead). The buffer parameter is removed in 2027.3.0. + void warn_write_buffer_deprecated(const LogString *platform, uint16_t address); +}; + +/// Gives a writer entity the write API of the WriterDevice it owns. The device is a member, not a base: +/// the mixin declares no virtual function, so an entity mixing it in gains no second vtable and all the +/// writer platforms share the single WriterDevice vtable instead of each emitting its own copy. +/// The forwarders keep `item->write_*()` working unchanged inside a write_lambda. +class WriterEntity { + public: + bool dispatched() const { return this->device_.dispatched(); } + bool write_single_register(uint16_t address, uint16_t value) { + return this->device_.write_single_register(address, value); + } + bool write_single_coil(uint16_t address, bool value) { return this->device_.write_single_coil(address, value); } + bool write_multiple_registers(uint16_t address, std::span values) { + return this->device_.write_multiple_registers(address, values); + } + bool write_multiple_coils(uint16_t address, std::span values) { + return this->device_.write_multiple_coils(address, values); + } + bool write_multiple_coils(uint16_t address, modbus::PackedBits bits) { + return this->device_.write_multiple_coils(address, bits); + } + bool queue_pdu(std::span pdu, modbus::CommandOptions options = {}) { + return this->device_.queue_pdu(pdu, options); + } + void clear_tx_queue_for_device() { this->device_.clear_tx_queue_for_device(); } + + protected: + bool send_raw_frame_deprecated_(std::span frame) { + return this->device_.send_raw_frame_deprecated(frame); + } + void set_controller_(ModbusController *controller) { this->device_.set_controller(controller); } + void clear_dispatched_() { this->device_.clear_dispatched(); } + void warn_write_buffer_deprecated_(const LogString *platform, uint16_t address) { + this->device_.warn_write_buffer_deprecated(platform, address); + } + + WriterDevice device_; +}; + /// A single modbus command. Each command is its own ModbusClientDevice: it sends its frame to the hub /// and the hub routes the response back to this object's on_modbus_* callbacks, so the controller no /// longer has to match responses to a FIFO queue. @@ -398,17 +507,16 @@ inline bool offline_retry_due(uint16_t update_counter, uint16_t module_offline_a class ModbusController final : public PollingComponent { public: + // The controller is not itself a modbus device - its commands and writer entities send as their own + // devices, built against this hub + address. + ModbusController(modbus::ModbusClientHub *hub, uint8_t address) : hub_(hub), address_(address) {} + void dump_config() override; // No loop() override: the hub owns transmit/receive timing and each command routes its own // response, so the controller never joins the looping components at all. void setup() override; void update() override; - // The controller is not itself a modbus device - its commands and writer entities send as their own - // devices. It only owns the hub + address so those senders can be built against them. - void set_parent(modbus::ModbusClientHub *hub) { this->hub_ = hub; } - void set_address(uint8_t address) { this->address_ = address; } - /// The hub and modbus address this controller talks to. Used to build commands/entities that send as /// their own device. modbus::ModbusClientHub *hub() const { return this->hub_; } diff --git a/esphome/components/modbus_controller/number/__init__.py b/esphome/components/modbus_controller/number/__init__.py index a43e10a51e..6a5b7041b8 100644 --- a/esphome/components/modbus_controller/number/__init__.py +++ b/esphome/components/modbus_controller/number/__init__.py @@ -3,6 +3,7 @@ from esphome.components import number from esphome.components.modbus.helpers import ( MODBUS_WRITE_REGISTER_TYPE, SENSOR_VALUE_TYPE, + RegisterValues, ) import esphome.config_validation as cv from esphome.const import ( @@ -13,6 +14,7 @@ from esphome.const import ( CONF_MULTIPLY, CONF_STEP, ) +from esphome.types import ConfigType from .. import ( ModbusItemBaseSchema, @@ -43,7 +45,7 @@ ModbusNumber = modbus_controller_ns.class_( ) -def validate_min_max(config): +def validate_min_max(config: ConfigType) -> ConfigType: if config[CONF_MAX_VALUE] <= config[CONF_MIN_VALUE]: raise cv.Invalid("max_value must be greater than min_value") if config[CONF_MIN_VALUE] < -16777215: @@ -53,7 +55,7 @@ def validate_min_max(config): return config -def validate_modbus_number(config): +def validate_modbus_number(config: ConfigType) -> ConfigType: # custom_command is the deprecated alias for custom_pdu (migrated later in final validate). has_custom = CONF_CUSTOM_PDU in config or CONF_CUSTOM_COMMAND in config if not has_custom and CONF_ADDRESS not in config: @@ -89,7 +91,7 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item -async def to_code(config): +async def to_code(config: ConfigType) -> None: byte_offset, reg_count = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], @@ -124,7 +126,7 @@ async def to_code(config): [ (ModbusNumber.operator("ptr"), "item"), (cg.float_, "x"), - (cg.std_vector.template(cg.uint16).operator("ref"), "payload"), + (RegisterValues.operator("ref"), "payload"), ], return_type=cg.optional.template(float), ) diff --git a/esphome/components/modbus_controller/number/modbus_number.cpp b/esphome/components/modbus_controller/number/modbus_number.cpp index 7903b2e317..e890a2a9ac 100644 --- a/esphome/components/modbus_controller/number/modbus_number.cpp +++ b/esphome/components/modbus_controller/number/modbus_number.cpp @@ -1,4 +1,3 @@ -#include #include "modbus_number.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -29,62 +28,73 @@ void ModbusNumber::parse_and_publish(std::span data) { } void ModbusNumber::control(float value) { - optional write_cmd; - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::RegisterValues data; float write_value = value; - // Is there are lambda configured? if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*(), override the value (return a value), or + // (deprecated) fill `data` with the register words to write. auto val = (*this->write_transform_func_)(this, value, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - write_value = val.value(); - } else { + if (this->dispatched()) { + this->publish_state(value); + return; + } + if (!data.empty()) { + // Deprecated buffer path (frozen): the lambda filled a legacy raw frame as words; pack it big-endian. + this->warn_write_buffer_deprecated_(LOG_STR("number"), this->start_address); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE + char hex_buf[format_hex_pretty_uint16_size(MODBUS_NUMBER_MAX_LOG_REGISTERS)]; +#endif + ESP_LOGV(TAG, "Modbus Number write raw: %s", + format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); + // Sized to hold RegisterValues at capacity, so a full buffer can never truncate into a valid frame. + StaticVector bytes; + for (uint16_t word : data) { + const auto word_bytes = decode_value(word); + bytes.push_back(word_bytes[0]); + bytes.push_back(word_bytes[1]); + } + if (!this->send_raw_frame_deprecated_(std::span(bytes.data(), bytes.size()))) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } + this->publish_state(value); + return; + } + if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; } + ESP_LOGV(TAG, "Value overwritten by lambda"); + write_value = val.value(); } else { write_value = this->multiply_by_ * write_value; } - if (!data.empty()) { -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_uint16_size(MODBUS_NUMBER_MAX_LOG_REGISTERS)]; -#endif - ESP_LOGV(TAG, "Modbus Number write raw: %s", - format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - write_cmd.emplace(ModbusCommandItem::create_custom_command( - this->parent_, data, - [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { - this->parent_->on_write_register_response(register_type, this->start_address, data); - })); - } else { - std::vector payload; - modbus::helpers::float_to_payload(payload, write_value, this->sensor_value_type); + modbus::helpers::float_to_payload(data, write_value, this->sensor_value_type); + // float_to_payload() appends nothing for RAW, so an empty payload must be caught before data[0] below. + if (data.empty()) { + ESP_LOGW(TAG, "No payload was created for updating number"); + return; + } - ESP_LOGD(TAG, - "Updating register: connected Sensor=%s start address=0x%X register count=%d new value=%.02f (val=%.02f)", - this->get_name().c_str(), this->start_address, this->register_count, value, write_value); + ESP_LOGD(TAG, + "Updating register: connected Sensor=%s start address=0x%X register count=%d new value=%.02f (val=%.02f)", + this->get_name().c_str(), this->start_address, this->register_count, value, write_value); - // Create and send the write command - if (this->register_count == 1 && !this->use_write_multiple_) { - write_cmd.emplace( - ModbusCommandItem::create_write_single_command(this->parent_, this->write_address(), payload[0])); - } else { - write_cmd.emplace(ModbusCommandItem::create_write_multiple_command(this->parent_, this->write_address(), - this->register_count, payload)); - } - // publish new value - write_cmd->on_data_func = [this, value](modbus::EntityType register_type, uint16_t start_address, - std::span data) { - // gets called when the write command is ack'd from the device - this->parent_->on_write_register_response(register_type, start_address, data); - this->publish_state(value); - }; + bool queued; + if (this->register_count == 1 && !this->use_write_multiple_) { + queued = this->write_single_register(this->write_address(), data[0]); + } else { + queued = this->write_multiple_registers(this->write_address(), data); + } + if (!queued) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; } - this->parent_->queue_command(std::move(*write_cmd)); this->publish_state(value); } void ModbusNumber::dump_config() { LOG_NUMBER(TAG, "Modbus Number", this); } diff --git a/esphome/components/modbus_controller/number/modbus_number.h b/esphome/components/modbus_controller/number/modbus_number.h index 538a982f80..59c76e18f2 100644 --- a/esphome/components/modbus_controller/number/modbus_number.h +++ b/esphome/components/modbus_controller/number/modbus_number.h @@ -10,7 +10,7 @@ namespace esphome::modbus_controller { using value_to_data_t = std::function(float); -class ModbusNumber final : public number::Number, public Component, public SensorItem { +class ModbusNumber final : public number::Number, public Component, public SensorItem, public WriterEntity { public: ModbusNumber(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, SensorValueType value_type, int register_count, bool force_new_range) { @@ -26,11 +26,11 @@ class ModbusNumber final : public number::Number, public Component, public Senso void dump_config() override; void parse_and_publish(std::span data) override; float get_setup_priority() const override { return setup_priority::HARDWARE; } - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } void set_write_multiply(float factor) { this->multiply_by_ = factor; } using transform_func_t = optional (*)(ModbusNumber *, float, std::span); - using write_transform_func_t = optional (*)(ModbusNumber *, float, std::vector &); + using write_transform_func_t = optional (*)(ModbusNumber *, float, modbus::RegisterValues &); void set_template(transform_func_t f) { this->transform_func_ = f; } void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } @@ -39,7 +39,6 @@ class ModbusNumber final : public number::Number, public Component, public Senso void control(float value) override; optional transform_func_{nullopt}; optional write_transform_func_{nullopt}; - ModbusController *parent_{nullptr}; float multiply_by_{1.0}; bool use_write_multiple_{false}; }; diff --git a/esphome/components/modbus_controller/output/__init__.py b/esphome/components/modbus_controller/output/__init__.py index 178c99caa1..34a0f488ec 100644 --- a/esphome/components/modbus_controller/output/__init__.py +++ b/esphome/components/modbus_controller/output/__init__.py @@ -1,8 +1,13 @@ import esphome.codegen as cg from esphome.components import output -from esphome.components.modbus.helpers import SENSOR_VALUE_TYPE +from esphome.components.modbus.helpers import ( + SENSOR_VALUE_TYPE, + PduBuffer, + RegisterValues, +) import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID, CONF_MULTIPLY +from esphome.types import ConfigType from .. import ( ModbusItemBaseSchema, @@ -73,7 +78,7 @@ CONFIG_SCHEMA = cv.typed_schema( ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: byte_offset, reg_count = modbus_calc_properties(config) # Binary Output write_template = None @@ -89,7 +94,7 @@ async def to_code(config): [ (ModbusBinaryOutput.operator("ptr"), "item"), (cg.bool_, "x"), - (cg.std_vector.template(cg.uint8).operator("ref"), "payload"), + (PduBuffer.operator("ref"), "payload"), ], return_type=cg.optional.template(bool), ) @@ -109,7 +114,7 @@ async def to_code(config): [ (ModbusFloatOutput.operator("ptr"), "item"), (cg.float_, "x"), - (cg.std_vector.template(cg.uint16).operator("ref"), "payload"), + (RegisterValues.operator("ref"), "payload"), ], return_type=cg.optional.template(float), ) diff --git a/esphome/components/modbus_controller/output/modbus_output.cpp b/esphome/components/modbus_controller/output/modbus_output.cpp index 48249f4387..b05d3889fd 100644 --- a/esphome/components/modbus_controller/output/modbus_output.cpp +++ b/esphome/components/modbus_controller/output/modbus_output.cpp @@ -2,6 +2,8 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include + namespace esphome::modbus_controller { static const char *const TAG = "modbus_controller.output"; @@ -13,25 +15,33 @@ static constexpr size_t MODBUS_OUTPUT_MAX_LOG_BYTES = 64; * */ void ModbusFloatOutput::write_state(float value) { - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::RegisterValues data; auto original_value = value; - // Is there are lambda configured? if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*(), override the value (return a value), or + // (deprecated) fill `data` with the register words to write. auto val = (*this->write_transform_func_)(this, value, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - value = val.value(); - } else { + if (this->dispatched()) { + return; + } + if (!data.empty()) { + // Deprecated buffer path (frozen): the lambda supplied the register words for the shared write below. + this->warn_write_buffer_deprecated_(LOG_STR("float output"), this->start_address); + } else if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; + } else { + ESP_LOGV(TAG, "Value overwritten by lambda"); + value = val.value(); } } else { value = this->multiply_by_ * value; } - // lambda didn't set payload + if (data.empty()) { modbus::helpers::float_to_payload(data, value, this->sensor_value_type); } @@ -57,16 +67,15 @@ void ModbusFloatOutput::write_state(float value) { return; } - // Create and send the write command - optional write_cmd; + bool queued; if (this->register_count == 1 && !this->use_write_multiple_) { - write_cmd.emplace( - ModbusCommandItem::create_write_single_command(this->parent_, this->start_address + this->offset, data[0])); + queued = this->write_single_register(this->write_address(), data[0]); } else { - write_cmd.emplace(ModbusCommandItem::create_write_multiple_command( - this->parent_, this->start_address + this->offset, data.size(), data)); + queued = this->write_multiple_registers(this->write_address(), data); + } + if (!queued) { + ESP_LOGW(TAG, "Modbus output write (address 0x%X) was refused by the hub", this->write_address()); } - this->parent_->queue_command(std::move(*write_cmd)); } void ModbusFloatOutput::dump_config() { @@ -81,50 +90,52 @@ void ModbusFloatOutput::dump_config() { // ModbusBinaryOutput void ModbusBinaryOutput::write_state(bool state) { - // This will be called every time the user requests a state change. - optional cmd; - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::helpers::PduBuffer data; - // Is there are lambda configured? if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*/queue_pdu(), override the value (return a value), + // or (deprecated) fill `data` with a custom PDU. auto val = (*this->write_transform_func_)(this, state, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - state = val.value(); - } else { + if (this->dispatched()) { + return; + } + if (!data.empty()) { + this->warn_write_buffer_deprecated_(LOG_STR("binary output"), this->start_address); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE + char hex_buf[format_hex_pretty_size(MODBUS_OUTPUT_MAX_LOG_BYTES)]; +#endif + ESP_LOGV(TAG, "Modbus binary output write raw: %s", + format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); + // The lambda filled a legacy raw frame (device address + function code + data). + if (!this->send_raw_frame_deprecated_(data)) { + ESP_LOGW(TAG, "Modbus output write (address 0x%X) was refused by the hub", this->write_address()); + } + return; + } + if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; } + ESP_LOGV(TAG, "Value overwritten by lambda"); + state = val.value(); } - if (!data.empty()) { -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_size(MODBUS_OUTPUT_MAX_LOG_BYTES)]; -#endif - ESP_LOGV(TAG, "Modbus binary output write raw: %s", - format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - cmd.emplace(ModbusCommandItem::create_custom_command( - this->parent_, data, - [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { - this->parent_->on_write_register_response(register_type, this->start_address, data); - })); + ESP_LOGV(TAG, "Write new state: value is %s, type is %d address = %X, offset = %x", ONOFF(state), + (int) this->register_type, this->start_address, this->offset); + // offset for coil and discrete inputs is the coil/register number not bytes + bool queued; + if (this->use_write_multiple_) { + std::array states{state}; + queued = this->write_multiple_coils(this->write_address(), states); } else { - ESP_LOGV(TAG, "Write new state: value is %s, type is %d address = %X, offset = %x", ONOFF(state), - (int) this->register_type, this->start_address, this->offset); - - // offset for coil and discrete inputs is the coil/register number not bytes - if (this->use_write_multiple_) { - std::vector states{state}; - cmd.emplace( - ModbusCommandItem::create_write_multiple_coils(this->parent_, this->start_address + this->offset, states)); - } else { - cmd.emplace( - ModbusCommandItem::create_write_single_coil(this->parent_, this->start_address + this->offset, state)); - } + queued = this->write_single_coil(this->write_address(), state); + } + if (!queued) { + ESP_LOGW(TAG, "Modbus output write (address 0x%X) was refused by the hub", this->write_address()); } - this->parent_->queue_command(std::move(*cmd)); } void ModbusBinaryOutput::dump_config() { diff --git a/esphome/components/modbus_controller/output/modbus_output.h b/esphome/components/modbus_controller/output/modbus_output.h index e79c442aa4..b942dcea62 100644 --- a/esphome/components/modbus_controller/output/modbus_output.h +++ b/esphome/components/modbus_controller/output/modbus_output.h @@ -8,26 +8,24 @@ namespace esphome::modbus_controller { -class ModbusFloatOutput final : public output::FloatOutput, public Component, public SensorItem { +class ModbusFloatOutput final : public output::FloatOutput, public Component, public SensorItem, public WriterEntity { public: ModbusFloatOutput(uint16_t start_address, uint8_t offset, SensorValueType value_type, int register_count) { this->register_type = modbus::EntityType::HOLDING; - this->set_address(start_address); - this->set_offset_from_start_address(offset); + this->set_address(start_address + offset); + this->set_offset_from_start_address(0); this->bitmask = 0xFFFFFFFF; this->register_count = register_count; this->sensor_value_type = value_type; - this->set_address(this->start_address + offset); - this->set_offset_from_start_address(0); } void dump_config() override; - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } void set_write_multiply(float factor) { this->multiply_by_ = factor; } // Do nothing void parse_and_publish(std::span data) override{}; - using write_transform_func_t = optional (*)(ModbusFloatOutput *, float, std::vector &); + using write_transform_func_t = optional (*)(ModbusFloatOutput *, float, modbus::RegisterValues &); void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } @@ -35,29 +33,28 @@ class ModbusFloatOutput final : public output::FloatOutput, public Component, pu void write_state(float value) override; optional write_transform_func_{nullopt}; - ModbusController *parent_{nullptr}; float multiply_by_{1.0}; bool use_write_multiple_{false}; }; -class ModbusBinaryOutput final : public output::BinaryOutput, public Component, public SensorItem { +class ModbusBinaryOutput final : public output::BinaryOutput, public Component, public SensorItem, public WriterEntity { public: ModbusBinaryOutput(uint16_t start_address, uint8_t offset) { this->register_type = modbus::EntityType::COIL; - this->set_address(start_address); + // A coil offset is a coil count; fold it into the address. + this->set_address(start_address + offset); this->bitmask = 0xFFFFFFFF; this->sensor_value_type = SensorValueType::BIT; this->register_count = 1; - this->set_address(this->start_address + offset); this->set_offset_from_start_address(0); } void dump_config() override; - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } // Do nothing void parse_and_publish(std::span data) override{}; - using write_transform_func_t = optional (*)(ModbusBinaryOutput *, bool, std::vector &); + using write_transform_func_t = optional (*)(ModbusBinaryOutput *, bool, modbus::helpers::PduBuffer &); void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } @@ -65,7 +62,6 @@ class ModbusBinaryOutput final : public output::BinaryOutput, public Component, void write_state(bool state) override; optional write_transform_func_{nullopt}; - ModbusController *parent_{nullptr}; bool use_write_multiple_{false}; }; diff --git a/esphome/components/modbus_controller/select/__init__.py b/esphome/components/modbus_controller/select/__init__.py index 1d77f9235d..07893e3303 100644 --- a/esphome/components/modbus_controller/select/__init__.py +++ b/esphome/components/modbus_controller/select/__init__.py @@ -1,8 +1,16 @@ +from collections.abc import Callable +from typing import Any + import esphome.codegen as cg from esphome.components import select -from esphome.components.modbus.helpers import SENSOR_VALUE_TYPE, TYPE_REGISTER_MAP +from esphome.components.modbus.helpers import ( + SENSOR_VALUE_TYPE, + TYPE_REGISTER_MAP, + RegisterValues, +) import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID, CONF_LAMBDA, CONF_OPTIMISTIC +from esphome.types import ConfigType from .. import ( ModbusController, @@ -29,8 +37,8 @@ ModbusSelect = modbus_controller_ns.class_( ) -def ensure_option_map(): - def validator(value): +def ensure_option_map() -> Callable[[Any], dict[str, int]]: + def validator(value: Any) -> dict[str, int]: cv.check_not_templatable(value) option = cv.All(cv.string_strict) mapping = cv.All(cv.int_range(-(2**63), 2**63 - 1)) @@ -47,7 +55,7 @@ def ensure_option_map(): return validator -def register_count_value_type_min(value): +def register_count_value_type_min(value: ConfigType) -> ConfigType: reg_count = value.get(CONF_REGISTER_COUNT) if reg_count is not None: value_type = value[CONF_VALUE_TYPE] @@ -87,7 +95,7 @@ CONFIG_SCHEMA = cv.All( ) -async def to_code(config): +async def to_code(config: ConfigType) -> None: value_type = config[CONF_VALUE_TYPE] reg_count = config.get(CONF_REGISTER_COUNT) if reg_count is None: @@ -132,7 +140,7 @@ async def to_code(config): (ModbusSelect.operator("const_ptr"), "item"), (cg.std_string.operator("const").operator("ref"), "x"), (cg.int64, "value"), - (cg.std_vector.template(cg.uint16).operator("ref"), "payload"), + (RegisterValues.operator("ref"), "payload"), ], return_type=cg.optional.template(cg.int64), ) diff --git a/esphome/components/modbus_controller/select/modbus_select.cpp b/esphome/components/modbus_controller/select/modbus_select.cpp index 0a9383b1b0..c1cc241d6b 100644 --- a/esphome/components/modbus_controller/select/modbus_select.cpp +++ b/esphome/components/modbus_controller/select/modbus_select.cpp @@ -46,35 +46,43 @@ void ModbusSelect::control(size_t index) { const char *option = this->option_at(index); ESP_LOGD(TAG, "Found value %lld for option '%s'", *mapval, option); - std::vector data; + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::RegisterValues data; if (this->write_transform_func_.has_value()) { - // Transform func requires string parameter for backward compatibility + // The lambda may drive the write itself via item->write_*(), override the mapping value (return a value), + // or (deprecated) fill `data` with the register words to write. Transform func requires string parameter + // for backward compatibility. auto val = (*this->write_transform_func_)(this, std::string(option), *mapval, data); - if (val.has_value()) { - mapval = val; - ESP_LOGV(TAG, "write_lambda returned mapping value %lld", *mapval); - } else { + if (this->dispatched()) { + if (this->optimistic_) + this->publish_state(index); + return; + } + if (!data.empty()) { + // Deprecated buffer path (frozen): the lambda supplied the register words for the shared write below. + this->warn_write_buffer_deprecated_(LOG_STR("select"), this->start_address); + } else if (!val.has_value()) { ESP_LOGD(TAG, "Communication handled by write_lambda - exiting control"); return; + } else { + mapval = val; + ESP_LOGV(TAG, "write_lambda returned mapping value %lld", *mapval); } } if (data.empty()) { modbus::helpers::number_to_payload(data, *mapval, this->sensor_value_type); - } else { - ESP_LOGV(TAG, "Using payload from write lambda"); + // number_to_payload() appends nothing for RAW. + if (data.empty()) { + ESP_LOGW(TAG, "No payload was created for updating select"); + return; + } } - if (data.empty()) { - ESP_LOGW(TAG, "No payload was created for updating select"); - return; - } - - // The command declares register_count registers, so the payload must be exactly that many words: - // a value type narrower than the declared width is zero-padded (the config deliberately allows - // register_count larger than the value type). Anything else would put a byte count on the wire - // that disagrees with the quantity field, which conformant devices reject. // register_count declares the READ range width - it may pull neighboring registers into one poll - // so a write covers exactly the registers the value occupies: the quantity comes from the payload, // never from register_count (padding to it would zero registers the user only declared for reading). @@ -86,16 +94,17 @@ void ModbusSelect::control(size_t index) { } const uint16_t write_address = this->write_address(); - optional write_cmd; + bool queued; if ((this->register_count == 1) && (!this->use_write_multiple_)) { - write_cmd.emplace(ModbusCommandItem::create_write_single_command(this->parent_, write_address, data[0])); + queued = this->write_single_register(write_address, data[0]); } else { - write_cmd.emplace( - ModbusCommandItem::create_write_multiple_command(this->parent_, write_address, data.size(), data)); + queued = this->write_multiple_registers(write_address, data); } - this->parent_->queue_command(std::move(*write_cmd)); - + if (!queued) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } if (this->optimistic_) this->publish_state(index); } diff --git a/esphome/components/modbus_controller/select/modbus_select.h b/esphome/components/modbus_controller/select/modbus_select.h index 41ebd4f658..c6ac76a45b 100644 --- a/esphome/components/modbus_controller/select/modbus_select.h +++ b/esphome/components/modbus_controller/select/modbus_select.h @@ -9,7 +9,7 @@ namespace esphome::modbus_controller { -class ModbusSelect final : public Component, public select::Select, public SensorItem { +class ModbusSelect final : public Component, public select::Select, public SensorItem, public WriterEntity { public: ModbusSelect(SensorValueType sensor_value_type, uint16_t start_address, uint8_t register_count, bool force_new_range, std::vector mapping) { @@ -26,9 +26,9 @@ class ModbusSelect final : public Component, public select::Select, public Senso using transform_func_t = optional (*)(ModbusSelect *const, int64_t, std::span); using write_transform_func_t = optional (*)(ModbusSelect *const, const std::string &, int64_t, - std::vector &); + modbus::RegisterValues &); - void set_parent(ModbusController *const parent) { this->parent_ = parent; } + void set_parent(ModbusController *const parent) { this->set_controller_(parent); } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } void set_template(transform_func_t f) { this->transform_func_ = f; } @@ -40,7 +40,6 @@ class ModbusSelect final : public Component, public select::Select, public Senso protected: std::vector mapping_{}; - ModbusController *parent_{nullptr}; bool use_write_multiple_{false}; bool optimistic_{false}; optional transform_func_{nullopt}; diff --git a/esphome/components/modbus_controller/switch/__init__.py b/esphome/components/modbus_controller/switch/__init__.py index dedd2ceedf..c52067f941 100644 --- a/esphome/components/modbus_controller/switch/__init__.py +++ b/esphome/components/modbus_controller/switch/__init__.py @@ -1,8 +1,9 @@ import esphome.codegen as cg from esphome.components import switch -from esphome.components.modbus.helpers import MODBUS_REGISTER_TYPE +from esphome.components.modbus.helpers import MODBUS_REGISTER_TYPE, PduBuffer import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ASSUMED_STATE, CONF_ID +from esphome.types import ConfigType from .. import ( ModbusItemBaseSchema, @@ -48,7 +49,7 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item -async def to_code(config): +async def to_code(config: ConfigType) -> None: byte_offset, _ = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], @@ -74,7 +75,7 @@ async def to_code(config): [ (ModbusSwitch.operator("ptr"), "item"), (cg.bool_, "x"), - (cg.std_vector.template(cg.uint8).operator("ref"), "payload"), + (PduBuffer.operator("ref"), "payload"), ], return_type=cg.optional.template(bool), ) diff --git a/esphome/components/modbus_controller/switch/modbus_switch.cpp b/esphome/components/modbus_controller/switch/modbus_switch.cpp index 810d904d85..c942ff1e6f 100644 --- a/esphome/components/modbus_controller/switch/modbus_switch.cpp +++ b/esphome/components/modbus_controller/switch/modbus_switch.cpp @@ -3,6 +3,8 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include + namespace esphome::modbus_controller { static const char *const TAG = "modbus_controller.switch"; @@ -58,57 +60,64 @@ void ModbusSwitch::parse_and_publish(std::span data) { } void ModbusSwitch::write_state(bool state) { - // This will be called every time the user requests a state change. - optional cmd; - std::vector data; - // Is there are lambda configured? + this->clear_dispatched_(); + // A new write supersedes this entity's own not-yet-sent writes: drop them (and detach any in-flight one) + // so a rapidly-changing value writes the latest, not every intermediate. + this->clear_tx_queue_for_device(); + modbus::helpers::PduBuffer data; if (this->write_transform_func_.has_value()) { - // data is passed by reference - // the lambda can fill the empty vector directly - // in that case the return value is ignored + // The lambda may drive the write itself via item->write_*/queue_pdu(), override the written value (return a + // value), or (deprecated) fill `data` with a custom PDU. auto val = (*this->write_transform_func_)(this, state, data); - if (val.has_value()) { - ESP_LOGV(TAG, "Value overwritten by lambda"); - state = val.value(); - } else { + if (this->dispatched()) { + this->publish_state(state); + return; + } + if (!data.empty()) { + this->warn_write_buffer_deprecated_(LOG_STR("switch"), this->start_address); +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE + char hex_buf[format_hex_pretty_size(MODBUS_SWITCH_MAX_LOG_BYTES)]; +#endif + ESP_LOGV(TAG, "Modbus Switch write raw: %s", + format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); + // The lambda filled a legacy raw frame (device address + function code + data). + if (!this->send_raw_frame_deprecated_(data)) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } + this->publish_state(state); + return; + } + if (!val.has_value()) { ESP_LOGV(TAG, "Communication handled by lambda - exiting control"); return; } + ESP_LOGV(TAG, "Value overwritten by lambda"); + state = val.value(); } - if (!data.empty()) { -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_size(MODBUS_SWITCH_MAX_LOG_BYTES)]; -#endif - ESP_LOGV(TAG, "Modbus Switch write raw: %s", - format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - cmd.emplace(ModbusCommandItem::create_custom_command( - this->parent_, data, - [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { - this->parent_->on_write_register_response(register_type, this->start_address, data); - })); - } else { - ESP_LOGV(TAG, "write_state '%s': new value = %s type = %d address = %X offset = %x", this->get_name().c_str(), - ONOFF(state), (int) this->register_type, this->start_address, this->offset); - if (this->register_type == modbus::EntityType::COIL) { - // offset for coil and discrete inputs is the coil/register number not bytes - if (this->use_write_multiple_) { - std::vector states{state}; - cmd.emplace(ModbusCommandItem::create_write_multiple_coils(this->parent_, this->write_address(), states)); - } else { - cmd.emplace(ModbusCommandItem::create_write_single_coil(this->parent_, this->write_address(), state)); - } + ESP_LOGV(TAG, "write_state '%s': new value = %s type = %d address = %X offset = %x", this->get_name().c_str(), + ONOFF(state), (int) this->register_type, this->start_address, this->offset); + bool queued; + if (this->register_type == EntityType::COIL) { + // offset for coil and discrete inputs is the coil/register number not bytes + if (this->use_write_multiple_) { + std::array states{state}; + queued = this->write_multiple_coils(this->write_address(), states); } else { - if (this->use_write_multiple_) { - std::vector bool_states(1, state ? (0xFFFF & this->bitmask) : 0); - cmd.emplace( - ModbusCommandItem::create_write_multiple_command(this->parent_, this->write_address(), 1, bool_states)); - } else { - cmd.emplace(ModbusCommandItem::create_write_single_command(this->parent_, this->write_address(), - state ? 0xFFFF & this->bitmask : 0u)); - } + queued = this->write_single_coil(this->write_address(), state); + } + } else { + if (this->use_write_multiple_) { + std::array states{static_cast(state ? (0xFFFF & this->bitmask) : 0)}; + queued = this->write_multiple_registers(this->write_address(), states); + } else { + queued = this->write_single_register(this->write_address(), state ? 0xFFFF & this->bitmask : 0u); } } - this->parent_->queue_command(std::move(*cmd)); + if (!queued) { + ESP_LOGW(TAG, "Modbus write for '%s' was refused by the hub; state not published", this->get_name().c_str()); + return; + } this->publish_state(state); } // ModbusSwitch end diff --git a/esphome/components/modbus_controller/switch/modbus_switch.h b/esphome/components/modbus_controller/switch/modbus_switch.h index c21a1939bc..1d3d03919f 100644 --- a/esphome/components/modbus_controller/switch/modbus_switch.h +++ b/esphome/components/modbus_controller/switch/modbus_switch.h @@ -8,7 +8,7 @@ namespace esphome::modbus_controller { -class ModbusSwitch final : public Component, public switch_::Switch, public SensorItem { +class ModbusSwitch final : public Component, public switch_::Switch, public SensorItem, public WriterEntity { public: ModbusSwitch(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, bool force_new_range) { @@ -30,17 +30,16 @@ class ModbusSwitch final : public Component, public switch_::Switch, public Sens void set_assumed_state(bool assumed_state); void set_state(bool state) { this->state = state; } void parse_and_publish(std::span data) override; - void set_parent(ModbusController *parent) { this->parent_ = parent; } + void set_parent(ModbusController *parent) { this->set_controller_(parent); } using transform_func_t = optional (*)(ModbusSwitch *, bool, std::span); - using write_transform_func_t = optional (*)(ModbusSwitch *, bool, std::vector &); + using write_transform_func_t = optional (*)(ModbusSwitch *, bool, modbus::helpers::PduBuffer &); void set_template(transform_func_t f) { this->publish_transform_func_ = f; } void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void set_use_write_mutiple(bool use_write_multiple) { this->use_write_multiple_ = use_write_multiple; } protected: bool assumed_state() override; - ModbusController *parent_{nullptr}; bool use_write_multiple_{false}; optional publish_transform_func_{nullopt}; optional write_transform_func_{nullopt}; diff --git a/tests/components/modbus_controller/command_payload_test.cpp b/tests/components/modbus_controller/command_payload_test.cpp index c125a44da5..a0a59f5106 100644 --- a/tests/components/modbus_controller/command_payload_test.cpp +++ b/tests/components/modbus_controller/command_payload_test.cpp @@ -13,7 +13,7 @@ namespace esphome::modbus_controller::testing { // malformed. Built at its true byte count, the oversize frame is refused by the hub's size check with // a log instead. TEST(ModbusCommandPayload, CoilWritePayloadIsExactSizedNotTruncated) { - ModbusController controller; + ModbusController controller(nullptr, 1); std::vector coils(modbus::MAX_NUM_OF_COILS_TO_WRITE + 1, true); auto cmd = ModbusCommandItem::create_write_multiple_coils(&controller, 0x10, coils); EXPECT_EQ(cmd.payload.size(), modbus::packed_bit_bytes(coils.size())); @@ -21,7 +21,7 @@ TEST(ModbusCommandPayload, CoilWritePayloadIsExactSizedNotTruncated) { // LSB-first packing with zeroed pad bits, matching the wire layout the PDU builders produce. TEST(ModbusCommandPayload, CoilWritePacksLsbFirstWithZeroPad) { - ModbusController controller; + ModbusController controller(nullptr, 1); const std::vector coils{true, false, true, true}; auto cmd = ModbusCommandItem::create_write_multiple_coils(&controller, 0x10, coils); ASSERT_EQ(cmd.payload.size(), 1u); diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml new file mode 100644 index 0000000000..86e17ea0d7 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml @@ -0,0 +1,97 @@ +esphome: + name: uart-mock-modbus-lambda-write + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg30 + type: uint16_t + initial_value: "0" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x30 + value_type: U_WORD + read_lambda: return id(reg30); + write_lambda: id(reg30) = x; return true; + +# A COIL-type switch (assumed_state, write-only) whose write_lambda ignores its own coil type and instead +# drives a HOLDING-REGISTER write on the mock server through the entity itself: `item` IS the command, so +# item->write_single_register() sends a register write from a coil entity (cross-type). Returning nothing +# (an empty optional) tells the write path the lambda already dispatched the frame - no default coil write. +switch: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "cross_switch" + register_type: coil + address: 0x00 + assumed_state: true + write_lambda: |- + item->write_single_register(0x30, x ? 1234 : 0); + return {}; + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_30" + address: 0x30 + register_type: holding + value_type: U_WORD + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index 707637cfc2..3dfeda9b37 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -969,10 +969,10 @@ async def test_uart_mock_modbus_client_read_write( @pytest.mark.xfail( strict=True, - reason="Byte-accurate register-offset writes require the modbus_controller " - "entity-device change; on dev the byte offset is folded into the address " - "(writes 0x12 instead of 0x11). The write and read assertions both flip via " - "the same switch-constructor fold. Remove this marker when that change merges.", + reason="Byte-accurate register-offset writes land in the follow-up offset fix; " + "until then the byte offset is folded into the address (writes 0x12 instead of " + "0x11). The write and read assertions both flip via the same switch-constructor " + "fold. Remove this marker when that change merges.", ) @pytest.mark.asyncio async def test_uart_mock_modbus_register_offset( @@ -1029,14 +1029,42 @@ async def test_uart_mock_modbus_register_offset( ) -@pytest.mark.xfail( - strict=True, - reason="The deprecated write buffer requires the modbus_controller " - "entity-device change; on dev a nullopt-returning write_lambda early-returns " - "before the buffer is used, so the write never happens. The warn-once " - "assertion matches the log substring 'write_lambda buffer'. Remove this " - "marker when that change merges.", -) +@pytest.mark.asyncio +async def test_uart_mock_modbus_lambda_write( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test a write_lambda that drives the write through the entity itself (item is the command). + + `cross_switch` is a coil-type switch whose write_lambda ignores its own type and calls + item->write_single_register(0x30, ...) - a register write issued from a coil entity. The lambda + returns an empty optional, so the write path detects the lambda already dispatched a frame and does + not fall back to the default coil write. Success is reg_30 reading back the value the lambda wrote, + which proves both the new item->write_* path and cross-type flexibility. + """ + + tracker = SensorTracker(["reg_30"]) + initial = tracker.expect("reg_30", 0) + wrote_30 = tracker.expect("reg_30", 1234) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + await tracker.await_change(initial, "reg_30", timeout=4.0) + + switch = find_entity(entities, "cross_switch", SwitchInfo) + assert switch is not None, "cross_switch not found" + client.switch_command(switch.key, True) + + # The coil switch's lambda wrote register 0x30 via item->write_single_register(); reg_30 must + # read back 1234. If the entity-as-command dispatch were broken, no register write would go out + # and this would time out. + await tracker.await_change(wrote_30, "reg_30", timeout=4.0) + + @pytest.mark.asyncio async def test_uart_mock_modbus_deprecated_write_buffer( yaml_config: str, @@ -1046,9 +1074,9 @@ async def test_uart_mock_modbus_deprecated_write_buffer( """Test the deprecated write_lambda buffer path still works, and warns once per entity. buf_number's write_lambda fills the old `payload` buffer with a legacy raw frame as words (device - address + function code + data) instead of calling item->write_*. Two writes must both land with the - legacy raw-frame semantics, and the one-time deprecation warning must fire exactly once per entity - regardless of how many writes happen. + address + function code + data) and returns {} instead of calling item->write_*. Both writes must + land - a filled buffer is sent, as the docs have always described - and the one-time deprecation + warning must fire exactly once per entity regardless of how many writes happen. """ warn_count = 0 From 3361d031de3f47741503ecfcd49202f2027c0e13 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 19:46:13 -0500 Subject: [PATCH 21/30] [api] Drop connection instead of crashing when overflow buffer allocation fails (#18802) --- esphome/components/api/api_frame_helper.cpp | 2 +- esphome/components/api/api_overflow_buffer.cpp | 16 ++++++++++++++-- esphome/components/api/api_overflow_buffer.h | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/api_frame_helper.cpp b/esphome/components/api/api_frame_helper.cpp index 7425304766..38da444a18 100644 --- a/esphome/components/api/api_frame_helper.cpp +++ b/esphome/components/api/api_frame_helper.cpp @@ -172,7 +172,7 @@ APIError APIFrameHelper::write_raw_iov_(const struct iovec *iov, int iovcnt, uin // Queue unsent data into overflow buffer if (!this->overflow_buf_.enqueue_iov(iov, iovcnt, total_write_len, static_cast(sent))) { - HELPER_LOG("Overflow buffer full, dropping connection"); + HELPER_LOG("Overflow buffer full or out of memory, dropping connection"); this->state_ = State::FAILED; return APIError::SOCKET_WRITE_FAILED; } diff --git a/esphome/components/api/api_overflow_buffer.cpp b/esphome/components/api/api_overflow_buffer.cpp index a57a2fb1bb..48d8fe18ba 100644 --- a/esphome/components/api/api_overflow_buffer.cpp +++ b/esphome/components/api/api_overflow_buffer.cpp @@ -1,6 +1,7 @@ #include "api_overflow_buffer.h" #ifdef USE_API #include +#include namespace esphome::api { @@ -61,9 +62,18 @@ bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, uint16_ return false; uint16_t buffer_size = total_len - skip; + // nothrow: a failed allocation returns nullptr so the connection is dropped + // cleanly instead of plain new's crash or abort on OOM // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - auto *entry = new Entry{new uint8_t[buffer_size], buffer_size, 0}; - this->queue_[this->tail_] = entry; + auto *data = new (std::nothrow) uint8_t[buffer_size]; + if (data == nullptr) + return false; + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + auto *entry = new (std::nothrow) Entry{data, buffer_size, 0}; + if (entry == nullptr) { + delete[] data; + return false; + } uint16_t to_skip = skip; uint16_t write_pos = 0; @@ -80,6 +90,8 @@ bool APIOverflowBuffer::enqueue_iov(const struct iovec *iov, int iovcnt, uint16_ } } + // Publish only after the copy completes so a half-built entry is never reachable + this->queue_[this->tail_] = entry; this->tail_ = (this->tail_ + 1) % API_MAX_SEND_QUEUE; this->count_++; return true; diff --git a/esphome/components/api/api_overflow_buffer.h b/esphome/components/api/api_overflow_buffer.h index 1227e83126..03a334b281 100644 --- a/esphome/components/api/api_overflow_buffer.h +++ b/esphome/components/api/api_overflow_buffer.h @@ -61,7 +61,7 @@ class APIOverflowBuffer { /// Enqueue unsent IOV data into the backlog. /// Copies iov data starting at byte offset `skip` into a new entry. - /// Returns false if the queue is full (caller should fail the connection). + /// Returns false if the queue is full or allocation fails (caller should fail the connection). bool enqueue_iov(const struct iovec *iov, int iovcnt, uint16_t total_len, uint16_t skip); protected: From 39177402ddabe5b474894a8f53860dea6a861f4f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 19:46:28 -0500 Subject: [PATCH 22/30] [api] Deprecate media player supports_pause field (#18801) --- esphome/components/api/api.proto | 3 ++- esphome/components/api/api_connection.cpp | 1 - esphome/components/api/api_pb2.cpp | 2 -- esphome/components/api/api_pb2.h | 3 +-- esphome/components/api/api_pb2_dump.cpp | 1 - 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 1942ff568b..c11700782e 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1654,7 +1654,8 @@ message ListEntitiesMediaPlayerResponse { bool disabled_by_default = 6; EntityCategory entity_category = 7; - bool supports_pause = 8; + // Deprecated in ESPHome 2026.9.0; use feature_flags instead. + bool supports_pause = 8 [deprecated = true]; repeated MediaPlayerSupportedFormat supported_formats = 9; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9b1026d2a9..7b0cb7069e 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1099,7 +1099,6 @@ uint16_t APIConnection::try_send_media_player_info(EntityBase *entity, APIConnec auto *media_player = static_cast(entity); ListEntitiesMediaPlayerResponse msg; auto traits = media_player->get_traits(); - msg.supports_pause = traits.get_supports_pause(); msg.feature_flags = traits.get_feature_flags(); for (auto &supported_format : traits.get_supported_formats()) { msg.supported_formats.emplace_back(); diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index b5062f9e9f..f56d791b67 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -2323,7 +2323,6 @@ uint8_t *ListEntitiesMediaPlayerResponse::encode(ProtoWriteBuffer &buffer PROTO_ #endif ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 6, this->disabled_by_default); ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 7, static_cast(this->entity_category)); - ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 8, this->supports_pause); for (auto &it : this->supported_formats) { ProtoEncode::encode_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 9, it); } @@ -2343,7 +2342,6 @@ uint32_t ListEntitiesMediaPlayerResponse::calculate_size() const { #endif size += ProtoSize::calc_bool(1, this->disabled_by_default); size += this->entity_category ? 2 : 0; - size += ProtoSize::calc_bool(1, this->supports_pause); if (!this->supported_formats.empty()) { for (const auto &it : this->supported_formats) { size += ProtoSize::calc_message_force(1, it.calculate_size()); diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 48e277fce1..bed28d2956 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1911,11 +1911,10 @@ class MediaPlayerSupportedFormat final : public ProtoMessage { class ListEntitiesMediaPlayerResponse final : public InfoResponseProtoMessage { public: static constexpr uint16_t MESSAGE_TYPE = 63; - static constexpr uint8_t ESTIMATED_SIZE = 80; + static constexpr uint8_t ESTIMATED_SIZE = 78; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_media_player_response"); } #endif - bool supports_pause{false}; std::vector supported_formats{}; uint32_t feature_flags{0}; uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 9f53438531..846c0ad652 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -1962,7 +1962,6 @@ const char *ListEntitiesMediaPlayerResponse::dump_to(DumpBuffer &out) const { #endif dump_field(out, ESPHOME_PSTR("disabled_by_default"), this->disabled_by_default); dump_field(out, ESPHOME_PSTR("entity_category"), static_cast(this->entity_category)); - dump_field(out, ESPHOME_PSTR("supports_pause"), this->supports_pause); for (const auto &it : this->supported_formats) { out.append(4, ' ').append_p(ESPHOME_PSTR("supported_formats")).append(": "); it.dump_to(out); From e458a38f89e7ccfc3d186d6d2f41708168e6f492 Mon Sep 17 00:00:00 2001 From: n-IA-hane <49248235+n-IA-hane@users.noreply.github.com> Date: Thu, 27 Aug 2026 02:55:39 +0200 Subject: [PATCH 23/30] [spi] Use PSRAM DMA for external buffers (#18699) --- esphome/components/spi/__init__.py | 59 +++++ esphome/components/spi/spi.h | 18 ++ esphome/components/spi/spi_esp_idf.cpp | 37 ++- esphome/core/defines.h | 1 + tests/component_tests/spi/test_psram_dma.py | 227 ++++++++++++++++++ .../spi_device/test.esp32-s3-idf.yaml | 10 + 6 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 tests/component_tests/spi/test_psram_dma.py create mode 100644 tests/components/spi_device/test.esp32-s3-idf.yaml diff --git a/esphome/components/spi/__init__.py b/esphome/components/spi/__init__.py index d7b85ee20d..dd2dce01c5 100644 --- a/esphome/components/spi/__init__.py +++ b/esphome/components/spi/__init__.py @@ -15,6 +15,7 @@ from esphome.components.esp32 import ( VARIANT_ESP32P4, VARIANT_ESP32S2, VARIANT_ESP32S3, + VARIANT_ESP32S31, only_on_variant, ) from esphome.config_helpers import filter_source_files_from_platform @@ -126,6 +127,7 @@ CONF_FORCE_SW = "force_sw" CONF_INTERFACE = "interface" CONF_INTERFACE_INDEX = "interface_index" CONF_RELEASE_DEVICE = "release_device" +CONF_PSRAM_DMA = "psram_dma" TYPE_SINGLE = "single" TYPE_QUAD = "quad" TYPE_OCTAL = "octal" @@ -136,6 +138,29 @@ TYPE_CLASS = { TYPE_OCTAL: OctalSPIComponent, } + +def _validate_psram_dma(value: Any) -> bool: + value = cv.boolean(value) + if not value: + return value + return cv.All( + cv.only_on_esp32, + cv.only_with_framework("esp-idf"), + only_on_variant( + supported=[ + VARIANT_ESP32C5, + VARIANT_ESP32C61, + VARIANT_ESP32P4, + VARIANT_ESP32S31, + VARIANT_ESP32S3, + ], + msg_prefix="PSRAM DMA", + ), + cv.require_framework_version(esp_idf=cv.Version(5, 5, 3)), + cv.requires_component("psram"), + )(value) + + # RP2040 SPI pin assignments are complicated; # refer to GPIO function select table in https://datasheets.raspberrypi.com/rp2040/rp2040-datasheet.pdf @@ -450,6 +475,7 @@ def spi_device_schema( SPI_MODE_OPTIONS, upper=True ), cv.Optional(CONF_RELEASE_DEVICE): cv.All(cv.boolean, cv.only_on_esp32), + cv.Optional(CONF_PSRAM_DMA): _validate_psram_dma, cs_pin_option(CONF_CS_PIN): pins.gpio_output_pin_schema, } ) @@ -471,6 +497,9 @@ async def register_spi_device( cg.add(var.set_mode(spi_mode)) if release_device := config.get(CONF_RELEASE_DEVICE): cg.add(var.set_release_device(release_device)) + if psram_dma := config.get(CONF_PSRAM_DMA): + cg.add_define("USE_SPI_PSRAM_DMA") + cg.add(var.set_psram_dma(psram_dma)) def final_validate_device_schema( @@ -498,6 +527,36 @@ def final_validate_device_schema( ) +def _walk_config(value: Any, path: tuple[Any, ...] = ()): + if isinstance(value, dict): + yield value, path + for key, child in value.items(): + yield from _walk_config(child, (*path, key)) + elif isinstance(value, list): + for index, child in enumerate(value): + yield from _walk_config(child, (*path, index)) + + +def _final_validate(config: Any) -> Any: + buses = config if isinstance(config, list) else [config] + software_bus_ids = { + bus[CONF_ID] for bus in buses if CONF_INTERFACE_INDEX not in bus + } + if not software_bus_ids: + return config + for candidate, path in _walk_config(fv.full_config.get()): + if ( + candidate.get(CONF_PSRAM_DMA) + and candidate.get(CONF_SPI_ID) in software_bus_ids + ): + with cv.prepend_path([cv.ROOT_CONFIG_PATH, *path, CONF_PSRAM_DMA]): + raise cv.Invalid("psram_dma requires a hardware SPI interface") + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + FILTER_SOURCE_FILES = filter_source_files_from_platform( { "spi_arduino.cpp": { diff --git a/esphome/components/spi/spi.h b/esphome/components/spi/spi.h index 17c59c895a..f8233c48d1 100644 --- a/esphome/components/spi/spi.h +++ b/esphome/components/spi/spi.h @@ -253,11 +253,18 @@ class SPIDelegate { // check if device is ready virtual bool is_ready(); +#ifdef USE_SPI_PSRAM_DMA + void set_psram_dma(bool enable) { this->psram_dma_ = enable; } +#endif + protected: SPIBitOrder bit_order_{BIT_ORDER_MSB_FIRST}; uint32_t data_rate_{1000000}; SPIMode mode_{MODE0}; GPIOPin *cs_pin_{NullPin::NULL_PIN}; +#ifdef USE_SPI_PSRAM_DMA + bool psram_dma_{false}; +#endif static SPIDelegate *const NULL_DELEGATE; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) }; @@ -397,6 +404,11 @@ class SPIClient { esph_log_d("spi_device", "mode %u, data_rate %ukHz", (unsigned) this->mode_, (unsigned) (this->data_rate_ / 1000)); this->delegate_ = this->parent_->register_device(this, this->mode_, this->bit_order_, this->data_rate_, this->cs_, this->release_device_, this->write_only_); +#ifdef USE_SPI_PSRAM_DMA + this->delegate_->set_psram_dma(this->psram_dma_); + if (this->psram_dma_) + esph_log_config("spi_device", "PSRAM DMA: enabled"); +#endif } virtual void spi_teardown() { @@ -407,6 +419,9 @@ class SPIClient { bool spi_is_ready() { return this->delegate_->is_ready(); } void set_release_device(bool release) { this->release_device_ = release; } void set_write_only(bool write_only) { this->write_only_ = write_only; } +#ifdef USE_SPI_PSRAM_DMA + void set_psram_dma(bool enable) { this->psram_dma_ = enable; } +#endif protected: SPIBitOrder bit_order_{BIT_ORDER_MSB_FIRST}; @@ -416,6 +431,9 @@ class SPIClient { GPIOPin *cs_{nullptr}; bool release_device_{false}; bool write_only_{false}; +#ifdef USE_SPI_PSRAM_DMA + bool psram_dma_{false}; +#endif SPIDelegate *delegate_{SPIDelegate::NULL_DELEGATE}; }; diff --git a/esphome/components/spi/spi_esp_idf.cpp b/esphome/components/spi/spi_esp_idf.cpp index d5d5053117..45d38c1719 100644 --- a/esphome/components/spi/spi_esp_idf.cpp +++ b/esphome/components/spi/spi_esp_idf.cpp @@ -1,12 +1,24 @@ #include "spi.h" #include +#ifdef USE_SPI_PSRAM_DMA +#include +#endif + namespace esphome::spi { #ifdef USE_ESP32 static const char *const TAG = "spi"; static const size_t MAX_TRANSFER_SIZE = 4092; // dictated by ESP-IDF API. +#ifdef USE_SPI_PSRAM_DMA +static uint32_t get_psram_dma_flags(bool enabled, const void *tx_buffer) { + if (enabled && tx_buffer != nullptr && esp_ptr_dma_ext_capable(tx_buffer)) + return SPI_TRANS_DMA_USE_PSRAM; + return 0; +} +#endif + class SPIDelegateHw : public SPIDelegate { public: SPIDelegateHw(SPIInterface channel, uint32_t data_rate, SPIBitOrder bit_order, SPIMode mode, GPIOPin *cs_pin, @@ -65,8 +77,13 @@ class SPIDelegateHw : public SPIDelegate { return; } spi_transaction_t desc = {}; - desc.flags = 0; +#ifdef USE_SPI_PSRAM_DMA + const uint32_t psram_flags = rxbuf == nullptr ? get_psram_dma_flags(this->psram_dma_, txbuf) : 0; +#endif while (length != 0) { +#ifdef USE_SPI_PSRAM_DMA + desc.flags = psram_flags; +#endif size_t const partial = std::min(length, MAX_TRANSFER_SIZE); desc.length = partial * 8; desc.rxlength = this->write_only_ ? 0 : partial * 8; @@ -81,6 +98,12 @@ class SPIDelegateHw : public SPIDelegate { ESP_LOGE(TAG, "Transmit failed - err %X", err); break; } +#ifdef USE_SPI_PSRAM_DMA + if ((desc.flags & SPI_TRANS_DMA_TX_FAIL) != 0) { + ESP_LOGE(TAG, "PSRAM DMA TX underflow"); + break; + } +#endif length -= partial; if (txbuf != nullptr) txbuf += partial; @@ -133,7 +156,13 @@ class SPIDelegateHw : public SPIDelegate { desc.base.rxlength = 0; desc.base.cmd = cmd; desc.base.addr = address; +#ifdef USE_SPI_PSRAM_DMA + const uint32_t transaction_flags = desc.base.flags | get_psram_dma_flags(this->psram_dma_, data); +#endif do { +#ifdef USE_SPI_PSRAM_DMA + desc.base.flags = transaction_flags; +#endif size_t chunk_size = std::min(length, MAX_TRANSFER_SIZE); if (data != nullptr && chunk_size != 0) { desc.base.length = chunk_size * 8; @@ -152,6 +181,12 @@ class SPIDelegateHw : public SPIDelegate { ESP_LOGE(TAG, "Transmit failed - err %X", err); return; } +#ifdef USE_SPI_PSRAM_DMA + if ((desc.base.flags & SPI_TRANS_DMA_TX_FAIL) != 0) { + ESP_LOGE(TAG, "PSRAM DMA TX underflow"); + return; + } +#endif // if more data is to be sent, skip the command and address phases. desc.command_bits = 0; desc.address_bits = 0; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 971ad7c8d9..bea2bed95f 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -355,6 +355,7 @@ #define USE_SPEAKER #define USE_SPEAKER_MEDIA_PLAYER_ON_OFF #define USE_SPI +#define USE_SPI_PSRAM_DMA #define USE_VOICE_ASSISTANT #define USE_WEBSERVER #define USE_WEBSERVER_AUTH diff --git a/tests/component_tests/spi/test_psram_dma.py b/tests/component_tests/spi/test_psram_dma.py new file mode 100644 index 0000000000..f721e0761b --- /dev/null +++ b/tests/component_tests/spi/test_psram_dma.py @@ -0,0 +1,227 @@ +"""Tests for SPI PSRAM DMA configuration validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.esp32 import ( + KEY_BOARD, + KEY_VARIANT, + VARIANT_ESP32, + VARIANT_ESP32S3, + VARIANT_ESP32S31, +) +from esphome.components.spi import ( + CONF_INTERFACE_INDEX, + CONF_PSRAM_DMA, + _final_validate, + spi_device_schema, +) +from esphome.config import Config +from esphome.const import CONF_ID, CONF_SPI_ID, KEY_FRAMEWORK_VERSION, PlatformFramework +from esphome.core import CORE, ID +from tests.component_tests.types import SetCoreConfigCallable + + +def _schema() -> cv.Schema: + return spi_device_schema( + cs_pin_required=False, + default_data_rate="1MHz", + default_mode="MODE0", + ) + + +def _stage( + set_core_config: SetCoreConfigCallable, + platform_framework: PlatformFramework, + variant: str, + version: cv.Version, +) -> None: + set_core_config( + platform_framework, + core_data={KEY_FRAMEWORK_VERSION: version}, + platform_data={KEY_BOARD: "test-board", KEY_VARIANT: variant}, + ) + CORE.loaded_integrations.add("psram") + + +def test_psram_dma_accepts_supported_idf_target( + set_core_config: SetCoreConfigCallable, +) -> None: + _stage( + set_core_config, + PlatformFramework.ESP32_IDF, + VARIANT_ESP32S3, + cv.Version(5, 5, 3), + ) + + config = _schema()({CONF_PSRAM_DMA: True}) + + assert config[CONF_PSRAM_DMA] is True + + +def test_psram_dma_accepts_esp32s31( + set_core_config: SetCoreConfigCallable, +) -> None: + _stage( + set_core_config, + PlatformFramework.ESP32_IDF, + VARIANT_ESP32S31, + cv.Version(6, 0, 0), + ) + + config = _schema()({CONF_PSRAM_DMA: True}) + + assert config[CONF_PSRAM_DMA] is True + + +def test_psram_dma_rejects_arduino( + set_core_config: SetCoreConfigCallable, +) -> None: + _stage( + set_core_config, + PlatformFramework.ESP32_ARDUINO, + VARIANT_ESP32S3, + cv.Version(5, 5, 3), + ) + + with pytest.raises(cv.Invalid, match="only available with framework"): + _schema()({CONF_PSRAM_DMA: True}) + + +def test_psram_dma_rejects_target_without_capability( + set_core_config: SetCoreConfigCallable, +) -> None: + _stage( + set_core_config, + PlatformFramework.ESP32_IDF, + VARIANT_ESP32, + cv.Version(5, 5, 3), + ) + + with pytest.raises(cv.Invalid, match="PSRAM DMA is only available"): + _schema()({CONF_PSRAM_DMA: True}) + + +def test_psram_dma_false_is_portable( + set_core_config: SetCoreConfigCallable, +) -> None: + _stage( + set_core_config, + PlatformFramework.ESP32_ARDUINO, + VARIANT_ESP32, + cv.Version(5, 5, 2), + ) + + config = _schema()({CONF_PSRAM_DMA: False}) + + assert config[CONF_PSRAM_DMA] is False + + +def test_psram_dma_rejects_older_idf( + set_core_config: SetCoreConfigCallable, +) -> None: + _stage( + set_core_config, + PlatformFramework.ESP32_IDF, + VARIANT_ESP32S3, + cv.Version(5, 5, 2), + ) + + with pytest.raises(cv.Invalid, match="requires at least framework version 5.5.3"): + _schema()({CONF_PSRAM_DMA: True}) + + +def test_psram_dma_requires_psram_component( + set_core_config: SetCoreConfigCallable, +) -> None: + _stage( + set_core_config, + PlatformFramework.ESP32_IDF, + VARIANT_ESP32S3, + cv.Version(5, 5, 3), + ) + CORE.loaded_integrations.remove("psram") + + with pytest.raises(cv.Invalid, match="requires component psram"): + _schema()({CONF_PSRAM_DMA: True}) + + +def _full_spi_config(*, hardware: bool, with_device: bool = True) -> tuple[Config, ID]: + bus_id = ID("spi_bus", is_declaration=True, type="SPIComponent") + bus = {CONF_ID: bus_id} + if hardware: + bus[CONF_INTERFACE_INDEX] = 0 + full = Config() + full["spi"] = [bus] + if with_device: + full["spi_device_test"] = { + CONF_SPI_ID: ID("spi_bus"), + CONF_PSRAM_DMA: True, + } + full.declare_ids.append((bus_id, ["spi", 0, CONF_ID])) + return full, ID("spi_bus", is_declaration=False, type="SPIComponent") + + +def test_psram_dma_accepts_hardware_spi( + set_core_config: SetCoreConfigCallable, +) -> None: + full_config, _ = _full_spi_config(hardware=True) + set_core_config(PlatformFramework.ESP32_IDF, full_config=full_config) + + _final_validate(full_config["spi"]) + + +def test_psram_dma_rejects_software_spi( + set_core_config: SetCoreConfigCallable, +) -> None: + full_config, _ = _full_spi_config(hardware=False) + set_core_config(PlatformFramework.ESP32_IDF, full_config=full_config) + + with pytest.raises(cv.Invalid, match="psram_dma requires a hardware SPI") as error: + _final_validate(full_config["spi"]) + + assert error.value.path[-2:] == ["spi_device_test", CONF_PSRAM_DMA] + + +def test_spi_bus_rejects_psram_dma_device_without_component_final_validation( + set_core_config: SetCoreConfigCallable, +) -> None: + full_config, bus_id = _full_spi_config(hardware=False, with_device=False) + full_config["device_without_final_validation"] = { + CONF_SPI_ID: bus_id, + CONF_PSRAM_DMA: True, + } + set_core_config(PlatformFramework.ESP32_IDF, full_config=full_config) + + with pytest.raises(cv.Invalid, match="psram_dma requires a hardware SPI") as error: + _final_validate(full_config["spi"]) + + assert error.value.path[-2:] == [ + "device_without_final_validation", + CONF_PSRAM_DMA, + ] + + +def test_psram_dma_accepts_hardware_device_with_mixed_buses( + set_core_config: SetCoreConfigCallable, +) -> None: + software_bus_id = ID("software_bus", is_declaration=True, type="SPIComponent") + hardware_bus_id = ID("hardware_bus", is_declaration=True, type="SPIComponent") + full_config = Config() + full_config["spi"] = [ + {CONF_ID: software_bus_id}, + {CONF_ID: hardware_bus_id, CONF_INTERFACE_INDEX: 0}, + ] + full_config["spi_device_test"] = { + CONF_SPI_ID: ID("hardware_bus"), + CONF_PSRAM_DMA: True, + } + full_config.declare_ids.extend( + ( + (software_bus_id, ["spi", 0, CONF_ID]), + (hardware_bus_id, ["spi", 1, CONF_ID]), + ) + ) + set_core_config(PlatformFramework.ESP32_IDF, full_config=full_config) + + _final_validate(full_config["spi"]) diff --git a/tests/components/spi_device/test.esp32-s3-idf.yaml b/tests/components/spi_device/test.esp32-s3-idf.yaml new file mode 100644 index 0000000000..41dc178c19 --- /dev/null +++ b/tests/components/spi_device/test.esp32-s3-idf.yaml @@ -0,0 +1,10 @@ +packages: + spi: !include ../../test_build_components/common/spi/esp32-s3-idf.yaml + common: !include common.yaml +psram: + mode: octal +spi_device: + - id: spi_device_psram_dma_test + psram_dma: true + data_rate: 1MHz + spi_mode: 0 From a1b29b0d0bb6d4e751e42ed1517255c66dfb0ef1 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:56:57 -0400 Subject: [PATCH 24/30] [external_components] Log when a source overrides built-in components (#18805) --- .../external_components/__init__.py | 34 ++++- .../external_components/test_init.py | 116 +++++++++++++++++- 2 files changed, 147 insertions(+), 3 deletions(-) diff --git a/esphome/components/external_components/__init__.py b/esphome/components/external_components/__init__.py index c892ec1112..504b1ae679 100644 --- a/esphome/components/external_components/__init__.py +++ b/esphome/components/external_components/__init__.py @@ -71,6 +71,33 @@ def _process_git_config(config: dict[str, Any], refresh: TimePeriodSeconds) -> P return components_dir +def _log_overridden_components( + conf: dict[str, Any], component_names: list[str] +) -> None: + overridden = [ + name + for name in component_names + if (loader.CORE_COMPONENTS_PATH / name / "__init__.py").is_file() + ] + if not overridden: + return + if conf[CONF_TYPE] == TYPE_GIT: + source = conf[CONF_URL] + if ref := conf.get(CONF_REF): + source = f"{source}@{ref}" + if path := conf.get(CONF_PATH): + source = f"{source} ({path})" + else: + source = conf[CONF_PATH] + _LOGGER.info( + "External components are overriding built-in components:\n" + " source: %s\n" + " components: %s", + source, + ", ".join(sorted(overridden)), + ) + + def _process_single_config(config: dict[str, Any]) -> None: conf = config[CONF_SOURCE] if conf[CONF_TYPE] == TYPE_GIT: @@ -84,8 +111,8 @@ def _process_single_config(config: dict[str, Any]) -> None: raise NotImplementedError if config[CONF_COMPONENTS] == "all": - num_components = len(list(components_dir.glob("*/__init__.py"))) - if num_components > 100: + component_names = [p.parent.name for p in components_dir.glob("*/__init__.py")] + if len(component_names) > 100: # Prevent accidentally including all components from an esphome fork/branch # In this case force the user to manually specify which components they want to include raise cv.Invalid( @@ -102,6 +129,9 @@ def _process_single_config(config: dict[str, Any]) -> None: [CONF_COMPONENTS, i], ) allowed_components = config[CONF_COMPONENTS] + component_names = allowed_components + + _log_overridden_components(conf, component_names) loader.install_meta_finder(components_dir, allowed_components=allowed_components) diff --git a/tests/component_tests/external_components/test_init.py b/tests/component_tests/external_components/test_init.py index d3813ecc75..78cb32af54 100644 --- a/tests/component_tests/external_components/test_init.py +++ b/tests/component_tests/external_components/test_init.py @@ -1,16 +1,21 @@ -"""Tests for the external_components skip-update behavior driven by CORE.skip_external_update.""" +"""Tests for the external_components config pass.""" +import logging from pathlib import Path from typing import Any from unittest.mock import MagicMock +import pytest + from esphome.components.external_components import do_external_components_pass from esphome.const import ( CONF_EXTERNAL_COMPONENTS, + CONF_PATH, CONF_REFRESH, CONF_SOURCE, CONF_URL, TYPE_GIT, + TYPE_LOCAL, ) from esphome.core import CORE, TimePeriodSeconds @@ -69,3 +74,112 @@ def test_external_components_normal_refresh( mock_clone_or_update.assert_called_once() call_args = mock_clone_or_update.call_args assert call_args.kwargs["refresh"] == TimePeriodSeconds(days=1) + + +def test_external_components_logs_built_in_override( + tmp_path: Path, + mock_clone_or_update: MagicMock, + mock_install_meta_finder: MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + """A source that provides a component with the same name as a built-in one logs an info message.""" + mock_clone_or_update.return_value = (tmp_path, None) + config = _make_config(tmp_path) + + for name in ("gpio", "some_custom_component"): + component_dir = tmp_path / "components" / name + component_dir.mkdir() + (component_dir / "__init__.py").write_text("# Test component") + + with caplog.at_level(logging.INFO): + do_external_components_pass(config) + + assert ( + "External components are overriding built-in components:\n" + " source: https://github.com/test/components\n" + " components: gpio" in caplog.text + ) + assert "some_custom_component" not in caplog.text + + +def test_external_components_override_log_includes_ref( + tmp_path: Path, + mock_clone_or_update: MagicMock, + mock_install_meta_finder: MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + """A git source with a ref logs the ref appended to the url.""" + mock_clone_or_update.return_value = (tmp_path, None) + config = _make_config(tmp_path) + config[CONF_EXTERNAL_COMPONENTS][0][CONF_SOURCE] = "github://test/components@main" + + component_dir = tmp_path / "components" / "gpio" + component_dir.mkdir() + (component_dir / "__init__.py").write_text("# Test component") + + with caplog.at_level(logging.INFO): + do_external_components_pass(config) + + assert " source: https://github.com/test/components.git@main\n" in caplog.text + + +def test_external_components_override_log_includes_git_path( + tmp_path: Path, + mock_clone_or_update: MagicMock, + mock_install_meta_finder: MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + """A git source with a subdirectory path logs the path after the url.""" + mock_clone_or_update.return_value = (tmp_path, None) + config = _make_config(tmp_path) + config[CONF_EXTERNAL_COMPONENTS][0][CONF_SOURCE][CONF_PATH] = "components" + + component_dir = tmp_path / "components" / "gpio" + component_dir.mkdir() + (component_dir / "__init__.py").write_text("# Test component") + + with caplog.at_level(logging.INFO): + do_external_components_pass(config) + + assert " source: https://github.com/test/components (components)\n" in caplog.text + + +def test_external_components_override_log_local_source( + tmp_path: Path, + mock_install_meta_finder: MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + """A local source logs its resolved path.""" + components_dir = tmp_path / "my_components" + gpio_dir = components_dir / "gpio" + gpio_dir.mkdir(parents=True) + (gpio_dir / "__init__.py").write_text("# Test component") + + CORE.config_path = tmp_path / "dummy.yaml" + config = { + CONF_EXTERNAL_COMPONENTS: [ + {CONF_SOURCE: {"type": TYPE_LOCAL, CONF_PATH: "my_components"}} + ] + } + + with caplog.at_level(logging.INFO): + do_external_components_pass(config) + + assert f" source: {components_dir}\n" in caplog.text + assert " components: gpio" in caplog.text + + +def test_external_components_no_override_no_log( + tmp_path: Path, + mock_clone_or_update: MagicMock, + mock_install_meta_finder: MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + """A source that only provides components not shipped with ESPHome logs nothing.""" + mock_clone_or_update.return_value = (tmp_path, None) + config = _make_config(tmp_path) + + with caplog.at_level(logging.INFO): + do_external_components_pass(config) + + assert "are overriding built-in components" not in caplog.text From b62305793dd5fc21771d9d6f0f03b3d5744f1a5e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 20:40:12 -0500 Subject: [PATCH 25/30] [improv_base] Bump Improv library to 1.2.7 (#18809) --- esphome/components/improv_base/__init__.py | 2 +- platformio.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/improv_base/__init__.py b/esphome/components/improv_base/__init__.py index f132dbacb0..412d143a48 100644 --- a/esphome/components/improv_base/__init__.py +++ b/esphome/components/improv_base/__init__.py @@ -43,4 +43,4 @@ async def setup_improv_core(var: MockObj, config: ConfigType, component: str) -> cg.add(var.set_next_url(_process_next_url(next_url))) cg.add_define(f"USE_{component.upper()}_NEXT_URL") - cg.add_library("improv/Improv", "1.2.6") + cg.add_library("improv/Improv", "1.2.7") diff --git a/platformio.ini b/platformio.ini index b2a36e687c..fcf7caa7c7 100644 --- a/platformio.ini +++ b/platformio.ini @@ -46,7 +46,7 @@ lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea esphome/noise-c@0.1.21 ; noise (api, ota) - improv/Improv@1.2.6 ; improv_serial / esp32_improv + improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library https://github.com/bitbank2/JPEGDEC.git#1.8.4 ; online_image @@ -248,7 +248,7 @@ lib_deps = ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt - improv/Improv@1.2.6 ; improv_serial + improv/Improv@1.2.7 ; improv_serial kikuchan98/pngle@1.1.0 ; online_image https://github.com/bitbank2/JPEGDEC.git#1.8.4 ; online_image build_flags = From e53870085b136c7fd880180ffb0334ab2bcb0035 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 21:13:47 -0500 Subject: [PATCH 26/30] [improv_serial] Add uart bus support and host integration test (#18794) --- esphome/components/host/__init__.py | 1 + esphome/components/improv_serial/__init__.py | 41 ++++- .../improv_serial/improv_serial_component.cpp | 12 +- .../improv_serial/improv_serial_component.h | 18 ++- esphome/core/defines.h | 2 + script/ci-custom.py | 2 + .../improv_serial/common-uart-bus.yaml | 11 ++ .../test-uart-bus.esp32-idf.yaml | 3 + .../test-uart-bus.esp8266-ard.yaml | 3 + .../external_components/wifi/__init__.py | 39 +++++ .../external_components/wifi/scan_list.h | 1 + .../wifi/wifi_component.cpp | 40 +++++ .../external_components/wifi/wifi_component.h | 77 ++++++++++ .../fixtures/improv_serial_uart.yaml | 40 +++++ tests/integration/log_utils.py | 43 ++++++ tests/integration/test_improv_serial_uart.py | 140 ++++++++++++++++++ 16 files changed, 463 insertions(+), 10 deletions(-) create mode 100644 tests/components/improv_serial/common-uart-bus.yaml create mode 100644 tests/components/improv_serial/test-uart-bus.esp32-idf.yaml create mode 100644 tests/components/improv_serial/test-uart-bus.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/external_components/wifi/__init__.py create mode 120000 tests/integration/fixtures/external_components/wifi/scan_list.h create mode 100644 tests/integration/fixtures/external_components/wifi/wifi_component.cpp create mode 100644 tests/integration/fixtures/external_components/wifi/wifi_component.h create mode 100644 tests/integration/fixtures/improv_serial_uart.yaml create mode 100644 tests/integration/log_utils.py create mode 100644 tests/integration/test_improv_serial_uart.py diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index 401bba5118..bd074ab6b5 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -50,6 +50,7 @@ async def to_code(config: ConfigType) -> None: cg.add_define("USE_ESPHOME_HOST_MAC_ADDRESS", config[CONF_MAC_ADDRESS].parts) cg.add_build_flag("-std=gnu++20") cg.add_define("ESPHOME_BOARD", "host") + cg.add_define("ESPHOME_VARIANT", "HOST") cg.add_define(ThreadModel.MULTI_ATOMICS) cg.add_platformio_option("platform", "platformio/native") cg.add_platformio_option("lib_ldf_mode", "off") diff --git a/esphome/components/improv_serial/__init__.py b/esphome/components/improv_serial/__init__.py index 40ef14c6bc..11e9f1ea62 100644 --- a/esphome/components/improv_serial/__init__.py +++ b/esphome/components/improv_serial/__init__.py @@ -1,9 +1,15 @@ import esphome.codegen as cg -from esphome.components import improv_base +from esphome.components import improv_base, uart from esphome.components.esp32 import VARIANT_ESP32S3, get_esp32_variant from esphome.components.logger import USB_CDC import esphome.config_validation as cv -from esphome.const import CONF_BAUD_RATE, CONF_HARDWARE_UART, CONF_ID, CONF_LOGGER +from esphome.const import ( + CONF_BAUD_RATE, + CONF_HARDWARE_UART, + CONF_ID, + CONF_LOGGER, + CONF_UART_ID, +) from esphome.core import CORE import esphome.final_validate as fv from esphome.types import ConfigType @@ -17,13 +23,35 @@ improv_serial_ns = cg.esphome_ns.namespace("improv_serial") ImprovSerialComponent = improv_serial_ns.class_("ImprovSerialComponent", cg.Component) CONFIG_SCHEMA = ( - cv.Schema({cv.GenerateID(): cv.declare_id(ImprovSerialComponent)}) + cv.Schema( + { + cv.GenerateID(): cv.declare_id(ImprovSerialComponent), + # YAML only: rewiring Improv onto another UART is not a knob for a + # visual editor and the device builder must not expose it + cv.Optional(CONF_UART_ID, visibility=cv.Visibility.YAML_ONLY): cv.use_id( + uart.UARTComponent + ), + } + ) .extend(improv_base.IMPROV_SCHEMA) .extend(cv.COMPONENT_SCHEMA) ) -def validate_logger(config: ConfigType) -> None: +_UART_FINAL_VALIDATE = uart.final_validate_device_schema( + "improv_serial", require_tx=True, require_rx=True +) + + +def validate_transport(config: ConfigType) -> None: + if CONF_UART_ID in config: + # A dedicated UART bus is used; the logger's serial settings are irrelevant, + # but the bus itself must be bidirectional and not claimed by another device + _UART_FINAL_VALIDATE(config) + return + # The host logger has no serial port for Improv to share + if CORE.is_host: + raise cv.Invalid("improv_serial on the host platform requires uart_id") logger_conf = fv.full_config.get()[CONF_LOGGER] if logger_conf[CONF_BAUD_RATE] == 0: raise cv.Invalid("improv_serial requires the logger baud_rate to be not 0") @@ -36,7 +64,7 @@ def validate_logger(config: ConfigType) -> None: ) -FINAL_VALIDATE_SCHEMA = validate_logger +FINAL_VALIDATE_SCHEMA = validate_transport async def to_code(config: ConfigType) -> None: @@ -44,3 +72,6 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await improv_base.setup_improv_core(var, config, "improv_serial") cg.add_define("USE_IMPROV_SERIAL") + if (uart_id := config.get(CONF_UART_ID)) is not None: + cg.add(var.set_uart(await cg.get_variable(uart_id))) + cg.add_define("USE_IMPROV_SERIAL_UART") diff --git a/esphome/components/improv_serial/improv_serial_component.cpp b/esphome/components/improv_serial/improv_serial_component.cpp index de9c7899cd..9c7745ee0a 100644 --- a/esphome/components/improv_serial/improv_serial_component.cpp +++ b/esphome/components/improv_serial/improv_serial_component.cpp @@ -15,7 +15,9 @@ static const char *const TAG = "improv_serial"; void ImprovSerialComponent::setup() { global_improv_serial_component = this; -#ifdef USE_ESP32 +#ifdef USE_IMPROV_SERIAL_UART + // Transport is a dedicated UART bus set via set_uart() in generated code +#elif defined(USE_ESP32) this->uart_num_ = logger::global_logger->get_uart_num(); this->uart_selection_ = logger::global_logger->get_uart(); #elif defined(USE_ARDUINO) @@ -89,7 +91,13 @@ void ImprovSerialComponent::write_data_(const uint8_t *data, const size_t size) } this->tx_header_[TX_CHECKSUM_IDX] = checksum; -#ifdef USE_ESP32 +#ifdef USE_IMPROV_SERIAL_UART + this->uart_->write_array(this->tx_header_, header_tx_len); + if (there_is_data) { + this->uart_->write_array(data, size); + this->uart_->write_array(&this->tx_header_[TX_CHECKSUM_IDX], 2); // Footer: checksum and newline + } +#elif defined(USE_ESP32) switch (this->uart_selection_) { case logger::UART_SELECTION_UART0: case logger::UART_SELECTION_UART1: diff --git a/esphome/components/improv_serial/improv_serial_component.h b/esphome/components/improv_serial/improv_serial_component.h index 00c40c4c7e..5a4eaaa945 100644 --- a/esphome/components/improv_serial/improv_serial_component.h +++ b/esphome/components/improv_serial/improv_serial_component.h @@ -10,7 +10,9 @@ #include #include -#ifdef USE_ESP32 +#ifdef USE_IMPROV_SERIAL_UART +#include "esphome/components/uart/uart_component.h" +#elif defined(USE_ESP32) #include #ifdef USE_LOGGER_USB_SERIAL_JTAG #include @@ -53,6 +55,10 @@ class ImprovSerialComponent final : public Component, public improv_base::Improv float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } +#ifdef USE_IMPROV_SERIAL_UART + void set_uart(uart::UARTComponent *uart) { this->uart_ = uart; } +#endif + protected: bool parse_improv_serial_byte_(uint8_t byte); bool parse_improv_payload_(improv::ImprovCommand &command); @@ -69,7 +75,11 @@ class ImprovSerialComponent final : public Component, public improv_base::Improv ESPHOME_ALWAYS_INLINE optional read_byte_() { optional byte; uint8_t data = 0; -#ifdef USE_ESP32 +#ifdef USE_IMPROV_SERIAL_UART + if (this->uart_->available() && this->uart_->read_byte(&data)) { + byte = data; + } +#elif defined(USE_ESP32) switch (this->uart_selection_) { case logger::UART_SELECTION_UART0: case logger::UART_SELECTION_UART1: @@ -129,7 +139,9 @@ class ImprovSerialComponent final : public Component, public improv_base::Improv '\n', }; -#ifdef USE_ESP32 +#ifdef USE_IMPROV_SERIAL_UART + uart::UARTComponent *uart_{nullptr}; +#elif defined(USE_ESP32) uart_port_t uart_num_; logger::UARTSelection uart_selection_{logger::UART_SELECTION_UART0}; #elif defined(USE_ARDUINO) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index bea2bed95f..5b73c43ccd 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -537,6 +537,8 @@ #ifdef USE_HOST #define USE_HTTP_REQUEST_RESPONSE +// Host only: the uart arm would shadow the native logger UART arms in other envs +#define USE_IMPROV_SERIAL_UART #define USE_SOCKET_IMPL_BSD_SOCKETS #define USE_ESPHOME_TASK_LOG_BUFFER #define ESPHOME_TASK_LOG_BUFFER_SIZE 64 diff --git a/script/ci-custom.py b/script/ci-custom.py index 2d2da20995..724a350884 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -247,6 +247,8 @@ def lint_ext_check(fname): "CLAUDE.md", "GEMINI.md", ".github/copilot-instructions.md", + # Symlink to the real wifi scan_list.h so the test stub cannot drift + "tests/integration/fixtures/external_components/wifi/scan_list.h", ] ) def lint_executable_bit(fname: Path) -> str | None: diff --git a/tests/components/improv_serial/common-uart-bus.yaml b/tests/components/improv_serial/common-uart-bus.yaml new file mode 100644 index 0000000000..41ee00fce0 --- /dev/null +++ b/tests/components/improv_serial/common-uart-bus.yaml @@ -0,0 +1,11 @@ +wifi: + ssid: MySSID + password: password1 + +# Serial logging off; on a dedicated UART bus improv_serial must not +# require the logger's serial settings +logger: + baud_rate: 0 + +improv_serial: + uart_id: uart_bus diff --git a/tests/components/improv_serial/test-uart-bus.esp32-idf.yaml b/tests/components/improv_serial/test-uart-bus.esp32-idf.yaml new file mode 100644 index 0000000000..235e3789a4 --- /dev/null +++ b/tests/components/improv_serial/test-uart-bus.esp32-idf.yaml @@ -0,0 +1,3 @@ +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + improv_serial: !include common-uart-bus.yaml diff --git a/tests/components/improv_serial/test-uart-bus.esp8266-ard.yaml b/tests/components/improv_serial/test-uart-bus.esp8266-ard.yaml new file mode 100644 index 0000000000..40a6b7f4fe --- /dev/null +++ b/tests/components/improv_serial/test-uart-bus.esp8266-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + improv_serial: !include common-uart-bus.yaml diff --git a/tests/integration/fixtures/external_components/wifi/__init__.py b/tests/integration/fixtures/external_components/wifi/__init__.py new file mode 100644 index 0000000000..109d56b035 --- /dev/null +++ b/tests/integration/fixtures/external_components/wifi/__init__.py @@ -0,0 +1,39 @@ +"""Host-only stub of the wifi component for integration tests. + +HOST-ONLY TEST COMPONENT: this shadows the real wifi component for EVERY +fixture that uses the shared external_components directory. Any host fixture +with a wifi block gets this stub, not the real component: fixed scan results, +is_connected() hardwired true, and save_wifi_sta that only logs. See +wifi_component.h for the full behavior. +""" + +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_ID, CONF_PASSWORD, CONF_SSID, CONF_USE_ADDRESS +from esphome.types import ConfigType + +CODEOWNERS = ["@esphome/tests"] + +wifi_ns = cg.esphome_ns.namespace("wifi") +WiFiComponent = wifi_ns.class_("WiFiComponent", cg.Component) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(WiFiComponent), + # Accepted for fixture realism; the stub ignores them + cv.Optional(CONF_SSID): cv.string, + cv.Optional(CONF_PASSWORD): cv.string, + # Read by StorageJSON via CORE.address whenever a wifi block exists + cv.Optional(CONF_USE_ADDRESS, default="localhost"): cv.string, + } +).extend(cv.COMPONENT_SCHEMA) + + +def check_placeholder_credentials(config: ConfigType) -> None: + """Compile-time hook the esphome CLI imports from the wifi module; no-op here.""" + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + cg.add_define("USE_WIFI") diff --git a/tests/integration/fixtures/external_components/wifi/scan_list.h b/tests/integration/fixtures/external_components/wifi/scan_list.h new file mode 120000 index 0000000000..fdef6e0be1 --- /dev/null +++ b/tests/integration/fixtures/external_components/wifi/scan_list.h @@ -0,0 +1 @@ +../../../../../esphome/components/wifi/scan_list.h \ No newline at end of file diff --git a/tests/integration/fixtures/external_components/wifi/wifi_component.cpp b/tests/integration/fixtures/external_components/wifi/wifi_component.cpp new file mode 100644 index 0000000000..d1e19a1a0a --- /dev/null +++ b/tests/integration/fixtures/external_components/wifi/wifi_component.cpp @@ -0,0 +1,40 @@ +#include "wifi_component.h" + +#include "esphome/core/log.h" + +namespace esphome::wifi { + +static const char *const TAG = "wifi_stub"; + +WiFiComponent *global_wifi_component = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +WiFiComponent::WiFiComponent() { global_wifi_component = this; } + +void WiFiComponent::setup() { ESP_LOGI(TAG, "Stub wifi ready"); } + +void WiFiComponent::dump_config() { ESP_LOGCONFIG(TAG, "Stub wifi"); } + +void WiFiComponent::start_scanning() { + // Duplicate TestNet entry (weaker) and a hidden entry exercise the + // should_show_scan_entry dedup and filtering logic + this->scan_result_.clear(); + this->scan_result_.emplace_back("TestNet", -50, true, false); + this->scan_result_.emplace_back("TestNet", -60, true, false); + this->scan_result_.emplace_back("OpenNet", -70, false, false); + this->scan_result_.emplace_back("", -40, false, true); + ESP_LOGI(TAG, "Scan complete with %zu results", this->scan_result_.size()); +} + +void WiFiComponent::set_sta(const WiFiAP &ap) { ESP_LOGI(TAG, "set_sta ssid=%s", ap.get_ssid().c_str()); } + +void WiFiComponent::start_connecting(const WiFiAP &ap) { + ESP_LOGI(TAG, "start_connecting ssid=%s", ap.get_ssid().c_str()); +} + +void WiFiComponent::clear_sta() { ESP_LOGI(TAG, "clear_sta"); } + +void WiFiComponent::save_wifi_sta(StringRef ssid, StringRef password) { + ESP_LOGI(TAG, "save_wifi_sta ssid=%s password_len=%zu", ssid.c_str(), password.size()); +} + +} // namespace esphome::wifi diff --git a/tests/integration/fixtures/external_components/wifi/wifi_component.h b/tests/integration/fixtures/external_components/wifi/wifi_component.h new file mode 100644 index 0000000000..a68f811ebd --- /dev/null +++ b/tests/integration/fixtures/external_components/wifi/wifi_component.h @@ -0,0 +1,77 @@ +#pragma once + +// ============================================================================ +// HOST-ONLY TEST COMPONENT — DO NOT COPY TO PRODUCTION CODE +// +// Stub of the real wifi component with just enough API surface for +// improv_serial to build and run on the host platform. Scan results are +// fixed, "connecting" succeeds immediately, and save_wifi_sta only logs so +// tests can assert on the log output. +// ============================================================================ + +#include "esphome/components/network/ip_address.h" +#include "esphome/core/component.h" +#include "esphome/core/string_ref.h" + +#include +#include + +namespace esphome::wifi { + +class WiFiAP { + public: + void set_ssid(const char *ssid) { this->ssid_ = ssid; } + void set_password(const char *password) { this->password_ = password; } + StringRef get_ssid() const { return StringRef(this->ssid_); } + StringRef get_password() const { return StringRef(this->password_); } + + protected: + std::string ssid_; + std::string password_; +}; + +class WiFiScanResult { + public: + WiFiScanResult(const char *ssid, int8_t rssi, bool with_auth, bool hidden) + : ssid_(ssid), rssi_(rssi), with_auth_(with_auth), hidden_(hidden) {} + StringRef get_ssid() const { return StringRef(this->ssid_); } + int8_t get_rssi() const { return this->rssi_; } + bool get_with_auth() const { return this->with_auth_; } + bool get_is_hidden() const { return this->hidden_; } + bool ssid_equals(const WiFiScanResult &other) const { return this->ssid_ == other.ssid_; } + + protected: + std::string ssid_; + int8_t rssi_; + bool with_auth_; + bool hidden_; +}; + +class WiFiComponent : public Component { + public: + WiFiComponent(); + void setup() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::WIFI; } + + bool has_sta() const { return false; } + bool is_disabled() const { return false; } + // Always connected so network::is_connected() keeps the API server accepting clients + bool is_connected() const { return true; } + void start_scanning(); + const std::vector &get_scan_result() const { return this->scan_result_; } + void set_sta(const WiFiAP &ap); + void start_connecting(const WiFiAP &ap); + void clear_sta(); + void save_wifi_sta(StringRef ssid, StringRef password); + // Called by network::util on any USE_WIFI build + const char *get_use_address() const { return "localhost"; } + network::IPAddresses get_ip_addresses() { return {}; } + + protected: + std::vector scan_result_; +}; + +extern WiFiComponent *global_wifi_component; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +} // namespace esphome::wifi diff --git a/tests/integration/fixtures/improv_serial_uart.yaml b/tests/integration/fixtures/improv_serial_uart.yaml new file mode 100644 index 0000000000..75ffe97809 --- /dev/null +++ b/tests/integration/fixtures/improv_serial_uart.yaml @@ -0,0 +1,40 @@ +esphome: + # Short name keeps the device info payload under uart_mock's 64 byte log cap + name: improv-uart + +host: +api: + actions: + - action: uart_inject + variables: + payload: int[] + then: + - uart_mock.inject_rx: + id: mock_uart + data: !lambda return std::vector(payload.begin(), payload.end()); + +logger: + level: DEBUG + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Host-only stub shadowing the real wifi component (see external_components/wifi) +wifi: + ssid: TestNet + password: password1 + +# Dummy uart entry so the uart component sources are part of the build; the +# actual bus used by improv_serial is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + id: mock_uart + baud_rate: 115200 + +improv_serial: + uart_id: mock_uart diff --git a/tests/integration/log_utils.py b/tests/integration/log_utils.py new file mode 100644 index 0000000000..0bfbb57b1f --- /dev/null +++ b/tests/integration/log_utils.py @@ -0,0 +1,43 @@ +"""Helpers for asserting on log output in integration tests.""" + +from __future__ import annotations + +import asyncio + + +class LineWaiter: + """Collects log lines and lets a test await one containing all needles. + + Pass ``callback`` as ``run_compiled``'s ``line_callback``; the callback runs + on the test's own event loop, so futures are resolved directly. Only one + ``wait_for`` may be outstanding at a time (tests await sequentially). + """ + + def __init__(self) -> None: + self.lines: list[str] = [] + self._needles: tuple[str, ...] = () + self._future: asyncio.Future | None = None + + def callback(self, line: str) -> None: + self.lines.append(line) + if ( + self._future is not None + and not self._future.done() + and all(n in line for n in self._needles) + ): + self._future.set_result(line) + self._future = None + + async def wait_for(self, *needles: str, timeout: float = 10.0) -> str: + """Return the first line, past or future, containing every needle.""" + for line in self.lines: + if all(n in line for n in needles): + return line + assert self._future is None or self._future.done(), "concurrent wait_for" + self._needles = needles + self._future = asyncio.get_running_loop().create_future() + try: + return await asyncio.wait_for(self._future, timeout) + finally: + self._future = None + self._needles = () diff --git a/tests/integration/test_improv_serial_uart.py b/tests/integration/test_improv_serial_uart.py new file mode 100644 index 0000000000..7dad5f74bd --- /dev/null +++ b/tests/integration/test_improv_serial_uart.py @@ -0,0 +1,140 @@ +"""Integration test for improv_serial over a mocked UART bus. + +Drives the improv serial protocol end to end on the host platform: +the fixture wires improv_serial to a uart_mock bus and shadows the wifi +component with a host stub. The test injects improv frames through an API +action and asserts on the framed responses that uart_mock logs as TX lines. + +Covered: + 1. Get Current State reports AUTHORIZED + 2. Get Device Info returns the firmware/device info RPC response + 3. Get Wi-Fi Networks returns deduplicated scan results and a terminator + 4. Wi-Fi Settings provisions: saves credentials and reports PROVISIONED +""" + +from __future__ import annotations + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + +# Improv serial framing (improv_serial_component.h) +IMPROV_HEADER = b"IMPROV" +IMPROV_VERSION = 1 +TYPE_CURRENT_STATE = 0x01 +TYPE_RPC = 0x03 +TYPE_RPC_RESPONSE = 0x04 + +# improv::Command values +CMD_GET_CURRENT_STATE = 0x02 +CMD_GET_DEVICE_INFO = 0x03 +CMD_GET_WIFI_NETWORKS = 0x04 +CMD_WIFI_SETTINGS = 0x01 + + +def build_rpc_frame(command: int, data: bytes = b"") -> list[int]: + """Build a full improv serial frame carrying one RPC command.""" + payload = bytes([command, len(data)]) + data + frame = IMPROV_HEADER + bytes([IMPROV_VERSION, TYPE_RPC, len(payload)]) + payload + checksum = sum(frame) & 0xFF + return list(frame + bytes([checksum]) + b"\n") + + +def state_frame_hex(state: int) -> str: + """Full 12 byte current-state frame as hex, checksum and newline included.""" + frame = IMPROV_HEADER + bytes([IMPROV_VERSION, TYPE_CURRENT_STATE, 1, state]) + checksum = sum(frame) & 0xFF + return ":".join(f"{b:02X}" for b in frame + bytes([checksum]) + b"\n") + + +def rpc_footer_hex(payload: bytes) -> str: + """Checksum and newline footer written after an RPC response payload.""" + header = IMPROV_HEADER + bytes([IMPROV_VERSION, TYPE_RPC_RESPONSE, len(payload)]) + checksum = (sum(header) + sum(payload)) & 0xFF + return f"{checksum:02X}:0A" + + +def wifi_settings_data(ssid: str, password: str) -> bytes: + ssid_b = ssid.encode() + pass_b = password.encode() + return bytes([len(ssid_b)]) + ssid_b + bytes([len(pass_b)]) + pass_b + + +def hex_of(text: str) -> str: + """Colon separated uppercase hex as logged by format_hex_pretty.""" + return ":".join(f"{b:02X}" for b in text.encode()) + + +@pytest.mark.asyncio +async def test_improv_serial_uart( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + waiter = LineWaiter() + + async with ( + run_compiled(yaml_config, line_callback=waiter.callback), + api_client_connected() as client, + ): + _entities, services = await client.list_entities_services() + inject = next(s for s in services if s.name == "uart_inject") + + # 1. Get Current State: expect the complete current-state frame reporting + # AUTHORIZED (0x02), checksum and newline included + await client.execute_service( + inject, {"payload": build_rpc_frame(CMD_GET_CURRENT_STATE)} + ) + await waiter.wait_for("uart_mock", f"TX 12 bytes: {state_frame_hex(0x02)}") + + # 2. Get Device Info: the always logged 9 byte response header, then the + # payload with the firmware name (must stay under uart_mock's 64 byte + # hex dump cap or the payload line reads "too large to log") + await client.execute_service( + inject, {"payload": build_rpc_frame(CMD_GET_DEVICE_INFO)} + ) + await waiter.wait_for("uart_mock", "TX 9 bytes: 49:4D:50:52:4F:56:01:04") + await waiter.wait_for("uart_mock", "TX ", hex_of("ESPHome")) + + # 3. Get Wi-Fi Networks: stub scan has TestNet twice (dedup keeps the + # stronger), OpenNet, and a hidden entry (filtered). Expect one response + # per visible network plus the empty terminator. + await client.execute_service( + inject, {"payload": build_rpc_frame(CMD_GET_WIFI_NETWORKS)} + ) + await waiter.wait_for("uart_mock", hex_of("TestNet")) + await waiter.wait_for("uart_mock", hex_of("OpenNet")) + # Terminator: all three writes of the response frame; 9 byte header, + # payload [0x04, 0x00, 0x00], then the checksum and newline footer + await waiter.wait_for("uart_mock", "TX 9 bytes: 49:4D:50:52:4F:56:01:04:03") + await waiter.wait_for("uart_mock", "TX 3 bytes: 04:00:00") + await waiter.wait_for( + "uart_mock", f"TX 2 bytes: {rpc_footer_hex(bytes([0x04, 0x00, 0x00]))}" + ) + testnet_count = sum( + 1 + for line in waiter.lines + if "uart_mock" in line and "TX " in line and hex_of("TestNet") in line + ) + assert testnet_count == 1, ( + f"Duplicate scan entry not deduplicated: {testnet_count} TestNet responses" + ) + + # 4. Wi-Fi Settings: stub connects immediately; expect the credentials + # saved, the PROVISIONED state frame (0x04), and the settings response + await client.execute_service( + inject, + { + "payload": build_rpc_frame( + CMD_WIFI_SETTINGS, wifi_settings_data("NewNet", "secret123") + ) + }, + ) + await waiter.wait_for("save_wifi_sta ssid=NewNet") + await waiter.wait_for("uart_mock", f"TX 12 bytes: {state_frame_hex(0x04)}") + # Settings RPC response with no URLs: payload [0x01, 0x00, 0x00] and footer + await waiter.wait_for("uart_mock", "TX 3 bytes: 01:00:00") + await waiter.wait_for( + "uart_mock", f"TX 2 bytes: {rpc_footer_hex(bytes([0x01, 0x00, 0x00]))}" + ) From 2104096f02b313d7dfdd4eedd6d91ff5b1898838 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" <3060199+jesserockz@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:42:30 -0500 Subject: [PATCH 27/30] [core] Prefetch PlatformIO packages in parallel (#18769) --- esphome/espidf/framework.py | 28 +- esphome/framework_helpers.py | 44 +- esphome/platformio/library.py | 13 +- esphome/platformio/prefetch.py | 600 +++++++++++ esphome/platformio/toolchain.py | 17 +- tests/unit_tests/test_espidf_framework.py | 20 +- tests/unit_tests/test_framework_helpers.py | 41 + tests/unit_tests/test_platformio_prefetch.py | 985 ++++++++++++++++++ tests/unit_tests/test_platformio_toolchain.py | 14 +- 9 files changed, 1713 insertions(+), 49 deletions(-) create mode 100644 esphome/platformio/prefetch.py create mode 100644 tests/unit_tests/test_platformio_prefetch.py diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 239d874dbd..6c2a285360 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -2,7 +2,6 @@ from collections.abc import Callable from ctypes.util import find_library -from functools import partial import json import logging import os @@ -24,16 +23,17 @@ from esphome.framework_helpers import ( create_venv, download_and_extract, download_from_mirrors, - download_with_resume, failure_reason, get_python_env_executable_path, get_system_python_path, + resume_fetch_job, rmdir, run_batch_downloads, run_command, run_command_ok, str_to_lst_of_str, tool_version_runs, + warn_prefetch_failures, ) from esphome.helpers import write_file_if_changed @@ -686,18 +686,6 @@ def _patch_tools_json_demote_unused_tools(framework_path: Path) -> None: ) -def _download_tool( - dist_path: Path, entry: dict, tracker: Callable[[int], None] -) -> None: - download_with_resume( - entry["url"], - dist_path / entry["dest"], - sha256=entry["sha256"], - size=entry["size"], - progress=tracker, - ) - - def _prefetch_idf_tool_archives( framework_path: Path, targets_str: str, @@ -775,15 +763,17 @@ def _prefetch_idf_tool_archives( ( entry["name"], entry["size"], - partial(_download_tool, dist_path, entry), + resume_fetch_job( + entry["url"], + dist_path / entry["dest"], + sha256=entry["sha256"], + size=entry["size"], + ), ) for entry in entries ], ) - for name, e in failures: - # failure_reason: a message-less exception must not log blank - _LOGGER.warning("Could not prefetch %s: %s", name, failure_reason(e)) - _LOGGER.debug("Prefetch failure detail", exc_info=e) + warn_prefetch_failures(failures) if len(failures) == len(entries): # A systematic fault, not one flaky mirror: the resume # workaround (#17703) is off for this whole install diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index aab7acc0e8..82bc0d3727 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -701,7 +701,7 @@ def _write_download_meta( _LOGGER.debug("Could not update download metadata %s: %s", meta, e) -def _content_length(resp: "requests.Response") -> int: +def content_length(resp: "requests.Response") -> int: """Return the response's Content-Length, or 0 when absent or malformed. 0 means "unknown", which downstream disables the progress bar and the @@ -744,7 +744,7 @@ def _stream_response_to_file( """ f.seek(offset) f.truncate(offset) - total_size = size or offset + _content_length(resp) + total_size = size or offset + content_length(resp) downloaded = offset own_bar: ProgressBar | None = None if progress is None: @@ -909,6 +909,19 @@ def _part_path(dest: Path) -> Path: return dest.with_name(dest.name + ".part") +def discard_partial_download(dest: Path) -> None: + """Remove ``dest`` and the resume sidecars of an abandoned download.""" + part = _part_path(dest) + for stale in (dest, part, part.with_name(part.name + ".meta")): + try: + stale.unlink() + except FileNotFoundError: + continue + except OSError as err: + # The caller's cache is never pruned; leave a trace + _LOGGER.debug("Could not remove %s: %s", stale, err) + + def _cancellable_sleep( delay: float, progress: Callable[[int], None] | None, done: int ) -> None: @@ -922,6 +935,31 @@ def _cancellable_sleep( time.sleep(min(0.5, remaining)) +def resume_fetch_job( + url: str, dest: PathType, **kwargs +) -> Callable[[Callable[[int], None]], None]: + """A ``run_batch_downloads`` job callable wrapping ``download_with_resume``. + + Forwards the runner's positional tracker as the ``progress`` keyword. + """ + + def fetch(tracker: Callable[[int], None]) -> None: + download_with_resume(url, dest, progress=tracker, **kwargs) + + return fetch + + +def warn_prefetch_failures( + failures: list[tuple[str, BaseException]], + message: str = "Could not prefetch %s: %s", +) -> None: + """Warn per failed batch-prefetch job; the caller's installer retries them.""" + for name, err in failures: + # failure_reason: a message-less exception must not log blank + _LOGGER.warning(message, name, failure_reason(err)) + _LOGGER.debug("Prefetch failure detail", exc_info=err) + + def download_with_resume( url: str, dest: PathType, @@ -1022,7 +1060,7 @@ def download_with_resume( streamed = True if offset == 0: validator = _response_validator(resp) - expected_total = _content_length(resp) + expected_total = content_length(resp) # Recorded so a later run can prove an If-Range # resume of this part file safe. _write_download_meta(meta, url, validator, expected_total) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 1792647d6b..306f07854e 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -35,6 +35,7 @@ from esphome.framework_helpers import ( failure_reason, rmdir, run_batch_downloads, + warn_prefetch_failures, ) _LOGGER = logging.getLogger(__name__) @@ -977,14 +978,10 @@ def _prefetch_wave( for c in components ], ) - for name, err in failures: - # The sequential call below retries and raises the real error - _LOGGER.warning( - "Prefetch of %s failed (retrying sequentially): %s", - name, - failure_reason(err), - ) - _LOGGER.debug("Prefetch failure detail", exc_info=err) + # The sequential call below retries and raises the real error + warn_prefetch_failures( + failures, "Prefetch of %s failed (retrying sequentially): %s" + ) except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught # Same policy as the ESP-IDF twin: the prefetch must never become a # new way for the build to fail diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py new file mode 100644 index 0000000000..f313c2f4d0 --- /dev/null +++ b/esphome/platformio/prefetch.py @@ -0,0 +1,600 @@ +"""Parallel prefetch of the packages a PlatformIO run would install. + +Downloads the archives concurrently into PlatformIO's own download cache +(identical ``compute_download_path`` keys) so the serial installer finds +them already cached. Runs in a subprocess like all PlatformIO execution: +loading a platform executes its code (pioarduino's penv setup rewrites +``sys.path``). A sentinel in the build dir lets warm builds skip the +spawn. Best-effort: any failure logs and PlatformIO downloads as before. +Across processes sharing a core dir every download destination is +serialized by a file lock; checksum-less URL downloads additionally +stage under a stable name and promote with an atomic rename. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import logging +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +from typing import Any + +from esphome.framework_helpers import ( + content_length, + discard_partial_download, + failure_reason, + resume_fetch_job, + run_batch_downloads, + warn_prefetch_failures, +) +from esphome.helpers import get_bool_env + +_LOGGER = logging.getLogger(__name__) + +# Concurrent registry resolutions / HEAD probes (each is network-bound) +_RESOLVE_WORKERS = 8 + +# A hung child must not block the build; downloads resume on the next run +_PREFETCH_TIMEOUT = 20 * 60 + +# Waiting on another process's URL download; past this, leave it to pio +_DOWNLOAD_LOCK_TIMEOUT = 60 + +# Child exit for a handled, already-warned failure; 1 would collide with +# the interpreter's own import-failure exit +_EXIT_HANDLED = 3 + +# Short lock-acquire slices so a waiting worker still observes Ctrl-C +_URI_LOCK_POLL = 1 + +# Resolution errored (vs a clean skip); suppresses the warm sentinel +_RESOLVE_FAILED = object() + + +def _sweep_stale_sidecars(download_dir: Path, expire_seconds: int) -> None: + """Prune resume sidecars pio's usage.db pruner cannot see. + + A version bump strands an aborted archive's sidecars forever. Lock + files stay: a held lock can carry an ancient mtime (O_TRUNC keeps + it), and unlinking one reopens the single-writer hole it guards. + """ + cutoff = time.time() - expire_seconds + try: + for f in download_dir.iterdir(): + if f.suffix not in (".part", ".meta", ".prefetch"): + continue + try: + if f.stat().st_mtime < cutoff: + f.unlink() + except OSError as err: + _LOGGER.debug("Could not remove %s: %s", f, err) + except OSError: + _LOGGER.debug("Could not sweep %s", download_dir, exc_info=True) + + +# Child records a no-work run; the parent skips the next spawn while valid +_SENTINEL_NAME = ".esphome_prefetch.json" +_SENTINEL_SCHEMA = 1 + + +def _ini_sha256(build_dir: Path) -> str: + return hashlib.sha256((build_dir / "platformio.ini").read_bytes()).hexdigest() + + +def _sentinel_state(build_dir: Path) -> dict[str, Any]: + """The environment fingerprint a sentinel must match to stay valid.""" + # Same fingerprint as the heal stamp: the sentinel's dirs die with its wipe + from esphome.platformio.toolchain import current_python_minor + + return { + "schema": _SENTINEL_SCHEMA, + "ini_sha256": _ini_sha256(build_dir), + "python": current_python_minor(), + "core_dir_env": os.environ.get("PLATFORMIO_CORE_DIR", ""), + } + + +def _prefetch_is_warm(build_dir: Path) -> bool: + """Whether the last prefetch found nothing to do and nothing changed since.""" + try: + data = json.loads((build_dir / _SENTINEL_NAME).read_text(encoding="utf-8")) + dirs = data.pop("dirs") + return ( + data == _sentinel_state(build_dir) + and bool(dirs) + and all(Path(d).is_dir() for d in dirs) + ) + except FileNotFoundError: + return False + except (OSError, ValueError, KeyError, AttributeError, TypeError): + _LOGGER.debug("Ignoring invalid prefetch sentinel", exc_info=True) + return False + + +def prefetch_platformio_packages() -> None: + """Warm PlatformIO's download cache for the current project, in parallel.""" + from esphome.core import CORE + from esphome.platformio.toolchain import ( + default_libdeps_dir, + heal_platformio_python_env, + ) + + # Heal first: its Python-version wipe would discard freshly warmed + # caches and the sentinel's dirs (the later heal call is a no-op) + heal_platformio_python_env() + build_dir = Path(CORE.build_path) + if _prefetch_is_warm(build_dir): + return + # The child is esphome itself: PYTHONPATH stays so it imports this + # tree's esphome (tests/integration pins the source tree through it) + env = dict(os.environ) + # Must match run_platformio_cli's default or warm builds re-resolve + # every library + env.setdefault("PLATFORMIO_LIBDEPS_DIR", default_libdeps_dir()) + # -v/-vv must reach the child's debug logging or the swallowed + # failure detail is undiagnosable in the field + env["ESPHOME_PREFETCH_LOG_LEVEL"] = str(logging.getLogger().getEffectiveLevel()) + if CORE.dashboard: + # The child's progress bar and log escaping key off CORE.dashboard + env["ESPHOME_PREFETCH_DASHBOARD"] = "1" + cmd = [ + sys.executable, + "-m", + "esphome.platformio.prefetch", + str(build_dir), + CORE.name, + ] + try: + proc = subprocess.run(cmd, env=env, check=False, timeout=_PREFETCH_TIMEOUT) + except subprocess.TimeoutExpired: + _LOGGER.warning("PlatformIO package prefetch timed out; continuing without it") + return + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # The prefetch must never become a new way for the build to fail + _LOGGER.warning("PlatformIO package prefetch skipped: %s", failure_reason(err)) + _LOGGER.debug("Prefetch failure detail", exc_info=True) + return + if proc.returncode == _EXIT_HANDLED: + # The child already warned with the reason; a second line is noise + _LOGGER.debug("Prefetch child reported a handled failure") + elif proc.returncode != 0: + # Exit 1 stays here: the interpreter exits 1 for import/module + # failures before main() ever runs, a wiring break worth a warning + _LOGGER.warning( + "PlatformIO package prefetch skipped (exit %d)", proc.returncode + ) + + +def _project_platform_and_config(ini: Path, env: str) -> tuple[str | None, Any]: + """The env's platform spec and the ProjectConfig for the given ini.""" + from platformio import app + from platformio.project.config import ProjectConfig + + # PlatformBase.config reads the default ProjectConfig; it must see + # this ini's env options + app.set_session_var("custom_project_conf", str(ini)) + config = ProjectConfig.get_instance(str(ini)) + return config.get(f"env:{env}", "platform", None), config + + +def _registry_jobs( + manager, specs, seen: set[str] +) -> tuple[list[tuple[str, int, Any]], int]: + """Resolve registry specs to ``(name, size, fetch)`` batch jobs. + + Mirrors PlatformIO's install path: best version, systype file, first + mirror, and the same sha1(url + checksum) download-cache key. Also + returns how many resolutions errored (a clean skip is not an error). + """ + from platformio.registry.mirror import RegistryFileMirrorIterator + + local = threading.local() + errors: list[str] = [] + + def _resolve(spec) -> tuple[str, int, str, Path, str] | object | None: + # One manager (and registry HTTP session) per worker thread; + # installed-state was already checked on the shared manager + if (mgr := getattr(local, "mgr", None)) is None: + mgr = local.mgr = manager.__class__() + try: + packages = mgr.search_registry_packages(spec) + if not packages: + _LOGGER.debug("%s is unknown to the registry", spec) + return None # let PlatformIO report it + package, version = mgr.find_best_registry_version(packages, spec) + if not package or not version: + _LOGGER.debug("%s has no matching registry version", spec) + return None + pkgfile = mgr.pick_compatible_pkg_file(version["files"]) + if not pkgfile: + _LOGGER.debug("%s has no file for this systype", spec) + return None + url, checksum = next(RegistryFileMirrorIterator(pkgfile["download_url"])) + checksum = checksum or pkgfile["checksum"]["sha256"] + dl_path = Path(mgr.compute_download_path(url, checksum)) + if dl_path.is_file(): + return None # cached from an earlier run + size = pkgfile.get("size") + if not size: + _LOGGER.debug("%s has no size; PlatformIO fetches it", spec) + return None # no size, no bar share + return f"{package['name']}@{version['name']}", size, url, dl_path, checksum + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # One flaky spec must not discard the rest of the batch + _LOGGER.debug("Could not resolve %s", spec, exc_info=True) + errors.append(failure_reason(err)) + return _RESOLVE_FAILED + + # Serial disk lookups on the shared manager: a fully warm build + # resolves nothing, and duplicate specs resolve once + unique: dict[tuple[str | None, str, str], Any] = {} + for s in specs: + if not s.uri and not manager.get_package(s): + unique.setdefault((s.owner, s.name, str(s.requirements)), s) + pending = list(unique.values()) + if not pending: + return [], 0 + # Serial resolutions (registry GET + mirror HEAD each) dominate + with ThreadPoolExecutor(max_workers=min(_RESOLVE_WORKERS, len(pending))) as ex: + results = list(ex.map(_resolve, pending)) + jobs: list[tuple[str, int, Any]] = [] + for res in results: + if res is None or res is _RESOLVE_FAILED: + continue + name, size, url, dl_path, checksum = res + if str(dl_path) in seen: + continue # duplicate spec; two workers must not share a .part + seen.add(str(dl_path)) + jobs.append( + (name, size, _registry_fetch_job(manager, url, dl_path, checksum, size)) + ) + if failed := len(errors): + # Visible once per build, naming a cause so an API break does not + # read as an outage; per-spec detail stays at debug + _LOGGER.warning( + "Could not resolve %d of %d PlatformIO package(s) (%s); " + "PlatformIO will download them serially", + failed, + len(pending), + errors[0], + ) + return jobs, failed + + +def _uri_jobs(manager, specs, seen: set[str]) -> tuple[list[tuple[str, int, Any]], int]: + """Jobs for direct-URL specs; a HEAD sizes each for the combined bar. + + Also returns how many HEAD probes errored (an absent length is not an + error). + """ + from esphome.net_retry import fetch_with_retry, http_request + + candidates: list[tuple[str, str, Path]] = [] + for spec in specs: + url = spec.uri + if not url or not url.startswith(("http://", "https://")): + continue # git+/file specs are cloned/copied, not downloaded + if url.split("#", 1)[0].endswith(".git"): + continue # bare-URL VCS spec; PlatformIO clones it + if manager.get_package(spec): + continue + # PlatformIO downloads URL specs with no checksum + dl_path = Path(manager.compute_download_path(url, "")) + if dl_path.is_file() or str(dl_path) in seen: + continue # cached, or another spec already claimed this .part + seen.add(str(dl_path)) + candidates.append((spec.name, url, dl_path)) + + errors: list[str] = [] + + def _head_size(url: str) -> int: + try: + resp = fetch_with_retry(url, lambda: http_request("HEAD", url, timeout=30)) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + _LOGGER.debug("HEAD %s failed", url, exc_info=True) + errors.append(failure_reason(err)) + return -1 + if not resp.ok: + # An error page's Content-Length is not a download size + _LOGGER.debug("HEAD %s returned %s", url, resp.status_code) + if resp.status_code in (401, 403, 408, 429) or resp.status_code >= 500: + # 401/403 included: registries rate-limit with them + errors.append(f"HTTP {resp.status_code}") + return -1 # transient; must not be cached as warm + # Permanent (405/501 HEAD-unsupported, 401/403/404): a clean + # skip so the warm sentinel is not disabled forever; pio run + # surfaces a genuinely broken URL when it downloads + return 0 + return content_length(resp) + + if not candidates: + return [], 0 + with ThreadPoolExecutor(max_workers=min(_RESOLVE_WORKERS, len(candidates))) as ex: + sizes = list(ex.map(_head_size, [url for _, url, _ in candidates])) + jobs: list[tuple[str, int, Any]] = [] + failed = 0 + for (name, url, dl_path), size in zip(candidates, sizes, strict=True): + if size < 0: + failed += 1 + elif size: + jobs.append((name, size, _uri_fetch_job(manager, url, dl_path, size))) + else: + # Missing or unusable Content-Length; visible under -v + _LOGGER.debug("%s reports no usable length; PlatformIO fetches it", url) + if failed: + _LOGGER.warning( + "Could not size %d of %d PlatformIO package URL(s) (%s); " + "PlatformIO will download them serially", + failed, + len(candidates), + errors[0], + ) + return jobs, failed + + +def _serialized_fetch_job( + dl_path: Path, lock_path: str, body: Any, unlocked_ok: bool = True +) -> Any: + """Wrap ``body`` so the shared destination is single-writer. + + Interleaved writers truncate each other's ``.part`` bytes (see + registry.py). The bounded poll observes Ctrl-C via the tracker; a + blown deadline is a clean skip (the holder's copy is what the build + needs). On a lock-less filesystem a sha256-verified body runs + unlocked with one warning; a checksum-less one + (``unlocked_ok=False``) is a counted failure instead. + """ + + def run(tracker: Any) -> None: + from filelock import FileLock, Timeout + + # fallback_to_soft would leave a stale marker on lock-less + # filesystems that blocks every later build (see git.py) + lock = FileLock(lock_path, fallback_to_soft=False) + deadline = time.monotonic() + _DOWNLOAD_LOCK_TIMEOUT + while True: + try: + lock.acquire(timeout=_URI_LOCK_POLL) + break + except Timeout: + tracker(0) # raises when the batch is cancelled + if time.monotonic() >= deadline: + # Another process is fetching this same file; its copy + # is what the build needs (a large framework archive + # can hold the lock far longer than this deadline) + _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) + return + except OSError as err: + if not unlocked_ok: + # A body with no checksum to catch interleaved corruption + raise + lock = None + _LOGGER.warning( + "Could not lock %s (%s); downloading unlocked", + dl_path.name, + err, + ) + break + try: + if dl_path.is_file(): + return # another process finished it while we waited + body(tracker) + finally: + if lock is not None: + lock.release() + + return run + + +# usage.db is a whole-file rewrite behind pio's self-unlinking LockFile; +# concurrent writers could reset every recorded entry +_REGISTER_LOCK = threading.Lock() + + +def _register_download(manager: Any, dl_path: Path) -> None: + """Hand the archive to pio's usage.db pruner; an unregistered one is + never expired (disk garbage, never a bad build).""" + try: + with _REGISTER_LOCK: + manager.set_download_utime(str(dl_path)) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + _LOGGER.debug("Could not register %s with pio's cache: %s", dl_path, err) + + +def _registry_fetch_job( + manager: Any, url: str, dl_path: Path, checksum: str, size: int +) -> Any: + """A locked fetch straight to the cache path; sha256 verifies it.""" + # .esphome.lock: pio's own LockFile(dl_path) owns .lock and + # deletes it on release, which would unlink a held filelock + fetch = _serialized_fetch_job( + dl_path, + f"{dl_path}.esphome.lock", + resume_fetch_job(url, dl_path, sha256=checksum, size=size), + ) + + def run(tracker: Any) -> None: + fetch(tracker) + if dl_path.is_file(): + # The deadline skip can end with no archive landed + _register_download(manager, dl_path) + + return run + + +def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: + """Fetch to a locked staging path, then rename into the cache. + + The stable staging name keeps resume working across interrupted + runs; the rename makes the promotion atomic. + """ + tmp = dl_path.with_name(f"{dl_path.name}.prefetch") + # attempts=2: the size is only a HEAD probe's word, and a HEAD/GET + # disagreement would otherwise re-download the archive five times + fetch = resume_fetch_job(url, tmp, size=size, attempts=2) + + def promote(tracker: Any) -> None: + fetch(tracker) + if (actual := tmp.stat().st_size) != size: + # A wrong-length checksum-less body must never be published + discard_partial_download(tmp) + raise ValueError(f"expected {size} bytes, fetched {actual}") + tmp.replace(dl_path) + + def run(tracker: Any) -> None: + _serialized_fetch_job(dl_path, f"{tmp}.lock", promote, unlocked_ok=False)( + tracker + ) + if dl_path.is_file(): + # Won or lost, the race is over; staging files left behind + # are dead weight PlatformIO's cache never prunes + discard_partial_download(tmp) + _register_download(manager, dl_path) + + return run + + +def _prefetch(build_dir: Path, env: str) -> None: + from platformio.dependencies import get_core_dependencies + from platformio.package.manager.library import LibraryPackageManager + from platformio.package.manager.platform import PlatformPackageManager + from platformio.package.meta import PackageSpec + from platformio.platform.factory import PlatformFactory + + platform_spec, config = _project_platform_and_config( + build_dir / "platformio.ini", env + ) + if not platform_spec: + # An env mismatch must not disable the feature with no trace + _LOGGER.debug( + "No platform for env %s in %s; nothing to prefetch", env, build_dir + ) + return + + # The platform (manifest plus build scripts) installs first and + # resolves the rest. Its setup may rewrite sys.path (pioarduino's penv + # setup does); restore it so later imports here still resolve. + saved_sys_path = list(sys.path) + pm = PlatformPackageManager() + _sweep_stale_sidecars(Path(pm.get_download_dir()), pm.DOWNLOAD_CACHE_EXPIRE) + pkg = pm.install(platform_spec, skip_dependencies=True) + p = PlatformFactory.new(pkg) + p.configure_project_packages(env, ["run"]) + sys.path[:] = saved_sys_path + + specs = [ + p.get_package_spec(name) + for name, opts in p.packages.items() + if not opts.get("optional") + ] + # PIO's build engine installs outside the platform package list; + # skipped when the platform lists it itself + if not any(s.name == "tool-scons" for s in specs): + specs.append( + PackageSpec( + owner="platformio", + name="tool-scons", + requirements=get_core_dependencies()["tool-scons"], + ) + ) + lib_deps = config.get(f"env:{env}", "lib_deps", []) + # pio run's storage dir for this env: installed libraries skip by + # disk lookup + libdeps_dir = Path(config.get("platformio", "libdeps_dir")) / env + lm = LibraryPackageManager(str(libdeps_dir)) + # A bare name is usually a framework built-in (WiFi, SPI); with no + # lib builders here to tell built-in from registry, skip it. The only + # cost is that an owner-less user library is not prefetched + lib_specs = [ + spec + for dep in lib_deps + if dep and not dep.startswith("$") + if (spec := PackageSpec(dep)).external or spec.owner + ] + + seen: set[str] = set() + jobs: list[tuple[str, int, Any]] = [] + unresolved = 0 + for mgr, batch in ((p.pm, specs), (lm, lib_specs)): + for build_jobs in (_registry_jobs, _uri_jobs): + batch_jobs, failed = build_jobs(mgr, batch, seen) + jobs += batch_jobs + unresolved += failed + + sentinel = build_dir / _SENTINEL_NAME + if not jobs: + if not unresolved: + # Record the no-work run so the parent skips the next spawn. + # A failed resolution is not "no work": a registry outage must + # not be cached as warm. + dirs = [config.get("platformio", "packages_dir")] + if lib_specs: + dirs.append(str(libdeps_dir)) + sentinel.write_text( + json.dumps({**_sentinel_state(build_dir), "dirs": dirs}), + encoding="utf-8", + ) + return + sentinel.unlink(missing_ok=True) + _LOGGER.info( + "Prefetching %d PlatformIO package(s): %s", + len(jobs), + ", ".join(name for name, _, _ in jobs), + ) + # PlatformIO retries failed packages itself, without resume + warn_prefetch_failures(run_batch_downloads("Downloading PlatformIO packages", jobs)) + + +def main(argv: list[str]) -> int: + """Subprocess entry point: ``prefetch ``.""" + from esphome.core import CORE + from esphome.log import setup_log + + raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") + try: + level = int(raw_level) if raw_level is not None else logging.INFO + except ValueError: + level = logging.INFO + # Mirror the parent's log setup: warnings keep their level prefix and + # color, and the download bar still draws under the dashboard + CORE.dashboard = get_bool_env("ESPHOME_PREFETCH_DASHBOARD") + setup_log(level) + # pio's managers attach their own handler and still propagate; without + # this every manager line also prints through the root handler. Their + # construction re-pins the logger to INFO, so a logger-level filter + # (which survives pio's handler reset) enforces a quiet level instead. + for cls_name in ( + "ToolPackageManager", + "LibraryPackageManager", + "PlatformPackageManager", + ): + manager_logger = logging.getLogger(cls_name.replace("Package", " ")) + manager_logger.propagate = False + manager_logger.addFilter(lambda record: record.levelno >= level) + if len(argv) != 2: + # A wiring bug, not a network failure; make it distinguishable + _LOGGER.warning("prefetch usage: ") + return 2 + build_dir, env = argv + try: + _prefetch(Path(build_dir), env) + except KeyboardInterrupt: + # Shared process group: exit quietly, no traceback on the terminal + _LOGGER.debug("Prefetch interrupted", exc_info=True) + return 130 + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # The parent treats any exit as warn-and-continue, never a failure + _LOGGER.warning("PlatformIO package prefetch skipped: %s", failure_reason(err)) + _LOGGER.debug("Prefetch failure detail", exc_info=True) + return _EXIT_HANDLED + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main(sys.argv[1:])) diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index cf2094dfe0..97b32420da 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -96,7 +96,7 @@ def _clean_platformio_python_env(config: "ProjectConfig", core_dir: Path) -> Non rmtree(penv) -def _current_python_minor() -> str: +def current_python_minor() -> str: """Return the running interpreter's ``major.minor`` (e.g. ``3.13``).""" return f"{sys.version_info.major}.{sys.version_info.minor}" @@ -161,7 +161,7 @@ def heal_platformio_python_env() -> None: def _check_platformio_python_stamp(config: "ProjectConfig") -> None: """Compare the stamp to the running interpreter; wipe and restamp on mismatch.""" - current = _current_python_minor() + current = current_python_minor() stamp_dir = _pio_stamp_dir(config) # Host the stamp/lock even before PlatformIO's first run creates the dir. stamp_dir.mkdir(parents=True, exist_ok=True) @@ -289,6 +289,12 @@ def copy_ccache_script() -> None: ) +def default_libdeps_dir() -> str: + """The PLATFORMIO_LIBDEPS_DIR value a pio run defaults to; the package + prefetch must resolve installed libraries against the same dir.""" + return str(CORE.relative_piolibdeps_path().absolute()) + + def run_platformio_cli(*args, **kwargs) -> str | int: # Re-provision the PlatformIO cache if the interpreter's major.minor changed # since it was last built; a stale platform otherwise rejects the new Python @@ -296,9 +302,7 @@ def run_platformio_cli(*args, **kwargs) -> str | int: heal_platformio_python_env() os.environ["PLATFORMIO_FORCE_COLOR"] = "true" os.environ["PLATFORMIO_BUILD_DIR"] = str(CORE.relative_pioenvs_path().absolute()) - os.environ.setdefault( - "PLATFORMIO_LIBDEPS_DIR", str(CORE.relative_piolibdeps_path().absolute()) - ) + os.environ.setdefault("PLATFORMIO_LIBDEPS_DIR", default_libdeps_dir()) # Suppress Python syntax warnings from third-party scripts during compilation os.environ.setdefault("PYTHONWARNINGS", "ignore::SyntaxWarning") # Increase uv retry count to handle transient network errors (default is 3) @@ -346,6 +350,9 @@ def run_platformio_cli_run(config, verbose, *args, **kwargs) -> str | int: def run_compile(config, verbose): + from esphome.platformio.prefetch import prefetch_platformio_packages + + prefetch_platformio_packages() args = [] if CONF_COMPILE_PROCESS_LIMIT in config[CONF_ESPHOME]: args += [f"-j{config[CONF_ESPHOME][CONF_COMPILE_PROCESS_LIMIT]}"] diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index 45a971ca01..afa4433aa1 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -911,7 +911,7 @@ def test_prefetch_leaves_unverifiable_entries_to_the_installer( "esphome.espidf.framework.run_command", return_value=(True, json.dumps(entries), ""), ), - patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.framework_helpers.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls, ): @@ -934,7 +934,7 @@ def test_prefetch_all_entries_unverifiable_is_a_noop(tmp_path: Path) -> None: "esphome.espidf.framework.run_command", return_value=(True, json.dumps(entries), ""), ), - patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.framework_helpers.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), ): _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) @@ -952,7 +952,7 @@ def test_prefetch_dedupes_entries_by_dest(tmp_path: Path) -> None: "esphome.espidf.framework.run_command", return_value=(True, json.dumps(entries), ""), ), - patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.framework_helpers.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch("esphome.framework_helpers._BatchDownloadProgress"), ): @@ -967,7 +967,7 @@ def test_prefetch_downloads_each_archive_with_resume(tmp_path: Path) -> None: "esphome.espidf.framework.run_command", return_value=(True, _PREFETCH_JSON, ""), ), - patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.framework_helpers.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls, ): @@ -1011,7 +1011,7 @@ def test_prefetch_downloads_archives_concurrently(tmp_path: Path) -> None: "esphome.espidf.framework.run_command", return_value=(True, json.dumps(entries), ""), ), - patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.framework_helpers.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch( "esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor @@ -1032,7 +1032,7 @@ def test_prefetch_skips_already_downloaded_archives(tmp_path: Path) -> None: "esphome.espidf.framework.run_command", return_value=(True, _PREFETCH_JSON, ""), ), - patch("esphome.espidf.framework.download_with_resume") as download, + patch("esphome.framework_helpers.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), ): _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) @@ -1065,7 +1065,7 @@ def test_prefetch_failures_never_raise( with ( patch("esphome.espidf.framework.run_command", return_value=run_result), patch( - "esphome.espidf.framework.download_with_resume", + "esphome.framework_helpers.download_with_resume", side_effect=download_error, ), patch("esphome.espidf.framework.get_system_python_path", return_value="python"), @@ -1087,7 +1087,7 @@ def test_prefetch_total_failure_logs_error( return_value=(True, _PREFETCH_JSON, ""), ), patch( - "esphome.espidf.framework.download_with_resume", + "esphome.framework_helpers.download_with_resume", side_effect=OSError("proxy refuses everything"), ), patch("esphome.espidf.framework.get_system_python_path", return_value="python"), @@ -1112,7 +1112,7 @@ def test_prefetch_one_failed_archive_does_not_stop_the_rest( return_value=(True, _PREFETCH_JSON, ""), ), patch( - "esphome.espidf.framework.download_with_resume", + "esphome.framework_helpers.download_with_resume", side_effect=_fail_cmake_download, ) as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), @@ -1133,7 +1133,7 @@ def test_prefetch_finishes_progress_bar_and_cancels_queue(tmp_path: Path) -> Non "esphome.espidf.framework.run_command", return_value=(True, _PREFETCH_JSON, ""), ), - patch("esphome.espidf.framework.download_with_resume"), + patch("esphome.framework_helpers.download_with_resume"), patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch("esphome.framework_helpers._BatchDownloadProgress") as progress_cls, patch("esphome.framework_helpers.ThreadPoolExecutor") as pool_cls, diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 8844212600..fcc5572f51 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2280,6 +2280,32 @@ class TestGetProjectCxxCompileFlags: assert get_project_cxx_compile_flags() == [] +def test_resume_fetch_job_threads_tracker(tmp_path: Path) -> None: + """The batch runner passes the tracker positionally; the shared adapter + must deliver it as download_with_resume's progress keyword.""" + from esphome.framework_helpers import resume_fetch_job + + with patch("esphome.framework_helpers.download_with_resume") as mock_download: + fetch = resume_fetch_job("https://x/a.zip", tmp_path / "a", sha256="ff", size=9) + tracker = lambda done: None # noqa: E731 + fetch(tracker) + mock_download.assert_called_once_with( + "https://x/a.zip", tmp_path / "a", progress=tracker, sha256="ff", size=9 + ) + + +def test_warn_prefetch_failures_names_each_failure( + caplog: pytest.LogCaptureFixture, +) -> None: + """The shared failure loop warns per job with the failure reason.""" + from esphome.framework_helpers import warn_prefetch_failures + + warn_prefetch_failures([("toolchain-x@1", OSError("down"))]) + assert "Could not prefetch toolchain-x@1: down" in caplog.text + warn_prefetch_failures([("lib", OSError("gone"))], "Prefetch of %s failed: %s") + assert "Prefetch of lib failed: gone" in caplog.text + + @pytest.mark.parametrize( ("platform", "input_path", "expected"), [ @@ -2312,3 +2338,18 @@ def test_strip_win_long_path_prefix( r"""``\\?\`` and ``\\?\UNC\`` prefixes are stripped only on win32.""" with patch("esphome.framework_helpers.sys.platform", platform): assert framework_helpers.strip_win_long_path_prefix(input_path) == expected + + +def test_discard_partial_download_logs_undeletable( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unremovable staging file leaves a debug trace; the caller's + cache is never pruned, so silence would hide unbounded growth.""" + dest = tmp_path / "archive" + dest.write_bytes(b"stale") + with ( + patch.object(Path, "unlink", side_effect=OSError("busy")), + caplog.at_level(logging.DEBUG), + ): + framework_helpers.discard_partial_download(dest) + assert "Could not remove" in caplog.text diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py new file mode 100644 index 0000000000..22e20d0bf6 --- /dev/null +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -0,0 +1,985 @@ +"""Tests for the parallel PlatformIO package prefetch.""" + +import errno +import json +import os +from pathlib import Path +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from filelock import Timeout +import pytest + +from esphome.core import CORE +import esphome.platformio.prefetch as pf + + +@pytest.fixture(autouse=True) +def _core(tmp_path: Path): + CORE.reset() + CORE.build_path = str(tmp_path) + CORE.name = "testenv" + pio_loggers = ("Tool Manager", "Library Manager", "Platform Manager") + saved_propagate = {n: pf.logging.getLogger(n).propagate for n in pio_loggers} + saved_filters = {n: list(pf.logging.getLogger(n).filters) for n in pio_loggers} + # The real setup_log would swap pytest's root-handler formatter + with patch("esphome.log.setup_log"): + yield + # main() flips these process-wide; keep the suite hermetic + for n, flag in saved_propagate.items(): + pf.logging.getLogger(n).propagate = flag + pf.logging.getLogger(n).filters[:] = saved_filters[n] + CORE.reset() + + +class _FakeSpec(SimpleNamespace): + """PackageSpec stand-in for the attributes the prefetch reads.""" + + def __init__( + self, *, owner=None, requirements=None, external=False, **kwargs + ) -> None: + super().__init__( + owner=owner, requirements=requirements, external=external, **kwargs + ) + + +def _fake_manager(tmp_path: Path) -> MagicMock: + m = MagicMock() + m.__class__ = lambda: m # _resolve constructs a same-class instance + m.get_package.return_value = None + m.search_registry_packages.return_value = [{"any": 1}] + m.find_best_registry_version.return_value = ( + {"name": "toolchain-xtensa"}, + { + "name": "2.0.0", + "files": [ + { + "download_url": "https://dl.example/t.tar.gz", + "checksum": {"sha256": "cafe"}, + "size": 1000, + } + ], + }, + ) + m.pick_compatible_pkg_file.side_effect = lambda files: files[0] + m.compute_download_path.side_effect = lambda url, checksum: str( + tmp_path / "dl" / f"{abs(hash((url, checksum)))}" + ) + return m + + +def _mirror_patch(): + return patch.dict( + "sys.modules", + { + "platformio.registry.mirror": SimpleNamespace( + RegistryFileMirrorIterator=lambda url: iter( + [("https://mirror.example/t.tar.gz", "beef")] + ) + ) + }, + ) + + +def test_registry_jobs_resolves_like_platformio(tmp_path: Path) -> None: + """A registry spec resolves to a job keyed by mirror URL and checksum.""" + m = _fake_manager(tmp_path) + with _mirror_patch(): + jobs, failed = pf._registry_jobs( + m, [_FakeSpec(uri=None, name="toolchain-xtensa")], set() + ) + assert failed == 0 + assert len(jobs) == 1 + name, size, fetch = jobs[0] + assert name == "toolchain-xtensa@2.0.0" + assert size == 1000 + m.compute_download_path.assert_called_once_with( + "https://mirror.example/t.tar.gz", "beef" + ) + assert callable(fetch) + + +@pytest.mark.parametrize( + ("method", "attr", "value"), + [ + ("get_package", "return_value", object()), # already installed + ("search_registry_packages", "return_value", []), # unknown package + ("find_best_registry_version", "return_value", (None, None)), # no match + ("pick_compatible_pkg_file", "side_effect", lambda files: None), # no file + ], +) +def test_registry_jobs_skips(tmp_path: Path, method, attr, value) -> None: + """Entries PlatformIO would not download produce no job.""" + m = _fake_manager(tmp_path) + setattr(getattr(m, method), attr, value) + with _mirror_patch(): + assert pf._registry_jobs(m, [_FakeSpec(uri=None, name="x")], set()) == ([], 0) + + +def test_registry_jobs_skips_cached_and_sizeless(tmp_path: Path) -> None: + """Cached or sizeless files are left to PlatformIO.""" + m = _fake_manager(tmp_path) + dl = Path(m.compute_download_path("https://mirror.example/t.tar.gz", "beef")) + dl.parent.mkdir(parents=True, exist_ok=True) + dl.touch() + with _mirror_patch(): + assert pf._registry_jobs(m, [_FakeSpec(uri=None, name="x")], set()) == ([], 0) + dl.unlink() + m.find_best_registry_version.return_value[1]["files"][0]["size"] = 0 + with _mirror_patch(): + assert pf._registry_jobs(m, [_FakeSpec(uri=None, name="x")], set()) == ([], 0) + + +def test_registry_jobs_dedupes_download_paths(tmp_path: Path) -> None: + """Duplicate specs resolve once and one archive yields one job (two + workers must never share a .part); nine specs against eight workers + also exercise the thread-local manager reuse.""" + m = _fake_manager(tmp_path) + specs = [_FakeSpec(uri=None, name="dup"), _FakeSpec(uri=None, name="dup")] + specs += [_FakeSpec(uri=None, name=f"n{i}") for i in range(8)] + with _mirror_patch(): + jobs, failed = pf._registry_jobs(m, specs, set()) + # the fake resolves every spec to the same mirror URL and checksum + assert failed == 0 + assert len(jobs) == 1 + assert m.search_registry_packages.call_count == 9 # dup resolved once + + +def test_registry_jobs_uri_specs_excluded(tmp_path: Path) -> None: + """URL specs never reach the registry resolution.""" + m = _fake_manager(tmp_path) + assert pf._registry_jobs( + m, [_FakeSpec(uri="https://x/y.zip", name="y")], set() + ) == ([], 0) + m.search_registry_packages.assert_not_called() + + +def test_registry_jobs_dedup_keeps_distinct_owners(tmp_path: Path) -> None: + """platformio/x and pioarduino/x are different packages.""" + m = _fake_manager(tmp_path) + specs = [ + _FakeSpec(uri=None, name="framework-x", owner="platformio"), + _FakeSpec(uri=None, name="framework-x", owner="pioarduino"), + ] + with _mirror_patch(): + pf._registry_jobs(m, specs, set()) + assert m.search_registry_packages.call_count == 2 + + +def test_registry_jobs_all_failed_warns_once( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A whole-batch failure is a systemic fault and must be visible.""" + m = _fake_manager(tmp_path) + m.search_registry_packages.side_effect = RuntimeError("registry down") + with _mirror_patch(): + jobs, failed = pf._registry_jobs( + m, + [_FakeSpec(uri=None, name="a"), _FakeSpec(uri=None, name="b")], + set(), + ) + assert (jobs, failed) == ([], 2) + # The aggregate warning names a cause so an API break does not read + # as a registry outage + assert "Could not resolve 2 of 2" in caplog.text + assert "registry down" in caplog.text + + +def test_uri_fetch_job_promotes_atomically(tmp_path: Path) -> None: + """Checksum-less URL archives land via a locked staging file and an + atomic rename (the stable name is what keeps .part resume working).""" + dl_path = tmp_path / "archive" + + def fake_download(url, dest, progress=None, **kwargs): + Path(dest).write_bytes(b"data") + + manager = MagicMock() + with patch( + "esphome.framework_helpers.download_with_resume", side_effect=fake_download + ): + pf._uri_fetch_job(manager, "https://x/a.zip", dl_path, 4)(lambda done: None) + assert dl_path.read_bytes() == b"data" + # The archive is handed to pio's usage.db pruner + manager.set_download_utime.assert_called_once_with(str(dl_path)) + # no orphaned staging file; the lock file may or may not persist + # (filelock removes it on release on some platforms) + leftovers = {f.name for f in tmp_path.iterdir()} + assert leftovers - {f"{dl_path.name}.prefetch.lock"} == {dl_path.name} + + +def test_registry_fetch_job_skips_when_cached(tmp_path: Path) -> None: + """A destination another process completed is not re-downloaded.""" + dl_path = tmp_path / "archive" + dl_path.write_bytes(b"done") + with patch("esphome.framework_helpers.download_with_resume") as mock_download: + pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + )(lambda done: None) + mock_download.assert_not_called() + assert dl_path.read_bytes() == b"done" + + +def test_registry_fetch_job_downloads_under_lock(tmp_path: Path) -> None: + """Registry downloads write the shared cache path under the same lock + the URL path uses; interleaved writers would corrupt the archive.""" + dl_path = tmp_path / "archive" + order: list[str] = [] + with ( + patch( + "esphome.framework_helpers.download_with_resume", + side_effect=lambda url, dest, progress=None, **kw: ( + order.append("fetch"), + Path(dest).write_bytes(b"data"), # registration needs a real file + ), + ), + patch( + "filelock.FileLock.acquire", + side_effect=lambda *a, **k: order.append("lock"), + ), + patch( + "filelock.FileLock.release", + side_effect=lambda *a, **k: order.append("unlock"), + ), + ): + manager = MagicMock() + pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( + lambda done: None + ) + # FileLock.__del__ may add a trailing release; the contract is the order + assert order[:2] == ["lock", "fetch"] + assert "unlock" in order[2:] + manager.set_download_utime.assert_called_once_with(str(dl_path)) + + +def test_lockless_filesystem_downloads_unlocked( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A filesystem without lock support (ENOSYS/EPERM) degrades to an + unlocked download with one warning, never a per-package failure.""" + dl_path = tmp_path / "archive" + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch( + "filelock.FileLock.acquire", + side_effect=OSError(errno.ENOSYS, "no locks"), + ), + patch("filelock.FileLock.release"), + ): + pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + )(lambda done: None) + mock_download.assert_called_once() + assert "downloading unlocked" in caplog.text + + +def test_uri_fetch_job_failed_download_keeps_staging(tmp_path: Path) -> None: + """A failed fetch keeps the .part staging bytes for the next resume.""" + dl_path = tmp_path / "archive" + part = tmp_path / "archive.prefetch.part" + part.write_bytes(b"partial") + with ( + patch( + "esphome.framework_helpers.download_with_resume", + side_effect=OSError("network gone"), + ), + pytest.raises(OSError, match="network gone"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(lambda done: None) + assert part.read_bytes() == b"partial" + assert not dl_path.exists() + + +def test_uri_fetch_job_rejects_wrong_length(tmp_path: Path) -> None: + """A checksum-less body of the wrong length is never published under a + cache key pio would trust forever.""" + dl_path = tmp_path / "archive" + + def fake_download(url, dest, progress=None, **kwargs): + Path(dest).write_bytes(b"short") + + with ( + patch( + "esphome.framework_helpers.download_with_resume", side_effect=fake_download + ), + pytest.raises(ValueError, match="expected 9999 bytes"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 9999)( + lambda done: None + ) + assert not dl_path.exists() + assert not (tmp_path / "archive.prefetch").exists() + + +def test_sweep_stale_sidecars(tmp_path: Path) -> None: + """Sidecars past pio's own expiry are pruned; fresh and foreign files + stay.""" + old_time = pf.time.time() - 110 + stale = tmp_path / "a.tar.gz.part" + stale.write_bytes(b"x") + os.utime(stale, (old_time, old_time)) + fresh = tmp_path / "b.tar.gz.part" + fresh.write_bytes(b"x") + keep = tmp_path / "c.tar.gz" + keep.write_bytes(b"x") + os.utime(keep, (old_time, old_time)) + # A held lock can carry an ancient mtime (O_TRUNC keeps it); locks + # must never be swept or the single-writer guarantee reopens + held_lock = tmp_path / "d.tar.gz.esphome.lock" + held_lock.write_bytes(b"") + os.utime(held_lock, (old_time, old_time)) + pf._sweep_stale_sidecars(tmp_path, 100) + assert not stale.exists() + assert fresh.exists() + assert keep.exists() + assert held_lock.exists() + pf._sweep_stale_sidecars(tmp_path / "missing", 100) # tolerated + + +def test_register_download_failure_leaves_a_trace( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failed usage.db registration is traced; an unregistered archive + is never pruned, so silence would hide the leak coming back.""" + manager = MagicMock() + manager.set_download_utime.side_effect = RuntimeError("db locked") + with caplog.at_level(pf.logging.DEBUG): + pf._register_download(manager, tmp_path / "a.tar.gz") + assert "Could not register" in caplog.text + + +def test_sweep_logs_unprunable_files( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A sidecar that cannot be removed leaves a trace; a sweep that never + prunes must not look like a clean sweep.""" + old_time = pf.time.time() - 110 + stale = tmp_path / "a.tar.gz.part" + stale.write_bytes(b"x") + os.utime(stale, (old_time, old_time)) + with ( + patch.object(Path, "unlink", side_effect=OSError("busy")), + caplog.at_level(pf.logging.DEBUG), + ): + pf._sweep_stale_sidecars(tmp_path, 100) + assert "Could not remove" in caplog.text + + +def test_uri_lock_failure_is_a_counted_failure(tmp_path: Path) -> None: + """The checksum-less URL path never degrades to an unlocked shared + write; interleaved right-length corruption would go undetected.""" + dl_path = tmp_path / "archive" + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch( + "filelock.FileLock.acquire", + side_effect=OSError(errno.ENOSYS, "no locks"), + ), + pytest.raises(OSError), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(lambda done: None) + mock_download.assert_not_called() + + +def test_uri_fetch_job_no_discard_without_a_file(tmp_path: Path) -> None: + """When no archive landed (degraded serialized run), the staging bytes + stay for the next resume instead of being discarded.""" + dl_path = tmp_path / "archive" + part = tmp_path / "archive.prefetch.part" + part.write_bytes(b"partial") + with patch.object(pf, "_serialized_fetch_job", return_value=lambda tracker: None): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(lambda done: None) + assert part.read_bytes() == b"partial" + + +def test_uri_fetch_job_waits_out_a_briefly_held_lock(tmp_path: Path) -> None: + """A lock freed within the deadline lets the job proceed normally.""" + dl_path = tmp_path / "archive" + + def fake_download(url, dest, progress=None, **kwargs): + Path(dest).write_bytes(b"data") + + with ( + patch( + "esphome.framework_helpers.download_with_resume", side_effect=fake_download + ), + patch("filelock.FileLock.acquire", side_effect=[Timeout("held"), None]), + patch("filelock.FileLock.release"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(lambda done: None) + assert dl_path.read_bytes() == b"data" + + +def test_lock_deadline_leaves_download_to_the_holder(tmp_path: Path) -> None: + """A lock held past the deadline means another process is fetching the + same file; skipping cleanly beats a misleading failure warning. The + tracker is still polled so a parked worker observes cancellation.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) + mock_download.assert_not_called() + assert ticks == [0] + assert not dl_path.exists() + + +def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: + """A registry job that lost the download race to another process + must not stamp a nonexistent archive into pio's usage.db.""" + manager = MagicMock() + dl_path = tmp_path / "archive" + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + ): + pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( + lambda done: None + ) + mock_download.assert_not_called() + manager.set_download_utime.assert_not_called() + + +def test_main_interrupt_exits_quietly(tmp_path: Path) -> None: + """Ctrl-C reaches the child via the shared process group; it must exit + without a traceback.""" + with ( + patch("esphome.log.setup_log"), + patch.object(pf, "_prefetch", side_effect=KeyboardInterrupt), + ): + assert pf.main([str(tmp_path), "testenv"]) == 130 + + +def test_main_bad_log_level_falls_back(tmp_path: Path) -> None: + with ( + patch.dict("os.environ", {"ESPHOME_PREFETCH_LOG_LEVEL": "verbose"}), + patch("esphome.log.setup_log") as mock_setup, + patch.object(pf, "_prefetch"), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + assert mock_setup.call_args[0][0] == pf.logging.INFO + + +def test_main_silences_pio_manager_propagation(tmp_path: Path) -> None: + """The pio manager loggers carry their own handler; propagation to + the root handler would print every install line twice.""" + with patch("esphome.log.setup_log"), patch.object(pf, "_prefetch"): + assert pf.main([str(tmp_path), "testenv"]) == 0 + for name in ("Tool Manager", "Library Manager", "Platform Manager"): + assert pf.logging.getLogger(name).propagate is False + + +def test_main_quiet_level_reaches_pio_manager_loggers(tmp_path: Path) -> None: + """Manager construction re-pins its logger to INFO, so a quiet run + needs the logger-level filter to keep per-package lines out.""" + with ( + patch.dict("os.environ", {"ESPHOME_PREFETCH_LOG_LEVEL": "30"}), + patch("esphome.log.setup_log"), + patch.object(pf, "_prefetch"), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + lib_logger = pf.logging.getLogger("Library Manager") + lib_logger.setLevel(pf.logging.INFO) # what pio's _setup_logger does + info = pf.logging.LogRecord("Library Manager", 20, __file__, 1, "x", (), None) + warning = pf.logging.LogRecord("Library Manager", 30, __file__, 1, "x", (), None) + # Logger.filter returns falsy to drop, the record itself to pass + assert not lib_logger.filter(info) + assert lib_logger.filter(warning) + + +def test_main_mirrors_parent_log_setup(tmp_path: Path) -> None: + """The child adopts the parent's dashboard flag and log formatter so + its warnings and progress bar match the parent's.""" + with ( + patch.dict( + "os.environ", + {"ESPHOME_PREFETCH_LOG_LEVEL": "30", "ESPHOME_PREFETCH_DASHBOARD": "1"}, + ), + patch("esphome.log.setup_log") as mock_setup, + patch.object(pf, "_prefetch"), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + mock_setup.assert_called_once_with(30) + assert CORE.dashboard is True + + +def test_uri_fetch_job_skips_when_another_process_won(tmp_path: Path) -> None: + """A lost race discards the staging files; the cache never prunes them.""" + dl_path = tmp_path / "archive" + dl_path.write_bytes(b"done") + stale = [ + tmp_path / "archive.prefetch", + tmp_path / "archive.prefetch.part", + tmp_path / "archive.prefetch.part.meta", + ] + for f in stale: + f.write_bytes(b"stale") + with patch("esphome.framework_helpers.download_with_resume") as mock_download: + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(lambda done: None) + mock_download.assert_not_called() + assert dl_path.read_bytes() == b"done" + assert not any(f.exists() for f in stale) + + +def test_registry_jobs_one_bad_spec_keeps_the_rest(tmp_path: Path) -> None: + """A flaky resolution counts as failed without discarding the batch.""" + m = _fake_manager(tmp_path) + m.search_registry_packages.side_effect = [ + RuntimeError("registry 500"), + [{"any": 1}], + ] + with _mirror_patch(): + jobs, failed = pf._registry_jobs( + m, + [_FakeSpec(uri=None, name="flaky"), _FakeSpec(uri=None, name="good")], + set(), + ) + assert failed == 1 + assert len(jobs) == 1 + + +def test_uri_jobs_head_sizes_the_bar(tmp_path: Path) -> None: + """HEAD sizes direct-URL specs; git and unreachable URLs are skipped.""" + m = _fake_manager(tmp_path) + resp = MagicMock() + resp.headers = {"content-length": "2222"} + with patch("esphome.net_retry.http_request", return_value=resp): + jobs, failed = pf._uri_jobs( + m, + [ + _FakeSpec(uri="https://x/big.zip", name="big"), + _FakeSpec(uri="git+https://x/repo.git", name="repo"), + _FakeSpec(uri="https://x/repo.git#v1", name="barevcs"), + _FakeSpec(uri=None, name="registry"), + ], + set(), + ) + assert failed == 0 + assert [(n, s) for n, s, _ in jobs] == [("big", 2222)] + # a successful HEAD with no Content-Length is a clean skip + resp.headers = {} + with patch("esphome.net_retry.http_request", return_value=resp): + assert pf._uri_jobs( + m, [_FakeSpec(uri="https://x/nolen.zip", name="nolen")], set() + ) == ([], 0) + + +def test_uri_jobs_head_failure_counts_as_unresolved( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Network errors and transient statuses count as unresolved (and warn); + any permanent error status is a clean skip so the sentinel can still + be written (pio run names a broken URL when it downloads).""" + m = _fake_manager(tmp_path) + spec = [_FakeSpec(uri="https://x/a.zip", name="a")] + with patch("esphome.net_retry.http_request", side_effect=OSError("no route")): + assert pf._uri_jobs(m, spec, set()) == ([], 1) + resp = MagicMock(ok=False, status_code=503) + resp.headers = {"content-length": "999"} + with patch("esphome.net_retry.http_request", return_value=resp): + assert pf._uri_jobs(m, spec, set()) == ([], 1) + # 403 is how registries rate-limit; it must not be cached as warm + resp = MagicMock(ok=False, status_code=403) + resp.headers = {"content-length": "999"} + with patch("esphome.net_retry.http_request", return_value=resp): + assert pf._uri_jobs(m, spec, set()) == ([], 1) + resp = MagicMock(ok=False, status_code=405) + resp.headers = {"content-length": "999"} + with patch("esphome.net_retry.http_request", return_value=resp): + assert pf._uri_jobs(m, spec, set()) == ([], 0) + assert "HEAD https://x/a.zip" not in caplog.text + resp = MagicMock(ok=False, status_code=404) + resp.headers = {"content-length": "999"} + with patch("esphome.net_retry.http_request", return_value=resp): + assert pf._uri_jobs(m, spec, set()) == ([], 0) + assert "returned 404" not in caplog.text + + +def test_uri_jobs_dedupes_duplicate_urls(tmp_path: Path) -> None: + """Two specs with one URL yield one HEAD and one job.""" + m = _fake_manager(tmp_path) + resp = MagicMock() + resp.headers = {"content-length": "5"} + with patch("esphome.net_retry.http_request", return_value=resp) as mock_head: + jobs, failed = pf._uri_jobs( + m, + [ + _FakeSpec(uri="https://x/a.zip", name="a"), + _FakeSpec(uri="https://x/a.zip", name="a"), + ], + set(), + ) + assert failed == 0 + assert len(jobs) == 1 + mock_head.assert_called_once() + + +def test_uri_jobs_skips_installed_cached_and_seen(tmp_path: Path) -> None: + m = _fake_manager(tmp_path) + m.get_package.return_value = object() + spec = [_FakeSpec(uri="https://x/a.zip", name="a")] + assert pf._uri_jobs(m, spec, set()) == ([], 0) + m.get_package.return_value = None + dl = Path(m.compute_download_path("https://x/a.zip", "")) + dl.parent.mkdir(parents=True, exist_ok=True) + dl.touch() + assert pf._uri_jobs(m, spec, set()) == ([], 0) + dl.unlink() + # a registry job already claimed this download path + assert pf._uri_jobs(m, spec, {str(dl)}) == ([], 0) + + +def test_prefetch_spawns_isolated_subprocess(tmp_path: Path) -> None: + """Heal runs first, then the subprocess spawns with pio run's libdeps + dir and the parent's PYTHONPATH preserved (the child is esphome).""" + proc = MagicMock(returncode=0) + order = MagicMock() + order.run.return_value = proc + with ( + patch( + "esphome.platformio.toolchain.heal_platformio_python_env", + order.heal, + ), + patch.object(pf.subprocess, "run", order.run) as mock_run, + patch.dict("os.environ", {"PYTHONPATH": "/leak"}), + ): + pf.prefetch_platformio_packages() + assert [c[0] for c in order.mock_calls[:2]] == ["heal", "run"] + (cmd,), kwargs = mock_run.call_args + assert cmd == [ + sys.executable, + "-m", + "esphome.platformio.prefetch", + str(CORE.build_path), + "testenv", + ] + assert kwargs["env"]["PLATFORMIO_LIBDEPS_DIR"] == str( + CORE.relative_piolibdeps_path().absolute() + ) + # The child is esphome itself; PYTHONPATH must survive so it imports + # the same tree (tests/integration pins the source tree through it) + assert kwargs["env"]["PYTHONPATH"] == "/leak" + assert "ESPHOME_PREFETCH_DASHBOARD" not in kwargs["env"] + assert kwargs["timeout"] == pf._PREFETCH_TIMEOUT + + +def test_prefetch_passes_dashboard_flag(tmp_path: Path) -> None: + """The dashboard flag reaches the child so its bar still draws.""" + CORE.dashboard = True + with ( + patch("esphome.platformio.toolchain.heal_platformio_python_env"), + patch.object( + pf.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + pf.prefetch_platformio_packages() + assert mock_run.call_args[1]["env"]["ESPHOME_PREFETCH_DASHBOARD"] == "1" + + +@pytest.mark.parametrize( + ("run_effect", "expected"), + [ + ( + {"side_effect": pf.subprocess.TimeoutExpired("cmd", pf._PREFETCH_TIMEOUT)}, + "prefetch timed out", + ), + ({"return_value": MagicMock(returncode=4)}, "prefetch skipped (exit 4)"), + # Exit 1 is the interpreter's own import-failure code, never quiet + ({"return_value": MagicMock(returncode=1)}, "prefetch skipped (exit 1)"), + ({"side_effect": OSError("no exec")}, "PlatformIO package prefetch skipped"), + ], +) +def test_prefetch_spawn_failures_warn_and_continue( + caplog: pytest.LogCaptureFixture, run_effect, expected +) -> None: + """Timeouts, nonzero exits, and spawn failures each warn, never raise.""" + with ( + patch("esphome.platformio.toolchain.heal_platformio_python_env"), + patch.object(pf.subprocess, "run", **run_effect), + ): + pf.prefetch_platformio_packages() + assert expected in caplog.text + + +def test_prefetch_child_handled_failure_is_quiet( + caplog: pytest.LogCaptureFixture, +) -> None: + """Exit _EXIT_HANDLED (3) means the child already warned with the + reason; the parent adds no second warning.""" + with ( + patch("esphome.platformio.toolchain.heal_platformio_python_env"), + patch.object( + pf.subprocess, "run", return_value=MagicMock(returncode=pf._EXIT_HANDLED) + ), + ): + pf.prefetch_platformio_packages() + assert "prefetch skipped" not in caplog.text + + +def test_main_guards_and_exits_nonzero(caplog: pytest.LogCaptureFixture) -> None: + """A swallowed failure still reaches the parent as a nonzero exit; the + parent warns and continues, never failing the build.""" + with patch.object(pf, "_prefetch", side_effect=RuntimeError("boom")): + assert pf.main(["/b", "testenv"]) == pf._EXIT_HANDLED + assert "PlatformIO package prefetch skipped" in caplog.text + + +def test_main_runs_prefetch(tmp_path: Path) -> None: + with patch.object(pf, "_prefetch") as mock_prefetch: + assert pf.main([str(tmp_path), "testenv"]) == 0 + mock_prefetch.assert_called_once_with(tmp_path, "testenv") + + +def test_main_bad_argv_is_a_distinct_exit( + caplog: pytest.LogCaptureFixture, +) -> None: + """A parent/child wiring bug must not look like a network failure.""" + with patch.object(pf, "_prefetch") as mock_prefetch: + assert pf.main(["only-one"]) == 2 + mock_prefetch.assert_not_called() + assert "prefetch usage" in caplog.text + + +def _write_ini(tmp_path: Path, body: str) -> None: + (tmp_path / "platformio.ini").write_text(body) + + +def _write_valid_sentinel(tmp_path: Path, dirs: list[str]) -> None: + (tmp_path / pf._SENTINEL_NAME).write_text( + json.dumps({**pf._sentinel_state(tmp_path), "dirs": dirs}), encoding="utf-8" + ) + + +def test_prefetch_no_platform_returns(tmp_path: Path) -> None: + _write_ini(tmp_path, "[env:testenv]\n") + with patch.object(pf, "_registry_jobs") as mock_jobs: + pf._prefetch(tmp_path, "testenv") + mock_jobs.assert_not_called() + + +def _pio_modules(tmp_path: Path, fake_platform, fake_pm, config, lib_captures=None): + # A bare MagicMock's get_download_dir would fspath to '' and point the + # sidecar sweep at the process cwd + fake_pm.get_download_dir.return_value = str(tmp_path / "downloads") + fake_pm.DOWNLOAD_CACHE_EXPIRE = 86400 * 30 + + def fake_lib_manager(storage_dir): + if lib_captures is not None: + lib_captures.append(storage_dir) + return _fake_manager(tmp_path) + + modules = { + "platformio": MagicMock(), + "platformio.app": MagicMock(), + "platformio.project": MagicMock(), + "platformio.project.config": MagicMock(), + "platformio.dependencies": SimpleNamespace( + get_core_dependencies=lambda: { + "tool-scons": "~4.0", + "contrib-piohome": "~3", + } + ), + "platformio.package": MagicMock(), + "platformio.package.manager": MagicMock(), + "platformio.package.manager.library": SimpleNamespace( + LibraryPackageManager=fake_lib_manager + ), + "platformio.package.manager.platform": SimpleNamespace( + PlatformPackageManager=lambda: fake_pm + ), + "platformio.package.meta": SimpleNamespace( + PackageSpec=lambda *a, **kw: _FakeSpec( + uri=None, + name=kw.get("name") or (a[0] if a else None), + owner=kw.get("owner") + or (str(a[0]).split("/")[0] if a and "/" in str(a[0]) else None), + external=bool(a and "://" in str(a[0])), + ) + ), + "platformio.platform": MagicMock(), + "platformio.platform.factory": SimpleNamespace( + PlatformFactory=SimpleNamespace(new=lambda pkg: fake_platform) + ), + } + modules[ + "platformio.project.config" + ].ProjectConfig.get_instance.return_value = config + return modules + + +def _fake_config(tmp_path: Path, env_options: dict): + config = MagicMock() + options = { + "libdeps_dir": str(tmp_path / "libdeps"), + "packages_dir": str(tmp_path / "packages"), + **env_options, + } + config.get.side_effect = lambda section, key, default=None: options.get( + key, default + ) + return config + + +def test_prefetch_all_cached_is_quiet_and_writes_sentinel(tmp_path: Path) -> None: + """A no-work run neither logs nor batches, and records the sentinel.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + (tmp_path / "packages").mkdir() + (tmp_path / "libdeps" / "testenv").mkdir(parents=True) + fake_platform = MagicMock() + fake_platform.packages = {} + config = _fake_config( + tmp_path, {"platform": "fake/p@1", "lib_deps": ["esphome/noise-c@1.0"]} + ) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + with ( + patch.dict("sys.modules", modules), + patch.object(pf, "_registry_jobs", return_value=([], 0)), + patch.object(pf, "_uri_jobs", return_value=([], 0)), + patch.object(pf, "run_batch_downloads") as mock_batch, + ): + pf._prefetch(tmp_path, "testenv") + mock_batch.assert_not_called() + assert pf._prefetch_is_warm(tmp_path) + + +def test_prefetch_failed_resolution_is_not_cached_as_warm(tmp_path: Path) -> None: + """A registry outage must not write the sentinel.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + (tmp_path / "packages").mkdir() + fake_platform = MagicMock() + fake_platform.packages = {} + config = _fake_config(tmp_path, {"platform": "fake/p@1"}) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + with ( + patch.dict("sys.modules", modules), + patch.object(pf, "_registry_jobs", return_value=([], 1)), + patch.object(pf, "_uri_jobs", return_value=([], 0)), + patch.object(pf, "run_batch_downloads") as mock_batch, + ): + pf._prefetch(tmp_path, "testenv") + mock_batch.assert_not_called() + assert not (tmp_path / pf._SENTINEL_NAME).exists() + + +def test_sentinel_invalidation(tmp_path: Path) -> None: + """Ini changes, missing dirs, and garbage sentinels all read as cold.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + pkg_dir = tmp_path / "packages" + pkg_dir.mkdir() + assert not pf._prefetch_is_warm(tmp_path) # no sentinel yet + _write_valid_sentinel(tmp_path, [str(pkg_dir)]) + assert pf._prefetch_is_warm(tmp_path) + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@2\n") + assert not pf._prefetch_is_warm(tmp_path) # ini changed + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + pkg_dir.rmdir() + assert not pf._prefetch_is_warm(tmp_path) # recorded dir gone + (tmp_path / pf._SENTINEL_NAME).write_text("not json", encoding="utf-8") + assert not pf._prefetch_is_warm(tmp_path) + + +def test_prefetch_warm_sentinel_skips_spawn(tmp_path: Path) -> None: + """A valid sentinel skips the subprocess entirely.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + pkg_dir = tmp_path / "packages" + pkg_dir.mkdir() + _write_valid_sentinel(tmp_path, [str(pkg_dir)]) + with ( + patch("esphome.platformio.toolchain.heal_platformio_python_env"), + patch.object(pf.subprocess, "run") as mock_run, + ): + pf.prefetch_platformio_packages() + mock_run.assert_not_called() + + +def test_prefetch_end_to_end_wiring( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Platform installs dep-free, non-optional packages plus tool-scons + resolve, libraries use the env libdeps dir, a platform sys.path rewrite + is undone, and failures warn by name.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/platform@1.0\n") + fake_platform = MagicMock() + fake_platform.packages = { + "toolchain-x": {"optional": False}, + "framework-y": {"optional": True}, + } + fake_platform.get_package_spec.side_effect = lambda name: _FakeSpec( + uri=None, name=name + ) + # Platform setup code rewrites sys.path (pioarduino penv); _prefetch + # must restore it + bogus = str(tmp_path / "penv-site-packages") + fake_platform.configure_project_packages.side_effect = lambda env, targets: ( + sys.path.insert(0, bogus) + ) + fake_pm = MagicMock() + config = _fake_config( + tmp_path, + { + "platform": "fake/platform@1.0", + # the bare built-in name and the interpolation are skipped; + # only the owner-qualified library resolves + "lib_deps": ["esphome/noise-c@1.0", "WiFi", "${common.lib_deps}"], + }, + ) + lib_dirs: list[str] = [] + modules = _pio_modules(tmp_path, fake_platform, fake_pm, config, lib_dirs) + (tmp_path / pf._SENTINEL_NAME).write_text("{}", encoding="utf-8") + captured: dict = {} + + def fake_registry_jobs(manager, specs, seen): + captured.setdefault("spec_batches", []).append([s.name for s in specs]) + return [("toolchain-x@1", 10, lambda t: None)], 0 + + with ( + patch.dict("sys.modules", modules), + patch.object(pf, "_registry_jobs", side_effect=fake_registry_jobs), + patch.object(pf, "_uri_jobs", return_value=([], 0)), + patch.object( + pf, + "run_batch_downloads", + return_value=[("toolchain-x@1", OSError("down"))], + ) as mock_batch, + ): + pf._prefetch(tmp_path, "testenv") + fake_pm.install.assert_called_once_with("fake/platform@1.0", skip_dependencies=True) + assert not (tmp_path / pf._SENTINEL_NAME).exists() # stale sentinel removed + fake_platform.configure_project_packages.assert_called_once_with("testenv", ["run"]) + assert bogus not in sys.path + # non-optional platform package + tool-scons (never piohome), then libs + assert captured["spec_batches"][0] == ["toolchain-x", "tool-scons"] + assert captured["spec_batches"][1] == ["esphome/noise-c@1.0"] + assert lib_dirs == [str(Path(tmp_path / "libdeps") / "testenv")] + mock_batch.assert_called_once() + assert "Could not prefetch toolchain-x@1" in caplog.text + + +def test_prefetch_skips_duplicate_tool_scons(tmp_path: Path) -> None: + """A platform that lists tool-scons itself does not get it appended.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + fake_platform = MagicMock() + fake_platform.packages = {"tool-scons": {"optional": False}} + fake_platform.get_package_spec.side_effect = lambda name: _FakeSpec( + uri=None, name=name + ) + config = _fake_config(tmp_path, {"platform": "fake/p@1"}) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + batches: list[list[str]] = [] + with ( + patch.dict("sys.modules", modules), + patch.object( + pf, + "_registry_jobs", + side_effect=lambda mgr, specs, seen: ( + batches.append([s.name for s in specs]) or ([], 0) + ), + ), + patch.object(pf, "_uri_jobs", return_value=([], 0)), + ): + pf._prefetch(tmp_path, "testenv") + assert batches[0] == ["tool-scons"] diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 63c40f3609..fb99ea9208 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -932,8 +932,13 @@ def test_run_compile(setup_core: Path, mock_run_platformio_cli_run: Mock) -> Non config = {CONF_ESPHOME: {CONF_COMPILE_PROCESS_LIMIT: 4}} mock_run_platformio_cli_run.return_value = 0 - toolchain.run_compile(config, verbose=True) + with patch( + "esphome.platformio.prefetch.prefetch_platformio_packages" + ) as mock_prefetch: + toolchain.run_compile(config, verbose=True) + # The only wiring of the prefetch into a build lives here + mock_prefetch.assert_called_once_with() mock_run_platformio_cli_run.assert_called_once_with(config, True, "-j4") @@ -947,7 +952,8 @@ def test_run_compile_without_process_limit( config = {CONF_ESPHOME: {}} mock_run_platformio_cli_run.return_value = 0 - toolchain.run_compile(config, verbose=False) + with patch("esphome.platformio.prefetch.prefetch_platformio_packages"): + toolchain.run_compile(config, verbose=False) mock_run_platformio_cli_run.assert_called_once_with(config, False) @@ -1677,8 +1683,8 @@ def pio_core_dir(tmp_path: Path) -> Path: def test_current_python_minor_matches_running_interpreter() -> None: - """_current_python_minor returns major.minor of the running interpreter.""" - assert toolchain._current_python_minor() == _CURRENT_MINOR + """current_python_minor returns major.minor of the running interpreter.""" + assert toolchain.current_python_minor() == _CURRENT_MINOR def test_pio_stamp_round_trip(tmp_path: Path) -> None: From a8094ed548f6fe2de6ba59a34b4c89f80f5e2e4d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 21:45:47 -0500 Subject: [PATCH 28/30] [docker] Parallelize the image's PlatformIO library preinstall (#18777) --- script/platformio_install_deps.py | 411 ++++++++++-- tests/script/test_platformio_install_deps.py | 649 +++++++++++++++++++ 2 files changed, 1013 insertions(+), 47 deletions(-) create mode 100644 tests/script/test_platformio_install_deps.py diff --git a/script/platformio_install_deps.py b/script/platformio_install_deps.py index 8f7261efc3..1c4fb28b30 100755 --- a/script/platformio_install_deps.py +++ b/script/platformio_install_deps.py @@ -3,58 +3,375 @@ # all platformio libraries in the global storage import argparse +from concurrent.futures import ThreadPoolExecutor import configparser +from contextlib import suppress +import os +from pathlib import Path +import queue import subprocess +import threading +import traceback -config = configparser.ConfigParser(inline_comment_prefixes=(";",)) +# esphome is not installed at this docker layer; pio's fs.rmtree is the +# same chmod-on-readonly shape its own installer uses +try: + from platformio import fs + from platformio.cache import ContentCache + from platformio.package.manager.base import BasePackageManager + from platformio.package.manager.library import LibraryPackageManager + from platformio.package.manager.tool import ToolPackageManager + from platformio.package.meta import PackageCompatibility -parser = argparse.ArgumentParser(description="") -parser.add_argument("file", help="Path to platformio.ini", nargs=1) -parser.add_argument("-l", "--libraries", help="Install libraries", action="store_true") -parser.add_argument("-p", "--platforms", help="Install platforms", action="store_true") -parser.add_argument("-t", "--tools", help="Install tools", action="store_true") + PARALLEL_AVAILABLE = True +except ImportError as err: # pragma: no cover + # A moved pio module must degrade to the serial pass, not kill the + # image build; the tripwire test makes the drift loud in CI + PARALLEL_AVAILABLE = False + IMPORT_ERROR = repr(err) -args = parser.parse_args() - -config.read(args.file) +# Network-bound downloads release the GIL, so the pool oversubscribes +# the cores. This bypasses pio's 500ms registry throttle and races its +# self-unlinking cache LockFiles; both are cache-only and self-healing. +MAX_WORKERS = 16 -libs = [] -tools = [] -platforms = [] -# Extract from every lib_deps key in all sections -for section in config.sections(): - conf = config[section] - if "lib_deps" in conf and args.libraries: - for lib_dep in conf["lib_deps"].splitlines(): - if not lib_dep: - # Empty line or comment - continue - if lib_dep.startswith("${"): - # Extending from another section - continue - if "@" not in lib_dep: - # No version pinned, this is an internal lib - continue - libs.append("-l") - libs.append(lib_dep) - if "platform" in conf and args.platforms: - platforms.append("-p") - platforms.append(conf["platform"]) - if "platform_packages" in conf and args.tools: - for tool in conf["platform_packages"].splitlines(): - if not tool: - # Empty line or comment - continue - if tool.startswith("${"): - # Extending from another section - continue - if tool.find("https://github.com") != -1: - split = tool.find("@") - tool = tool[split + 1 :] - tools.append("-t") - tools.append(tool) +class CleanupError(RuntimeError): + """A torn destination could not be removed; the serial pass would + trust it, so the build must fail rather than bake a corrupt image.""" -subprocess.check_call( - ["platformio", "pkg", "install", "-g", *libs, *platforms, *tools], close_fds=False -) + +class LockReleaseError(RuntimeError): + """The manager lock could not be released; the serial pass would + block on it, so the build must fail with the cause named.""" + + +def parse_specs(path: str, args: argparse.Namespace) -> tuple[list, list, list]: + """Extract lib/platform/tool specs from every section of a platformio.ini.""" + config = configparser.ConfigParser(inline_comment_prefixes=(";",)) + if not config.read(path): + # ConfigParser silently ignores unreadable files; an empty spec + # list would build an image with no dependencies at all + raise SystemExit(f"Could not read {path}") + libs = [] + tools = [] + platforms = [] + for section in config.sections(): + conf = config[section] + if "lib_deps" in conf and args.libraries: + for lib_dep in conf["lib_deps"].splitlines(): + if not lib_dep: + # Empty line or comment + continue + if lib_dep.startswith("${"): + # Extending from another section + continue + if "@" not in lib_dep: + # No version pinned, this is an internal lib + continue + libs.append(lib_dep) + if "platform" in conf and args.platforms: + platforms.append(conf["platform"]) + if "platform_packages" in conf and args.tools: + for tool in conf["platform_packages"].splitlines(): + if not tool: + # Empty line or comment + continue + if tool.startswith("${"): + # Extending from another section + continue + if tool.find("https://github.com") != -1: + split = tool.find("@") + tool = tool[split + 1 :] + tools.append(tool) + # Exact-string dedupe only: name-level dedupe would change which + # version conflicts the pkg install pass reconciles + return ( + list(dict.fromkeys(libs)), + list(dict.fromkeys(platforms)), + list(dict.fromkeys(tools)), + ) + + +def piopm_matches(package_dir: str, spec) -> list[Path]: + """Dirs whose .piopm metadata names this spec; a positive match beats + guessing the manifest-derived dirname from the registry name.""" + want = (BasePackageManager.ensure_spec(spec).name or "").lower() + matches: list[Path] = [] + if not want: + return matches + try: + entries = list(Path(package_dir).iterdir()) + except FileNotFoundError: + return matches + for d in entries: + if not d.is_dir(): + continue # pio's get_installed skips files and *.pio-link too + try: + meta = fs.load_json(str(d / ".piopm")) + except FileNotFoundError: + continue # no metadata means pio does not trust it either + except (OSError, ValueError): + if d.name.lower() == want: + # A corrupt .piopm under this spec's own name would crash + # pio's whole storage scan; remove it + matches.append(d) + continue + mspec = meta.get("spec") or {} + if (mspec.get("name") or meta.get("name") or "").lower() == want: + matches.append(d) + return matches + + +def remove_dir(spec, dest: Path) -> None: + # fs.rmtree never raises (errors go to a printing onexc handler); + # only the destination's absence proves the cleanup worked + fs.rmtree(str(dest)) + if dest.exists(): + # Failing the build beats baking a corrupt image + raise CleanupError( + f"could not remove the failed pre-install of {spec} at {dest}" + ) + print(f"Removed torn destination {dest}", flush=True) + + +def cleanup_or_die(mgr, spec) -> None: + """Cleanup that did not demonstrably succeed must fail the build.""" + try: + clean_torn(mgr, spec) + except CleanupError: + raise + except Exception as err: # noqa: BLE001 + raise CleanupError(f"cleanup failed for {spec}: {err!r}") from err + + +def clean_torn(mgr, spec) -> None: + """Remove a torn destination so the serial pass cannot trust it.""" + pkg = None + with suppress(Exception): + # get_package memoizes a pre-install snapshot; reset to see the + # torn dir. It also recognizes manifest-only legacy dirs pio's + # storage scan would trust, which the .piopm fallback cannot see. + mgr.memcache_reset() + pkg = mgr.get_package(spec) + if pkg is not None: + remove_dir(spec, Path(pkg.path)) + elif dests := piopm_matches(mgr.package_dir, spec): + # A .piopm naming this spec is the exact shape the serial pass + # trusts; a dir without one is overwritten by pio's own install + for dest in dests: + remove_dir(spec, dest) + else: + print(f"No resolvable destination to clean for {spec}", flush=True) + + +def spec_key(spec) -> str | None: + """The destination identity of a spec: PlatformIO installs by package + name, so two specs sharing a name share a directory. ``None`` means + the name could not be derived; such a spec must stay out of the wave + (a raw-string key would break the one-per-destination guarantee).""" + name = BasePackageManager.ensure_spec(spec).name + return name.lower() if name else None + + +def dependency_specs(manager, specs: list) -> list: + """``(spec, compatibility)`` registry dependencies of installed + packages, from local manifest reads. Name-only dependencies + (platform-bundled libs like SPI) stay with the ``pkg install`` pass; + the compatibility qualifiers mirror pio's install_dependency, so a + qualified dep resolves to the same package the serial pass picks.""" + return [ + (manager.dependency_to_spec(dep), PackageCompatibility.from_dependency(dep)) + for spec in specs + if (pkg := manager.get_package(spec)) is not None + for dep in manager.get_pkg_dependencies(pkg) or [] + if dep.get("owner") or dep.get("version") + ] + + +def parallel_install(manager_cls, specs: list, prior_names: set | None = None) -> None: + """Best-effort parallel top-level install. + + PlatformIO's own installer downloads and unpacks one package at a time + on one core. Dependencies are skipped (two packages sharing one must + not extract into the same directory from two threads) and failures are + only reported: the stock ``pkg install`` pass afterwards installs + whatever is missing and is the authority on the final state. + """ + if not specs: + return + manager = manager_cls(None) + # One spec per destination: two threads must not extract into the + # same directory. Second versions of a name and URL specs (their dir + # comes from the archive manifest) stay with the pkg install pass. + seen_names: set = prior_names if prior_names is not None else set() + # Wave-1 items are strings; dependency waves carry (spec, compatibility) + pairs = [item if isinstance(item, tuple) else (item, None) for item in specs] + unique = {} + for spec, compat in pairs: + # Normalize once: a dependency's URL version surfaces as spec.uri + parsed = BasePackageManager.ensure_spec(spec) + if parsed.uri: + continue + if (key := spec_key(parsed)) is None: + # No name, no destination identity; leave it to the serial pass + print(f"Skipping unresolvable spec {spec!r} in the wave", flush=True) + continue + unique.setdefault(key, (spec, compat)) # first-wins, like pio's walk + pending = [ + (spec, compat) + for spec, compat in unique.values() + if not manager.get_package(spec) + ] + if not pending: + # Nothing to install, but a warm store's dependencies must still + # feed the next wave (a transitive dep may be missing) + _next_wave(manager_cls, manager, unique, seen_names) + return + workers = min(len(pending), MAX_WORKERS) + # One manager per worker (_install mutates instance state); built + # serially because construction rewires the shared manager logger + managers: queue.SimpleQueue = queue.SimpleQueue() + for _ in range(workers): + managers.put(manager_cls(None)) + local = threading.local() + + def install_one(item) -> bool: + spec, compat = item + if (mgr := getattr(local, "mgr", None)) is None: + mgr = local.mgr = managers.get_nowait() + try: + mgr._install( # noqa: SLF001 + spec, skip_dependencies=True, compatibility=compat + ) + return True + except Exception as err: # noqa: BLE001 + print(f"Pre-install of {spec} failed ({err!r})", flush=True) + cleanup_or_die(mgr, spec) + return False + except BaseException: + # A worker SystemExit (main() guards against it) must not skip + # the cleanup and leave a torn dir the serial pass trusts + cleanup_or_die(mgr, spec) + raise + + print(f"Preinstalling {len(pending)} package(s) with {workers} workers", flush=True) + # The serial getter calls create pio's lazy dirs (made without + # exist_ok) before cold-cache workers can race the creation + manager.get_download_dir() + manager.get_tmp_dir() + ContentCache("http") + cwd = Path.cwd() + manager.lock() + try: + with ThreadPoolExecutor(max_workers=workers) as ex: + futures = [ex.submit(install_one, item) for item in pending] + # The with-block joined every future; drain them all so a + # concurrent CleanupError is never dropped + errors = [err for f in futures if (err := f.exception()) is not None] + for err in errors: + # Every failure is on the record; the raised one is a summary + print(f"Wave failure: {err!r}", flush=True) + if errors: + raise next((e for e in errors if isinstance(e, CleanupError)), errors[0]) + results = [f.result() for f in futures] + finally: + try: + manager.unlock() + except Exception as unlock_err: # noqa: BLE001 + # A held flock would hang the serial pass in another process; + # failing loudly beats an unexplained stuck docker build. Any + # in-flight error stays attached as the context. + raise LockReleaseError( + f"could not release the manager lock: {unlock_err!r}" + ) from unlock_err + # Worker postinstall scripts chdir process-wide (pio's fs.cd); + # restore between waves. The serial pass pins its own cwd. + with suppress(OSError): + os.chdir(cwd) + if failures := len(results) - sum(results): + # The stock pass retries CLI specs and re-walks installed + # packages' dependencies, so failed deps retry too + print( + f"Pre-install failed for {failures} of {len(results)} package(s); " + "pkg install retries them serially", + flush=True, + ) + + # Waves skip dependencies (a shared one must not extract from two + # threads); the installed manifests feed the next wave + _next_wave(manager_cls, manager, unique, seen_names) + + +def _next_wave(manager_cls, manager, unique: dict, seen_names: set) -> None: + """Queue the dependency wave for every requested spec, installed or + freshly waved; a warm store can still be missing a transitive dep. + Terminates without a cap: each wave admits only never-seen names.""" + seen_names.update(unique) + # The pre-wave get_package calls memoized an empty storage snapshot + manager.memcache_reset() + next_specs = [ + item + for item in dependency_specs(manager, [spec for spec, _ in unique.values()]) + if spec_key(item[0]) not in seen_names + ] + if next_specs: + parallel_install(manager_cls, next_specs, seen_names) + + +def build_cli_args(libs: list, platforms: list, tools: list) -> list: + return [ + arg + for flag, specs in (("-l", libs), ("-p", platforms), ("-t", tools)) + for spec in specs + for arg in (flag, spec) + ] + + +def main() -> None: + parser = argparse.ArgumentParser(description="") + parser.add_argument("file", help="Path to platformio.ini", nargs=1) + parser.add_argument( + "-l", "--libraries", help="Install libraries", action="store_true" + ) + parser.add_argument( + "-p", "--platforms", help="Install platforms", action="store_true" + ) + parser.add_argument("-t", "--tools", help="Install tools", action="store_true") + args = parser.parse_args() + start_cwd = Path.cwd() + libs, platforms, tools = parse_specs(args.file[0], args) + + # Platforms stay serial: PlatformPackageManager.install runs an + # on_installed hook the private _install path would skip + if PARALLEL_AVAILABLE: + wave_groups = [(ToolPackageManager, tools), (LibraryPackageManager, libs)] + else: # pragma: no cover + wave_groups = [] + print( + f"PlatformIO layout changed ({IMPORT_ERROR}); serial install only", + flush=True, + ) + for manager_cls, specs in wave_groups: + try: + parallel_install(manager_cls, specs) + except (CleanupError, LockReleaseError, KeyboardInterrupt): + # A torn package or a held lock must fail the build + raise + except BaseException: # noqa: BLE001 + # BaseException: a worker postinstall's SystemExit must not + # skip the authoritative serial pass (partial deps, exit 0) + print("Parallel preinstall failed, falling back to serial", flush=True) + traceback.print_exc() + + # Postinstall scripts chdir process-wide (pio's fs.cd captures its + # restore path at construction); pin the authoritative pass's cwd + subprocess.check_call( + ["platformio", "pkg", "install", "-g", *build_cli_args(libs, platforms, tools)], + close_fds=False, + cwd=start_cwd, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py new file mode 100644 index 0000000000..a263d7937f --- /dev/null +++ b/tests/script/test_platformio_install_deps.py @@ -0,0 +1,649 @@ +"""Tests for script/platformio_install_deps.py.""" + +from argparse import Namespace +import importlib.util +import inspect +from pathlib import Path +import shutil +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from platformio import fs +from platformio.cache import ContentCache +from platformio.exception import InvalidJSONFile +from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageCompatibility, PackageItem, PackageSpec +import pytest +from semantic_version import Version + +_SCRIPT = Path(__file__).parents[2] / "script" / "platformio_install_deps.py" + + +def _load_script(): + spec = importlib.util.spec_from_file_location("platformio_install_deps", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # The real ContentCache would create dirs under the user's core dir + module.ContentCache = lambda *_: None + return module + + +def test_spec_key_collapses_destinations() -> None: + """Two specs delivering one package share a directory and one key.""" + mod = _load_script() + assert mod.spec_key("esphome/noise-c @ 0.1.21") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.21") == "noise-c" + assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( + "esp32async/asynctcp @ 3.5.0" + ) + url = "https://github.com/pioarduino/platform-espressif32/releases/download/{v}/platform-espressif32.zip" + assert mod.spec_key(url.format(v="55.03.311")) == mod.spec_key( + url.format(v="54.03.20") + ) + + +def test_parse_specs_and_cli_args(tmp_path: Path) -> None: + """Parsing skips unpinned and interpolated entries; the CLI rebuild + keeps the original flag pairing.""" + ini = tmp_path / "platformio.ini" + ini.write_text( + "[env:a]\n" + "platform = fake/platform@1\n" + "lib_deps =\n" + " esphome/noise-c @ 0.1.21\n" + " ${common.lib_deps}\n" + " internal_lib\n" + "[env:b]\n" + "lib_deps =\n" + " esphome/noise-c @ 0.1.21\n" + ) + mod = _load_script() + args = Namespace(libraries=True, platforms=True, tools=False) + libs, platforms, tools = mod.parse_specs(str(ini), args) + # exact-string duplicates collapse; distinct version pins survive + assert libs == ["esphome/noise-c @ 0.1.21"] + assert platforms == ["fake/platform@1"] + assert tools == [] + assert mod.build_cli_args(libs, platforms, tools) == [ + "-l", + "esphome/noise-c @ 0.1.21", + "-p", + "fake/platform@1", + ] + + +class _FakeManager: + """Scripted manager_cls: records installs, raises on demand.""" + + installed: set = set() + fail: set = set() + calls: list = [] + lock_events: list = [] + base_dir: str = "" # per-test tmp base; set by _reset_fake + + def __init__(self, package_dir) -> None: + assert package_dir is None + + @staticmethod + def _key(spec) -> str: + return spec if isinstance(spec, str) else str(spec) + + def get_package(self, spec): + if self._key(spec) in self.installed: + return SimpleNamespace(path="/tmp/fake-pkg", spec=self._key(spec)) + return None + + def memcache_reset(self) -> None: + type(self).resets = getattr(type(self), "resets", 0) + 1 + + @property + def package_dir(self) -> str: + return str(Path(type(self).base_dir) / "packages") + + def get_download_dir(self) -> str: + return str(Path(type(self).base_dir) / "downloads") + + def get_tmp_dir(self) -> str: + return str(Path(type(self).base_dir) / "tmp") + + def lock(self) -> None: + type(self).lock_events.append("lock") + + def unlock(self) -> None: + type(self).lock_events.append("unlock") + + def _install(self, spec, skip_dependencies, compatibility=None): + assert skip_dependencies is True + if self._key(spec) in self.fail: + raise RuntimeError("boom") + type(self).calls.append(spec) + type(self).compat_calls.append((self._key(spec), compatibility)) + type(self).installed.add(self._key(spec)) # atomic under the GIL + + def get_pkg_dependencies(self, pkg): + return getattr(type(self), "deps", {}).get(pkg.spec) + + dependency_to_spec = staticmethod(BasePackageManager.dependency_to_spec) + + +def _reset_fake(base_dir: str = "", **kwargs) -> type: + # A fresh subclass per test: nothing leaks between tests through the + # class-level scripted state + return type( + "_ScriptedManager", + (_FakeManager,), + { + "base_dir": base_dir, + "installed": kwargs.get("installed", set()), + "fail": kwargs.get("fail", set()), + "calls": [], + "compat_calls": [], + "lock_events": [], + }, + ) + + +def test_parallel_install_empty_specs_is_a_no_op(tmp_path: Path) -> None: + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + mod.parallel_install(cls, []) + assert cls.calls == [] and cls.lock_events == [] + + +def test_parallel_install_behavior(tmp_path: Path) -> None: + """Duplicates collapse to one install, installed specs are filtered, + URL specs stay out of the wave, and the lock wraps the pool.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), installed={"esphome/already @ 1.0"}) + mod.parallel_install( + cls, + [ + "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.21", + "esphome/already @ 1.0", + "https://x/framework.tar.xz", + ], + ) + assert cls.calls == ["esphome/noise-c @ 0.1.21"] + assert cls.lock_events == ["lock", "unlock"] + + +def test_parallel_install_failure_cleans_torn_destination( + tmp_path: Path, capsys +) -> None: + """A failed install resets the memcache, removes what get_package can + see, and reports; the others still install.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + + removed = [] + + torn = str(tmp_path / "packages" / "torn-pkg") # never created; only rmtree'd + + def get_package(self, spec): + if spec == "esphome/bad @ 1.0" and getattr(cls, "resets", 0): + return SimpleNamespace(path=torn, spec=spec) + return _FakeManager.get_package(self, spec) + + cls.get_package = get_package # throwaway subclass; nothing to restore + with patch.object(mod.fs, "rmtree", side_effect=removed.append): + mod.parallel_install(cls, ["esphome/bad @ 1.0", "esphome/good @ 1.0"]) + assert "esphome/good @ 1.0" in cls.calls + assert removed == [torn] + out = capsys.readouterr().out + assert "Pre-install of esphome/bad @ 1.0 failed" in out + assert "Pre-install failed for 1 of 2 package(s)" in out + + +def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: + """Dependencies of wave-installed packages install in a second wave, + deduped by name; name-only platform libs stay with the serial pass.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + cls.deps = { + "esphome/noise-c @ 0.1.21": [ + {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, + {"name": "SPI"}, + ], + "esphome/wg @ 1.0": [ + {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, + ], + } + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21", "esphome/wg @ 1.0"]) + assert len(cls.calls) == 3 # the shared dep installs exactly once + assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} + # Wave-1 strings carry no compatibility; the dependency wave does + compats = dict(cls.compat_calls) + assert compats["esphome/noise-c @ 0.1.21"] is None + dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) + assert dep_compat is not None # mirrors pio's install_dependency + + +def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: + """A dependency pinned to a URL surfaces as spec.uri; it must stay out + of the wave like string URL specs do.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + cls.deps = { + "esphome/noise-c @ 0.1.21": [ + {"name": "vendored", "version": "https://github.com/x/y.git"}, + ], + } + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} + + +def test_failed_cleanup_fails_the_build(tmp_path: Path) -> None: + """A torn destination still on disk after rmtree must fail the build: + fs.rmtree never raises (its onexc handler prints), so only the + destination's absence proves the cleanup worked.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + torn = tmp_path / "packages" / "torn-pkg" + torn.mkdir(parents=True) + + def get_package(self, spec): + if getattr(cls, "resets", 0): + return SimpleNamespace(path=str(torn), spec=spec) + return None + + cls.get_package = get_package # throwaway subclass; nothing to restore + with ( + patch.object(mod.fs, "rmtree", lambda path: None), # onexc swallowed + pytest.raises(mod.CleanupError, match="could not remove"), + ): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert cls.lock_events == ["lock", "unlock"] # still released + + +def test_unverifiable_torn_destination_fails_the_build(tmp_path: Path) -> None: + """When the scan fails, the spec's own .piopm decides: an unremovable + leftover fails the build.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + dest = Path(cls.base_dir) / "packages" / "bad" + dest.mkdir(parents=True) + (dest / ".piopm").write_text('{"spec": {"owner": "esphome", "name": "bad"}}') + + def bad_reset(self): + raise OSError("scan broken") + + cls.memcache_reset = bad_reset + with ( + patch.object(mod.fs, "rmtree", lambda path: None), # onexc swallowed + pytest.raises(mod.CleanupError, match="could not remove"), + ): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + + +def test_unverifiable_scan_without_leftover_degrades(tmp_path: Path, capsys) -> None: + """A failing scan with no destination on disk is never a build + failure blaming this spec.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + resets = {"n": 0} + + def bad_reset(self): + # Fail clean_torn's reset; the coordinator's later reset works + resets["n"] += 1 + if resets["n"] <= 1: + raise OSError("scan broken") + + cls.memcache_reset = bad_reset + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert "No resolvable destination to clean" in capsys.readouterr().out + + +def test_unresolvable_torn_destination_is_printed(tmp_path: Path, capsys) -> None: + """A failed install with no resolvable package prints, so an invisible + torn directory is at least traceable.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert "No resolvable destination to clean" in capsys.readouterr().out + + +def test_unparsable_torn_destination_is_removed(tmp_path: Path, capsys) -> None: + """A torn dir get_package cannot resolve but whose .piopm names the + spec is removed instead of surviving into the serial pass.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + dest = Path(cls.base_dir) / "packages" / "bad" + dest.mkdir(parents=True) + (dest / ".piopm").write_text('{"spec": {"owner": "esphome", "name": "bad"}}') + + with patch.object(mod.fs, "rmtree", shutil.rmtree): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert not dest.exists() + assert "Removed torn destination" in capsys.readouterr().out + + +def test_parse_specs_tools_branch(tmp_path: Path) -> None: + """platform_packages parsing keeps owner'd tools and rewrites github + URL pins to bare URLs the wave then skips via parsed.uri.""" + mod = _load_script() + ini = tmp_path / "platformio.ini" + ini.write_text( + "[env:t]\n" + "platform_packages =\n" + " ${common.platform_packages}\n" + " platformio/tool-scons@~4.40801.0\n" + " framework-arduinopico@https://github.com/earlephilhower/arduino-pico/releases/download/6.0.0/rp2040-6.0.0.zip\n" + ) + args = Namespace(libraries=False, platforms=False, tools=True) + libs, platforms, tools = mod.parse_specs(str(ini), args) + assert libs == [] and platforms == [] + assert tools == [ + "platformio/tool-scons@~4.40801.0", + "https://github.com/earlephilhower/arduino-pico/releases/download/6.0.0/rp2040-6.0.0.zip", + ] + assert mod.build_cli_args([], [], tools)[:2] == ["-t", tools[0]] + + +def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: + """Already-installed top-level packages still feed the dependency + wave; a warm store can be missing a transitive dep.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.21"}) + cls.deps = { + "esphome/noise-c @ 0.1.21": [ + {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, + ], + } + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] + + +def test_worker_system_exit_still_cleans(tmp_path: Path, capsys) -> None: + """A worker SystemExit runs the torn cleanup before propagating; the + serial pass must never trust its leftovers.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + torn = tmp_path / "packages" / "torn-pkg" + torn.mkdir(parents=True) + + def exiting_install(self, spec, skip_dependencies, compatibility=None): + raise SystemExit(0) + + def get_package(self, spec): + if getattr(cls, "resets", 0): + return SimpleNamespace(path=str(torn), spec=spec) + return None + + cls._install = exiting_install + cls.get_package = get_package + + def real_rmtree(path): + Path(path).rmdir() + + with ( + patch.object(mod.fs, "rmtree", real_rmtree), + pytest.raises(SystemExit), + ): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert not torn.exists() + + +def test_unlock_failure_is_fatal(tmp_path: Path) -> None: + """A failed unlock must fail the build: the serial pass in another + process would block on the held flock.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + + def bad_unlock(self): + raise OSError("flock broke") + + cls.unlock = bad_unlock + with pytest.raises(mod.LockReleaseError, match="manager lock"): + mod.parallel_install(cls, ["esphome/good @ 1.0"]) + + +def test_unlock_failure_keeps_inflight_error_as_context(tmp_path: Path) -> None: + """An in-flight CleanupError stays attached when the unlock fault + takes over the raise.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + torn = tmp_path / "packages" / "bad" + torn.mkdir(parents=True) + + def get_package(self, spec): + if getattr(cls, "resets", 0): + return SimpleNamespace(path=str(torn), spec=spec) + return None + + def bad_unlock(self): + raise OSError("flock broke") + + cls.get_package = get_package + cls.unlock = bad_unlock + with ( + patch.object(mod.fs, "rmtree", lambda path: None), # leaves torn + pytest.raises(mod.LockReleaseError) as err, + ): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert isinstance(err.value.__cause__.__context__, mod.CleanupError) + + +def test_chdir_failure_does_not_fail_the_wave(tmp_path: Path, monkeypatch) -> None: + """A lost cwd is suppressed: further waves may misbehave and fall to + the serial pass, whose cwd is pinned.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + monkeypatch.setattr(mod.os, "chdir", MagicMock(side_effect=OSError("gone"))) + mod.parallel_install(cls, ["esphome/good @ 1.0"]) + assert cls.calls == ["esphome/good @ 1.0"] + + +def test_piopm_match_removes_manifest_named_torn_dir(tmp_path: Path, capsys) -> None: + """A torn dir named by its manifest (not the registry spec) is found + through its .piopm and removed.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + torn = tmp_path / "packages" / "ManifestName" + torn.mkdir(parents=True) + (torn / ".piopm").write_text('{"spec": {"owner": "esphome", "name": "bad"}}') + innocent = tmp_path / "packages" / "innocent" + innocent.mkdir() + (innocent / ".piopm").write_text('{"spec": {"owner": "o", "name": "other"}}') + with patch.object(mod.fs, "rmtree", shutil.rmtree): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert not torn.exists() + assert innocent.exists() # another package's valid metadata survives + assert "Removed torn destination" in capsys.readouterr().out + + +def test_unscannable_package_dir_fails_the_build(tmp_path: Path) -> None: + """A storage dir the cleanup cannot scan is not proof of cleanliness.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + real_iterdir = Path.iterdir + + def broken_iterdir(self): + if self.name == "packages": + raise PermissionError("denied") + return real_iterdir(self) + + with ( + patch.object(Path, "iterdir", broken_iterdir), + pytest.raises(mod.CleanupError, match="cleanup failed"), + ): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + + +def test_stray_file_in_package_dir_is_ignored(tmp_path: Path) -> None: + """A plain file (or a pio-link) beside the packages is skipped by + pio's own scan and must never hard-fail the build.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + (tmp_path / "packages").mkdir(parents=True) + (tmp_path / "packages" / "stray.pio-link").write_text("x") + (tmp_path / "packages" / "no-metadata").mkdir() # pio overwrites these + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert (tmp_path / "packages" / "stray.pio-link").exists() + assert (tmp_path / "packages" / "no-metadata").exists() + + +def test_unreadable_piopm_dir_is_removed(tmp_path: Path) -> None: + """A persistently corrupt .piopm under this spec's own name would + crash pio's storage scan; the dir is removed rather than left to + break the serial pass.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + torn = tmp_path / "packages" / "bad" + torn.mkdir(parents=True) + (torn / ".piopm").write_text("{not json") + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert not torn.exists() + + +def test_unreadable_piopm_under_other_name_survives(tmp_path: Path) -> None: + """A corrupt .piopm in another package's dir may be a worker mid-copy; + a failing spec must not remove a directory it does not own.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + other = tmp_path / "packages" / "innocent" + other.mkdir(parents=True) + (other / ".piopm").write_text("{not json") + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + assert other.exists() + + +def test_unexpected_cleanup_class_becomes_cleanup_error(tmp_path: Path) -> None: + """Cleanup failures of any class fail the build; nothing may be + downgraded to the serial fallback over a torn directory.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path), fail={"esphome/bad @ 1.0"}) + + with ( + patch.object( + mod, "piopm_matches", MagicMock(side_effect=ValueError("bad spec")) + ), + pytest.raises(mod.CleanupError, match="cleanup failed"), + ): + mod.parallel_install(cls, ["esphome/bad @ 1.0"]) + + +def test_main_cleanup_error_fails_before_generic_fallback(tmp_path: Path) -> None: + """A CleanupError must escape main's serial fallback: the clause order + decides whether a stuck torn package fails the image build.""" + mod = _load_script() + ini = tmp_path / "platformio.ini" + ini.write_text("[env:t]\nlib_deps =\n esphome/x @ 1.0\n") + with ( + patch.object( + mod, "parallel_install", side_effect=mod.CleanupError("stuck torn pkg") + ), + patch.object(mod.subprocess, "check_call"), + patch.object(sys, "argv", ["platformio_install_deps.py", str(ini), "-l"]), + pytest.raises(mod.CleanupError), + ): + mod.main() + + +def test_main_generic_failure_still_runs_serial_pass(tmp_path: Path) -> None: + """A non-CleanupError wave failure prints, dumps the traceback, and + still reaches the authoritative serial pass with the pinned cwd.""" + mod = _load_script() + ini = tmp_path / "platformio.ini" + ini.write_text("[env:t]\nlib_deps =\n esphome/x @ 1.0\n") + with ( + patch.object(mod, "parallel_install", side_effect=RuntimeError("boom")), + patch.object(mod.subprocess, "check_call") as mock_call, + patch.object(sys, "argv", ["platformio_install_deps.py", str(ini), "-l"]), + ): + mod.main() + mock_call.assert_called_once() + args, kwargs = mock_call.call_args + assert args[0][:4] == ["platformio", "pkg", "install", "-g"] + assert "esphome/x @ 1.0" in args[0] + assert kwargs["cwd"] == Path.cwd() + + +def test_content_cache_creates_its_dir(tmp_path: Path, monkeypatch) -> None: + """The cold-cache hardening relies on ContentCache.__init__ creating + the namespace dir; pin the side effect, not mere callability.""" + monkeypatch.setenv("PLATFORMIO_CACHE_DIR", str(tmp_path / "cache")) + ContentCache("http") + assert (tmp_path / "cache" / "http").is_dir() + + +def test_piopm_matches_without_name_matches_nothing(tmp_path: Path) -> None: + """A spec with no derivable name can never match a directory.""" + mod = _load_script() + assert mod.piopm_matches(str(tmp_path), "") == [] + + +def test_unresolvable_spec_stays_out_of_the_wave(tmp_path: Path, capsys) -> None: + """A spec with no derivable name is left to the serial pass; a raw + string key would break the one-per-destination dedupe.""" + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + nameless = PackageSpec(requirements="^1.0") + mod.parallel_install(cls, [nameless]) + assert cls.calls == [] + assert "Skipping unresolvable spec" in capsys.readouterr().out + + +def test_parallel_install_unlocks_when_pool_fails(tmp_path: Path) -> None: + mod = _load_script() + cls = _reset_fake(str(tmp_path)) + with ( + patch.object(mod, "ThreadPoolExecutor", side_effect=RuntimeError("no")), + pytest.raises(RuntimeError), + ): + mod.parallel_install(cls, ["esphome/a @ 1.0"]) + assert cls.lock_events == ["lock", "unlock"] + + +def test_parse_specs_unreadable_ini_fails_loudly(tmp_path: Path) -> None: + """A bad path must not silently build an image with no dependencies.""" + mod = _load_script() + args = Namespace(libraries=True, platforms=False, tools=False) + with pytest.raises(SystemExit): + mod.parse_specs(str(tmp_path / "missing.ini"), args) + + +def test_platformio_surface_for_install_deps_script() -> None: + """A PlatformIO bump that changes these members must fail here, not + silently turn the docker image's parallel preinstall into a no-op.""" + # The script calls these positionally; pin the positions, not just + # membership, so a parameter reorder trips the wire too + params = inspect.signature(PackageManagerInstallMixin._install).parameters + assert list(params)[1] == "spec" + assert "skip_dependencies" in params + assert "compatibility" in params + for cls in (ToolPackageManager, LibraryPackageManager): + assert list(inspect.signature(cls.__init__).parameters)[1] == "package_dir" + for name in ( + "lock", + "unlock", + "get_package", + "memcache_reset", + "get_pkg_dependencies", + "dependency_to_spec", + "get_download_dir", + "get_tmp_dir", + ): + assert callable(getattr(BasePackageManager, name)) + # Losing any of these turns the wave into main()'s silent serial + # fallback: ensure_spec runs in the coordinator, the spec attributes + # feed the dedupe, cleanup, and dependency filters + assert callable(BasePackageManager.ensure_spec) + spec = PackageSpec("owner/name @ ^1.0") + assert spec.name == "name" + assert spec.owner == "owner" + assert spec.uri is None + assert spec.external is False + assert Version("1.5.0") in spec.requirements + # The failure-cleanup path degrades to a single line if these vanish + assert callable(fs.rmtree) + assert callable(fs.load_json) + # piopm_matches only tolerates a corrupt .piopm through this base; + # losing it would flip a wave failure from degrade to build failure + assert issubclass(InvalidJSONFile, ValueError) + assert PackageItem("pkg-dir").path == "pkg-dir" + assert callable(PackageCompatibility.from_dependency) From e55a8aeabe97e21cac878f8933d132a0c95e1b8b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 21:49:12 -0500 Subject: [PATCH 29/30] [api] Drop connection instead of crashing when buffer allocation fails (#18803) --- esphome/components/api/api_buffer.cpp | 11 ++++- esphome/components/api/api_buffer.h | 32 +++++------- esphome/components/api/api_connection.cpp | 49 ++++++++++++++----- esphome/components/api/api_connection.h | 22 +++------ .../components/api/api_connection_buffer.h | 26 ++++++++-- .../components/api/api_frame_helper_noise.cpp | 27 +++++++--- .../api/api_frame_helper_plaintext.cpp | 5 +- .../components/api/bench_list_entities.cpp | 12 ++--- .../components/api/bench_log_response.cpp | 8 +-- .../components/api/bench_plaintext_frame.cpp | 8 +-- .../components/api/bench_proto_decode.cpp | 2 +- .../components/api/bench_proto_encode.cpp | 24 ++++----- .../components/api/bench_proto_proxy.cpp | 10 ++-- .../components/api/bench_proto_varint.cpp | 6 +-- .../components/api/test_proto_mac_varint.cpp | 2 +- 15 files changed, 148 insertions(+), 96 deletions(-) diff --git a/esphome/components/api/api_buffer.cpp b/esphome/components/api/api_buffer.cpp index 6db18b0365..fc45a4e971 100644 --- a/esphome/components/api/api_buffer.cpp +++ b/esphome/components/api/api_buffer.cpp @@ -1,13 +1,20 @@ #include "api_buffer.h" +#include namespace esphome::api { -void APIBuffer::grow_(size_t n) { - auto new_data = make_buffer(n); +bool APIBuffer::grow_(size_t n) { + // nothrow (no zero-fill) so OOM is reportable; plain new aborts instead + // (NEW_OOM_ABORT on ESP8266 Arduino, exception stub on ESP-IDF). + // RAMAllocator is no fit here: unique_ptr needs delete[]-compatible memory. + std::unique_ptr new_data(new (std::nothrow) uint8_t[n]); + if (new_data == nullptr) + return false; if (this->size_) std::memcpy(new_data.get(), this->data_.get(), this->size_); this->data_ = std::move(new_data); this->capacity_ = n; + return true; } } // namespace esphome::api diff --git a/esphome/components/api/api_buffer.h b/esphome/components/api/api_buffer.h index 1d0cccf61c..396dadbe58 100644 --- a/esphome/components/api/api_buffer.h +++ b/esphome/components/api/api_buffer.h @@ -9,16 +9,6 @@ namespace esphome::api { -/// Helper to use make_unique_for_overwrite where available (skips zero-fill), -/// falling back to make_unique on older GCC (ESP8266, LibreTiny). -inline std::unique_ptr make_buffer(size_t n) { -#if defined(USE_ESP8266) || defined(USE_LIBRETINY) - return std::make_unique(n); -#else - return std::make_unique_for_overwrite(n); -#endif -} - /// Byte buffer that skips zero-initialization on resize(). /// /// std::vector::resize() zero-fills new bytes via memset. For the @@ -36,23 +26,23 @@ inline std::unique_ptr make_buffer(size_t n) { class APIBuffer { public: void clear() { this->size_ = 0; } - inline void reserve(size_t n) ESPHOME_ALWAYS_INLINE { - if (n > this->capacity_) - this->grow_(n); - } - inline void resize(size_t n) ESPHOME_ALWAYS_INLINE { - this->reserve(n); - this->size_ = n; // no zero-fill - } + /// Returns false if allocation fails; the buffer is left unchanged. + [[nodiscard]] inline bool reserve(size_t n) ESPHOME_ALWAYS_INLINE { return n <= this->capacity_ || this->grow_(n); } + /// Returns false if allocation fails; the buffer is left unchanged. No zero-fill. + [[nodiscard]] inline bool resize(size_t n) ESPHOME_ALWAYS_INLINE { return this->reserve_and_resize(n, n); } /// Reserve capacity for max(reserve_size, new_size) bytes, then set size to new_size. /// Single grow_ check regardless of argument order. - inline void reserve_and_resize(size_t reserve_size, size_t new_size) ESPHOME_ALWAYS_INLINE { - this->reserve(std::max(reserve_size, new_size)); + /// Returns false if allocation fails; the buffer is left unchanged. + [[nodiscard]] inline bool reserve_and_resize(size_t reserve_size, size_t new_size) ESPHOME_ALWAYS_INLINE { + if (!this->reserve(std::max(reserve_size, new_size))) + return false; this->size_ = new_size; + return true; } uint8_t *data() { return this->data_.get(); } const uint8_t *data() const { return this->data_.get(); } size_t size() const { return this->size_; } + size_t capacity() const { return this->capacity_; } bool empty() const { return this->size_ == 0; } uint8_t &operator[](size_t i) { return this->data_[i]; } const uint8_t &operator[](size_t i) const { return this->data_[i]; } @@ -64,7 +54,7 @@ class APIBuffer { } protected: - void grow_(size_t n); + bool grow_(size_t n); std::unique_ptr data_; size_t size_{0}; size_t capacity_{0}; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 7b0cb7069e..bc088ca473 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1,6 +1,6 @@ #include "api_connection.h" #ifdef USE_API -#include "api_connection_buffer.h" // for encode_to_buffer / get_batch_delay_ms_ inlines +#include "api_connection_buffer.h" // for the APIServer-dependent APIConnection inlines #ifdef USE_API_NOISE #include "api_frame_helper_noise.h" #endif @@ -2239,10 +2239,17 @@ bool APIConnection::send_message_(uint32_t payload_size, uint16_t message_type, this->log_send_message_(proto_msg->message_name(), proto_msg->dump_to(dump_buf)); } #endif + if (!this->prepare_first_message_buffer(payload_size)) [[unlikely]] { + this->fatal_out_of_memory_(); + return false; + } auto &shared_buf = this->parent_->get_shared_buffer_ref(); - this->prepare_first_message_buffer(shared_buf, payload_size); size_t write_start = shared_buf.size(); - shared_buf.resize(write_start + payload_size); +#ifdef ESPHOME_DEBUG_API + assert(shared_buf.capacity() >= write_start + payload_size); +#endif + // Capacity reserved above, cannot fail + (void) shared_buf.resize(write_start + payload_size); ProtoWriteBuffer buffer{&shared_buf, write_start}; encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); return this->send_buffer(ProtoWriteBuffer{&shared_buf}, message_type); @@ -2278,6 +2285,9 @@ void APIConnection::on_no_setup_connection() { this->on_fatal_error(); this->log_client_(ESPHOME_LOG_LEVEL_DEBUG, LOG_STR("no connection setup")); } +void APIConnection::fatal_out_of_memory_() { + this->fatal_error_with_log_(LOG_STR("Out of memory"), APIError::OUT_OF_MEMORY); +} void APIConnection::on_fatal_error() { // Don't close socket here - keep it open so getpeername() works for logging // Socket will be closed when client is removed from the list in APIServer::loop() @@ -2292,16 +2302,25 @@ bool APIConnection::schedule_message_front_(EntityBase *entity, uint16_t message bool APIConnection::send_message_smart_(EntityBase *entity, uint16_t message_type, uint8_t estimated_size, uint8_t aux_data_index) { if (this->should_send_immediately_(message_type) && this->helper_->can_write_without_blocking()) { - auto &shared_buf = this->parent_->get_shared_buffer_ref(); - this->prepare_first_message_buffer(shared_buf, estimated_size); + // No local for the shared buffer here: keeping it live across + // dispatch_message_ costs a register and spills message_type into the + // batching path's dedup loop (measured on x86 GCC -Os) + if (!this->prepare_first_message_buffer(estimated_size)) [[unlikely]] { + this->fatal_out_of_memory_(); + return false; + } DeferredBatch::BatchItem item{entity, message_type, estimated_size, aux_data_index}; if (this->dispatch_message_(item, MAX_BATCH_PACKET_SIZE, true) && - this->send_buffer(ProtoWriteBuffer{&shared_buf}, message_type)) { + this->send_buffer(ProtoWriteBuffer{&this->parent_->get_shared_buffer_ref()}, message_type)) { #ifdef HAS_PROTO_MESSAGE_DUMP this->log_batch_item_(item); #endif return true; } + // An OOM during the immediate attempt marks the connection for removal; + // don't queue more work (schedule_message_'s push_back may allocate again) + if (this->flags_.remove) [[unlikely]] + return false; } return this->schedule_message_(entity, message_type, estimated_size, aux_data_index); } @@ -2351,7 +2370,11 @@ void APIConnection::process_batch_() { total_estimated_size = MAX_BATCH_PACKET_SIZE; } - this->prepare_first_message_buffer(shared_buf, header_padding, total_estimated_size); + if (!this->prepare_first_message_buffer(header_padding, total_estimated_size)) [[unlikely]] { + this->fatal_out_of_memory_(); + this->clear_batch_(); + return; + } // Fast path for single message - buffer already allocated above if (num_items == 1) { @@ -2366,8 +2389,10 @@ void APIConnection::process_batch_() { #endif this->clear_batch_(); } else if (payload_size == 0) { - // Message too large to fit in available space - ESP_LOGW(TAG, "Message too large to send: type=%u", item.message_type); + // payload_size == 0 with remove set means encoding hit OOM and the + // connection is being dropped; warn only for a genuinely oversized message + if (!this->flags_.remove) + ESP_LOGW(TAG, "Message too large to send: type=%u", item.message_type); this->clear_batch_(); } return; @@ -2430,8 +2455,10 @@ void APIConnection::process_batch_multi_(APIBuffer &shared_buf, size_t num_items if (items_processed > 0) { // Add footer space for the last message (for Noise protocol MAC) - if (footer_size > 0) { - shared_buf.resize(shared_buf.size() + footer_size); + if (footer_size > 0 && !shared_buf.resize(shared_buf.size() + footer_size)) [[unlikely]] { + this->fatal_out_of_memory_(); + this->clear_batch_(); + return; } // Send all collected messages diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 5a554f4857..a4c49dccf4 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -352,22 +352,13 @@ class APIConnection final : public APIServerConnectionBase { } } - void prepare_first_message_buffer(APIBuffer &shared_buf, size_t header_padding, size_t total_size) { - shared_buf.clear(); - // Reserve space for header padding + message + footer - // - Header padding: space for protocol headers (7 bytes for Noise, 6 for Plaintext) - // - Footer: space for MAC (16 bytes for Noise, 0 for Plaintext) - // Reserve full size but only set initial size to header padding - // so message encoding starts at the correct position - shared_buf.reserve_and_resize(total_size, header_padding); - } + /// Clear the shared write buffer and reserve space for the first message. + /// Returns false if the allocation fails (out of memory). + /// Defined in api_connection_buffer.h (needs APIServer complete). + [[nodiscard]] bool prepare_first_message_buffer(size_t header_padding, size_t total_size); // Convenience overload - computes frame overhead internally - void prepare_first_message_buffer(APIBuffer &shared_buf, size_t payload_size) { - const uint8_t header_padding = this->helper_->frame_header_padding(); - const uint8_t footer_size = this->helper_->frame_footer_size(); - this->prepare_first_message_buffer(shared_buf, header_padding, payload_size + header_padding + footer_size); - } + [[nodiscard]] bool prepare_first_message_buffer(size_t payload_size); bool try_to_clear_buffer(bool log_out_of_space) { if (this->flags_.remove) @@ -853,6 +844,9 @@ class APIConnection final : public APIServerConnectionBase { this->on_fatal_error(); this->log_warning_(message, err); } + // Shared cold path for buffer allocation failures — noinline keeps the + // OOM handling out of the hot send paths + void __attribute__((noinline)) fatal_out_of_memory_(); }; } // namespace esphome::api diff --git a/esphome/components/api/api_connection_buffer.h b/esphome/components/api/api_connection_buffer.h index 1dd8a162e4..08520249bf 100644 --- a/esphome/components/api/api_connection_buffer.h +++ b/esphome/components/api/api_connection_buffer.h @@ -3,8 +3,8 @@ #include "esphome/core/defines.h" #ifdef USE_API -// Inline APIConnection methods that need APIServer complete. Include this -// instead of api_connection.h when calling encode_to_buffer or get_batch_delay_ms_. +// Inline APIConnection members that need APIServer complete. Include this +// instead of api_connection.h when calling them. #include "api_connection.h" #include "api_server.h" @@ -41,7 +41,10 @@ inline uint16_t ESPHOME_ALWAYS_INLINE APIConnection::encode_to_buffer(uint32_t c return 0; auto &shared_buf = conn->parent_->get_shared_buffer_ref(); - shared_buf.resize(shared_buf.size() + to_add); + if (!shared_buf.resize(shared_buf.size() + to_add)) [[unlikely]] { + conn->fatal_out_of_memory_(); + return 0; + } ProtoWriteBuffer buffer{&shared_buf, shared_buf.size() - calculated_size}; encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); @@ -50,5 +53,22 @@ inline uint16_t ESPHOME_ALWAYS_INLINE APIConnection::encode_to_buffer(uint32_t c inline uint32_t APIConnection::get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } +inline bool APIConnection::prepare_first_message_buffer(size_t header_padding, size_t total_size) { + auto &shared_buf = this->parent_->get_shared_buffer_ref(); + shared_buf.clear(); + // Reserve space for header padding + message + footer + // - Header padding: space for protocol headers (7 bytes for Noise, 6 for Plaintext) + // - Footer: space for MAC (16 bytes for Noise, 0 for Plaintext) + // Reserve full size but only set initial size to header padding + // so message encoding starts at the correct position + return shared_buf.reserve_and_resize(total_size, header_padding); +} + +inline bool APIConnection::prepare_first_message_buffer(size_t payload_size) { + const uint8_t header_padding = this->helper_->frame_header_padding(); + const uint8_t footer_size = this->helper_->frame_footer_size(); + return this->prepare_first_message_buffer(header_padding, payload_size + header_padding + footer_size); +} + } // namespace esphome::api #endif diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 9c4cc2aa78..138dbdddba 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -68,7 +68,10 @@ APIError APINoiseFrameHelper::init() { // init prologue size_t old_size = prologue_.size(); - prologue_.resize(old_size + PROLOGUE_INIT_LEN); + if (!prologue_.resize(old_size + PROLOGUE_INIT_LEN)) [[unlikely]] { + state_ = State::FAILED; + return APIError::OUT_OF_MEMORY; + } #ifdef USE_ESP8266 memcpy_P(prologue_.data() + old_size, PROLOGUE_INIT, PROLOGUE_INIT_LEN); #else @@ -202,7 +205,10 @@ APIError APINoiseFrameHelper::try_read_frame_() { // During handshake, rx_buf_.size() is used in prologue construction, so // the buffer must be exactly msg_size to avoid prologue mismatch.) uint16_t alloc_size = msg_size + (is_data ? RX_BUF_NULL_TERMINATOR : 0); - this->rx_buf_.resize(alloc_size); + if (!this->rx_buf_.resize(alloc_size)) [[unlikely]] { + state_ = State::FAILED; + return APIError::OUT_OF_MEMORY; + } if (rx_buf_len_ < msg_size) { // more data to read @@ -269,7 +275,10 @@ APIError APINoiseFrameHelper::state_action_client_hello_() { // Resize for: existing prologue + 2 size bytes + frame data size_t old_size = this->prologue_.size(); size_t rx_size = this->rx_buf_.size(); - this->prologue_.resize(old_size + 2 + rx_size); + if (!this->prologue_.resize(old_size + 2 + rx_size)) [[unlikely]] { + state_ = State::FAILED; + return APIError::OUT_OF_MEMORY; + } this->prologue_[old_size] = (uint8_t) (rx_size >> 8); this->prologue_[old_size + 1] = (uint8_t) rx_size; if (rx_size > 0) { @@ -477,13 +486,15 @@ APIError APINoiseFrameHelper::write_protobuf_packet(uint16_t type, ProtoWriteBuf assert(this->state_ == State::DATA); #endif + APIBuffer *buf = buffer.get_buffer(); // Resize buffer to include footer space for Noise MAC - if (this->frame_footer_size_) - buffer.get_buffer()->resize(buffer.get_buffer()->size() + this->frame_footer_size_); + if (this->frame_footer_size_ && !buf->resize(buf->size() + this->frame_footer_size_)) [[unlikely]] { + state_ = State::FAILED; + return APIError::OUT_OF_MEMORY; + } - uint16_t payload_size = - static_cast(buffer.get_buffer()->size() - HEADER_PADDING - this->frame_footer_size_); - uint8_t *buf_start = buffer.get_buffer()->data(); + uint16_t payload_size = static_cast(buf->size() - HEADER_PADDING - this->frame_footer_size_); + uint8_t *buf_start = buf->data(); uint16_t encrypted_len; APIError aerr = this->encrypt_noise_message_(buf_start, payload_size, type, encrypted_len); if (aerr != APIError::OK) diff --git a/esphome/components/api/api_frame_helper_plaintext.cpp b/esphome/components/api/api_frame_helper_plaintext.cpp index 09ace7294a..d4e3354fa0 100644 --- a/esphome/components/api/api_frame_helper_plaintext.cpp +++ b/esphome/components/api/api_frame_helper_plaintext.cpp @@ -172,7 +172,10 @@ APIError APIPlaintextFrameHelper::try_read_frame_() { // Reserve space for body (+ null terminator so protobuf StringRef fields // can be safely null-terminated in-place after decode) - this->rx_buf_.resize(this->rx_header_parsed_len_ + RX_BUF_NULL_TERMINATOR); + if (!this->rx_buf_.resize(this->rx_header_parsed_len_ + RX_BUF_NULL_TERMINATOR)) [[unlikely]] { + state_ = State::FAILED; + return APIError::OUT_OF_MEMORY; + } if (rx_buf_len_ < rx_header_parsed_len_) { // more data to read diff --git a/tests/benchmarks/components/api/bench_list_entities.cpp b/tests/benchmarks/components/api/bench_list_entities.cpp index 02cef50d70..4c445c2bb6 100644 --- a/tests/benchmarks/components/api/bench_list_entities.cpp +++ b/tests/benchmarks/components/api/bench_list_entities.cpp @@ -49,7 +49,7 @@ static void Encode_ListEntitiesSensorResponse(benchmark::State &state) { auto msg = make_sensor_response(); APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -69,7 +69,7 @@ static void CalcAndEncode_ListEntitiesSensorResponse(benchmark::State &state) { for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); } @@ -117,7 +117,7 @@ static void Encode_ListEntitiesBinarySensorResponse(benchmark::State &state) { auto msg = make_binary_sensor_response(); APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -137,7 +137,7 @@ static void CalcAndEncode_ListEntitiesBinarySensorResponse(benchmark::State &sta for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); } @@ -202,7 +202,7 @@ static void Encode_ListEntitiesLightResponse(benchmark::State &state) { auto msg = make_light_response(); APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -222,7 +222,7 @@ static void CalcAndEncode_ListEntitiesLightResponse(benchmark::State &state) { for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); } diff --git a/tests/benchmarks/components/api/bench_log_response.cpp b/tests/benchmarks/components/api/bench_log_response.cpp index 4ef57987be..f9060af65c 100644 --- a/tests/benchmarks/components/api/bench_log_response.cpp +++ b/tests/benchmarks/components/api/bench_log_response.cpp @@ -23,7 +23,7 @@ static void Encode_LogResponse_Typical(benchmark::State &state) { msg.level = enums::LOG_LEVEL_DEBUG; msg.set_message(reinterpret_cast(kTypicalLogLine), strlen(kTypicalLogLine)); uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -42,7 +42,7 @@ static void Encode_LogResponse_Short(benchmark::State &state) { msg.level = enums::LOG_LEVEL_INFO; msg.set_message(reinterpret_cast(kShortLogLine), strlen(kShortLogLine)); uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -84,7 +84,7 @@ static void CalcAndEncode_LogResponse_Typical(benchmark::State &state) { for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); } @@ -105,7 +105,7 @@ static void CalcAndEncode_LogResponse_Typical_Fresh(benchmark::State &state) { for (int i = 0; i < kInnerIterations; i++) { APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); benchmark::DoNotOptimize(buffer.data()); diff --git a/tests/benchmarks/components/api/bench_plaintext_frame.cpp b/tests/benchmarks/components/api/bench_plaintext_frame.cpp index 74c640a093..07b479290c 100644 --- a/tests/benchmarks/components/api/bench_plaintext_frame.cpp +++ b/tests/benchmarks/components/api/bench_plaintext_frame.cpp @@ -33,7 +33,7 @@ static void PlaintextFrame_WriteSensorState(benchmark::State &state) { // Pre-init buffer to typical TCP MSS size to avoid benchmarking // heap allocation — in real use the buffer is reused across writes. APIBuffer buffer; - buffer.reserve(1460); + (void) buffer.reserve(1460); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -44,7 +44,7 @@ static void PlaintextFrame_WriteSensorState(benchmark::State &state) { msg.missing_state = false; uint32_t size = msg.calculate_size(); - buffer.resize(padding + size); + (void) buffer.resize(padding + size); ProtoWriteBuffer writer(&buffer, padding); msg.encode(writer); @@ -70,7 +70,7 @@ static void PlaintextFrame_WriteBatch5(benchmark::State &state) { // Pre-init buffer to typical TCP MSS size to avoid benchmarking // heap allocation — in real use the buffer is reused across writes. APIBuffer buffer; - buffer.reserve(1460); + (void) buffer.reserve(1460); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -85,7 +85,7 @@ static void PlaintextFrame_WriteBatch5(benchmark::State &state) { msg.missing_state = false; uint32_t size = msg.calculate_size(); - buffer.resize(offset + padding + size + footer); + (void) buffer.resize(offset + padding + size + footer); ProtoWriteBuffer writer(&buffer, offset + padding); msg.encode(writer); diff --git a/tests/benchmarks/components/api/bench_proto_decode.cpp b/tests/benchmarks/components/api/bench_proto_decode.cpp index 961c629f2a..0268e98035 100644 --- a/tests/benchmarks/components/api/bench_proto_decode.cpp +++ b/tests/benchmarks/components/api/bench_proto_decode.cpp @@ -16,7 +16,7 @@ static constexpr int kInnerIterations = 2000; template static APIBuffer encode_message(const T &msg) { APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); return buffer; diff --git a/tests/benchmarks/components/api/bench_proto_encode.cpp b/tests/benchmarks/components/api/bench_proto_encode.cpp index 1e2efcd281..e1383e8990 100644 --- a/tests/benchmarks/components/api/bench_proto_encode.cpp +++ b/tests/benchmarks/components/api/bench_proto_encode.cpp @@ -19,7 +19,7 @@ static void Encode_SensorStateResponse(benchmark::State &state) { msg.state = 23.5f; msg.missing_state = false; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -60,7 +60,7 @@ static void CalcAndEncode_SensorStateResponse(benchmark::State &state) { for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); } @@ -84,7 +84,7 @@ static void CalcAndEncode_SensorStateResponse_Fresh(benchmark::State &state) { for (int i = 0; i < kInnerIterations; i++) { APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); benchmark::DoNotOptimize(buffer.data()); @@ -103,7 +103,7 @@ static void Encode_BinarySensorStateResponse(benchmark::State &state) { msg.state = true; msg.missing_state = false; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -126,7 +126,7 @@ static void Encode_HelloResponse(benchmark::State &state) { msg.server_info = StringRef::from_lit("esphome v2026.3.0"); msg.name = StringRef::from_lit("living-room-sensor"); uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -158,7 +158,7 @@ static void Encode_LightStateResponse(benchmark::State &state) { msg.warm_white = 0.0f; msg.effect = StringRef::from_lit("rainbow"); uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -243,7 +243,7 @@ static void Encode_DeviceInfoResponse(benchmark::State &state) { auto msg = make_device_info_response(); APIBuffer buffer; uint32_t total_size = msg.calculate_size(); - buffer.resize(total_size); + (void) buffer.resize(total_size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -264,7 +264,7 @@ static void CalcAndEncode_DeviceInfoResponse(benchmark::State &state) { for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); } @@ -285,7 +285,7 @@ static void CalcAndEncode_DeviceInfoResponse_Fresh(benchmark::State &state) { for (int i = 0; i < kInnerIterations; i++) { APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); benchmark::DoNotOptimize(buffer.data()); @@ -335,7 +335,7 @@ static void Encode_BLERawAdvs12(benchmark::State &state) { auto msg = make_ble_raw_advs_12(); APIBuffer buffer; uint32_t total_size = msg.calculate_size(); - buffer.resize(total_size); + (void) buffer.resize(total_size); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -355,7 +355,7 @@ static void CalcAndEncode_BLERawAdvs12(benchmark::State &state) { for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); } @@ -372,7 +372,7 @@ static void CalcAndEncode_BLERawAdvs12_Fresh(benchmark::State &state) { for (int i = 0; i < kInnerIterations; i++) { APIBuffer buffer; uint32_t size = msg.calculate_size(); - buffer.resize(size); + (void) buffer.resize(size); ProtoWriteBuffer writer(&buffer, 0); msg.encode(writer); benchmark::DoNotOptimize(buffer.data()); diff --git a/tests/benchmarks/components/api/bench_proto_proxy.cpp b/tests/benchmarks/components/api/bench_proto_proxy.cpp index fa3191a969..05bbcc73dd 100644 --- a/tests/benchmarks/components/api/bench_proto_proxy.cpp +++ b/tests/benchmarks/components/api/bench_proto_proxy.cpp @@ -16,7 +16,7 @@ static constexpr int kInnerIterations = 2000; // Encodes `src` into `out`. Caller owns `out` and must keep it alive across // the decode loop (decoded messages may store pointers back into its bytes). template static void encode_into(APIBuffer &out, const T &src) { - out.resize(src.calculate_size()); + (void) out.resize(src.calculate_size()); ProtoWriteBuffer writer(&out, 0); src.encode(writer); } @@ -33,7 +33,7 @@ static void Encode_ZWaveProxyFrame(benchmark::State &state) { msg.data = kZWaveFrameData; msg.data_len = sizeof(kZWaveFrameData); APIBuffer buffer; - buffer.resize(msg.calculate_size()); + (void) buffer.resize(msg.calculate_size()); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -111,7 +111,7 @@ static void Encode_SerialProxyDataReceived(benchmark::State &state) { msg.instance = 0; msg.set_data(kSerialPayload, kSerialPayloadSize); APIBuffer buffer; - buffer.resize(msg.calculate_size()); + (void) buffer.resize(msg.calculate_size()); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -171,7 +171,7 @@ static void Encode_InfraredRFReceiveEvent(benchmark::State &state) { msg.key = 0xDEADBEEF; msg.timings = &get_ir_timings_100(); APIBuffer buffer; - buffer.resize(msg.calculate_size()); + (void) buffer.resize(msg.calculate_size()); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -254,7 +254,7 @@ static APIBuffer build_infrared_rf_transmit_wire() { put_varint(1); APIBuffer buf; - buf.resize(len); + (void) buf.resize(len); std::memcpy(buf.data(), bytes, len); return buf; } diff --git a/tests/benchmarks/components/api/bench_proto_varint.cpp b/tests/benchmarks/components/api/bench_proto_varint.cpp index 0b5ccc2b7d..ea7fd99aa5 100644 --- a/tests/benchmarks/components/api/bench_proto_varint.cpp +++ b/tests/benchmarks/components/api/bench_proto_varint.cpp @@ -58,7 +58,7 @@ BENCHMARK(ProtoVarInt_Parse_FiveByte); static void Encode_Varint_Small(benchmark::State &state) { APIBuffer buffer; - buffer.resize(16); + (void) buffer.resize(16); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -73,7 +73,7 @@ BENCHMARK(Encode_Varint_Small); static void Encode_Varint_Large(benchmark::State &state) { APIBuffer buffer; - buffer.resize(16); + (void) buffer.resize(16); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { @@ -88,7 +88,7 @@ BENCHMARK(Encode_Varint_Large); static void Encode_Varint_MaxUint32(benchmark::State &state) { APIBuffer buffer; - buffer.resize(16); + (void) buffer.resize(16); for (auto _ : state) { for (int i = 0; i < kInnerIterations; i++) { diff --git a/tests/components/api/test_proto_mac_varint.cpp b/tests/components/api/test_proto_mac_varint.cpp index f2a63e96f6..9ea6ce1cd9 100644 --- a/tests/components/api/test_proto_mac_varint.cpp +++ b/tests/components/api/test_proto_mac_varint.cpp @@ -54,7 +54,7 @@ static void verify_mac(uint64_t mac, size_t expected_bytes) { size_t ref_len = reference_encode(mac, ref_buf); APIBuffer api_buf; - api_buf.resize(16); + ASSERT_TRUE(api_buf.resize(16)); uint8_t *pos = api_buf.data(); #ifdef ESPHOME_DEBUG_API uint8_t *proto_debug_end_ = api_buf.data() + api_buf.size(); From b99e7f5ae281c1577a11bacf77851dd2e014c111 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 22:02:02 -0500 Subject: [PATCH 30/30] [core] Install prefetched PlatformIO packages with parallel extraction (#18775) --- esphome/core/config.py | 11 +- esphome/helpers.py | 9 + esphome/platformio/prefetch.py | 512 +++++++++-- tests/unit_tests/core/test_config.py | 28 - tests/unit_tests/test_helpers.py | 24 + tests/unit_tests/test_platformio_prefetch.py | 882 +++++++++++++++++-- 6 files changed, 1288 insertions(+), 178 deletions(-) diff --git a/esphome/core/config.py b/esphome/core/config.py index 472ca64c9a..67a7b5210e 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -55,6 +55,7 @@ from esphome.helpers import ( cpp_string_escape, fnv1a_32bit_hash, get_str_env, + get_usable_cpu_count, walk_files, ) from esphome.types import ConfigType @@ -205,16 +206,6 @@ def valid_project_name(value: str): return value -def get_usable_cpu_count() -> int: - """Return the number of CPUs that can be used for processes. - On Python 3.13+ this is the number of CPUs that can be used for processes. - On older Python versions this is the number of CPUs. - """ - return ( - os.process_cpu_count() if hasattr(os, "process_cpu_count") else os.cpu_count() - ) - - if "ESPHOME_DEFAULT_COMPILE_PROCESS_LIMIT" in os.environ: _compile_process_limit_default = min( int(os.environ["ESPHOME_DEFAULT_COMPILE_PROCESS_LIMIT"]), get_usable_cpu_count() diff --git a/esphome/helpers.py b/esphome/helpers.py index 4397111c2e..a38fcaf821 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -402,6 +402,15 @@ def sort_ip_addresses(address_list: list[str]) -> list[str]: return [socket.getnameinfo(r[4], socket.NI_NUMERICHOST)[0] for r in res] +def get_usable_cpu_count() -> int: + """Return the number of CPUs usable by this process (affinity-aware + on Python 3.13+); 1 when the count is undeterminable.""" + count = ( + os.process_cpu_count() if hasattr(os, "process_cpu_count") else os.cpu_count() + ) + return count or 1 + + def get_bool_env(var, default=False): """Read a boolean env var: the ``cv.boolean`` spellings plus ``1``/``0``; anything else falls through to ``bool(value)``.""" diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index f313c2f4d0..ef8c27c9aa 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -1,29 +1,35 @@ -"""Parallel prefetch of the packages a PlatformIO run would install. +"""Parallel prefetch and install of the packages a PlatformIO run needs. Downloads the archives concurrently into PlatformIO's own download cache -(identical ``compute_download_path`` keys) so the serial installer finds -them already cached. Runs in a subprocess like all PlatformIO execution: -loading a platform executes its code (pioarduino's penv setup rewrites -``sys.path``). A sentinel in the build dir lets warm builds skip the -spawn. Best-effort: any failure logs and PlatformIO downloads as before. -Across processes sharing a core dir every download destination is -serialized by a file lock; checksum-less URL downloads additionally -stage under a stable name and promote with an atomic rename. +(identical ``compute_download_path`` keys), then installs them through +PlatformIO's own ``_install`` with one worker per usable core, so +extraction (the serial, single-core half of a cold install) parallelizes +too and ``pio run`` finds every package already installed. Runs in a +subprocess like all PlatformIO execution: loading a platform executes +its code (pioarduino's penv setup rewrites ``sys.path``). A sentinel in +the build dir lets warm builds skip the spawn. Best-effort: any failure +logs and PlatformIO downloads and installs as before. Across processes +sharing a core dir every download destination is serialized by a file +lock; checksum-less URL downloads additionally stage under a stable +name and promote with an atomic rename. """ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor +from contextlib import suppress import hashlib import json import logging import os from pathlib import Path +from queue import SimpleQueue +import signal import subprocess import sys import threading import time -from typing import Any +from typing import Any, NamedTuple from esphome.framework_helpers import ( content_length, @@ -33,7 +39,7 @@ from esphome.framework_helpers import ( run_batch_downloads, warn_prefetch_failures, ) -from esphome.helpers import get_bool_env +from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree _LOGGER = logging.getLogger(__name__) @@ -78,6 +84,18 @@ def _sweep_stale_sidecars(download_dir: Path, expire_seconds: int) -> None: _LOGGER.debug("Could not sweep %s", download_dir, exc_info=True) +class _Resolved(NamedTuple): + """A registry spec resolved to its archive; ``cached`` skips the download.""" + + spec: Any + name: str + size: int + url: str + dl_path: Path + checksum: str + cached: bool + + # Child records a no-work run; the parent skips the next spawn while valid _SENTINEL_NAME = ".esphome_prefetch.json" _SENTINEL_SCHEMA = 1 @@ -151,24 +169,75 @@ def prefetch_platformio_packages() -> None: CORE.name, ] try: - proc = subprocess.run(cmd, env=env, check=False, timeout=_PREFETCH_TIMEOUT) - except subprocess.TimeoutExpired: - _LOGGER.warning("PlatformIO package prefetch timed out; continuing without it") - return + # Not a with-block: the lifetime spans the wait/terminate arms + proc = subprocess.Popen(cmd, env=env) # pylint: disable=consider-using-with except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught # The prefetch must never become a new way for the build to fail _LOGGER.warning("PlatformIO package prefetch skipped: %s", failure_reason(err)) _LOGGER.debug("Prefetch failure detail", exc_info=True) return - if proc.returncode == _EXIT_HANDLED: + try: + returncode = proc.wait(timeout=_PREFETCH_TIMEOUT) + except subprocess.TimeoutExpired: + _stop_child(proc) + _LOGGER.warning("PlatformIO package prefetch timed out; continuing without it") + return + except BaseException as err: + # SIGKILL (subprocess.run's choice on interrupt) could land inside + # a package-directory copy pio run would then trust; ask first + _stop_child(proc) + if isinstance(err, Exception): + # An unexpected wait() failure must degrade, not fail the build + _LOGGER.warning( + "PlatformIO package prefetch skipped: %s", failure_reason(err) + ) + return + raise + if returncode == _EXIT_HANDLED: # The child already warned with the reason; a second line is noise _LOGGER.debug("Prefetch child reported a handled failure") - elif proc.returncode != 0: + elif returncode != 0: # Exit 1 stays here: the interpreter exits 1 for import/module # failures before main() ever runs, a wiring break worth a warning - _LOGGER.warning( - "PlatformIO package prefetch skipped (exit %d)", proc.returncode - ) + _LOGGER.warning("PlatformIO package prefetch skipped (exit %d)", returncode) + + +def _stop_child(proc: subprocess.Popen) -> None: + """Stop the child without cutting an in-flight package install short. + + Wait first (a terminal interrupt already unwinds the child), then + SIGTERM for the clean unwind main() installs, then kill. On Windows + terminate() cannot reach the handler, so its arm is a plain wait. + """ + if proc.poll() is None: + _LOGGER.info("Waiting for the prefetch child to finish its current install") + try: + with suppress(subprocess.TimeoutExpired): + proc.wait(timeout=5) + return + if sys.platform != "win32": + proc.terminate() + with suppress(subprocess.TimeoutExpired): + proc.wait(timeout=30) + return + proc.kill() + proc.wait(timeout=5) + # The kill can land mid-copy; the uncertainty must be visible + _LOGGER.warning("Prefetch child killed; a package install may be incomplete") + except KeyboardInterrupt: + # Kill so an interrupted stop cannot orphan a still-writing child + # (BaseException: a further interrupt must not skip the kill), + # then re-raise so the build aborts + with suppress(BaseException): + proc.kill() + proc.wait(timeout=5) + if proc.poll() is None: + _LOGGER.warning("The prefetch child could not be confirmed stopped") + raise + except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught + # A surviving child may still be writing packages pio run trusts + _LOGGER.warning("The prefetch child could not be confirmed stopped") + _LOGGER.debug("Stop detail", exc_info=True) def _project_platform_and_config(ini: Path, env: str) -> tuple[str | None, Any]: @@ -183,25 +252,36 @@ def _project_platform_and_config(ini: Path, env: str) -> tuple[str | None, Any]: return config.get(f"env:{env}", "platform", None), config +def _sibling_manager(manager: Any) -> Any: + """A same-store manager equivalent to the shared one.""" + # Hard read: a renamed attribute must fail loudly, not silently drop + # the qualifiers wave-1 installs resolve with; is-not-None so a falsy + # PackageCompatibility still propagates + if (compatibility := manager.compatibility) is not None: + return manager.__class__(manager.package_dir, compatibility=compatibility) + return manager.__class__(manager.package_dir) + + def _registry_jobs( - manager, specs, seen: set[str] -) -> tuple[list[tuple[str, int, Any]], int]: + manager: Any, specs: list[Any], seen: set[str] +) -> tuple[list[tuple[str, int, Any]], int, list[tuple[str, Any]]]: """Resolve registry specs to ``(name, size, fetch)`` batch jobs. Mirrors PlatformIO's install path: best version, systype file, first mirror, and the same sha1(url + checksum) download-cache key. Also - returns how many resolutions errored (a clean skip is not an error). + returns how many resolutions errored (a clean skip is not an error) + and the ``(name, spec)`` pairs whose archives will be installable. """ from platformio.registry.mirror import RegistryFileMirrorIterator local = threading.local() errors: list[str] = [] - def _resolve(spec) -> tuple[str, int, str, Path, str] | object | None: + def _resolve(spec) -> _Resolved | object | None: # One manager (and registry HTTP session) per worker thread; # installed-state was already checked on the shared manager if (mgr := getattr(local, "mgr", None)) is None: - mgr = local.mgr = manager.__class__() + mgr = local.mgr = _sibling_manager(manager) try: packages = mgr.search_registry_packages(spec) if not packages: @@ -218,13 +298,13 @@ def _registry_jobs( url, checksum = next(RegistryFileMirrorIterator(pkgfile["download_url"])) checksum = checksum or pkgfile["checksum"]["sha256"] dl_path = Path(mgr.compute_download_path(url, checksum)) - if dl_path.is_file(): - return None # cached from an earlier run + cached = dl_path.is_file() # fetched by an earlier run size = pkgfile.get("size") - if not size: + if not cached and not size: _LOGGER.debug("%s has no size; PlatformIO fetches it", spec) return None # no size, no bar share - return f"{package['name']}@{version['name']}", size, url, dl_path, checksum + name = f"{package['name']}@{version['name']}" + return _Resolved(spec, name, size or 0, url, dl_path, checksum, cached) except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught # One flaky spec must not discard the rest of the batch _LOGGER.debug("Could not resolve %s", spec, exc_info=True) @@ -239,20 +319,27 @@ def _registry_jobs( unique.setdefault((s.owner, s.name, str(s.requirements)), s) pending = list(unique.values()) if not pending: - return [], 0 + return [], 0, [] # Serial resolutions (registry GET + mirror HEAD each) dominate with ThreadPoolExecutor(max_workers=min(_RESOLVE_WORKERS, len(pending))) as ex: results = list(ex.map(_resolve, pending)) jobs: list[tuple[str, int, Any]] = [] + installable: list[tuple[str, Any]] = [] for res in results: if res is None or res is _RESOLVE_FAILED: continue - name, size, url, dl_path, checksum = res - if str(dl_path) in seen: - continue # duplicate spec; two workers must not share a .part - seen.add(str(dl_path)) + installable.append((res.name, res.spec)) + if res.cached or str(res.dl_path) in seen: + continue # already fetched, or a duplicate must not share a .part + seen.add(str(res.dl_path)) jobs.append( - (name, size, _registry_fetch_job(manager, url, dl_path, checksum, size)) + ( + res.name, + res.size, + _registry_fetch_job( + manager, res.url, res.dl_path, res.checksum, res.size + ), + ) ) if failed := len(errors): # Visible once per build, naming a cause so an API break does not @@ -264,18 +351,22 @@ def _registry_jobs( len(pending), errors[0], ) - return jobs, failed + return jobs, failed, installable -def _uri_jobs(manager, specs, seen: set[str]) -> tuple[list[tuple[str, int, Any]], int]: +def _uri_jobs( + manager: Any, specs: list[Any], seen: set[str] +) -> tuple[list[tuple[str, int, Any]], int, list[tuple[str, Any]]]: """Jobs for direct-URL specs; a HEAD sizes each for the combined bar. Also returns how many HEAD probes errored (an absent length is not an - error). + error) and the ``(name, spec)`` pairs whose archives will be + installable. """ from esphome.net_retry import fetch_with_retry, http_request - candidates: list[tuple[str, str, Path]] = [] + candidates: list[tuple[str, str, Path, Any]] = [] + installable: list[tuple[str, Any]] = [] for spec in specs: url = spec.uri if not url or not url.startswith(("http://", "https://")): @@ -284,12 +375,21 @@ def _uri_jobs(manager, specs, seen: set[str]) -> tuple[list[tuple[str, int, Any] continue # bare-URL VCS spec; PlatformIO clones it if manager.get_package(spec): continue + name = spec.name or url.rsplit("/", 1)[-1] # PlatformIO downloads URL specs with no checksum dl_path = Path(manager.compute_download_path(url, "")) - if dl_path.is_file() or str(dl_path) in seen: - continue # cached, or another spec already claimed this .part + if dl_path.is_file(): + if spec.has_custom_name(): + # Only a custom name (Foo=https://...) is the destination + # dir; a URI-derived name's destination comes from the + # archive manifest, so its dedupe key could collide with + # another name and race one directory. pio run installs it. + installable.append((name, spec)) # fetched by an earlier run + continue + if str(dl_path) in seen: + continue # another spec already claimed this .part seen.add(str(dl_path)) - candidates.append((spec.name, url, dl_path)) + candidates.append((spec.name, url, dl_path, spec)) errors: list[str] = [] @@ -314,16 +414,19 @@ def _uri_jobs(manager, specs, seen: set[str]) -> tuple[list[tuple[str, int, Any] return content_length(resp) if not candidates: - return [], 0 + return [], 0, installable with ThreadPoolExecutor(max_workers=min(_RESOLVE_WORKERS, len(candidates))) as ex: - sizes = list(ex.map(_head_size, [url for _, url, _ in candidates])) + sizes = list(ex.map(_head_size, [url for _, url, _, _ in candidates])) jobs: list[tuple[str, int, Any]] = [] failed = 0 - for (name, url, dl_path), size in zip(candidates, sizes, strict=True): + for (name, url, dl_path, spec), size in zip(candidates, sizes, strict=True): if size < 0: failed += 1 elif size: jobs.append((name, size, _uri_fetch_job(manager, url, dl_path, size))) + if spec.has_custom_name(): + # See above: derived-name specs stay with pio run's installer + installable.append((name, spec)) else: # Missing or unusable Content-Length; visible under -v _LOGGER.debug("%s reports no usable length; PlatformIO fetches it", url) @@ -335,7 +438,7 @@ def _uri_jobs(manager, specs, seen: set[str]) -> tuple[list[tuple[str, int, Any] len(candidates), errors[0], ) - return jobs, failed + return jobs, failed, installable def _serialized_fetch_job( @@ -460,11 +563,233 @@ def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: return run +# (name, spec) from wave 1, (name, spec, compatibility) from dep waves +_Entry = tuple[str, Any] | tuple[str, Any, Any] + + +def _dependency_entries( + manager: Any, entries: list[_Entry], seen_names: set[str] +) -> list[_Entry]: + """Registry dependencies of the installed entries, one per new name. + + Mostly local manifest reads; the builtin probe walks installed + platforms (each may run platform code). Name-only platform libs stay + with pio run. + """ + + # Hard read: losing this filter would pre-install incompatible + # packages pio run then trusts + compatibility = manager.compatibility + # Tool managers have no builtin table; the contract test pins the name + is_builtin = getattr(manager, "is_builtin_lib", None) + deps: dict[str, Any] = {} + skipped = 0 + for name, spec, *_ in entries: + try: + deps_of = _entry_dependencies(manager, spec, compatibility, is_builtin) + except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught + # One unreadable manifest must not drop the group's whole wave + _LOGGER.debug("Skipping dependencies of %s", name, exc_info=True) + skipped += 1 + continue + for key, entry in deps_of: + if key not in seen_names: + deps.setdefault(key, entry) + if skipped: + # Visible at default verbosity: a dropped subtree silently + # degrades the wave; per-entry detail stays at debug + _LOGGER.warning( + "Could not read dependencies of %d of %d package(s)", + skipped, + len(entries), + ) + return list(deps.values()) + + +def _entry_dependencies( + manager: Any, spec: Any, compatibility: Any, is_builtin: Any +) -> list[tuple[str, _Entry]]: + from platformio.package.meta import PackageCompatibility + + out: list[tuple[str, _Entry]] = [] + if (pkg := manager.get_package(spec)) is None: + # Only successful installs are walked, so this is a real anomaly + # (stale memcache, name/dir mismatch, a pio API change); raising + # folds it into the caller's aggregate dropped-subtree warning + raise RuntimeError(f"just-installed {spec} is not resolvable") + for dep in manager.get_pkg_dependencies(pkg) or []: + if not (dep.get("owner") or dep.get("version")): + continue + if compatibility and not PackageCompatibility.from_dependency( + dep + ).is_compatible(compatibility): + continue # pio's install_dependency would skip it too + dspec = manager.dependency_to_spec(dep) + if ( + is_builtin + and not dspec.owner + and not dspec.external + and is_builtin(dspec.name) + ): + # pio's LibraryPackageManager.install_dependency skips + # builtins; a registry copy would shadow the bundled one + continue + if not (key := (dspec.name or "").lower()): + _LOGGER.debug("Dependency %r of %s has no name; left to pio run", dep, spec) + continue + if manager.get_package(dspec) is not None: + continue # already installed + # Carry the dep's compatibility so _install searches the + # registry qualified, exactly like pio's install_dependency + out.append( + (key, (dspec.name, dspec, PackageCompatibility.from_dependency(dep))) + ) + return out + + +def _clean_failed_install(mgr: Any, name: str, spec: Any) -> None: + # A post-copy failure leaves a package pio run would trust; remove it + # so pio run genuinely reinstalls it + try: + mgr.memcache_reset() + if (pkg := mgr.get_package(spec)) is not None: + # Dropping the metadata is the invariant: pio's own install + # overwrites a metadata-less dir, so a stuck tree cannot be + # trusted. The rmtree is best-effort tidiness. + (Path(pkg.path) / ".piopm").unlink(missing_ok=True) + with suppress(OSError): + rmtree(pkg.path) + else: + # Nothing was moved into place; the common failure shape + _LOGGER.debug("No on-disk install of %s to remove", name) + except Exception as cleanup_err: # noqa: BLE001 # pylint: disable=broad-exception-caught + _LOGGER.warning( + "Could not remove the failed install of %s: %s", + name, + failure_reason(cleanup_err), + ) + + +def _preinstall( + manager: Any, entries: list[_Entry], seen_names: set[str] | None = None +) -> None: + """Install downloaded packages in parallel via pio's own ``_install``. + + ``entries`` are ``_Entry`` tuples, one per destination directory. + The lock is held around each wave's pool, safe only because pio's + private ``_install`` never re-acquires it (a same-process re-lock + would hang, not fail). Waves skip dependencies; the installed + manifests feed the next wave. Any failure falls back to pio run. + """ + workers = min(get_usable_cpu_count(), len(entries)) + # One manager per worker (_install mutates instance state); built + # serially because construction rewires the shared manager logger + managers: SimpleQueue = SimpleQueue() + for _ in range(workers): + managers.put(_sibling_manager(manager)) + local = threading.local() + + def _install_one(entry) -> bool: + # Wave-1 entries are (name, spec); dependency waves add compatibility + name, spec, *rest = entry + compat = rest[0] if rest else None + if (mgr := getattr(local, "mgr", None)) is None: + # at most `workers` pool threads, one dequeue each + mgr = local.mgr = managers.get_nowait() + try: + mgr._install( # pylint: disable=protected-access # noqa: SLF001 + spec, skip_dependencies=True, compatibility=compat + ) + return True + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + _LOGGER.warning("Could not pre-install %s: %s", name, failure_reason(err)) + _LOGGER.debug("Pre-install failure detail", exc_info=True) + _clean_failed_install(mgr, name, spec) + return False + except BaseException: + # A SystemExit from a postinstall must not skip the cleanup + # and leave a torn dir pio run trusts + _clean_failed_install(mgr, name, spec) + raise + + _LOGGER.info( + "Installing %d PlatformIO package(s) with %d extraction worker(s): %s", + len(entries), + workers, + ", ".join(name for name, *_ in entries), + ) + # Postinstall scripts chdir process-globally; the cwd is restored + # after the pool. Concurrent postinstalls can still race pio's + # non-reentrant fs.cd mid-pool; that install fails, warns, and is + # redone serially by pio run. Suppress interleaved progress bars. + os.environ.setdefault("PLATFORMIO_DISABLE_PROGRESSBAR", "true") + # get_tmp_dir/get_download_dir create without exist_ok; racing workers + # would FileExistsError, so create them serially first. Concurrent + # usage.db updates can drop download bookkeeping; never a bad build. + manager.get_tmp_dir() + manager.get_download_dir() + cwd = Path.cwd() + manager.lock() + try: + with ThreadPoolExecutor(max_workers=workers) as ex: + try: + results = list(ex.map(_install_one, entries)) + except BaseException: + # Drop queued installs; in-flight ones finish so no + # package directory is left half copied + ex.shutdown(wait=True, cancel_futures=True) + raise + finally: + # Cleanup must not mask an in-flight exception or skip a step + # Each step runs even if an earlier one fails, and none may + # displace the in-flight exception (SIGTERM's SystemExit + # included) with a downgradeable one + wave_ok = True + for step, label in ( + (manager.memcache_reset, "reset the storage cache"), + (manager.unlock, "release the manager lock"), + (lambda: os.chdir(cwd), "restore the working dir"), + ): + try: + step() + except Exception: # noqa: BLE001,PERF203 # pylint: disable=broad-exception-caught + wave_ok = False + _LOGGER.warning("Could not %s", label) + _LOGGER.debug("Teardown detail", exc_info=True) + if len(entries) > 1 and not any(results): + # A systematic fault, not one bad archive; pio run installs serially + _LOGGER.warning( + "Could not pre-install any of %d PlatformIO package(s)", len(entries) + ) + + seen = seen_names if seen_names is not None else set() + # All entries join seen (failures must not be re-queued); only + # successful installs feed the dependency walk + seen.update(name.split("@", 1)[0].lower() for name, *_ in entries) + installed = [e for e, ok in zip(entries, results, strict=True) if ok] + if not wave_ok: + # A stale cache, an unknown lock state, or a lost cwd would + # poison the next wave; pio run installs the rest cleanly + _LOGGER.warning("Skipping the dependency wave") + return + # The builtin probe may construct platforms whose setup rewrites + # sys.path (see _prefetch); restore it for later imports + saved_sys_path = list(sys.path) + try: + next_entries = _dependency_entries(manager, installed, seen) + finally: + sys.path[:] = saved_sys_path + if next_entries: + # Terminates without a cap: every wave admits only never-seen + # names, so a cycle yields an empty next wave + _preinstall(manager, next_entries, seen) + + def _prefetch(build_dir: Path, env: str) -> None: from platformio.dependencies import get_core_dependencies from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager - from platformio.package.meta import PackageSpec + from platformio.package.meta import PackageCompatibility, PackageSpec from platformio.platform.factory import PlatformFactory platform_spec, config = _project_platform_and_config( @@ -504,10 +829,16 @@ def _prefetch(build_dir: Path, env: str) -> None: ) ) lib_deps = config.get(f"env:{env}", "lib_deps", []) - # pio run's storage dir for this env: installed libraries skip by - # disk lookup + # pio run's storage dir for this env, with its compatibility + # qualifiers: an unqualified library install could land a different + # owner's package pio run would then trust + qualifiers: dict[str, Any] = {"platforms": [p.name]} + if framework := config.get(f"env:{env}", "framework", None): + qualifiers["frameworks"] = framework libdeps_dir = Path(config.get("platformio", "libdeps_dir")) / env - lm = LibraryPackageManager(str(libdeps_dir)) + lm = LibraryPackageManager( + str(libdeps_dir), compatibility=PackageCompatibility(**qualifiers) + ) # A bare name is usually a framework built-in (WiFi, SPI); with no # lib builders here to tell built-in from registry, skip it. The only # cost is that an owner-less user library is not prefetched @@ -520,35 +851,71 @@ def _prefetch(build_dir: Path, env: str) -> None: seen: set[str] = set() jobs: list[tuple[str, int, Any]] = [] + groups: list[tuple[Any, list[tuple[str, Any]]]] = [] unresolved = 0 for mgr, batch in ((p.pm, specs), (lm, lib_specs)): + entries: list[tuple[str, Any]] = [] for build_jobs in (_registry_jobs, _uri_jobs): - batch_jobs, failed = build_jobs(mgr, batch, seen) + batch_jobs, failed, installable = build_jobs(mgr, batch, seen) jobs += batch_jobs unresolved += failed + entries += installable + if entries: + groups.append((mgr, entries)) sentinel = build_dir / _SENTINEL_NAME - if not jobs: - if not unresolved: - # Record the no-work run so the parent skips the next spawn. - # A failed resolution is not "no work": a registry outage must - # not be cached as warm. - dirs = [config.get("platformio", "packages_dir")] - if lib_specs: - dirs.append(str(libdeps_dir)) - sentinel.write_text( - json.dumps({**_sentinel_state(build_dir), "dirs": dirs}), - encoding="utf-8", - ) - return - sentinel.unlink(missing_ok=True) - _LOGGER.info( - "Prefetching %d PlatformIO package(s): %s", - len(jobs), - ", ".join(name for name, _, _ in jobs), - ) - # PlatformIO retries failed packages itself, without resume - warn_prefetch_failures(run_batch_downloads("Downloading PlatformIO packages", jobs)) + if jobs or groups: + # Real work invalidates any previous no-work record + sentinel.unlink(missing_ok=True) + failed_names: set[str] = set() + if jobs: + _LOGGER.info( + "Prefetching %d PlatformIO package(s): %s", + len(jobs), + ", ".join(name for name, _, _ in jobs), + ) + # PlatformIO retries failed packages itself, without resume + failures = run_batch_downloads("Downloading PlatformIO packages", jobs) + warn_prefetch_failures(failures) + failed_names = {name for name, _ in failures} + elif not groups and not unresolved: + # Record the no-work run so the parent skips the next spawn. + # A failed resolution is not "no work": a registry outage must + # not be cached as warm. + dirs = [config.get("platformio", "packages_dir")] + if lib_specs: + dirs.append(str(libdeps_dir)) + sentinel.write_text( + json.dumps({**_sentinel_state(build_dir), "dirs": dirs}), + encoding="utf-8", + ) + + for mgr, entries in groups: + # One install per destination: pio derives the directory from + # the package name, so key on the name part + to_install = { + name.split("@", 1)[0].lower(): (name, spec) + for name, spec in entries + if name not in failed_names + } + if to_install: + try: + _preinstall(mgr, list(to_install.values())) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + # Each group degrades independently; pio run installs + # whatever this one did not + _LOGGER.warning( + "Pre-install failed for the %s group: %s", + mgr.__class__.__name__, + failure_reason(err), + ) + _LOGGER.debug("Pre-install group failure detail", exc_info=True) + + +def _sigterm(_signum, _frame) -> None: + # Raised in the main thread: the pool's BaseException arm cancels + # queued installs while in-flight copies finish, then finally runs + raise SystemExit(143) def main(argv: list[str]) -> int: @@ -556,6 +923,7 @@ def main(argv: list[str]) -> int: from esphome.core import CORE from esphome.log import setup_log + signal.signal(signal.SIGTERM, _sigterm) raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") try: level = int(raw_level) if raw_level is not None else logging.INFO diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index e620f8ec7f..68b165c0d0 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -3,7 +3,6 @@ from collections.abc import Callable import os from pathlib import Path -import types from typing import Any from unittest.mock import MagicMock, Mock, patch @@ -705,33 +704,6 @@ def test_include_file_with_c_header( assert '#include "c_library.h"' in mock_raw_statement.text -def test_get_usable_cpu_count() -> None: - """Test get_usable_cpu_count returns CPU count.""" - count = config.get_usable_cpu_count() - assert isinstance(count, int) - assert count > 0 - - -def test_get_usable_cpu_count_with_process_cpu_count() -> None: - """Test get_usable_cpu_count uses process_cpu_count when available.""" - # Test with process_cpu_count (Python 3.13+) - # Create a mock os module with process_cpu_count - - mock_os = types.SimpleNamespace(process_cpu_count=lambda: 8, cpu_count=lambda: 4) - - with patch("esphome.core.config.os", mock_os): - # When process_cpu_count exists, it should be used - count = config.get_usable_cpu_count() - assert count == 8 - - # Test fallback to cpu_count when process_cpu_count not available - mock_os_no_process = types.SimpleNamespace(cpu_count=lambda: 4) - - with patch("esphome.core.config.os", mock_os_no_process): - count = config.get_usable_cpu_count() - assert count == 4 - - def test_list_target_platforms(tmp_path: Path) -> None: """Test _list_target_platforms returns available platforms.""" # Create mock components directory structure diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index 683fef22cf..53c326e0d0 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -4,6 +4,7 @@ import os from pathlib import Path import socket import stat +import types from unittest.mock import MagicMock, patch from aioesphomeapi.host_resolver import AddrInfo, IPv4Sockaddr, IPv6Sockaddr @@ -1154,3 +1155,26 @@ def test_progressbar_interrupt_keeps_finished_bar_done(monkeypatch) -> None: def test_format_duration(seconds: float, expected: str) -> None: """Test that durations are rendered as short human-readable strings.""" assert helpers.format_duration(seconds) == expected + + +def test_get_usable_cpu_count() -> None: + """Returns a positive int on the real host.""" + count = helpers.get_usable_cpu_count() + assert isinstance(count, int) + assert count > 0 + + +def test_get_usable_cpu_count_sources() -> None: + """Prefers process_cpu_count, falls back to cpu_count, degrades to 1.""" + mock_os = types.SimpleNamespace(process_cpu_count=lambda: 8, cpu_count=lambda: 4) + with patch("esphome.helpers.os", mock_os): + assert helpers.get_usable_cpu_count() == 8 + + mock_os_no_process = types.SimpleNamespace(cpu_count=lambda: 4) + with patch("esphome.helpers.os", mock_os_no_process): + assert helpers.get_usable_cpu_count() == 4 + + # An undeterminable count degrades to one worker, never zero + mock_os_unknown = types.SimpleNamespace(cpu_count=lambda: None) + with patch("esphome.helpers.os", mock_os_unknown): + assert helpers.get_usable_cpu_count() == 1 diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 22e20d0bf6..91fb78c6af 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1,14 +1,24 @@ """Tests for the parallel PlatformIO package prefetch.""" import errno +import inspect import json +import logging import os from pathlib import Path +import signal import sys +import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch from filelock import Timeout +from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageCompatibility, PackageSpec import pytest from esphome.core import CORE @@ -20,13 +30,20 @@ def _core(tmp_path: Path): CORE.reset() CORE.build_path = str(tmp_path) CORE.name = "testenv" + saved_bar = os.environ.get("PLATFORMIO_DISABLE_PROGRESSBAR") + saved_sigterm = signal.getsignal(signal.SIGTERM) pio_loggers = ("Tool Manager", "Library Manager", "Platform Manager") saved_propagate = {n: pf.logging.getLogger(n).propagate for n in pio_loggers} saved_filters = {n: list(pf.logging.getLogger(n).filters) for n in pio_loggers} # The real setup_log would swap pytest's root-handler formatter with patch("esphome.log.setup_log"): yield - # main() flips these process-wide; keep the suite hermetic + # _preinstall and main() set these process-wide; keep the suite hermetic + if saved_bar is None: + os.environ.pop("PLATFORMIO_DISABLE_PROGRESSBAR", None) + else: + os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = saved_bar + signal.signal(signal.SIGTERM, saved_sigterm) for n, flag in saved_propagate.items(): pf.logging.getLogger(n).propagate = flag pf.logging.getLogger(n).filters[:] = saved_filters[n] @@ -37,17 +54,31 @@ class _FakeSpec(SimpleNamespace): """PackageSpec stand-in for the attributes the prefetch reads.""" def __init__( - self, *, owner=None, requirements=None, external=False, **kwargs + self, + *, + uri=None, + owner=None, + requirements=None, + external=False, + custom_name=False, + **kwargs, ) -> None: super().__init__( - owner=owner, requirements=requirements, external=external, **kwargs + uri=uri, owner=owner, requirements=requirements, external=external, **kwargs ) + self._custom_name = custom_name + + def has_custom_name(self) -> bool: + return self._custom_name def _fake_manager(tmp_path: Path) -> MagicMock: m = MagicMock() - m.__class__ = lambda: m # _resolve constructs a same-class instance + # _resolve and _preinstall construct same-class instances + m.__class__ = lambda package_dir=None, **kwargs: m m.get_package.return_value = None + m.compatibility = None + m.is_builtin_lib.return_value = False m.search_registry_packages.return_value = [{"any": 1}] m.find_best_registry_version.return_value = ( {"name": "toolchain-xtensa"}, @@ -86,11 +117,12 @@ def test_registry_jobs_resolves_like_platformio(tmp_path: Path) -> None: """A registry spec resolves to a job keyed by mirror URL and checksum.""" m = _fake_manager(tmp_path) with _mirror_patch(): - jobs, failed = pf._registry_jobs( - m, [_FakeSpec(uri=None, name="toolchain-xtensa")], set() + jobs, failed, installable = pf._registry_jobs( + m, [_FakeSpec(name="toolchain-xtensa")], set() ) assert failed == 0 assert len(jobs) == 1 + assert [n for n, _ in installable] == ["toolchain-xtensa@2.0.0"] name, size, fetch = jobs[0] assert name == "toolchain-xtensa@2.0.0" assert size == 1000 @@ -114,21 +146,32 @@ def test_registry_jobs_skips(tmp_path: Path, method, attr, value) -> None: m = _fake_manager(tmp_path) setattr(getattr(m, method), attr, value) with _mirror_patch(): - assert pf._registry_jobs(m, [_FakeSpec(uri=None, name="x")], set()) == ([], 0) + assert pf._registry_jobs(m, [_FakeSpec(name="x")], set()) == ( + [], + 0, + [], + ) def test_registry_jobs_skips_cached_and_sizeless(tmp_path: Path) -> None: - """Cached or sizeless files are left to PlatformIO.""" + """A cached archive needs no download but is still installable; a + sizeless uncached one is left to PlatformIO entirely.""" m = _fake_manager(tmp_path) dl = Path(m.compute_download_path("https://mirror.example/t.tar.gz", "beef")) dl.parent.mkdir(parents=True, exist_ok=True) dl.touch() with _mirror_patch(): - assert pf._registry_jobs(m, [_FakeSpec(uri=None, name="x")], set()) == ([], 0) + jobs, failed, installable = pf._registry_jobs(m, [_FakeSpec(name="x")], set()) + assert (jobs, failed) == ([], 0) + assert [n for n, _ in installable] == ["toolchain-xtensa@2.0.0"] dl.unlink() m.find_best_registry_version.return_value[1]["files"][0]["size"] = 0 with _mirror_patch(): - assert pf._registry_jobs(m, [_FakeSpec(uri=None, name="x")], set()) == ([], 0) + assert pf._registry_jobs(m, [_FakeSpec(name="x")], set()) == ( + [], + 0, + [], + ) def test_registry_jobs_dedupes_download_paths(tmp_path: Path) -> None: @@ -136,10 +179,10 @@ def test_registry_jobs_dedupes_download_paths(tmp_path: Path) -> None: workers must never share a .part); nine specs against eight workers also exercise the thread-local manager reuse.""" m = _fake_manager(tmp_path) - specs = [_FakeSpec(uri=None, name="dup"), _FakeSpec(uri=None, name="dup")] - specs += [_FakeSpec(uri=None, name=f"n{i}") for i in range(8)] + specs = [_FakeSpec(name="dup"), _FakeSpec(name="dup")] + specs += [_FakeSpec(name=f"n{i}") for i in range(8)] with _mirror_patch(): - jobs, failed = pf._registry_jobs(m, specs, set()) + jobs, failed, _installable = pf._registry_jobs(m, specs, set()) # the fake resolves every spec to the same mirror URL and checksum assert failed == 0 assert len(jobs) == 1 @@ -151,7 +194,7 @@ def test_registry_jobs_uri_specs_excluded(tmp_path: Path) -> None: m = _fake_manager(tmp_path) assert pf._registry_jobs( m, [_FakeSpec(uri="https://x/y.zip", name="y")], set() - ) == ([], 0) + ) == ([], 0, []) m.search_registry_packages.assert_not_called() @@ -159,8 +202,8 @@ def test_registry_jobs_dedup_keeps_distinct_owners(tmp_path: Path) -> None: """platformio/x and pioarduino/x are different packages.""" m = _fake_manager(tmp_path) specs = [ - _FakeSpec(uri=None, name="framework-x", owner="platformio"), - _FakeSpec(uri=None, name="framework-x", owner="pioarduino"), + _FakeSpec(name="framework-x", owner="platformio"), + _FakeSpec(name="framework-x", owner="pioarduino"), ] with _mirror_patch(): pf._registry_jobs(m, specs, set()) @@ -174,12 +217,12 @@ def test_registry_jobs_all_failed_warns_once( m = _fake_manager(tmp_path) m.search_registry_packages.side_effect = RuntimeError("registry down") with _mirror_patch(): - jobs, failed = pf._registry_jobs( + jobs, failed, installable = pf._registry_jobs( m, - [_FakeSpec(uri=None, name="a"), _FakeSpec(uri=None, name="b")], + [_FakeSpec(name="a"), _FakeSpec(name="b")], set(), ) - assert (jobs, failed) == ([], 2) + assert (jobs, failed, installable) == ([], 2, []) # The aggregate warning names a cause so an API break does not read # as a registry outage assert "Could not resolve 2 of 2" in caplog.text @@ -533,13 +576,14 @@ def test_registry_jobs_one_bad_spec_keeps_the_rest(tmp_path: Path) -> None: [{"any": 1}], ] with _mirror_patch(): - jobs, failed = pf._registry_jobs( + jobs, failed, installable = pf._registry_jobs( m, - [_FakeSpec(uri=None, name="flaky"), _FakeSpec(uri=None, name="good")], + [_FakeSpec(name="flaky"), _FakeSpec(name="good")], set(), ) assert failed == 1 assert len(jobs) == 1 + assert len(installable) == 1 def test_uri_jobs_head_sizes_the_bar(tmp_path: Path) -> None: @@ -548,24 +592,25 @@ def test_uri_jobs_head_sizes_the_bar(tmp_path: Path) -> None: resp = MagicMock() resp.headers = {"content-length": "2222"} with patch("esphome.net_retry.http_request", return_value=resp): - jobs, failed = pf._uri_jobs( + jobs, failed, installable = pf._uri_jobs( m, [ - _FakeSpec(uri="https://x/big.zip", name="big"), + _FakeSpec(uri="https://x/big.zip", name="big", custom_name=True), _FakeSpec(uri="git+https://x/repo.git", name="repo"), _FakeSpec(uri="https://x/repo.git#v1", name="barevcs"), - _FakeSpec(uri=None, name="registry"), + _FakeSpec(name="registry"), ], set(), ) assert failed == 0 assert [(n, s) for n, s, _ in jobs] == [("big", 2222)] + assert [n for n, _ in installable] == ["big"] # a successful HEAD with no Content-Length is a clean skip resp.headers = {} with patch("esphome.net_retry.http_request", return_value=resp): assert pf._uri_jobs( m, [_FakeSpec(uri="https://x/nolen.zip", name="nolen")], set() - ) == ([], 0) + ) == ([], 0, []) def test_uri_jobs_head_failure_counts_as_unresolved( @@ -577,25 +622,25 @@ def test_uri_jobs_head_failure_counts_as_unresolved( m = _fake_manager(tmp_path) spec = [_FakeSpec(uri="https://x/a.zip", name="a")] with patch("esphome.net_retry.http_request", side_effect=OSError("no route")): - assert pf._uri_jobs(m, spec, set()) == ([], 1) + assert pf._uri_jobs(m, spec, set()) == ([], 1, []) resp = MagicMock(ok=False, status_code=503) resp.headers = {"content-length": "999"} with patch("esphome.net_retry.http_request", return_value=resp): - assert pf._uri_jobs(m, spec, set()) == ([], 1) + assert pf._uri_jobs(m, spec, set()) == ([], 1, []) # 403 is how registries rate-limit; it must not be cached as warm resp = MagicMock(ok=False, status_code=403) resp.headers = {"content-length": "999"} with patch("esphome.net_retry.http_request", return_value=resp): - assert pf._uri_jobs(m, spec, set()) == ([], 1) + assert pf._uri_jobs(m, spec, set()) == ([], 1, []) resp = MagicMock(ok=False, status_code=405) resp.headers = {"content-length": "999"} with patch("esphome.net_retry.http_request", return_value=resp): - assert pf._uri_jobs(m, spec, set()) == ([], 0) + assert pf._uri_jobs(m, spec, set()) == ([], 0, []) assert "HEAD https://x/a.zip" not in caplog.text resp = MagicMock(ok=False, status_code=404) resp.headers = {"content-length": "999"} with patch("esphome.net_retry.http_request", return_value=resp): - assert pf._uri_jobs(m, spec, set()) == ([], 0) + assert pf._uri_jobs(m, spec, set()) == ([], 0, []) assert "returned 404" not in caplog.text @@ -605,7 +650,7 @@ def test_uri_jobs_dedupes_duplicate_urls(tmp_path: Path) -> None: resp = MagicMock() resp.headers = {"content-length": "5"} with patch("esphome.net_retry.http_request", return_value=resp) as mock_head: - jobs, failed = pf._uri_jobs( + jobs, failed, _installable = pf._uri_jobs( m, [ _FakeSpec(uri="https://x/a.zip", name="a"), @@ -621,22 +666,26 @@ def test_uri_jobs_dedupes_duplicate_urls(tmp_path: Path) -> None: def test_uri_jobs_skips_installed_cached_and_seen(tmp_path: Path) -> None: m = _fake_manager(tmp_path) m.get_package.return_value = object() - spec = [_FakeSpec(uri="https://x/a.zip", name="a")] - assert pf._uri_jobs(m, spec, set()) == ([], 0) + spec = [_FakeSpec(uri="https://x/a.zip", name="a", custom_name=True)] + assert pf._uri_jobs(m, spec, set()) == ([], 0, []) m.get_package.return_value = None dl = Path(m.compute_download_path("https://x/a.zip", "")) dl.parent.mkdir(parents=True, exist_ok=True) dl.touch() - assert pf._uri_jobs(m, spec, set()) == ([], 0) + # cached: no download job, but still installable + jobs, failed, installable = pf._uri_jobs(m, spec, set()) + assert (jobs, failed) == ([], 0) + assert [n for n, _ in installable] == ["a"] dl.unlink() # a registry job already claimed this download path - assert pf._uri_jobs(m, spec, {str(dl)}) == ([], 0) + assert pf._uri_jobs(m, spec, {str(dl)}) == ([], 0, []) def test_prefetch_spawns_isolated_subprocess(tmp_path: Path) -> None: """Heal runs first, then the subprocess spawns with pio run's libdeps dir and the parent's PYTHONPATH preserved (the child is esphome).""" - proc = MagicMock(returncode=0) + proc = MagicMock() + proc.wait.return_value = 0 order = MagicMock() order.run.return_value = proc with ( @@ -644,7 +693,7 @@ def test_prefetch_spawns_isolated_subprocess(tmp_path: Path) -> None: "esphome.platformio.toolchain.heal_platformio_python_env", order.heal, ), - patch.object(pf.subprocess, "run", order.run) as mock_run, + patch.object(pf.subprocess, "Popen", order.run) as mock_run, patch.dict("os.environ", {"PYTHONPATH": "/leak"}), ): pf.prefetch_platformio_packages() @@ -664,57 +713,425 @@ def test_prefetch_spawns_isolated_subprocess(tmp_path: Path) -> None: # the same tree (tests/integration pins the source tree through it) assert kwargs["env"]["PYTHONPATH"] == "/leak" assert "ESPHOME_PREFETCH_DASHBOARD" not in kwargs["env"] - assert kwargs["timeout"] == pf._PREFETCH_TIMEOUT + proc.wait.assert_called_once_with(timeout=pf._PREFETCH_TIMEOUT) + + +def test_stop_child_windows_never_terminates() -> None: + """The Windows TerminateProcess cannot reach the SIGTERM handler, so + the graceful arm becomes a plain longer wait.""" + proc = MagicMock() + proc.wait.side_effect = [pf.subprocess.TimeoutExpired("x", 1), 0] + with patch.object(pf.sys, "platform", "win32"): + pf._stop_child(proc) + proc.terminate.assert_not_called() + proc.kill.assert_not_called() + + +def test_stop_child_surviving_child_warns(caplog: pytest.LogCaptureFixture) -> None: + """A child that outlives kill() may still be writing packages pio run + trusts; that must be visible at default verbosity.""" + timeout = pf.subprocess.TimeoutExpired("cmd", 5) + proc = MagicMock() + proc.poll.return_value = None # still running: the wait is announced + proc.wait.side_effect = [timeout, timeout, timeout] + with ( + patch.object(pf.sys, "platform", "linux"), + caplog.at_level(pf.logging.INFO), + ): + pf._stop_child(proc) + assert "Waiting for the prefetch child" in caplog.text + assert "could not be confirmed stopped" in caplog.text + + +def test_dependency_entries_isolate_a_bad_manifest(tmp_path: Path) -> None: + """One unreadable manifest skips that entry only, never the group.""" + m = _fake_manager(tmp_path) + m.get_package.side_effect = lambda spec: ( + SimpleNamespace(spec=spec) + if getattr(spec, "name", "") in ("bad", "good") + else None + ) + + def deps_for(pkg): + if pkg.spec.name == "bad": + raise RuntimeError("manifest unreadable") + return [{"owner": "o", "name": "dep", "version": "^1"}] + + m.get_pkg_dependencies.side_effect = deps_for + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) + entries = pf._dependency_entries( + m, + [ + ("bad@1", _FakeSpec(name="bad")), + ("good@1", _FakeSpec(name="good")), + ], + set(), + ) + assert [name for name, *_ in entries] == ["dep"] + + +def test_dependency_entries_skip_nameless_spec( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A dependency whose spec has no name has no destination identity; + the drop is diagnosable under -v.""" + m = _fake_manager(tmp_path) + m.get_package.side_effect = lambda spec: ( + SimpleNamespace(spec=spec) if getattr(spec, "name", "") == "top" else None + ) + m.get_pkg_dependencies.return_value = [{"owner": "o", "version": "^1"}] + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=None) + with caplog.at_level(logging.DEBUG): + assert ( + pf._dependency_entries(m, [("top@1", _FakeSpec(name="top"))], set()) == [] + ) + assert "has no name; left to pio run" in caplog.text + + +def test_dependency_entries_filter_seen_names(tmp_path: Path) -> None: + """A dependency already waved under its name is not queued again.""" + m = _fake_manager(tmp_path) + m.get_package.side_effect = lambda spec: ( + SimpleNamespace(spec=spec) if getattr(spec, "name", "") == "top" else None + ) + m.get_pkg_dependencies.return_value = [ + {"owner": "o", "name": "dep", "version": "^1"} + ] + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) + assert pf._dependency_entries(m, [("top@1", _FakeSpec(name="top"))], {"dep"}) == [] + + +def test_preinstall_cleanup_cannot_displace_the_inflight_error( + tmp_path: Path, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failing unlock or cwd restore must not replace the pool's own + exception (SIGTERM's SystemExit included) with a downgradeable one.""" + m = _fake_manager(tmp_path) + m._install.side_effect = SystemExit(143) + m.unlock.side_effect = RuntimeError("flock broke") + real_chdir = pf.os.chdir + monkeypatch.setattr(pf.os, "chdir", MagicMock(side_effect=OSError("cwd removed"))) + try: + with pytest.raises(SystemExit): + pf._preinstall(m, [("a@1", _FakeSpec(name="a"))]) + finally: + monkeypatch.setattr(pf.os, "chdir", real_chdir) + assert "Could not release the manager lock" in caplog.text + + +def test_preinstall_memcache_failure_leaves_a_trace( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failing cache reset warns and skips the dependency wave; the + wave itself still completes.""" + m = _fake_manager(tmp_path) + m.memcache_reset.side_effect = RuntimeError("cache broken") + pf._preinstall(m, [("a@1", _FakeSpec(name="a"))]) + assert "Could not reset the storage cache" in caplog.text + assert "Skipping the dependency wave" in caplog.text + m.get_pkg_dependencies.assert_not_called() + + +def test_prefetch_wait_failure_degrades(caplog: pytest.LogCaptureFixture) -> None: + """An unexpected wait() failure warns and continues; the prefetch must + never become a new way for the build to fail.""" + proc = MagicMock() + proc.wait.side_effect = [RuntimeError("wait broke"), 0] + with ( + patch("esphome.platformio.toolchain.heal_platformio_python_env"), + patch.object(pf.subprocess, "Popen", return_value=proc), + ): + pf.prefetch_platformio_packages() + assert "prefetch skipped" in caplog.text + + +def test_preinstall_stuck_lock_skips_dependency_wave( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failed unlock leaves the lock state unknown; the recursive wave + would install under a lock() that silently no-ops.""" + m = _fake_manager(tmp_path) + m.unlock.side_effect = RuntimeError("flock broke") + pf._preinstall(m, [("a@1", _FakeSpec(name="a"))]) + assert "Skipping the dependency wave" in caplog.text + m.get_pkg_dependencies.assert_not_called() + + +def test_preinstall_lost_cwd_warns_and_skips_wave( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failed cwd restore is process-global state loss: it warns and + the rest is left to pio run from a clean process.""" + m = _fake_manager(tmp_path) + with patch.object(pf.os, "chdir", side_effect=OSError("cwd gone")): + pf._preinstall(m, [("a@1", _FakeSpec(name="a"))]) + assert "Could not restore the working dir" in caplog.text + assert "Skipping the dependency wave" in caplog.text + m.get_pkg_dependencies.assert_not_called() + + +def test_stop_child_interrupted_and_still_alive_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + """An interrupt triggers a best-effort kill, warns when the child + cannot be confirmed dead, and re-raises so the build aborts.""" + proc = MagicMock() + proc.wait.side_effect = KeyboardInterrupt() + proc.poll.return_value = None + with pytest.raises(KeyboardInterrupt): + pf._stop_child(proc) + proc.kill.assert_called_once_with() + assert "could not be confirmed stopped" in caplog.text + + +def test_uri_derived_name_spec_downloads_but_never_installs(tmp_path: Path) -> None: + """A URL spec whose name is derived from the URI installs into a dir + named by the archive manifest, not the derived name; its archive is + prefetched, but the install stays with pio run.""" + m = _fake_manager(tmp_path) + resp = MagicMock(ok=True) + resp.headers = {"content-length": "4"} + with patch("esphome.net_retry.http_request", return_value=resp): + jobs, failed, installable = pf._uri_jobs( + m, [_FakeSpec(uri="https://x/v1.zip", name="v1")], set() + ) + assert failed == 0 + assert len(jobs) == 1 # still prefetched + assert installable == [] + # Cached-from-an-earlier-run archives are skipped the same way + dl = Path(m.compute_download_path("https://x/v1.zip", "")) + dl.parent.mkdir(parents=True, exist_ok=True) + dl.touch() + assert pf._uri_jobs(m, [_FakeSpec(uri="https://x/v1.zip", name="v1")], set()) == ( + [], + 0, + [], + ) + + +def test_dependency_entries_warn_when_all_reads_fail( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Every manifest read failing is a systematic fault (a pio API + break), not one bad package; the waves must not vanish silently.""" + m = _fake_manager(tmp_path) + m.get_package.side_effect = lambda spec: SimpleNamespace(spec=spec) + m.get_pkg_dependencies.side_effect = RuntimeError("api break") + assert ( + pf._dependency_entries( + m, + [ + ("a@1", _FakeSpec(name="a")), + ("b@1", _FakeSpec(name="b")), + ], + set(), + ) + == [] + ) + assert "Could not read dependencies of 2 of 2" in caplog.text + + +def _proc(wait_effect) -> MagicMock: + proc = MagicMock() + if isinstance(wait_effect, BaseException): + proc.wait.side_effect = [wait_effect, 0] + else: + proc.wait.return_value = wait_effect + return proc def test_prefetch_passes_dashboard_flag(tmp_path: Path) -> None: """The dashboard flag reaches the child so its bar still draws.""" CORE.dashboard = True + proc = MagicMock() + proc.wait.return_value = 0 with ( patch("esphome.platformio.toolchain.heal_platformio_python_env"), - patch.object( - pf.subprocess, "run", return_value=MagicMock(returncode=0) - ) as mock_run, + patch.object(pf.subprocess, "Popen", return_value=proc) as mock_popen, ): pf.prefetch_platformio_packages() - assert mock_run.call_args[1]["env"]["ESPHOME_PREFETCH_DASHBOARD"] == "1" + assert mock_popen.call_args[1]["env"]["ESPHOME_PREFETCH_DASHBOARD"] == "1" @pytest.mark.parametrize( - ("run_effect", "expected"), + ("wait_effect", "spawn_error", "expected"), [ - ( - {"side_effect": pf.subprocess.TimeoutExpired("cmd", pf._PREFETCH_TIMEOUT)}, - "prefetch timed out", - ), - ({"return_value": MagicMock(returncode=4)}, "prefetch skipped (exit 4)"), + ("timeout", None, "prefetch timed out"), + (4, None, "prefetch skipped (exit 4)"), # Exit 1 is the interpreter's own import-failure code, never quiet - ({"return_value": MagicMock(returncode=1)}, "prefetch skipped (exit 1)"), - ({"side_effect": OSError("no exec")}, "PlatformIO package prefetch skipped"), + (1, None, "prefetch skipped (exit 1)"), + (None, OSError("no exec"), "PlatformIO package prefetch skipped"), ], ) def test_prefetch_spawn_failures_warn_and_continue( - caplog: pytest.LogCaptureFixture, run_effect, expected + caplog: pytest.LogCaptureFixture, wait_effect, spawn_error, expected ) -> None: - """Timeouts, nonzero exits, and spawn failures each warn, never raise.""" + """Timeouts, nonzero exits, and spawn failures each warn, never raise; + a timed-out child is stopped gracefully. The mock is built per test: + a collection-time mock's consumable side_effect breaks reruns.""" + if spawn_error is not None: + popen_effect = {"side_effect": spawn_error} + elif wait_effect == "timeout": + popen_effect = { + "return_value": _proc( + pf.subprocess.TimeoutExpired("cmd", pf._PREFETCH_TIMEOUT) + ) + } + else: + popen_effect = {"return_value": _proc(wait_effect)} with ( patch("esphome.platformio.toolchain.heal_platformio_python_env"), - patch.object(pf.subprocess, "run", **run_effect), + patch.object(pf.subprocess, "Popen", **popen_effect), ): pf.prefetch_platformio_packages() assert expected in caplog.text +def test_stop_child_waits_terminates_then_kills() -> None: + """The stop sequence waits for a self-unwinding child first, then + SIGTERMs, and kills only a child that will not stop. The platform is + pinned: on Windows the terminate arm is deliberately skipped.""" + timeout = pf.subprocess.TimeoutExpired("cmd", 5) + with patch.object(pf.sys, "platform", "linux"): + # Child already unwinding from its own SIGINT: no signals at all + proc = MagicMock() + proc.wait.return_value = 0 + pf._stop_child(proc) + proc.terminate.assert_not_called() + # Child needs the SIGTERM unwind + proc = MagicMock() + proc.wait.side_effect = [timeout, 0] + pf._stop_child(proc) + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + # Child ignoring SIGTERM is killed with a bounded reap + proc = MagicMock() + proc.wait.side_effect = [timeout, timeout, 0] + pf._stop_child(proc) + proc.kill.assert_called_once_with() + # An interrupt mid-stop re-raises so the build aborts + proc = MagicMock() + proc.wait.side_effect = KeyboardInterrupt() + with pytest.raises(KeyboardInterrupt): + pf._stop_child(proc) + + +def test_preinstall_failure_removes_torn_destination(tmp_path: Path) -> None: + """A failed install removes whatever get_package can see so pio run + genuinely reinstalls it; a cleanup failure warns.""" + m = _fake_manager(tmp_path) + m._install.side_effect = RuntimeError("postinstall failed") + # cleanup lookup first, then the dependency-wave lookup + m.get_package.side_effect = [SimpleNamespace(path=str(tmp_path / "torn")), None] + removed: list[str] = [] + with patch.object(pf, "rmtree", side_effect=removed.append): + pf._preinstall(m, [("bad@1", _FakeSpec(name="bad"))]) + assert removed == [str(tmp_path / "torn")] + + +def test_preinstall_system_exit_still_cleans(tmp_path: Path) -> None: + """A worker SystemExit runs the torn cleanup before propagating.""" + m = _fake_manager(tmp_path) + m._install.side_effect = SystemExit(143) + m.get_package.side_effect = [SimpleNamespace(path=str(tmp_path / "torn")), None] + removed: list[str] = [] + with ( + patch.object(pf, "rmtree", side_effect=removed.append), + pytest.raises(SystemExit), + ): + pf._preinstall(m, [("bad@1", _FakeSpec(name="bad"))]) + assert removed == [str(tmp_path / "torn")] + + +def test_preinstall_stuck_tree_drops_metadata( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unremovable torn tree loses its .piopm so pio run reinstalls + it instead of trusting it forever.""" + m = _fake_manager(tmp_path) + m._install.side_effect = RuntimeError("boom") + torn = tmp_path / "torn" + torn.mkdir() + (torn / ".piopm").write_text("{}") + m.get_package.side_effect = [SimpleNamespace(path=str(torn)), None] + with patch.object(pf, "rmtree", side_effect=OSError("busy")): + pf._preinstall(m, [("bad@1", _FakeSpec(name="bad"))]) + assert not (torn / ".piopm").exists() + assert torn.exists() # tidiness is best-effort; metadata is the invariant + + +def test_preinstall_cleanup_failure_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + m = _fake_manager(tmp_path) + m._install.side_effect = RuntimeError("boom") + m.get_package.side_effect = [OSError("scan failed"), None] + pf._preinstall(m, [("bad@1", _FakeSpec(name="bad"))]) + assert "Could not remove the failed install of bad@1" in caplog.text + + +def test_dependency_entries_honor_compatibility(tmp_path: Path) -> None: + """A dependency pio's install_dependency would skip as incompatible is + not pre-installed either.""" + m = _fake_manager(tmp_path) + m.compatibility = PackageCompatibility(platforms=["espressif32"]) + # only the top-level entry is installed; the deps are not + m.get_package.side_effect = lambda spec: ( + SimpleNamespace(spec=spec) if getattr(spec, "name", "") == "top" else None + ) + m.get_pkg_dependencies.return_value = [ + {"owner": "o", "name": "espdep", "version": "^1", "platforms": ["espressif32"]}, + {"owner": "o", "name": "avrdep", "version": "^1", "platforms": ["atmelavr"]}, + ] + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) + entries = pf._dependency_entries(m, [("top@1", _FakeSpec(name="top"))], set()) + assert [name for name, *_ in entries] == ["espdep"] + + +def test_dependency_entries_skip_builtin_libs(tmp_path: Path) -> None: + """An owner-less versioned dep naming a framework builtin (the dict + manifest form of SPI/Wire) is skipped like pio's install_dependency; + a registry copy would shadow the bundled library.""" + m = _fake_manager(tmp_path) + m.is_builtin_lib.side_effect = lambda name: name == "SPI" + m.get_package.side_effect = lambda spec: ( + SimpleNamespace(spec=spec) if getattr(spec, "name", "") == "top" else None + ) + m.get_pkg_dependencies.return_value = [ + {"name": "SPI", "version": "*"}, + {"name": "realdep", "version": "^1"}, + ] + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) + entries = pf._dependency_entries(m, [("top@1", _FakeSpec(name="top"))], set()) + assert [name for name, *_ in entries] == ["realdep"] + + +def test_prefetch_interrupt_stops_child_gracefully() -> None: + """On Ctrl-C the stop sequence waits first; a child that exits on its + own is never signalled, and the interrupt re-raises.""" + proc = MagicMock() + proc.wait.side_effect = [KeyboardInterrupt(), 0] + with ( + patch("esphome.platformio.toolchain.heal_platformio_python_env"), + patch.object(pf.subprocess, "Popen", return_value=proc), + pytest.raises(KeyboardInterrupt), + ): + pf.prefetch_platformio_packages() + # the stop sequence's first wait saw the child exit on its own + proc.terminate.assert_not_called() + proc.kill.assert_not_called() + + def test_prefetch_child_handled_failure_is_quiet( caplog: pytest.LogCaptureFixture, ) -> None: """Exit _EXIT_HANDLED (3) means the child already warned with the reason; the parent adds no second warning.""" + proc = MagicMock() + proc.wait.return_value = pf._EXIT_HANDLED with ( patch("esphome.platformio.toolchain.heal_platformio_python_env"), - patch.object( - pf.subprocess, "run", return_value=MagicMock(returncode=pf._EXIT_HANDLED) - ), + patch.object(pf.subprocess, "Popen", return_value=proc), ): pf.prefetch_platformio_packages() assert "prefetch skipped" not in caplog.text @@ -767,7 +1184,7 @@ def _pio_modules(tmp_path: Path, fake_platform, fake_pm, config, lib_captures=No fake_pm.get_download_dir.return_value = str(tmp_path / "downloads") fake_pm.DOWNLOAD_CACHE_EXPIRE = 86400 * 30 - def fake_lib_manager(storage_dir): + def fake_lib_manager(storage_dir, **kwargs): if lib_captures is not None: lib_captures.append(storage_dir) return _fake_manager(tmp_path) @@ -798,7 +1215,8 @@ def _pio_modules(tmp_path: Path, fake_platform, fake_pm, config, lib_captures=No owner=kw.get("owner") or (str(a[0]).split("/")[0] if a and "/" in str(a[0]) else None), external=bool(a and "://" in str(a[0])), - ) + ), + PackageCompatibility=SimpleNamespace, ), "platformio.platform": MagicMock(), "platformio.platform.factory": SimpleNamespace( @@ -837,12 +1255,14 @@ def test_prefetch_all_cached_is_quiet_and_writes_sentinel(tmp_path: Path) -> Non modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) with ( patch.dict("sys.modules", modules), - patch.object(pf, "_registry_jobs", return_value=([], 0)), - patch.object(pf, "_uri_jobs", return_value=([], 0)), + patch.object(pf, "_registry_jobs", return_value=([], 0, [])), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), patch.object(pf, "run_batch_downloads") as mock_batch, + patch.object(pf, "_preinstall") as mock_install, ): pf._prefetch(tmp_path, "testenv") mock_batch.assert_not_called() + mock_install.assert_not_called() assert pf._prefetch_is_warm(tmp_path) @@ -856,8 +1276,8 @@ def test_prefetch_failed_resolution_is_not_cached_as_warm(tmp_path: Path) -> Non modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) with ( patch.dict("sys.modules", modules), - patch.object(pf, "_registry_jobs", return_value=([], 1)), - patch.object(pf, "_uri_jobs", return_value=([], 0)), + patch.object(pf, "_registry_jobs", return_value=([], 1, [])), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), patch.object(pf, "run_batch_downloads") as mock_batch, ): pf._prefetch(tmp_path, "testenv") @@ -890,10 +1310,10 @@ def test_prefetch_warm_sentinel_skips_spawn(tmp_path: Path) -> None: _write_valid_sentinel(tmp_path, [str(pkg_dir)]) with ( patch("esphome.platformio.toolchain.heal_platformio_python_env"), - patch.object(pf.subprocess, "run") as mock_run, + patch.object(pf.subprocess, "Popen") as mock_popen, ): pf.prefetch_platformio_packages() - mock_run.assert_not_called() + mock_popen.assert_not_called() def test_prefetch_end_to_end_wiring( @@ -922,6 +1342,7 @@ def test_prefetch_end_to_end_wiring( tmp_path, { "platform": "fake/platform@1.0", + "framework": "arduino", # the bare built-in name and the interpolation are skipped; # only the owner-qualified library resolves "lib_deps": ["esphome/noise-c@1.0", "WiFi", "${common.lib_deps}"], @@ -934,17 +1355,22 @@ def test_prefetch_end_to_end_wiring( def fake_registry_jobs(manager, specs, seen): captured.setdefault("spec_batches", []).append([s.name for s in specs]) - return [("toolchain-x@1", 10, lambda t: None)], 0 + return ( + [("toolchain-x@1", 10, lambda t: None)], + 0, + [("toolchain-x@1", specs[0])], + ) with ( patch.dict("sys.modules", modules), patch.object(pf, "_registry_jobs", side_effect=fake_registry_jobs), - patch.object(pf, "_uri_jobs", return_value=([], 0)), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), patch.object( pf, "run_batch_downloads", return_value=[("toolchain-x@1", OSError("down"))], ) as mock_batch, + patch.object(pf, "_preinstall") as mock_install, ): pf._prefetch(tmp_path, "testenv") fake_pm.install.assert_called_once_with("fake/platform@1.0", skip_dependencies=True) @@ -957,6 +1383,279 @@ def test_prefetch_end_to_end_wiring( assert lib_dirs == [str(Path(tmp_path / "libdeps") / "testenv")] mock_batch.assert_called_once() assert "Could not prefetch toolchain-x@1" in caplog.text + # every installable failed its download; nothing to pre-install + mock_install.assert_not_called() + + +def test_prefetch_installs_cached_archives_without_downloads( + tmp_path: Path, +) -> None: + """Archives already in the download cache still pre-install (in + parallel) even when there is nothing to download, and no sentinel is + written until everything is installed.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + fake_platform = MagicMock() + fake_platform.packages = {} + config = _fake_config(tmp_path, {"platform": "fake/p@1"}) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + spec = _FakeSpec(name="cachedpkg") + with ( + patch.dict("sys.modules", modules), + patch.object( + pf, + "_registry_jobs", + side_effect=[([], 0, [("cachedpkg@1", spec)]), ([], 0, [])], + ), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), + patch.object(pf, "run_batch_downloads") as mock_batch, + patch.object(pf, "_preinstall") as mock_install, + ): + pf._prefetch(tmp_path, "testenv") + mock_batch.assert_not_called() + assert mock_install.call_count == 1 + assert mock_install.call_args[0][1] == [("cachedpkg@1", spec)] + assert not (tmp_path / pf._SENTINEL_NAME).exists() + + +def test_preinstall_extracts_in_parallel_under_one_lock(tmp_path: Path) -> None: + """The manager lock wraps the whole batch; per-thread managers share + its package dir; one failing install leaves the rest alone.""" + m = _fake_manager(tmp_path) + installed: list[str] = [] + + def fake_install(spec, skip_dependencies, compatibility=None): + # Dependencies must be skipped: a shared dep extracted from two + # threads would race one destination dir + assert skip_dependencies is True + if spec.name == "bad": + raise RuntimeError("corrupt archive") + installed.append(spec.name) + + m._install.side_effect = fake_install + entries = [ + ("a@1", _FakeSpec(name="a")), + ("bad@1", _FakeSpec(name="bad")), + ("b@1", _FakeSpec(name="b")), + ] + pf._preinstall(m, entries) + assert sorted(installed) == ["a", "b"] + m.lock.assert_called_once_with() + m.unlock.assert_called_once_with() + assert m.memcache_reset.call_count >= 1 + + +def test_preinstall_all_failed_warns_once( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Every install failing is a systemic fault, not archive noise.""" + m = _fake_manager(tmp_path) + m._install.side_effect = AttributeError("_install went away") + pf._preinstall( + m, + [ + ("a@1", _FakeSpec(name="a")), + ("b@1", _FakeSpec(name="b")), + ], + ) + assert "Could not pre-install a@1" in caplog.text + assert "Could not pre-install any of 2" in caplog.text + + +def test_preinstall_dedupes_names_across_entries(tmp_path: Path) -> None: + """Two entries with one name install once (one destination dir).""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + fake_platform = MagicMock() + fake_platform.packages = {} + config = _fake_config(tmp_path, {"platform": "fake/p@1"}) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + s1 = _FakeSpec(name="dup") + s2 = _FakeSpec(name="dup") + with ( + patch.dict("sys.modules", modules), + patch.object( + pf, + "_registry_jobs", + side_effect=[([], 0, [("pkg@1", s1), ("pkg@1", s2)]), ([], 0, [])], + ), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), + patch.object(pf, "_preinstall") as mock_install, + ): + pf._prefetch(tmp_path, "testenv") + mock_install.assert_called_once() + (entry,) = mock_install.call_args[0][1] + assert entry[0] == "pkg@1" + assert entry[1] is s2 # the dict comprehension keeps the last duplicate + + +def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: + """Dependencies of installed packages install in a follow-up wave, + deduped by name; name-only platform libs stay with pio run.""" + m = _fake_manager(tmp_path) + installed: list[str] = [] + m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( + installed.append(spec.name if hasattr(spec, "name") else str(spec)) + ) + pkg = SimpleNamespace(spec="noise-c") + m.get_package.side_effect = lambda spec: ( + pkg if getattr(spec, "name", None) == "noise-c" else None + ) + m.get_pkg_dependencies.return_value = [ + {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, + {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, + {"name": "SPI"}, + ] + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) + pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out + # The dep wave carries its compatibility so _install searches qualified + dep_call = m._install.call_args_list[-1] + assert dep_call.kwargs["compatibility"] is not None + + +def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: + """A dependency whose name matches an already-waved entry is not + reinstalled.""" + m = _fake_manager(tmp_path) + m.get_package.side_effect = lambda spec: SimpleNamespace(spec=spec) + m.get_pkg_dependencies.return_value = [ + {"owner": "esphome", "name": "noise-c", "version": "^0.1"}, + ] + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) + installed: list[str] = [] + m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( + installed.append(getattr(spec, "name", str(spec))) + ) + pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + assert installed == ["noise-c"] + + +def test_preinstall_uses_distinct_managers_in_parallel(tmp_path: Path) -> None: + """Each worker thread gets its own pre-built manager and installs + genuinely overlap (the barrier deadlocks a serial pool). The worker + count is pinned so a 1-CPU host cannot serialize the pool.""" + barrier = threading.Barrier(2, timeout=5) + used: set = set() + + class _WaveManager: + package_dir = str(tmp_path) + compatibility = None + + def __init__(self, package_dir, **kwargs) -> None: + assert package_dir == str(tmp_path) + + def lock(self) -> None: + pass + + def unlock(self) -> None: + pass + + def memcache_reset(self) -> None: + pass + + def get_tmp_dir(self) -> str: + return str(tmp_path) + + def get_download_dir(self) -> str: + return str(tmp_path) + + def get_package(self, spec): + return None + + def get_pkg_dependencies(self, pkg): + return None + + def _install(self, spec, skip_dependencies, compatibility=None) -> None: + used.add(id(self)) + barrier.wait() + + seed = _WaveManager(str(tmp_path)) + with patch.object(pf, "get_usable_cpu_count", return_value=2): + pf._preinstall( + seed, + [ + ("a@1", _FakeSpec(name="a")), + ("b@1", _FakeSpec(name="b")), + ], + ) + assert len(used) == 2 + assert id(seed) not in used + + +def test_sibling_manager_and_sigterm() -> None: + """Sibling managers inherit compatibility; SIGTERM raises SystemExit.""" + calls = [] + m = MagicMock(package_dir="p", compatibility="qual") + m.__class__ = lambda package_dir, **kw: calls.append((package_dir, kw)) + pf._sibling_manager(m) + m.compatibility = None + pf._sibling_manager(m) + assert calls == [("p", {"compatibility": "qual"}), ("p", {})] + with pytest.raises(SystemExit): + pf._sigterm(15, None) + + +def test_dependency_entries_skip_installed(tmp_path: Path) -> None: + """A dependency a previous build installed stays off the destructive + failure path.""" + m = _fake_manager(tmp_path) + m.get_package.side_effect = lambda spec: SimpleNamespace(spec=spec) + m.get_pkg_dependencies.return_value = [ + {"owner": "o", "name": "already", "version": "^1"}, + ] + m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) + assert pf._dependency_entries(m, [("top@1", _FakeSpec(name="top"))], set()) == [] + + +def test_group_failure_does_not_skip_other_groups(tmp_path: Path) -> None: + """One group's pre-install failure degrades that group only.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + fake_platform = MagicMock() + fake_platform.packages = {} + config = _fake_config(tmp_path, {"platform": "fake/p@1"}) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + s1 = _FakeSpec(name="toolpkg") + s2 = _FakeSpec(name="libpkg") + with ( + patch.dict("sys.modules", modules), + patch.object( + pf, + "_registry_jobs", + side_effect=[ + ([], 0, [("toolpkg@1", s1)]), + ([], 0, [("libpkg@1", s2)]), + ], + ), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), + patch.object( + pf, "_preinstall", side_effect=[RuntimeError("group down"), None] + ) as mock_install, + ): + pf._prefetch(tmp_path, "testenv") + assert mock_install.call_count == 2 + + +def test_preinstall_unlocks_even_when_pool_fails(tmp_path: Path) -> None: + """A failure inside the pool cancels queued installs and releases the + lock; a failing executor construction still releases it.""" + m = _fake_manager(tmp_path) + boom = MagicMock() + boom.__enter__.return_value = boom + boom.map.side_effect = RuntimeError("no threads") + with ( + patch.object(pf, "ThreadPoolExecutor", return_value=boom), + pytest.raises(RuntimeError), + ): + pf._preinstall(m, [("a@1", _FakeSpec(name="a"))]) + m.unlock.assert_called_once_with() + assert boom.shutdown.call_args_list[0][1].get("cancel_futures") is True + m.reset_mock() + # A failing executor construction still releases the lock + with ( + patch.object(pf, "ThreadPoolExecutor", side_effect=RuntimeError("no")), + pytest.raises(RuntimeError), + ): + pf._preinstall(m, [("a@1", _FakeSpec(name="a"))]) + m.unlock.assert_called_once_with() def test_prefetch_skips_duplicate_tool_scons(tmp_path: Path) -> None: @@ -976,10 +1675,57 @@ def test_prefetch_skips_duplicate_tool_scons(tmp_path: Path) -> None: pf, "_registry_jobs", side_effect=lambda mgr, specs, seen: ( - batches.append([s.name for s in specs]) or ([], 0) + batches.append([s.name for s in specs]) or ([], 0, []) ), ), - patch.object(pf, "_uri_jobs", return_value=([], 0)), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), ): pf._prefetch(tmp_path, "testenv") assert batches[0] == ["tool-scons"] + + +def test_platformio_private_api_contract() -> None: + """The pinned PlatformIO still exposes what the pre-install drives. + + Also load-bearing but unpinnable by introspection: pio's private + _install must never re-acquire the manager's inter-process lock + (locking lives in the public install()); a re-lock would hang the + child for the full prefetch timeout, so re-check it on any bump. + + Everything else in this module mocks the managers, so this is the one + test that fails loudly when a requirements bump changes the private + surface instead of silently degrading the prefetch to a no-op. + """ + params = inspect.signature(PackageManagerInstallMixin._install).parameters + assert "spec" in params + assert "skip_dependencies" in params + assert "compatibility" in params + for cls in (ToolPackageManager, LibraryPackageManager, PlatformPackageManager): + assert "package_dir" in inspect.signature(cls.__init__).parameters + for name in ( + "lock", + "unlock", + "memcache_reset", + "get_package", + "compute_download_path", + "get_pkg_dependencies", + "dependency_to_spec", + ): + assert callable(getattr(BasePackageManager, name)) + # The dependency wave mirrors install_dependency's builtin skip + assert callable(LibraryPackageManager.is_builtin_lib) + # The pre-install passes these positionally / by keyword + assert "compatibility" in inspect.signature(BasePackageManager.__init__).parameters + lib_params = inspect.signature(LibraryPackageManager.__init__).parameters + # Capability, not implementation: an explicit compatibility= parameter + # would serve the call site just as well as **kwargs forwarding + assert "compatibility" in lib_params or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in lib_params.values() + ) + assert callable(PackageCompatibility.from_dependency) + assert callable(PackageCompatibility.is_compatible) + # Every URL spec derives a name from the URI; only a custom name + # (Foo=https://...) is also the destination dir the wave installs into + derived = PackageSpec("https://x/y/archive/master.zip") + assert derived.name and not derived.has_custom_name() + assert PackageSpec("Foo=https://x/y/archive/master.zip").has_custom_name()