From 51ea97deffbac5c2d1379ce20424c74d6b2259b5 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:38:55 +1200 Subject: [PATCH 01/53] [esp32_ble] Reference count BLE advertising (#18943) --- esphome/components/esp32_ble/ble.cpp | 35 +++++++++++++++---- esphome/components/esp32_ble/ble.h | 13 +++++++ .../esp32_ble_beacon/esp32_ble_beacon.cpp | 2 ++ .../components/esp32_ble_server/__init__.py | 12 +++++++ .../esp32_ble_server/ble_server.cpp | 21 +++++++++-- .../components/esp32_ble_server/ble_server.h | 11 ++++++ .../esp32_improv/esp32_improv_component.cpp | 20 ++++++++++- .../esp32_improv/esp32_improv_component.h | 3 ++ .../esp32_ble_server/config/improv_only.yaml | 13 +++++++ .../config/manufacturer_data_only.yaml | 9 +++++ .../esp32_ble_server/config/own_service.yaml | 14 ++++++++ .../esp32_ble_server/test_esp32_ble_server.py | 28 +++++++++++++++ 12 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/esp32_ble_server/config/improv_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/own_service.yaml diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 6e6fb0e30d..fc95760cf8 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -100,21 +100,38 @@ void ESP32BLE::disable() { #ifdef USE_ESP32_BLE_ADVERTISING void ESP32BLE::advertising_start() { this->advertising_init_(); - if (!this->is_active()) + this->advertising_ref_count_++; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_stop() { + if (this->advertising_ref_count_ == 0) return; - this->advertising_->start(); + this->advertising_ref_count_--; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_refresh() { + if (this->advertising_ == nullptr || !this->is_active()) + return; + // Advertise while any component still needs it, otherwise stop + if (this->advertising_ref_count_ == 0) { + this->advertising_->stop(); + } else { + this->advertising_->start(); + } } void ESP32BLE::advertising_set_service_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_service_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_manufacturer_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_manufacturer_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_service_data_and_name(std::span data, bool include_name) { @@ -136,7 +153,7 @@ void ESP32BLE::advertising_set_service_data_and_name(std::span da this->advertising_->set_service_data(data); } - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_register_raw_advertisement_callback(std::function &&callback) { @@ -147,13 +164,13 @@ void ESP32BLE::advertising_register_raw_advertisement_callback(std::functionadvertising_init_(); this->advertising_->add_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_remove_service_uuid(ESPBTUUID uuid) { this->advertising_init_(); this->advertising_->remove_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } #endif @@ -575,6 +592,10 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { } this->state_ = BLE_COMPONENT_STATE_ACTIVE; +#ifdef USE_ESP32_BLE_ADVERTISING + // Requests made before the stack was up (or before it was re-enabled) take effect now + this->advertising_refresh(); +#endif } } diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 2a355a6c8b..7d2d0438a4 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -114,7 +114,17 @@ class ESP32BLE final : public Component { void set_name(const char *name) { this->name_ = name; } #ifdef USE_ESP32_BLE_ADVERTISING + /** Request advertising on behalf of a component. + * + * Requests are reference counted: advertising runs until every component that called + * advertising_start() has released it again with advertising_stop(). Each component must + * pair its calls, so nothing advertises until something actually asks for it. + */ void advertising_start(); + /// Release a request made with advertising_start(); advertising stops at the last release. + void advertising_stop(); + /// Apply the current payload and request count: advertise while requested, otherwise stop. + void advertising_refresh(); void advertising_set_service_data(const std::vector &data); void advertising_set_manufacturer_data(const std::vector &data); void advertising_set_appearance(uint16_t appearance) { this->appearance_ = appearance; } @@ -226,6 +236,9 @@ class ESP32BLE final : public Component { // 1-byte aligned members (grouped together to minimize padding) BLEComponentState state_{BLE_COMPONENT_STATE_OFF}; // 1 byte (uint8_t enum) bool enable_on_boot_{}; // 1 byte +#ifdef USE_ESP32_BLE_ADVERTISING + uint8_t advertising_ref_count_{0}; // 1 byte, number of components requesting advertising +#endif #ifdef ESPHOME_ESP32_BLE_EXTENDED_AUTH_PARAMS optional auth_req_mode_; diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp index 9f1723430b..ab728f9f6f 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp @@ -67,6 +67,8 @@ void ESP32BLEBeacon::setup() { this->on_advertise_(); } }); + // A beacon always needs the device to advertise, and never releases the request + global_ble->advertising_start(); } void ESP32BLEBeacon::on_advertise_() { diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 855a3be29b..d8095cd702 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -596,6 +596,18 @@ async def to_code(config): cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) cg.add(var.set_max_clients(config[CONF_MAX_CLIENTS])) + # Only advertise for the server itself when the configuration gives clients something to + # find. A server that is auto-loaded purely to host a runtime service (esp32_improv) stays + # silent until that service asks for advertising. + cg.add( + var.set_advertising_required( + CONF_MANUFACTURER_DATA in config + or any( + not uuid_is(service_config[CONF_UUID], DEVICE_INFORMATION_SERVICE_UUID) + for service_config in config[CONF_SERVICES] + ) + ) + ) if CONF_MANUFACTURER_DATA in config: cg.add(var.set_manufacturer_data(config[CONF_MANUFACTURER_DATA])) for service_config in config[CONF_SERVICES]: diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp index 2dea1666bb..45679b9b98 100644 --- a/esphome/components/esp32_ble_server/ble_server.cpp +++ b/esphome/components/esp32_ble_server/ble_server.cpp @@ -81,6 +81,7 @@ void BLEServer::loop() { if (this->device_information_service_->is_running()) { this->state_ = RUNNING; this->restart_advertising_(); + this->request_advertising_(); ESP_LOGD(TAG, "BLE server setup successfully"); } else if (this->device_information_service_->is_created()) { this->device_information_service_->start(); @@ -98,6 +99,20 @@ void BLEServer::restart_advertising_() { } } +void BLEServer::request_advertising_() { + if (!this->advertising_required_ || this->advertising_requested_) + return; + this->advertising_requested_ = true; + this->parent_->advertising_start(); +} + +void BLEServer::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + this->parent_->advertising_stop(); +} + BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles) { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE char uuid_buf[esp32_ble::UUID_STR_LEN]; @@ -170,7 +185,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga this->add_client_(param->connect.conn_id); // Resume advertising so additional clients can discover and connect if (this->client_count_ < this->max_clients_) { - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); } this->dispatch_callbacks_(CallbackType::ON_CONNECT, param->connect.conn_id); break; @@ -178,7 +193,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga case ESP_GATTS_DISCONNECT_EVT: { ESP_LOGD(TAG, "BLE Client disconnected"); this->remove_client_(param->disconnect.conn_id); - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); this->dispatch_callbacks_(CallbackType::ON_DISCONNECT, param->disconnect.conn_id); break; } @@ -226,6 +241,8 @@ void BLEServer::remove_client_(uint16_t conn_id) { } void BLEServer::ble_before_disabled_event_handler() { + // Advertising is re-requested once the server is running again after BLE is re-enabled + this->release_advertising_(); // Delete all clients this->client_count_ = 0; // Delete all services diff --git a/esphome/components/esp32_ble_server/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h index fdd92812cd..7869c73cc5 100644 --- a/esphome/components/esp32_ble_server/ble_server.h +++ b/esphome/components/esp32_ble_server/ble_server.h @@ -38,6 +38,13 @@ class BLEServer final : public Component, public Parented { this->restart_advertising_(); } + /** Whether this server needs the device to advertise so clients can find and connect to it. + * + * False for a server that only hosts services created at runtime (e.g. esp32_improv), which + * request advertising themselves for as long as they need it. + */ + void set_advertising_required(bool required) { this->advertising_required_ = required; } + void set_max_clients(uint8_t max_clients) { this->max_clients_ = max_clients; } uint8_t get_max_clients() const { return this->max_clients_; } @@ -82,6 +89,8 @@ class BLEServer final : public Component, public Parented { }; void restart_advertising_(); + void request_advertising_(); + void release_advertising_(); int8_t find_client_index_(uint16_t conn_id) const; void add_client_(uint16_t conn_id); @@ -93,6 +102,8 @@ class BLEServer final : public Component, public Parented { std::vector manufacturer_data_{}; esp_gatt_if_t gatts_if_{0}; bool registered_{false}; + bool advertising_required_{true}; + bool advertising_requested_{false}; uint16_t clients_[USE_ESP32_BLE_MAX_CONNECTIONS]{}; uint8_t client_count_{0}; diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 4756fba637..9ec6eb7bab 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -112,6 +112,7 @@ void ESP32ImprovComponent::loop() { this->state_callback_.call(this->state_, this->error_state_); #endif } + this->release_advertising_(); this->incoming_data_.clear(); return; } @@ -143,8 +144,9 @@ void ESP32ImprovComponent::loop() { ESP_LOGV(TAG, "Starting with device name advertising"); this->advertising_device_name_ = true; this->last_name_adv_time_ = App.get_loop_component_start_time(); + // Set the payload before requesting, so advertising starts exactly once esp32_ble::global_ble->advertising_set_service_data_and_name(std::span{}, true); - esp32_ble::global_ble->advertising_start(); + this->request_advertising_(); // Set initial state based on whether we have an authorizer this->set_state_(this->get_initial_state_(), false); @@ -326,6 +328,8 @@ void ESP32ImprovComponent::stop() { this->set_timeout("end-service", STOP_ADVERTISING_DELAY, [this] { if (this->state_ == improv::STATE_STOPPED || this->service_ == nullptr) return; + // Release first so removing the service UUID does not restart advertising on the way out + this->release_advertising_(); this->service_->stop(); this->set_state_(improv::STATE_STOPPED); }); @@ -520,6 +524,20 @@ void ESP32ImprovComponent::update_advertising_type_() { } } +void ESP32ImprovComponent::request_advertising_() { + if (this->advertising_requested_) + return; + this->advertising_requested_ = true; + esp32_ble::global_ble->advertising_start(); +} + +void ESP32ImprovComponent::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + esp32_ble::global_ble->advertising_stop(); +} + improv::State ESP32ImprovComponent::get_initial_state_() const { #ifdef USE_BINARY_SENSOR // If we have an authorizer, start in awaiting authorization state diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index 414948c977..a40d60552a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -104,8 +104,11 @@ class ESP32ImprovComponent final : public Component, public improv_base::ImprovB bool status_indicator_state_{false}; uint32_t last_name_adv_time_{0}; bool advertising_device_name_{false}; + bool advertising_requested_{false}; void set_status_indicator_state_(bool state); void update_advertising_type_(); + void request_advertising_(); + void release_advertising_(); void set_state_(improv::State state, bool update_advertising = true); void set_error_(improv::Error error); diff --git a/tests/component_tests/esp32_ble_server/config/improv_only.yaml b/tests/component_tests/esp32_ble_server/config/improv_only.yaml new file mode 100644 index 0000000000..8a5c3ba638 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/improv_only.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32 + +wifi: + ssid: MySSID + password: password1 + +# esp32_ble_server is only auto-loaded here, so it has no services of its own. +esp32_improv: + authorizer: none diff --git a/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml new file mode 100644 index 0000000000..b7bdae4af7 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + manufacturer_data: [0x72, 0x04, 0x00, 0x23] diff --git a/tests/component_tests/esp32_ble_server/config/own_service.yaml b/tests/component_tests/esp32_ble_server/config/own_service.yaml new file mode 100644 index 0000000000..c7ef0287b0 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/own_service.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + services: + - uuid: 2a24b789-7aab-4535-af3e-ee76a35cc12d + characteristics: + - uuid: cad48e28-7fbe-41cf-bae9-d77a6c233423 + read: true + value: [1, 2, 3, 4] diff --git a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py index 88307d0dcf..4b7ab79a81 100644 --- a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py +++ b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py @@ -1,5 +1,10 @@ """Tests for esp32_ble_server configuration helpers.""" +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + import pytest from esphome.components.esp32_ble_server import ( @@ -45,3 +50,26 @@ def test_uuid_is_matches_descriptor_short_strings(uuid16) -> None: assert uuid_is(uuid16, uuid16) assert uuid_is(f"{uuid16:04X}", uuid16) assert uuid_is(f"{uuid16:08X}", uuid16) + + +@pytest.mark.parametrize( + ("config_file", "required"), + [ + # Auto-loaded by esp32_improv only: nothing to find until Improv asks for it + ("improv_only.yaml", False), + # The configuration defines a service clients are meant to connect to + ("own_service.yaml", True), + # Manufacturer data is only useful if it is actually broadcast + ("manufacturer_data_only.yaml", True), + ], +) +def test_advertising_required( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + required: bool, +) -> None: + """The server only requests advertising when the configuration needs it.""" + main_cpp = generate_main(component_config_path(config_file)) + + assert f"set_advertising_required({str(required).lower()})" in main_cpp From ce87bf9b17f5f93171e62a1f6ecbc1ade6ce1132 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:05:26 -0400 Subject: [PATCH 02/53] Bump platformdirs from 4.11.5 to 4.11.7 (#18976) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a510a2c60..cd3f7446f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.5 # native esp-idf toolchain global cache dir +platformdirs==4.11.7 # native esp-idf toolchain global cache dir ninja==1.13.2 # native esp8266 arduino toolchain build driver filelock==3.32.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From d1829c495d2c982eb2f2845406ccfe5b74bd2f64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:05:36 -0400 Subject: [PATCH 03/53] Bump prek from 0.5.0 to 0.5.1 (#18977) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index b1309ec63b..897445a4cb 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.8 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.5.0 # also change in .github/workflows/ci.yml when updating +prek==0.5.1 # also change in .github/workflows/ci.yml when updating # Unit tests pytest==9.1.1 From b66822d9bd0f741b8d40264e019264d9910fc377 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:21:47 +1000 Subject: [PATCH 04/53] [ai] Advice to agents to limit verbiage (#18980) --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index e932c50f32..15b92c4deb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -553,6 +553,7 @@ file does, and it is the authority when they disagree. The most useful starting 4. **Lint:** Run `prek` to ensure code is compliant. 5. **Commit:** Commit your changes. There is no strict format for commit messages. 6. **Pull Request:** Submit a PR against the `dev` branch. The Pull Request title must start with a `[tag]` prefix. For component work, use the component name (e.g., `[display] Fix bug`, `[abc123] Add new component`); for changes to shared/core code that isn't tied to a single component, use `[core]` (e.g., `[core] Add validator`). Update documentation, examples, and add `CODEOWNERS` entries as needed. Pull requests should always be made using the `.github/PULL_REQUEST_TEMPLATE.md` template - fill out all sections completely without removing any parts of the template. + 7. **Comments:** When commenting on GitHub PRs or issues, don't tag contributors, especially bots. Avoid referring to list items (e.g. from reviews) with the form #nn - this will be interpreted by GitHub as a reference to issue or PR nn. Keep comments short and exclude irrelevant details, backstories, restatement of previous comments and anything that is already obvious to the reader. * **Documentation Contributions:** * Documentation is hosted in the separate `esphome/esphome.io` repository. From 13dbbcaa32e94423ff5bf9fe62b6f56cb073a6c5 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sat, 5 Sep 2026 06:00:25 -0500 Subject: [PATCH 05/53] [usb_uart] Keep the comm interface number valid when its claim fails (#18968) --- esphome/components/usb_uart/usb_uart.cpp | 12 +++++++----- esphome/components/usb_uart/usb_uart.h | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/esphome/components/usb_uart/usb_uart.cpp b/esphome/components/usb_uart/usb_uart.cpp index cf66e4c369..60b7fe4e9c 100644 --- a/esphome/components/usb_uart/usb_uart.cpp +++ b/esphome/components/usb_uart/usb_uart.cpp @@ -434,11 +434,12 @@ void USBUartTypeCdcAcm::on_connected() { auto err_comm = usb_host_interface_claim(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number, 0); if (err_comm != ESP_OK) { + // Continue anyway: the interface number stays valid for CDC request addressing ESP_LOGW(TAG, "Could not claim comm interface %d: %s", channel->cdc_dev_.interrupt_interface_number, esp_err_to_name(err_comm)); - channel->cdc_dev_.interrupt_interface_number = 0xFF; // Mark as unavailable, but continue anyway } else { ESP_LOGD(TAG, "Claimed comm interface %d", channel->cdc_dev_.interrupt_interface_number); + channel->cdc_dev_.interrupt_interface_claimed = true; } } auto err = @@ -465,14 +466,15 @@ void USBUartTypeCdcAcm::on_disconnected() { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); } - if (channel->cdc_dev_.notify_ep != nullptr) { + // Only tear down the notify pipe when we claimed its interface ourselves; + // no transfer is ever submitted on it, so there is nothing else to cancel. + if (channel->cdc_dev_.notify_ep != nullptr && channel->cdc_dev_.interrupt_interface_claimed) { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); } - if (channel->cdc_dev_.interrupt_interface_number != 0xFF && - channel->cdc_dev_.interrupt_interface_number != channel->cdc_dev_.bulk_interface_number) { + if (channel->cdc_dev_.interrupt_interface_claimed) { usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number); - channel->cdc_dev_.interrupt_interface_number = 0xFF; + channel->cdc_dev_.interrupt_interface_claimed = false; } usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.bulk_interface_number); // Reset the input and output started flags to their initial state to avoid the possibility of spurious restarts diff --git a/esphome/components/usb_uart/usb_uart.h b/esphome/components/usb_uart/usb_uart.h index 00b34fb942..9d87bf964c 100644 --- a/esphome/components/usb_uart/usb_uart.h +++ b/esphome/components/usb_uart/usb_uart.h @@ -34,7 +34,10 @@ struct CdcEps { const usb_ep_desc_t *in_ep; const usb_ep_desc_t *out_ep; uint8_t bulk_interface_number; + // Also the wIndex target for CDC class requests (SET_LINE_CODING etc.), so it + // must remain valid even when the interface itself is not claimed. uint8_t interrupt_interface_number; + bool interrupt_interface_claimed{false}; }; enum CH34xChipType : uint8_t { From 84f78831f95442f124c2f652b644611b15143fbe Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:07:15 +0200 Subject: [PATCH 06/53] Bump bundled esphome-device-builder to 1.14.1 (#18981) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7952616496..2d4ddbef5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 RUN \ platformio settings set enable_telemetry No \ From ae187f81f25fcce1869a8f128c3a929ec07c7f89 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:44:37 +1000 Subject: [PATCH 07/53] [wifi] Allow a forced roam check (#17349) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Claude --- esphome/components/wifi/__init__.py | 16 ++++++++- esphome/components/wifi/automation.h | 5 +++ esphome/components/wifi/wifi_component.cpp | 38 +++++++++++++++------- esphome/components/wifi/wifi_component.h | 6 ++++ tests/components/wifi/common.yaml | 1 + 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index b8c6d774ac..1691dcc293 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -66,13 +66,14 @@ from esphome.const import ( ) from esphome.core import ( CORE, + ID, CoroPriority, EsphomeError, HexInt, coroutine_with_priority, ) import esphome.final_validate as fv -from esphome.types import ConfigType +from esphome.types import ConfigType, TemplateArgsType from . import wpa2_eap @@ -208,6 +209,7 @@ WiFiEnabledCondition = wifi_ns.class_("WiFiEnabledCondition", Condition) WiFiAPActiveCondition = wifi_ns.class_("WiFiAPActiveCondition", Condition) WiFiEnableAction = wifi_ns.class_("WiFiEnableAction", automation.Action) WiFiDisableAction = wifi_ns.class_("WiFiDisableAction", automation.Action) +WiFiRoamAction = wifi_ns.class_("WiFiRoamAction", automation.Action) WiFiConfigureAction = wifi_ns.class_( "WiFiConfigureAction", automation.Action, cg.Component ) @@ -820,6 +822,18 @@ async def wifi_disable_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg) +@automation.register_action( + "wifi.roam", WiFiRoamAction, cv.Schema({}), synchronous=True +) +async def wifi_roam_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> cg.MockObj: + return cg.new_Pvariable(action_id, template_arg) + + KEEP_SCAN_RESULTS_KEY = "wifi_keep_scan_results" RUNTIME_POWER_SAVE_KEY = "wifi_runtime_power_save" RUNTIME_ROAMING_SUPPRESSION_KEY = "wifi_runtime_roaming_suppression" diff --git a/esphome/components/wifi/automation.h b/esphome/components/wifi/automation.h index e63faa18ab..c14341330f 100644 --- a/esphome/components/wifi/automation.h +++ b/esphome/components/wifi/automation.h @@ -31,6 +31,11 @@ template class WiFiDisableAction final : public Action { void play(const Ts &...x) override { global_wifi_component->disable(); } }; +template class WiFiRoamAction final : public Action { + public: + void play(const Ts &...x) override { global_wifi_component->force_roam_check(); } +}; + template class WiFiConfigureAction final : public Action, public Component { public: TEMPLATABLE_VALUE(std::string, ssid) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 694e616476..f9e80995e1 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -846,17 +846,18 @@ void WiFiComponent::loop() { this->notify_connect_state_listeners_(); #endif - // Post-connect roaming: check for better AP - if (this->post_connect_roaming_) { - if (this->is_roaming_scan_active()) { - if (this->scan_done_) { - this->process_roaming_scan_(); - } - // else: scan in progress, wait - } else if (this->roaming_state_ == RoamingState::IDLE && this->roaming_attempts_ < ROAMING_MAX_ATTEMPTS && - now - this->roaming_last_check_ >= ROAMING_CHECK_INTERVAL && !this->roaming_suppressed_()) { - this->check_roaming_(now); + // Post-connect roaming: check for better AP. A scan may have been started by an + // explicit force_roam_check() even when post_connect_roaming_ is disabled, so the + // scan must always be consumed here to avoid leaving roaming_state_ stuck. + if (this->is_roaming_scan_active()) { + if (this->scan_done_) { + this->process_roaming_scan_(); } + // else: scan in progress, wait + } else if (this->post_connect_roaming_ && this->roaming_state_ == RoamingState::IDLE && + this->roaming_attempts_ < ROAMING_MAX_ATTEMPTS && + now - this->roaming_last_check_ >= ROAMING_CHECK_INTERVAL && !this->roaming_suppressed_()) { + this->check_roaming_(now); } } break; @@ -2463,6 +2464,17 @@ void WiFiComponent::notify_scan_results_listeners_() { } #endif // USE_WIFI_SCAN_RESULTS_LISTENERS +void WiFiComponent::force_roam_check() { + if (!this->is_connected() || this->roaming_state_ != RoamingState::IDLE || this->roaming_suppressed_()) { + ESP_LOGD(TAG, "Roam check requested, but not able to check now"); + return; + } + // Reset the attempt counter so a prior run of failed roams doesn't block this explicit request + // Note that this re-arms automatic roaming if enabled. + this->roaming_attempts_ = 0; + this->check_roaming_(millis()); +} + void WiFiComponent::check_roaming_(uint32_t now) { // Guard: not for hidden networks (may not appear in scan) const WiFiAP *selected = this->get_selected_sta_(); @@ -2484,7 +2496,11 @@ void WiFiComponent::check_roaming_(uint32_t now) { ESP_LOGD(TAG, "Roam scan (%d dBm, attempt %u/%u)", rssi, this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); this->roaming_state_ = RoamingState::SCANNING; - this->wifi_scan_start_(this->passive_scan_); + if (!this->wifi_scan_start_(this->passive_scan_)) { + // Scan failed to start (e.g. busy) - don't get stuck in SCANNING forever + ESP_LOGD(TAG, "Roam scan failed to start"); + this->roaming_state_ = RoamingState::IDLE; + } } void WiFiComponent::process_roaming_scan_() { diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 63df9fbfa5..94fdd9bc14 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -565,6 +565,12 @@ class WiFiComponent final : public Component { void set_keep_scan_results(bool keep_scan_results) { this->keep_scan_results_ = keep_scan_results; } void set_post_connect_roaming(bool enabled) { this->post_connect_roaming_ = enabled; } + /** Force an immediate post-connect roaming check, bypassing the periodic interval and the + * per-connection attempt limit. Does nothing (besides a debug log) if not connected, if a + * roam scan or connect is already in progress, or if roaming is currently suppressed. + */ + void force_roam_check(); + #ifdef USE_WIFI_CONNECT_TRIGGER Trigger<> *get_connect_trigger() { return &this->connect_trigger_; } #endif diff --git a/tests/components/wifi/common.yaml b/tests/components/wifi/common.yaml index 10b68347eb..10a8a61c66 100644 --- a/tests/components/wifi/common.yaml +++ b/tests/components/wifi/common.yaml @@ -14,6 +14,7 @@ esphome: condition: wifi.ap_active then: - logger.log: "WiFi AP is active!" + - wifi.roam wifi: networks: From 3ef7460fca9e5326b5d51e0e3bf51c1bcb8abde6 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:25:58 +0000 Subject: [PATCH 08/53] Bump bundled esphome-device-builder to 1.14.2 (#18988) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d4ddbef5d..b5170864a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 RUN \ platformio settings set enable_telemetry No \ From e3dd2f44a45bc7200566393db51fa17eb0a5edf1 Mon Sep 17 00:00:00 2001 From: elwin loomis Date: Sat, 5 Sep 2026 16:08:21 -0500 Subject: [PATCH 09/53] [mipi_dsi] Let IDF pick the DPHY PLL reference clock (#18984) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> --- esphome/components/mipi_dsi/mipi_dsi.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index 0850b50c85..0150cc2544 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -35,8 +35,8 @@ void MipiDsi::setup() { .bus_id = 0, // index from 0, specify the DSI host to use .num_data_lanes = this->lanes_, // Number of data lanes to use, can't set a value that exceeds the chip's capability - .phy_clk_src = MIPI_DSI_PHY_CLK_SRC_DEFAULT, // Clock source for the DPHY - .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps + // phy_clk_src left at 0 to enable runtime auto-select. + .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps }; auto err = esp_lcd_new_dsi_bus(&bus_config, &this->bus_handle_); if (err != ESP_OK) { From e5200db6fd6008da8a1e4b88d8b99e463aae0759 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:28:02 +0200 Subject: [PATCH 10/53] Bump bundled esphome-device-builder to 1.14.3 (#18996) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5170864a3..e875851bfb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 RUN \ platformio settings set enable_telemetry No \ From 8e1044e8ea35aa959121170d7a8000fcbf90aed2 Mon Sep 17 00:00:00 2001 From: Ricardo Sanz Date: Sun, 6 Sep 2026 23:03:07 +0200 Subject: [PATCH 11/53] [climate][template] New template climate component (#14455) --- esphome/components/climate/__init__.py | 13 + .../components/template/climate/__init__.py | 465 ++++++++++++++++++ .../components/template/climate/automation.h | 57 +++ .../template/climate/template_climate.cpp | 164 ++++++ .../template/climate/template_climate.h | 92 ++++ esphome/config_validation.py | 1 + .../template/test_template_climate.py | 145 ++++++ tests/components/climate/common.yaml | 3 +- tests/components/template/common-base.yaml | 113 +++++ .../fixtures/template_climate_basic.yaml | 72 +++ .../template_climate_custom_modes.yaml | 47 ++ .../template_climate_nonoptimistic.yaml | 56 +++ .../template_climate_on_control_ordering.yaml | 26 + .../template_climate_publish_all_fields.yaml | 63 +++ .../template_climate_sensor_push.yaml | 49 ++ .../template_climate_set_actions.yaml | 89 ++++ ...emplate_climate_two_point_temperature.yaml | 52 ++ .../test_template_climate_basic.py | 146 ++++++ .../test_template_climate_custom_modes.py | 98 ++++ .../test_template_climate_nonoptimistic.py | 107 ++++ ...st_template_climate_on_control_ordering.py | 83 ++++ ...est_template_climate_publish_all_fields.py | 96 ++++ .../test_template_climate_sensor_push.py | 88 ++++ .../test_template_climate_set_actions.py | 114 +++++ ..._template_climate_two_point_temperature.py | 118 +++++ 25 files changed, 2355 insertions(+), 2 deletions(-) create mode 100644 esphome/components/template/climate/__init__.py create mode 100644 esphome/components/template/climate/automation.h create mode 100644 esphome/components/template/climate/template_climate.cpp create mode 100644 esphome/components/template/climate/template_climate.h create mode 100644 tests/component_tests/template/test_template_climate.py create mode 100644 tests/integration/fixtures/template_climate_basic.yaml create mode 100644 tests/integration/fixtures/template_climate_custom_modes.yaml create mode 100644 tests/integration/fixtures/template_climate_nonoptimistic.yaml create mode 100644 tests/integration/fixtures/template_climate_on_control_ordering.yaml create mode 100644 tests/integration/fixtures/template_climate_publish_all_fields.yaml create mode 100644 tests/integration/fixtures/template_climate_sensor_push.yaml create mode 100644 tests/integration/fixtures/template_climate_set_actions.yaml create mode 100644 tests/integration/fixtures/template_climate_two_point_temperature.yaml create mode 100644 tests/integration/test_template_climate_basic.py create mode 100644 tests/integration/test_template_climate_custom_modes.py create mode 100644 tests/integration/test_template_climate_nonoptimistic.py create mode 100644 tests/integration/test_template_climate_on_control_ordering.py create mode 100644 tests/integration/test_template_climate_publish_all_fields.py create mode 100644 tests/integration/test_template_climate_sensor_push.py create mode 100644 tests/integration/test_template_climate_set_actions.py create mode 100644 tests/integration/test_template_climate_two_point_temperature.py diff --git a/esphome/components/climate/__init__.py b/esphome/components/climate/__init__.py index 80dd913fba..3fbca1a6d0 100644 --- a/esphome/components/climate/__init__.py +++ b/esphome/components/climate/__init__.py @@ -125,6 +125,19 @@ CLIMATE_SWING_MODES = { validate_climate_swing_mode = cv.enum(CLIMATE_SWING_MODES, upper=True) +ClimateAction = climate_ns.enum("ClimateAction") +CLIMATE_ACTIONS = { + "OFF": ClimateAction.CLIMATE_ACTION_OFF, + "COOLING": ClimateAction.CLIMATE_ACTION_COOLING, + "HEATING": ClimateAction.CLIMATE_ACTION_HEATING, + "IDLE": ClimateAction.CLIMATE_ACTION_IDLE, + "DRYING": ClimateAction.CLIMATE_ACTION_DRYING, + "FAN": ClimateAction.CLIMATE_ACTION_FAN, + "DEFROSTING": ClimateAction.CLIMATE_ACTION_DEFROSTING, +} + +validate_climate_action = cv.enum(CLIMATE_ACTIONS, upper=True) + CONF_MIN_HUMIDITY = "min_humidity" CONF_MAX_HUMIDITY = "max_humidity" CONF_TARGET_HUMIDITY = "target_humidity" diff --git a/esphome/components/template/climate/__init__.py b/esphome/components/template/climate/__init__.py new file mode 100644 index 0000000000..c39ea8f80e --- /dev/null +++ b/esphome/components/template/climate/__init__.py @@ -0,0 +1,465 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import climate, sensor +from esphome.components.climate import climate_ns +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTION, + CONF_CURRENT_TEMPERATURE, + CONF_CUSTOM_FAN_MODE, + CONF_CUSTOM_FAN_MODES, + CONF_CUSTOM_PRESET, + CONF_CUSTOM_PRESETS, + CONF_FAN_MODE, + CONF_HUMIDITY_SENSOR, + CONF_ID, + CONF_INITIAL_STATE, + CONF_MODE, + CONF_OPTIMISTIC, + CONF_PRESET, + CONF_RESTORE_MODE, + CONF_SENSOR, + CONF_SUPPORTED_FAN_MODES, + CONF_SUPPORTED_MODES, + CONF_SUPPORTED_PRESETS, + CONF_SUPPORTED_SWING_MODES, + CONF_SWING_MODE, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.types import ConfigType + +from .. import template_ns + +CONF_CURRENT_HUMIDITY = "current_humidity" +CONF_TARGET_HUMIDITY = "target_humidity" +CONF_SUPPORTS_ACTION = "supports_action" +CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE = "supports_two_point_target_temperature" +CONF_SUPPORTS_TARGET_HUMIDITY = "supports_target_humidity" +CONF_SUPPORTS_CURRENT_TEMPERATURE = "supports_current_temperature" +CONF_SUPPORTS_CURRENT_HUMIDITY = "supports_current_humidity" +CONF_SET_MODE_ACTION = "set_mode_action" +CONF_SET_TARGET_TEMPERATURE_ACTION = "set_target_temperature_action" +CONF_SET_TARGET_TEMPERATURE_LOW_ACTION = "set_target_temperature_low_action" +CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION = "set_target_temperature_high_action" +CONF_SET_TARGET_HUMIDITY_ACTION = "set_target_humidity_action" +CONF_SET_FAN_MODE_ACTION = "set_fan_mode_action" +CONF_SET_CUSTOM_FAN_MODE_ACTION = "set_custom_fan_mode_action" +CONF_SET_SWING_MODE_ACTION = "set_swing_mode_action" +CONF_SET_PRESET_ACTION = "set_preset_action" +CONF_SET_CUSTOM_PRESET_ACTION = "set_custom_preset_action" + +TemplateClimate = template_ns.class_("TemplateClimate", climate.Climate, cg.Component) +TemplateClimatePublishAction = template_ns.class_( + "TemplateClimatePublishAction", + automation.Action, + cg.Parented.template(TemplateClimate), +) + +TemplateClimateRestoreMode = template_ns.enum( + "TemplateClimateRestoreMode", is_class=True +) +CLIMATE_RESTORE_MODES = { + "NO_RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + "RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +} + +# Per-field actions that forward a requested value on. The third item is the type of `x`. +SET_ACTIONS = ( + (CONF_SET_MODE_ACTION, "get_set_mode_trigger", climate.ClimateMode), + ( + CONF_SET_TARGET_TEMPERATURE_ACTION, + "get_set_target_temperature_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + "get_set_target_temperature_low_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + "get_set_target_temperature_high_trigger", + cg.float_, + ), + (CONF_SET_TARGET_HUMIDITY_ACTION, "get_set_target_humidity_trigger", cg.float_), + (CONF_SET_FAN_MODE_ACTION, "get_set_fan_mode_trigger", climate.ClimateFanMode), + ( + CONF_SET_CUSTOM_FAN_MODE_ACTION, + "get_set_custom_fan_mode_trigger", + cg.StringRef, + ), + ( + CONF_SET_SWING_MODE_ACTION, + "get_set_swing_mode_trigger", + climate.ClimateSwingMode, + ), + (CONF_SET_PRESET_ACTION, "get_set_preset_trigger", climate.ClimatePreset), + (CONF_SET_CUSTOM_PRESET_ACTION, "get_set_custom_preset_trigger", cg.StringRef), +) + +# supports_* keys have no default so that an omitted key can mean "derive it from the sensor or +# set action that makes the trait useful", which is not expressible once a default fills it in. +DERIVED_SUPPORTS = ( + (CONF_SUPPORTS_CURRENT_TEMPERATURE, (CONF_SENSOR,)), + (CONF_SUPPORTS_CURRENT_HUMIDITY, (CONF_HUMIDITY_SENSOR,)), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + ), + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, (CONF_SET_TARGET_HUMIDITY_ACTION,)), +) + + +# Custom fan modes/presets are opaque user-defined strings with no build-time correctness check +# elsewhere (Climate::set_supported_custom_fan_modes()/set_supported_custom_presets() don't block +# empty entries), so reject empty ones here -- they could never be selected at runtime anyway. +validate_custom_climate_string = cv.All(cv.string_strict, cv.Length(min=1)) + + +def _validate_two_point(config: ConfigType) -> ConfigType: + has_low = CONF_TARGET_TEMPERATURE_LOW in config + has_high = CONF_TARGET_TEMPERATURE_HIGH in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE_LOW}' and '{CONF_TARGET_TEMPERATURE_HIGH}' must be used together" + ) + if (has_low or has_high) and CONF_TARGET_TEMPERATURE in config: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' cannot be used together with " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}'" + ) + return config + + +def _validate_set_actions(config: ConfigType) -> ConfigType: + has_low = CONF_SET_TARGET_TEMPERATURE_LOW_ACTION in config + has_high = CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}' and " + f"'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}' must be used together" + ) + if (has_low or has_high) and CONF_SET_TARGET_TEMPERATURE_ACTION in config: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_ACTION}' cannot be used together with " + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}'/'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}'" + ) + return config + + +def _resolve_supports(config: ConfigType) -> ConfigType: + # An explicit true stays valid without either, since climate.template.publish can report the + # value; an explicit false that contradicts the configuration is an error, not a silent override. + for key, sources in DERIVED_SUPPORTS: + configured = [source for source in sources if source in config] + if key not in config: + config[key] = bool(configured) + elif not config[key] and configured: + raise cv.Invalid( + f"'{key}' cannot be false while '{configured[0]}' is configured", + path=[key], + ) + return config + + +def _validate_initial_state(config: ConfigType) -> ConfigType: + # Climate keeps target_temperature and target_temperature_low in a union, so writing the wrong + # one of the pair corrupts the setpoint with no runtime complaint. + if (initial_state := config.get(CONF_INITIAL_STATE)) is None: + return config + + two_point = config[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] + if two_point and CONF_TARGET_TEMPERATURE in initial_state: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' is not available while " + f"'{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' is enabled; use " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}' instead", + path=[CONF_INITIAL_STATE, CONF_TARGET_TEMPERATURE], + ) + if not two_point: + for key in (CONF_TARGET_TEMPERATURE_LOW, CONF_TARGET_TEMPERATURE_HIGH): + if key in initial_state: + raise cv.Invalid( + f"'{key}' requires '{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' to be enabled", + path=[CONF_INITIAL_STATE, key], + ) + if ( + CONF_TARGET_HUMIDITY in initial_state + and not config[CONF_SUPPORTS_TARGET_HUMIDITY] + ): + raise cv.Invalid( + f"'{CONF_TARGET_HUMIDITY}' requires '{CONF_SUPPORTS_TARGET_HUMIDITY}' to be enabled", + path=[CONF_INITIAL_STATE, CONF_TARGET_HUMIDITY], + ) + return config + + +# Same settable fields as climate.template.publish, minus current_temperature/current_humidity/ +# action: those are reported values (from a sensor or the device), not meaningful static defaults. +INITIAL_STATE_SCHEMA = cv.All( + cv.Schema( + { + cv.Optional(CONF_MODE): climate.validate_climate_mode, + cv.Optional(CONF_TARGET_TEMPERATURE): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.temperature, + cv.Optional(CONF_TARGET_HUMIDITY): cv.percentage_int, + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): climate.validate_climate_fan_mode, + cv.Exclusive( + CONF_CUSTOM_FAN_MODE, "fan_mode" + ): validate_custom_climate_string, + cv.Optional(CONF_SWING_MODE): climate.validate_climate_swing_mode, + cv.Exclusive(CONF_PRESET, "preset"): climate.validate_climate_preset, + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): validate_custom_climate_string, + } + ), + _validate_two_point, +) + +CONFIG_SCHEMA = cv.All( + climate.climate_schema(TemplateClimate) + .extend( + { + cv.Optional(CONF_SENSOR): cv.use_id(sensor.Sensor), + cv.Optional(CONF_HUMIDITY_SENSOR): cv.use_id(sensor.Sensor), + # action only ever arrives through climate.template.publish, so unlike the other + # supports_* keys there is no set action to derive it from. + cv.Optional(CONF_SUPPORTS_ACTION, default=False): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_HUMIDITY): cv.boolean, + cv.Optional(CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_TARGET_HUMIDITY): cv.boolean, + cv.Required(CONF_SUPPORTED_MODES): cv.All( + cv.ensure_list(climate.validate_climate_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_FAN_MODES): cv.All( + cv.ensure_list(climate.validate_climate_fan_mode), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_FAN_MODES): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_SWING_MODES): cv.All( + cv.ensure_list(climate.validate_climate_swing_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_PRESETS): cv.All( + cv.ensure_list(climate.validate_climate_preset), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_PRESETS): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_OPTIMISTIC, default=True): cv.boolean, + cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum( + CLIMATE_RESTORE_MODES, upper=True + ), + cv.Optional(CONF_INITIAL_STATE): INITIAL_STATE_SCHEMA, + cv.Optional(CONF_SET_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_HUMIDITY_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_FAN_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_CUSTOM_FAN_MODE_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_SWING_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_PRESET_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_CUSTOM_PRESET_ACTION): automation.validate_automation( + single=True + ), + } + ) + .extend(cv.COMPONENT_SCHEMA), + _validate_set_actions, + _resolve_supports, + _validate_initial_state, +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await climate.register_climate(var, config) + + if (sens := config.get(CONF_SENSOR)) is not None: + cg.add(var.set_sensor(await cg.get_variable(sens))) + + if (sens := config.get(CONF_HUMIDITY_SENSOR)) is not None: + cg.add(var.set_humidity_sensor(await cg.get_variable(sens))) + + for key, flag in ( + (CONF_SUPPORTS_ACTION, climate_ns.CLIMATE_SUPPORTS_ACTION), + ( + CONF_SUPPORTS_CURRENT_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_CURRENT_TEMPERATURE, + ), + (CONF_SUPPORTS_CURRENT_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_CURRENT_HUMIDITY), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_TARGET_HUMIDITY), + ): + if config[key]: + cg.add(var.add_feature_flags(flag)) + + for mode in config[CONF_SUPPORTED_MODES]: + cg.add(var.add_supported_mode(mode)) + + for mode in config.get(CONF_SUPPORTED_FAN_MODES, []): + cg.add(var.add_supported_fan_mode(mode)) + + if CONF_CUSTOM_FAN_MODES in config: + cg.add( + var.set_supported_custom_fan_modes( + cg.ArrayInitializer(*config[CONF_CUSTOM_FAN_MODES]) + ) + ) + + for mode in config.get(CONF_SUPPORTED_SWING_MODES, []): + cg.add(var.add_supported_swing_mode(mode)) + + for preset in config.get(CONF_SUPPORTED_PRESETS, []): + cg.add(var.add_supported_preset(preset)) + + if CONF_CUSTOM_PRESETS in config: + cg.add( + var.set_supported_custom_presets( + cg.ArrayInitializer(*config[CONF_CUSTOM_PRESETS]) + ) + ) + + for key, trigger_getter, arg_type in SET_ACTIONS: + if (conf := config.get(key)) is not None: + await automation.build_automation( + getattr(var, trigger_getter)(), [(arg_type, "x")], conf + ) + + cg.add(var.set_optimistic(config[CONF_OPTIMISTIC])) + cg.add(var.set_restore_mode(config[CONF_RESTORE_MODE])) + + if (initial_state := config.get(CONF_INITIAL_STATE)) is not None: + if (v := initial_state.get(CONF_MODE)) is not None: + cg.add(var.set_mode(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add(var.set_target_temperature_high(v)) + if (v := initial_state.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(v)) + if (v := initial_state.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(v)) + if (v := initial_state.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(v)) + if (v := initial_state.get(CONF_SWING_MODE)) is not None: + cg.add(var.set_swing_mode(v)) + if (v := initial_state.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(v)) + if (v := initial_state.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(v)) + + +CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.use_id(TemplateClimate), + cv.Optional(CONF_CURRENT_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_CURRENT_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_TARGET_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_MODE): cv.templatable(climate.validate_climate_mode), + cv.Optional(CONF_ACTION): cv.templatable(climate.validate_climate_action), + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): cv.templatable( + climate.validate_climate_fan_mode + ), + cv.Exclusive(CONF_CUSTOM_FAN_MODE, "fan_mode"): cv.templatable( + validate_custom_climate_string + ), + cv.Optional(CONF_SWING_MODE): cv.templatable( + climate.validate_climate_swing_mode + ), + cv.Exclusive(CONF_PRESET, "preset"): cv.templatable( + climate.validate_climate_preset + ), + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): cv.templatable( + validate_custom_climate_string + ), + } + ), + _validate_two_point, +) + + +@automation.register_action( + "climate.template.publish", + TemplateClimatePublishAction, + CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA, + synchronous=True, +) +async def climate_template_publish_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + + if (v := config.get(CONF_CURRENT_TEMPERATURE)) is not None: + cg.add(var.set_current_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_CURRENT_HUMIDITY)) is not None: + cg.add(var.set_current_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add( + var.set_target_temperature_high(await cg.templatable(v, args, cg.float_)) + ) + if (v := config.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_MODE)) is not None: + cg.add(var.set_mode(await cg.templatable(v, args, climate.ClimateMode))) + if (v := config.get(CONF_ACTION)) is not None: + cg.add(var.set_action(await cg.templatable(v, args, climate.ClimateAction))) + if (v := config.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(await cg.templatable(v, args, climate.ClimateFanMode))) + if (v := config.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(await cg.templatable(v, args, cg.std_string))) + if (v := config.get(CONF_SWING_MODE)) is not None: + cg.add( + var.set_swing_mode(await cg.templatable(v, args, climate.ClimateSwingMode)) + ) + if (v := config.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(await cg.templatable(v, args, climate.ClimatePreset))) + if (v := config.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(await cg.templatable(v, args, cg.std_string))) + + return var diff --git a/esphome/components/template/climate/automation.h b/esphome/components/template/climate/automation.h new file mode 100644 index 0000000000..49a79ace2f --- /dev/null +++ b/esphome/components/template/climate/automation.h @@ -0,0 +1,57 @@ +#pragma once + +#include "template_climate.h" +#include "esphome/core/automation.h" + +namespace esphome::template_ { + +template +class TemplateClimatePublishAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(float, current_temperature) + TEMPLATABLE_VALUE(float, current_humidity) + TEMPLATABLE_VALUE(float, target_temperature) + TEMPLATABLE_VALUE(float, target_temperature_low) + TEMPLATABLE_VALUE(float, target_temperature_high) + TEMPLATABLE_VALUE(float, target_humidity) + TEMPLATABLE_VALUE(climate::ClimateMode, mode) + TEMPLATABLE_VALUE(climate::ClimateAction, action) + TEMPLATABLE_VALUE(climate::ClimateFanMode, fan_mode) + TEMPLATABLE_VALUE(std::string, custom_fan_mode) + TEMPLATABLE_VALUE(climate::ClimateSwingMode, swing_mode) + TEMPLATABLE_VALUE(climate::ClimatePreset, preset) + TEMPLATABLE_VALUE(std::string, custom_preset) + + void play(const Ts &...x) override { + if (this->current_temperature_.has_value()) + this->parent_->current_temperature = this->current_temperature_.value(x...); + if (this->current_humidity_.has_value()) + this->parent_->current_humidity = this->current_humidity_.value(x...); + if (this->target_temperature_.has_value()) + this->parent_->set_target_temperature(this->target_temperature_.value(x...)); + if (this->target_temperature_low_.has_value()) + this->parent_->set_target_temperature_low(this->target_temperature_low_.value(x...)); + if (this->target_temperature_high_.has_value()) + this->parent_->set_target_temperature_high(this->target_temperature_high_.value(x...)); + if (this->target_humidity_.has_value()) + this->parent_->set_target_humidity(this->target_humidity_.value(x...)); + if (this->mode_.has_value()) + this->parent_->set_mode(this->mode_.value(x...)); + if (this->action_.has_value()) + this->parent_->action = this->action_.value(x...); + if (this->fan_mode_.has_value()) + this->parent_->set_fan_mode(this->fan_mode_.value(x...)); + if (this->custom_fan_mode_.has_value()) + this->parent_->set_custom_fan_mode(StringRef(this->custom_fan_mode_.value(x...))); + if (this->swing_mode_.has_value()) + this->parent_->set_swing_mode(this->swing_mode_.value(x...)); + if (this->preset_.has_value()) + this->parent_->set_preset(this->preset_.value(x...)); + if (this->custom_preset_.has_value()) + this->parent_->set_custom_preset(StringRef(this->custom_preset_.value(x...))); + + this->parent_->publish_state(); + } +}; + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.cpp b/esphome/components/template/climate/template_climate.cpp new file mode 100644 index 0000000000..a7a4d2ccab --- /dev/null +++ b/esphome/components/template/climate/template_climate.cpp @@ -0,0 +1,164 @@ +#include "template_climate.h" +#include "esphome/core/log.h" + +namespace esphome::template_ { + +static const char *const TAG = "template.climate"; + +void TemplateClimate::setup() { + if (this->restore_mode_ == TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE) { + auto restore = this->restore_state_(); + if (restore.has_value()) { + restore->apply(this); + } + } + + // Sensors publish every reading, not just changes, so only re-publish when the value moved. + // NAN means the sensor went unavailable and is passed through rather than dropped; the second + // check stops an unavailable sensor re-publishing forever, since NAN never equals NAN. +#ifdef USE_SENSOR + if (this->sensor_ != nullptr) { + this->current_temperature = this->sensor_->state; + this->sensor_->add_on_state_callback([this](float state) { + if (state != this->current_temperature && !(std::isnan(state) && std::isnan(this->current_temperature))) { + this->current_temperature = state; + this->publish_state(); + } + }); + } + + if (this->humidity_sensor_ != nullptr) { + this->current_humidity = this->humidity_sensor_->state; + this->humidity_sensor_->add_on_state_callback([this](float state) { + if (state != this->current_humidity && !(std::isnan(state) && std::isnan(this->current_humidity))) { + this->current_humidity = state; + this->publish_state(); + } + }); + } +#endif +} + +void TemplateClimate::dump_config() { + LOG_CLIMATE("", "Template Climate", this); + ESP_LOGCONFIG(TAG, " Optimistic: %s", YESNO(this->optimistic_)); +} + +void TemplateClimate::control(const climate::ClimateCall &call) { + // Each field present fires its set_*_action; on_control sees the whole call. optimistic: true + // also applies the values right away, false waits for a climate.template.publish report. + if (auto mode = call.get_mode()) { + if (this->optimistic_) + this->mode = *mode; + this->set_mode_trigger_.trigger(*mode); + } + + if (auto target_temp = call.get_target_temperature()) { + if (this->optimistic_) + this->target_temperature = *target_temp; + this->set_target_temperature_trigger_.trigger(*target_temp); + } + + if (auto target_temp_low = call.get_target_temperature_low()) { + if (this->optimistic_) + this->target_temperature_low = *target_temp_low; + this->set_target_temperature_low_trigger_.trigger(*target_temp_low); + } + + if (auto target_temp_high = call.get_target_temperature_high()) { + if (this->optimistic_) + this->target_temperature_high = *target_temp_high; + this->set_target_temperature_high_trigger_.trigger(*target_temp_high); + } + + if (auto target_humidity = call.get_target_humidity()) { + if (this->optimistic_) + this->target_humidity = *target_humidity; + this->set_target_humidity_trigger_.trigger(*target_humidity); + } + + if (auto fan_mode = call.get_fan_mode()) { + if (this->optimistic_) + this->set_fan_mode_(*fan_mode); + this->set_fan_mode_trigger_.trigger(*fan_mode); + } + + if (call.has_custom_fan_mode()) { + if (this->optimistic_) + this->set_custom_fan_mode_(call.get_custom_fan_mode()); + this->set_custom_fan_mode_trigger_.trigger(call.get_custom_fan_mode()); + } + + if (auto swing_mode = call.get_swing_mode()) { + if (this->optimistic_) + this->swing_mode = *swing_mode; + this->set_swing_mode_trigger_.trigger(*swing_mode); + } + + if (auto preset = call.get_preset()) { + if (this->optimistic_) + this->set_preset_(*preset); + this->set_preset_trigger_.trigger(*preset); + } + + if (call.has_custom_preset()) { + if (this->optimistic_) + this->set_custom_preset_(call.get_custom_preset()); + this->set_custom_preset_trigger_.trigger(call.get_custom_preset()); + } + + if (this->optimistic_) + this->publish_state(); +} + +// A climate.template.publish report (and initial_state:) never goes through ClimateCall::validate_(), +// so check here instead -- otherwise a typo is published as state the receiving end will reject. +void TemplateClimate::set_mode(climate::ClimateMode mode) { + if (!this->traits_.supports_mode(mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported mode %u", this->get_name().c_str(), static_cast(mode)); + return; + } + this->mode = mode; +} + +void TemplateClimate::set_swing_mode(climate::ClimateSwingMode swing_mode) { + if (!this->traits_.supports_swing_mode(swing_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported swing mode %u", this->get_name().c_str(), static_cast(swing_mode)); + return; + } + this->swing_mode = swing_mode; +} + +void TemplateClimate::set_fan_mode(climate::ClimateFanMode fan_mode) { + if (!this->traits_.supports_fan_mode(fan_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported fan mode %u", this->get_name().c_str(), static_cast(fan_mode)); + return; + } + this->set_fan_mode_(fan_mode); +} + +void TemplateClimate::set_preset(climate::ClimatePreset preset) { + if (!this->traits_.supports_preset(preset)) { + ESP_LOGW(TAG, "'%s' - Unsupported preset %u", this->get_name().c_str(), static_cast(preset)); + return; + } + this->set_preset_(preset); +} + +void TemplateClimate::set_custom_fan_mode(StringRef mode) { + if (this->find_custom_fan_mode_(mode.c_str(), mode.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom fan mode '%s'", this->get_name().c_str(), mode.c_str()); + return; + } + this->set_custom_fan_mode_(mode); +} + +void TemplateClimate::set_custom_preset(StringRef preset) { + if (this->find_custom_preset_(preset.c_str(), preset.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom preset '%s'", this->get_name().c_str(), preset.c_str()); + return; + } + this->set_custom_preset_(preset); +} + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.h b/esphome/components/template/climate/template_climate.h new file mode 100644 index 0000000000..5448488c34 --- /dev/null +++ b/esphome/components/template/climate/template_climate.h @@ -0,0 +1,92 @@ +#pragma once + +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/components/climate/climate.h" +#ifdef USE_SENSOR +#include "esphome/components/sensor/sensor.h" +#endif + +namespace esphome::template_ { + +enum class TemplateClimateRestoreMode { + TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +}; + +class TemplateClimate final : public climate::Climate, public Component { + public: + void setup() override; + void dump_config() override; + + climate::ClimateTraits traits() override { return this->traits_; } + + void add_feature_flags(uint32_t flags) { this->traits_.add_feature_flags(flags); } + +#ifdef USE_SENSOR + // The matching feature flag is added from codegen, so the configuration alone decides it. + void set_sensor(sensor::Sensor *sensor) { this->sensor_ = sensor; } + void set_humidity_sensor(sensor::Sensor *sensor) { this->humidity_sensor_ = sensor; } +#endif + + void add_supported_mode(climate::ClimateMode mode) { this->traits_.add_supported_mode(mode); } + void add_supported_fan_mode(climate::ClimateFanMode mode) { this->traits_.add_supported_fan_mode(mode); } + void add_supported_swing_mode(climate::ClimateSwingMode mode) { this->traits_.add_supported_swing_mode(mode); } + void add_supported_preset(climate::ClimatePreset preset) { this->traits_.add_supported_preset(preset); } + + void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } + void set_restore_mode(TemplateClimateRestoreMode restore_mode) { this->restore_mode_ = restore_mode; } + + // Fired from control() for each field the call carries, so a device-backed config can forward + // it on. Which of these are configured also decides the two-point/target-humidity traits. + Trigger *get_set_mode_trigger() { return &this->set_mode_trigger_; } + Trigger *get_set_target_temperature_trigger() { return &this->set_target_temperature_trigger_; } + Trigger *get_set_target_temperature_low_trigger() { return &this->set_target_temperature_low_trigger_; } + Trigger *get_set_target_temperature_high_trigger() { return &this->set_target_temperature_high_trigger_; } + Trigger *get_set_target_humidity_trigger() { return &this->set_target_humidity_trigger_; } + Trigger *get_set_fan_mode_trigger() { return &this->set_fan_mode_trigger_; } + Trigger *get_set_custom_fan_mode_trigger() { return &this->set_custom_fan_mode_trigger_; } + Trigger *get_set_swing_mode_trigger() { return &this->set_swing_mode_trigger_; } + Trigger *get_set_preset_trigger() { return &this->set_preset_trigger_; } + Trigger *get_set_custom_preset_trigger() { return &this->set_custom_preset_trigger_; } + + // Used by TemplateClimatePublishAction, which is not a Climate subclass and so cannot reach the + // protected setters, and by codegen to apply `initial_state:` before setup() runs. + void set_target_temperature(float value) { this->target_temperature = value; } + void set_target_temperature_low(float value) { this->target_temperature_low = value; } + void set_target_temperature_high(float value) { this->target_temperature_high = value; } + void set_target_humidity(float value) { this->target_humidity = value; } + void set_mode(climate::ClimateMode mode); + void set_swing_mode(climate::ClimateSwingMode mode); + void set_fan_mode(climate::ClimateFanMode mode); + void set_custom_fan_mode(const char *mode) { this->set_custom_fan_mode(StringRef(mode)); } + void set_custom_fan_mode(StringRef mode); + void set_preset(climate::ClimatePreset preset); + void set_custom_preset(const char *preset) { this->set_custom_preset(StringRef(preset)); } + void set_custom_preset(StringRef preset); + + protected: + void control(const climate::ClimateCall &call) override; + + climate::ClimateTraits traits_; + bool optimistic_{false}; + TemplateClimateRestoreMode restore_mode_{TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE}; + +#ifdef USE_SENSOR + sensor::Sensor *sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; +#endif + + Trigger set_mode_trigger_; + Trigger set_target_temperature_trigger_; + Trigger set_target_temperature_low_trigger_; + Trigger set_target_temperature_high_trigger_; + Trigger set_target_humidity_trigger_; + Trigger set_fan_mode_trigger_; + Trigger set_custom_fan_mode_trigger_; + Trigger set_swing_mode_trigger_; + Trigger set_preset_trigger_; + Trigger set_custom_preset_trigger_; +}; + +} // namespace esphome::template_ diff --git a/esphome/config_validation.py b/esphome/config_validation.py index aff39201e8..685a9d04b3 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -133,6 +133,7 @@ Upper = vol.Upper Length = vol.Length Exclusive = vol.Exclusive Inclusive = vol.Inclusive +Unique = vol.Unique ALLOW_EXTRA = vol.ALLOW_EXTRA UNDEFINED = vol.UNDEFINED RequiredFieldInvalid = vol.RequiredFieldInvalid diff --git a/tests/component_tests/template/test_template_climate.py b/tests/component_tests/template/test_template_climate.py new file mode 100644 index 0000000000..304991ea64 --- /dev/null +++ b/tests/component_tests/template/test_template_climate.py @@ -0,0 +1,145 @@ +"""Tests for template climate config validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.template.climate import ( + CONF_SET_TARGET_HUMIDITY_ACTION, + CONF_SET_TARGET_TEMPERATURE_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SUPPORTS_CURRENT_HUMIDITY, + CONF_SUPPORTS_CURRENT_TEMPERATURE, + CONF_SUPPORTS_TARGET_HUMIDITY, + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + CONF_TARGET_HUMIDITY, + _resolve_supports, + _validate_initial_state, + _validate_set_actions, +) +from esphome.const import ( + CONF_HUMIDITY_SENSOR, + CONF_INITIAL_STATE, + CONF_SENSOR, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.types import ConfigType + + +def test_supports_current_temperature_derived_from_sensor() -> None: + config: ConfigType = {CONF_SENSOR: "some_sensor"} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_without_sensor() -> None: + assert _resolve_supports({})[CONF_SUPPORTS_CURRENT_TEMPERATURE] is False + + +def test_supports_current_temperature_explicit_true_without_sensor_allowed() -> None: + # The value can still be reported with climate.template.publish. + config: ConfigType = {CONF_SUPPORTS_CURRENT_TEMPERATURE: True} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_supports_current_humidity_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_HUMIDITY_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_HUMIDITY: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_two_point_derived_from_set_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + assert _resolve_supports(config)[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] is True + + +def test_two_point_false_with_set_action_rejected() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_target_humidity_derived_from_set_action() -> None: + config: ConfigType = {CONF_SET_TARGET_HUMIDITY_ACTION: [{}]} + assert _resolve_supports(config)[CONF_SUPPORTS_TARGET_HUMIDITY] is True + + +def test_set_target_temperature_low_requires_high() -> None: + config: ConfigType = {CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}]} + with pytest.raises(cv.Invalid, match="must be used together"): + _validate_set_actions(config) + + +def test_set_target_temperature_conflicts_with_two_point_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + with pytest.raises(cv.Invalid, match="cannot be used together"): + _validate_set_actions(config) + + +def test_initial_state_target_temperature_rejected_with_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_TEMPERATURE: 21.0}, + } + with pytest.raises(cv.Invalid, match="is not available"): + _validate_initial_state(config) + + +def test_initial_state_two_point_values_rejected_without_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + }, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_target_humidity_rejected_without_support() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_HUMIDITY: 50}, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_matching_two_point_accepted() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: True, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + CONF_TARGET_HUMIDITY: 50, + }, + } + assert _validate_initial_state(config) is config diff --git a/tests/components/climate/common.yaml b/tests/components/climate/common.yaml index c28fde8eeb..49386a16d5 100644 --- a/tests/components/climate/common.yaml +++ b/tests/components/climate/common.yaml @@ -30,8 +30,7 @@ climate: - switch.turn_on: climate_heater_switch - switch.turn_off: climate_cooler_switch # Thermostat-based climate so climate.control: action variants get build - # coverage (bang_bang doesn't support fan modes, presets, etc.). Climate - # has no template platform, so thermostat is the right vehicle. + # coverage (bang_bang doesn't support fan modes, presets, etc.). - platform: thermostat id: climate_test_thermostat name: Test Thermostat diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index 92a1fc8eda..02aedaf167 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -25,6 +25,27 @@ esphome: away: !lambda "return true;" is_on: !lambda "return false;" + - climate.template.publish: + id: template_climate + current_temperature: 21.0 + mode: HEAT + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE + target_temperature: 22.0 + + # Templated + - climate.template.publish: + id: template_climate + current_temperature: !lambda "return 21.5f;" + mode: !lambda "return climate::CLIMATE_MODE_COOL;" + target_temperature: !lambda "return 23.0f;" + + - climate.template.publish: + id: template_climate_custom_modes + custom_fan_mode: "turbo" + custom_preset: "eco_plus" + # Test C++ API: set_template() with stateless lambda (no captures) # NOTE: set_template() is not intended to be a public API, but we test it to ensure it doesn't break. - lambda: |- @@ -513,6 +534,98 @@ alarm_control_panel: codes: - "1234" +climate: + - platform: template + id: template_climate + name: "Template Climate" + optimistic: true + sensor: template_template_sens + supports_action: true + supports_current_humidity: true + restore_mode: NO_RESTORE + initial_state: + mode: HEAT + target_temperature: 21.0 + fan_mode: LOW + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_action: + - logger.log: + format: "set_target_temperature_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.1f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + on_control: + - logger.log: "on_control fired" + on_state: + - logger.log: "on_state fired" + + - platform: template + id: template_climate_custom_modes + name: "Template Climate Custom Modes" + optimistic: true + sensor: template_template_sens + supported_modes: + - "OFF" + - HEAT + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + initial_state: + custom_fan_mode: eco + custom_preset: max + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + water_heater: - platform: template id: template_water_heater diff --git a/tests/integration/fixtures/template_climate_basic.yaml b/tests/integration/fixtures/template_climate_basic.yaml new file mode 100644 index 0000000000..51558b4875 --- /dev/null +++ b/tests/integration/fixtures/template_climate_basic.yaml @@ -0,0 +1,72 @@ +esphome: + name: tmpl-clim-basic + on_boot: + - climate.template.publish: + id: test_climate + action: IDLE +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Basic Climate + optimistic: true + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 55.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: "OFF" + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE diff --git a/tests/integration/fixtures/template_climate_custom_modes.yaml b/tests/integration/fixtures/template_climate_custom_modes.yaml new file mode 100644 index 0000000000..9dbfe60cb9 --- /dev/null +++ b/tests/integration/fixtures/template_climate_custom_modes.yaml @@ -0,0 +1,47 @@ +esphome: + name: tmpl-clim-custom +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Custom Mode Climate + optimistic: true + sensor: test_climate_current_temperature + supported_modes: + - "OFF" + - HEAT + - COOL + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + on_control: + - lambda: |- + if (x.has_custom_fan_mode()) + ESP_LOGD("test", "on_control custom_fan_mode=%s", x.get_custom_fan_mode().c_str()); + if (x.has_custom_preset()) + ESP_LOGD("test", "on_control custom_preset=%s", x.get_custom_preset().c_str()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + custom_fan_mode: "eco" + custom_preset: "max" diff --git a/tests/integration/fixtures/template_climate_nonoptimistic.yaml b/tests/integration/fixtures/template_climate_nonoptimistic.yaml new file mode 100644 index 0000000000..2b0c7ee132 --- /dev/null +++ b/tests/integration/fixtures/template_climate_nonoptimistic.yaml @@ -0,0 +1,56 @@ +esphome: + name: tmpl-clim-nonopt +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Template Climate Nonoptimistic + optimistic: false + supported_modes: + - "OFF" + - HEAT + - COOL + - FAN_ONLY + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + - AWAY + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +button: + - platform: template + id: simulate_device_confirmation + name: Simulate Device Confirmation + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + target_temperature: 22.5 + fan_mode: HIGH + swing_mode: VERTICAL + preset: AWAY diff --git a/tests/integration/fixtures/template_climate_on_control_ordering.yaml b/tests/integration/fixtures/template_climate_on_control_ordering.yaml new file mode 100644 index 0000000000..8366a6d21e --- /dev/null +++ b/tests/integration/fixtures/template_climate_on_control_ordering.yaml @@ -0,0 +1,26 @@ +esphome: + name: tmpl-clim-oc-order +host: +api: +logger: + +# on_control fires with the full ClimateCall (arg `x`) from the base Climate component's +# ClimateCall::perform(), before validate_()/control() run -- so when the lambda action below +# runs, the entity's own .mode is still the OLD value, even though x.get_mode() already reports +# the NEW requested value. on_state fires afterward, once control() has applied it. +climate: + - platform: template + id: test_climate + name: Test On Control Ordering + optimistic: true + supported_modes: + - "OFF" + - HEAT + on_control: + - lambda: |- + ESP_LOGD("test", "on_control requested_mode=%d current_mode_before_apply=%d", + x.get_mode().has_value() ? (int) *x.get_mode() : -1, + (int) id(test_climate).mode); + on_state: + - lambda: |- + ESP_LOGD("test", "on_state mode=%d", (int) x.mode); diff --git a/tests/integration/fixtures/template_climate_publish_all_fields.yaml b/tests/integration/fixtures/template_climate_publish_all_fields.yaml new file mode 100644 index 0000000000..e57fcc4508 --- /dev/null +++ b/tests/integration/fixtures/template_climate_publish_all_fields.yaml @@ -0,0 +1,63 @@ +esphome: + name: tmpl-clim-publish-all +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Publish All Fields + optimistic: true + # current_temperature/current_humidity/action are only sent over the API at all if their + # trait is advertised: current_temperature/current_humidity because a sensor/humidity_sensor + # is referenced below, action because supports_action is set. The sensors' fixed readings + # match what climate.template.publish pushes, so the sensor callback (guarded to only publish + # on an actual change) doesn't produce an extra, unexpected state update of its own. + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + supported_fan_modes: + - AUTO + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + on_control: + # Should never fire in this test: climate.template.publish is a pure bypass and must not + # re-trigger on_control as if the entity were freshly commanded. + - logger.log: "on_control fired" + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 20.0f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 60.0f;" + update_interval: 10ms + +button: + - platform: template + id: publish_all + name: Publish All + on_press: + - climate.template.publish: + id: test_climate + current_temperature: 20.0 + current_humidity: 60.0 + target_temperature: 23.0 + mode: HEAT + action: HEATING + fan_mode: HIGH + swing_mode: VERTICAL + preset: ECO diff --git a/tests/integration/fixtures/template_climate_sensor_push.yaml b/tests/integration/fixtures/template_climate_sensor_push.yaml new file mode 100644 index 0000000000..1fc004335d --- /dev/null +++ b/tests/integration/fixtures/template_climate_sensor_push.yaml @@ -0,0 +1,49 @@ +esphome: + name: tmpl-clim-sensor-push +host: +api: +logger: + +# No lambda/update_interval: these sensors only ever report a value when a button below +# publishes one (standing in for e.g. a BLE scan callback in a real config). +sensor: + - platform: template + id: room_temperature + name: Room Temperature + - platform: template + id: room_humidity + name: Room Humidity + +climate: + - platform: template + id: test_climate + name: Test Sensor Push Climate + optimistic: true + sensor: room_temperature + humidity_sensor: room_humidity + supported_modes: + - "OFF" + - HEAT + +button: + - platform: template + id: publish_temperature + name: Publish Temperature + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_temperature_same + name: Publish Temperature Same Value + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_humidity + name: Publish Humidity + on_press: + - sensor.template.publish: + id: room_humidity + state: 65.0 diff --git a/tests/integration/fixtures/template_climate_set_actions.yaml b/tests/integration/fixtures/template_climate_set_actions.yaml new file mode 100644 index 0000000000..b247367f64 --- /dev/null +++ b/tests/integration/fixtures/template_climate_set_actions.yaml @@ -0,0 +1,89 @@ +esphome: + name: tmpl-clim-set-act +host: +api: +logger: + +# Every settable field forwards its requested value to a set_*_action. supports_two_point and +# supports_target_humidity are not declared here: they are derived from the low/high and humidity +# set actions being present. +climate: + - platform: template + id: test_climate + name: Test Set Actions + optimistic: false + restore_mode: NO_RESTORE + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + custom_fan_modes: + - turbo + custom_presets: + - eco_plus + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_low_action: + - logger.log: + format: "set_target_temperature_low_action %.1f" + args: ["x"] + set_target_temperature_high_action: + - logger.log: + format: "set_target_temperature_high_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.0f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + +button: + - platform: template + id: report_device_state + name: Report Device State + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + + - platform: template + id: report_unsupported_mode + name: Report Unsupported Mode + on_press: + - climate.template.publish: + id: test_climate + mode: DRY diff --git a/tests/integration/fixtures/template_climate_two_point_temperature.yaml b/tests/integration/fixtures/template_climate_two_point_temperature.yaml new file mode 100644 index 0000000000..ec10785ee8 --- /dev/null +++ b/tests/integration/fixtures/template_climate_two_point_temperature.yaml @@ -0,0 +1,52 @@ +esphome: + name: tmpl-clim-two-point +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Two-Point Heatpump + optimistic: true + sensor: test_climate_current_temperature + supports_two_point_target_temperature: true + supports_target_humidity: true + supported_modes: + - "OFF" + - HEAT_COOL + - HEAT + - COOL + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature_low().has_value()) + ESP_LOGD("test", "on_control target_temperature_low=%.1f", *x.get_target_temperature_low()); + if (x.get_target_temperature_high().has_value()) + ESP_LOGD("test", "on_control target_temperature_high=%.1f", *x.get_target_temperature_high()); + if (x.get_target_humidity().has_value()) + ESP_LOGD("test", "on_control target_humidity=%.1f", *x.get_target_humidity()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 21.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT_COOL + target_temperature_low: 18.0 + target_temperature_high: 24.0 + target_humidity: 50.0 diff --git a/tests/integration/test_template_climate_basic.py b/tests/integration/test_template_climate_basic.py new file mode 100644 index 0000000000..431fd4e3e8 --- /dev/null +++ b/tests/integration/test_template_climate_basic.py @@ -0,0 +1,146 @@ +"""Integration test for template climate: sensor-pushed measured values, on_control + publish +for the settable ones. + +current_temperature/current_humidity are pushed by a referenced sensor/humidity_sensor (no +polling); action is set once at boot via climate.template.publish, since it has no sensor +equivalent. mode/target_temperature/fan_mode/swing_mode/preset are plain internal state: +on_control fires exactly once per command (never before the first one), and +climate.template.publish simulates the device reporting its own state independent of any prior +command -- that report is authoritative, overriding whatever was optimistically applied earlier. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-basic" + + +@pytest.mark.asyncio +async def test_template_climate_basic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Sensor-pushed measured values, on_control + publish for settable ones.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + # Advertised capabilities come straight from the supported_*/custom_* config lists. + assert ClimateMode.OFF in test_climate.supported_modes + assert ClimateMode.HEAT in test_climate.supported_modes + assert ClimateMode.COOL in test_climate.supported_modes + + assert ClimateFanMode.AUTO in test_climate.supported_fan_modes + assert ClimateFanMode.LOW in test_climate.supported_fan_modes + assert ClimateFanMode.HIGH in test_climate.supported_fan_modes + + assert ClimateSwingMode.OFF in test_climate.supported_swing_modes + assert ClimateSwingMode.VERTICAL in test_climate.supported_swing_modes + + assert ClimatePreset.NONE in test_climate.supported_presets + assert ClimatePreset.ECO in test_climate.supported_presets + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.current_temperature == pytest.approx(22.5, abs=0.1) + assert initial.current_humidity == pytest.approx(55.0, abs=0.1) + assert initial.action == ClimateAction.IDLE + assert initial.mode == ClimateMode.OFF + # Nothing was commanded yet: on_control must not have fired. + assert not log_lines + + # Commands apply optimistically and on_control fires with the same values. + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + client.climate_command(test_climate.key, target_temperature=22.5) + state = await wait_for_climate_state() + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.HIGH) + state = await wait_for_climate_state() + assert state.fan_mode == ClimateFanMode.HIGH + + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + state = await wait_for_climate_state() + assert state.swing_mode == ClimateSwingMode.VERTICAL + + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + state = await wait_for_climate_state() + assert state.preset == ClimatePreset.ECO + + await asyncio.sleep(0.2) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + # Exactly one on_control log line per command, none extra (e.g. from a stray republish). + assert len(log_lines) == 5 + + # measured values are untouched by any of the above (no set action exists for them). + assert state.current_temperature == pytest.approx(22.5, abs=0.1) + assert state.current_humidity == pytest.approx(55.0, abs=0.1) + assert state.action == ClimateAction.IDLE + + # The device's report is authoritative and overrides everything commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.OFF + assert state.fan_mode == ClimateFanMode.AUTO + assert state.swing_mode == ClimateSwingMode.OFF + assert state.preset == ClimatePreset.NONE diff --git a/tests/integration/test_template_climate_custom_modes.py b/tests/integration/test_template_climate_custom_modes.py new file mode 100644 index 0000000000..4817fe1ddf --- /dev/null +++ b/tests/integration/test_template_climate_custom_modes.py @@ -0,0 +1,98 @@ +"""Integration test for template climate: custom fan modes and presets. + +Same on_control (forward) + climate.template.publish (device report, authoritative) pattern as +the enum-based mode/preset fields, but for the custom string variants. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-custom" + + +@pytest.mark.asyncio +async def test_template_climate_custom_modes( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Custom fan mode/preset: traits, on_control forwarding, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + assert set(test_climate.supported_custom_fan_modes) == { + "turbo", + "silent", + "eco", + } + assert set(test_climate.supported_custom_presets) == { + "eco_plus", + "power_save", + "max", + } + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.custom_fan_mode == "" + assert initial.custom_preset == "" + + client.climate_command(test_climate.key, custom_fan_mode="turbo") + state = await wait_for_climate_state() + assert state.custom_fan_mode == "turbo" + + client.climate_command(test_climate.key, custom_preset="power_save") + state = await wait_for_climate_state() + assert state.custom_preset == "power_save" + + await asyncio.sleep(0.2) + assert any("on_control custom_fan_mode=turbo" in line for line in log_lines) + assert any("on_control custom_preset=power_save" in line for line in log_lines) + + # The device's report is authoritative and overrides what was commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.custom_fan_mode == "eco" + assert state.custom_preset == "max" diff --git a/tests/integration/test_template_climate_nonoptimistic.py b/tests/integration/test_template_climate_nonoptimistic.py new file mode 100644 index 0000000000..e922ec31b9 --- /dev/null +++ b/tests/integration/test_template_climate_nonoptimistic.py @@ -0,0 +1,107 @@ +"""Integration test for template climate: optimistic: false. + +A command still fires on_control (so a real device-backed config can forward it out), but must +NOT change the entity's own state -- only an explicit climate.template.publish call (standing in +for the device confirming the command actually took effect) does that. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-nonopt" + + +@pytest.mark.asyncio +async def test_template_climate_nonoptimistic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Nonoptimistic: a command doesn't change state until explicitly published.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + state_updates: list[aioesphomeapi.ClimateState] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + confirm_button = require_entity( + entities, "simulate_device_confirmation", ButtonInfo + ) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.mode == ClimateMode.OFF + + # Send every settable field in one command. on_control must fire with all of them, but + # nothing may be applied to the entity's own state -- no ClimateState update at all. + client.climate_command( + test_climate.key, + mode=ClimateMode.HEAT, + target_temperature=22.5, + fan_mode=ClimateFanMode.HIGH, + swing_mode=ClimateSwingMode.VERTICAL, + preset=ClimatePreset.AWAY, + ) + await asyncio.sleep(0.3) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + assert not state_updates, ( + "optimistic: false must not publish a state until climate.template.publish reports it" + ) + + # The device confirms the command actually took effect. + client.button_command(confirm_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.AWAY diff --git a/tests/integration/test_template_climate_on_control_ordering.py b/tests/integration/test_template_climate_on_control_ordering.py new file mode 100644 index 0000000000..8d212b3ccb --- /dev/null +++ b/tests/integration/test_template_climate_on_control_ordering.py @@ -0,0 +1,83 @@ +"""Integration test: on_control fires before control()/on_state, with the full ClimateCall. + +on_control's lambda argument exposes get_mode()/etc. on the *requested* ClimateCall, while the +entity's own .mode field still reflects the state *before* control() applies the change -- +proving the firing order is on_control, then control(), then on_state. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-oc-order" + + +@pytest.mark.asyncio +async def test_template_climate_on_control_ordering( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """on_control sees the requested value while the entity's own state is still the old one.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line or "on_state " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + await asyncio.sleep(0.2) + + # on_control saw the new requested mode (3 == CLIMATE_MODE_HEAT) while the entity's own + # state was still the old one (0 == CLIMATE_MODE_OFF) -- proving it fired before control(). + assert any( + "on_control requested_mode=3 current_mode_before_apply=0" in line + for line in log_lines + ) + # on_state fired afterward, reporting the now-applied mode. + assert any("on_state mode=3" in line for line in log_lines) + + control_index = next( + i for i, line in enumerate(log_lines) if "on_control " in line + ) + state_index = next(i for i, line in enumerate(log_lines) if "on_state " in line) + assert control_index < state_index, "on_control must fire before on_state" diff --git a/tests/integration/test_template_climate_publish_all_fields.py b/tests/integration/test_template_climate_publish_all_fields.py new file mode 100644 index 0000000000..9c4262b311 --- /dev/null +++ b/tests/integration/test_template_climate_publish_all_fields.py @@ -0,0 +1,96 @@ +"""Integration test for template climate: climate.template.publish covering every field at once. + +A single climate.template.publish call resolves into exactly one ClimateState update, and never +triggers on_control (which would misrepresent a device state report as a fresh command). This also +exercises that a sensor/humidity_sensor whose reading matches what's about to be published doesn't +sneak in an extra state update of its own (the sensor callback only re-publishes on an actual +change). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-publish-all" + + +@pytest.mark.asyncio +async def test_template_climate_publish_all_fields( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """One climate.template.publish call setting every field resolves to one state update.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + on_control_count = 0 + + def on_log_line(line: str) -> None: + nonlocal on_control_count + if "on_control fired" in line: + on_control_count += 1 + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + publish_button = require_entity(entities, "publish_all", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.button_command(publish_button.key) + try: + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + except TimeoutError: + pytest.fail("Timeout waiting for the published climate state") + + assert state.current_temperature == pytest.approx(20.0, abs=0.1) + assert state.current_humidity == pytest.approx(60.0, abs=0.1) + assert state.target_temperature == pytest.approx(23.0, abs=0.1) + assert state.mode == ClimateMode.HEAT + assert state.action == ClimateAction.HEATING + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.ECO + + # Give any stray extra update (there shouldn't be one) a moment to arrive. + await asyncio.sleep(0.2) + assert len(state_updates) == 1, ( + f"Expected exactly one ClimateState update, got {len(state_updates)}" + ) + assert on_control_count == 0, ( + "climate.template.publish must not trigger on_control" + ) diff --git a/tests/integration/test_template_climate_sensor_push.py b/tests/integration/test_template_climate_sensor_push.py new file mode 100644 index 0000000000..1db4da81ed --- /dev/null +++ b/tests/integration/test_template_climate_sensor_push.py @@ -0,0 +1,88 @@ +"""Integration test for template climate: current_temperature/current_humidity live sensor push. + +A *later* change to a backing sensor's value -- not just its initial reading at boot -- propagates +into a new climate state via add_on_state_callback. Re-publishing the same sensor value again must +not cause a redundant climate state update. +""" + +from __future__ import annotations + +import asyncio +import math + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-sensor-push" + + +@pytest.mark.asyncio +async def test_template_climate_sensor_push( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A later change to the backing sensor pushes a new climate state; an unchanged republish does not.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + publish_temp = require_entity(entities, "publish_temperature", ButtonInfo) + publish_temp_same = require_entity( + entities, "publish_temperature_same", ButtonInfo + ) + publish_humidity = require_entity(entities, "publish_humidity", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Neither backing sensor has published anything yet. + assert math.isnan(initial.current_temperature) + assert math.isnan(initial.current_humidity) + + # A later sensor reading -- not the initial one -- pushes a new climate state. + client.button_command(publish_temp.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_temperature == pytest.approx(24.0, abs=0.1) + + client.button_command(publish_humidity.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_humidity == pytest.approx(65.0, abs=0.1) + + # Re-publishing the same temperature must not cause a redundant climate state update. + updates_before = len(state_updates) + client.button_command(publish_temp_same.key) + await asyncio.sleep(0.3) + assert len(state_updates) == updates_before, ( + "Re-publishing an unchanged sensor reading must not republish the climate state" + ) diff --git a/tests/integration/test_template_climate_set_actions.py b/tests/integration/test_template_climate_set_actions.py new file mode 100644 index 0000000000..0b1eb80874 --- /dev/null +++ b/tests/integration/test_template_climate_set_actions.py @@ -0,0 +1,114 @@ +"""Integration test: each settable field forwards its value to the matching set_*_action. + +With optimistic: false the entity state stays put until climate.template.publish reports the +device's actual state back, so the actions are the only thing that reacts to a command. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-set-act" + + +@pytest.mark.asyncio +async def test_template_climate_set_actions( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Every set_*_action fires with the requested value; state waits for a publish.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "_action " in line or "Unsupported" in line: + log_lines.append(line) + + def logged(fragment: str) -> bool: + return any(fragment in line for line in log_lines) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + report_button = require_entity(entities, "report_device_state", ButtonInfo) + unsupported_button = require_entity( + entities, "report_unsupported_mode", ButtonInfo + ) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Both traits are derived from the low/high and humidity set actions, not declared. + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + client.climate_command( + test_climate.key, target_temperature_low=18.0, target_temperature_high=24.0 + ) + client.climate_command(test_climate.key, target_humidity=55) + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.LOW) + client.climate_command(test_climate.key, custom_fan_mode="turbo") + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + client.climate_command(test_climate.key, custom_preset="eco_plus") + + for _ in range(50): + await asyncio.sleep(0.1) + if logged("set_custom_preset_action eco_plus"): + break + + assert logged("set_mode_action 3") # CLIMATE_MODE_HEAT + assert logged("set_target_temperature_low_action 18.0") + assert logged("set_target_temperature_high_action 24.0") + assert logged("set_target_humidity_action 55") + assert logged("set_fan_mode_action 3") # CLIMATE_FAN_LOW + assert logged("set_custom_fan_mode_action turbo") + assert logged("set_swing_mode_action 2") # CLIMATE_SWING_VERTICAL + assert logged("set_preset_action 5") # CLIMATE_PRESET_ECO + assert logged("set_custom_preset_action eco_plus") + + # optimistic: false, so none of the commands above touched the entity's own state -- + # a device report is what actually moves it. + client.button_command(report_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + + # A publish naming a mode outside supported_modes warns instead of publishing it. + client.button_command(unsupported_button.key) + for _ in range(50): + await asyncio.sleep(0.1) + if logged("Unsupported mode"): + break + assert logged("Unsupported mode") diff --git a/tests/integration/test_template_climate_two_point_temperature.py b/tests/integration/test_template_climate_two_point_temperature.py new file mode 100644 index 0000000000..9270b59ffc --- /dev/null +++ b/tests/integration/test_template_climate_two_point_temperature.py @@ -0,0 +1,118 @@ +"""Integration tests for template climate: two-point target temperature + humidity. + +Covers the supports_two_point_target_temperature/supports_target_humidity boolean flags plus +on_control (forwarding commands out) and climate.template.publish (the device reporting its own +authoritative state, independent of any prior command -- e.g. a device that owns its own setpoint, +changed via a physical remote). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-two-point" + + +@pytest.mark.asyncio +async def test_template_climate_two_point_temperature( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Two-point target temperature + humidity: booleans, on_control, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + test_climate = climate_infos[0] + assert test_climate.name == "Test Two-Point Heatpump" + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Nothing has been published yet: settable fields have no sensor to seed them from, so + # the entity starts at ESPHome's plain defaults. current_temperature is pushed by the + # referenced sensor, which has already settled by the time we get here. + assert initial.mode == ClimateMode.OFF + assert initial.current_temperature == pytest.approx(21.0, abs=0.1) + + # The device reports its actual state for the first time. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT_COOL + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) + + # Commands apply optimistically (settable fields are plain internal state), and on_control + # fires with the same values so a real config could forward them to the device. + client.climate_command( + test_climate.key, target_temperature_low=19.0, target_temperature_high=25.0 + ) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(19.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(25.0, abs=0.1) + await asyncio.sleep(0.2) + assert any( + "on_control target_temperature_low=19.0" in line for line in log_lines + ) + assert any( + "on_control target_temperature_high=25.0" in line for line in log_lines + ) + + client.climate_command(test_climate.key, target_humidity=45.0) + state = await wait_for_climate_state() + assert state.target_humidity == pytest.approx(45.0, abs=0.1) + await asyncio.sleep(0.2) + assert any("on_control target_humidity=45.0" in line for line in log_lines) + + # The device's next report is authoritative and overrides whatever was optimistically + # applied above -- this is the whole point of climate.template.publish: a device that owns + # its own state (e.g. changed by a physical remote) always wins. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) From 833dd0e812ecf6e413022bd23b9d4e098245d715 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 6 Sep 2026 23:59:40 +0200 Subject: [PATCH 12/53] [ota] Offer encryption with the api key so enabling it works over OTA (#18979) --- THREAT_MODEL.md | 55 ++- esphome/__main__.py | 14 +- esphome/components/api/__init__.py | 4 +- esphome/components/api/api_connection.cpp | 6 +- .../components/api/api_frame_helper_noise.cpp | 2 +- esphome/components/api/api_server.cpp | 37 +- esphome/components/api/api_server.h | 12 +- esphome/components/esphome/ota/__init__.py | 130 +++--- .../components/esphome/ota/ota_esphome.cpp | 83 ++-- esphome/components/esphome/ota/ota_esphome.h | 13 +- .../esphome/ota/ota_esphome_noise.cpp | 91 +++-- esphome/components/noise/__init__.py | 36 +- esphome/components/noise/noise.cpp | 9 + esphome/components/noise/noise.h | 16 +- esphome/components/noise/noise_handshake.cpp | 5 +- esphome/components/noise/noise_handshake.h | 6 +- esphome/core/defines.h | 3 + esphome/espota2.py | 122 +++++- esphome/wizard.py | 18 +- .../noise/test_encryption_key.py | 14 +- tests/component_tests/ota/test_esphome_ota.py | 242 +++++++++--- .../ota/test_esphome_ota_api_key_offer.yaml | 11 + ...st_esphome_ota_api_key_offer_password.yaml | 12 + .../test_esphome_ota_encryption_required.yaml | 12 + .../ota/test_esphome_ota_own_key.yaml | 11 + .../ota/test_esphome_ota_plain.yaml | 9 + .../ota/test_esphome_ota_runtime_api_key.yaml | 10 + .../components/noise/test_noise_handshake.cpp | 18 +- .../noise/test_noise_primitives.cpp | 13 +- tests/components/ota/api_key_offer.yaml | 12 + tests/components/ota/api_runtime_key.yaml | 10 + .../ota/test-api_key_offer.esp32-idf.yaml | 2 + .../ota/test-api_key_offer.esp8266-ard.yaml | 2 + .../ota/test-api_runtime_key.esp32-idf.yaml | 2 + .../ota/test-api_runtime_key.esp8266-ard.yaml | 2 + tests/integration/conftest.py | 7 + tests/integration/const.py | 7 + .../host_ota_api_key_offer_with_password.yaml | 12 + .../host_ota_provisioned_api_key.yaml | 10 + .../test_api_zero_psk_provisioning.py | 51 ++- tests/integration/test_host_ota.py | 373 ++++++++++++------ tests/unit_tests/test_espota2_noise.py | 136 ++++++- tests/unit_tests/test_main.py | 114 +++++- tests/unit_tests/test_wizard.py | 31 +- 44 files changed, 1342 insertions(+), 443 deletions(-) create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_encryption_required.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_own_key.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_plain.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml create mode 100644 tests/components/ota/api_key_offer.yaml create mode 100644 tests/components/ota/api_runtime_key.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp8266-ard.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml create mode 100644 tests/integration/fixtures/host_ota_provisioned_api_key.yaml diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index b4f557e55b..11656ff0b7 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -125,30 +125,47 @@ design is optimal or that it will not change. ## OTA update encryption The `esphome` OTA platform optionally encrypts updates with the same Noise -`NNpsk0` pattern the native API uses; one key protects the device. With an -`encryption:` block configured the guarantees are: the firmware image is -confidential in transit, the uploader is authenticated by the pre-shared key, -and the plaintext negotiation preceding the handshake is bound into the -handshake prologue, so stripping or tampering with it fails the first MAC. -Both ends fail closed with no override: a device built with a key refuses +`NNpsk0` pattern the native API uses; one key protects the device. A device +whose `api:` block has an encryption key, static in the YAML or provisioned at +runtime, compiles in the transport and offers it on every OTA connection once +it holds a key, so an uploader presenting that key gets the guarantees below +even without an `ota: encryption:` block; only that block makes the device +require encryption. The guarantees are: the firmware image is confidential in +transit, the uploader is authenticated by the pre-shared key, and the plaintext +negotiation preceding the handshake is bound into the handshake prologue, so +stripping or tampering with it fails the first MAC. With `ota: encryption:` +configured both ends fail closed with no override: the device refuses plaintext uploads, and the CLI refuses to send plaintext when a key is -configured. +configured. Without that block the CLI tries a static api key when the device +offers and, until 2027.3.0, falls back to plaintext with a warning when the +offer is missing or the handshake fails; a runtime provisioned key never +reaches the CLI, so those uploads stay plaintext. -Defeating any of that without the key is in scope: a keyed device accepting a -plaintext or downgraded upload, getting past the MAC, or recovering image -contents from captured traffic. +Defeating any of that without the key is in scope: a device that requires +encryption accepting a plaintext or downgraded upload, getting past the MAC, +or recovering image contents from captured traffic. The following are **not** vulnerabilities, by design: -- Plaintext OTA on a device with no `encryption:` block. That is the - documented default, authenticated (if at all) by the OTA password. -- The enablement window: turning encryption on takes one last upload of the - encryption-enabled firmware over the existing plaintext channel, with the - pre-existing plaintext exposure. -- The web OTA `/update` endpoint alongside encryption. The `web_server` - component keeps it always reachable, and `captive_portal:` auto-loads it - for the fallback AP window; validation warns about both combinations, and - the operator keeps the recovery path. +- Plaintext OTA on a device with no `ota: encryption:` block, including one + that offers encryption because it has an api key. That is the documented + default, authenticated (if at all) by the OTA password. An uploader that + takes the offer skips the password; the key authenticates it. With a + runtime provisioned key and no `provisioning:` window, whoever provisions + the key gains that upload path too; validation warns about the pair. +- The CLI plaintext fallback until 2027.3.0: without `ota: encryption:` an + active attacker who strips the offer or breaks the handshake can make a + keyed CLI upload plaintext, with the pre-existing plaintext exposure. A + device that requires encryption still refuses that upload. +- The enablement window: firmware built with a static api key already offers + encryption, so turning on `ota: encryption:` is itself an encrypted upload. + Older firmware needs one last plaintext upload of an offering build, with + the pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. With the `web_server` + or `prometheus` component the shared listener is always up, so the endpoint + stays reachable and validation warns about that combination; + `captive_portal:` alone brings the listener up only for the fallback AP + window, which is the intended recovery path, so that is not warned about. - CLI retry behavior on transport or MAC failures; every attempt renegotiates a fresh handshake with fresh ephemerals, so retrying does not weaken authentication. diff --git a/esphome/__main__.py b/esphome/__main__.py index b3d58ad13b..30e97f55eb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1335,12 +1335,14 @@ def _upload_via_native_api( break from esphome import espota2 + from esphome.components.noise import static_encryption_key remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) # Fail closed: an encryption block whose key did not resolve must never # fall back to a plaintext upload noise_psk = None + plaintext_fallback = False if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: noise_psk = encryption_conf.get(CONF_KEY) if not noise_psk: @@ -1351,6 +1353,10 @@ def _upload_via_native_api( # Ensure the key is a string, as required by the underlying OTA implementation. # It arrives here as a SensitiveStr which aioesphomeapi rejects. noise_psk = str(noise_psk) + elif api_key := static_encryption_key(config.get(CONF_API) or {}): + # Remove before 2027.3.0: the api key is tried, falling back to plaintext + noise_psk = str(api_key) + plaintext_fallback = True def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1382,7 +1388,13 @@ def _upload_via_native_api( _validate_bootloader_binary(binary) return espota2.run_ota( - network_devices, remote_port, password, binary, ota_type, noise_psk + network_devices, + remote_port, + password, + binary, + ota_type, + noise_psk, + plaintext_fallback=plaintext_fallback, ) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3568318dad..6202e127bf 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -14,6 +14,7 @@ from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, encryption_schema, + new_psk_progmem, validate_encryption_key, ) from esphome.config_helpers import filter_source_files_from_defines, get_logger_level @@ -589,8 +590,7 @@ async def to_code(config: ConfigType) -> None: if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None: if key := encryption_config.get(CONF_KEY): - decoded = decode_encryption_key(key) - cg.add(var.set_noise_psk(list(decoded))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) cg.add_define("USE_API_NOISE_PSK_FROM_YAML") else: # No key provided, but encryption desired diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9c609aa047..da4b7d7702 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2161,7 +2161,10 @@ void APIConnection::on_homeassistant_action_response(const HomeassistantActionRe bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg) { NoiseEncryptionSetKeyResponse resp; resp.success = false; - +#ifdef USE_API_NOISE_PSK_FROM_YAML + // A yaml key cannot be changed at runtime, so no decode or save path is built + ESP_LOGW(TAG, "Key set in YAML"); +#else #ifdef USE_PROVISIONING // Refuse to set a key once the provisioning window has closed (defense in depth; // such connections are already rejected at hello). @@ -2196,6 +2199,7 @@ bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptio } #endif } +#endif // USE_API_NOISE_PSK_FROM_YAML return this->send_message(resp); } diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 138dbdddba..29b2858aee 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -548,7 +548,7 @@ APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) { * @return 0 on success, -1 on error (check errno) */ APIError APINoiseFrameHelper::init_handshake_() { - int err = this->handshake_.init(this->ctx_.get_psk(), prologue_.data(), prologue_.size()); + int err = this->handshake_.init(this->ctx_, prologue_.data(), prologue_.size()); APIError aerr = handle_noise_error_(err, LOG_STR("noise_handshake_init"), APIError::HANDSHAKESTATE_SETUP_FAILED); if (aerr != APIError::OK) return aerr; diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 43d35363d3..78ebe5c38e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -41,13 +41,13 @@ void APIServer::setup() { ControllerRegistry::register_controller(this); #ifdef USE_API_NOISE + // Always reserve the slot: flash preferences are positional on esp8266, so + // a yaml key build must keep the layout of a runtime key build uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); - #ifndef USE_API_NOISE_PSK_FROM_YAML - // Only load saved PSK if not set from YAML - if (this->load_and_apply_noise_psk_()) { + // A cleared record loads fine but holds no key + if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { ESP_LOGD(TAG, "Loaded saved Noise PSK"); } #endif @@ -550,6 +550,7 @@ const std::vector &APIServer::get_sta #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active) { if (!this->noise_pref_.save(&new_psk)) { @@ -583,22 +584,19 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString } bool APIServer::load_and_apply_noise_psk_() { - SavedNoisePsk saved{}; - if (!this->noise_pref_.load(&saved)) + // Load into a temp so a failed read cannot disturb the key in use + SavedNoisePsk loaded{}; + if (!this->noise_pref_.load(&loaded)) return false; - this->set_noise_psk(saved.psk); + this->saved_psk_ = loaded; + // An unprovisioned device stores the reserved all-zeros key, which is no key + const bool has_key = !noise::NoiseContext::is_all_zeros(this->saved_psk_.psk); + this->noise_ctx_.set_psk(has_key ? this->saved_psk_.psk.data() : nullptr); return true; } bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else - auto &old_psk = this->noise_ctx_.get_psk(); - if (std::equal(old_psk.begin(), old_psk.end(), psk.begin())) { + if (this->saved_psk_.psk == psk) { ESP_LOGW(TAG, "New PSK matches old"); return true; } @@ -614,15 +612,8 @@ bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { } #endif return result; -#endif } bool APIServer::clear_noise_psk(bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else SavedNoisePsk empty_psk{}; bool result = this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"), make_active); @@ -634,8 +625,8 @@ bool APIServer::clear_noise_psk(bool make_active) { } #endif return result; -#endif } +#endif // USE_API_NOISE_PSK_FROM_YAML #endif #ifdef USE_HOMEASSISTANT_TIME diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 072a583901..618ea4eb11 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -76,9 +76,14 @@ class APIServer final : public Component, APIBuffer &get_shared_buffer_ref() { return shared_write_buffer_; } #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML + // Runtime key changes exist for the provisioning path only (not lambdas); + // with a yaml key they compile out bool save_noise_psk(noise::psk_t psk, bool make_active = true); bool clear_noise_psk(bool make_active = true); - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } noise::NoiseContext &get_noise_ctx() { return this->noise_ctx_; } #endif // USE_API_NOISE @@ -275,10 +280,12 @@ class APIServer final : public Component, #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active); // Load saved PSK from preferences and apply it. Returns true on success. bool load_and_apply_noise_psk_(); +#endif // USE_API_NOISE_PSK_FROM_YAML #endif // USE_API_NOISE #ifdef USE_API_HOMEASSISTANT_STATES // Helper methods to reduce code duplication @@ -358,6 +365,9 @@ class APIServer final : public Component, #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; +#ifndef USE_API_NOISE_PSK_FROM_YAML + SavedNoisePsk saved_psk_{}; // backs noise_ctx_ for a runtime provisioned key +#endif ESPPreferenceObject noise_pref_; #endif // USE_API_NOISE }; diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 1fec9e5c9b..f5eb878260 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -2,12 +2,12 @@ import logging import esphome.codegen as cg from esphome.components.noise import ( - decode_encryption_key, encryption_schema, - is_reserved_key, + new_psk_progmem, + static_encryption_key, ) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code -from esphome.config_helpers import merge_config +from esphome.config_helpers import filter_source_files_from_defines, merge_config import esphome.config_validation as cv from esphome.const import ( CONF_API, @@ -31,7 +31,6 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" -CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -41,11 +40,10 @@ DEPENDENCIES = ["network"] def AUTO_LOAD(config: ConfigType) -> list[str]: - """Auto-load noise only when encryption is configured.""" + """Auto-load noise only when encryption is configured; the api key offer + inherits it from the api component.""" base = ["sha256", "socket"] - # A falsy config is a tooling probe for the maximal set (None from - # dependency resolution, {} from the components-graph platform probe); - # a validated config always carries defaults, never empty + # A falsy config is a tooling probe for the maximal set if not config or CONF_ENCRYPTION in config: return base + ["noise"] return base @@ -132,12 +130,56 @@ def ota_esphome_final_validate(config: ConfigType) -> None: _validate_no_password_with_encryption(ota_conf) if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: _resolve_encryption_key(encryption_conf, api_conf) - if any( - conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf - ) and any( - CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + elif CONF_PASSWORD in ota_conf and static_encryption_key(api_conf) is not None: + _LOGGER.warning( + "'%s' %s wastes significant flash and RAM (about 3.5 KB and 60 " + "bytes plus the password on the heap): the device already offers " + "encryption with the '%s' %s %s, which authenticates any uploader " + "that takes it, and a password only matters for uploaders without " + "encryption support; remove '%s' and add '%s' under '%s' so " + "uploads use the key and encryption is required", + CONF_OTA, + CONF_PASSWORD, + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_PASSWORD, + CONF_ENCRYPTION, + CONF_OTA, + ) + elif ( + CONF_PASSWORD in ota_conf + and CONF_ENCRYPTION in api_conf + and not api_conf[CONF_ENCRYPTION].get(CONF_KEY) + ): + # The CLI still needs the password; whoever provisions the key skips it + _LOGGER.warning( + "The '%s' %s %s provisioned at runtime also authenticates OTA " + "uploads once provisioned; '%s' %s then only guards plaintext " + "uploads. Whoever provisions the key can upload firmware " + "without the password, so add a 'provisioning:' block to limit " + "when that is possible", + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_OTA, + CONF_PASSWORD, + ) + # web_server and prometheus keep the shared listener up; the captive + # portal's copy only exists on the fallback AP and is the recovery path + if ( + (CONF_WEB_SERVER in full_conf or "prometheus" in full_conf) + and any(conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf) + and any( + CONF_ENCRYPTION in conf + for conf in merged_ota_esphome_configs_by_port.values() + ) ): - _warn_web_server_ota(full_conf) + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -152,33 +194,11 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) -def _warn_web_server_ota(full_conf: ConfigType) -> None: - """The web_server ota platform accepts the same image over plaintext HTTP - with basic auth, bypassing the encryption; warn rather than fail so the - operator keeps the recovery path.""" - if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: - # The captive_portal auto-load: the endpoint only exists while the - # fallback AP is active - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform (auto-loaded " - "by captive_portal); the plaintext /update endpoint stays " - "reachable while the fallback AP is active", - CONF_WEB_SERVER, - ) - else: - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform; its " - "plaintext /update endpoint accepts the same image", - CONF_WEB_SERVER, - ) - - def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: """Resolve the one encryption key per device into the ota block. An explicit ota key must match the api key, a bare block inherits it, - a runtime provisioned api key cannot be inherited, and the all-zeros - provisioning sentinel is rejected (the device treats it as no key). + a runtime provisioned api key cannot be inherited. """ api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) if ota_key := encryption_conf.get(CONF_KEY): @@ -201,11 +221,6 @@ def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) - ) else: encryption_conf[CONF_KEY] = api_key - if is_reserved_key(encryption_conf[CONF_KEY]): - raise cv.Invalid( - f"The all-zeros {CONF_KEY} is reserved and provides no protection; " - f"generate a real key with: openssl rand -base64 32" - ) # Also called on merged same-port configs in final validate, where schemas @@ -267,15 +282,9 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate -def FILTER_SOURCE_FILES() -> list[str]: - """Filter out the noise transport when no ota entry configures encryption.""" - for ota_conf in CORE.config.get(CONF_OTA, []): - if ( - ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME - and ota_conf.get(CONF_ENCRYPTION) is not None - ): - return [] - return ["ota_esphome_noise.cpp"] +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} +) @coroutine_with_priority(CoroPriority.OTA_UPDATES) @@ -296,11 +305,24 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") - if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: - # A missing key was resolved from the api component in final validate. - key = encryption_conf[CONF_KEY] + # One key per device: an api encryption block supplies it (static or + # runtime) and offers; the ota block only adds the requirement + api_conf = CORE.config.get(CONF_API) or {} + encryption_conf = config.get(CONF_ENCRYPTION) + own_key = None + if encryption_conf is not None and static_encryption_key(api_conf) is None: + own_key = encryption_conf[CONF_KEY] + if own_key is not None: cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + elif CONF_ENCRYPTION in api_conf: + cg.add_define("USE_OTA_ENCRYPTION") + cg.add_define("USE_OTA_ENCRYPTION_FROM_API") + if static_encryption_key(api_conf) is None: + # The key arrives at runtime, so the offer has to look for it + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 396a47bc52..1005ed214b 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,4 +1,7 @@ #include "ota_esphome.h" +#ifdef USE_OTA_ENCRYPTION_FROM_API +#include "esphome/components/api/api_server.h" +#endif #ifdef USE_OTA #ifdef USE_OTA_PASSWORD #include "esphome/components/sha256/sha256.h" @@ -26,6 +29,16 @@ namespace esphome { static const char *const TAG = "esphome.ota"; + +#ifdef USE_OTA_ENCRYPTION +const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { +#ifdef USE_OTA_ENCRYPTION_FROM_API + return api::global_api_server->get_noise_ctx(); +#else + return this->noise_ctx_; +#endif +} +#endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -97,18 +110,30 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, "Over-The-Air updates:\n" " Address: %s:%u\n" - " Version: %d", - network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION); + " Version: %d" +#ifdef USE_OTA_ENCRYPTION + "\n Encryption: %s" +#endif + , + network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION +#ifdef USE_OTA_ENCRYPTION_REQUIRED + , + LOG_STR_LITERAL("required") +#elif defined(USE_OTA_ENCRYPTION_PROVISIONED) + // A runtime provisioned key may not exist yet + , + this->noise_context_().has_psk() ? LOG_STR_LITERAL("offered, plaintext accepted") + : LOG_STR_LITERAL("offered once the api key is provisioned") +#elif defined(USE_OTA_ENCRYPTION) + , + LOG_STR_LITERAL("offered, plaintext accepted") +#endif + ); #ifdef USE_OTA_PASSWORD if (!this->password_.empty()) { ESP_LOGCONFIG(TAG, " Password configured"); } #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { - ESP_LOGCONFIG(TAG, " Encryption configured"); - } -#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -154,10 +179,22 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +// Noise needs the extended protocol: the prologue binds the 2-byte feature ack +static constexpr uint8_t CLIENT_NOISE_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +inline bool ESPHomeOTAComponent::extended_proto_() const { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // FEATURE_READ already refused every client without the extended protocol + return true; +#else + return (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; +#endif +} + void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. /// @@ -241,12 +278,9 @@ void ESPHomeOTAComponent::handle_handshake_() { this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); -#ifdef USE_OTA_ENCRYPTION - // Fail closed: with a PSK configured the client must negotiate encryption - // (which requires the extended protocol); refuse plaintext uploads. - static constexpr uint8_t NOISE_REQUIRED_FEATURES = - CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; - if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // `ota: encryption:` requires the client to negotiate encryption + if ((this->ota_features_ & CLIENT_NOISE_FEATURES) != CLIENT_NOISE_FEATURES) { ESP_LOGW(TAG, "Client does not support encryption"); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); return; @@ -261,18 +295,21 @@ void ESPHomeOTAComponent::handle_handshake_() { // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte // legacy response. - this->extended_proto_ = (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; - if (this->extended_proto_) { + if (this->extended_proto_()) { static_assert(HANDSHAKE_BUF_SIZE >= 2, "handshake_buf_ must hold the 2-byte extended-protocol feature ack"); this->handshake_buf_[0] = ota::OTA_RESPONSE_FEATURE_FLAGS; this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // A runtime provisioned key may not exist yet + if (this->noise_context_().has_psk()) { this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; } +#elif defined(USE_OTA_ENCRYPTION) + // A yaml key always exists: validation rejects the all-zeros key + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; #endif } else { this->handshake_buf_[0] = @@ -284,15 +321,15 @@ void ESPHomeOTAComponent::handle_handshake_() { case OTAState::FEATURE_ACK: { static constexpr size_t STANDARD_PROTO_ACK_SIZE = 1; static constexpr size_t EXTENDED_PROTO_ACK_SIZE = 2; - const size_t ack_size = this->extended_proto_ ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; + const size_t ack_size = this->extended_proto_() ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } #ifdef USE_OTA_ENCRYPTION - // With a PSK configured the rest of the session runs inside the noise - // transport; the client sends the first handshake frame next, so there - // is nothing to do until data arrives. - if (this->noise_ctx_.has_psk()) { + // Latch the offer actually sent: a key activating between the two + // states must not start a session the client never expects + if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -412,7 +449,7 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge auth OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); - if (this->extended_proto_) { + if (this->extended_proto_()) { // Read ota type, 1 byte if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index fd164b8138..c6f710b3fc 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,8 +44,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#ifdef USE_OTA_ENCRYPTION - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif /// Manually set the port OTA should listen on @@ -85,9 +86,12 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; + // The api server's live context when the api has encryption, else our own + const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); + size_t noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len); bool noise_try_write_frame_(); void noise_send_reject_(const LogString *reason); ssize_t noise_decrypt_(uint8_t *buf, size_t len); @@ -144,7 +148,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION +#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#endif std::unique_ptr noise_; #endif // USE_OTA_ENCRYPTION @@ -166,6 +172,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; + // Derived from the feature byte; storing it would pad the trailing bytes + bool extended_proto_() const; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; @@ -179,7 +187,6 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint8_t auth_buf_pos_{0}; uint8_t auth_type_{0}; // Store auth type to know which hasher to use #endif // USE_OTA_PASSWORD - bool extended_proto_{false}; }; } // namespace esphome diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7f8331cf96..7401413d6d 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -3,6 +3,7 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise.h" #include "esphome/components/ota/ota_backend.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -40,24 +41,17 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // A provisioned key cleared between the offer and here is not guarded: the + // session runs on the zero key load_psk fills in and fails the client's MAC. + // Default-init: the frame buffer is written before it is read // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); - if (this->noise_ == nullptr) { - ESP_LOGW(TAG, "Session allocation failed"); - this->cleanup_connection_(); - return false; - } - + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + PROLOGUE_FEATURE_ACK_LEN]; -#ifdef USE_ESP8266 - memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#else - std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#endif + progmem_memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; // Magic bytes, already validated in MAGIC_READ std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); @@ -71,9 +65,13 @@ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; *p++ = server_feature_flags; - int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + // The caller only starts a session when the context holds a key + int err = this->noise_ == nullptr ? NOISE_ERROR_NO_MEMORY + : this->noise_->handshake.init(this->noise_context_(), prologue, sizeof(prologue)); if (err != 0) { - ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + // Raw noise codes throughout: the name table would cost flash in builds + // where only the OTA uses noise + ESP_LOGW(TAG, "Session init: %d", err); this->cleanup_connection_(); return false; } @@ -105,14 +103,16 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { s.frame_pos = 0; s.frame_len = 0; if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { - ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + ESP_LOGW(TAG, "Client rejected the handshake: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); this->cleanup_connection_(); return false; } int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); if (err != 0) { - ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); - this->noise_send_reject_(noise::reject_reason_for(err)); + // A MAC failure here almost always means the uploader has a different key + const LogString *reason = noise::reject_reason_for(err); + ESP_LOGW(TAG, "Handshake read: %s (%d)", LOG_STR_ARG(reason), err); + this->noise_send_reject_(reason); this->cleanup_connection_(); return false; } @@ -123,7 +123,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { int err = s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); if (err != 0) { - ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake write: %d", err); this->cleanup_connection_(); return false; } @@ -138,7 +138,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { int err = s.handshake.split(s.send_cipher, s.recv_cipher); if (err != 0) { - ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake split: %d", err); this->cleanup_connection_(); return false; } @@ -154,33 +154,41 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { } } +/// Payload length from a frame header, or 0 (logged) when the indicator or +/// the length is out of range. Callers pass min_len >= 1 so 0 is never valid. +size_t ESPHomeOTAComponent::noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len) { + const size_t payload_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || payload_len < min_len || payload_len > max_len) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], payload_len); + return 0; + } + return payload_len; +} + /// Non-blocking read of one handshake frame into the session buffer. bool ESPHomeOTAComponent::noise_try_read_frame_() { NoiseSession &s = *this->noise_; - while (s.frame_pos < noise::FRAME_HEADER_SIZE) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { - return false; + while (true) { + // The header first, then the body once the header says how long it is + const uint16_t want = s.frame_len == 0 ? noise::FRAME_HEADER_SIZE : s.frame_len; + if (s.frame_pos < want) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, want - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise"))) { + return false; + } + s.frame_pos += read; + continue; } - s.frame_pos += read; - } - if (s.frame_len == 0) { - const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); - if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { - ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + if (s.frame_len != 0) { + return true; + } + const size_t payload_len = this->noise_frame_payload_len_(s.frame_buf, 1, 1 + noise::MAX_HANDSHAKE_SIZE); + if (payload_len == 0) { this->cleanup_connection_(); return false; } s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; } - while (s.frame_pos < s.frame_len) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { - return false; - } - s.frame_pos += read; - } - return true; } /// Non-blocking write of the pending session-buffer frame. @@ -214,7 +222,7 @@ ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { noise_buffer_set_inout(mbuf, buf, len, len); int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Decrypt: %d", err); return -1; } return mbuf.size; @@ -229,9 +237,8 @@ ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min if (!this->readall_(header, sizeof(header))) { return -1; } - const size_t ciphertext_len = encode_uint16(header[1], header[2]); - if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { - ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + const size_t ciphertext_len = this->noise_frame_payload_len_(header, min_ciphertext, max_ciphertext); + if (ciphertext_len == 0) { return -1; } if (!this->readall_(buf, ciphertext_len)) { @@ -267,7 +274,7 @@ bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Encrypt: %d", err); return false; } noise::write_frame_header(frame, mbuf.size); diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 0f9328a482..a1d9444fc0 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -4,7 +4,9 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_KEY +from esphome.const import CONF_ENCRYPTION, CONF_KEY +from esphome.core import ID +from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] @@ -23,6 +25,14 @@ def validate_encryption_key(value: Any) -> str: if len(decoded) != 32: raise cv.Invalid("Encryption key must be base64 and 32 bytes long") + if not any(decoded): + # The device treats the all-zeros key as no key at all (it is the + # provisioning sentinel), so it must never reach a build + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"omit the {CONF_KEY} to provision it at runtime, or generate a real " + "key with: openssl rand -base64 32" + ) # Return original data for roundtrip conversion return value @@ -45,15 +55,6 @@ def decode_encryption_key(value: str) -> bytes: return decoded -def is_reserved_key(value: str) -> bool: - """Whether the key is the reserved all-zeros provisioning sentinel. - - The device treats it as no key configured, so consumers that require a - real key must reject it. - """ - return not any(decode_encryption_key(value)) - - ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), @@ -61,6 +62,21 @@ ENCRYPTION_SCHEMA = cv.Schema( ) +def static_encryption_key(conf: ConfigType) -> str | None: + """The build time key of a component config; None without one or when + the key is provisioned at runtime.""" + return (conf.get(CONF_ENCRYPTION) or {}).get(CONF_KEY) or None + + +def new_psk_progmem(parent_id: ID, key: str) -> MockObj: + """Emit the decoded key as a PROGMEM array; the component keeps a pointer + so the key never occupies RAM.""" + return cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decode_encryption_key(key)), + ) + + def encryption_schema(config: ConfigType | None) -> ConfigType: # A bare `encryption:` block is valid; a missing key means the consumer # falls back to its keyless behavior (api provisioning, ota inheriting diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 95fab322db..4806706167 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,5 +1,6 @@ #include "noise.h" #ifdef USE_NOISE +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -15,6 +16,14 @@ namespace esphome::noise { static const char *const TAG = "noise"; +void NoiseContext::load_psk(psk_t &out) const { + if (this->psk_ == nullptr) { + out.fill(0); + return; + } + progmem_memcpy(out.data(), this->psk_, out.size()); +} + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index f9da8d35b8..1033d5423c 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -23,16 +23,16 @@ class NoiseContext { } return acc == 0; } - void set_psk(psk_t psk) { - this->psk_ = psk; - this->has_psk_ = !is_all_zeros(psk); - } - const psk_t &get_psk() const { return this->psk_; } - bool has_psk() const { return this->has_psk_; } + /// psk points at 32 bytes that outlive the context (PROGMEM or caller owned + /// RAM); nullptr means no key. Runtime callers map the all-zeros key to + /// nullptr themselves; validation keeps it out of yaml. + void set_psk(const uint8_t *psk) { this->psk_ = psk; } + /// Copy the key out (flash-aware on ESP8266); all zeros when none is set. + void load_psk(psk_t &out) const; + bool has_psk() const { return this->psk_ != nullptr; } protected: - psk_t psk_{}; - bool has_psk_{false}; + const uint8_t *psk_{nullptr}; }; /// Convert a noise error code to a readable error diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 6d426de012..cc7fa603c4 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -20,7 +20,7 @@ NoiseResponderHandshake::~NoiseResponderHandshake() { } } -int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len) { +int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len) { if (this->handshake_ != nullptr) { noise_handshakestate_free(this->handshake_); this->handshake_ = nullptr; @@ -44,6 +44,9 @@ int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, siz HANDSHAKE_STEP_LOG("noise_handshakestate_new_by_id", err); return err; } + // noise-c keeps its own copy, so the key only passes through the stack here + psk_t psk; + ctx.load_psk(psk); err = noise_handshakestate_set_pre_shared_key(this->handshake_, psk.data(), psk.size()); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_set_pre_shared_key", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index 30596f35c2..bf1aa8cb7f 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -36,9 +36,9 @@ class NoiseResponderHandshake { NoiseResponderHandshake(const NoiseResponderHandshake &) = delete; NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; - /// Create and start the handshake with the given PSK and prologue. A - /// repeated call frees the previous handshake state and starts over. - [[nodiscard]] int init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len); + /// Create and start the handshake with the context's PSK and the prologue. + /// A repeated call frees the previous handshake state and starts over. + [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. [[nodiscard]] Action action() const; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 526adf74f0..9dd1e0ced6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -244,6 +244,9 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION +#define USE_OTA_ENCRYPTION_FROM_API +#define USE_OTA_ENCRYPTION_PROVISIONED +#define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ac4cbeeb7c..ce403c398d 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -202,6 +202,49 @@ class OTANetworkError(OTAError): """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" +# Remove before 2027.3.0 +class OTAEncryptionFallback(OTAError): + """The encrypted attempt failed and the caller may retry in plaintext.""" + + +# Remove before 2027.3.0 +PLAINTEXT_FALLBACK_NOTICE = ( + "A device with an api encryption key offers encryption after this " + "install; add 'encryption:' under 'ota: platform: esphome' to require it. " + "This plaintext fallback is removed in 2027.3.0." +) + + +# Remove before 2027.3.0 +class _EncryptionAttempt: + """The key an upload tries and whether it may fall back to plaintext; + a rejected handshake falls back at once, a transport fault only on repeat.""" + + def __init__(self, noise_psk: str | None, plaintext_fallback: bool) -> None: + self.noise_psk = noise_psk + self.plaintext_fallback = plaintext_fallback + self.handshake_faults = 0 + + def handshake_fault_falls_back(self) -> bool: + self.handshake_faults += 1 + return self.plaintext_fallback and self.handshake_faults >= 2 + + def downgrade(self, reason: str) -> None: + _LOGGER.warning( + "%s. Retrying in plaintext; a device that requires encryption " + "refuses it. %s", + reason, + PLAINTEXT_FALLBACK_NOTICE, + ) + self.noise_psk = None + self.plaintext_fallback = False + + +# Remove before 2027.3.0: only the fallback decision needs this distinction +class OTAHandshakeNetworkError(OTANetworkError): + """A transport failure inside the noise handshake; retrying encrypted may succeed.""" + + def _committed_error(err: OTANetworkError) -> OTAError: """Wrap a network failure that happened once the device had the full image. @@ -464,6 +507,7 @@ def perform_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> None: # Validate up front; an out-of-range value would only surface as a # ValueError deep inside send_check, bypassing OTAError handling @@ -528,19 +572,28 @@ def perform_ota( else: features = 0 - if noise_psk: - # Fail closed: never fall back to a plaintext upload when an - # encryption key is configured, an active attacker could otherwise - # strip the feature flag and capture the image (it contains the wifi - # credentials and the api encryption key). - if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if noise_psk and not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if plaintext_fallback: + # Remove before 2027.3.0: older firmware that cannot encrypt still + # gets its update on this connection + _LOGGER.warning( + "The device did not offer OTA encryption; continuing in plaintext. %s", + PLAINTEXT_FALLBACK_NOTICE, + ) + noise_psk = None + else: + # Fail closed: an attacker could otherwise strip the offer and + # capture the image (wifi credentials, api key) raise OTAError( "An OTA encryption key is configured but the device did not " "offer encryption; refusing to send the image in plaintext. " - "If the running firmware predates OTA encryption, first update " - "it without the 'ota: encryption:' block (over a trusted " - "network or via USB), then restore the block and upload again." + "The running firmware predates ESPHome 2026.9.0 or has no " + "'api: encryption: key'. With an api key, install once " + "without the 'ota: encryption:' block (that build offers " + "encryption), then restore it; otherwise flash by serial or " + "the web_server OTA platform." ) + if noise_psk: # The prologue binds every negotiation byte both sides saw, so any # tampering with the plaintext preamble breaks the handshake. prologue = ( @@ -549,8 +602,18 @@ def perform_ota( + bytes([RESPONSE_OK, version, features_to_send]) + bytes([RESPONSE_FEATURE_FLAGS, features]) ) + # Built outside the try: a local failure must never downgrade the upload sock = NoiseSocketWrapper(sock, noise_psk, prologue) - sock.do_handshake() + try: + sock.do_handshake() + except OTANetworkError as err: + # A transport fault: retry encrypted before considering plaintext + raise OTAHandshakeNetworkError(str(err)) from err + except OTAError as err: + # Remove before 2027.3.0 + if plaintext_fallback: + raise OTAEncryptionFallback(str(err)) from err + raise _LOGGER.info("Encrypted connection established") if ota_type != OTA_TYPE_UPDATE_APP: @@ -757,6 +820,7 @@ def run_ota_impl_( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -795,7 +859,9 @@ def run_ota_impl_( total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" reached_device = False - for attempt in range(total_attempts): + attempt = 0 + encryption = _EncryptionAttempt(noise_psk, plaintext_fallback) + while attempt < total_attempts: af, socktype, _, _, sa = res[attempt % len(res)] if reached_device or attempt >= len(res): _LOGGER.info( @@ -815,17 +881,40 @@ def run_ota_impl_( sock.close() _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) last_error = f"connecting to {sa[0]} failed: {err}" + attempt += 1 continue _LOGGER.info("Connected to %s", sa[0]) reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) + perform_ota( + sock, + password, + file_handle, + filename, + ota_type, + encryption.noise_psk, + encryption.plaintext_fallback, + ) + except OTAEncryptionFallback as err: + # Same address and attempt budget: not a network retry + last_error = str(err) + encryption.downgrade(last_error) + continue + except OTAHandshakeNetworkError as err: + last_error = str(err) + if encryption.handshake_fault_falls_back(): + encryption.downgrade(last_error) + continue + _LOGGER.warning("%s", last_error) + attempt += 1 + continue except OTANetworkError as err: # Transient network failure; retry last_error = str(err) _LOGGER.warning("%s", last_error) + attempt += 1 continue except OTAError as err: # Device-reported error (wrong password, wrong flash size, ...); @@ -847,10 +936,17 @@ def run_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: try: return run_ota_impl_( - remote_host, remote_port, password, filename, ota_type, noise_psk + remote_host, + remote_port, + password, + filename, + ota_type, + noise_psk, + plaintext_fallback, ) except OTAError as err: _LOGGER.error(err) diff --git a/esphome/wizard.py b/esphome/wizard.py index f7706928e9..897d5f60a1 100644 --- a/esphome/wizard.py +++ b/esphome/wizard.py @@ -148,11 +148,13 @@ def wizard_file(**kwargs: Unpack[WizardFileKwargs]) -> str: if "api_encryption_key" in kwargs: config += f' encryption:\n key: "{kwargs["api_encryption_key"]}"\n' - # Configure OTA + # The api key also secures OTA; a password only serves older uploaders config += "\nota:\n" config += " - platform: esphome\n" if "ota_password" in kwargs: config += f' password: "{kwargs["ota_password"]}"' + elif "api_encryption_key" in kwargs: + config += " encryption:" # Configuring wifi config += "\n\nwifi:\n" @@ -529,20 +531,9 @@ def wizard(path: Path) -> int: safe_print() safe_print("You'll need this key when adding the device to Home Assistant.") sleep(1) - - safe_print() - safe_print( - f"Do you want to set a {color(AnsiFore.GREEN, 'password')} for OTA updates? " - "This can be insecure if you do not trust the WiFi network." - ) - safe_print() - sleep(0.25) - safe_print("Press ENTER for no password") - ota_password = safe_input(color(AnsiFore.BOLD_WHITE, "(password): ")) else: ssid, psk = "", "" api_encryption_key = None - ota_password = "" kwargs = { "path": path, @@ -553,10 +544,9 @@ def wizard(path: Path) -> int: "psk": psk, "type": "basic", } + # The api key also secures OTA updates, so the wizard sets no OTA password if api_encryption_key: kwargs["api_encryption_key"] = api_encryption_key - if ota_password: - kwargs["ota_password"] = ota_password if not wizard_write(**kwargs): return 1 diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 10f1eb3d4c..2b79bd5464 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,11 +5,7 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import ( - decode_encryption_key, - is_reserved_key, - validate_encryption_key, -) +from esphome.components.noise import decode_encryption_key, validate_encryption_key KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -41,6 +37,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: decode_encryption_key("AAECAw==") -def test_is_reserved_key() -> None: - assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") - assert not is_reserved_key(KEY) +def test_validate_encryption_key_rejects_all_zeros() -> None: + """The all-zeros key is the provisioning sentinel the device treats as no + key, so it never reaches a build.""" + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + validate_encryption_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 873f162555..d3092294dc 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable import logging from typing import Any @@ -14,6 +15,7 @@ from esphome.components.esphome.ota import ( _validate_no_password_with_encryption, ota_esphome_final_validate, ) +from esphome.components.noise import static_encryption_key from esphome.const import ( CONF_API, CONF_ENCRYPTION, @@ -115,7 +117,6 @@ def test_non_esphome_ota_unaffected() -> None: API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" -ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" def test_encryption_key_inherited_from_api() -> None: @@ -197,36 +198,6 @@ def test_encryption_without_any_key_rejected() -> None: fv.full_config.reset(token) -def test_encryption_explicit_all_zeros_key_rejected() -> None: - """The all-zeros key is the provisioning sentinel; the device would treat - it as no PSK and accept plaintext, so it must fail validation.""" - full_conf = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) - ], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - -def test_encryption_inherited_all_zeros_key_rejected() -> None: - """An all-zeros api key must not silently disable ota encryption either.""" - full_conf = { - CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, - CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: """Same-port configs with different encryption keys raise.""" full_conf = { @@ -295,13 +266,14 @@ def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None fv.full_config.reset(token) +@pytest.mark.parametrize("component", ["web_server", "prometheus"]) def test_encryption_with_web_server_ota_warns( - caplog: pytest.LogCaptureFixture, + caplog: pytest.LogCaptureFixture, component: str ) -> None: - """With the web_server component the plaintext /update endpoint is always - on; the combination validates with a warning.""" + """web_server and prometheus keep the shared listener up, so the + plaintext /update endpoint is always on and the combination warns.""" full_conf = { - "web_server": {}, + component: {}, CONF_OTA: [ _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, @@ -316,12 +288,12 @@ def test_encryption_with_web_server_ota_warns( fv.full_config.reset(token) -def test_encryption_with_captive_portal_web_server_ota_warns( +def test_encryption_with_captive_portal_does_not_warn( caplog: pytest.LogCaptureFixture, ) -> None: """captive_portal auto-loads the web_server ota platform without the - web_server component; encryption stays usable and only warns, so the - fallback AP recovery path is not lost.""" + web_server component; its endpoint only exists while the fallback AP is + active and is the intended recovery path, so there is no warning.""" full_conf = { "captive_portal": {}, CONF_OTA: [ @@ -333,7 +305,10 @@ def test_encryption_with_captive_portal_web_server_ota_warns( try: with caplog.at_level(logging.WARNING): ota_esphome_final_validate({}) - assert any("captive_portal" in record.message for record in caplog.records) + assert not any( + "OTA encryption does not cover" in record.message + for record in caplog.records + ) esphome_conf = next( conf for conf in fv.full_config.get()[CONF_OTA] @@ -344,6 +319,100 @@ def test_encryption_with_captive_portal_web_server_ota_warns( fv.full_config.reset(token) +def test_password_with_api_key_warns(caplog: pytest.LogCaptureFixture) -> None: + """A static api key makes the device offer encryption and the CLI take + it, so the password is dead weight; the config validates with a warning.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("wastes significant flash" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_password_with_runtime_api_key_warns_differently( + caplog: pytest.LogCaptureFixture, +) -> None: + """The CLI still needs the password, but the provisioned key also + authenticates uploads; the warning says so without the flash advice.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + messages = [r.message for r in caplog.records] + assert any("provisioned at runtime also authenticates" in m for m in messages) + assert not any("wastes significant flash" in m for m in messages) + finally: + fv.full_config.reset(token) + + +def test_password_without_api_key_no_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """Without an api key there is no offer, so nothing to warn about.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("authenticates" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_web_server_component_without_ota_platform_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """The web_server component alone has no /update endpoint.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any( + "OTA encryption does not cover" in r.message for r in caplog.records + ) + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_platform_alone_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the web_server component starts the shared listener, so the ota + platform on its own never exposes /update.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("plaintext /update" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + def test_web_server_ota_without_encryption_unaffected() -> None: """web_server ota stays valid alongside an unencrypted esphome entry.""" full_conf = { @@ -370,20 +439,87 @@ def test_auto_load_pulls_noise_only_for_encryption() -> None: assert "noise" in AUTO_LOAD({}) -def test_filter_source_files_excludes_noise_without_encryption() -> None: - """The noise transport source compiles only for encrypted builds.""" - old_config = CORE.config - try: - CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} - assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] - CORE.config = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) - ] - } - assert FILTER_SOURCE_FILES() == [] - finally: - CORE.config = old_config +def test_static_encryption_key() -> None: + """Only a build-time key counts; a runtime provisioned one does not.""" + assert static_encryption_key({}) is None + assert static_encryption_key({CONF_ENCRYPTION: {}}) is None + assert static_encryption_key({CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) == API_KEY + + +@pytest.mark.parametrize( + ("yaml_name", "defines_present", "defines_absent"), + [ + # An api key alone compiles the transport in without requiring it; + # the device uses the api server's key, not a copy + ( + "api_key_offer", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A password still guards plaintext uploads on an offering device + ( + "api_key_offer_password", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # The ota encryption block is what makes the device refuse plaintext + ( + "encryption_required", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + }, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # Without api encryption the ota key is the device's own + ( + "own_key", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, + {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A key provisioned at runtime lives in the api server; the device + # offers with it once provisioned and never requires it + ( + "runtime_api_key", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + {"USE_OTA_ENCRYPTION_REQUIRED"}, + ), + # No api encryption at all keeps the noise glue out of the build + ( + "plain", + set(), + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + ), + ], +) +def test_encryption_offer_codegen( + generate_main: Callable[[str], str], + yaml_name: str, + defines_present: set[str], + defines_absent: set[str], +) -> None: + main_cpp = generate_main( + f"tests/component_tests/ota/test_esphome_ota_{yaml_name}.yaml" + ) + defines = {define.name for define in CORE.defines} + assert defines_present <= defines + assert not (defines_absent & defines) + encrypted = "USE_OTA_ENCRYPTION" in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) + # The noise transport source compiles only when the define is set + assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) def test_password_with_encryption_rejected() -> None: diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml new file mode 100644 index 0000000000..ca26eb9f46 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-offer + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml new file mode 100644 index 0000000000..1e23975690 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-offer-password + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml new file mode 100644 index 0000000000..36690038d8 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-encryption-required + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + encryption: diff --git a/tests/component_tests/ota/test_esphome_ota_own_key.yaml b/tests/component_tests/ota/test_esphome_ota_own_key.yaml new file mode 100644 index 0000000000..b6d1e4200d --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_own_key.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-own-key + +host: + +api: + +ota: + - platform: esphome + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/component_tests/ota/test_esphome_ota_plain.yaml b/tests/component_tests/ota/test_esphome_ota_plain.yaml new file mode 100644 index 0000000000..c5ca7afcf0 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_plain.yaml @@ -0,0 +1,9 @@ +esphome: + name: ota-plain + +host: + +api: + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml new file mode 100644 index 0000000000..8825335141 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: ota-runtime-key + +host: + +api: + encryption: + +ota: + - platform: esphome diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index d879a26c43..f2081f2965 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -68,6 +68,14 @@ class Initiator { static const uint8_t PROLOGUE[] = {'t', 'e', 's', 't', 'p', 'r', 'o', 'l', 'o', 'g', 'u', 'e'}; +// The context only points at the key and init() copies it before returning, +// so a temporary context over a temporary key is safe within one call +static NoiseContext ctx_for(const psk_t &psk) { + NoiseContext ctx; + ctx.set_psk(psk.data()); + return ctx; +} + static psk_t make_psk(uint8_t seed) { psk_t psk; for (size_t i = 0; i < psk.size(); i++) { @@ -102,7 +110,7 @@ TEST(NoiseResponderHandshakeTest, MessageMethodsErrorBeforeInit) { TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); @@ -155,8 +163,8 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // proves the restart took effect; the old state surviving would fail the // MAC here. NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); - ASSERT_EQ(responder.init(make_psk(9), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(9)), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(make_psk(9), PROLOGUE, sizeof(PROLOGUE)); @@ -168,7 +176,7 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { TEST(NoiseResponderHandshakeTest, WrongPskFailsWithMacFailure) { NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); Initiator initiator(make_psk(200), PROLOGUE, sizeof(PROLOGUE)); uint8_t msg[MAX_HANDSHAKE_SIZE]; @@ -185,7 +193,7 @@ TEST(NoiseResponderHandshakeTest, MismatchedPrologueFailsWithMacFailure) { // tampered preamble must fail even with the right key. const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); static const uint8_t TAMPERED[] = {'x'}; Initiator initiator(psk, TAMPERED, sizeof(TAMPERED)); diff --git a/tests/components/noise/test_noise_primitives.cpp b/tests/components/noise/test_noise_primitives.cpp index 018be9f717..8687c4b963 100644 --- a/tests/components/noise/test_noise_primitives.cpp +++ b/tests/components/noise/test_noise_primitives.cpp @@ -17,12 +17,17 @@ TEST(NoiseContextTest, AllZerosPskIsReserved) { EXPECT_FALSE(NoiseContext::is_all_zeros(psk)); NoiseContext ctx; + psk_t loaded; EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(zeros); - EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, zeros); + ctx.set_psk(psk.data()); EXPECT_TRUE(ctx.has_psk()); - EXPECT_EQ(ctx.get_psk(), psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, psk); + // Callers map the reserved key to nullptr; the context just stores what it is given + ctx.set_psk(nullptr); + EXPECT_FALSE(ctx.has_psk()); } TEST(WireFormatTest, FrameHeaderIsIndicatorPlusBigEndianLength) { diff --git a/tests/components/ota/api_key_offer.yaml b/tests/components/ota/api_key_offer.yaml new file mode 100644 index 0000000000..8d1814bf7e --- /dev/null +++ b/tests/components/ota/api_key_offer.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3290 + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/components/ota/api_runtime_key.yaml b/tests/components/ota/api_runtime_key.yaml new file mode 100644 index 0000000000..8976c92f96 --- /dev/null +++ b/tests/components/ota/api_runtime_key.yaml @@ -0,0 +1,10 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + +ota: + - platform: esphome + port: 3291 diff --git a/tests/components/ota/test-api_key_offer.esp32-idf.yaml b/tests/components/ota/test-api_key_offer.esp32-idf.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_key_offer.esp8266-ard.yaml b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp32-idf.yaml b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6777e6cabc..15c5860879 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -162,6 +162,13 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) +@pytest.fixture +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Host preferences persist per device name; give the test its own so a + provisioned key never leaks into another run.""" + monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) + + @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" diff --git a/tests/integration/const.py b/tests/integration/const.py index 6876bbd443..e35d4673af 100644 --- a/tests/integration/const.py +++ b/tests/integration/const.py @@ -9,6 +9,13 @@ API_CONNECTION_TIMEOUT = 30.0 # seconds PORT_WAIT_TIMEOUT = 30.0 # seconds PORT_POLL_INTERVAL = 0.1 # seconds +# The well-known all-zeros provisioning PSK, a key to provision over it, and +# the time the device takes to activate a newly saved key (100 ms timer plus +# margin) +ZERO_PSK = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +PROVISIONING_PSK = b"bm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm4=" +KEY_ACTIVATION_DELAY = 0.5 # seconds + # Process shutdown timeouts SIGINT_TIMEOUT = 5.0 # seconds SIGTERM_TIMEOUT = 2.0 # seconds diff --git a/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml new file mode 100644 index 0000000000..1dedcc9ee1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + password: "hunter2" +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml new file mode 100644 index 0000000000..aa0a9a66c9 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index bcea2a2471..f315335d1b 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -10,34 +10,40 @@ from __future__ import annotations import asyncio import base64 +import socket from aioesphomeapi import InvalidEncryptionKeyAPIError, RequiresEncryptionAPIError import pytest -from .types import APIClientConnectedFactory, RunCompiledFunction +from .conftest import run_binary_and_wait_for_port +from .const import KEY_ACTIVATION_DELAY, LOCALHOST, PROVISIONING_PSK, ZERO_PSK +from .types import ( + APIClientConnectedFactory, + CompileFunction, + ConfigWriter, + RunCompiledFunction, +) -# The well-known provisioning PSK: base64 of 32 zero bytes -ZERO_PSK = base64.b64encode(bytes(32)).decode() -# A real key to provision -NEW_KEY = base64.b64encode(b"n" * 32) -# Time for the device to activate a newly saved key (100ms timer plus margin) -KEY_ACTIVATION_DELAY = 0.5 - - -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so every run starts unprovisioned.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) +pytestmark = pytest.mark.usefixtures("isolated_preferences") +NEW_KEY = PROVISIONING_PSK @pytest.mark.asyncio async def test_api_zero_psk_provisioning( yaml_config: str, - run_compiled: RunCompiledFunction, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], api_client_connected: APIClientConnectedFactory, ) -> None: - """Exercise the reject paths, then provision a key over the zero-PSK channel.""" - async with run_compiled(yaml_config): + """Exercise the reject paths, provision a key over the zero-PSK channel, + and check the key comes back from preferences on the next boot.""" + port, port_socket = reserved_tcp_port + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + port_socket.close() + + async with run_binary_and_wait_for_port(binary_path, LOCALHOST, port): # --- Pre-provisioning reject paths (device state is unchanged) --- # A wrong (non-zero) PSK fails against the zero provisioning PSK @@ -97,6 +103,19 @@ async def test_api_zero_psk_provisioning( async with api_client_connected(timeout=5) as client: await client.device_info() + # The key is loaded from preferences on the next boot + lines: list[str] = [] + async with run_binary_and_wait_for_port( + binary_path, LOCALHOST, port, line_callback=lines.append + ): + async with api_client_connected(noise_psk=NEW_KEY.decode()) as client: + device_info = await client.device_info() + assert device_info.api_encryption_provisionable is False + with pytest.raises(InvalidEncryptionKeyAPIError): + async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client: + await client.device_info() + assert any("Loaded saved Noise PSK" in line for line in lines) + @pytest.mark.asyncio async def test_api_zero_psk_provisioning_plaintext( diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 4e74814534..f8c122c6e1 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -8,9 +8,12 @@ instance covers the FD_CLOEXEC path. from __future__ import annotations import asyncio +import base64 from collections.abc import Generator from contextlib import contextmanager +from dataclasses import dataclass import functools +from pathlib import Path import socket import pytest @@ -18,10 +21,18 @@ import pytest from esphome import espota2 from .conftest import run_binary, wait_and_connect_api_client -from .const import LOCALHOST, PORT_POLL_INTERVAL, PORT_WAIT_TIMEOUT -from .types import CompileFunction, ConfigWriter +from .const import ( + KEY_ACTIVATION_DELAY, + LOCALHOST, + PORT_POLL_INTERVAL, + PORT_WAIT_TIMEOUT, + PROVISIONING_PSK, + ZERO_PSK, +) +from .types import APIClientConnectedFactory, CompileFunction, ConfigWriter DEVICE_NAME = "host-ota-test" +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @contextmanager @@ -35,6 +46,14 @@ def _reserve_port() -> Generator[tuple[int, socket.socket]]: s.close() +async def _wait_for_line(lines: list[str], needle: str, timeout: float = 5.0) -> None: + """The config dump prints after every setup, a little after the api port + opens, so wait for it rather than assert on the lines seen so far.""" + async with asyncio.timeout(timeout): + while not any(needle in line for line in lines): + await asyncio.sleep(PORT_POLL_INTERVAL) + + async def _wait_for_port(host: str, port: int, timeout: float) -> None: """Poll until a TCP port accepts connections, or raise TimeoutError.""" loop = asyncio.get_running_loop() @@ -51,6 +70,102 @@ async def _wait_for_port(host: str, port: int, timeout: float) -> None: raise TimeoutError(f"Port {port} on {host} did not open within {timeout}s") +async def _build( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> tuple[int, int, Path]: + """Reserve an OTA port, compile the fixture with it, and release both + ports right before the binary is started.""" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + config_path = await write_yaml_config( + yaml_config.replace("__OTA_PORT__", str(ota_port)) + ) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + return api_port, ota_port, binary_path + + +async def _run_ota( + ota_port: int, + password: str | None, + binary_path: Path, + noise_psk: str | None, + plaintext_fallback: bool = False, +) -> int: + """espota2 is blocking; run it in the executor and return its exit code.""" + rc, _ = await asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + password, + binary_path, + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, + ), + ) + return rc + + +@dataclass +class _Device: + """A running host binary and the checks every successful OTA repeats: + a safe reboot, the api port back up, and the pid preserved by execv.""" + + api_port: int + ota_port: int + binary_path: Path + proc: asyncio.subprocess.Process | None = None + reboots: int = 0 + + def __post_init__(self) -> None: + self._rebooted = asyncio.Event() + + def on_log(self, line: str) -> None: + if "Rebooting safely" in line: + self.reboots += 1 + self._rebooted.set() + + async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: + async with asyncio.timeout(timeout): + while self.reboots < count: + self._rebooted.clear() + await self._rebooted.wait() + + async def ota( + self, + password: str | None, + noise_psk: str | None, + msg: str, + plaintext_fallback: bool = False, + ) -> None: + """Upload, then expect the re-exec with the pid preserved.""" + pid_before = self.proc.pid + expected_reboots = self.reboots + 1 + rc = await _run_ota( + self.ota_port, password, self.binary_path, noise_psk, plaintext_fallback + ) + assert rc == 0, msg + await self.wait_reboot(expected_reboots) + await _wait_for_port(LOCALHOST, self.api_port, PORT_WAIT_TIMEOUT) + assert self.proc.returncode is None, "process exited instead of execing" + assert self.proc.pid == pid_before + + async def refused_ota( + self, password: str | None, noise_psk: str | None, msg: str + ) -> None: + """Upload must fail and the device must keep running.""" + rc = await _run_ota(self.ota_port, password, self.binary_path, noise_psk) + assert rc == 1, msg + await asyncio.sleep(0.5) + assert self.proc.returncode is None, "process died on rejected OTA" + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -59,57 +174,34 @@ async def test_host_ota_self_update( reserved_tcp_port: tuple[int, socket.socket], ) -> None: """Self-OTA: upload the running binary back to itself, expect re-exec.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + staged = asyncio.Event() - loop = asyncio.get_running_loop() - ota_staged = loop.create_future() - rebooted = loop.create_future() + def on_log(line: str) -> None: + if "OTA staged at" in line: + staged.set() + dev.on_log(line) - def on_log(line: str) -> None: - if not ota_staged.done() and "OTA staged at" in line: - ota_staged.set_result(True) - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_before = await client.device_info() + assert info_before.name == DEVICE_NAME - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - async with wait_and_connect_api_client(port=api_port) as client: - info_before = await client.device_info() - assert info_before.name == DEVICE_NAME + await dev.ota(None, None, "espota2 reported failure") + assert staged.is_set() - # espota2 is blocking; run in executor. - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "espota2 reported failure" + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_after = await client.device_info() + assert info_after.name == info_before.name - await asyncio.wait_for(ota_staged, timeout=10.0) - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - - # execv preserves pid; mismatch means external respawn. - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info_after = await client.device_info() - assert info_after.name == DEVICE_NAME - assert info_after.name == info_before.name - - # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "second OTA failed -- listener leaked across execv" - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.pid == pid_before + # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). + await dev.ota(None, None, "second OTA failed -- listener leaked across execv") @pytest.mark.asyncio @@ -121,51 +213,110 @@ async def test_host_ota_encrypted( ) -> None: """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" pytest.importorskip("aioesphomeapi.noise") - noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await dev.refused_ota( + None, None, "plaintext upload to an encrypted device must fail" + ) + await dev.ota(None, API_KEY, "encrypted OTA reported failure") - loop = asyncio.get_running_loop() - rebooted = loop.create_future() - def on_log(line: str) -> None: - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) +@pytest.mark.asyncio +async def test_host_ota_api_key_offer_with_password( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + caplog: pytest.LogCaptureFixture, +) -> None: + """With only an api key the device offers encryption without requiring + it: the password still guards plaintext uploads, the key alone + authenticates an encrypted one, and until 2027.3.0 a failed encrypted + attempt falls back to plaintext.""" + pytest.importorskip("aioesphomeapi.noise") + wrong_key = base64.b64encode(b"w" * 32).decode() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "Encryption: offered") - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid + await dev.refused_ota( + None, None, "plaintext upload without the password must fail" + ) + await dev.ota( + "hunter2", None, "plaintext upload with the password must succeed" + ) + await dev.ota(None, API_KEY, "encrypted upload with the api key must succeed") - # A plaintext upload must be refused with the device unharmed - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + # Remove before 2027.3.0: a wrong key falls back to plaintext, which + # the password still guards + with caplog.at_level("WARNING", logger="esphome.espota2"): + await dev.ota( + "hunter2", + wrong_key, + "the plaintext retry with the password must succeed", + plaintext_fallback=True, ) - assert rc == 1, "plaintext upload to an encrypted device must fail" - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected plaintext OTA" + assert any("Retrying in plaintext" in r.message for r in caplog.records) + await dev.ota( + None, + API_KEY, + "the right api key encrypts without touching the fallback", + plaintext_fallback=True, + ) - # The encrypted upload goes through and the device re-execs - rc, _ = await loop.run_in_executor( - None, - functools.partial( - espota2.run_ota, - LOCALHOST, - ota_port, - None, - binary_path, - noise_psk=noise_psk, - ), - ) - assert rc == 0, "encrypted OTA reported failure" - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_preferences") +async def test_host_ota_provisioned_api_key( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """A key provisioned over the api feeds the OTA offer: plaintext works + while unprovisioned, the provisioned key encrypts, the key loaded from + preferences on the next boot keeps encrypting, and plaintext stays + accepted because only the ota block requires encryption.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "once the api key is provisioned") + + await dev.ota( + None, None, "plaintext upload to an unprovisioned device must succeed" + ) + + async with api_client_connected( + port=dev.api_port, noise_psk=ZERO_PSK + ) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + key = PROVISIONING_PSK.decode() + await dev.ota( + None, key, "encrypted upload with the provisioned key must succeed" + ) + await dev.ota(None, key, "the key loaded at boot must feed the OTA offer") + await dev.ota(None, None, "plaintext must stay accepted on an offering device") @pytest.mark.asyncio @@ -177,33 +328,25 @@ async def test_host_ota_rejects_garbage( integration_test_dir, ) -> None: """Bogus payload is rejected and the device keeps running.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # 192 bytes that are neither ELF nor Mach-O. + bogus_path = integration_test_dir / "bogus.bin" + bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) - # 192 bytes that are neither ELF nor Mach-O. - bogus_path = integration_test_dir / "bogus.bin" - bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) + async with run_binary(dev.binary_path) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + rc = await _run_ota(dev.ota_port, None, bogus_path, None) + assert rc == 1 + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected OTA" + assert proc.pid == pid_before - api_socket.close() - ota_socket.close() - - async with run_binary(binary_path) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - - loop = asyncio.get_running_loop() - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, bogus_path - ) - assert rc == 1 - - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected OTA" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info = await client.device_info() - assert info.name == DEVICE_NAME + async with wait_and_connect_api_client(port=dev.api_port) as client: + info = await client.device_info() + assert info.name == DEVICE_NAME diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py index 5b43d05530..439220f09c 100644 --- a/tests/unit_tests/test_espota2_noise.py +++ b/tests/unit_tests/test_espota2_noise.py @@ -10,12 +10,15 @@ when the installed aioesphomeapi predates the noise module. from __future__ import annotations import base64 +from collections.abc import Callable import hashlib import io +import logging from pathlib import Path import socket import sys import threading +from typing import Any from unittest.mock import Mock, patch import pytest @@ -65,8 +68,12 @@ class FakeEncryptedDevice(threading.Thread): offer_noise: bool = True, require_noise: bool = True, prologue_features_override: int | None = None, + connections: int = 1, + drop_handshakes: int = 0, ) -> None: super().__init__(daemon=True) + self.connections = connections + self.drop_handshakes = drop_handshakes # hang up mid-handshake this many times self.psk = psk self.version = version self.offer_noise = offer_noise @@ -81,10 +88,11 @@ class FakeEncryptedDevice(threading.Thread): def run(self) -> None: try: - sock, _ = self.listener.accept() - sock.settimeout(10) - with sock: - self._serve(sock) + for _ in range(self.connections): + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) except Exception as err: # noqa: BLE001 - surfaced via join_and_check self.error = err finally: @@ -109,8 +117,23 @@ class FakeEncryptedDevice(threading.Thread): return server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) - if not (self.offer_noise and noise_negotiated): - return # the client fails closed; nothing further arrives + if not (noise_negotiated and self.offer_noise): + # A device that does not require encryption continues in + # plaintext whatever the client asked for, like older firmware + try: + self._transfer( + lambda byte: sock.sendall(bytes([byte])), + lambda length: _recv_exact(sock, length), + lambda remaining: _recv_exact( + sock, min(remaining, espota2.UPLOAD_BLOCK_SIZE) + ), + ) + except ConnectionError: + # A keyed client without fallback fails closed and hangs up + if noise_negotiated and not self.offer_noise: + return + raise + return from cryptography.exceptions import InvalidTag from noise.connection import NoiseConnection @@ -134,6 +157,9 @@ class FakeEncryptedDevice(threading.Thread): msg1 = _recv_frame(sock) assert msg1[0] == 0x00 + if self.drop_handshakes > 0: + self.drop_handshakes -= 1 + return # a transport fault: the socket closes with no reply try: proto.read_message(msg1[1:]) except InvalidTag: @@ -149,6 +175,20 @@ class FakeEncryptedDevice(threading.Thread): assert len(plaintext) == length, "control units must be one per frame" return plaintext + def recv_data(_remaining: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + return plaintext + + self._transfer(send_byte, recv_unit, recv_data) + + def _transfer( + self, + send_byte: Callable[[int], None], + recv_unit: Callable[[int], bytes], + recv_data: Callable[[int], bytes], + ) -> None: + """The post-handshake exchange, identical over both transports.""" send_byte(espota2.RESPONSE_AUTH_OK) recv_unit(1) # ota type size = int.from_bytes(recv_unit(4), "big") @@ -159,9 +199,7 @@ class FakeEncryptedDevice(threading.Thread): received = b"" acked = 0 while len(received) < size: - plaintext = proto.decrypt(_recv_frame(sock)) - assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT - received += plaintext + received += recv_data(size - len(received)) if self.version >= espota2.OTA_VERSION_2_0: while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( len(received) == size and acked < size @@ -176,7 +214,10 @@ class FakeEncryptedDevice(threading.Thread): def _upload( - device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None + device: FakeEncryptedDevice, + firmware: bytes, + noise_psk: str | None, + plaintext_fallback: bool = False, ) -> None: device.start() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -184,12 +225,35 @@ def _upload( sock.connect(("127.0.0.1", device.port)) try: espota2.perform_ota( - sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + sock, + None, + io.BytesIO(firmware), + Path("firmware.bin"), + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, ) finally: sock.close() +def _run_ota( + device: FakeEncryptedDevice, firmware: bytes, tmp_path: Path, noise_psk: str +) -> int: + """Drive the retry loop, which is where the plaintext fallback reconnects.""" + path = tmp_path / "firmware.bin" + path.write_bytes(firmware) + device.start() + rc, _ = espota2.run_ota( + "127.0.0.1", + device.port, + None, + path, + noise_psk=noise_psk, + plaintext_fallback=True, + ) + return rc + + def test_encrypted_upload_success() -> None: """A full encrypted v2 upload spanning several 8192-byte blocks.""" pytest.importorskip("aioesphomeapi.noise") @@ -240,6 +304,56 @@ def test_client_fails_closed_when_device_lacks_encryption() -> None: device.join_and_check() +# Remove before 2027.3.0 +def test_fallback_when_device_does_not_offer(caplog: pytest.LogCaptureFixture) -> None: + """The api key is tried opportunistically; an older device that cannot + encrypt still gets its update, with a warning.""" + firmware = b"firmware" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + _upload(device, firmware, PSK, plaintext_fallback=True) + device.join_and_check() + assert device.received == firmware + assert any("fallback is removed in 2027.3.0" in r.message for r in caplog.records) + + +# Remove before 2027.3.0 +@pytest.mark.parametrize( + ("device_kwargs", "expected_rc", "fell_back"), + [ + # A wrong key against an offering device reconnects in plaintext + ({"psk": OTHER_PSK, "require_noise": False, "connections": 2}, 0, True), + # The plaintext retry is refused by a device that requires encryption + ({"psk": OTHER_PSK, "require_noise": True, "connections": 2}, 1, True), + # A dropped connection inside the handshake is retried encrypted + ({"require_noise": False, "connections": 2, "drop_handshakes": 1}, 0, False), + # A second transport fault inside the handshake falls back + ({"require_noise": False, "connections": 3, "drop_handshakes": 2}, 0, True), + ], + ids=["wrong_key", "wrong_key_required", "one_fault", "two_faults"], +) +def test_fallback_through_the_retry_loop( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, + device_kwargs: dict[str, Any], + expected_rc: int, + fell_back: bool, +) -> None: + pytest.importorskip("aioesphomeapi.noise") + firmware = b"firmware" + device = FakeEncryptedDevice(**device_kwargs) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + rc = _run_ota(device, firmware, tmp_path, PSK) + device.join_and_check() + assert rc == expected_rc + assert (device.received == firmware) is (expected_rc == 0) + assert ( + any("Retrying in plaintext" in r.message for r in caplog.records) is fell_back + ) + if expected_rc == 1: + assert any("requires an encrypted OTA" in r.message for r in caplog.records) + + def test_plaintext_client_gets_encryption_required_error() -> None: """A client without a key gets the device's 0x94 error message.""" device = FakeEncryptedDevice() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 5372a7203d..8fb9b7376e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -2108,7 +2108,13 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + "secret", + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2140,10 +2146,77 @@ def test_upload_program_ota_encryption_key( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=False, ) +def test_upload_program_ota_api_key_opportunistic( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Without an ota encryption block the api key is tried with a plaintext + fallback (removed in 2027.3.0).""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: key}}, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=True, + ) + + +@pytest.mark.parametrize( + "api_conf", + [{}, {CONF_ENCRYPTION: {}}], + ids=["no_encryption", "runtime_key"], +) +def test_upload_program_ota_no_usable_api_key_stays_plaintext( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + api_conf: dict[str, Any], +) -> None: + """A missing or runtime provisioned api key gives the uploader nothing + to try.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_API: api_conf, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert mock_run_ota.call_args.args[5] is None + assert mock_run_ota.call_args.kwargs == {"plaintext_fallback": False} + + def test_upload_program_ota_encryption_without_key_fails_closed( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2194,7 +2267,13 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + Path("custom.bin"), + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2250,6 +2329,7 @@ def test_upload_program_ota_partition_table_with_file_arg( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2312,6 +2392,7 @@ def test_upload_program_ota_partition_table_mqttip( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2500,6 +2581,7 @@ def test_upload_program_ota_bootloader_with_file_arg( bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, None, + plaintext_fallback=False, ) @@ -2988,7 +3070,13 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -3038,7 +3126,13 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.50"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5211,6 +5305,7 @@ def test_upload_program_ota_static_ip_with_mqttip( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5261,6 +5356,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5438,7 +5534,13 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) diff --git a/tests/unit_tests/test_wizard.py b/tests/unit_tests/test_wizard.py index 244e4eb5a1..f57ae71ae6 100644 --- a/tests/unit_tests/test_wizard.py +++ b/tests/unit_tests/test_wizard.py @@ -37,7 +37,6 @@ def wizard_answers() -> list[str]: "nodemcuv2", # board "SSID", # ssid "psk", # wifi password - "", # ota password (empty for no password) ] @@ -101,6 +100,25 @@ def test_config_file_should_include_ota(default_config: dict[str, Any]): assert "ota:" in config +def test_config_file_should_use_encryption_when_api_key_set( + default_config: dict[str, Any], +): + """ + With an API encryption key and no OTA password the OTA block reuses the key + """ + # Given + default_config["api_encryption_key"] = ( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + ) + + # When + config = wz.wizard_file(**default_config) + + # Then + assert "ota:\n - platform: esphome\n encryption:" in config + assert "password" not in config.split("ota:")[1].split("wifi:")[0] + + def test_config_file_should_include_ota_when_password_set( default_config: dict[str, Any], ): @@ -630,15 +648,15 @@ def test_wizard_write_protects_existing_config( assert config_file.read_text() == original_content -def test_wizard_accepts_ota_password( +def test_wizard_uses_the_api_key_for_ota( tmp_path: Path, monkeypatch: MonkeyPatch, wizard_answers: list[str] ): """ - The wizard should pass ota_password to wizard_write when the user provides one + The wizard generates an api key and does not ask for an OTA password; + the key secures OTA updates """ # Given - wizard_answers[5] = "my_ota_password" # Set OTA password config_file = tmp_path / "test.yaml" input_mock = MagicMock(side_effect=wizard_answers) monkeypatch.setattr("builtins.input", input_mock) @@ -653,8 +671,9 @@ def test_wizard_accepts_ota_password( # Then assert retval == 0 call_kwargs = wizard_write_mock.call_args.kwargs - assert "ota_password" in call_kwargs - assert call_kwargs["ota_password"] == "my_ota_password" + assert "api_encryption_key" in call_kwargs + assert "ota_password" not in call_kwargs + assert input_mock.call_count == len(wizard_answers) def test_wizard_accepts_rpipico_board(tmp_path: Path, monkeypatch: MonkeyPatch): From 9c00f13606886643a5e9ff8195a08621dd10e9af Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:05:16 +0000 Subject: [PATCH 13/53] Bump bundled esphome-device-builder to 1.14.4 (#19006) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e875851bfb..da76ab7b6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 RUN \ platformio settings set enable_telemetry No \ From 688af60cbfa4289fb3e13b0284622d34dbf77366 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:12:52 +0200 Subject: [PATCH 14/53] [noise] Bump noise-c to 0.1.24 and libsodium to 1.10021.6 (#18989) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index a1d9444fc0..4de706120e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.21") + cg.add_library("esphome/noise-c", "0.1.24") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.4") + cg.add_library("esphome/libsodium", "1.10021.6") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index fcf7caa7c7..779a05e7de 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.21 ; used by noise (api, ota) + esphome/noise-c@0.1.24 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index a263d7937f..4f7f5a4a4c 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.21") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.21") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.21"] + assert libs == ["esphome/noise-c @ 0.1.24"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.21", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.24", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.21"] + assert cls.calls == ["esphome/noise-c @ 0.1.24"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.21"] is None + assert compats["esphome/noise-c @ 0.1.24"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.21"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 379ef52ebd..fb79885736 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1576,7 +1576,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1596,7 +1596,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 20c7dcb1ddf6a70aaf75ff418499835c3e99228e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:26:44 +0200 Subject: [PATCH 15/53] [mdns] Guard LEAmDNS main loop calls against lwIP re-entrancy on ESP8266 (#18990) --- esphome/components/mdns/__init__.py | 2 + esphome/components/mdns/mdns_esp8266.cpp | 51 +++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index f039bb69f0..c8020104b3 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -192,6 +192,8 @@ async def to_code(config: ConfigType) -> None: if CORE.using_arduino: if CORE.is_esp8266: cg.add_library("ESP8266mDNS", None) + # No MDNS global in the build; mdns_esp8266.cpp owns a guarded MDNSResponder + cg.add_build_flag("-DNO_GLOBAL_MDNS") elif CORE.is_rp2: cg.add_library("LEAmDNS", None) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index 1f0b3c9519..0e600d3bac 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -13,8 +13,47 @@ namespace esphome::mdns { +// Main-loop calls into LEAmDNS that send (update() and close(); begin(), addService() and +// the scheduled restart never reach a send) can yield inside UdpContext::sendTimeout(); a +// packet arriving then re-enters LEAmDNS from lwIP on the same UdpContext and both sides +// free the same tx pbufs (#18760). Received packets stay queued during such a call and are +// processed from the main loop afterwards. +class GuardedMDNSResponder : public ::esp8266::MDNSImplementation::MDNSResponder { + public: + void update_guarded() { this->run_guarded_(&GuardedMDNSResponder::update); } + void close_guarded() { this->run_guarded_(&GuardedMDNSResponder::close); } + + private: + void run_guarded_(bool (GuardedMDNSResponder::*fn)()) { + UdpContext *ctx = this->m_pUDPContext; + if (ctx == nullptr) { + (this->*fn)(); + return; + } + // Set every time: a restart replaces the context together with its stock handler. Only + // begin() and the scheduled netif callback restart, never update() or close(), so the + // context cannot change underneath this call. + ctx->onRx([this]() { + if (!this->in_loop_call_) { + this->_callProcess(); + } + }); + this->in_loop_call_ = true; + (this->*fn)(); + // close() releases the context; a yield in here queues further packets for this loop too + while (this->m_pUDPContext != nullptr && this->m_pUDPContext->next()) { + this->_parseMessage(); + } + this->in_loop_call_ = false; + } + + volatile bool in_loop_call_{false}; +}; + +static GuardedMDNSResponder mdns_responder; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + static void register_esp8266(MDNSComponent *, StaticVector &services) { - MDNS.begin(App.get_name().c_str()); + mdns_responder.begin(App.get_name().c_str()); for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is @@ -30,10 +69,10 @@ static void register_esp8266(MDNSComponent *, StaticVectoris_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) return; #endif - MDNS.update(); + mdns_responder.update_guarded(); }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } @@ -81,7 +120,7 @@ void MDNSComponent::on_ip_state(const network::IPAddresses &ips, const network:: #endif void MDNSComponent::on_shutdown() { - MDNS.close(); + mdns_responder.close_guarded(); delay(10); } From 8966567be072926e211b1ca717b7b1d628be7b15 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:28:31 +0200 Subject: [PATCH 16/53] [core] Show the other downloader's progress while a prefetch job waits on its lock (#18983) --- esphome/framework_helpers.py | 61 +++++++++++++- esphome/platformio/prefetch.py | 87 ++++++++++---------- esphome/platformio/registry.py | 67 ++++++++++----- tests/unit_tests/conftest.py | 39 ++++++++- tests/unit_tests/test_framework_helpers.py | 17 ++++ tests/unit_tests/test_platformio_prefetch.py | 87 ++++++++++++++++++-- tests/unit_tests/test_platformio_registry.py | 73 ++++++++++++++-- 7 files changed, 348 insertions(+), 83 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 82bc0d3727..fc2a18a6ec 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -23,6 +23,7 @@ from esphome.net_retry import ( ) if TYPE_CHECKING: + from filelock import FileLock import requests PathType = str | os.PathLike @@ -909,6 +910,61 @@ def _part_path(dest: Path) -> Path: return dest.with_name(dest.name + ".part") +def downloaded_bytes(dest: Path, size: int | None = None) -> int: + """Bytes of ``dest`` on disk (its ``.part`` while streaming), capped at ``size``.""" + done = 0 + for candidate in (_part_path(dest), dest): + try: + done = candidate.stat().st_size + break + except FileNotFoundError: + continue + return done if size is None else min(done, size) + + +# Short lock-acquire slices so a waiting worker still observes Ctrl-C +_DOWNLOAD_LOCK_POLL = 1 + +# Waiting on another process's download; past this the caller leaves the +# file to its holder (the later sequential install waits on the same lock) +DOWNLOAD_LOCK_TIMEOUT = 60 + + +class DownloadLockUnavailable(OSError): + """The lock file cannot be used at all (a lock-less filesystem).""" + + +def wait_for_download_lock( + lock: "FileLock", + tracker: Callable[[int], None], + on_disk: Callable[[], int], + name: str, +) -> None: + """Acquire ``lock``, reporting ``on_disk()`` to ``tracker`` each poll so the + bar follows the holder's download. Raises filelock's ``Timeout`` once + ``DOWNLOAD_LOCK_TIMEOUT`` seconds pass.""" + from filelock import Timeout + + deadline = time.monotonic() + DOWNLOAD_LOCK_TIMEOUT + waiting = False + while True: + try: + lock.acquire(timeout=_DOWNLOAD_LOCK_POLL) + return + except Timeout: + pass + except OSError as err: + # Distinct from an OSError out of on_disk(), which must not + # read as "locks unsupported" + raise DownloadLockUnavailable(*err.args) from err + if not waiting: + waiting = True + _LOGGER.info("Waiting for another process downloading %s", name) + tracker(on_disk()) # raises when the batch is cancelled + if time.monotonic() >= deadline: + raise Timeout(lock.lock_file) + + def discard_partial_download(dest: Path) -> None: """Remove ``dest`` and the resume sidecars of an abandoned download.""" part = _part_path(dest) @@ -1319,10 +1375,7 @@ def download_from_mirrors( ) # Tick with the bytes already on disk so a combined bar holds # steady during the backoff instead of rewinding to zero - done = 0 - if progress is not None: - part = _part_path(path_target) - done = part.stat().st_size if part.is_file() else 0 + done = downloaded_bytes(path_target) if progress is not None else 0 _cancellable_sleep(delay, progress, done) # 3. Report every attempted URL if all mirrors failed. failures spans diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 5097239065..17a06cb9c1 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -33,11 +33,14 @@ import time from typing import Any, NamedTuple from esphome.framework_helpers import ( + DownloadLockUnavailable, content_length, discard_partial_download, + downloaded_bytes, failure_reason, resume_fetch_job, run_batch_downloads, + wait_for_download_lock, warn_prefetch_failures, ) from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree @@ -61,16 +64,10 @@ _RESOLVE_WORKERS = 8 # A hung child must not block the build; downloads resume on the next run _PREFETCH_TIMEOUT = 20 * 60 -# Waiting on another process's URL download; past this, leave it to pio -_DOWNLOAD_LOCK_TIMEOUT = 60 - # Child exit for a handled, already-warned failure; 1 would collide with # the interpreter's own import-failure exit _EXIT_HANDLED = 3 -# Short lock-acquire slices so a waiting worker still observes Ctrl-C -_URI_LOCK_POLL = 1 - # Resolution errored (vs a clean skip); suppresses the warm sentinel _RESOLVE_FAILED = object() @@ -462,51 +459,54 @@ def _uri_jobs( def _serialized_fetch_job( - dl_path: Path, lock_path: str, body: Any, unlocked_ok: bool = True + dl_path: Path, + lock_path: str, + body: Any, + size: int, + stream_dest: Path | None = None, + unlocked_ok: bool = True, ) -> Any: - """Wrap ``body`` so the shared destination is single-writer. - - Interleaved writers truncate each other's ``.part`` bytes (see - registry.py). The bounded poll observes Ctrl-C via the tracker; a - blown deadline is a clean skip (the holder's copy is what the build - needs). On a lock-less filesystem a sha256-verified body runs - unlocked with one warning; a checksum-less one - (``unlocked_ok=False``) is a counted failure instead. + """Wrap ``body`` so the shared destination is single-writer (interleaved + writers truncate each other's ``.part``, see registry.py). A blown deadline + is a clean skip. On a lock-less filesystem a sha256-verified body runs + unlocked with one warning; a checksum-less one (``unlocked_ok=False``) fails. """ + def on_disk() -> int: + # A URL job's holder streams beside the staging path until it + # promotes; after that only dl_path is left + done = downloaded_bytes(dl_path, size) + if not done and stream_dest is not None: + done = downloaded_bytes(stream_dest, size) + return done + def run(tracker: Any) -> None: from filelock import FileLock, Timeout # fallback_to_soft would leave a stale marker on lock-less # filesystems that blocks every later build (see git.py) lock = FileLock(lock_path, fallback_to_soft=False) - deadline = time.monotonic() + _DOWNLOAD_LOCK_TIMEOUT - while True: - try: - lock.acquire(timeout=_URI_LOCK_POLL) - break - except Timeout: - tracker(0) # raises when the batch is cancelled - if time.monotonic() >= deadline: - # Another process is fetching this same file; its copy - # is what the build needs (a large framework archive - # can hold the lock far longer than this deadline) - _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) - return - except OSError as err: - if not unlocked_ok: - # A body with no checksum to catch interleaved corruption - raise - lock = None - _LOGGER.warning( - "Could not lock %s (%s); downloading unlocked", - dl_path.name, - err, - ) - break + try: + wait_for_download_lock(lock, tracker, on_disk, dl_path.name) + except Timeout: + # The holder's copy is what the build needs (a large + # framework archive can outlast this deadline) + _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) + return + except DownloadLockUnavailable as err: + if not unlocked_ok: + # A body with no checksum to catch interleaved corruption + raise + lock = None + _LOGGER.warning( + "Could not lock %s (%s); downloading unlocked", + dl_path.name, + err, + ) try: if dl_path.is_file(): - return # another process finished it while we waited + tracker(size) # another process finished it while we waited + return body(tracker) finally: if lock is not None: @@ -540,6 +540,7 @@ def _registry_fetch_job( dl_path, f"{dl_path}.esphome.lock", resume_fetch_job(url, dl_path, sha256=checksum, size=size), + size, ) def run(tracker: Any) -> None: @@ -571,9 +572,9 @@ def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: tmp.replace(dl_path) def run(tracker: Any) -> None: - _serialized_fetch_job(dl_path, f"{tmp}.lock", promote, unlocked_ok=False)( - tracker - ) + _serialized_fetch_job( + dl_path, f"{tmp}.lock", promote, size, tmp, unlocked_ok=False + )(tracker) if dl_path.is_file(): # Won or lost, the race is over; staging files left behind # are dead weight PlatformIO's cache never prunes diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index 9538a28ff4..75df82da0e 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -17,8 +17,10 @@ from esphome.framework_helpers import ( archive_extract_all, download_from_mirrors, download_with_resume, + downloaded_bytes, rmdir, run_batch_downloads, + wait_for_download_lock, ) from esphome.net_retry import fetch_with_retry, http_request @@ -164,11 +166,17 @@ class _PendingArchive(NamedTuple): name: str version: str dest: Path + archive: Path url: str sha256: str size: int +def _archive_path(downloads_dir: Path, name: str, version: str) -> Path: + """The one archive path the prefetch and the sequential install share.""" + return downloads_dir / f"{name}-{version}" + + def _already_installed(dest: Path) -> bool: """Whether ``dest`` holds a completed install (extraction marker).""" return (dest / ".esphome_extracted").is_file() @@ -187,18 +195,18 @@ def prefetch_packages( lock as ``install_package``: the archive's ``.part`` file is shared, and two concurrent writers would truncate each other's bytes. """ - from filelock import FileLock + from filelock import FileLock, Timeout pending: list[_PendingArchive] = [] - seen: set[str] = set() + seen: set[Path] = set() for name, version, dest, mirrors in packages: if mirrors or (dest / ".esphome_extracted").is_file(): continue - archive_name = f"{name}-{version}" - if archive_name in seen: + archive = _archive_path(downloads_dir, name, version) + if archive in seen: # A duplicate entry would race itself between two workers continue - seen.add(archive_name) + seen.add(archive) try: url, sha256, size = registry_download(name, version) except EsphomeError as err: @@ -207,10 +215,9 @@ def prefetch_packages( continue if not size: continue - archive = downloads_dir / archive_name if archive.is_file() and archive.stat().st_size == size: continue - pending.append(_PendingArchive(name, version, dest, url, sha256, size)) + pending.append(_PendingArchive(name, version, dest, archive, url, sha256, size)) if len(pending) < 2: return downloads_dir.mkdir(parents=True, exist_ok=True) @@ -222,20 +229,36 @@ def prefetch_packages( def _fetch(entry: _PendingArchive, tracker: Callable[[int], None]) -> None: entry.dest.parent.mkdir(parents=True, exist_ok=True) - with FileLock(f"{entry.dest}.lock", fallback_to_soft=False): - # Marker re-check: a concurrent build may have installed (and - # deleted the archive of) this package while we waited; - # re-downloading would orphan a fresh copy in downloads_dir - # no branch: the thread tracer misses the skip edge; both - # arms of _already_installed are pinned directly - if not _already_installed(entry.dest): # pragma: no branch - download_with_resume( - entry.url, - downloads_dir / f"{entry.name}-{entry.version}", - sha256=entry.sha256, - size=entry.size, - progress=tracker, - ) + + def on_disk() -> int: + if done := downloaded_bytes(entry.archive, entry.size): + return done + # The holder deletes the archive once it has installed it + return entry.size if _already_installed(entry.dest) else 0 + + lock = FileLock(f"{entry.dest}.lock", fallback_to_soft=False) + try: + wait_for_download_lock(lock, tracker, on_disk, entry.name) + except Timeout: + # install_package waits on this same lock and verifies the + # holder's copy + _LOGGER.debug("Leaving %s to its current downloader", entry.name) + return + try: + if _already_installed(entry.dest): + # A concurrent build installed it while we waited; a + # re-download would orphan a fresh copy in downloads_dir + tracker(entry.size) + return + download_with_resume( + entry.url, + entry.archive, + sha256=entry.sha256, + size=entry.size, + progress=tracker, + ) + finally: + lock.release() failures = run_batch_downloads( "Downloading packages", @@ -288,7 +311,7 @@ def install_package( rmdir(dest, msg=f"Clean up incomplete {name} install") # Persistent location so an interrupted download resumes across runs. downloads_dir.mkdir(parents=True, exist_ok=True) - archive = downloads_dir / f"{name}-{version}" + archive = _archive_path(downloads_dir, name, version) _LOGGER.info("Downloading %s %s ...", name, version) if mirrors: _LOGGER.warning( diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 9de8f715ef..ad9c0bb11f 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -9,7 +9,7 @@ not be part of a unit test suite. """ -from collections.abc import Generator +from collections.abc import Callable, Generator import os from pathlib import Path import sys @@ -137,3 +137,40 @@ def mock_get_component() -> Generator[Mock, None, None]: """Mock get_component for config module.""" with patch("esphome.config.get_component") as mock: yield mock + + +@pytest.fixture +def held_lock() -> Callable[..., Callable[..., None]]: + """Factory for a ``FileLock.acquire`` fake held by another downloader. + + Each poll writes the next chunk to ``part`` (or runs it, for a callable) + and raises ``Timeout``; when the chunks run out the part is removed, + ``land()`` runs, and the acquire succeeds (also for any later job, so + ``land`` must be idempotent). + """ + from filelock import Timeout + + def make( + part: Path, + chunks: list[bytes | Callable[[], None]], + land: Callable[[], None], + ) -> Callable[..., None]: + polls = iter(chunks) + + def acquire(*args, **kwargs) -> None: + try: + chunk = next(polls) + except StopIteration: + part.unlink(missing_ok=True) + land() + return + if callable(chunk): + chunk() + else: + part.parent.mkdir(parents=True, exist_ok=True) + part.write_bytes(chunk) + raise Timeout("held") + + return acquire + + return make diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index fcc5572f51..22b34c9df5 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2353,3 +2353,20 @@ def test_discard_partial_download_logs_undeletable( ): framework_helpers.discard_partial_download(dest) assert "Could not remove" in caplog.text + + +def test_downloaded_bytes_reports_what_is_on_disk(tmp_path: Path) -> None: + """Part file first, then the landed file, both capped at size; else 0.""" + dest = tmp_path / "archive" + assert framework_helpers.downloaded_bytes(dest, 4) == 0 + part = tmp_path / "archive.part" + part.write_bytes(b"ab") + assert framework_helpers.downloaded_bytes(dest, 4) == 2 + part.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 + part.unlink() + dest.write_bytes(b"abc") + assert framework_helpers.downloaded_bytes(dest, 4) == 3 + assert framework_helpers.downloaded_bytes(dest) == 3 + dest.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index fb79885736..77490fd861 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -454,23 +454,96 @@ def test_uri_fetch_job_waits_out_a_briefly_held_lock(tmp_path: Path) -> None: assert dl_path.read_bytes() == b"data" -def test_lock_deadline_leaves_download_to_the_holder(tmp_path: Path) -> None: - """A lock held past the deadline means another process is fetching the - same file; skipping cleanly beats a misleading failure warning. The - tracker is still polled so a parked worker observes cancellation.""" +@pytest.mark.parametrize("staged", [b"", b"ab"]) +def test_lock_deadline_leaves_download_to_the_holder( + tmp_path: Path, staged: bytes +) -> None: + """A lock held past the deadline is another process's download; skip + cleanly, polling the tracker with what the holder has staged so far.""" dl_path = tmp_path / "archive" + (tmp_path / "archive.prefetch.part").write_bytes(staged) ticks: list[int] = [] with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) mock_download.assert_not_called() - assert ticks == [0] + assert ticks == [len(staged)] assert not dl_path.exists() +@pytest.mark.parametrize( + ("job", "part_name", "chunks", "expected"), + [ + ( + lambda dl_path: pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + ), + "archive.part", + [b"a", b"abc"], + [1, 3, 4], + ), + ( + lambda dl_path: pf._uri_fetch_job( + MagicMock(), "https://x/a.zip", dl_path, 4 + ), + "archive.prefetch.part", + [b"ab"], + [2, 4], + ), + ], + ids=["registry", "uri"], +) +def test_lock_wait_reports_the_holders_progress( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + held_lock, + job, + part_name: str, + chunks: list[bytes], + expected: list[int], +) -> None: + """A waiting job reports the holder's part file (the staging one for a + URL job), then the full size once the holder lands the archive.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / part_name, chunks, lambda: dl_path.write_bytes(b"abcd") + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + caplog.at_level(logging.INFO), + ): + job(dl_path)(ticks.append) + mock_download.assert_not_called() + assert ticks == expected + assert caplog.text.count("Waiting for another process downloading archive") == 1 + + +def test_uri_lock_wait_prefers_the_landed_archive(tmp_path: Path, held_lock) -> None: + """Between the holder's promotion rename and its release the staging + part is gone; the landed cache file is credited instead of 0.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / "archive.prefetch.part", + [b"ab", lambda: dl_path.write_bytes(b"abcd")], + lambda: None, + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) + mock_download.assert_not_called() + assert ticks == [2, 4, 4] + + def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: """A registry job that lost the download race to another process must not stamp a nonexistent archive into pio's usage.db.""" @@ -479,7 +552,7 @@ def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( lambda done: None diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 6ba8691c4e..9d5f6c4ce5 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -8,6 +8,7 @@ import os from pathlib import Path from unittest.mock import MagicMock, patch +from filelock import Timeout import pytest from esphome.core import EsphomeError @@ -540,16 +541,13 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: dest = tmp_path / "a" dest.mkdir() - from contextlib import contextmanager - - @contextmanager - def marker_appears_under_lock(path, **kwargs): + def marker_appears_under_lock(*args, **kwargs): # Simulates the concurrent build finishing while we waited (dest / ".esphome_extracted").touch() - yield with ( - patch("filelock.FileLock", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.acquire", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.release"), patch.object(registry, "download_with_resume") as mock_download, patch.object( registry, "registry_download", side_effect=_resolve_for({"a": 10}) @@ -559,6 +557,69 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: mock_download.assert_not_called() +def test_prefetch_packages_waits_with_the_holders_progress( + tmp_path: Path, held_lock +) -> None: + """A worker parked on another build's lock reports that build's part + file, then the full size once the marker appears.""" + dest = tmp_path / "a" + dest.mkdir() + ticks: list[int] = [] + part = tmp_path / "dl" / "a-1.0.part" + + def installed_and_pruned() -> None: + # install_package touches the marker, then unlinks the archive + (dest / ".esphome_extracted").touch() + part.unlink() + + acquire = held_lock( + part, + [lambda: None, b"abc", installed_and_pruned], + (dest / ".esphome_extracted").touch, + ) + + def fake_batch(header, jobs): + for _name, _size, fetch in jobs: + fetch(ticks.append) + return [] + + with ( + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + patch.object(registry, "run_batch_downloads", side_effect=fake_batch), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", dest, []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + assert ticks == [0, 3, 10, 10] + mock_download.assert_called_once() + + +def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( + tmp_path: Path, +) -> None: + """Past the deadline the worker skips; install_package waits on the same + lock later and verifies whatever the holder produced.""" + with ( + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", tmp_path / "a", []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + def test_already_installed_probe(tmp_path: Path) -> None: """Both arms of the marker probe the prefetch worker keys on.""" dest = tmp_path / "pkg" From d58b37faa1eff3324dd6c9389c864d5c3576eadf Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:29:20 +1200 Subject: [PATCH 17/53] [esp32_hosted] Add ESP-NOW-over-hosted shim for the ESP32-P4 (#17712) --- esphome/components/esp32_hosted/__init__.py | 36 ++ .../esp32_hosted/esp_now_hosted.cpp | 467 ++++++++++++++++++ .../esp32_hosted/esp_now_hosted_rpc.h | 128 +++++ esphome/components/espnow/__init__.py | 20 + esphome/core/defines.h | 1 + script/ci-custom.py | 17 +- .../test-espnow.esp32-p4-idf.yaml | 5 + tests/unit_tests/components/test_espnow.py | 48 ++ 8 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 esphome/components/esp32_hosted/esp_now_hosted.cpp create mode 100644 esphome/components/esp32_hosted/esp_now_hosted_rpc.h create mode 100644 tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml create mode 100644 tests/unit_tests/components/test_espnow.py diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index ab9455250c..21626e432b 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -37,6 +37,25 @@ CONF_HANDSHAKE_PIN = "handshake_pin" CONF_SDIO_FREQUENCY = "sdio_frequency" CONF_SPI_MODE = "spi_mode" +# ESP-NOW-over-hosted shim (esp_now_hosted.cpp). esp-hosted proxies esp_wifi.h +# but not esp_now.h (espressif/esp-hosted-mcu#19), and esp_wifi_remote injects +# the esp_now.h header on the ESP32-P4 host with no implementation, leaving the +# esp_now_* symbols undefined at link. On a P4 host, esp_now_hosted.cpp DEFINES +# those symbols and forwards each call to the co-processor over esp-hosted's +# CustomRpc "peer data transfer" channel, so ESPHome's `espnow` component links +# and runs unchanged (proven on a Tab5, 2026-07-20). The .cpp is guarded to +# CONFIG_IDF_TARGET_ESP32P4 so it compiles to nothing on hosts with a native +# ESP-NOW stack. CustomRpc needs these two host-side Kconfig options. Host +# registers 3 handlers (RESP, RECV, SEND); the coprocessor registers 1 (REQ); +# we ask for 8 to leave room for other CustomRpc extensions alongside. +# +# The coprocessor must run the matching custom firmware (a parallel effort in +# esphome/esp-hosted-firmware). esp_now_hosted_rpc.h here is the canonical copy +# of the wire contract and MUST stay byte-identical to the copy that coprocessor +# firmware uses — the packed structs are the on-wire layout, so any divergence +# silently corrupts every ESP-NOW frame. +_MAX_CUSTOM_MSG_HANDLERS = 8 + # Shared fields for both transport modes BASE_SCHEMA = cv.Schema( { @@ -262,6 +281,23 @@ async def to_code(config: ConfigType) -> None: else: _configure_spi(config) + # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see + # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not + # gated on the `espnow` component being present: the shim is tiny and the + # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options + # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so + # coupling the two keeps the build consistent. When `espnow` is absent the + # symbols are simply unused and never register a callback at runtime. + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + add_define("USE_ESP_NOW_HOSTED") + # esp-hosted's CustomRpc ("peer data transfer") path — off by default. + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS + ) + # Place the transport mempool in PSRAM. Required on memory-tight host # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM # mempool allocation fails at boot with `sdio_mempool_create` assert. diff --git a/esphome/components/esp32_hosted/esp_now_hosted.cpp b/esphome/components/esp32_hosted/esp_now_hosted.cpp new file mode 100644 index 0000000000..ad29b208fe --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted.cpp @@ -0,0 +1,467 @@ +/* + * esp_now_hosted — host-side shim implementing over esp-hosted + * CustomRpc, so ESPHome's `espnow` component can run on a radio-less host + * (e.g. the ESP32-P4) whose radio lives on an esp-hosted co-processor. + * + * A radio-less host has no native ESP-NOW. esp_wifi_remote INJECTS the full + * esp_now.h header (types + declarations) but ships NO implementation, so every + * esp_now_* symbol is an undefined reference at link time. This translation + * unit provides those definitions; each forwards to the co-processor over + * CustomRpc (see esphome/esp-hosted-firmware for the matching coprocessor + * handlers). No esp-hosted or esp_wifi_remote source is patched, and there is no + * duplicate-symbol clash because nothing else defines these symbols here. + * + * See esp_now_hosted_rpc.h for the wire protocol. + */ + +#include "sdkconfig.h" + +// Only build the shim on the radio-less host. On chips with a native ESP-NOW +// stack (S3, C6, …) the real symbols exist and this file must stay empty to +// avoid duplicate definitions. +#if defined(CONFIG_IDF_TARGET_ESP32P4) + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" + +#include "esp_idf_version.h" +#include "esp_log.h" +#include "esp_timer.h" + +#include // injected declarations we are now DEFINING +#include // wifi_pkt_rx_ctrl_t, wifi_tx_info_t + +// esp_hosted_misc.h (host) ships WITHOUT an extern "C" guard, so including it +// from C++ would give its declarations C++ linkage and the real C symbols in +// libesp_hosted would go unresolved at link. Wrap it. (Verified vs +// esp_hosted 2.12.9.) +extern "C" { +#include "esp_hosted_misc.h" // esp_hosted_{send_custom_data,register_custom_callback} +} + +#include "esp_now_hosted_rpc.h" + +namespace { + +const char *const TAG = "esp_now_hosted"; + +// One outstanding request at a time. ESPHome drives esp_now_* from the main +// loop; the matching response and the async RECV/SEND events all arrive on the +// single esp-hosted RPC RX thread. Serializing requests keeps the shared +// response slot race-free; a sequence number stops a late/stale response from +// being mistaken for ours. +SemaphoreHandle_t g_req_mutex = nullptr; +SemaphoreHandle_t g_resp_sem = nullptr; // given when the matching RESP lands +bool g_setup_done = false; // set only after setup fully succeeds +uint8_t g_seq = 0; +volatile uint8_t g_expect_seq = 0; +volatile int32_t g_resp_status = 0; +uint8_t g_resp_ret[16]; +volatile uint16_t g_resp_ret_len = 0; + +// Written from the main loop (register/unregister/deinit), read from the +// esp-hosted RX thread (on_recv/on_send). volatile for the same reason the +// g_resp_* globals are: force the RX thread to observe an updated pointer +// (e.g. a nulling by esp_now_deinit) rather than a cached one. +volatile esp_now_recv_cb_t g_recv_cb = nullptr; +volatile esp_now_send_cb_t g_send_cb = nullptr; + +// Local mirror of the co-processor's peer table. ESPHome's espnow component +// calls esp_now_is_peer_exist() on the main loop for every received frame +// (twice) and every send; forwarding each as a blocking RPC round-trip stalls +// the loop. The shim is the only path that mutates the co-processor peer table +// (add/del/deinit all go through here), so this mirror is authoritative and +// esp_now_is_peer_exist() can answer from it with no round-trip. +// +// esp_now_* are public C symbols: any component or user lambda may call them, +// and although ESPHome's espnow touches peers only from the main loop today +// (its RX/TX callbacks merely enqueue), the shim cannot rely on that. A short +// spinlock keeps the mirror consistent from any task/core, matching native +// esp_now_*'s own internal thread-safety. The critical sections are a bounded +// (<=20-entry) scan, so they stay tiny. ESP_NOW_MAX_TOTAL_PEER_NUM is 20. +constexpr size_t ESP_NOW_HOSTED_MAX_PEERS = 20; +uint8_t g_peer_cache[ESP_NOW_HOSTED_MAX_PEERS][6]; +size_t g_peer_count = 0; +portMUX_TYPE g_peer_lock = portMUX_INITIALIZER_UNLOCKED; + +// Caller must hold g_peer_lock. +int peer_cache_find_locked(const uint8_t *mac) { + for (size_t i = 0; i < g_peer_count; i++) { + if (memcmp(g_peer_cache[i], mac, 6) == 0) + return static_cast(i); + } + return -1; +} + +bool peer_cache_contains(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const bool found = peer_cache_find_locked(mac) >= 0; + portEXIT_CRITICAL(&g_peer_lock); + return found; +} + +void peer_cache_add(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + if (peer_cache_find_locked(mac) < 0 && g_peer_count < ESP_NOW_HOSTED_MAX_PEERS) + memcpy(g_peer_cache[g_peer_count++], mac, 6); + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_remove(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const int idx = peer_cache_find_locked(mac); + if (idx >= 0) { + g_peer_count--; + if (static_cast(idx) != g_peer_count) // move the last entry into the gap + memcpy(g_peer_cache[idx], g_peer_cache[g_peer_count], 6); + } + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_clear() { + portENTER_CRITICAL(&g_peer_lock); + g_peer_count = 0; + portEXIT_CRITICAL(&g_peer_lock); +} + +// ── CustomRpc event handlers (run on the esp-hosted RPC RX thread) ────────── +// Keep them short and non-blocking. In particular they MUST NOT call back into +// any esp_now_* shim function: that would try to take g_req_mutex / wait on the +// RX thread that delivers the response, and deadlock. + +void on_resp(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + if (len < sizeof(esp_now_hosted_resp_t)) { + ESP_LOGW(TAG, "RESP too short: %u bytes", static_cast(len)); + return; + } + const auto *r = reinterpret_cast(data); + if (r->seq != g_expect_seq) { // late response from a timed-out request (expected) + ESP_LOGV(TAG, "dropping stale RESP seq %u (want %u)", r->seq, g_expect_seq); + return; + } + g_resp_status = r->status; + uint16_t rl = r->ret_len; + if (rl > sizeof(g_resp_ret)) { + // Larger than any real opcode return — a likely wire-format drift signal. + ESP_LOGW(TAG, "RESP ret_len %u exceeds buffer, clamping (wire drift?)", rl); + rl = sizeof(g_resp_ret); + } + if (len >= sizeof(esp_now_hosted_resp_t) + rl) { + memcpy(g_resp_ret, r->ret, rl); + } else { + // Truncated frame: fail closed. Never hand the caller stale bytes left in + // g_resp_ret by a previous response, and don't let request() report a + // zeroed payload as success — override the status to an error. + ESP_LOGW(TAG, "RESP truncated: claims %u ret bytes, frame too short", rl); + rl = 0; + g_resp_status = ESP_ERR_INVALID_RESPONSE; + } + g_resp_ret_len = rl; + xSemaphoreGive(g_resp_sem); +} + +void on_recv(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once: esp_now_unregister_recv_cb()/deinit() (via + // the espnow component's disable()) can null it on the main loop between the + // guard and the call, which would otherwise turn the call into a null-deref. + const esp_now_recv_cb_t cb = g_recv_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_recv_evt_t)) { + ESP_LOGW(TAG, "RECV too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); + if (len < sizeof(esp_now_hosted_recv_evt_t) + e->data_len) { + ESP_LOGW(TAG, "RECV data_len %u exceeds frame", e->data_len); + return; + } + + // ESPHome dereferences info->rx_ctrl->{rssi,timestamp}; give it a real one. + wifi_pkt_rx_ctrl_t rx_ctrl; + memset(&rx_ctrl, 0, sizeof(rx_ctrl)); + rx_ctrl.rssi = e->rssi; + rx_ctrl.channel = e->channel; + rx_ctrl.timestamp = static_cast(esp_timer_get_time()); + + esp_now_recv_info_t info; + info.src_addr = const_cast(e->src_addr); + info.des_addr = const_cast(e->des_addr); + info.rx_ctrl = &rx_ctrl; + cb(&info, e->data, static_cast(e->data_len)); +} + +void on_send(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once (see on_recv): disable()/deinit() can null it + // on the main loop concurrently with this RX-thread callback. + const esp_now_send_cb_t cb = g_send_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_send_evt_t)) { + ESP_LOGW(TAG, "SEND evt too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0) + // IDF >= 5.5: esp_now_send_cb_t takes esp_now_send_info_t (== wifi_tx_info_t), + // whose des_addr is a POINTER (not an inline array). Point it at the event's + // MAC (valid for this callback) — do NOT memcpy into it (that writes NULL and + // faults). ESPHome reads only info->des_addr. + esp_now_send_info_t si; + memset(&si, 0, sizeof(si)); + si.des_addr = const_cast(e->des_addr); + cb(&si, static_cast(e->status)); +#else + cb(e->des_addr, static_cast(e->status)); +#endif +} + +esp_err_t ensure_setup() { + // Gate on g_setup_done, not on g_req_mutex: a failure part-way through (a + // semaphore that did not allocate, a callback that did not register) must not + // leave a later call thinking setup completed. Semaphore creation is guarded + // so a retry after a partial failure does not leak the earlier handles. + if (g_setup_done) + return ESP_OK; + if (g_req_mutex == nullptr) + g_req_mutex = xSemaphoreCreateMutex(); + if (g_resp_sem == nullptr) + g_resp_sem = xSemaphoreCreateBinary(); + if (g_req_mutex == nullptr || g_resp_sem == nullptr) + return ESP_ERR_NO_MEM; + esp_err_t err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RESP, on_resp, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RECV, on_recv, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_SEND, on_send, nullptr)) != ESP_OK) + return err; + g_setup_done = true; + return ESP_OK; +} + +// Send one request envelope. With wait=true (default) block until the matching +// response (or timeout); with wait=false return as soon as the frame is handed +// to the transport (fire-and-forget, used by esp_now_send). +// +// `tail` is an optional second chunk written straight after `payload`. Callers +// with a fixed header plus a bulk body (esp_now_send) pass the two separately +// so they never need a build buffer of their own: both chunks are laid into the +// request buffer here, under g_req_mutex, which keeps concurrent callers from +// racing and saves a full copy of the body on every transmit. +esp_err_t request(uint8_t opcode, const void *payload, uint16_t plen, void *ret, uint16_t ret_cap, uint16_t *ret_len, + bool wait = true, const void *tail = nullptr, uint16_t tail_len = 0) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + if (plen > ESP_NOW_HOSTED_MAX_PAYLOAD || tail_len > ESP_NOW_HOSTED_MAX_PAYLOAD - plen) + return ESP_ERR_INVALID_SIZE; + const uint16_t total_len = static_cast(plen + tail_len); + + if (xSemaphoreTake(g_req_mutex, portMAX_DELAY) != pdTRUE) + return ESP_FAIL; + + static uint8_t buf[sizeof(esp_now_hosted_req_t) + ESP_NOW_HOSTED_MAX_PAYLOAD]; // guarded by g_req_mutex + auto *req = reinterpret_cast(buf); + req->opcode = opcode; + req->seq = ++g_seq; + req->payload_len = total_len; + if (plen != 0) + memcpy(req->payload, payload, plen); + if (tail_len != 0) + memcpy(req->payload + plen, tail, tail_len); + g_expect_seq = req->seq; + + xSemaphoreTake(g_resp_sem, 0); // drain any stale signal before sending + err = esp_hosted_send_custom_data(ESP_NOW_HOSTED_MSG_REQ, buf, sizeof(esp_now_hosted_req_t) + total_len); + if (err != ESP_OK) { + xSemaphoreGive(g_req_mutex); + return err; + } + if (!wait) { + // Fire-and-forget (esp_now_send): the co-processor enqueues the frame and + // reports the real TX result later via the async SEND event, exactly like + // native esp_now_send. Returning here keeps the main loop off the ~100 ms+ + // RPC round-trip. The matching RESP is ignored (seq won't match the next + // waited request, so on_resp drops it). + xSemaphoreGive(g_req_mutex); + return ESP_OK; + } + if (xSemaphoreTake(g_resp_sem, pdMS_TO_TICKS(ESP_NOW_HOSTED_TIMEOUT_MS)) != pdTRUE) { + ESP_LOGW(TAG, "opcode %u timed out", opcode); + xSemaphoreGive(g_req_mutex); + return ESP_ERR_TIMEOUT; + } + + const int32_t status = g_resp_status; + if (ret != nullptr && ret_cap != 0) { + uint16_t n = g_resp_ret_len < ret_cap ? g_resp_ret_len : ret_cap; + memcpy(ret, const_cast(g_resp_ret), n); + if (ret_len != nullptr) + *ret_len = n; + } + xSemaphoreGive(g_req_mutex); + return static_cast(status); +} + +} // namespace + +// ── The surface, defined for the radio-less host ──────────────── +extern "C" { + +esp_err_t esp_now_init(void) { return request(ESP_NOW_HOSTED_OP_INIT, nullptr, 0, nullptr, 0, nullptr); } + +esp_err_t esp_now_deinit(void) { + g_recv_cb = nullptr; + g_send_cb = nullptr; + peer_cache_clear(); // the co-processor drops all peers on deinit + return request(ESP_NOW_HOSTED_OP_DEINIT, nullptr, 0, nullptr, 0, nullptr); +} + +esp_err_t esp_now_get_version(uint32_t *version) { + uint32_t v = 0; + uint16_t rl = 0; + esp_err_t err = request(ESP_NOW_HOSTED_OP_GET_VERSION, nullptr, 0, &v, sizeof(v), &rl); + if (version != nullptr) + *version = v; + return err; +} + +esp_err_t esp_now_register_recv_cb(esp_now_recv_cb_t cb) { + // Only arm the callback once the CustomRpc handlers are actually registered, + // so a failed setup leaves g_recv_cb null rather than falsely "registered". + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_recv_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_recv_cb(void) { + g_recv_cb = nullptr; + return ESP_OK; +} +esp_err_t esp_now_register_send_cb(esp_now_send_cb_t cb) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_send_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_send_cb(void) { + g_send_cb = nullptr; + return ESP_OK; +} + +static esp_err_t add_or_mod_peer(uint8_t opcode, const esp_now_peer_info_t *peer, bool wait) { + if (peer == nullptr) + return ESP_ERR_ESPNOW_ARG; + esp_now_hosted_peer_t p; + memset(&p, 0, sizeof(p)); + memcpy(p.peer_addr, peer->peer_addr, 6); + memcpy(p.lmk, peer->lmk, 16); + p.channel = peer->channel; + p.ifidx = static_cast(peer->ifidx); + p.encrypt = peer->encrypt ? 1 : 0; + return request(opcode, &p, sizeof(p), nullptr, 0, nullptr, wait); +} +esp_err_t esp_now_add_peer(const esp_now_peer_info_t *peer) { + // Fire-and-forget (wait=false): adding a peer is a blocking RPC round-trip, + // and ESPHome's espnow calls it on the main loop when a device joins the mesh + // — under co-processor load that stalls the UI (peer-churn stutter). Issue it + // without waiting and mirror it locally. Safe against a following + // esp_now_send to the same peer: both ride the same in-order CustomRpc + // channel (mutex-serialized on the host) and the co-processor processes REQs + // FIFO, so ADD_PEER is applied before the SEND. Trade-off: a co-processor-side + // failure (e.g. peer table full) is no longer reported synchronously — the + // same limitation as esp_now_send — but ESPHome only adds peers it validated. + esp_err_t err = add_or_mod_peer(ESP_NOW_HOSTED_OP_ADD_PEER, peer, /*wait=*/false); + if (err == ESP_OK) + peer_cache_add(peer->peer_addr); // keep the local mirror in sync + return err; +} +esp_err_t esp_now_mod_peer(const esp_now_peer_info_t *peer) { + // mod_peer changes a peer's parameters, not its existence, so the cache is + // unaffected. Kept synchronous — it is not on any hot path (espnow never + // calls it), so the extra round-trip does not matter and the status is useful. + return add_or_mod_peer(ESP_NOW_HOSTED_OP_MOD_PEER, peer, /*wait=*/true); +} + +esp_err_t esp_now_del_peer(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return ESP_ERR_ESPNOW_ARG; + // Fire-and-forget for the same reason as add_peer (peer churn on the main + // loop). Removal is order-independent, so this is strictly safe. + esp_err_t err = request(ESP_NOW_HOSTED_OP_DEL_PEER, peer_addr, 6, nullptr, 0, nullptr, /*wait=*/false); + if (err == ESP_OK) + peer_cache_remove(peer_addr); // keep the local mirror in sync + return err; +} + +bool esp_now_is_peer_exist(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return false; + // Answered from the local mirror — no RPC round-trip. ESPHome's espnow calls + // this on the main loop for every received frame and every send, so a + // blocking round-trip here would stall rendering under mesh traffic. + return peer_cache_contains(peer_addr); +} + +esp_err_t esp_now_send(const uint8_t *peer_addr, const uint8_t *data, size_t len) { + if (len > ESP_NOW_HOSTED_MAX_FRAME) + return ESP_ERR_ESPNOW_ARG; + if (data == nullptr && len != 0) // native esp_now_send treats this as an arg error + return ESP_ERR_ESPNOW_ARG; + // Only the small fixed header is built here; the caller's frame goes over as + // the request tail, so request() lays both into its own buffer under + // g_req_mutex. esp_now_send is a public C symbol and may be called from any + // task, and a shared build buffer here would let two callers corrupt each + // other's frame. Passing the body through also drops a full-frame copy per + // transmit, on the path this shim exists to keep quick. + uint8_t hdr[sizeof(esp_now_hosted_send_req_t)]; + auto *s = reinterpret_cast(hdr); + s->has_addr = peer_addr != nullptr ? 1 : 0; + if (peer_addr != nullptr) + memcpy(s->peer_addr, peer_addr, 6); + else + memset(s->peer_addr, 0, 6); + s->data_len = static_cast(len); + // Fire-and-forget (wait=false): native esp_now_send returns once the frame is + // queued, with the real TX result delivered later through the send callback. + // The co-processor mirrors that — it acks enqueue immediately and reports the + // outcome via the async SEND event (on_send -> on_send_report). Waiting for + // the RPC RESP here would block the main loop for the full round-trip on + // every transmit. + return request(ESP_NOW_HOSTED_OP_SEND, hdr, sizeof(hdr), nullptr, 0, nullptr, /*wait=*/false, data, + static_cast(len)); +} + +esp_err_t esp_now_set_pmk(const uint8_t *pmk) { + if (pmk == nullptr) + return ESP_ERR_ESPNOW_ARG; + return request(ESP_NOW_HOSTED_OP_SET_PMK, pmk, 16, nullptr, 0, nullptr); +} + +// Remainder of the surface. Not used by ESPHome's espnow component +// today; provided so the whole header links and future callers get a defined +// (if unimplemented) symbol rather than a link error. Wire them through +// CustomRpc if a use case appears. +esp_err_t esp_now_get_peer(const uint8_t * /*peer_addr*/, esp_now_peer_info_t * /*peer*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_now_fetch_peer(bool /*from_head*/, esp_now_peer_info_t * /*peer*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_get_peer_num(esp_now_peer_num_t * /*num*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_set_wake_window(uint16_t /*window*/) { + return ESP_ERR_NOT_SUPPORTED; // power-save wake window is not forwarded; don't claim success +} +esp_err_t esp_now_set_peer_rate_config(const uint8_t * /*peer_addr*/, esp_now_rate_config_t * /*cfg*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_wifi_config_espnow_rate(wifi_interface_t /*ifx*/, wifi_phy_rate_t /*rate*/) { + return ESP_ERR_NOT_SUPPORTED; +} + +} // extern "C" + +#endif // CONFIG_IDF_TARGET_ESP32P4 diff --git a/esphome/components/esp32_hosted/esp_now_hosted_rpc.h b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h new file mode 100644 index 0000000000..bf68c759ee --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h @@ -0,0 +1,128 @@ +/* + * esp_now_hosted — ESP-NOW-over-CustomRpc wire protocol. + * + * Shared, byte-for-byte-identical contract between: + * - the host shim (esphome/components/esp32_hosted/esp_now_hosted.cpp) + * - the coprocessor firmware (esphome/esp-hosted-firmware) + * + * It rides esp-hosted's CustomRpc channel (RPC ID 388, "peer data transfer", + * available since esp-hosted v2.8.1), teaching the radio-less host <-> radio + * co-processor link to carry esp_now.h, which esp-hosted itself does not proxy + * (Espressif issue espressif/esp-hosted-mcu#19). + * + * KEEP THE TWO COPIES IN SYNC. The canonical copy lives here; the coprocessor + * firmware uses a verbatim copy. Both sides are little-endian, so these packed + * structs are wire-compatible with no byte-swapping. + */ + +#ifndef ESP_NOW_HOSTED_RPC_H +#define ESP_NOW_HOSTED_RPC_H + +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* ── CustomRpc message IDs (any uint32_t except 0xFFFFFFFF) ────────────────── + * One REQ handler slot on the device; three event handler slots on the host. + * The bytes spell "now" + index, a private range unlikely to clash with other + * CustomRpc users (e.g. the stock peer_data_transfer example's 1..6). */ +#define ESP_NOW_HOSTED_MSG_REQ 0x6E6F7701u /* host -> device : request envelope */ +#define ESP_NOW_HOSTED_MSG_RESP 0x6E6F7702u /* device -> host : reply to a REQ */ +#define ESP_NOW_HOSTED_MSG_RECV 0x6E6F7703u /* device -> host : async RX frame */ +#define ESP_NOW_HOSTED_MSG_SEND 0x6E6F7704u /* device -> host : async TX status */ + +/* ── Request opcodes ────────────────────────────────────────────────────── */ +enum { + ESP_NOW_HOSTED_OP_INIT = 1, /* esp_now_init + register device recv/send cbs */ + ESP_NOW_HOSTED_OP_DEINIT = 2, /* unregister cbs + esp_now_deinit */ + ESP_NOW_HOSTED_OP_ADD_PEER = 3, /* payload: esp_now_hosted_peer_t */ + ESP_NOW_HOSTED_OP_DEL_PEER = 4, /* payload: 6-byte peer MAC */ + ESP_NOW_HOSTED_OP_IS_PEER_EXIST = 5, /* payload: 6-byte MAC; ret: 1 byte bool */ + ESP_NOW_HOSTED_OP_SEND = 6, /* payload: esp_now_hosted_send_req_t */ + ESP_NOW_HOSTED_OP_GET_VERSION = 7, /* ret: uint32 version */ + ESP_NOW_HOSTED_OP_SET_PMK = 8, /* payload: 16-byte PMK */ + ESP_NOW_HOSTED_OP_MOD_PEER = 9, /* payload: esp_now_hosted_peer_t */ +}; + +/* Largest ESP-NOW payload we forward. ESP-NOW v2 (IDF >= 5.4) is 1470 B; well + * under esp-hosted's 8166 B CustomRpc cap, so the shim never truncates. */ +#define ESP_NOW_HOSTED_MAX_FRAME 1470u +/* Envelope slack for the largest opcode payload (a SEND req wrapping a frame). */ +#define ESP_NOW_HOSTED_MAX_PAYLOAD (ESP_NOW_HOSTED_MAX_FRAME + 16u) +/* Host request/response round-trip timeout over the transport. Generous: + * normal RTT is sub-millisecond, but Wi-Fi/BLE contention on the co-processor + * can stall the RX thread. */ +#define ESP_NOW_HOSTED_TIMEOUT_MS 2000 + +/* ── Envelopes ──────────────────────────────────────────────────────────── */ + +/* These payloads are shared verbatim with the C co-processor firmware, so they + * use C's `typedef struct {...} name;` idiom rather than C++ `using` aliases, + * which would not compile there. Silence clang-tidy's modernize-use-using for + * the shared struct block. */ +// NOLINTBEGIN(modernize-use-using) +typedef struct { + uint8_t opcode; /* one of ESP_NOW_HOSTED_OP_* */ + uint8_t seq; /* wraps 0..255; echoed in the response for matching */ + uint16_t payload_len; /* bytes of opcode-specific payload that follow */ + uint8_t payload[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_req_t; + +typedef struct { + uint8_t opcode; /* echoes the request opcode */ + uint8_t seq; /* echoes the request seq */ + int32_t status; /* esp_err_t from the native call on the co-processor */ + uint16_t ret_len; /* bytes of return payload that follow */ + uint8_t ret[]; /* flexible (e.g. version u32, is_peer_exist bool) */ +} __attribute__((packed)) esp_now_hosted_resp_t; + +/* ── Opcode payloads ────────────────────────────────────────────────────── */ + +/* esp_now_peer_info_t minus the host-only `priv` pointer, which is meaningless + * across the transport and never set by ESPHome's espnow component. */ +typedef struct { + uint8_t peer_addr[6]; + uint8_t lmk[16]; + uint8_t channel; /* 0 = current channel */ + uint8_t ifidx; /* wifi_interface_t (0=STA, 1=AP) */ + uint8_t encrypt; /* bool */ +} __attribute__((packed)) esp_now_hosted_peer_t; + +typedef struct { + uint8_t has_addr; /* 0 => peer_addr is NULL (broadcast to all peers) */ + uint8_t peer_addr[6]; + uint16_t data_len; + uint8_t data[]; /* flexible, up to ESP_NOW_HOSTED_MAX_FRAME */ +} __attribute__((packed)) esp_now_hosted_send_req_t; + +/* ── Async events (device -> host) ──────────────────────────────────────── */ + +/* Reconstructed on the host into an esp_now_recv_info_t + a minimal + * wifi_pkt_rx_ctrl_t. ESPHome's espnow reads info->src_addr, info->des_addr, + * info->rx_ctrl->rssi and info->rx_ctrl->timestamp. */ +typedef struct { + uint8_t src_addr[6]; + uint8_t des_addr[6]; + int8_t rssi; + uint8_t channel; + uint16_t data_len; + uint8_t data[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_recv_evt_t; + +typedef struct { + uint8_t des_addr[6]; + uint8_t status; /* esp_now_send_status_t (0 = success) */ +} __attribute__((packed)) esp_now_hosted_send_evt_t; +// NOLINTEND(modernize-use-using) + +#ifdef __cplusplus +} +#endif + +#endif /* ESP_NOW_HOSTED_RPC_H */ diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 5541a6ee97..14d099ec06 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -3,6 +3,7 @@ from typing import Any from esphome import automation, core import esphome.codegen as cg from esphome.components import wifi +from esphome.components.esp32 import VARIANT_ESP32P4, get_esp32_variant from esphome.components.udp import CONF_ON_RECEIVE import esphome.config_validation as cv from esphome.const import ( @@ -17,6 +18,7 @@ from esphome.const import ( ) from esphome.core import CORE, HexInt from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] @@ -132,6 +134,24 @@ CONFIG_SCHEMA = cv.All( ) +def _validate_variant(config: ConfigType) -> ConfigType: + # ESP-NOW rides the Wi-Fi PHY. Radio-less esp32 variants have no native + # ESP-NOW; only the ESP32-P4 has a path, via the esp32_hosted shim that + # supplies the esp_now_* symbols. Fail here with a clear message instead of + # letting the build reach an "undefined reference to esp_now_*" link error. + variant = get_esp32_variant() + if wifi.variant_has_wifi(variant): + return config + if variant != VARIANT_ESP32P4: + raise cv.Invalid(f"ESP-NOW is not supported on {variant} (no Wi-Fi radio)") + if "esp32_hosted" not in fv.full_config.get(): + raise cv.Invalid(f"ESP-NOW on {variant} requires the esp32_hosted component") + return config + + +FINAL_VALIDATE_SCHEMA = _validate_variant + + async def _trigger_to_code(config: ConfigType) -> MockObj: if address := config.get(CONF_ADDRESS): address = address.parts diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9dd1e0ced6..eaece6d5ff 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_ESP32_HOSTED #define USE_ESP32_HOSTED_HTTP_UPDATE #define USE_ESP32_IMPROV_STATE_CALLBACK +#define USE_ESP_NOW_HOSTED #define USE_EVENT #define USE_FAN #define USE_GPIO_BINARY_SENSOR_INTERRUPT diff --git a/script/ci-custom.py b/script/ci-custom.py index f481fda860..e2b7cd8d37 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -294,6 +294,9 @@ def highlight(s): "esphome/components/socket/headers.h", "esphome/core/defines.h", "esphome/components/http_request/httplib.h", + # Shared C wire header (byte-identical with the co-processor firmware); + # these are protocol constants and constexpr is C++-only. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_no_defines(fname, match): @@ -816,6 +819,10 @@ def lint_relative_py_import(fname: Path, line, col, content): "esphome/components/host/helpers.cpp", "esphome/components/zephyr/helpers.cpp", "esphome/components/http_request/httplib.h", + # Global extern "C" esp_now_* linker symbols + shared C wire header; + # neither can live in a C++ namespace. + "esphome/components/esp32_hosted/esp_now_hosted.cpp", + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: @@ -841,7 +848,15 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) +@lint_content_check( + include=["*.h"], + exclude=[ + "esphome/core/entity_types.h", + # Shared C wire header; uses a classic #ifndef guard for portability + # across the co-processor firmware repo it stays byte-identical with. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + ], +) def lint_pragma_once(fname, content): if "#pragma once" not in content: return ( diff --git a/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml new file mode 100644 index 0000000000..fab0a64ab8 --- /dev/null +++ b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml @@ -0,0 +1,5 @@ +# Exercises the ESP-NOW-over-hosted shim: on the ESP32-P4 host, esp32_hosted +# supplies the esp_now_* symbols that the espnow component links against. +packages: + esp32_hosted: !include common.yaml + espnow: !include ../espnow/common.yaml diff --git a/tests/unit_tests/components/test_espnow.py b/tests/unit_tests/components/test_espnow.py new file mode 100644 index 0000000000..21305c2b33 --- /dev/null +++ b/tests/unit_tests/components/test_espnow.py @@ -0,0 +1,48 @@ +"""Tests for the espnow component's final validation.""" + +import pytest + +from esphome.components.esp32.const import ( + VARIANT_ESP32C3, + VARIANT_ESP32H2, + VARIANT_ESP32P4, +) +from esphome.components.espnow import _validate_variant +import esphome.config_validation as cv +import esphome.final_validate as fv +from esphome.types import ConfigType + + +def _run( + monkeypatch, variant: str, full_config: dict, config: ConfigType +) -> ConfigType: + monkeypatch.setattr("esphome.components.espnow.get_esp32_variant", lambda: variant) + token = fv.full_config.set(full_config) + try: + return _validate_variant(config) + finally: + fv.full_config.reset(token) + + +def test_variant_with_native_wifi_passes(monkeypatch) -> None: + """A variant with a native Wi-Fi PHY needs no shim; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32C3, {}, config) is config + + +def test_radioless_non_p4_variant_rejected(monkeypatch) -> None: + """Radio-less variants without any ESP-NOW path are rejected outright.""" + with pytest.raises(cv.Invalid, match="not supported"): + _run(monkeypatch, VARIANT_ESP32H2, {}, {}) + + +def test_p4_without_esp32_hosted_rejected(monkeypatch) -> None: + """The P4 needs the esp32_hosted shim to supply the esp_now_* symbols.""" + with pytest.raises(cv.Invalid, match="esp32_hosted"): + _run(monkeypatch, VARIANT_ESP32P4, {}, {}) + + +def test_p4_with_esp32_hosted_passes(monkeypatch) -> None: + """The P4 with esp32_hosted present validates; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32P4, {"esp32_hosted": {}}, config) is config From 3321566cc010c3a4e774a78e1d81d887c8e02879 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 3 Sep 2026 07:12:36 -0500 Subject: [PATCH 18/53] [remote_transmitter] Fix BK7231N build by limiting the PWM path to BK7238 (#18958) --- esphome/components/remote_transmitter/__init__.py | 14 +++++--------- .../remote_transmitter/remote_transmitter.h | 9 +++++---- .../remote_transmitter_bk72xx.cpp | 11 +++++++---- .../remote_transmitter_libretiny_isr.cpp | 10 +++++----- .../remote_transmitter/test_non_blocking_gate.py | 2 +- .../remote_transmitter/test.bk72xx-ard.yaml | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index cb2aebec91..58392c48ab 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -4,11 +4,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base from esphome.components.libretiny import get_libretiny_family -from esphome.components.libretiny.const import ( - FAMILY_BK7231N, - FAMILY_BK7238, - FAMILY_RTL8720C, -) +from esphome.components.libretiny.const import FAMILY_BK7238, FAMILY_RTL8720C from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -49,7 +45,9 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) -_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7231N, FAMILY_BK7238) +# Keep in sync with the USE_LIBRETINY_VARIANT_RTL8720C / REMOTE_TRANSMITTER_BK_PWM gates in +# remote_transmitter.h, which decide where set_non_blocking() is declared +_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7238) def _validate_non_blocking_platform(value: bool) -> bool: @@ -59,9 +57,7 @@ def _validate_non_blocking_platform(value: bool) -> bool: return cv.boolean(value) if CORE.is_libretiny and get_libretiny_family() in _NON_BLOCKING_LIBRETINY_FAMILIES: return cv.boolean(value) - raise cv.Invalid( - "non_blocking is only supported on ESP32, RTL8720C, BK7231N and BK7238" - ) + raise cv.Invalid("non_blocking is only supported on ESP32, RTL8720C and BK7238") MULTI_CONF = True diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index 313b26364d..4db4e80a60 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -12,10 +12,11 @@ #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 -// The BK7231N-style PWM block (hardware shadow-load duty updates) enables the ISR-driven -// transmitter on these families; family-level proxy for the SDK's CFG_SOC_NAME gate. -// See remote_transmitter_bk72xx.cpp. -#if defined(USE_LIBRETINY_VARIANT_BK7231N) || defined(USE_LIBRETINY_VARIANT_BK7238) +// Enables the ISR-driven transmitter on Beken. Gated on BK7238 alone: the shadow-load PWM +// block is shared with BK7231N, but LibreTiny builds that family against an older BDK whose +// PWM driver has no pwm_init_param()/pwm_start(). See remote_transmitter_bk72xx.cpp. +// Keep in sync with _NON_BLOCKING_LIBRETINY_FAMILIES in __init__.py. +#ifdef USE_LIBRETINY_VARIANT_BK7238 #define REMOTE_TRANSMITTER_BK_PWM #endif diff --git a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp index 0081ae47b3..822389ccf9 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp @@ -9,10 +9,13 @@ // with the core's fixes for type-name collisions between the two #include -// Only the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) -// supports glitch-free per-edge duty updates; older SoCs compile the generic bit-bang -// implementation (remote_transmitter.cpp) instead, and this file compiles to nothing. -// REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h. +// Needs the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) +// for glitch-free per-edge duty updates, and an SDK exposing pwm_init_param()/pwm_start(). +// BK7231N has the block but LibreTiny builds it against an older BDK offering only the +// sddev_control API (CMD_PWM_INIT_PARAM), so it stays on the generic bit-bang path until +// someone can add and validate that path on real hardware. Every other Beken SoC lacks the +// block. REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h; when it is +// unset this file compiles to nothing and remote_transmitter.cpp is used instead. namespace esphome::remote_transmitter { diff --git a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp index 003cdfa986..fad91f593f 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp @@ -3,11 +3,11 @@ #include "esphome/core/hal.h" #include "esphome/core/log.h" -// Envelope chain shared by the LibreTiny families that pace transmission from a hardware -// timer interrupt: RTL8720C (gtimer) and the BK7231N-style PWM block (BKTIMER1). Everything -// platform-specific sits behind five hooks implemented in the per-family files -- carrier -// setup, duty writes, one-shot arming and timer stop. Families without a usable timer keep -// the generic bit-bang implementation and compile none of this. +// Envelope chain shared by the LibreTiny families that pace transmission from a hardware timer +// interrupt: RTL8720C (gtimer) and BK7238 (BKTIMER1). Everything platform-specific sits behind +// five hooks implemented in the per-family files -- carrier setup, duty writes, one-shot arming +// and timer stop. Families without a usable timer keep the generic bit-bang implementation and +// compile none of this. #if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) namespace esphome::remote_transmitter { diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py index ee2769e177..525ab3329e 100644 --- a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -26,7 +26,7 @@ from ..types import SetCoreConfigCallable (PlatformFramework.ESP32_IDF, None, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), - (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, False), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7238, True), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231T, False), (PlatformFramework.ESP8266_ARDUINO, None, False), diff --git a/tests/components/remote_transmitter/test.bk72xx-ard.yaml b/tests/components/remote_transmitter/test.bk72xx-ard.yaml index ea2feafda9..f3e2da9daf 100644 --- a/tests/components/remote_transmitter/test.bk72xx-ard.yaml +++ b/tests/components/remote_transmitter/test.bk72xx-ard.yaml @@ -2,7 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO26 carrier_duty_percent: 50% - # non_blocking is bk7231n/bk7238-only; the CI board is a BK7252 + # non_blocking is bk7238-only; the CI board is a BK7252, so this builds the bit-bang path packages: buttons: !include common-buttons.yaml From 657116a213de452fc191772c06e20aec09d16446 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:15:06 +0000 Subject: [PATCH 19/53] Bump bundled esphome-device-builder to 1.14.0 (#18960) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0da8048c57..7952616496 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 RUN \ platformio settings set enable_telemetry No \ From e47247486ba238f16f958a3298e08c43d309e6c7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 3 Sep 2026 21:16:36 +0200 Subject: [PATCH 20/53] [esp8266] Drop Arduino framework versions before 3.0.0 (#18917) to Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/arduino8266/framework.py | 13 ++--- esphome/components/climate/climate.cpp | 4 +- esphome/components/debug/debug_component.cpp | 4 +- esphome/components/debug/debug_component.h | 4 +- esphome/components/debug/debug_esp8266.cpp | 2 - esphome/components/debug/sensor.py | 7 +-- esphome/components/esp8266/__init__.py | 57 ++++++------------- .../nextion/nextion_upload_arduino.cpp | 6 -- esphome/components/wifi/wifi_component.h | 5 -- .../wifi/wifi_component_esp8266.cpp | 10 +--- esphome/core/log.h | 14 ----- .../components/esp8266/test_boards.py | 17 +----- .../esp8266/test_framework_version.py | 23 ++++++++ .../unit_tests/test_arduino8266_framework.py | 17 ++---- 14 files changed, 62 insertions(+), 121 deletions(-) create mode 100644 tests/unit_tests/components/esp8266/test_framework_version.py diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 1edbe4b36f..663002b3b1 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -44,8 +44,7 @@ def get_arduino8266_tools_path() -> Path: return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) -# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the -# encoder below cannot name 3.0.0/3.0.1 either (see its docstring) +# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0 MIN_FRAMEWORK_VERSION = Version(3, 1, 1) @@ -53,20 +52,16 @@ def framework_package_version(ver: Version) -> str: """Map an Arduino core version to its registry package version (3.1.2 -> 3.30102.0; the leading 3 is the package major). - Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor - at MIN_FRAMEWORK_VERSION. + Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION. """ if ver.major > 3: raise EsphomeError( f"Arduino core {ver} is not supported yet; " "the newest known core series is 3.x" ) - if ver <= Version(2, 6, 2): - # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same - # boundary as _format_framework_arduino_version's era guard) + if ver.major < 3: raise EsphomeError( - f"Arduino core {ver} uses an older package encoding than this " - "helper implements (newer than 2.6.2)" + f"Arduino core {ver} is not supported; ESPHome requires core 3.x" ) return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 34684a87e1..f80de151b1 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -368,8 +368,8 @@ optional Climate::restore_state_() { } void Climate::save_state_(const ClimateTraits &traits) { -#if (defined(USE_ESP32) || (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0))) && \ - !defined(CLANG_TIDY) +#if (defined(USE_ESP32) || defined(USE_ESP8266)) && !defined(CLANG_TIDY) +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wclass-memaccess" #define TEMP_IGNORE_MEMACCESS #endif diff --git a/esphome/components/debug/debug_component.cpp b/esphome/components/debug/debug_component.cpp index 9020c261c2..97f4522c62 100644 --- a/esphome/components/debug/debug_component.cpp +++ b/esphome/components/debug/debug_component.cpp @@ -22,9 +22,9 @@ void DebugComponent::dump_config() { LOG_SENSOR(" ", "Free space on heap", this->free_sensor_); LOG_SENSOR(" ", "Largest free heap block", this->block_sensor_); LOG_SENSOR(" ", "CPU frequency", this->cpu_frequency_sensor_); -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#ifdef USE_ESP8266 LOG_SENSOR(" ", "Heap fragmentation", this->fragmentation_sensor_); -#endif // defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#endif // USE_ESP8266 #endif // USE_SENSOR char device_info_buffer[DEVICE_INFO_BUFFER_SIZE]; diff --git a/esphome/components/debug/debug_component.h b/esphome/components/debug/debug_component.h index 20798cf600..b05029f878 100644 --- a/esphome/components/debug/debug_component.h +++ b/esphome/components/debug/debug_component.h @@ -35,7 +35,7 @@ class DebugComponent final : public PollingComponent { #ifdef USE_SENSOR void set_free_sensor(sensor::Sensor *free_sensor) { free_sensor_ = free_sensor; } void set_block_sensor(sensor::Sensor *block_sensor) { block_sensor_ = block_sensor; } -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) void set_fragmentation_sensor(sensor::Sensor *fragmentation_sensor) { fragmentation_sensor_ = fragmentation_sensor; } #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) @@ -61,7 +61,7 @@ class DebugComponent final : public PollingComponent { sensor::Sensor *free_sensor_{nullptr}; sensor::Sensor *block_sensor_{nullptr}; -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) sensor::Sensor *fragmentation_sensor_{nullptr}; #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) diff --git a/esphome/components/debug/debug_esp8266.cpp b/esphome/components/debug/debug_esp8266.cpp index 272123dfc0..acce28818c 100644 --- a/esphome/components/debug/debug_esp8266.cpp +++ b/esphome/components/debug/debug_esp8266.cpp @@ -159,12 +159,10 @@ void DebugComponent::update_platform_() { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->block_sensor_->publish_state(ESP.getMaxFreeBlockSize()); } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) if (this->fragmentation_sensor_ != nullptr) { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->fragmentation_sensor_->publish_state(ESP.getHeapFragmentation()); } -#endif #endif } diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index 72e2efebc2..e53cb0d1e4 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -52,12 +52,9 @@ CONFIG_SCHEMA = { ), cv.Optional(CONF_FRAGMENTATION): cv.All( cv.Any( - cv.All( - cv.only_on_esp8266, - cv.require_framework_version(esp8266_arduino=cv.Version(2, 5, 2)), - ), + cv.only_on_esp8266, cv.only_on_esp32, - msg="This feature is only available on ESP8266 (Arduino 2.5.2+) and ESP32", + msg="This feature is only available on ESP8266 and ESP32", ), sensor.sensor_schema( unit_of_measurement=UNIT_PERCENT, diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 63665e7681..19dbb68f29 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -35,7 +35,7 @@ from esphome.platformio.toolchain import copy_ccache_script from esphome.storage_json import StorageJSON from esphome.types import ConfigType -from .boards import BOARDS, ESP8266_LD_SCRIPTS, board_ld_script +from .boards import BOARDS, board_ld_script from .const import ( CONF_EARLY_PIN_INIT, CONF_ENABLE_SERIAL, @@ -43,8 +43,6 @@ from .const import ( CONF_RESTORE_FROM_FLASH, KEY_BOARD, KEY_ESP8266, - KEY_FLASH_SIZE, - KEY_LDSCRIPT, KEY_PIN_INITIAL_STATES, KEY_SERIAL1_REQUIRED, KEY_SERIAL_REQUIRED, @@ -133,10 +131,6 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/esp8266/Arduino/releases) version to # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 - if ver <= cv.Version(2, 4, 1): - return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - if ver <= cv.Version(2, 6, 2): - return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" # Same encoding the native toolchain uses for its package download, so a # version bump cannot drift between the two paths. from esphome.arduino8266.framework import framework_package_version @@ -159,11 +153,9 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # - https://github.com/esp8266/Arduino/releases # - https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(3, 1, 2) -# The platformio/espressif8266 version to use for arduino 2 framework versions +# The platformio/espressif8266 version to use for arduino 3 framework versions # - https://github.com/platformio/platform-espressif8266/releases # - https://api.registry.platformio.org/v3/packages/platformio/platform/espressif8266 -ARDUINO_2_PLATFORM_VERSION = cv.Version(2, 6, 3) -# for arduino 3 framework versions ARDUINO_3_PLATFORM_VERSION = cv.Version(3, 2, 0) # for arduino 4 framework versions ARDUINO_4_PLATFORM_VERSION = cv.Version(4, 2, 1) @@ -188,6 +180,14 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: version = cv.Version.parse(cv.version_number(value[CONF_VERSION])) source = value.get(CONF_SOURCE, None) + if version < cv.Version(3, 0, 0): + raise cv.Invalid( + f"Arduino framework {version} is no longer supported; ESPHome requires " + f"C++20, which needs Arduino core 3.x. Use the recommended version " + f"({RECOMMENDED_ARDUINO_FRAMEWORK_VERSION}).", + path=[CONF_VERSION], + ) + value[CONF_VERSION] = str(version) value[CONF_SOURCE] = source or _format_framework_arduino_version(version) @@ -195,12 +195,8 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: if platform_version is None: if version >= cv.Version(3, 1, 0): platform_version = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION)) - elif version >= cv.Version(3, 0, 0): - platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) - elif version >= cv.Version(2, 5, 0): - platform_version = _parse_platform_version(str(ARDUINO_2_PLATFORM_VERSION)) else: - platform_version = _parse_platform_version(str(cv.Version(1, 8, 0))) + platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) value[CONF_PLATFORM_VERSION] = platform_version if version != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION: @@ -289,29 +285,11 @@ def check_rosetta() -> None: ) -def _choose_ld_script(board: str, ver: cv.Version) -> str | None: - """The flash ld to pin for this board and core, or None for cores - without ld-script support.""" - board_data = BOARDS[board] - ld_scripts = ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]] - if ver <= cv.Version(2, 3, 0): - # No ld script support - return None - if ver <= cv.Version(2, 4, 2): - # Old ld script path; the modern per-board override names do not - # exist in this core's SDK, so the override cannot be honored. - # Substituting the size default would move _FS_end and the - # preferences sector, wiping flash-backed state on flash. - if KEY_LDSCRIPT in board_data: - raise EsphomeError( - f"Board {board} requires its {board_data[KEY_LDSCRIPT]} " - f"flash layout, which Arduino core {ver} cannot honor; " - "use a core newer than 2.4.2" - ) - return ld_scripts[0] +def _choose_ld_script(board: str) -> str: + """The flash ld to pin for this board.""" # A per-board override preserves a layout the board shipped with # (see d1_wroom_02 in boards.py) - return board_ld_script(board_data) + return board_ld_script(BOARDS[board]) @coroutine_with_priority(CoroPriority.PLATFORM) @@ -435,10 +413,9 @@ async def to_code(config: ConfigType) -> None: ) if config[CONF_BOARD] in BOARDS: - ld_script = _choose_ld_script(config[CONF_BOARD], ver) - - if ld_script is not None: - cg.add_platformio_option("board_build.ldscript", ld_script) + cg.add_platformio_option( + "board_build.ldscript", _choose_ld_script(config[CONF_BOARD]) + ) CORE.add_job(add_pin_initial_states_array) CORE.add_job(finalize_waveform_config) diff --git a/esphome/components/nextion/nextion_upload_arduino.cpp b/esphome/components/nextion/nextion_upload_arduino.cpp index f02f32d5ca..944fa1db47 100644 --- a/esphome/components/nextion/nextion_upload_arduino.cpp +++ b/esphome/components/nextion/nextion_upload_arduino.cpp @@ -209,14 +209,8 @@ bool Nextion::upload_tft(uint32_t baud_rate, bool exit_reparse) { http_client.setTimeout(this->tft_upload_http_timeout_); bool begin_status = false; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 7, 0) http_client.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); -#elif USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) - http_client.setFollowRedirects(true); -#endif -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) http_client.setRedirectLimit(3); -#endif begin_status = http_client.begin(*this->get_wifi_client_(), this->tft_url_.c_str()); if (!begin_status) { this->connection_state_.is_updating_ = false; diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index cfdbc1a968..63df9fbfa5 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -40,11 +40,6 @@ #include #include -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(2, 4, 0) -extern "C" { -#include -}; -#endif #endif #ifdef USE_RP2 diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index b4a91fb3cd..031da1b355 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -21,7 +21,6 @@ extern "C" { #include "lwip/apps/sntp.h" #include "lwip/netif.h" // struct netif #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) #include "LwipDhcpServer.h" #if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) #include @@ -30,7 +29,6 @@ extern "C" { #define wifi_softap_set_dhcps_lease_time(time) dhcpSoftAP.set_dhcps_lease_time(time) #define wifi_softap_set_dhcps_offer_option(offer, mode) dhcpSoftAP.set_dhcps_offer_option(offer, mode) #endif -#endif } #include "esphome/core/application.h" @@ -293,7 +291,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { conf.bssid_set = 0; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) if (ap.password_.empty()) { conf.threshold.authmode = AUTH_OPEN; } else { @@ -310,7 +307,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { } } conf.threshold.rssi = -127; -#endif ETS_UART_INTR_DISABLE(); bool ret = wifi_station_set_config_current(&conf); @@ -602,7 +598,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) case EVENT_OPMODE_CHANGED: { auto it = event->event_info.opmode_changed; ESP_LOGV(TAG, "Changed Mode old=%s new=%s", LOG_STR_ARG(get_op_mode_str(it.old_opmode)), @@ -620,7 +615,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#endif default: break; } @@ -705,7 +699,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.bssid = nullptr; config.channel = 0; config.show_hidden = 1; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE; // Use shorter dwell times for roaming scans - we only need to detect strong // nearby APs, not do a thorough survey. This also reduces off-channel time @@ -724,7 +717,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.scan_time.active.min = roaming ? SCAN_ACTIVE_MIN_ROAMING_MS : SCAN_ACTIVE_MIN_DEFAULT_MS; config.scan_time.active.max = roaming ? SCAN_ACTIVE_MAX_ROAMING_MS : SCAN_ACTIVE_MAX_DEFAULT_MS; } -#endif bool ret = wifi_station_scan(&config, &WiFiComponent::s_wifi_scan_done_callback); if (!ret) { ESP_LOGV(TAG, "wifi_station_scan failed"); @@ -830,7 +822,7 @@ bool WiFiComponent::wifi_ap_ip_config_(const optional &manual_ip) { return false; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) +#if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) dhcpSoftAP.begin(&info); #endif diff --git a/esphome/core/log.h b/esphome/core/log.h index 272e516808..14d24412ef 100644 --- a/esphome/core/log.h +++ b/esphome/core/log.h @@ -18,7 +18,6 @@ #ifdef USE_STORE_LOG_STR_IN_FLASH #include "WString.h" -#include "esphome/core/defines.h" // for USE_ARDUINO_VERSION_CODE #endif // Include ESP-IDF/Arduino based logging methods here so they don't undefine ours later @@ -177,20 +176,7 @@ struct LogString; #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 0) #define LOG_STR_ARG(s) ((PGM_P) (s)) -#else -// Pre-Arduino 2.5, we can't pass a PSTR() to printf(). Emulate support by copying the message to a -// local buffer first. String length is limited to 63 characters. -// https://github.com/esp8266/Arduino/commit/6280e98b0360f85fdac2b8f10707fffb4f6e6e31 -#define LOG_STR_ARG(s) \ - ({ \ - char __buf[64]; \ - __buf[63] = '\0'; \ - strncpy_P(__buf, (PGM_P) (s), 63); \ - __buf; \ - }) -#endif #define LOG_STR(s) (reinterpret_cast(PSTR(s))) #define LOG_STR_LITERAL(s) LOG_STR_ARG(LOG_STR(s)) diff --git a/tests/unit_tests/components/esp8266/test_boards.py b/tests/unit_tests/components/esp8266/test_boards.py index df0e536d42..78213a762a 100644 --- a/tests/unit_tests/components/esp8266/test_boards.py +++ b/tests/unit_tests/components/esp8266/test_boards.py @@ -1,11 +1,7 @@ """Tests for the per-board linker-script rule.""" -import pytest - from esphome.components.esp8266 import _choose_ld_script from esphome.components.esp8266.boards import BOARDS, board_ld_script -import esphome.config_validation as cv -from esphome.core import EsphomeError def test_d1_wroom_02_keeps_its_shipped_layout() -> None: @@ -21,13 +17,6 @@ def test_default_boards_use_the_flash_size_layout() -> None: def test_choose_ld_script_paths() -> None: - """Old cores get the size default, overriding boards hard-error there - (a substituted layout would wipe flash-backed state), modern cores - honor the override.""" - assert _choose_ld_script("nodemcuv2", cv.Version(2, 3, 0)) is None - assert _choose_ld_script("nodemcuv2", cv.Version(2, 4, 2)) == "eagle.flash.4m.ld" - assert _choose_ld_script("d1_wroom_02", cv.Version(2, 7, 4)) == ( - "eagle.flash.2m64.ld" - ) - with pytest.raises(EsphomeError, match="cannot honor"): - _choose_ld_script("d1_wroom_02", cv.Version(2, 4, 2)) + """Default boards get the size layout, overriding boards keep theirs.""" + assert _choose_ld_script("nodemcuv2") == "eagle.flash.4m.ld" + assert _choose_ld_script("d1_wroom_02") == "eagle.flash.2m64.ld" diff --git a/tests/unit_tests/components/esp8266/test_framework_version.py b/tests/unit_tests/components/esp8266/test_framework_version.py new file mode 100644 index 0000000000..0107aff8dd --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_framework_version.py @@ -0,0 +1,23 @@ +"""Tests for the Arduino framework version floor.""" + +import pytest + +from esphome.components.esp8266 import _arduino_check_versions +import esphome.config_validation as cv +from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION + + +def test_versions_before_3_are_rejected() -> None: + with pytest.raises(cv.Invalid, match="no longer supported") as excinfo: + _arduino_check_versions({CONF_VERSION: "2.7.4"}) + assert excinfo.value.path == [CONF_VERSION] + + +def test_supported_versions_pass() -> None: + value = _arduino_check_versions({CONF_VERSION: "3.0.2"}) + assert value[CONF_VERSION] == "3.0.2" + assert "espressif8266@3.2.0" in value[CONF_PLATFORM_VERSION] + + value = _arduino_check_versions({CONF_VERSION: "recommended"}) + assert value[CONF_VERSION] == "3.1.2" + assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION] diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index bd0a620e10..9f415344ae 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -21,17 +21,12 @@ def _build_path(tmp_path: Path) -> None: def test_framework_package_version() -> None: assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" - # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path) - assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0" # A future major bump needs its own encoding, not a doomed registry lookup with pytest.raises(EsphomeError, match="not supported yet"): framework.framework_package_version(cv.Version(4, 0, 0)) - # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release - # keeps this encoding - with pytest.raises(EsphomeError, match="older package encoding"): - framework.framework_package_version(cv.Version(2, 6, 2)) - assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0" - assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0" + # Cores before 3.x cannot build ESPHome (C++20) and are rejected + with pytest.raises(EsphomeError, match="requires core 3"): + framework.framework_package_version(cv.Version(2, 7, 4)) def test_format_framework_arduino_version_pins_all_series() -> None: @@ -39,10 +34,10 @@ def test_format_framework_arduino_version_pins_all_series() -> None: era, including the 4.x rejection it now shares with the installer.""" from esphome.components.esp8266 import _format_framework_arduino_version as fmt - assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0" - assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0" - assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0" assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" + # Pre-3 cores are rejected with the version line anchored + with pytest.raises(cv.Invalid, match="requires core 3"): + fmt(cv.Version(2, 7, 4)) # Anchored to the framework version line, not a bare EsphomeError with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: fmt(cv.Version(4, 0, 0)) From cb0c2bdaca67440249b415f4bb831c3906b83bbe Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:38:55 +1200 Subject: [PATCH 21/53] [esp32_ble] Reference count BLE advertising (#18943) --- esphome/components/esp32_ble/ble.cpp | 35 +++++++++++++++---- esphome/components/esp32_ble/ble.h | 13 +++++++ .../esp32_ble_beacon/esp32_ble_beacon.cpp | 2 ++ .../components/esp32_ble_server/__init__.py | 12 +++++++ .../esp32_ble_server/ble_server.cpp | 21 +++++++++-- .../components/esp32_ble_server/ble_server.h | 11 ++++++ .../esp32_improv/esp32_improv_component.cpp | 20 ++++++++++- .../esp32_improv/esp32_improv_component.h | 3 ++ .../esp32_ble_server/config/improv_only.yaml | 13 +++++++ .../config/manufacturer_data_only.yaml | 9 +++++ .../esp32_ble_server/config/own_service.yaml | 14 ++++++++ .../esp32_ble_server/test_esp32_ble_server.py | 28 +++++++++++++++ 12 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/esp32_ble_server/config/improv_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/own_service.yaml diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 6e6fb0e30d..fc95760cf8 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -100,21 +100,38 @@ void ESP32BLE::disable() { #ifdef USE_ESP32_BLE_ADVERTISING void ESP32BLE::advertising_start() { this->advertising_init_(); - if (!this->is_active()) + this->advertising_ref_count_++; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_stop() { + if (this->advertising_ref_count_ == 0) return; - this->advertising_->start(); + this->advertising_ref_count_--; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_refresh() { + if (this->advertising_ == nullptr || !this->is_active()) + return; + // Advertise while any component still needs it, otherwise stop + if (this->advertising_ref_count_ == 0) { + this->advertising_->stop(); + } else { + this->advertising_->start(); + } } void ESP32BLE::advertising_set_service_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_service_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_manufacturer_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_manufacturer_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_service_data_and_name(std::span data, bool include_name) { @@ -136,7 +153,7 @@ void ESP32BLE::advertising_set_service_data_and_name(std::span da this->advertising_->set_service_data(data); } - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_register_raw_advertisement_callback(std::function &&callback) { @@ -147,13 +164,13 @@ void ESP32BLE::advertising_register_raw_advertisement_callback(std::functionadvertising_init_(); this->advertising_->add_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_remove_service_uuid(ESPBTUUID uuid) { this->advertising_init_(); this->advertising_->remove_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } #endif @@ -575,6 +592,10 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { } this->state_ = BLE_COMPONENT_STATE_ACTIVE; +#ifdef USE_ESP32_BLE_ADVERTISING + // Requests made before the stack was up (or before it was re-enabled) take effect now + this->advertising_refresh(); +#endif } } diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 2a355a6c8b..7d2d0438a4 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -114,7 +114,17 @@ class ESP32BLE final : public Component { void set_name(const char *name) { this->name_ = name; } #ifdef USE_ESP32_BLE_ADVERTISING + /** Request advertising on behalf of a component. + * + * Requests are reference counted: advertising runs until every component that called + * advertising_start() has released it again with advertising_stop(). Each component must + * pair its calls, so nothing advertises until something actually asks for it. + */ void advertising_start(); + /// Release a request made with advertising_start(); advertising stops at the last release. + void advertising_stop(); + /// Apply the current payload and request count: advertise while requested, otherwise stop. + void advertising_refresh(); void advertising_set_service_data(const std::vector &data); void advertising_set_manufacturer_data(const std::vector &data); void advertising_set_appearance(uint16_t appearance) { this->appearance_ = appearance; } @@ -226,6 +236,9 @@ class ESP32BLE final : public Component { // 1-byte aligned members (grouped together to minimize padding) BLEComponentState state_{BLE_COMPONENT_STATE_OFF}; // 1 byte (uint8_t enum) bool enable_on_boot_{}; // 1 byte +#ifdef USE_ESP32_BLE_ADVERTISING + uint8_t advertising_ref_count_{0}; // 1 byte, number of components requesting advertising +#endif #ifdef ESPHOME_ESP32_BLE_EXTENDED_AUTH_PARAMS optional auth_req_mode_; diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp index 9f1723430b..ab728f9f6f 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp @@ -67,6 +67,8 @@ void ESP32BLEBeacon::setup() { this->on_advertise_(); } }); + // A beacon always needs the device to advertise, and never releases the request + global_ble->advertising_start(); } void ESP32BLEBeacon::on_advertise_() { diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 855a3be29b..d8095cd702 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -596,6 +596,18 @@ async def to_code(config): cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) cg.add(var.set_max_clients(config[CONF_MAX_CLIENTS])) + # Only advertise for the server itself when the configuration gives clients something to + # find. A server that is auto-loaded purely to host a runtime service (esp32_improv) stays + # silent until that service asks for advertising. + cg.add( + var.set_advertising_required( + CONF_MANUFACTURER_DATA in config + or any( + not uuid_is(service_config[CONF_UUID], DEVICE_INFORMATION_SERVICE_UUID) + for service_config in config[CONF_SERVICES] + ) + ) + ) if CONF_MANUFACTURER_DATA in config: cg.add(var.set_manufacturer_data(config[CONF_MANUFACTURER_DATA])) for service_config in config[CONF_SERVICES]: diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp index 2dea1666bb..45679b9b98 100644 --- a/esphome/components/esp32_ble_server/ble_server.cpp +++ b/esphome/components/esp32_ble_server/ble_server.cpp @@ -81,6 +81,7 @@ void BLEServer::loop() { if (this->device_information_service_->is_running()) { this->state_ = RUNNING; this->restart_advertising_(); + this->request_advertising_(); ESP_LOGD(TAG, "BLE server setup successfully"); } else if (this->device_information_service_->is_created()) { this->device_information_service_->start(); @@ -98,6 +99,20 @@ void BLEServer::restart_advertising_() { } } +void BLEServer::request_advertising_() { + if (!this->advertising_required_ || this->advertising_requested_) + return; + this->advertising_requested_ = true; + this->parent_->advertising_start(); +} + +void BLEServer::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + this->parent_->advertising_stop(); +} + BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles) { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE char uuid_buf[esp32_ble::UUID_STR_LEN]; @@ -170,7 +185,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga this->add_client_(param->connect.conn_id); // Resume advertising so additional clients can discover and connect if (this->client_count_ < this->max_clients_) { - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); } this->dispatch_callbacks_(CallbackType::ON_CONNECT, param->connect.conn_id); break; @@ -178,7 +193,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga case ESP_GATTS_DISCONNECT_EVT: { ESP_LOGD(TAG, "BLE Client disconnected"); this->remove_client_(param->disconnect.conn_id); - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); this->dispatch_callbacks_(CallbackType::ON_DISCONNECT, param->disconnect.conn_id); break; } @@ -226,6 +241,8 @@ void BLEServer::remove_client_(uint16_t conn_id) { } void BLEServer::ble_before_disabled_event_handler() { + // Advertising is re-requested once the server is running again after BLE is re-enabled + this->release_advertising_(); // Delete all clients this->client_count_ = 0; // Delete all services diff --git a/esphome/components/esp32_ble_server/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h index fdd92812cd..7869c73cc5 100644 --- a/esphome/components/esp32_ble_server/ble_server.h +++ b/esphome/components/esp32_ble_server/ble_server.h @@ -38,6 +38,13 @@ class BLEServer final : public Component, public Parented { this->restart_advertising_(); } + /** Whether this server needs the device to advertise so clients can find and connect to it. + * + * False for a server that only hosts services created at runtime (e.g. esp32_improv), which + * request advertising themselves for as long as they need it. + */ + void set_advertising_required(bool required) { this->advertising_required_ = required; } + void set_max_clients(uint8_t max_clients) { this->max_clients_ = max_clients; } uint8_t get_max_clients() const { return this->max_clients_; } @@ -82,6 +89,8 @@ class BLEServer final : public Component, public Parented { }; void restart_advertising_(); + void request_advertising_(); + void release_advertising_(); int8_t find_client_index_(uint16_t conn_id) const; void add_client_(uint16_t conn_id); @@ -93,6 +102,8 @@ class BLEServer final : public Component, public Parented { std::vector manufacturer_data_{}; esp_gatt_if_t gatts_if_{0}; bool registered_{false}; + bool advertising_required_{true}; + bool advertising_requested_{false}; uint16_t clients_[USE_ESP32_BLE_MAX_CONNECTIONS]{}; uint8_t client_count_{0}; diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 4756fba637..9ec6eb7bab 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -112,6 +112,7 @@ void ESP32ImprovComponent::loop() { this->state_callback_.call(this->state_, this->error_state_); #endif } + this->release_advertising_(); this->incoming_data_.clear(); return; } @@ -143,8 +144,9 @@ void ESP32ImprovComponent::loop() { ESP_LOGV(TAG, "Starting with device name advertising"); this->advertising_device_name_ = true; this->last_name_adv_time_ = App.get_loop_component_start_time(); + // Set the payload before requesting, so advertising starts exactly once esp32_ble::global_ble->advertising_set_service_data_and_name(std::span{}, true); - esp32_ble::global_ble->advertising_start(); + this->request_advertising_(); // Set initial state based on whether we have an authorizer this->set_state_(this->get_initial_state_(), false); @@ -326,6 +328,8 @@ void ESP32ImprovComponent::stop() { this->set_timeout("end-service", STOP_ADVERTISING_DELAY, [this] { if (this->state_ == improv::STATE_STOPPED || this->service_ == nullptr) return; + // Release first so removing the service UUID does not restart advertising on the way out + this->release_advertising_(); this->service_->stop(); this->set_state_(improv::STATE_STOPPED); }); @@ -520,6 +524,20 @@ void ESP32ImprovComponent::update_advertising_type_() { } } +void ESP32ImprovComponent::request_advertising_() { + if (this->advertising_requested_) + return; + this->advertising_requested_ = true; + esp32_ble::global_ble->advertising_start(); +} + +void ESP32ImprovComponent::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + esp32_ble::global_ble->advertising_stop(); +} + improv::State ESP32ImprovComponent::get_initial_state_() const { #ifdef USE_BINARY_SENSOR // If we have an authorizer, start in awaiting authorization state diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index 414948c977..a40d60552a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -104,8 +104,11 @@ class ESP32ImprovComponent final : public Component, public improv_base::ImprovB bool status_indicator_state_{false}; uint32_t last_name_adv_time_{0}; bool advertising_device_name_{false}; + bool advertising_requested_{false}; void set_status_indicator_state_(bool state); void update_advertising_type_(); + void request_advertising_(); + void release_advertising_(); void set_state_(improv::State state, bool update_advertising = true); void set_error_(improv::Error error); diff --git a/tests/component_tests/esp32_ble_server/config/improv_only.yaml b/tests/component_tests/esp32_ble_server/config/improv_only.yaml new file mode 100644 index 0000000000..8a5c3ba638 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/improv_only.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32 + +wifi: + ssid: MySSID + password: password1 + +# esp32_ble_server is only auto-loaded here, so it has no services of its own. +esp32_improv: + authorizer: none diff --git a/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml new file mode 100644 index 0000000000..b7bdae4af7 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + manufacturer_data: [0x72, 0x04, 0x00, 0x23] diff --git a/tests/component_tests/esp32_ble_server/config/own_service.yaml b/tests/component_tests/esp32_ble_server/config/own_service.yaml new file mode 100644 index 0000000000..c7ef0287b0 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/own_service.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + services: + - uuid: 2a24b789-7aab-4535-af3e-ee76a35cc12d + characteristics: + - uuid: cad48e28-7fbe-41cf-bae9-d77a6c233423 + read: true + value: [1, 2, 3, 4] diff --git a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py index 88307d0dcf..4b7ab79a81 100644 --- a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py +++ b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py @@ -1,5 +1,10 @@ """Tests for esp32_ble_server configuration helpers.""" +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + import pytest from esphome.components.esp32_ble_server import ( @@ -45,3 +50,26 @@ def test_uuid_is_matches_descriptor_short_strings(uuid16) -> None: assert uuid_is(uuid16, uuid16) assert uuid_is(f"{uuid16:04X}", uuid16) assert uuid_is(f"{uuid16:08X}", uuid16) + + +@pytest.mark.parametrize( + ("config_file", "required"), + [ + # Auto-loaded by esp32_improv only: nothing to find until Improv asks for it + ("improv_only.yaml", False), + # The configuration defines a service clients are meant to connect to + ("own_service.yaml", True), + # Manufacturer data is only useful if it is actually broadcast + ("manufacturer_data_only.yaml", True), + ], +) +def test_advertising_required( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + required: bool, +) -> None: + """The server only requests advertising when the configuration needs it.""" + main_cpp = generate_main(component_config_path(config_file)) + + assert f"set_advertising_required({str(required).lower()})" in main_cpp From e36445fa5feb4db65586186ec823a225339b8885 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sat, 5 Sep 2026 06:00:25 -0500 Subject: [PATCH 22/53] [usb_uart] Keep the comm interface number valid when its claim fails (#18968) --- esphome/components/usb_uart/usb_uart.cpp | 12 +++++++----- esphome/components/usb_uart/usb_uart.h | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/esphome/components/usb_uart/usb_uart.cpp b/esphome/components/usb_uart/usb_uart.cpp index cf66e4c369..60b7fe4e9c 100644 --- a/esphome/components/usb_uart/usb_uart.cpp +++ b/esphome/components/usb_uart/usb_uart.cpp @@ -434,11 +434,12 @@ void USBUartTypeCdcAcm::on_connected() { auto err_comm = usb_host_interface_claim(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number, 0); if (err_comm != ESP_OK) { + // Continue anyway: the interface number stays valid for CDC request addressing ESP_LOGW(TAG, "Could not claim comm interface %d: %s", channel->cdc_dev_.interrupt_interface_number, esp_err_to_name(err_comm)); - channel->cdc_dev_.interrupt_interface_number = 0xFF; // Mark as unavailable, but continue anyway } else { ESP_LOGD(TAG, "Claimed comm interface %d", channel->cdc_dev_.interrupt_interface_number); + channel->cdc_dev_.interrupt_interface_claimed = true; } } auto err = @@ -465,14 +466,15 @@ void USBUartTypeCdcAcm::on_disconnected() { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); } - if (channel->cdc_dev_.notify_ep != nullptr) { + // Only tear down the notify pipe when we claimed its interface ourselves; + // no transfer is ever submitted on it, so there is nothing else to cancel. + if (channel->cdc_dev_.notify_ep != nullptr && channel->cdc_dev_.interrupt_interface_claimed) { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); } - if (channel->cdc_dev_.interrupt_interface_number != 0xFF && - channel->cdc_dev_.interrupt_interface_number != channel->cdc_dev_.bulk_interface_number) { + if (channel->cdc_dev_.interrupt_interface_claimed) { usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number); - channel->cdc_dev_.interrupt_interface_number = 0xFF; + channel->cdc_dev_.interrupt_interface_claimed = false; } usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.bulk_interface_number); // Reset the input and output started flags to their initial state to avoid the possibility of spurious restarts diff --git a/esphome/components/usb_uart/usb_uart.h b/esphome/components/usb_uart/usb_uart.h index 00b34fb942..9d87bf964c 100644 --- a/esphome/components/usb_uart/usb_uart.h +++ b/esphome/components/usb_uart/usb_uart.h @@ -34,7 +34,10 @@ struct CdcEps { const usb_ep_desc_t *in_ep; const usb_ep_desc_t *out_ep; uint8_t bulk_interface_number; + // Also the wIndex target for CDC class requests (SET_LINE_CODING etc.), so it + // must remain valid even when the interface itself is not claimed. uint8_t interrupt_interface_number; + bool interrupt_interface_claimed{false}; }; enum CH34xChipType : uint8_t { From 745eb3010910a400c14527d0dd8e1fd5fa7ac984 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:07:15 +0200 Subject: [PATCH 23/53] Bump bundled esphome-device-builder to 1.14.1 (#18981) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7952616496..2d4ddbef5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 RUN \ platformio settings set enable_telemetry No \ From 7089dae3b63f57db6435202c14668001d0f0b595 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:25:58 +0000 Subject: [PATCH 24/53] Bump bundled esphome-device-builder to 1.14.2 (#18988) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d4ddbef5d..b5170864a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 RUN \ platformio settings set enable_telemetry No \ From 011497d6eeed511d55ec556db334f154f9dfc514 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:28:02 +0200 Subject: [PATCH 25/53] Bump bundled esphome-device-builder to 1.14.3 (#18996) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5170864a3..e875851bfb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 RUN \ platformio settings set enable_telemetry No \ From 18220e0b3940727d2f0657247ba0b749403b9db4 Mon Sep 17 00:00:00 2001 From: Ricardo Sanz Date: Sun, 6 Sep 2026 23:03:07 +0200 Subject: [PATCH 26/53] [climate][template] New template climate component (#14455) --- esphome/components/climate/__init__.py | 13 + .../components/template/climate/__init__.py | 465 ++++++++++++++++++ .../components/template/climate/automation.h | 57 +++ .../template/climate/template_climate.cpp | 164 ++++++ .../template/climate/template_climate.h | 92 ++++ esphome/config_validation.py | 1 + .../template/test_template_climate.py | 145 ++++++ tests/components/climate/common.yaml | 3 +- tests/components/template/common-base.yaml | 113 +++++ .../fixtures/template_climate_basic.yaml | 72 +++ .../template_climate_custom_modes.yaml | 47 ++ .../template_climate_nonoptimistic.yaml | 56 +++ .../template_climate_on_control_ordering.yaml | 26 + .../template_climate_publish_all_fields.yaml | 63 +++ .../template_climate_sensor_push.yaml | 49 ++ .../template_climate_set_actions.yaml | 89 ++++ ...emplate_climate_two_point_temperature.yaml | 52 ++ .../test_template_climate_basic.py | 146 ++++++ .../test_template_climate_custom_modes.py | 98 ++++ .../test_template_climate_nonoptimistic.py | 107 ++++ ...st_template_climate_on_control_ordering.py | 83 ++++ ...est_template_climate_publish_all_fields.py | 96 ++++ .../test_template_climate_sensor_push.py | 88 ++++ .../test_template_climate_set_actions.py | 114 +++++ ..._template_climate_two_point_temperature.py | 118 +++++ 25 files changed, 2355 insertions(+), 2 deletions(-) create mode 100644 esphome/components/template/climate/__init__.py create mode 100644 esphome/components/template/climate/automation.h create mode 100644 esphome/components/template/climate/template_climate.cpp create mode 100644 esphome/components/template/climate/template_climate.h create mode 100644 tests/component_tests/template/test_template_climate.py create mode 100644 tests/integration/fixtures/template_climate_basic.yaml create mode 100644 tests/integration/fixtures/template_climate_custom_modes.yaml create mode 100644 tests/integration/fixtures/template_climate_nonoptimistic.yaml create mode 100644 tests/integration/fixtures/template_climate_on_control_ordering.yaml create mode 100644 tests/integration/fixtures/template_climate_publish_all_fields.yaml create mode 100644 tests/integration/fixtures/template_climate_sensor_push.yaml create mode 100644 tests/integration/fixtures/template_climate_set_actions.yaml create mode 100644 tests/integration/fixtures/template_climate_two_point_temperature.yaml create mode 100644 tests/integration/test_template_climate_basic.py create mode 100644 tests/integration/test_template_climate_custom_modes.py create mode 100644 tests/integration/test_template_climate_nonoptimistic.py create mode 100644 tests/integration/test_template_climate_on_control_ordering.py create mode 100644 tests/integration/test_template_climate_publish_all_fields.py create mode 100644 tests/integration/test_template_climate_sensor_push.py create mode 100644 tests/integration/test_template_climate_set_actions.py create mode 100644 tests/integration/test_template_climate_two_point_temperature.py diff --git a/esphome/components/climate/__init__.py b/esphome/components/climate/__init__.py index 80dd913fba..3fbca1a6d0 100644 --- a/esphome/components/climate/__init__.py +++ b/esphome/components/climate/__init__.py @@ -125,6 +125,19 @@ CLIMATE_SWING_MODES = { validate_climate_swing_mode = cv.enum(CLIMATE_SWING_MODES, upper=True) +ClimateAction = climate_ns.enum("ClimateAction") +CLIMATE_ACTIONS = { + "OFF": ClimateAction.CLIMATE_ACTION_OFF, + "COOLING": ClimateAction.CLIMATE_ACTION_COOLING, + "HEATING": ClimateAction.CLIMATE_ACTION_HEATING, + "IDLE": ClimateAction.CLIMATE_ACTION_IDLE, + "DRYING": ClimateAction.CLIMATE_ACTION_DRYING, + "FAN": ClimateAction.CLIMATE_ACTION_FAN, + "DEFROSTING": ClimateAction.CLIMATE_ACTION_DEFROSTING, +} + +validate_climate_action = cv.enum(CLIMATE_ACTIONS, upper=True) + CONF_MIN_HUMIDITY = "min_humidity" CONF_MAX_HUMIDITY = "max_humidity" CONF_TARGET_HUMIDITY = "target_humidity" diff --git a/esphome/components/template/climate/__init__.py b/esphome/components/template/climate/__init__.py new file mode 100644 index 0000000000..c39ea8f80e --- /dev/null +++ b/esphome/components/template/climate/__init__.py @@ -0,0 +1,465 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import climate, sensor +from esphome.components.climate import climate_ns +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTION, + CONF_CURRENT_TEMPERATURE, + CONF_CUSTOM_FAN_MODE, + CONF_CUSTOM_FAN_MODES, + CONF_CUSTOM_PRESET, + CONF_CUSTOM_PRESETS, + CONF_FAN_MODE, + CONF_HUMIDITY_SENSOR, + CONF_ID, + CONF_INITIAL_STATE, + CONF_MODE, + CONF_OPTIMISTIC, + CONF_PRESET, + CONF_RESTORE_MODE, + CONF_SENSOR, + CONF_SUPPORTED_FAN_MODES, + CONF_SUPPORTED_MODES, + CONF_SUPPORTED_PRESETS, + CONF_SUPPORTED_SWING_MODES, + CONF_SWING_MODE, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.types import ConfigType + +from .. import template_ns + +CONF_CURRENT_HUMIDITY = "current_humidity" +CONF_TARGET_HUMIDITY = "target_humidity" +CONF_SUPPORTS_ACTION = "supports_action" +CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE = "supports_two_point_target_temperature" +CONF_SUPPORTS_TARGET_HUMIDITY = "supports_target_humidity" +CONF_SUPPORTS_CURRENT_TEMPERATURE = "supports_current_temperature" +CONF_SUPPORTS_CURRENT_HUMIDITY = "supports_current_humidity" +CONF_SET_MODE_ACTION = "set_mode_action" +CONF_SET_TARGET_TEMPERATURE_ACTION = "set_target_temperature_action" +CONF_SET_TARGET_TEMPERATURE_LOW_ACTION = "set_target_temperature_low_action" +CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION = "set_target_temperature_high_action" +CONF_SET_TARGET_HUMIDITY_ACTION = "set_target_humidity_action" +CONF_SET_FAN_MODE_ACTION = "set_fan_mode_action" +CONF_SET_CUSTOM_FAN_MODE_ACTION = "set_custom_fan_mode_action" +CONF_SET_SWING_MODE_ACTION = "set_swing_mode_action" +CONF_SET_PRESET_ACTION = "set_preset_action" +CONF_SET_CUSTOM_PRESET_ACTION = "set_custom_preset_action" + +TemplateClimate = template_ns.class_("TemplateClimate", climate.Climate, cg.Component) +TemplateClimatePublishAction = template_ns.class_( + "TemplateClimatePublishAction", + automation.Action, + cg.Parented.template(TemplateClimate), +) + +TemplateClimateRestoreMode = template_ns.enum( + "TemplateClimateRestoreMode", is_class=True +) +CLIMATE_RESTORE_MODES = { + "NO_RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + "RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +} + +# Per-field actions that forward a requested value on. The third item is the type of `x`. +SET_ACTIONS = ( + (CONF_SET_MODE_ACTION, "get_set_mode_trigger", climate.ClimateMode), + ( + CONF_SET_TARGET_TEMPERATURE_ACTION, + "get_set_target_temperature_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + "get_set_target_temperature_low_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + "get_set_target_temperature_high_trigger", + cg.float_, + ), + (CONF_SET_TARGET_HUMIDITY_ACTION, "get_set_target_humidity_trigger", cg.float_), + (CONF_SET_FAN_MODE_ACTION, "get_set_fan_mode_trigger", climate.ClimateFanMode), + ( + CONF_SET_CUSTOM_FAN_MODE_ACTION, + "get_set_custom_fan_mode_trigger", + cg.StringRef, + ), + ( + CONF_SET_SWING_MODE_ACTION, + "get_set_swing_mode_trigger", + climate.ClimateSwingMode, + ), + (CONF_SET_PRESET_ACTION, "get_set_preset_trigger", climate.ClimatePreset), + (CONF_SET_CUSTOM_PRESET_ACTION, "get_set_custom_preset_trigger", cg.StringRef), +) + +# supports_* keys have no default so that an omitted key can mean "derive it from the sensor or +# set action that makes the trait useful", which is not expressible once a default fills it in. +DERIVED_SUPPORTS = ( + (CONF_SUPPORTS_CURRENT_TEMPERATURE, (CONF_SENSOR,)), + (CONF_SUPPORTS_CURRENT_HUMIDITY, (CONF_HUMIDITY_SENSOR,)), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + ), + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, (CONF_SET_TARGET_HUMIDITY_ACTION,)), +) + + +# Custom fan modes/presets are opaque user-defined strings with no build-time correctness check +# elsewhere (Climate::set_supported_custom_fan_modes()/set_supported_custom_presets() don't block +# empty entries), so reject empty ones here -- they could never be selected at runtime anyway. +validate_custom_climate_string = cv.All(cv.string_strict, cv.Length(min=1)) + + +def _validate_two_point(config: ConfigType) -> ConfigType: + has_low = CONF_TARGET_TEMPERATURE_LOW in config + has_high = CONF_TARGET_TEMPERATURE_HIGH in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE_LOW}' and '{CONF_TARGET_TEMPERATURE_HIGH}' must be used together" + ) + if (has_low or has_high) and CONF_TARGET_TEMPERATURE in config: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' cannot be used together with " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}'" + ) + return config + + +def _validate_set_actions(config: ConfigType) -> ConfigType: + has_low = CONF_SET_TARGET_TEMPERATURE_LOW_ACTION in config + has_high = CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}' and " + f"'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}' must be used together" + ) + if (has_low or has_high) and CONF_SET_TARGET_TEMPERATURE_ACTION in config: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_ACTION}' cannot be used together with " + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}'/'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}'" + ) + return config + + +def _resolve_supports(config: ConfigType) -> ConfigType: + # An explicit true stays valid without either, since climate.template.publish can report the + # value; an explicit false that contradicts the configuration is an error, not a silent override. + for key, sources in DERIVED_SUPPORTS: + configured = [source for source in sources if source in config] + if key not in config: + config[key] = bool(configured) + elif not config[key] and configured: + raise cv.Invalid( + f"'{key}' cannot be false while '{configured[0]}' is configured", + path=[key], + ) + return config + + +def _validate_initial_state(config: ConfigType) -> ConfigType: + # Climate keeps target_temperature and target_temperature_low in a union, so writing the wrong + # one of the pair corrupts the setpoint with no runtime complaint. + if (initial_state := config.get(CONF_INITIAL_STATE)) is None: + return config + + two_point = config[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] + if two_point and CONF_TARGET_TEMPERATURE in initial_state: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' is not available while " + f"'{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' is enabled; use " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}' instead", + path=[CONF_INITIAL_STATE, CONF_TARGET_TEMPERATURE], + ) + if not two_point: + for key in (CONF_TARGET_TEMPERATURE_LOW, CONF_TARGET_TEMPERATURE_HIGH): + if key in initial_state: + raise cv.Invalid( + f"'{key}' requires '{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' to be enabled", + path=[CONF_INITIAL_STATE, key], + ) + if ( + CONF_TARGET_HUMIDITY in initial_state + and not config[CONF_SUPPORTS_TARGET_HUMIDITY] + ): + raise cv.Invalid( + f"'{CONF_TARGET_HUMIDITY}' requires '{CONF_SUPPORTS_TARGET_HUMIDITY}' to be enabled", + path=[CONF_INITIAL_STATE, CONF_TARGET_HUMIDITY], + ) + return config + + +# Same settable fields as climate.template.publish, minus current_temperature/current_humidity/ +# action: those are reported values (from a sensor or the device), not meaningful static defaults. +INITIAL_STATE_SCHEMA = cv.All( + cv.Schema( + { + cv.Optional(CONF_MODE): climate.validate_climate_mode, + cv.Optional(CONF_TARGET_TEMPERATURE): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.temperature, + cv.Optional(CONF_TARGET_HUMIDITY): cv.percentage_int, + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): climate.validate_climate_fan_mode, + cv.Exclusive( + CONF_CUSTOM_FAN_MODE, "fan_mode" + ): validate_custom_climate_string, + cv.Optional(CONF_SWING_MODE): climate.validate_climate_swing_mode, + cv.Exclusive(CONF_PRESET, "preset"): climate.validate_climate_preset, + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): validate_custom_climate_string, + } + ), + _validate_two_point, +) + +CONFIG_SCHEMA = cv.All( + climate.climate_schema(TemplateClimate) + .extend( + { + cv.Optional(CONF_SENSOR): cv.use_id(sensor.Sensor), + cv.Optional(CONF_HUMIDITY_SENSOR): cv.use_id(sensor.Sensor), + # action only ever arrives through climate.template.publish, so unlike the other + # supports_* keys there is no set action to derive it from. + cv.Optional(CONF_SUPPORTS_ACTION, default=False): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_HUMIDITY): cv.boolean, + cv.Optional(CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_TARGET_HUMIDITY): cv.boolean, + cv.Required(CONF_SUPPORTED_MODES): cv.All( + cv.ensure_list(climate.validate_climate_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_FAN_MODES): cv.All( + cv.ensure_list(climate.validate_climate_fan_mode), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_FAN_MODES): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_SWING_MODES): cv.All( + cv.ensure_list(climate.validate_climate_swing_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_PRESETS): cv.All( + cv.ensure_list(climate.validate_climate_preset), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_PRESETS): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_OPTIMISTIC, default=True): cv.boolean, + cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum( + CLIMATE_RESTORE_MODES, upper=True + ), + cv.Optional(CONF_INITIAL_STATE): INITIAL_STATE_SCHEMA, + cv.Optional(CONF_SET_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_HUMIDITY_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_FAN_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_CUSTOM_FAN_MODE_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_SWING_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_PRESET_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_CUSTOM_PRESET_ACTION): automation.validate_automation( + single=True + ), + } + ) + .extend(cv.COMPONENT_SCHEMA), + _validate_set_actions, + _resolve_supports, + _validate_initial_state, +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await climate.register_climate(var, config) + + if (sens := config.get(CONF_SENSOR)) is not None: + cg.add(var.set_sensor(await cg.get_variable(sens))) + + if (sens := config.get(CONF_HUMIDITY_SENSOR)) is not None: + cg.add(var.set_humidity_sensor(await cg.get_variable(sens))) + + for key, flag in ( + (CONF_SUPPORTS_ACTION, climate_ns.CLIMATE_SUPPORTS_ACTION), + ( + CONF_SUPPORTS_CURRENT_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_CURRENT_TEMPERATURE, + ), + (CONF_SUPPORTS_CURRENT_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_CURRENT_HUMIDITY), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_TARGET_HUMIDITY), + ): + if config[key]: + cg.add(var.add_feature_flags(flag)) + + for mode in config[CONF_SUPPORTED_MODES]: + cg.add(var.add_supported_mode(mode)) + + for mode in config.get(CONF_SUPPORTED_FAN_MODES, []): + cg.add(var.add_supported_fan_mode(mode)) + + if CONF_CUSTOM_FAN_MODES in config: + cg.add( + var.set_supported_custom_fan_modes( + cg.ArrayInitializer(*config[CONF_CUSTOM_FAN_MODES]) + ) + ) + + for mode in config.get(CONF_SUPPORTED_SWING_MODES, []): + cg.add(var.add_supported_swing_mode(mode)) + + for preset in config.get(CONF_SUPPORTED_PRESETS, []): + cg.add(var.add_supported_preset(preset)) + + if CONF_CUSTOM_PRESETS in config: + cg.add( + var.set_supported_custom_presets( + cg.ArrayInitializer(*config[CONF_CUSTOM_PRESETS]) + ) + ) + + for key, trigger_getter, arg_type in SET_ACTIONS: + if (conf := config.get(key)) is not None: + await automation.build_automation( + getattr(var, trigger_getter)(), [(arg_type, "x")], conf + ) + + cg.add(var.set_optimistic(config[CONF_OPTIMISTIC])) + cg.add(var.set_restore_mode(config[CONF_RESTORE_MODE])) + + if (initial_state := config.get(CONF_INITIAL_STATE)) is not None: + if (v := initial_state.get(CONF_MODE)) is not None: + cg.add(var.set_mode(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add(var.set_target_temperature_high(v)) + if (v := initial_state.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(v)) + if (v := initial_state.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(v)) + if (v := initial_state.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(v)) + if (v := initial_state.get(CONF_SWING_MODE)) is not None: + cg.add(var.set_swing_mode(v)) + if (v := initial_state.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(v)) + if (v := initial_state.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(v)) + + +CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.use_id(TemplateClimate), + cv.Optional(CONF_CURRENT_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_CURRENT_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_TARGET_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_MODE): cv.templatable(climate.validate_climate_mode), + cv.Optional(CONF_ACTION): cv.templatable(climate.validate_climate_action), + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): cv.templatable( + climate.validate_climate_fan_mode + ), + cv.Exclusive(CONF_CUSTOM_FAN_MODE, "fan_mode"): cv.templatable( + validate_custom_climate_string + ), + cv.Optional(CONF_SWING_MODE): cv.templatable( + climate.validate_climate_swing_mode + ), + cv.Exclusive(CONF_PRESET, "preset"): cv.templatable( + climate.validate_climate_preset + ), + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): cv.templatable( + validate_custom_climate_string + ), + } + ), + _validate_two_point, +) + + +@automation.register_action( + "climate.template.publish", + TemplateClimatePublishAction, + CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA, + synchronous=True, +) +async def climate_template_publish_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + + if (v := config.get(CONF_CURRENT_TEMPERATURE)) is not None: + cg.add(var.set_current_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_CURRENT_HUMIDITY)) is not None: + cg.add(var.set_current_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add( + var.set_target_temperature_high(await cg.templatable(v, args, cg.float_)) + ) + if (v := config.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_MODE)) is not None: + cg.add(var.set_mode(await cg.templatable(v, args, climate.ClimateMode))) + if (v := config.get(CONF_ACTION)) is not None: + cg.add(var.set_action(await cg.templatable(v, args, climate.ClimateAction))) + if (v := config.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(await cg.templatable(v, args, climate.ClimateFanMode))) + if (v := config.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(await cg.templatable(v, args, cg.std_string))) + if (v := config.get(CONF_SWING_MODE)) is not None: + cg.add( + var.set_swing_mode(await cg.templatable(v, args, climate.ClimateSwingMode)) + ) + if (v := config.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(await cg.templatable(v, args, climate.ClimatePreset))) + if (v := config.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(await cg.templatable(v, args, cg.std_string))) + + return var diff --git a/esphome/components/template/climate/automation.h b/esphome/components/template/climate/automation.h new file mode 100644 index 0000000000..49a79ace2f --- /dev/null +++ b/esphome/components/template/climate/automation.h @@ -0,0 +1,57 @@ +#pragma once + +#include "template_climate.h" +#include "esphome/core/automation.h" + +namespace esphome::template_ { + +template +class TemplateClimatePublishAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(float, current_temperature) + TEMPLATABLE_VALUE(float, current_humidity) + TEMPLATABLE_VALUE(float, target_temperature) + TEMPLATABLE_VALUE(float, target_temperature_low) + TEMPLATABLE_VALUE(float, target_temperature_high) + TEMPLATABLE_VALUE(float, target_humidity) + TEMPLATABLE_VALUE(climate::ClimateMode, mode) + TEMPLATABLE_VALUE(climate::ClimateAction, action) + TEMPLATABLE_VALUE(climate::ClimateFanMode, fan_mode) + TEMPLATABLE_VALUE(std::string, custom_fan_mode) + TEMPLATABLE_VALUE(climate::ClimateSwingMode, swing_mode) + TEMPLATABLE_VALUE(climate::ClimatePreset, preset) + TEMPLATABLE_VALUE(std::string, custom_preset) + + void play(const Ts &...x) override { + if (this->current_temperature_.has_value()) + this->parent_->current_temperature = this->current_temperature_.value(x...); + if (this->current_humidity_.has_value()) + this->parent_->current_humidity = this->current_humidity_.value(x...); + if (this->target_temperature_.has_value()) + this->parent_->set_target_temperature(this->target_temperature_.value(x...)); + if (this->target_temperature_low_.has_value()) + this->parent_->set_target_temperature_low(this->target_temperature_low_.value(x...)); + if (this->target_temperature_high_.has_value()) + this->parent_->set_target_temperature_high(this->target_temperature_high_.value(x...)); + if (this->target_humidity_.has_value()) + this->parent_->set_target_humidity(this->target_humidity_.value(x...)); + if (this->mode_.has_value()) + this->parent_->set_mode(this->mode_.value(x...)); + if (this->action_.has_value()) + this->parent_->action = this->action_.value(x...); + if (this->fan_mode_.has_value()) + this->parent_->set_fan_mode(this->fan_mode_.value(x...)); + if (this->custom_fan_mode_.has_value()) + this->parent_->set_custom_fan_mode(StringRef(this->custom_fan_mode_.value(x...))); + if (this->swing_mode_.has_value()) + this->parent_->set_swing_mode(this->swing_mode_.value(x...)); + if (this->preset_.has_value()) + this->parent_->set_preset(this->preset_.value(x...)); + if (this->custom_preset_.has_value()) + this->parent_->set_custom_preset(StringRef(this->custom_preset_.value(x...))); + + this->parent_->publish_state(); + } +}; + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.cpp b/esphome/components/template/climate/template_climate.cpp new file mode 100644 index 0000000000..a7a4d2ccab --- /dev/null +++ b/esphome/components/template/climate/template_climate.cpp @@ -0,0 +1,164 @@ +#include "template_climate.h" +#include "esphome/core/log.h" + +namespace esphome::template_ { + +static const char *const TAG = "template.climate"; + +void TemplateClimate::setup() { + if (this->restore_mode_ == TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE) { + auto restore = this->restore_state_(); + if (restore.has_value()) { + restore->apply(this); + } + } + + // Sensors publish every reading, not just changes, so only re-publish when the value moved. + // NAN means the sensor went unavailable and is passed through rather than dropped; the second + // check stops an unavailable sensor re-publishing forever, since NAN never equals NAN. +#ifdef USE_SENSOR + if (this->sensor_ != nullptr) { + this->current_temperature = this->sensor_->state; + this->sensor_->add_on_state_callback([this](float state) { + if (state != this->current_temperature && !(std::isnan(state) && std::isnan(this->current_temperature))) { + this->current_temperature = state; + this->publish_state(); + } + }); + } + + if (this->humidity_sensor_ != nullptr) { + this->current_humidity = this->humidity_sensor_->state; + this->humidity_sensor_->add_on_state_callback([this](float state) { + if (state != this->current_humidity && !(std::isnan(state) && std::isnan(this->current_humidity))) { + this->current_humidity = state; + this->publish_state(); + } + }); + } +#endif +} + +void TemplateClimate::dump_config() { + LOG_CLIMATE("", "Template Climate", this); + ESP_LOGCONFIG(TAG, " Optimistic: %s", YESNO(this->optimistic_)); +} + +void TemplateClimate::control(const climate::ClimateCall &call) { + // Each field present fires its set_*_action; on_control sees the whole call. optimistic: true + // also applies the values right away, false waits for a climate.template.publish report. + if (auto mode = call.get_mode()) { + if (this->optimistic_) + this->mode = *mode; + this->set_mode_trigger_.trigger(*mode); + } + + if (auto target_temp = call.get_target_temperature()) { + if (this->optimistic_) + this->target_temperature = *target_temp; + this->set_target_temperature_trigger_.trigger(*target_temp); + } + + if (auto target_temp_low = call.get_target_temperature_low()) { + if (this->optimistic_) + this->target_temperature_low = *target_temp_low; + this->set_target_temperature_low_trigger_.trigger(*target_temp_low); + } + + if (auto target_temp_high = call.get_target_temperature_high()) { + if (this->optimistic_) + this->target_temperature_high = *target_temp_high; + this->set_target_temperature_high_trigger_.trigger(*target_temp_high); + } + + if (auto target_humidity = call.get_target_humidity()) { + if (this->optimistic_) + this->target_humidity = *target_humidity; + this->set_target_humidity_trigger_.trigger(*target_humidity); + } + + if (auto fan_mode = call.get_fan_mode()) { + if (this->optimistic_) + this->set_fan_mode_(*fan_mode); + this->set_fan_mode_trigger_.trigger(*fan_mode); + } + + if (call.has_custom_fan_mode()) { + if (this->optimistic_) + this->set_custom_fan_mode_(call.get_custom_fan_mode()); + this->set_custom_fan_mode_trigger_.trigger(call.get_custom_fan_mode()); + } + + if (auto swing_mode = call.get_swing_mode()) { + if (this->optimistic_) + this->swing_mode = *swing_mode; + this->set_swing_mode_trigger_.trigger(*swing_mode); + } + + if (auto preset = call.get_preset()) { + if (this->optimistic_) + this->set_preset_(*preset); + this->set_preset_trigger_.trigger(*preset); + } + + if (call.has_custom_preset()) { + if (this->optimistic_) + this->set_custom_preset_(call.get_custom_preset()); + this->set_custom_preset_trigger_.trigger(call.get_custom_preset()); + } + + if (this->optimistic_) + this->publish_state(); +} + +// A climate.template.publish report (and initial_state:) never goes through ClimateCall::validate_(), +// so check here instead -- otherwise a typo is published as state the receiving end will reject. +void TemplateClimate::set_mode(climate::ClimateMode mode) { + if (!this->traits_.supports_mode(mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported mode %u", this->get_name().c_str(), static_cast(mode)); + return; + } + this->mode = mode; +} + +void TemplateClimate::set_swing_mode(climate::ClimateSwingMode swing_mode) { + if (!this->traits_.supports_swing_mode(swing_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported swing mode %u", this->get_name().c_str(), static_cast(swing_mode)); + return; + } + this->swing_mode = swing_mode; +} + +void TemplateClimate::set_fan_mode(climate::ClimateFanMode fan_mode) { + if (!this->traits_.supports_fan_mode(fan_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported fan mode %u", this->get_name().c_str(), static_cast(fan_mode)); + return; + } + this->set_fan_mode_(fan_mode); +} + +void TemplateClimate::set_preset(climate::ClimatePreset preset) { + if (!this->traits_.supports_preset(preset)) { + ESP_LOGW(TAG, "'%s' - Unsupported preset %u", this->get_name().c_str(), static_cast(preset)); + return; + } + this->set_preset_(preset); +} + +void TemplateClimate::set_custom_fan_mode(StringRef mode) { + if (this->find_custom_fan_mode_(mode.c_str(), mode.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom fan mode '%s'", this->get_name().c_str(), mode.c_str()); + return; + } + this->set_custom_fan_mode_(mode); +} + +void TemplateClimate::set_custom_preset(StringRef preset) { + if (this->find_custom_preset_(preset.c_str(), preset.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom preset '%s'", this->get_name().c_str(), preset.c_str()); + return; + } + this->set_custom_preset_(preset); +} + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.h b/esphome/components/template/climate/template_climate.h new file mode 100644 index 0000000000..5448488c34 --- /dev/null +++ b/esphome/components/template/climate/template_climate.h @@ -0,0 +1,92 @@ +#pragma once + +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/components/climate/climate.h" +#ifdef USE_SENSOR +#include "esphome/components/sensor/sensor.h" +#endif + +namespace esphome::template_ { + +enum class TemplateClimateRestoreMode { + TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +}; + +class TemplateClimate final : public climate::Climate, public Component { + public: + void setup() override; + void dump_config() override; + + climate::ClimateTraits traits() override { return this->traits_; } + + void add_feature_flags(uint32_t flags) { this->traits_.add_feature_flags(flags); } + +#ifdef USE_SENSOR + // The matching feature flag is added from codegen, so the configuration alone decides it. + void set_sensor(sensor::Sensor *sensor) { this->sensor_ = sensor; } + void set_humidity_sensor(sensor::Sensor *sensor) { this->humidity_sensor_ = sensor; } +#endif + + void add_supported_mode(climate::ClimateMode mode) { this->traits_.add_supported_mode(mode); } + void add_supported_fan_mode(climate::ClimateFanMode mode) { this->traits_.add_supported_fan_mode(mode); } + void add_supported_swing_mode(climate::ClimateSwingMode mode) { this->traits_.add_supported_swing_mode(mode); } + void add_supported_preset(climate::ClimatePreset preset) { this->traits_.add_supported_preset(preset); } + + void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } + void set_restore_mode(TemplateClimateRestoreMode restore_mode) { this->restore_mode_ = restore_mode; } + + // Fired from control() for each field the call carries, so a device-backed config can forward + // it on. Which of these are configured also decides the two-point/target-humidity traits. + Trigger *get_set_mode_trigger() { return &this->set_mode_trigger_; } + Trigger *get_set_target_temperature_trigger() { return &this->set_target_temperature_trigger_; } + Trigger *get_set_target_temperature_low_trigger() { return &this->set_target_temperature_low_trigger_; } + Trigger *get_set_target_temperature_high_trigger() { return &this->set_target_temperature_high_trigger_; } + Trigger *get_set_target_humidity_trigger() { return &this->set_target_humidity_trigger_; } + Trigger *get_set_fan_mode_trigger() { return &this->set_fan_mode_trigger_; } + Trigger *get_set_custom_fan_mode_trigger() { return &this->set_custom_fan_mode_trigger_; } + Trigger *get_set_swing_mode_trigger() { return &this->set_swing_mode_trigger_; } + Trigger *get_set_preset_trigger() { return &this->set_preset_trigger_; } + Trigger *get_set_custom_preset_trigger() { return &this->set_custom_preset_trigger_; } + + // Used by TemplateClimatePublishAction, which is not a Climate subclass and so cannot reach the + // protected setters, and by codegen to apply `initial_state:` before setup() runs. + void set_target_temperature(float value) { this->target_temperature = value; } + void set_target_temperature_low(float value) { this->target_temperature_low = value; } + void set_target_temperature_high(float value) { this->target_temperature_high = value; } + void set_target_humidity(float value) { this->target_humidity = value; } + void set_mode(climate::ClimateMode mode); + void set_swing_mode(climate::ClimateSwingMode mode); + void set_fan_mode(climate::ClimateFanMode mode); + void set_custom_fan_mode(const char *mode) { this->set_custom_fan_mode(StringRef(mode)); } + void set_custom_fan_mode(StringRef mode); + void set_preset(climate::ClimatePreset preset); + void set_custom_preset(const char *preset) { this->set_custom_preset(StringRef(preset)); } + void set_custom_preset(StringRef preset); + + protected: + void control(const climate::ClimateCall &call) override; + + climate::ClimateTraits traits_; + bool optimistic_{false}; + TemplateClimateRestoreMode restore_mode_{TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE}; + +#ifdef USE_SENSOR + sensor::Sensor *sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; +#endif + + Trigger set_mode_trigger_; + Trigger set_target_temperature_trigger_; + Trigger set_target_temperature_low_trigger_; + Trigger set_target_temperature_high_trigger_; + Trigger set_target_humidity_trigger_; + Trigger set_fan_mode_trigger_; + Trigger set_custom_fan_mode_trigger_; + Trigger set_swing_mode_trigger_; + Trigger set_preset_trigger_; + Trigger set_custom_preset_trigger_; +}; + +} // namespace esphome::template_ diff --git a/esphome/config_validation.py b/esphome/config_validation.py index aff39201e8..685a9d04b3 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -133,6 +133,7 @@ Upper = vol.Upper Length = vol.Length Exclusive = vol.Exclusive Inclusive = vol.Inclusive +Unique = vol.Unique ALLOW_EXTRA = vol.ALLOW_EXTRA UNDEFINED = vol.UNDEFINED RequiredFieldInvalid = vol.RequiredFieldInvalid diff --git a/tests/component_tests/template/test_template_climate.py b/tests/component_tests/template/test_template_climate.py new file mode 100644 index 0000000000..304991ea64 --- /dev/null +++ b/tests/component_tests/template/test_template_climate.py @@ -0,0 +1,145 @@ +"""Tests for template climate config validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.template.climate import ( + CONF_SET_TARGET_HUMIDITY_ACTION, + CONF_SET_TARGET_TEMPERATURE_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SUPPORTS_CURRENT_HUMIDITY, + CONF_SUPPORTS_CURRENT_TEMPERATURE, + CONF_SUPPORTS_TARGET_HUMIDITY, + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + CONF_TARGET_HUMIDITY, + _resolve_supports, + _validate_initial_state, + _validate_set_actions, +) +from esphome.const import ( + CONF_HUMIDITY_SENSOR, + CONF_INITIAL_STATE, + CONF_SENSOR, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.types import ConfigType + + +def test_supports_current_temperature_derived_from_sensor() -> None: + config: ConfigType = {CONF_SENSOR: "some_sensor"} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_without_sensor() -> None: + assert _resolve_supports({})[CONF_SUPPORTS_CURRENT_TEMPERATURE] is False + + +def test_supports_current_temperature_explicit_true_without_sensor_allowed() -> None: + # The value can still be reported with climate.template.publish. + config: ConfigType = {CONF_SUPPORTS_CURRENT_TEMPERATURE: True} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_supports_current_humidity_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_HUMIDITY_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_HUMIDITY: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_two_point_derived_from_set_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + assert _resolve_supports(config)[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] is True + + +def test_two_point_false_with_set_action_rejected() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_target_humidity_derived_from_set_action() -> None: + config: ConfigType = {CONF_SET_TARGET_HUMIDITY_ACTION: [{}]} + assert _resolve_supports(config)[CONF_SUPPORTS_TARGET_HUMIDITY] is True + + +def test_set_target_temperature_low_requires_high() -> None: + config: ConfigType = {CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}]} + with pytest.raises(cv.Invalid, match="must be used together"): + _validate_set_actions(config) + + +def test_set_target_temperature_conflicts_with_two_point_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + with pytest.raises(cv.Invalid, match="cannot be used together"): + _validate_set_actions(config) + + +def test_initial_state_target_temperature_rejected_with_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_TEMPERATURE: 21.0}, + } + with pytest.raises(cv.Invalid, match="is not available"): + _validate_initial_state(config) + + +def test_initial_state_two_point_values_rejected_without_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + }, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_target_humidity_rejected_without_support() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_HUMIDITY: 50}, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_matching_two_point_accepted() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: True, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + CONF_TARGET_HUMIDITY: 50, + }, + } + assert _validate_initial_state(config) is config diff --git a/tests/components/climate/common.yaml b/tests/components/climate/common.yaml index c28fde8eeb..49386a16d5 100644 --- a/tests/components/climate/common.yaml +++ b/tests/components/climate/common.yaml @@ -30,8 +30,7 @@ climate: - switch.turn_on: climate_heater_switch - switch.turn_off: climate_cooler_switch # Thermostat-based climate so climate.control: action variants get build - # coverage (bang_bang doesn't support fan modes, presets, etc.). Climate - # has no template platform, so thermostat is the right vehicle. + # coverage (bang_bang doesn't support fan modes, presets, etc.). - platform: thermostat id: climate_test_thermostat name: Test Thermostat diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index 92a1fc8eda..02aedaf167 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -25,6 +25,27 @@ esphome: away: !lambda "return true;" is_on: !lambda "return false;" + - climate.template.publish: + id: template_climate + current_temperature: 21.0 + mode: HEAT + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE + target_temperature: 22.0 + + # Templated + - climate.template.publish: + id: template_climate + current_temperature: !lambda "return 21.5f;" + mode: !lambda "return climate::CLIMATE_MODE_COOL;" + target_temperature: !lambda "return 23.0f;" + + - climate.template.publish: + id: template_climate_custom_modes + custom_fan_mode: "turbo" + custom_preset: "eco_plus" + # Test C++ API: set_template() with stateless lambda (no captures) # NOTE: set_template() is not intended to be a public API, but we test it to ensure it doesn't break. - lambda: |- @@ -513,6 +534,98 @@ alarm_control_panel: codes: - "1234" +climate: + - platform: template + id: template_climate + name: "Template Climate" + optimistic: true + sensor: template_template_sens + supports_action: true + supports_current_humidity: true + restore_mode: NO_RESTORE + initial_state: + mode: HEAT + target_temperature: 21.0 + fan_mode: LOW + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_action: + - logger.log: + format: "set_target_temperature_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.1f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + on_control: + - logger.log: "on_control fired" + on_state: + - logger.log: "on_state fired" + + - platform: template + id: template_climate_custom_modes + name: "Template Climate Custom Modes" + optimistic: true + sensor: template_template_sens + supported_modes: + - "OFF" + - HEAT + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + initial_state: + custom_fan_mode: eco + custom_preset: max + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + water_heater: - platform: template id: template_water_heater diff --git a/tests/integration/fixtures/template_climate_basic.yaml b/tests/integration/fixtures/template_climate_basic.yaml new file mode 100644 index 0000000000..51558b4875 --- /dev/null +++ b/tests/integration/fixtures/template_climate_basic.yaml @@ -0,0 +1,72 @@ +esphome: + name: tmpl-clim-basic + on_boot: + - climate.template.publish: + id: test_climate + action: IDLE +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Basic Climate + optimistic: true + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 55.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: "OFF" + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE diff --git a/tests/integration/fixtures/template_climate_custom_modes.yaml b/tests/integration/fixtures/template_climate_custom_modes.yaml new file mode 100644 index 0000000000..9dbfe60cb9 --- /dev/null +++ b/tests/integration/fixtures/template_climate_custom_modes.yaml @@ -0,0 +1,47 @@ +esphome: + name: tmpl-clim-custom +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Custom Mode Climate + optimistic: true + sensor: test_climate_current_temperature + supported_modes: + - "OFF" + - HEAT + - COOL + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + on_control: + - lambda: |- + if (x.has_custom_fan_mode()) + ESP_LOGD("test", "on_control custom_fan_mode=%s", x.get_custom_fan_mode().c_str()); + if (x.has_custom_preset()) + ESP_LOGD("test", "on_control custom_preset=%s", x.get_custom_preset().c_str()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + custom_fan_mode: "eco" + custom_preset: "max" diff --git a/tests/integration/fixtures/template_climate_nonoptimistic.yaml b/tests/integration/fixtures/template_climate_nonoptimistic.yaml new file mode 100644 index 0000000000..2b0c7ee132 --- /dev/null +++ b/tests/integration/fixtures/template_climate_nonoptimistic.yaml @@ -0,0 +1,56 @@ +esphome: + name: tmpl-clim-nonopt +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Template Climate Nonoptimistic + optimistic: false + supported_modes: + - "OFF" + - HEAT + - COOL + - FAN_ONLY + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + - AWAY + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +button: + - platform: template + id: simulate_device_confirmation + name: Simulate Device Confirmation + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + target_temperature: 22.5 + fan_mode: HIGH + swing_mode: VERTICAL + preset: AWAY diff --git a/tests/integration/fixtures/template_climate_on_control_ordering.yaml b/tests/integration/fixtures/template_climate_on_control_ordering.yaml new file mode 100644 index 0000000000..8366a6d21e --- /dev/null +++ b/tests/integration/fixtures/template_climate_on_control_ordering.yaml @@ -0,0 +1,26 @@ +esphome: + name: tmpl-clim-oc-order +host: +api: +logger: + +# on_control fires with the full ClimateCall (arg `x`) from the base Climate component's +# ClimateCall::perform(), before validate_()/control() run -- so when the lambda action below +# runs, the entity's own .mode is still the OLD value, even though x.get_mode() already reports +# the NEW requested value. on_state fires afterward, once control() has applied it. +climate: + - platform: template + id: test_climate + name: Test On Control Ordering + optimistic: true + supported_modes: + - "OFF" + - HEAT + on_control: + - lambda: |- + ESP_LOGD("test", "on_control requested_mode=%d current_mode_before_apply=%d", + x.get_mode().has_value() ? (int) *x.get_mode() : -1, + (int) id(test_climate).mode); + on_state: + - lambda: |- + ESP_LOGD("test", "on_state mode=%d", (int) x.mode); diff --git a/tests/integration/fixtures/template_climate_publish_all_fields.yaml b/tests/integration/fixtures/template_climate_publish_all_fields.yaml new file mode 100644 index 0000000000..e57fcc4508 --- /dev/null +++ b/tests/integration/fixtures/template_climate_publish_all_fields.yaml @@ -0,0 +1,63 @@ +esphome: + name: tmpl-clim-publish-all +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Publish All Fields + optimistic: true + # current_temperature/current_humidity/action are only sent over the API at all if their + # trait is advertised: current_temperature/current_humidity because a sensor/humidity_sensor + # is referenced below, action because supports_action is set. The sensors' fixed readings + # match what climate.template.publish pushes, so the sensor callback (guarded to only publish + # on an actual change) doesn't produce an extra, unexpected state update of its own. + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + supported_fan_modes: + - AUTO + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + on_control: + # Should never fire in this test: climate.template.publish is a pure bypass and must not + # re-trigger on_control as if the entity were freshly commanded. + - logger.log: "on_control fired" + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 20.0f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 60.0f;" + update_interval: 10ms + +button: + - platform: template + id: publish_all + name: Publish All + on_press: + - climate.template.publish: + id: test_climate + current_temperature: 20.0 + current_humidity: 60.0 + target_temperature: 23.0 + mode: HEAT + action: HEATING + fan_mode: HIGH + swing_mode: VERTICAL + preset: ECO diff --git a/tests/integration/fixtures/template_climate_sensor_push.yaml b/tests/integration/fixtures/template_climate_sensor_push.yaml new file mode 100644 index 0000000000..1fc004335d --- /dev/null +++ b/tests/integration/fixtures/template_climate_sensor_push.yaml @@ -0,0 +1,49 @@ +esphome: + name: tmpl-clim-sensor-push +host: +api: +logger: + +# No lambda/update_interval: these sensors only ever report a value when a button below +# publishes one (standing in for e.g. a BLE scan callback in a real config). +sensor: + - platform: template + id: room_temperature + name: Room Temperature + - platform: template + id: room_humidity + name: Room Humidity + +climate: + - platform: template + id: test_climate + name: Test Sensor Push Climate + optimistic: true + sensor: room_temperature + humidity_sensor: room_humidity + supported_modes: + - "OFF" + - HEAT + +button: + - platform: template + id: publish_temperature + name: Publish Temperature + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_temperature_same + name: Publish Temperature Same Value + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_humidity + name: Publish Humidity + on_press: + - sensor.template.publish: + id: room_humidity + state: 65.0 diff --git a/tests/integration/fixtures/template_climate_set_actions.yaml b/tests/integration/fixtures/template_climate_set_actions.yaml new file mode 100644 index 0000000000..b247367f64 --- /dev/null +++ b/tests/integration/fixtures/template_climate_set_actions.yaml @@ -0,0 +1,89 @@ +esphome: + name: tmpl-clim-set-act +host: +api: +logger: + +# Every settable field forwards its requested value to a set_*_action. supports_two_point and +# supports_target_humidity are not declared here: they are derived from the low/high and humidity +# set actions being present. +climate: + - platform: template + id: test_climate + name: Test Set Actions + optimistic: false + restore_mode: NO_RESTORE + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + custom_fan_modes: + - turbo + custom_presets: + - eco_plus + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_low_action: + - logger.log: + format: "set_target_temperature_low_action %.1f" + args: ["x"] + set_target_temperature_high_action: + - logger.log: + format: "set_target_temperature_high_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.0f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + +button: + - platform: template + id: report_device_state + name: Report Device State + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + + - platform: template + id: report_unsupported_mode + name: Report Unsupported Mode + on_press: + - climate.template.publish: + id: test_climate + mode: DRY diff --git a/tests/integration/fixtures/template_climate_two_point_temperature.yaml b/tests/integration/fixtures/template_climate_two_point_temperature.yaml new file mode 100644 index 0000000000..ec10785ee8 --- /dev/null +++ b/tests/integration/fixtures/template_climate_two_point_temperature.yaml @@ -0,0 +1,52 @@ +esphome: + name: tmpl-clim-two-point +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Two-Point Heatpump + optimistic: true + sensor: test_climate_current_temperature + supports_two_point_target_temperature: true + supports_target_humidity: true + supported_modes: + - "OFF" + - HEAT_COOL + - HEAT + - COOL + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature_low().has_value()) + ESP_LOGD("test", "on_control target_temperature_low=%.1f", *x.get_target_temperature_low()); + if (x.get_target_temperature_high().has_value()) + ESP_LOGD("test", "on_control target_temperature_high=%.1f", *x.get_target_temperature_high()); + if (x.get_target_humidity().has_value()) + ESP_LOGD("test", "on_control target_humidity=%.1f", *x.get_target_humidity()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 21.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT_COOL + target_temperature_low: 18.0 + target_temperature_high: 24.0 + target_humidity: 50.0 diff --git a/tests/integration/test_template_climate_basic.py b/tests/integration/test_template_climate_basic.py new file mode 100644 index 0000000000..431fd4e3e8 --- /dev/null +++ b/tests/integration/test_template_climate_basic.py @@ -0,0 +1,146 @@ +"""Integration test for template climate: sensor-pushed measured values, on_control + publish +for the settable ones. + +current_temperature/current_humidity are pushed by a referenced sensor/humidity_sensor (no +polling); action is set once at boot via climate.template.publish, since it has no sensor +equivalent. mode/target_temperature/fan_mode/swing_mode/preset are plain internal state: +on_control fires exactly once per command (never before the first one), and +climate.template.publish simulates the device reporting its own state independent of any prior +command -- that report is authoritative, overriding whatever was optimistically applied earlier. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-basic" + + +@pytest.mark.asyncio +async def test_template_climate_basic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Sensor-pushed measured values, on_control + publish for settable ones.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + # Advertised capabilities come straight from the supported_*/custom_* config lists. + assert ClimateMode.OFF in test_climate.supported_modes + assert ClimateMode.HEAT in test_climate.supported_modes + assert ClimateMode.COOL in test_climate.supported_modes + + assert ClimateFanMode.AUTO in test_climate.supported_fan_modes + assert ClimateFanMode.LOW in test_climate.supported_fan_modes + assert ClimateFanMode.HIGH in test_climate.supported_fan_modes + + assert ClimateSwingMode.OFF in test_climate.supported_swing_modes + assert ClimateSwingMode.VERTICAL in test_climate.supported_swing_modes + + assert ClimatePreset.NONE in test_climate.supported_presets + assert ClimatePreset.ECO in test_climate.supported_presets + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.current_temperature == pytest.approx(22.5, abs=0.1) + assert initial.current_humidity == pytest.approx(55.0, abs=0.1) + assert initial.action == ClimateAction.IDLE + assert initial.mode == ClimateMode.OFF + # Nothing was commanded yet: on_control must not have fired. + assert not log_lines + + # Commands apply optimistically and on_control fires with the same values. + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + client.climate_command(test_climate.key, target_temperature=22.5) + state = await wait_for_climate_state() + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.HIGH) + state = await wait_for_climate_state() + assert state.fan_mode == ClimateFanMode.HIGH + + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + state = await wait_for_climate_state() + assert state.swing_mode == ClimateSwingMode.VERTICAL + + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + state = await wait_for_climate_state() + assert state.preset == ClimatePreset.ECO + + await asyncio.sleep(0.2) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + # Exactly one on_control log line per command, none extra (e.g. from a stray republish). + assert len(log_lines) == 5 + + # measured values are untouched by any of the above (no set action exists for them). + assert state.current_temperature == pytest.approx(22.5, abs=0.1) + assert state.current_humidity == pytest.approx(55.0, abs=0.1) + assert state.action == ClimateAction.IDLE + + # The device's report is authoritative and overrides everything commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.OFF + assert state.fan_mode == ClimateFanMode.AUTO + assert state.swing_mode == ClimateSwingMode.OFF + assert state.preset == ClimatePreset.NONE diff --git a/tests/integration/test_template_climate_custom_modes.py b/tests/integration/test_template_climate_custom_modes.py new file mode 100644 index 0000000000..4817fe1ddf --- /dev/null +++ b/tests/integration/test_template_climate_custom_modes.py @@ -0,0 +1,98 @@ +"""Integration test for template climate: custom fan modes and presets. + +Same on_control (forward) + climate.template.publish (device report, authoritative) pattern as +the enum-based mode/preset fields, but for the custom string variants. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-custom" + + +@pytest.mark.asyncio +async def test_template_climate_custom_modes( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Custom fan mode/preset: traits, on_control forwarding, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + assert set(test_climate.supported_custom_fan_modes) == { + "turbo", + "silent", + "eco", + } + assert set(test_climate.supported_custom_presets) == { + "eco_plus", + "power_save", + "max", + } + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.custom_fan_mode == "" + assert initial.custom_preset == "" + + client.climate_command(test_climate.key, custom_fan_mode="turbo") + state = await wait_for_climate_state() + assert state.custom_fan_mode == "turbo" + + client.climate_command(test_climate.key, custom_preset="power_save") + state = await wait_for_climate_state() + assert state.custom_preset == "power_save" + + await asyncio.sleep(0.2) + assert any("on_control custom_fan_mode=turbo" in line for line in log_lines) + assert any("on_control custom_preset=power_save" in line for line in log_lines) + + # The device's report is authoritative and overrides what was commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.custom_fan_mode == "eco" + assert state.custom_preset == "max" diff --git a/tests/integration/test_template_climate_nonoptimistic.py b/tests/integration/test_template_climate_nonoptimistic.py new file mode 100644 index 0000000000..e922ec31b9 --- /dev/null +++ b/tests/integration/test_template_climate_nonoptimistic.py @@ -0,0 +1,107 @@ +"""Integration test for template climate: optimistic: false. + +A command still fires on_control (so a real device-backed config can forward it out), but must +NOT change the entity's own state -- only an explicit climate.template.publish call (standing in +for the device confirming the command actually took effect) does that. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-nonopt" + + +@pytest.mark.asyncio +async def test_template_climate_nonoptimistic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Nonoptimistic: a command doesn't change state until explicitly published.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + state_updates: list[aioesphomeapi.ClimateState] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + confirm_button = require_entity( + entities, "simulate_device_confirmation", ButtonInfo + ) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.mode == ClimateMode.OFF + + # Send every settable field in one command. on_control must fire with all of them, but + # nothing may be applied to the entity's own state -- no ClimateState update at all. + client.climate_command( + test_climate.key, + mode=ClimateMode.HEAT, + target_temperature=22.5, + fan_mode=ClimateFanMode.HIGH, + swing_mode=ClimateSwingMode.VERTICAL, + preset=ClimatePreset.AWAY, + ) + await asyncio.sleep(0.3) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + assert not state_updates, ( + "optimistic: false must not publish a state until climate.template.publish reports it" + ) + + # The device confirms the command actually took effect. + client.button_command(confirm_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.AWAY diff --git a/tests/integration/test_template_climate_on_control_ordering.py b/tests/integration/test_template_climate_on_control_ordering.py new file mode 100644 index 0000000000..8d212b3ccb --- /dev/null +++ b/tests/integration/test_template_climate_on_control_ordering.py @@ -0,0 +1,83 @@ +"""Integration test: on_control fires before control()/on_state, with the full ClimateCall. + +on_control's lambda argument exposes get_mode()/etc. on the *requested* ClimateCall, while the +entity's own .mode field still reflects the state *before* control() applies the change -- +proving the firing order is on_control, then control(), then on_state. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-oc-order" + + +@pytest.mark.asyncio +async def test_template_climate_on_control_ordering( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """on_control sees the requested value while the entity's own state is still the old one.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line or "on_state " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + await asyncio.sleep(0.2) + + # on_control saw the new requested mode (3 == CLIMATE_MODE_HEAT) while the entity's own + # state was still the old one (0 == CLIMATE_MODE_OFF) -- proving it fired before control(). + assert any( + "on_control requested_mode=3 current_mode_before_apply=0" in line + for line in log_lines + ) + # on_state fired afterward, reporting the now-applied mode. + assert any("on_state mode=3" in line for line in log_lines) + + control_index = next( + i for i, line in enumerate(log_lines) if "on_control " in line + ) + state_index = next(i for i, line in enumerate(log_lines) if "on_state " in line) + assert control_index < state_index, "on_control must fire before on_state" diff --git a/tests/integration/test_template_climate_publish_all_fields.py b/tests/integration/test_template_climate_publish_all_fields.py new file mode 100644 index 0000000000..9c4262b311 --- /dev/null +++ b/tests/integration/test_template_climate_publish_all_fields.py @@ -0,0 +1,96 @@ +"""Integration test for template climate: climate.template.publish covering every field at once. + +A single climate.template.publish call resolves into exactly one ClimateState update, and never +triggers on_control (which would misrepresent a device state report as a fresh command). This also +exercises that a sensor/humidity_sensor whose reading matches what's about to be published doesn't +sneak in an extra state update of its own (the sensor callback only re-publishes on an actual +change). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-publish-all" + + +@pytest.mark.asyncio +async def test_template_climate_publish_all_fields( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """One climate.template.publish call setting every field resolves to one state update.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + on_control_count = 0 + + def on_log_line(line: str) -> None: + nonlocal on_control_count + if "on_control fired" in line: + on_control_count += 1 + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + publish_button = require_entity(entities, "publish_all", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.button_command(publish_button.key) + try: + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + except TimeoutError: + pytest.fail("Timeout waiting for the published climate state") + + assert state.current_temperature == pytest.approx(20.0, abs=0.1) + assert state.current_humidity == pytest.approx(60.0, abs=0.1) + assert state.target_temperature == pytest.approx(23.0, abs=0.1) + assert state.mode == ClimateMode.HEAT + assert state.action == ClimateAction.HEATING + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.ECO + + # Give any stray extra update (there shouldn't be one) a moment to arrive. + await asyncio.sleep(0.2) + assert len(state_updates) == 1, ( + f"Expected exactly one ClimateState update, got {len(state_updates)}" + ) + assert on_control_count == 0, ( + "climate.template.publish must not trigger on_control" + ) diff --git a/tests/integration/test_template_climate_sensor_push.py b/tests/integration/test_template_climate_sensor_push.py new file mode 100644 index 0000000000..1db4da81ed --- /dev/null +++ b/tests/integration/test_template_climate_sensor_push.py @@ -0,0 +1,88 @@ +"""Integration test for template climate: current_temperature/current_humidity live sensor push. + +A *later* change to a backing sensor's value -- not just its initial reading at boot -- propagates +into a new climate state via add_on_state_callback. Re-publishing the same sensor value again must +not cause a redundant climate state update. +""" + +from __future__ import annotations + +import asyncio +import math + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-sensor-push" + + +@pytest.mark.asyncio +async def test_template_climate_sensor_push( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A later change to the backing sensor pushes a new climate state; an unchanged republish does not.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + publish_temp = require_entity(entities, "publish_temperature", ButtonInfo) + publish_temp_same = require_entity( + entities, "publish_temperature_same", ButtonInfo + ) + publish_humidity = require_entity(entities, "publish_humidity", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Neither backing sensor has published anything yet. + assert math.isnan(initial.current_temperature) + assert math.isnan(initial.current_humidity) + + # A later sensor reading -- not the initial one -- pushes a new climate state. + client.button_command(publish_temp.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_temperature == pytest.approx(24.0, abs=0.1) + + client.button_command(publish_humidity.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_humidity == pytest.approx(65.0, abs=0.1) + + # Re-publishing the same temperature must not cause a redundant climate state update. + updates_before = len(state_updates) + client.button_command(publish_temp_same.key) + await asyncio.sleep(0.3) + assert len(state_updates) == updates_before, ( + "Re-publishing an unchanged sensor reading must not republish the climate state" + ) diff --git a/tests/integration/test_template_climate_set_actions.py b/tests/integration/test_template_climate_set_actions.py new file mode 100644 index 0000000000..0b1eb80874 --- /dev/null +++ b/tests/integration/test_template_climate_set_actions.py @@ -0,0 +1,114 @@ +"""Integration test: each settable field forwards its value to the matching set_*_action. + +With optimistic: false the entity state stays put until climate.template.publish reports the +device's actual state back, so the actions are the only thing that reacts to a command. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-set-act" + + +@pytest.mark.asyncio +async def test_template_climate_set_actions( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Every set_*_action fires with the requested value; state waits for a publish.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "_action " in line or "Unsupported" in line: + log_lines.append(line) + + def logged(fragment: str) -> bool: + return any(fragment in line for line in log_lines) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + report_button = require_entity(entities, "report_device_state", ButtonInfo) + unsupported_button = require_entity( + entities, "report_unsupported_mode", ButtonInfo + ) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Both traits are derived from the low/high and humidity set actions, not declared. + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + client.climate_command( + test_climate.key, target_temperature_low=18.0, target_temperature_high=24.0 + ) + client.climate_command(test_climate.key, target_humidity=55) + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.LOW) + client.climate_command(test_climate.key, custom_fan_mode="turbo") + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + client.climate_command(test_climate.key, custom_preset="eco_plus") + + for _ in range(50): + await asyncio.sleep(0.1) + if logged("set_custom_preset_action eco_plus"): + break + + assert logged("set_mode_action 3") # CLIMATE_MODE_HEAT + assert logged("set_target_temperature_low_action 18.0") + assert logged("set_target_temperature_high_action 24.0") + assert logged("set_target_humidity_action 55") + assert logged("set_fan_mode_action 3") # CLIMATE_FAN_LOW + assert logged("set_custom_fan_mode_action turbo") + assert logged("set_swing_mode_action 2") # CLIMATE_SWING_VERTICAL + assert logged("set_preset_action 5") # CLIMATE_PRESET_ECO + assert logged("set_custom_preset_action eco_plus") + + # optimistic: false, so none of the commands above touched the entity's own state -- + # a device report is what actually moves it. + client.button_command(report_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + + # A publish naming a mode outside supported_modes warns instead of publishing it. + client.button_command(unsupported_button.key) + for _ in range(50): + await asyncio.sleep(0.1) + if logged("Unsupported mode"): + break + assert logged("Unsupported mode") diff --git a/tests/integration/test_template_climate_two_point_temperature.py b/tests/integration/test_template_climate_two_point_temperature.py new file mode 100644 index 0000000000..9270b59ffc --- /dev/null +++ b/tests/integration/test_template_climate_two_point_temperature.py @@ -0,0 +1,118 @@ +"""Integration tests for template climate: two-point target temperature + humidity. + +Covers the supports_two_point_target_temperature/supports_target_humidity boolean flags plus +on_control (forwarding commands out) and climate.template.publish (the device reporting its own +authoritative state, independent of any prior command -- e.g. a device that owns its own setpoint, +changed via a physical remote). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-two-point" + + +@pytest.mark.asyncio +async def test_template_climate_two_point_temperature( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Two-point target temperature + humidity: booleans, on_control, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + test_climate = climate_infos[0] + assert test_climate.name == "Test Two-Point Heatpump" + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Nothing has been published yet: settable fields have no sensor to seed them from, so + # the entity starts at ESPHome's plain defaults. current_temperature is pushed by the + # referenced sensor, which has already settled by the time we get here. + assert initial.mode == ClimateMode.OFF + assert initial.current_temperature == pytest.approx(21.0, abs=0.1) + + # The device reports its actual state for the first time. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT_COOL + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) + + # Commands apply optimistically (settable fields are plain internal state), and on_control + # fires with the same values so a real config could forward them to the device. + client.climate_command( + test_climate.key, target_temperature_low=19.0, target_temperature_high=25.0 + ) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(19.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(25.0, abs=0.1) + await asyncio.sleep(0.2) + assert any( + "on_control target_temperature_low=19.0" in line for line in log_lines + ) + assert any( + "on_control target_temperature_high=25.0" in line for line in log_lines + ) + + client.climate_command(test_climate.key, target_humidity=45.0) + state = await wait_for_climate_state() + assert state.target_humidity == pytest.approx(45.0, abs=0.1) + await asyncio.sleep(0.2) + assert any("on_control target_humidity=45.0" in line for line in log_lines) + + # The device's next report is authoritative and overrides whatever was optimistically + # applied above -- this is the whole point of climate.template.publish: a device that owns + # its own state (e.g. changed by a physical remote) always wins. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) From 95ab3fb4f29aed85f762f2699484fea9e756b4f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 6 Sep 2026 23:59:40 +0200 Subject: [PATCH 27/53] [ota] Offer encryption with the api key so enabling it works over OTA (#18979) --- THREAT_MODEL.md | 55 ++- esphome/__main__.py | 14 +- esphome/components/api/__init__.py | 4 +- esphome/components/api/api_connection.cpp | 6 +- .../components/api/api_frame_helper_noise.cpp | 2 +- esphome/components/api/api_server.cpp | 37 +- esphome/components/api/api_server.h | 12 +- esphome/components/esphome/ota/__init__.py | 130 +++--- .../components/esphome/ota/ota_esphome.cpp | 83 ++-- esphome/components/esphome/ota/ota_esphome.h | 13 +- .../esphome/ota/ota_esphome_noise.cpp | 91 +++-- esphome/components/noise/__init__.py | 36 +- esphome/components/noise/noise.cpp | 9 + esphome/components/noise/noise.h | 16 +- esphome/components/noise/noise_handshake.cpp | 5 +- esphome/components/noise/noise_handshake.h | 6 +- esphome/core/defines.h | 3 + esphome/espota2.py | 122 +++++- esphome/wizard.py | 18 +- .../noise/test_encryption_key.py | 14 +- tests/component_tests/ota/test_esphome_ota.py | 242 +++++++++--- .../ota/test_esphome_ota_api_key_offer.yaml | 11 + ...st_esphome_ota_api_key_offer_password.yaml | 12 + .../test_esphome_ota_encryption_required.yaml | 12 + .../ota/test_esphome_ota_own_key.yaml | 11 + .../ota/test_esphome_ota_plain.yaml | 9 + .../ota/test_esphome_ota_runtime_api_key.yaml | 10 + .../components/noise/test_noise_handshake.cpp | 18 +- .../noise/test_noise_primitives.cpp | 13 +- tests/components/ota/api_key_offer.yaml | 12 + tests/components/ota/api_runtime_key.yaml | 10 + .../ota/test-api_key_offer.esp32-idf.yaml | 2 + .../ota/test-api_key_offer.esp8266-ard.yaml | 2 + .../ota/test-api_runtime_key.esp32-idf.yaml | 2 + .../ota/test-api_runtime_key.esp8266-ard.yaml | 2 + tests/integration/conftest.py | 7 + tests/integration/const.py | 7 + .../host_ota_api_key_offer_with_password.yaml | 12 + .../host_ota_provisioned_api_key.yaml | 10 + .../test_api_zero_psk_provisioning.py | 51 ++- tests/integration/test_host_ota.py | 373 ++++++++++++------ tests/unit_tests/test_espota2_noise.py | 136 ++++++- tests/unit_tests/test_main.py | 114 +++++- tests/unit_tests/test_wizard.py | 31 +- 44 files changed, 1342 insertions(+), 443 deletions(-) create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_encryption_required.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_own_key.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_plain.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml create mode 100644 tests/components/ota/api_key_offer.yaml create mode 100644 tests/components/ota/api_runtime_key.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp8266-ard.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml create mode 100644 tests/integration/fixtures/host_ota_provisioned_api_key.yaml diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index b4f557e55b..11656ff0b7 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -125,30 +125,47 @@ design is optimal or that it will not change. ## OTA update encryption The `esphome` OTA platform optionally encrypts updates with the same Noise -`NNpsk0` pattern the native API uses; one key protects the device. With an -`encryption:` block configured the guarantees are: the firmware image is -confidential in transit, the uploader is authenticated by the pre-shared key, -and the plaintext negotiation preceding the handshake is bound into the -handshake prologue, so stripping or tampering with it fails the first MAC. -Both ends fail closed with no override: a device built with a key refuses +`NNpsk0` pattern the native API uses; one key protects the device. A device +whose `api:` block has an encryption key, static in the YAML or provisioned at +runtime, compiles in the transport and offers it on every OTA connection once +it holds a key, so an uploader presenting that key gets the guarantees below +even without an `ota: encryption:` block; only that block makes the device +require encryption. The guarantees are: the firmware image is confidential in +transit, the uploader is authenticated by the pre-shared key, and the plaintext +negotiation preceding the handshake is bound into the handshake prologue, so +stripping or tampering with it fails the first MAC. With `ota: encryption:` +configured both ends fail closed with no override: the device refuses plaintext uploads, and the CLI refuses to send plaintext when a key is -configured. +configured. Without that block the CLI tries a static api key when the device +offers and, until 2027.3.0, falls back to plaintext with a warning when the +offer is missing or the handshake fails; a runtime provisioned key never +reaches the CLI, so those uploads stay plaintext. -Defeating any of that without the key is in scope: a keyed device accepting a -plaintext or downgraded upload, getting past the MAC, or recovering image -contents from captured traffic. +Defeating any of that without the key is in scope: a device that requires +encryption accepting a plaintext or downgraded upload, getting past the MAC, +or recovering image contents from captured traffic. The following are **not** vulnerabilities, by design: -- Plaintext OTA on a device with no `encryption:` block. That is the - documented default, authenticated (if at all) by the OTA password. -- The enablement window: turning encryption on takes one last upload of the - encryption-enabled firmware over the existing plaintext channel, with the - pre-existing plaintext exposure. -- The web OTA `/update` endpoint alongside encryption. The `web_server` - component keeps it always reachable, and `captive_portal:` auto-loads it - for the fallback AP window; validation warns about both combinations, and - the operator keeps the recovery path. +- Plaintext OTA on a device with no `ota: encryption:` block, including one + that offers encryption because it has an api key. That is the documented + default, authenticated (if at all) by the OTA password. An uploader that + takes the offer skips the password; the key authenticates it. With a + runtime provisioned key and no `provisioning:` window, whoever provisions + the key gains that upload path too; validation warns about the pair. +- The CLI plaintext fallback until 2027.3.0: without `ota: encryption:` an + active attacker who strips the offer or breaks the handshake can make a + keyed CLI upload plaintext, with the pre-existing plaintext exposure. A + device that requires encryption still refuses that upload. +- The enablement window: firmware built with a static api key already offers + encryption, so turning on `ota: encryption:` is itself an encrypted upload. + Older firmware needs one last plaintext upload of an offering build, with + the pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. With the `web_server` + or `prometheus` component the shared listener is always up, so the endpoint + stays reachable and validation warns about that combination; + `captive_portal:` alone brings the listener up only for the fallback AP + window, which is the intended recovery path, so that is not warned about. - CLI retry behavior on transport or MAC failures; every attempt renegotiates a fresh handshake with fresh ephemerals, so retrying does not weaken authentication. diff --git a/esphome/__main__.py b/esphome/__main__.py index b3d58ad13b..30e97f55eb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1335,12 +1335,14 @@ def _upload_via_native_api( break from esphome import espota2 + from esphome.components.noise import static_encryption_key remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) # Fail closed: an encryption block whose key did not resolve must never # fall back to a plaintext upload noise_psk = None + plaintext_fallback = False if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: noise_psk = encryption_conf.get(CONF_KEY) if not noise_psk: @@ -1351,6 +1353,10 @@ def _upload_via_native_api( # Ensure the key is a string, as required by the underlying OTA implementation. # It arrives here as a SensitiveStr which aioesphomeapi rejects. noise_psk = str(noise_psk) + elif api_key := static_encryption_key(config.get(CONF_API) or {}): + # Remove before 2027.3.0: the api key is tried, falling back to plaintext + noise_psk = str(api_key) + plaintext_fallback = True def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1382,7 +1388,13 @@ def _upload_via_native_api( _validate_bootloader_binary(binary) return espota2.run_ota( - network_devices, remote_port, password, binary, ota_type, noise_psk + network_devices, + remote_port, + password, + binary, + ota_type, + noise_psk, + plaintext_fallback=plaintext_fallback, ) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3568318dad..6202e127bf 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -14,6 +14,7 @@ from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, encryption_schema, + new_psk_progmem, validate_encryption_key, ) from esphome.config_helpers import filter_source_files_from_defines, get_logger_level @@ -589,8 +590,7 @@ async def to_code(config: ConfigType) -> None: if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None: if key := encryption_config.get(CONF_KEY): - decoded = decode_encryption_key(key) - cg.add(var.set_noise_psk(list(decoded))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) cg.add_define("USE_API_NOISE_PSK_FROM_YAML") else: # No key provided, but encryption desired diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9c609aa047..da4b7d7702 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2161,7 +2161,10 @@ void APIConnection::on_homeassistant_action_response(const HomeassistantActionRe bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg) { NoiseEncryptionSetKeyResponse resp; resp.success = false; - +#ifdef USE_API_NOISE_PSK_FROM_YAML + // A yaml key cannot be changed at runtime, so no decode or save path is built + ESP_LOGW(TAG, "Key set in YAML"); +#else #ifdef USE_PROVISIONING // Refuse to set a key once the provisioning window has closed (defense in depth; // such connections are already rejected at hello). @@ -2196,6 +2199,7 @@ bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptio } #endif } +#endif // USE_API_NOISE_PSK_FROM_YAML return this->send_message(resp); } diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 138dbdddba..29b2858aee 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -548,7 +548,7 @@ APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) { * @return 0 on success, -1 on error (check errno) */ APIError APINoiseFrameHelper::init_handshake_() { - int err = this->handshake_.init(this->ctx_.get_psk(), prologue_.data(), prologue_.size()); + int err = this->handshake_.init(this->ctx_, prologue_.data(), prologue_.size()); APIError aerr = handle_noise_error_(err, LOG_STR("noise_handshake_init"), APIError::HANDSHAKESTATE_SETUP_FAILED); if (aerr != APIError::OK) return aerr; diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 43d35363d3..78ebe5c38e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -41,13 +41,13 @@ void APIServer::setup() { ControllerRegistry::register_controller(this); #ifdef USE_API_NOISE + // Always reserve the slot: flash preferences are positional on esp8266, so + // a yaml key build must keep the layout of a runtime key build uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); - #ifndef USE_API_NOISE_PSK_FROM_YAML - // Only load saved PSK if not set from YAML - if (this->load_and_apply_noise_psk_()) { + // A cleared record loads fine but holds no key + if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { ESP_LOGD(TAG, "Loaded saved Noise PSK"); } #endif @@ -550,6 +550,7 @@ const std::vector &APIServer::get_sta #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active) { if (!this->noise_pref_.save(&new_psk)) { @@ -583,22 +584,19 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString } bool APIServer::load_and_apply_noise_psk_() { - SavedNoisePsk saved{}; - if (!this->noise_pref_.load(&saved)) + // Load into a temp so a failed read cannot disturb the key in use + SavedNoisePsk loaded{}; + if (!this->noise_pref_.load(&loaded)) return false; - this->set_noise_psk(saved.psk); + this->saved_psk_ = loaded; + // An unprovisioned device stores the reserved all-zeros key, which is no key + const bool has_key = !noise::NoiseContext::is_all_zeros(this->saved_psk_.psk); + this->noise_ctx_.set_psk(has_key ? this->saved_psk_.psk.data() : nullptr); return true; } bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else - auto &old_psk = this->noise_ctx_.get_psk(); - if (std::equal(old_psk.begin(), old_psk.end(), psk.begin())) { + if (this->saved_psk_.psk == psk) { ESP_LOGW(TAG, "New PSK matches old"); return true; } @@ -614,15 +612,8 @@ bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { } #endif return result; -#endif } bool APIServer::clear_noise_psk(bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else SavedNoisePsk empty_psk{}; bool result = this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"), make_active); @@ -634,8 +625,8 @@ bool APIServer::clear_noise_psk(bool make_active) { } #endif return result; -#endif } +#endif // USE_API_NOISE_PSK_FROM_YAML #endif #ifdef USE_HOMEASSISTANT_TIME diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 072a583901..618ea4eb11 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -76,9 +76,14 @@ class APIServer final : public Component, APIBuffer &get_shared_buffer_ref() { return shared_write_buffer_; } #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML + // Runtime key changes exist for the provisioning path only (not lambdas); + // with a yaml key they compile out bool save_noise_psk(noise::psk_t psk, bool make_active = true); bool clear_noise_psk(bool make_active = true); - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } noise::NoiseContext &get_noise_ctx() { return this->noise_ctx_; } #endif // USE_API_NOISE @@ -275,10 +280,12 @@ class APIServer final : public Component, #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active); // Load saved PSK from preferences and apply it. Returns true on success. bool load_and_apply_noise_psk_(); +#endif // USE_API_NOISE_PSK_FROM_YAML #endif // USE_API_NOISE #ifdef USE_API_HOMEASSISTANT_STATES // Helper methods to reduce code duplication @@ -358,6 +365,9 @@ class APIServer final : public Component, #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; +#ifndef USE_API_NOISE_PSK_FROM_YAML + SavedNoisePsk saved_psk_{}; // backs noise_ctx_ for a runtime provisioned key +#endif ESPPreferenceObject noise_pref_; #endif // USE_API_NOISE }; diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 1fec9e5c9b..f5eb878260 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -2,12 +2,12 @@ import logging import esphome.codegen as cg from esphome.components.noise import ( - decode_encryption_key, encryption_schema, - is_reserved_key, + new_psk_progmem, + static_encryption_key, ) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code -from esphome.config_helpers import merge_config +from esphome.config_helpers import filter_source_files_from_defines, merge_config import esphome.config_validation as cv from esphome.const import ( CONF_API, @@ -31,7 +31,6 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" -CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -41,11 +40,10 @@ DEPENDENCIES = ["network"] def AUTO_LOAD(config: ConfigType) -> list[str]: - """Auto-load noise only when encryption is configured.""" + """Auto-load noise only when encryption is configured; the api key offer + inherits it from the api component.""" base = ["sha256", "socket"] - # A falsy config is a tooling probe for the maximal set (None from - # dependency resolution, {} from the components-graph platform probe); - # a validated config always carries defaults, never empty + # A falsy config is a tooling probe for the maximal set if not config or CONF_ENCRYPTION in config: return base + ["noise"] return base @@ -132,12 +130,56 @@ def ota_esphome_final_validate(config: ConfigType) -> None: _validate_no_password_with_encryption(ota_conf) if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: _resolve_encryption_key(encryption_conf, api_conf) - if any( - conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf - ) and any( - CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + elif CONF_PASSWORD in ota_conf and static_encryption_key(api_conf) is not None: + _LOGGER.warning( + "'%s' %s wastes significant flash and RAM (about 3.5 KB and 60 " + "bytes plus the password on the heap): the device already offers " + "encryption with the '%s' %s %s, which authenticates any uploader " + "that takes it, and a password only matters for uploaders without " + "encryption support; remove '%s' and add '%s' under '%s' so " + "uploads use the key and encryption is required", + CONF_OTA, + CONF_PASSWORD, + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_PASSWORD, + CONF_ENCRYPTION, + CONF_OTA, + ) + elif ( + CONF_PASSWORD in ota_conf + and CONF_ENCRYPTION in api_conf + and not api_conf[CONF_ENCRYPTION].get(CONF_KEY) + ): + # The CLI still needs the password; whoever provisions the key skips it + _LOGGER.warning( + "The '%s' %s %s provisioned at runtime also authenticates OTA " + "uploads once provisioned; '%s' %s then only guards plaintext " + "uploads. Whoever provisions the key can upload firmware " + "without the password, so add a 'provisioning:' block to limit " + "when that is possible", + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_OTA, + CONF_PASSWORD, + ) + # web_server and prometheus keep the shared listener up; the captive + # portal's copy only exists on the fallback AP and is the recovery path + if ( + (CONF_WEB_SERVER in full_conf or "prometheus" in full_conf) + and any(conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf) + and any( + CONF_ENCRYPTION in conf + for conf in merged_ota_esphome_configs_by_port.values() + ) ): - _warn_web_server_ota(full_conf) + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -152,33 +194,11 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) -def _warn_web_server_ota(full_conf: ConfigType) -> None: - """The web_server ota platform accepts the same image over plaintext HTTP - with basic auth, bypassing the encryption; warn rather than fail so the - operator keeps the recovery path.""" - if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: - # The captive_portal auto-load: the endpoint only exists while the - # fallback AP is active - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform (auto-loaded " - "by captive_portal); the plaintext /update endpoint stays " - "reachable while the fallback AP is active", - CONF_WEB_SERVER, - ) - else: - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform; its " - "plaintext /update endpoint accepts the same image", - CONF_WEB_SERVER, - ) - - def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: """Resolve the one encryption key per device into the ota block. An explicit ota key must match the api key, a bare block inherits it, - a runtime provisioned api key cannot be inherited, and the all-zeros - provisioning sentinel is rejected (the device treats it as no key). + a runtime provisioned api key cannot be inherited. """ api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) if ota_key := encryption_conf.get(CONF_KEY): @@ -201,11 +221,6 @@ def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) - ) else: encryption_conf[CONF_KEY] = api_key - if is_reserved_key(encryption_conf[CONF_KEY]): - raise cv.Invalid( - f"The all-zeros {CONF_KEY} is reserved and provides no protection; " - f"generate a real key with: openssl rand -base64 32" - ) # Also called on merged same-port configs in final validate, where schemas @@ -267,15 +282,9 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate -def FILTER_SOURCE_FILES() -> list[str]: - """Filter out the noise transport when no ota entry configures encryption.""" - for ota_conf in CORE.config.get(CONF_OTA, []): - if ( - ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME - and ota_conf.get(CONF_ENCRYPTION) is not None - ): - return [] - return ["ota_esphome_noise.cpp"] +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} +) @coroutine_with_priority(CoroPriority.OTA_UPDATES) @@ -296,11 +305,24 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") - if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: - # A missing key was resolved from the api component in final validate. - key = encryption_conf[CONF_KEY] + # One key per device: an api encryption block supplies it (static or + # runtime) and offers; the ota block only adds the requirement + api_conf = CORE.config.get(CONF_API) or {} + encryption_conf = config.get(CONF_ENCRYPTION) + own_key = None + if encryption_conf is not None and static_encryption_key(api_conf) is None: + own_key = encryption_conf[CONF_KEY] + if own_key is not None: cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + elif CONF_ENCRYPTION in api_conf: + cg.add_define("USE_OTA_ENCRYPTION") + cg.add_define("USE_OTA_ENCRYPTION_FROM_API") + if static_encryption_key(api_conf) is None: + # The key arrives at runtime, so the offer has to look for it + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 396a47bc52..1005ed214b 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,4 +1,7 @@ #include "ota_esphome.h" +#ifdef USE_OTA_ENCRYPTION_FROM_API +#include "esphome/components/api/api_server.h" +#endif #ifdef USE_OTA #ifdef USE_OTA_PASSWORD #include "esphome/components/sha256/sha256.h" @@ -26,6 +29,16 @@ namespace esphome { static const char *const TAG = "esphome.ota"; + +#ifdef USE_OTA_ENCRYPTION +const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { +#ifdef USE_OTA_ENCRYPTION_FROM_API + return api::global_api_server->get_noise_ctx(); +#else + return this->noise_ctx_; +#endif +} +#endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -97,18 +110,30 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, "Over-The-Air updates:\n" " Address: %s:%u\n" - " Version: %d", - network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION); + " Version: %d" +#ifdef USE_OTA_ENCRYPTION + "\n Encryption: %s" +#endif + , + network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION +#ifdef USE_OTA_ENCRYPTION_REQUIRED + , + LOG_STR_LITERAL("required") +#elif defined(USE_OTA_ENCRYPTION_PROVISIONED) + // A runtime provisioned key may not exist yet + , + this->noise_context_().has_psk() ? LOG_STR_LITERAL("offered, plaintext accepted") + : LOG_STR_LITERAL("offered once the api key is provisioned") +#elif defined(USE_OTA_ENCRYPTION) + , + LOG_STR_LITERAL("offered, plaintext accepted") +#endif + ); #ifdef USE_OTA_PASSWORD if (!this->password_.empty()) { ESP_LOGCONFIG(TAG, " Password configured"); } #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { - ESP_LOGCONFIG(TAG, " Encryption configured"); - } -#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -154,10 +179,22 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +// Noise needs the extended protocol: the prologue binds the 2-byte feature ack +static constexpr uint8_t CLIENT_NOISE_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +inline bool ESPHomeOTAComponent::extended_proto_() const { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // FEATURE_READ already refused every client without the extended protocol + return true; +#else + return (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; +#endif +} + void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. /// @@ -241,12 +278,9 @@ void ESPHomeOTAComponent::handle_handshake_() { this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); -#ifdef USE_OTA_ENCRYPTION - // Fail closed: with a PSK configured the client must negotiate encryption - // (which requires the extended protocol); refuse plaintext uploads. - static constexpr uint8_t NOISE_REQUIRED_FEATURES = - CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; - if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // `ota: encryption:` requires the client to negotiate encryption + if ((this->ota_features_ & CLIENT_NOISE_FEATURES) != CLIENT_NOISE_FEATURES) { ESP_LOGW(TAG, "Client does not support encryption"); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); return; @@ -261,18 +295,21 @@ void ESPHomeOTAComponent::handle_handshake_() { // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte // legacy response. - this->extended_proto_ = (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; - if (this->extended_proto_) { + if (this->extended_proto_()) { static_assert(HANDSHAKE_BUF_SIZE >= 2, "handshake_buf_ must hold the 2-byte extended-protocol feature ack"); this->handshake_buf_[0] = ota::OTA_RESPONSE_FEATURE_FLAGS; this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // A runtime provisioned key may not exist yet + if (this->noise_context_().has_psk()) { this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; } +#elif defined(USE_OTA_ENCRYPTION) + // A yaml key always exists: validation rejects the all-zeros key + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; #endif } else { this->handshake_buf_[0] = @@ -284,15 +321,15 @@ void ESPHomeOTAComponent::handle_handshake_() { case OTAState::FEATURE_ACK: { static constexpr size_t STANDARD_PROTO_ACK_SIZE = 1; static constexpr size_t EXTENDED_PROTO_ACK_SIZE = 2; - const size_t ack_size = this->extended_proto_ ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; + const size_t ack_size = this->extended_proto_() ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } #ifdef USE_OTA_ENCRYPTION - // With a PSK configured the rest of the session runs inside the noise - // transport; the client sends the first handshake frame next, so there - // is nothing to do until data arrives. - if (this->noise_ctx_.has_psk()) { + // Latch the offer actually sent: a key activating between the two + // states must not start a session the client never expects + if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -412,7 +449,7 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge auth OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); - if (this->extended_proto_) { + if (this->extended_proto_()) { // Read ota type, 1 byte if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index fd164b8138..c6f710b3fc 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,8 +44,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#ifdef USE_OTA_ENCRYPTION - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif /// Manually set the port OTA should listen on @@ -85,9 +86,12 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; + // The api server's live context when the api has encryption, else our own + const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); + size_t noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len); bool noise_try_write_frame_(); void noise_send_reject_(const LogString *reason); ssize_t noise_decrypt_(uint8_t *buf, size_t len); @@ -144,7 +148,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION +#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#endif std::unique_ptr noise_; #endif // USE_OTA_ENCRYPTION @@ -166,6 +172,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; + // Derived from the feature byte; storing it would pad the trailing bytes + bool extended_proto_() const; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; @@ -179,7 +187,6 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint8_t auth_buf_pos_{0}; uint8_t auth_type_{0}; // Store auth type to know which hasher to use #endif // USE_OTA_PASSWORD - bool extended_proto_{false}; }; } // namespace esphome diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7f8331cf96..7401413d6d 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -3,6 +3,7 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise.h" #include "esphome/components/ota/ota_backend.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -40,24 +41,17 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // A provisioned key cleared between the offer and here is not guarded: the + // session runs on the zero key load_psk fills in and fails the client's MAC. + // Default-init: the frame buffer is written before it is read // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); - if (this->noise_ == nullptr) { - ESP_LOGW(TAG, "Session allocation failed"); - this->cleanup_connection_(); - return false; - } - + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + PROLOGUE_FEATURE_ACK_LEN]; -#ifdef USE_ESP8266 - memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#else - std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#endif + progmem_memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; // Magic bytes, already validated in MAGIC_READ std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); @@ -71,9 +65,13 @@ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; *p++ = server_feature_flags; - int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + // The caller only starts a session when the context holds a key + int err = this->noise_ == nullptr ? NOISE_ERROR_NO_MEMORY + : this->noise_->handshake.init(this->noise_context_(), prologue, sizeof(prologue)); if (err != 0) { - ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + // Raw noise codes throughout: the name table would cost flash in builds + // where only the OTA uses noise + ESP_LOGW(TAG, "Session init: %d", err); this->cleanup_connection_(); return false; } @@ -105,14 +103,16 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { s.frame_pos = 0; s.frame_len = 0; if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { - ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + ESP_LOGW(TAG, "Client rejected the handshake: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); this->cleanup_connection_(); return false; } int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); if (err != 0) { - ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); - this->noise_send_reject_(noise::reject_reason_for(err)); + // A MAC failure here almost always means the uploader has a different key + const LogString *reason = noise::reject_reason_for(err); + ESP_LOGW(TAG, "Handshake read: %s (%d)", LOG_STR_ARG(reason), err); + this->noise_send_reject_(reason); this->cleanup_connection_(); return false; } @@ -123,7 +123,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { int err = s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); if (err != 0) { - ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake write: %d", err); this->cleanup_connection_(); return false; } @@ -138,7 +138,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { int err = s.handshake.split(s.send_cipher, s.recv_cipher); if (err != 0) { - ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake split: %d", err); this->cleanup_connection_(); return false; } @@ -154,33 +154,41 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { } } +/// Payload length from a frame header, or 0 (logged) when the indicator or +/// the length is out of range. Callers pass min_len >= 1 so 0 is never valid. +size_t ESPHomeOTAComponent::noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len) { + const size_t payload_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || payload_len < min_len || payload_len > max_len) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], payload_len); + return 0; + } + return payload_len; +} + /// Non-blocking read of one handshake frame into the session buffer. bool ESPHomeOTAComponent::noise_try_read_frame_() { NoiseSession &s = *this->noise_; - while (s.frame_pos < noise::FRAME_HEADER_SIZE) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { - return false; + while (true) { + // The header first, then the body once the header says how long it is + const uint16_t want = s.frame_len == 0 ? noise::FRAME_HEADER_SIZE : s.frame_len; + if (s.frame_pos < want) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, want - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise"))) { + return false; + } + s.frame_pos += read; + continue; } - s.frame_pos += read; - } - if (s.frame_len == 0) { - const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); - if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { - ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + if (s.frame_len != 0) { + return true; + } + const size_t payload_len = this->noise_frame_payload_len_(s.frame_buf, 1, 1 + noise::MAX_HANDSHAKE_SIZE); + if (payload_len == 0) { this->cleanup_connection_(); return false; } s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; } - while (s.frame_pos < s.frame_len) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { - return false; - } - s.frame_pos += read; - } - return true; } /// Non-blocking write of the pending session-buffer frame. @@ -214,7 +222,7 @@ ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { noise_buffer_set_inout(mbuf, buf, len, len); int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Decrypt: %d", err); return -1; } return mbuf.size; @@ -229,9 +237,8 @@ ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min if (!this->readall_(header, sizeof(header))) { return -1; } - const size_t ciphertext_len = encode_uint16(header[1], header[2]); - if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { - ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + const size_t ciphertext_len = this->noise_frame_payload_len_(header, min_ciphertext, max_ciphertext); + if (ciphertext_len == 0) { return -1; } if (!this->readall_(buf, ciphertext_len)) { @@ -267,7 +274,7 @@ bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Encrypt: %d", err); return false; } noise::write_frame_header(frame, mbuf.size); diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 0f9328a482..a1d9444fc0 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -4,7 +4,9 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_KEY +from esphome.const import CONF_ENCRYPTION, CONF_KEY +from esphome.core import ID +from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] @@ -23,6 +25,14 @@ def validate_encryption_key(value: Any) -> str: if len(decoded) != 32: raise cv.Invalid("Encryption key must be base64 and 32 bytes long") + if not any(decoded): + # The device treats the all-zeros key as no key at all (it is the + # provisioning sentinel), so it must never reach a build + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"omit the {CONF_KEY} to provision it at runtime, or generate a real " + "key with: openssl rand -base64 32" + ) # Return original data for roundtrip conversion return value @@ -45,15 +55,6 @@ def decode_encryption_key(value: str) -> bytes: return decoded -def is_reserved_key(value: str) -> bool: - """Whether the key is the reserved all-zeros provisioning sentinel. - - The device treats it as no key configured, so consumers that require a - real key must reject it. - """ - return not any(decode_encryption_key(value)) - - ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), @@ -61,6 +62,21 @@ ENCRYPTION_SCHEMA = cv.Schema( ) +def static_encryption_key(conf: ConfigType) -> str | None: + """The build time key of a component config; None without one or when + the key is provisioned at runtime.""" + return (conf.get(CONF_ENCRYPTION) or {}).get(CONF_KEY) or None + + +def new_psk_progmem(parent_id: ID, key: str) -> MockObj: + """Emit the decoded key as a PROGMEM array; the component keeps a pointer + so the key never occupies RAM.""" + return cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decode_encryption_key(key)), + ) + + def encryption_schema(config: ConfigType | None) -> ConfigType: # A bare `encryption:` block is valid; a missing key means the consumer # falls back to its keyless behavior (api provisioning, ota inheriting diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 95fab322db..4806706167 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,5 +1,6 @@ #include "noise.h" #ifdef USE_NOISE +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -15,6 +16,14 @@ namespace esphome::noise { static const char *const TAG = "noise"; +void NoiseContext::load_psk(psk_t &out) const { + if (this->psk_ == nullptr) { + out.fill(0); + return; + } + progmem_memcpy(out.data(), this->psk_, out.size()); +} + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index f9da8d35b8..1033d5423c 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -23,16 +23,16 @@ class NoiseContext { } return acc == 0; } - void set_psk(psk_t psk) { - this->psk_ = psk; - this->has_psk_ = !is_all_zeros(psk); - } - const psk_t &get_psk() const { return this->psk_; } - bool has_psk() const { return this->has_psk_; } + /// psk points at 32 bytes that outlive the context (PROGMEM or caller owned + /// RAM); nullptr means no key. Runtime callers map the all-zeros key to + /// nullptr themselves; validation keeps it out of yaml. + void set_psk(const uint8_t *psk) { this->psk_ = psk; } + /// Copy the key out (flash-aware on ESP8266); all zeros when none is set. + void load_psk(psk_t &out) const; + bool has_psk() const { return this->psk_ != nullptr; } protected: - psk_t psk_{}; - bool has_psk_{false}; + const uint8_t *psk_{nullptr}; }; /// Convert a noise error code to a readable error diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 6d426de012..cc7fa603c4 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -20,7 +20,7 @@ NoiseResponderHandshake::~NoiseResponderHandshake() { } } -int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len) { +int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len) { if (this->handshake_ != nullptr) { noise_handshakestate_free(this->handshake_); this->handshake_ = nullptr; @@ -44,6 +44,9 @@ int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, siz HANDSHAKE_STEP_LOG("noise_handshakestate_new_by_id", err); return err; } + // noise-c keeps its own copy, so the key only passes through the stack here + psk_t psk; + ctx.load_psk(psk); err = noise_handshakestate_set_pre_shared_key(this->handshake_, psk.data(), psk.size()); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_set_pre_shared_key", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index 30596f35c2..bf1aa8cb7f 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -36,9 +36,9 @@ class NoiseResponderHandshake { NoiseResponderHandshake(const NoiseResponderHandshake &) = delete; NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; - /// Create and start the handshake with the given PSK and prologue. A - /// repeated call frees the previous handshake state and starts over. - [[nodiscard]] int init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len); + /// Create and start the handshake with the context's PSK and the prologue. + /// A repeated call frees the previous handshake state and starts over. + [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. [[nodiscard]] Action action() const; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 526adf74f0..9dd1e0ced6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -244,6 +244,9 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION +#define USE_OTA_ENCRYPTION_FROM_API +#define USE_OTA_ENCRYPTION_PROVISIONED +#define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ac4cbeeb7c..ce403c398d 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -202,6 +202,49 @@ class OTANetworkError(OTAError): """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" +# Remove before 2027.3.0 +class OTAEncryptionFallback(OTAError): + """The encrypted attempt failed and the caller may retry in plaintext.""" + + +# Remove before 2027.3.0 +PLAINTEXT_FALLBACK_NOTICE = ( + "A device with an api encryption key offers encryption after this " + "install; add 'encryption:' under 'ota: platform: esphome' to require it. " + "This plaintext fallback is removed in 2027.3.0." +) + + +# Remove before 2027.3.0 +class _EncryptionAttempt: + """The key an upload tries and whether it may fall back to plaintext; + a rejected handshake falls back at once, a transport fault only on repeat.""" + + def __init__(self, noise_psk: str | None, plaintext_fallback: bool) -> None: + self.noise_psk = noise_psk + self.plaintext_fallback = plaintext_fallback + self.handshake_faults = 0 + + def handshake_fault_falls_back(self) -> bool: + self.handshake_faults += 1 + return self.plaintext_fallback and self.handshake_faults >= 2 + + def downgrade(self, reason: str) -> None: + _LOGGER.warning( + "%s. Retrying in plaintext; a device that requires encryption " + "refuses it. %s", + reason, + PLAINTEXT_FALLBACK_NOTICE, + ) + self.noise_psk = None + self.plaintext_fallback = False + + +# Remove before 2027.3.0: only the fallback decision needs this distinction +class OTAHandshakeNetworkError(OTANetworkError): + """A transport failure inside the noise handshake; retrying encrypted may succeed.""" + + def _committed_error(err: OTANetworkError) -> OTAError: """Wrap a network failure that happened once the device had the full image. @@ -464,6 +507,7 @@ def perform_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> None: # Validate up front; an out-of-range value would only surface as a # ValueError deep inside send_check, bypassing OTAError handling @@ -528,19 +572,28 @@ def perform_ota( else: features = 0 - if noise_psk: - # Fail closed: never fall back to a plaintext upload when an - # encryption key is configured, an active attacker could otherwise - # strip the feature flag and capture the image (it contains the wifi - # credentials and the api encryption key). - if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if noise_psk and not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if plaintext_fallback: + # Remove before 2027.3.0: older firmware that cannot encrypt still + # gets its update on this connection + _LOGGER.warning( + "The device did not offer OTA encryption; continuing in plaintext. %s", + PLAINTEXT_FALLBACK_NOTICE, + ) + noise_psk = None + else: + # Fail closed: an attacker could otherwise strip the offer and + # capture the image (wifi credentials, api key) raise OTAError( "An OTA encryption key is configured but the device did not " "offer encryption; refusing to send the image in plaintext. " - "If the running firmware predates OTA encryption, first update " - "it without the 'ota: encryption:' block (over a trusted " - "network or via USB), then restore the block and upload again." + "The running firmware predates ESPHome 2026.9.0 or has no " + "'api: encryption: key'. With an api key, install once " + "without the 'ota: encryption:' block (that build offers " + "encryption), then restore it; otherwise flash by serial or " + "the web_server OTA platform." ) + if noise_psk: # The prologue binds every negotiation byte both sides saw, so any # tampering with the plaintext preamble breaks the handshake. prologue = ( @@ -549,8 +602,18 @@ def perform_ota( + bytes([RESPONSE_OK, version, features_to_send]) + bytes([RESPONSE_FEATURE_FLAGS, features]) ) + # Built outside the try: a local failure must never downgrade the upload sock = NoiseSocketWrapper(sock, noise_psk, prologue) - sock.do_handshake() + try: + sock.do_handshake() + except OTANetworkError as err: + # A transport fault: retry encrypted before considering plaintext + raise OTAHandshakeNetworkError(str(err)) from err + except OTAError as err: + # Remove before 2027.3.0 + if plaintext_fallback: + raise OTAEncryptionFallback(str(err)) from err + raise _LOGGER.info("Encrypted connection established") if ota_type != OTA_TYPE_UPDATE_APP: @@ -757,6 +820,7 @@ def run_ota_impl_( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -795,7 +859,9 @@ def run_ota_impl_( total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" reached_device = False - for attempt in range(total_attempts): + attempt = 0 + encryption = _EncryptionAttempt(noise_psk, plaintext_fallback) + while attempt < total_attempts: af, socktype, _, _, sa = res[attempt % len(res)] if reached_device or attempt >= len(res): _LOGGER.info( @@ -815,17 +881,40 @@ def run_ota_impl_( sock.close() _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) last_error = f"connecting to {sa[0]} failed: {err}" + attempt += 1 continue _LOGGER.info("Connected to %s", sa[0]) reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) + perform_ota( + sock, + password, + file_handle, + filename, + ota_type, + encryption.noise_psk, + encryption.plaintext_fallback, + ) + except OTAEncryptionFallback as err: + # Same address and attempt budget: not a network retry + last_error = str(err) + encryption.downgrade(last_error) + continue + except OTAHandshakeNetworkError as err: + last_error = str(err) + if encryption.handshake_fault_falls_back(): + encryption.downgrade(last_error) + continue + _LOGGER.warning("%s", last_error) + attempt += 1 + continue except OTANetworkError as err: # Transient network failure; retry last_error = str(err) _LOGGER.warning("%s", last_error) + attempt += 1 continue except OTAError as err: # Device-reported error (wrong password, wrong flash size, ...); @@ -847,10 +936,17 @@ def run_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: try: return run_ota_impl_( - remote_host, remote_port, password, filename, ota_type, noise_psk + remote_host, + remote_port, + password, + filename, + ota_type, + noise_psk, + plaintext_fallback, ) except OTAError as err: _LOGGER.error(err) diff --git a/esphome/wizard.py b/esphome/wizard.py index f7706928e9..897d5f60a1 100644 --- a/esphome/wizard.py +++ b/esphome/wizard.py @@ -148,11 +148,13 @@ def wizard_file(**kwargs: Unpack[WizardFileKwargs]) -> str: if "api_encryption_key" in kwargs: config += f' encryption:\n key: "{kwargs["api_encryption_key"]}"\n' - # Configure OTA + # The api key also secures OTA; a password only serves older uploaders config += "\nota:\n" config += " - platform: esphome\n" if "ota_password" in kwargs: config += f' password: "{kwargs["ota_password"]}"' + elif "api_encryption_key" in kwargs: + config += " encryption:" # Configuring wifi config += "\n\nwifi:\n" @@ -529,20 +531,9 @@ def wizard(path: Path) -> int: safe_print() safe_print("You'll need this key when adding the device to Home Assistant.") sleep(1) - - safe_print() - safe_print( - f"Do you want to set a {color(AnsiFore.GREEN, 'password')} for OTA updates? " - "This can be insecure if you do not trust the WiFi network." - ) - safe_print() - sleep(0.25) - safe_print("Press ENTER for no password") - ota_password = safe_input(color(AnsiFore.BOLD_WHITE, "(password): ")) else: ssid, psk = "", "" api_encryption_key = None - ota_password = "" kwargs = { "path": path, @@ -553,10 +544,9 @@ def wizard(path: Path) -> int: "psk": psk, "type": "basic", } + # The api key also secures OTA updates, so the wizard sets no OTA password if api_encryption_key: kwargs["api_encryption_key"] = api_encryption_key - if ota_password: - kwargs["ota_password"] = ota_password if not wizard_write(**kwargs): return 1 diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 10f1eb3d4c..2b79bd5464 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,11 +5,7 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import ( - decode_encryption_key, - is_reserved_key, - validate_encryption_key, -) +from esphome.components.noise import decode_encryption_key, validate_encryption_key KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -41,6 +37,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: decode_encryption_key("AAECAw==") -def test_is_reserved_key() -> None: - assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") - assert not is_reserved_key(KEY) +def test_validate_encryption_key_rejects_all_zeros() -> None: + """The all-zeros key is the provisioning sentinel the device treats as no + key, so it never reaches a build.""" + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + validate_encryption_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 873f162555..d3092294dc 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable import logging from typing import Any @@ -14,6 +15,7 @@ from esphome.components.esphome.ota import ( _validate_no_password_with_encryption, ota_esphome_final_validate, ) +from esphome.components.noise import static_encryption_key from esphome.const import ( CONF_API, CONF_ENCRYPTION, @@ -115,7 +117,6 @@ def test_non_esphome_ota_unaffected() -> None: API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" -ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" def test_encryption_key_inherited_from_api() -> None: @@ -197,36 +198,6 @@ def test_encryption_without_any_key_rejected() -> None: fv.full_config.reset(token) -def test_encryption_explicit_all_zeros_key_rejected() -> None: - """The all-zeros key is the provisioning sentinel; the device would treat - it as no PSK and accept plaintext, so it must fail validation.""" - full_conf = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) - ], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - -def test_encryption_inherited_all_zeros_key_rejected() -> None: - """An all-zeros api key must not silently disable ota encryption either.""" - full_conf = { - CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, - CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: """Same-port configs with different encryption keys raise.""" full_conf = { @@ -295,13 +266,14 @@ def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None fv.full_config.reset(token) +@pytest.mark.parametrize("component", ["web_server", "prometheus"]) def test_encryption_with_web_server_ota_warns( - caplog: pytest.LogCaptureFixture, + caplog: pytest.LogCaptureFixture, component: str ) -> None: - """With the web_server component the plaintext /update endpoint is always - on; the combination validates with a warning.""" + """web_server and prometheus keep the shared listener up, so the + plaintext /update endpoint is always on and the combination warns.""" full_conf = { - "web_server": {}, + component: {}, CONF_OTA: [ _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, @@ -316,12 +288,12 @@ def test_encryption_with_web_server_ota_warns( fv.full_config.reset(token) -def test_encryption_with_captive_portal_web_server_ota_warns( +def test_encryption_with_captive_portal_does_not_warn( caplog: pytest.LogCaptureFixture, ) -> None: """captive_portal auto-loads the web_server ota platform without the - web_server component; encryption stays usable and only warns, so the - fallback AP recovery path is not lost.""" + web_server component; its endpoint only exists while the fallback AP is + active and is the intended recovery path, so there is no warning.""" full_conf = { "captive_portal": {}, CONF_OTA: [ @@ -333,7 +305,10 @@ def test_encryption_with_captive_portal_web_server_ota_warns( try: with caplog.at_level(logging.WARNING): ota_esphome_final_validate({}) - assert any("captive_portal" in record.message for record in caplog.records) + assert not any( + "OTA encryption does not cover" in record.message + for record in caplog.records + ) esphome_conf = next( conf for conf in fv.full_config.get()[CONF_OTA] @@ -344,6 +319,100 @@ def test_encryption_with_captive_portal_web_server_ota_warns( fv.full_config.reset(token) +def test_password_with_api_key_warns(caplog: pytest.LogCaptureFixture) -> None: + """A static api key makes the device offer encryption and the CLI take + it, so the password is dead weight; the config validates with a warning.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("wastes significant flash" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_password_with_runtime_api_key_warns_differently( + caplog: pytest.LogCaptureFixture, +) -> None: + """The CLI still needs the password, but the provisioned key also + authenticates uploads; the warning says so without the flash advice.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + messages = [r.message for r in caplog.records] + assert any("provisioned at runtime also authenticates" in m for m in messages) + assert not any("wastes significant flash" in m for m in messages) + finally: + fv.full_config.reset(token) + + +def test_password_without_api_key_no_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """Without an api key there is no offer, so nothing to warn about.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("authenticates" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_web_server_component_without_ota_platform_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """The web_server component alone has no /update endpoint.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any( + "OTA encryption does not cover" in r.message for r in caplog.records + ) + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_platform_alone_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the web_server component starts the shared listener, so the ota + platform on its own never exposes /update.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("plaintext /update" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + def test_web_server_ota_without_encryption_unaffected() -> None: """web_server ota stays valid alongside an unencrypted esphome entry.""" full_conf = { @@ -370,20 +439,87 @@ def test_auto_load_pulls_noise_only_for_encryption() -> None: assert "noise" in AUTO_LOAD({}) -def test_filter_source_files_excludes_noise_without_encryption() -> None: - """The noise transport source compiles only for encrypted builds.""" - old_config = CORE.config - try: - CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} - assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] - CORE.config = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) - ] - } - assert FILTER_SOURCE_FILES() == [] - finally: - CORE.config = old_config +def test_static_encryption_key() -> None: + """Only a build-time key counts; a runtime provisioned one does not.""" + assert static_encryption_key({}) is None + assert static_encryption_key({CONF_ENCRYPTION: {}}) is None + assert static_encryption_key({CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) == API_KEY + + +@pytest.mark.parametrize( + ("yaml_name", "defines_present", "defines_absent"), + [ + # An api key alone compiles the transport in without requiring it; + # the device uses the api server's key, not a copy + ( + "api_key_offer", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A password still guards plaintext uploads on an offering device + ( + "api_key_offer_password", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # The ota encryption block is what makes the device refuse plaintext + ( + "encryption_required", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + }, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # Without api encryption the ota key is the device's own + ( + "own_key", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, + {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A key provisioned at runtime lives in the api server; the device + # offers with it once provisioned and never requires it + ( + "runtime_api_key", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + {"USE_OTA_ENCRYPTION_REQUIRED"}, + ), + # No api encryption at all keeps the noise glue out of the build + ( + "plain", + set(), + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + ), + ], +) +def test_encryption_offer_codegen( + generate_main: Callable[[str], str], + yaml_name: str, + defines_present: set[str], + defines_absent: set[str], +) -> None: + main_cpp = generate_main( + f"tests/component_tests/ota/test_esphome_ota_{yaml_name}.yaml" + ) + defines = {define.name for define in CORE.defines} + assert defines_present <= defines + assert not (defines_absent & defines) + encrypted = "USE_OTA_ENCRYPTION" in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) + # The noise transport source compiles only when the define is set + assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) def test_password_with_encryption_rejected() -> None: diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml new file mode 100644 index 0000000000..ca26eb9f46 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-offer + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml new file mode 100644 index 0000000000..1e23975690 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-offer-password + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml new file mode 100644 index 0000000000..36690038d8 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-encryption-required + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + encryption: diff --git a/tests/component_tests/ota/test_esphome_ota_own_key.yaml b/tests/component_tests/ota/test_esphome_ota_own_key.yaml new file mode 100644 index 0000000000..b6d1e4200d --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_own_key.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-own-key + +host: + +api: + +ota: + - platform: esphome + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/component_tests/ota/test_esphome_ota_plain.yaml b/tests/component_tests/ota/test_esphome_ota_plain.yaml new file mode 100644 index 0000000000..c5ca7afcf0 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_plain.yaml @@ -0,0 +1,9 @@ +esphome: + name: ota-plain + +host: + +api: + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml new file mode 100644 index 0000000000..8825335141 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: ota-runtime-key + +host: + +api: + encryption: + +ota: + - platform: esphome diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index d879a26c43..f2081f2965 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -68,6 +68,14 @@ class Initiator { static const uint8_t PROLOGUE[] = {'t', 'e', 's', 't', 'p', 'r', 'o', 'l', 'o', 'g', 'u', 'e'}; +// The context only points at the key and init() copies it before returning, +// so a temporary context over a temporary key is safe within one call +static NoiseContext ctx_for(const psk_t &psk) { + NoiseContext ctx; + ctx.set_psk(psk.data()); + return ctx; +} + static psk_t make_psk(uint8_t seed) { psk_t psk; for (size_t i = 0; i < psk.size(); i++) { @@ -102,7 +110,7 @@ TEST(NoiseResponderHandshakeTest, MessageMethodsErrorBeforeInit) { TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); @@ -155,8 +163,8 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // proves the restart took effect; the old state surviving would fail the // MAC here. NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); - ASSERT_EQ(responder.init(make_psk(9), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(9)), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(make_psk(9), PROLOGUE, sizeof(PROLOGUE)); @@ -168,7 +176,7 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { TEST(NoiseResponderHandshakeTest, WrongPskFailsWithMacFailure) { NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); Initiator initiator(make_psk(200), PROLOGUE, sizeof(PROLOGUE)); uint8_t msg[MAX_HANDSHAKE_SIZE]; @@ -185,7 +193,7 @@ TEST(NoiseResponderHandshakeTest, MismatchedPrologueFailsWithMacFailure) { // tampered preamble must fail even with the right key. const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); static const uint8_t TAMPERED[] = {'x'}; Initiator initiator(psk, TAMPERED, sizeof(TAMPERED)); diff --git a/tests/components/noise/test_noise_primitives.cpp b/tests/components/noise/test_noise_primitives.cpp index 018be9f717..8687c4b963 100644 --- a/tests/components/noise/test_noise_primitives.cpp +++ b/tests/components/noise/test_noise_primitives.cpp @@ -17,12 +17,17 @@ TEST(NoiseContextTest, AllZerosPskIsReserved) { EXPECT_FALSE(NoiseContext::is_all_zeros(psk)); NoiseContext ctx; + psk_t loaded; EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(zeros); - EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, zeros); + ctx.set_psk(psk.data()); EXPECT_TRUE(ctx.has_psk()); - EXPECT_EQ(ctx.get_psk(), psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, psk); + // Callers map the reserved key to nullptr; the context just stores what it is given + ctx.set_psk(nullptr); + EXPECT_FALSE(ctx.has_psk()); } TEST(WireFormatTest, FrameHeaderIsIndicatorPlusBigEndianLength) { diff --git a/tests/components/ota/api_key_offer.yaml b/tests/components/ota/api_key_offer.yaml new file mode 100644 index 0000000000..8d1814bf7e --- /dev/null +++ b/tests/components/ota/api_key_offer.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3290 + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/components/ota/api_runtime_key.yaml b/tests/components/ota/api_runtime_key.yaml new file mode 100644 index 0000000000..8976c92f96 --- /dev/null +++ b/tests/components/ota/api_runtime_key.yaml @@ -0,0 +1,10 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + +ota: + - platform: esphome + port: 3291 diff --git a/tests/components/ota/test-api_key_offer.esp32-idf.yaml b/tests/components/ota/test-api_key_offer.esp32-idf.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_key_offer.esp8266-ard.yaml b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp32-idf.yaml b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6777e6cabc..15c5860879 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -162,6 +162,13 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) +@pytest.fixture +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Host preferences persist per device name; give the test its own so a + provisioned key never leaks into another run.""" + monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) + + @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" diff --git a/tests/integration/const.py b/tests/integration/const.py index 6876bbd443..e35d4673af 100644 --- a/tests/integration/const.py +++ b/tests/integration/const.py @@ -9,6 +9,13 @@ API_CONNECTION_TIMEOUT = 30.0 # seconds PORT_WAIT_TIMEOUT = 30.0 # seconds PORT_POLL_INTERVAL = 0.1 # seconds +# The well-known all-zeros provisioning PSK, a key to provision over it, and +# the time the device takes to activate a newly saved key (100 ms timer plus +# margin) +ZERO_PSK = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +PROVISIONING_PSK = b"bm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm4=" +KEY_ACTIVATION_DELAY = 0.5 # seconds + # Process shutdown timeouts SIGINT_TIMEOUT = 5.0 # seconds SIGTERM_TIMEOUT = 2.0 # seconds diff --git a/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml new file mode 100644 index 0000000000..1dedcc9ee1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + password: "hunter2" +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml new file mode 100644 index 0000000000..aa0a9a66c9 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index bcea2a2471..f315335d1b 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -10,34 +10,40 @@ from __future__ import annotations import asyncio import base64 +import socket from aioesphomeapi import InvalidEncryptionKeyAPIError, RequiresEncryptionAPIError import pytest -from .types import APIClientConnectedFactory, RunCompiledFunction +from .conftest import run_binary_and_wait_for_port +from .const import KEY_ACTIVATION_DELAY, LOCALHOST, PROVISIONING_PSK, ZERO_PSK +from .types import ( + APIClientConnectedFactory, + CompileFunction, + ConfigWriter, + RunCompiledFunction, +) -# The well-known provisioning PSK: base64 of 32 zero bytes -ZERO_PSK = base64.b64encode(bytes(32)).decode() -# A real key to provision -NEW_KEY = base64.b64encode(b"n" * 32) -# Time for the device to activate a newly saved key (100ms timer plus margin) -KEY_ACTIVATION_DELAY = 0.5 - - -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so every run starts unprovisioned.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) +pytestmark = pytest.mark.usefixtures("isolated_preferences") +NEW_KEY = PROVISIONING_PSK @pytest.mark.asyncio async def test_api_zero_psk_provisioning( yaml_config: str, - run_compiled: RunCompiledFunction, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], api_client_connected: APIClientConnectedFactory, ) -> None: - """Exercise the reject paths, then provision a key over the zero-PSK channel.""" - async with run_compiled(yaml_config): + """Exercise the reject paths, provision a key over the zero-PSK channel, + and check the key comes back from preferences on the next boot.""" + port, port_socket = reserved_tcp_port + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + port_socket.close() + + async with run_binary_and_wait_for_port(binary_path, LOCALHOST, port): # --- Pre-provisioning reject paths (device state is unchanged) --- # A wrong (non-zero) PSK fails against the zero provisioning PSK @@ -97,6 +103,19 @@ async def test_api_zero_psk_provisioning( async with api_client_connected(timeout=5) as client: await client.device_info() + # The key is loaded from preferences on the next boot + lines: list[str] = [] + async with run_binary_and_wait_for_port( + binary_path, LOCALHOST, port, line_callback=lines.append + ): + async with api_client_connected(noise_psk=NEW_KEY.decode()) as client: + device_info = await client.device_info() + assert device_info.api_encryption_provisionable is False + with pytest.raises(InvalidEncryptionKeyAPIError): + async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client: + await client.device_info() + assert any("Loaded saved Noise PSK" in line for line in lines) + @pytest.mark.asyncio async def test_api_zero_psk_provisioning_plaintext( diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 4e74814534..f8c122c6e1 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -8,9 +8,12 @@ instance covers the FD_CLOEXEC path. from __future__ import annotations import asyncio +import base64 from collections.abc import Generator from contextlib import contextmanager +from dataclasses import dataclass import functools +from pathlib import Path import socket import pytest @@ -18,10 +21,18 @@ import pytest from esphome import espota2 from .conftest import run_binary, wait_and_connect_api_client -from .const import LOCALHOST, PORT_POLL_INTERVAL, PORT_WAIT_TIMEOUT -from .types import CompileFunction, ConfigWriter +from .const import ( + KEY_ACTIVATION_DELAY, + LOCALHOST, + PORT_POLL_INTERVAL, + PORT_WAIT_TIMEOUT, + PROVISIONING_PSK, + ZERO_PSK, +) +from .types import APIClientConnectedFactory, CompileFunction, ConfigWriter DEVICE_NAME = "host-ota-test" +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @contextmanager @@ -35,6 +46,14 @@ def _reserve_port() -> Generator[tuple[int, socket.socket]]: s.close() +async def _wait_for_line(lines: list[str], needle: str, timeout: float = 5.0) -> None: + """The config dump prints after every setup, a little after the api port + opens, so wait for it rather than assert on the lines seen so far.""" + async with asyncio.timeout(timeout): + while not any(needle in line for line in lines): + await asyncio.sleep(PORT_POLL_INTERVAL) + + async def _wait_for_port(host: str, port: int, timeout: float) -> None: """Poll until a TCP port accepts connections, or raise TimeoutError.""" loop = asyncio.get_running_loop() @@ -51,6 +70,102 @@ async def _wait_for_port(host: str, port: int, timeout: float) -> None: raise TimeoutError(f"Port {port} on {host} did not open within {timeout}s") +async def _build( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> tuple[int, int, Path]: + """Reserve an OTA port, compile the fixture with it, and release both + ports right before the binary is started.""" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + config_path = await write_yaml_config( + yaml_config.replace("__OTA_PORT__", str(ota_port)) + ) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + return api_port, ota_port, binary_path + + +async def _run_ota( + ota_port: int, + password: str | None, + binary_path: Path, + noise_psk: str | None, + plaintext_fallback: bool = False, +) -> int: + """espota2 is blocking; run it in the executor and return its exit code.""" + rc, _ = await asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + password, + binary_path, + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, + ), + ) + return rc + + +@dataclass +class _Device: + """A running host binary and the checks every successful OTA repeats: + a safe reboot, the api port back up, and the pid preserved by execv.""" + + api_port: int + ota_port: int + binary_path: Path + proc: asyncio.subprocess.Process | None = None + reboots: int = 0 + + def __post_init__(self) -> None: + self._rebooted = asyncio.Event() + + def on_log(self, line: str) -> None: + if "Rebooting safely" in line: + self.reboots += 1 + self._rebooted.set() + + async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: + async with asyncio.timeout(timeout): + while self.reboots < count: + self._rebooted.clear() + await self._rebooted.wait() + + async def ota( + self, + password: str | None, + noise_psk: str | None, + msg: str, + plaintext_fallback: bool = False, + ) -> None: + """Upload, then expect the re-exec with the pid preserved.""" + pid_before = self.proc.pid + expected_reboots = self.reboots + 1 + rc = await _run_ota( + self.ota_port, password, self.binary_path, noise_psk, plaintext_fallback + ) + assert rc == 0, msg + await self.wait_reboot(expected_reboots) + await _wait_for_port(LOCALHOST, self.api_port, PORT_WAIT_TIMEOUT) + assert self.proc.returncode is None, "process exited instead of execing" + assert self.proc.pid == pid_before + + async def refused_ota( + self, password: str | None, noise_psk: str | None, msg: str + ) -> None: + """Upload must fail and the device must keep running.""" + rc = await _run_ota(self.ota_port, password, self.binary_path, noise_psk) + assert rc == 1, msg + await asyncio.sleep(0.5) + assert self.proc.returncode is None, "process died on rejected OTA" + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -59,57 +174,34 @@ async def test_host_ota_self_update( reserved_tcp_port: tuple[int, socket.socket], ) -> None: """Self-OTA: upload the running binary back to itself, expect re-exec.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + staged = asyncio.Event() - loop = asyncio.get_running_loop() - ota_staged = loop.create_future() - rebooted = loop.create_future() + def on_log(line: str) -> None: + if "OTA staged at" in line: + staged.set() + dev.on_log(line) - def on_log(line: str) -> None: - if not ota_staged.done() and "OTA staged at" in line: - ota_staged.set_result(True) - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_before = await client.device_info() + assert info_before.name == DEVICE_NAME - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - async with wait_and_connect_api_client(port=api_port) as client: - info_before = await client.device_info() - assert info_before.name == DEVICE_NAME + await dev.ota(None, None, "espota2 reported failure") + assert staged.is_set() - # espota2 is blocking; run in executor. - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "espota2 reported failure" + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_after = await client.device_info() + assert info_after.name == info_before.name - await asyncio.wait_for(ota_staged, timeout=10.0) - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - - # execv preserves pid; mismatch means external respawn. - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info_after = await client.device_info() - assert info_after.name == DEVICE_NAME - assert info_after.name == info_before.name - - # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "second OTA failed -- listener leaked across execv" - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.pid == pid_before + # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). + await dev.ota(None, None, "second OTA failed -- listener leaked across execv") @pytest.mark.asyncio @@ -121,51 +213,110 @@ async def test_host_ota_encrypted( ) -> None: """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" pytest.importorskip("aioesphomeapi.noise") - noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await dev.refused_ota( + None, None, "plaintext upload to an encrypted device must fail" + ) + await dev.ota(None, API_KEY, "encrypted OTA reported failure") - loop = asyncio.get_running_loop() - rebooted = loop.create_future() - def on_log(line: str) -> None: - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) +@pytest.mark.asyncio +async def test_host_ota_api_key_offer_with_password( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + caplog: pytest.LogCaptureFixture, +) -> None: + """With only an api key the device offers encryption without requiring + it: the password still guards plaintext uploads, the key alone + authenticates an encrypted one, and until 2027.3.0 a failed encrypted + attempt falls back to plaintext.""" + pytest.importorskip("aioesphomeapi.noise") + wrong_key = base64.b64encode(b"w" * 32).decode() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "Encryption: offered") - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid + await dev.refused_ota( + None, None, "plaintext upload without the password must fail" + ) + await dev.ota( + "hunter2", None, "plaintext upload with the password must succeed" + ) + await dev.ota(None, API_KEY, "encrypted upload with the api key must succeed") - # A plaintext upload must be refused with the device unharmed - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + # Remove before 2027.3.0: a wrong key falls back to plaintext, which + # the password still guards + with caplog.at_level("WARNING", logger="esphome.espota2"): + await dev.ota( + "hunter2", + wrong_key, + "the plaintext retry with the password must succeed", + plaintext_fallback=True, ) - assert rc == 1, "plaintext upload to an encrypted device must fail" - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected plaintext OTA" + assert any("Retrying in plaintext" in r.message for r in caplog.records) + await dev.ota( + None, + API_KEY, + "the right api key encrypts without touching the fallback", + plaintext_fallback=True, + ) - # The encrypted upload goes through and the device re-execs - rc, _ = await loop.run_in_executor( - None, - functools.partial( - espota2.run_ota, - LOCALHOST, - ota_port, - None, - binary_path, - noise_psk=noise_psk, - ), - ) - assert rc == 0, "encrypted OTA reported failure" - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_preferences") +async def test_host_ota_provisioned_api_key( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """A key provisioned over the api feeds the OTA offer: plaintext works + while unprovisioned, the provisioned key encrypts, the key loaded from + preferences on the next boot keeps encrypting, and plaintext stays + accepted because only the ota block requires encryption.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "once the api key is provisioned") + + await dev.ota( + None, None, "plaintext upload to an unprovisioned device must succeed" + ) + + async with api_client_connected( + port=dev.api_port, noise_psk=ZERO_PSK + ) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + key = PROVISIONING_PSK.decode() + await dev.ota( + None, key, "encrypted upload with the provisioned key must succeed" + ) + await dev.ota(None, key, "the key loaded at boot must feed the OTA offer") + await dev.ota(None, None, "plaintext must stay accepted on an offering device") @pytest.mark.asyncio @@ -177,33 +328,25 @@ async def test_host_ota_rejects_garbage( integration_test_dir, ) -> None: """Bogus payload is rejected and the device keeps running.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # 192 bytes that are neither ELF nor Mach-O. + bogus_path = integration_test_dir / "bogus.bin" + bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) - # 192 bytes that are neither ELF nor Mach-O. - bogus_path = integration_test_dir / "bogus.bin" - bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) + async with run_binary(dev.binary_path) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + rc = await _run_ota(dev.ota_port, None, bogus_path, None) + assert rc == 1 + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected OTA" + assert proc.pid == pid_before - api_socket.close() - ota_socket.close() - - async with run_binary(binary_path) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - - loop = asyncio.get_running_loop() - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, bogus_path - ) - assert rc == 1 - - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected OTA" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info = await client.device_info() - assert info.name == DEVICE_NAME + async with wait_and_connect_api_client(port=dev.api_port) as client: + info = await client.device_info() + assert info.name == DEVICE_NAME diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py index 5b43d05530..439220f09c 100644 --- a/tests/unit_tests/test_espota2_noise.py +++ b/tests/unit_tests/test_espota2_noise.py @@ -10,12 +10,15 @@ when the installed aioesphomeapi predates the noise module. from __future__ import annotations import base64 +from collections.abc import Callable import hashlib import io +import logging from pathlib import Path import socket import sys import threading +from typing import Any from unittest.mock import Mock, patch import pytest @@ -65,8 +68,12 @@ class FakeEncryptedDevice(threading.Thread): offer_noise: bool = True, require_noise: bool = True, prologue_features_override: int | None = None, + connections: int = 1, + drop_handshakes: int = 0, ) -> None: super().__init__(daemon=True) + self.connections = connections + self.drop_handshakes = drop_handshakes # hang up mid-handshake this many times self.psk = psk self.version = version self.offer_noise = offer_noise @@ -81,10 +88,11 @@ class FakeEncryptedDevice(threading.Thread): def run(self) -> None: try: - sock, _ = self.listener.accept() - sock.settimeout(10) - with sock: - self._serve(sock) + for _ in range(self.connections): + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) except Exception as err: # noqa: BLE001 - surfaced via join_and_check self.error = err finally: @@ -109,8 +117,23 @@ class FakeEncryptedDevice(threading.Thread): return server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) - if not (self.offer_noise and noise_negotiated): - return # the client fails closed; nothing further arrives + if not (noise_negotiated and self.offer_noise): + # A device that does not require encryption continues in + # plaintext whatever the client asked for, like older firmware + try: + self._transfer( + lambda byte: sock.sendall(bytes([byte])), + lambda length: _recv_exact(sock, length), + lambda remaining: _recv_exact( + sock, min(remaining, espota2.UPLOAD_BLOCK_SIZE) + ), + ) + except ConnectionError: + # A keyed client without fallback fails closed and hangs up + if noise_negotiated and not self.offer_noise: + return + raise + return from cryptography.exceptions import InvalidTag from noise.connection import NoiseConnection @@ -134,6 +157,9 @@ class FakeEncryptedDevice(threading.Thread): msg1 = _recv_frame(sock) assert msg1[0] == 0x00 + if self.drop_handshakes > 0: + self.drop_handshakes -= 1 + return # a transport fault: the socket closes with no reply try: proto.read_message(msg1[1:]) except InvalidTag: @@ -149,6 +175,20 @@ class FakeEncryptedDevice(threading.Thread): assert len(plaintext) == length, "control units must be one per frame" return plaintext + def recv_data(_remaining: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + return plaintext + + self._transfer(send_byte, recv_unit, recv_data) + + def _transfer( + self, + send_byte: Callable[[int], None], + recv_unit: Callable[[int], bytes], + recv_data: Callable[[int], bytes], + ) -> None: + """The post-handshake exchange, identical over both transports.""" send_byte(espota2.RESPONSE_AUTH_OK) recv_unit(1) # ota type size = int.from_bytes(recv_unit(4), "big") @@ -159,9 +199,7 @@ class FakeEncryptedDevice(threading.Thread): received = b"" acked = 0 while len(received) < size: - plaintext = proto.decrypt(_recv_frame(sock)) - assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT - received += plaintext + received += recv_data(size - len(received)) if self.version >= espota2.OTA_VERSION_2_0: while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( len(received) == size and acked < size @@ -176,7 +214,10 @@ class FakeEncryptedDevice(threading.Thread): def _upload( - device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None + device: FakeEncryptedDevice, + firmware: bytes, + noise_psk: str | None, + plaintext_fallback: bool = False, ) -> None: device.start() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -184,12 +225,35 @@ def _upload( sock.connect(("127.0.0.1", device.port)) try: espota2.perform_ota( - sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + sock, + None, + io.BytesIO(firmware), + Path("firmware.bin"), + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, ) finally: sock.close() +def _run_ota( + device: FakeEncryptedDevice, firmware: bytes, tmp_path: Path, noise_psk: str +) -> int: + """Drive the retry loop, which is where the plaintext fallback reconnects.""" + path = tmp_path / "firmware.bin" + path.write_bytes(firmware) + device.start() + rc, _ = espota2.run_ota( + "127.0.0.1", + device.port, + None, + path, + noise_psk=noise_psk, + plaintext_fallback=True, + ) + return rc + + def test_encrypted_upload_success() -> None: """A full encrypted v2 upload spanning several 8192-byte blocks.""" pytest.importorskip("aioesphomeapi.noise") @@ -240,6 +304,56 @@ def test_client_fails_closed_when_device_lacks_encryption() -> None: device.join_and_check() +# Remove before 2027.3.0 +def test_fallback_when_device_does_not_offer(caplog: pytest.LogCaptureFixture) -> None: + """The api key is tried opportunistically; an older device that cannot + encrypt still gets its update, with a warning.""" + firmware = b"firmware" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + _upload(device, firmware, PSK, plaintext_fallback=True) + device.join_and_check() + assert device.received == firmware + assert any("fallback is removed in 2027.3.0" in r.message for r in caplog.records) + + +# Remove before 2027.3.0 +@pytest.mark.parametrize( + ("device_kwargs", "expected_rc", "fell_back"), + [ + # A wrong key against an offering device reconnects in plaintext + ({"psk": OTHER_PSK, "require_noise": False, "connections": 2}, 0, True), + # The plaintext retry is refused by a device that requires encryption + ({"psk": OTHER_PSK, "require_noise": True, "connections": 2}, 1, True), + # A dropped connection inside the handshake is retried encrypted + ({"require_noise": False, "connections": 2, "drop_handshakes": 1}, 0, False), + # A second transport fault inside the handshake falls back + ({"require_noise": False, "connections": 3, "drop_handshakes": 2}, 0, True), + ], + ids=["wrong_key", "wrong_key_required", "one_fault", "two_faults"], +) +def test_fallback_through_the_retry_loop( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, + device_kwargs: dict[str, Any], + expected_rc: int, + fell_back: bool, +) -> None: + pytest.importorskip("aioesphomeapi.noise") + firmware = b"firmware" + device = FakeEncryptedDevice(**device_kwargs) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + rc = _run_ota(device, firmware, tmp_path, PSK) + device.join_and_check() + assert rc == expected_rc + assert (device.received == firmware) is (expected_rc == 0) + assert ( + any("Retrying in plaintext" in r.message for r in caplog.records) is fell_back + ) + if expected_rc == 1: + assert any("requires an encrypted OTA" in r.message for r in caplog.records) + + def test_plaintext_client_gets_encryption_required_error() -> None: """A client without a key gets the device's 0x94 error message.""" device = FakeEncryptedDevice() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 5372a7203d..8fb9b7376e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -2108,7 +2108,13 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + "secret", + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2140,10 +2146,77 @@ def test_upload_program_ota_encryption_key( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=False, ) +def test_upload_program_ota_api_key_opportunistic( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Without an ota encryption block the api key is tried with a plaintext + fallback (removed in 2027.3.0).""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: key}}, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=True, + ) + + +@pytest.mark.parametrize( + "api_conf", + [{}, {CONF_ENCRYPTION: {}}], + ids=["no_encryption", "runtime_key"], +) +def test_upload_program_ota_no_usable_api_key_stays_plaintext( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + api_conf: dict[str, Any], +) -> None: + """A missing or runtime provisioned api key gives the uploader nothing + to try.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_API: api_conf, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert mock_run_ota.call_args.args[5] is None + assert mock_run_ota.call_args.kwargs == {"plaintext_fallback": False} + + def test_upload_program_ota_encryption_without_key_fails_closed( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2194,7 +2267,13 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + Path("custom.bin"), + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2250,6 +2329,7 @@ def test_upload_program_ota_partition_table_with_file_arg( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2312,6 +2392,7 @@ def test_upload_program_ota_partition_table_mqttip( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2500,6 +2581,7 @@ def test_upload_program_ota_bootloader_with_file_arg( bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, None, + plaintext_fallback=False, ) @@ -2988,7 +3070,13 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -3038,7 +3126,13 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.50"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5211,6 +5305,7 @@ def test_upload_program_ota_static_ip_with_mqttip( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5261,6 +5356,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5438,7 +5534,13 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) diff --git a/tests/unit_tests/test_wizard.py b/tests/unit_tests/test_wizard.py index 244e4eb5a1..f57ae71ae6 100644 --- a/tests/unit_tests/test_wizard.py +++ b/tests/unit_tests/test_wizard.py @@ -37,7 +37,6 @@ def wizard_answers() -> list[str]: "nodemcuv2", # board "SSID", # ssid "psk", # wifi password - "", # ota password (empty for no password) ] @@ -101,6 +100,25 @@ def test_config_file_should_include_ota(default_config: dict[str, Any]): assert "ota:" in config +def test_config_file_should_use_encryption_when_api_key_set( + default_config: dict[str, Any], +): + """ + With an API encryption key and no OTA password the OTA block reuses the key + """ + # Given + default_config["api_encryption_key"] = ( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + ) + + # When + config = wz.wizard_file(**default_config) + + # Then + assert "ota:\n - platform: esphome\n encryption:" in config + assert "password" not in config.split("ota:")[1].split("wifi:")[0] + + def test_config_file_should_include_ota_when_password_set( default_config: dict[str, Any], ): @@ -630,15 +648,15 @@ def test_wizard_write_protects_existing_config( assert config_file.read_text() == original_content -def test_wizard_accepts_ota_password( +def test_wizard_uses_the_api_key_for_ota( tmp_path: Path, monkeypatch: MonkeyPatch, wizard_answers: list[str] ): """ - The wizard should pass ota_password to wizard_write when the user provides one + The wizard generates an api key and does not ask for an OTA password; + the key secures OTA updates """ # Given - wizard_answers[5] = "my_ota_password" # Set OTA password config_file = tmp_path / "test.yaml" input_mock = MagicMock(side_effect=wizard_answers) monkeypatch.setattr("builtins.input", input_mock) @@ -653,8 +671,9 @@ def test_wizard_accepts_ota_password( # Then assert retval == 0 call_kwargs = wizard_write_mock.call_args.kwargs - assert "ota_password" in call_kwargs - assert call_kwargs["ota_password"] == "my_ota_password" + assert "api_encryption_key" in call_kwargs + assert "ota_password" not in call_kwargs + assert input_mock.call_count == len(wizard_answers) def test_wizard_accepts_rpipico_board(tmp_path: Path, monkeypatch: MonkeyPatch): From 96b1a03ea493a7281158907c6dd98184a48c05f2 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:05:16 +0000 Subject: [PATCH 28/53] Bump bundled esphome-device-builder to 1.14.4 (#19006) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e875851bfb..da76ab7b6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 RUN \ platformio settings set enable_telemetry No \ From c1aa41f276e4bc2b05f4b45031229623d93ab84a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:12:52 +0200 Subject: [PATCH 29/53] [noise] Bump noise-c to 0.1.24 and libsodium to 1.10021.6 (#18989) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index a1d9444fc0..4de706120e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.21") + cg.add_library("esphome/noise-c", "0.1.24") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.4") + cg.add_library("esphome/libsodium", "1.10021.6") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index fcf7caa7c7..779a05e7de 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.21 ; used by noise (api, ota) + esphome/noise-c@0.1.24 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index a263d7937f..4f7f5a4a4c 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.21") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.21") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.21"] + assert libs == ["esphome/noise-c @ 0.1.24"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.21", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.24", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.21"] + assert cls.calls == ["esphome/noise-c @ 0.1.24"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.21"] is None + assert compats["esphome/noise-c @ 0.1.24"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.21"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 379ef52ebd..fb79885736 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1576,7 +1576,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1596,7 +1596,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 5d2ddc658c3db2431fb71dfc78dc2df885f1cf78 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:26:44 +0200 Subject: [PATCH 30/53] [mdns] Guard LEAmDNS main loop calls against lwIP re-entrancy on ESP8266 (#18990) --- esphome/components/mdns/__init__.py | 2 + esphome/components/mdns/mdns_esp8266.cpp | 51 +++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index f039bb69f0..c8020104b3 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -192,6 +192,8 @@ async def to_code(config: ConfigType) -> None: if CORE.using_arduino: if CORE.is_esp8266: cg.add_library("ESP8266mDNS", None) + # No MDNS global in the build; mdns_esp8266.cpp owns a guarded MDNSResponder + cg.add_build_flag("-DNO_GLOBAL_MDNS") elif CORE.is_rp2: cg.add_library("LEAmDNS", None) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index 1f0b3c9519..0e600d3bac 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -13,8 +13,47 @@ namespace esphome::mdns { +// Main-loop calls into LEAmDNS that send (update() and close(); begin(), addService() and +// the scheduled restart never reach a send) can yield inside UdpContext::sendTimeout(); a +// packet arriving then re-enters LEAmDNS from lwIP on the same UdpContext and both sides +// free the same tx pbufs (#18760). Received packets stay queued during such a call and are +// processed from the main loop afterwards. +class GuardedMDNSResponder : public ::esp8266::MDNSImplementation::MDNSResponder { + public: + void update_guarded() { this->run_guarded_(&GuardedMDNSResponder::update); } + void close_guarded() { this->run_guarded_(&GuardedMDNSResponder::close); } + + private: + void run_guarded_(bool (GuardedMDNSResponder::*fn)()) { + UdpContext *ctx = this->m_pUDPContext; + if (ctx == nullptr) { + (this->*fn)(); + return; + } + // Set every time: a restart replaces the context together with its stock handler. Only + // begin() and the scheduled netif callback restart, never update() or close(), so the + // context cannot change underneath this call. + ctx->onRx([this]() { + if (!this->in_loop_call_) { + this->_callProcess(); + } + }); + this->in_loop_call_ = true; + (this->*fn)(); + // close() releases the context; a yield in here queues further packets for this loop too + while (this->m_pUDPContext != nullptr && this->m_pUDPContext->next()) { + this->_parseMessage(); + } + this->in_loop_call_ = false; + } + + volatile bool in_loop_call_{false}; +}; + +static GuardedMDNSResponder mdns_responder; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + static void register_esp8266(MDNSComponent *, StaticVector &services) { - MDNS.begin(App.get_name().c_str()); + mdns_responder.begin(App.get_name().c_str()); for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is @@ -30,10 +69,10 @@ static void register_esp8266(MDNSComponent *, StaticVectoris_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) return; #endif - MDNS.update(); + mdns_responder.update_guarded(); }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } @@ -81,7 +120,7 @@ void MDNSComponent::on_ip_state(const network::IPAddresses &ips, const network:: #endif void MDNSComponent::on_shutdown() { - MDNS.close(); + mdns_responder.close_guarded(); delay(10); } From e0e85db822309dd8fe17043552405c8c8b8374af Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:28:31 +0200 Subject: [PATCH 31/53] [core] Show the other downloader's progress while a prefetch job waits on its lock (#18983) --- esphome/framework_helpers.py | 61 +++++++++++++- esphome/platformio/prefetch.py | 87 ++++++++++---------- esphome/platformio/registry.py | 67 ++++++++++----- tests/unit_tests/conftest.py | 39 ++++++++- tests/unit_tests/test_framework_helpers.py | 17 ++++ tests/unit_tests/test_platformio_prefetch.py | 87 ++++++++++++++++++-- tests/unit_tests/test_platformio_registry.py | 73 ++++++++++++++-- 7 files changed, 348 insertions(+), 83 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 82bc0d3727..fc2a18a6ec 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -23,6 +23,7 @@ from esphome.net_retry import ( ) if TYPE_CHECKING: + from filelock import FileLock import requests PathType = str | os.PathLike @@ -909,6 +910,61 @@ def _part_path(dest: Path) -> Path: return dest.with_name(dest.name + ".part") +def downloaded_bytes(dest: Path, size: int | None = None) -> int: + """Bytes of ``dest`` on disk (its ``.part`` while streaming), capped at ``size``.""" + done = 0 + for candidate in (_part_path(dest), dest): + try: + done = candidate.stat().st_size + break + except FileNotFoundError: + continue + return done if size is None else min(done, size) + + +# Short lock-acquire slices so a waiting worker still observes Ctrl-C +_DOWNLOAD_LOCK_POLL = 1 + +# Waiting on another process's download; past this the caller leaves the +# file to its holder (the later sequential install waits on the same lock) +DOWNLOAD_LOCK_TIMEOUT = 60 + + +class DownloadLockUnavailable(OSError): + """The lock file cannot be used at all (a lock-less filesystem).""" + + +def wait_for_download_lock( + lock: "FileLock", + tracker: Callable[[int], None], + on_disk: Callable[[], int], + name: str, +) -> None: + """Acquire ``lock``, reporting ``on_disk()`` to ``tracker`` each poll so the + bar follows the holder's download. Raises filelock's ``Timeout`` once + ``DOWNLOAD_LOCK_TIMEOUT`` seconds pass.""" + from filelock import Timeout + + deadline = time.monotonic() + DOWNLOAD_LOCK_TIMEOUT + waiting = False + while True: + try: + lock.acquire(timeout=_DOWNLOAD_LOCK_POLL) + return + except Timeout: + pass + except OSError as err: + # Distinct from an OSError out of on_disk(), which must not + # read as "locks unsupported" + raise DownloadLockUnavailable(*err.args) from err + if not waiting: + waiting = True + _LOGGER.info("Waiting for another process downloading %s", name) + tracker(on_disk()) # raises when the batch is cancelled + if time.monotonic() >= deadline: + raise Timeout(lock.lock_file) + + def discard_partial_download(dest: Path) -> None: """Remove ``dest`` and the resume sidecars of an abandoned download.""" part = _part_path(dest) @@ -1319,10 +1375,7 @@ def download_from_mirrors( ) # Tick with the bytes already on disk so a combined bar holds # steady during the backoff instead of rewinding to zero - done = 0 - if progress is not None: - part = _part_path(path_target) - done = part.stat().st_size if part.is_file() else 0 + done = downloaded_bytes(path_target) if progress is not None else 0 _cancellable_sleep(delay, progress, done) # 3. Report every attempted URL if all mirrors failed. failures spans diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 5097239065..17a06cb9c1 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -33,11 +33,14 @@ import time from typing import Any, NamedTuple from esphome.framework_helpers import ( + DownloadLockUnavailable, content_length, discard_partial_download, + downloaded_bytes, failure_reason, resume_fetch_job, run_batch_downloads, + wait_for_download_lock, warn_prefetch_failures, ) from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree @@ -61,16 +64,10 @@ _RESOLVE_WORKERS = 8 # A hung child must not block the build; downloads resume on the next run _PREFETCH_TIMEOUT = 20 * 60 -# Waiting on another process's URL download; past this, leave it to pio -_DOWNLOAD_LOCK_TIMEOUT = 60 - # Child exit for a handled, already-warned failure; 1 would collide with # the interpreter's own import-failure exit _EXIT_HANDLED = 3 -# Short lock-acquire slices so a waiting worker still observes Ctrl-C -_URI_LOCK_POLL = 1 - # Resolution errored (vs a clean skip); suppresses the warm sentinel _RESOLVE_FAILED = object() @@ -462,51 +459,54 @@ def _uri_jobs( def _serialized_fetch_job( - dl_path: Path, lock_path: str, body: Any, unlocked_ok: bool = True + dl_path: Path, + lock_path: str, + body: Any, + size: int, + stream_dest: Path | None = None, + unlocked_ok: bool = True, ) -> Any: - """Wrap ``body`` so the shared destination is single-writer. - - Interleaved writers truncate each other's ``.part`` bytes (see - registry.py). The bounded poll observes Ctrl-C via the tracker; a - blown deadline is a clean skip (the holder's copy is what the build - needs). On a lock-less filesystem a sha256-verified body runs - unlocked with one warning; a checksum-less one - (``unlocked_ok=False``) is a counted failure instead. + """Wrap ``body`` so the shared destination is single-writer (interleaved + writers truncate each other's ``.part``, see registry.py). A blown deadline + is a clean skip. On a lock-less filesystem a sha256-verified body runs + unlocked with one warning; a checksum-less one (``unlocked_ok=False``) fails. """ + def on_disk() -> int: + # A URL job's holder streams beside the staging path until it + # promotes; after that only dl_path is left + done = downloaded_bytes(dl_path, size) + if not done and stream_dest is not None: + done = downloaded_bytes(stream_dest, size) + return done + def run(tracker: Any) -> None: from filelock import FileLock, Timeout # fallback_to_soft would leave a stale marker on lock-less # filesystems that blocks every later build (see git.py) lock = FileLock(lock_path, fallback_to_soft=False) - deadline = time.monotonic() + _DOWNLOAD_LOCK_TIMEOUT - while True: - try: - lock.acquire(timeout=_URI_LOCK_POLL) - break - except Timeout: - tracker(0) # raises when the batch is cancelled - if time.monotonic() >= deadline: - # Another process is fetching this same file; its copy - # is what the build needs (a large framework archive - # can hold the lock far longer than this deadline) - _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) - return - except OSError as err: - if not unlocked_ok: - # A body with no checksum to catch interleaved corruption - raise - lock = None - _LOGGER.warning( - "Could not lock %s (%s); downloading unlocked", - dl_path.name, - err, - ) - break + try: + wait_for_download_lock(lock, tracker, on_disk, dl_path.name) + except Timeout: + # The holder's copy is what the build needs (a large + # framework archive can outlast this deadline) + _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) + return + except DownloadLockUnavailable as err: + if not unlocked_ok: + # A body with no checksum to catch interleaved corruption + raise + lock = None + _LOGGER.warning( + "Could not lock %s (%s); downloading unlocked", + dl_path.name, + err, + ) try: if dl_path.is_file(): - return # another process finished it while we waited + tracker(size) # another process finished it while we waited + return body(tracker) finally: if lock is not None: @@ -540,6 +540,7 @@ def _registry_fetch_job( dl_path, f"{dl_path}.esphome.lock", resume_fetch_job(url, dl_path, sha256=checksum, size=size), + size, ) def run(tracker: Any) -> None: @@ -571,9 +572,9 @@ def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: tmp.replace(dl_path) def run(tracker: Any) -> None: - _serialized_fetch_job(dl_path, f"{tmp}.lock", promote, unlocked_ok=False)( - tracker - ) + _serialized_fetch_job( + dl_path, f"{tmp}.lock", promote, size, tmp, unlocked_ok=False + )(tracker) if dl_path.is_file(): # Won or lost, the race is over; staging files left behind # are dead weight PlatformIO's cache never prunes diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index 9538a28ff4..75df82da0e 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -17,8 +17,10 @@ from esphome.framework_helpers import ( archive_extract_all, download_from_mirrors, download_with_resume, + downloaded_bytes, rmdir, run_batch_downloads, + wait_for_download_lock, ) from esphome.net_retry import fetch_with_retry, http_request @@ -164,11 +166,17 @@ class _PendingArchive(NamedTuple): name: str version: str dest: Path + archive: Path url: str sha256: str size: int +def _archive_path(downloads_dir: Path, name: str, version: str) -> Path: + """The one archive path the prefetch and the sequential install share.""" + return downloads_dir / f"{name}-{version}" + + def _already_installed(dest: Path) -> bool: """Whether ``dest`` holds a completed install (extraction marker).""" return (dest / ".esphome_extracted").is_file() @@ -187,18 +195,18 @@ def prefetch_packages( lock as ``install_package``: the archive's ``.part`` file is shared, and two concurrent writers would truncate each other's bytes. """ - from filelock import FileLock + from filelock import FileLock, Timeout pending: list[_PendingArchive] = [] - seen: set[str] = set() + seen: set[Path] = set() for name, version, dest, mirrors in packages: if mirrors or (dest / ".esphome_extracted").is_file(): continue - archive_name = f"{name}-{version}" - if archive_name in seen: + archive = _archive_path(downloads_dir, name, version) + if archive in seen: # A duplicate entry would race itself between two workers continue - seen.add(archive_name) + seen.add(archive) try: url, sha256, size = registry_download(name, version) except EsphomeError as err: @@ -207,10 +215,9 @@ def prefetch_packages( continue if not size: continue - archive = downloads_dir / archive_name if archive.is_file() and archive.stat().st_size == size: continue - pending.append(_PendingArchive(name, version, dest, url, sha256, size)) + pending.append(_PendingArchive(name, version, dest, archive, url, sha256, size)) if len(pending) < 2: return downloads_dir.mkdir(parents=True, exist_ok=True) @@ -222,20 +229,36 @@ def prefetch_packages( def _fetch(entry: _PendingArchive, tracker: Callable[[int], None]) -> None: entry.dest.parent.mkdir(parents=True, exist_ok=True) - with FileLock(f"{entry.dest}.lock", fallback_to_soft=False): - # Marker re-check: a concurrent build may have installed (and - # deleted the archive of) this package while we waited; - # re-downloading would orphan a fresh copy in downloads_dir - # no branch: the thread tracer misses the skip edge; both - # arms of _already_installed are pinned directly - if not _already_installed(entry.dest): # pragma: no branch - download_with_resume( - entry.url, - downloads_dir / f"{entry.name}-{entry.version}", - sha256=entry.sha256, - size=entry.size, - progress=tracker, - ) + + def on_disk() -> int: + if done := downloaded_bytes(entry.archive, entry.size): + return done + # The holder deletes the archive once it has installed it + return entry.size if _already_installed(entry.dest) else 0 + + lock = FileLock(f"{entry.dest}.lock", fallback_to_soft=False) + try: + wait_for_download_lock(lock, tracker, on_disk, entry.name) + except Timeout: + # install_package waits on this same lock and verifies the + # holder's copy + _LOGGER.debug("Leaving %s to its current downloader", entry.name) + return + try: + if _already_installed(entry.dest): + # A concurrent build installed it while we waited; a + # re-download would orphan a fresh copy in downloads_dir + tracker(entry.size) + return + download_with_resume( + entry.url, + entry.archive, + sha256=entry.sha256, + size=entry.size, + progress=tracker, + ) + finally: + lock.release() failures = run_batch_downloads( "Downloading packages", @@ -288,7 +311,7 @@ def install_package( rmdir(dest, msg=f"Clean up incomplete {name} install") # Persistent location so an interrupted download resumes across runs. downloads_dir.mkdir(parents=True, exist_ok=True) - archive = downloads_dir / f"{name}-{version}" + archive = _archive_path(downloads_dir, name, version) _LOGGER.info("Downloading %s %s ...", name, version) if mirrors: _LOGGER.warning( diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 9de8f715ef..ad9c0bb11f 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -9,7 +9,7 @@ not be part of a unit test suite. """ -from collections.abc import Generator +from collections.abc import Callable, Generator import os from pathlib import Path import sys @@ -137,3 +137,40 @@ def mock_get_component() -> Generator[Mock, None, None]: """Mock get_component for config module.""" with patch("esphome.config.get_component") as mock: yield mock + + +@pytest.fixture +def held_lock() -> Callable[..., Callable[..., None]]: + """Factory for a ``FileLock.acquire`` fake held by another downloader. + + Each poll writes the next chunk to ``part`` (or runs it, for a callable) + and raises ``Timeout``; when the chunks run out the part is removed, + ``land()`` runs, and the acquire succeeds (also for any later job, so + ``land`` must be idempotent). + """ + from filelock import Timeout + + def make( + part: Path, + chunks: list[bytes | Callable[[], None]], + land: Callable[[], None], + ) -> Callable[..., None]: + polls = iter(chunks) + + def acquire(*args, **kwargs) -> None: + try: + chunk = next(polls) + except StopIteration: + part.unlink(missing_ok=True) + land() + return + if callable(chunk): + chunk() + else: + part.parent.mkdir(parents=True, exist_ok=True) + part.write_bytes(chunk) + raise Timeout("held") + + return acquire + + return make diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index fcc5572f51..22b34c9df5 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2353,3 +2353,20 @@ def test_discard_partial_download_logs_undeletable( ): framework_helpers.discard_partial_download(dest) assert "Could not remove" in caplog.text + + +def test_downloaded_bytes_reports_what_is_on_disk(tmp_path: Path) -> None: + """Part file first, then the landed file, both capped at size; else 0.""" + dest = tmp_path / "archive" + assert framework_helpers.downloaded_bytes(dest, 4) == 0 + part = tmp_path / "archive.part" + part.write_bytes(b"ab") + assert framework_helpers.downloaded_bytes(dest, 4) == 2 + part.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 + part.unlink() + dest.write_bytes(b"abc") + assert framework_helpers.downloaded_bytes(dest, 4) == 3 + assert framework_helpers.downloaded_bytes(dest) == 3 + dest.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index fb79885736..77490fd861 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -454,23 +454,96 @@ def test_uri_fetch_job_waits_out_a_briefly_held_lock(tmp_path: Path) -> None: assert dl_path.read_bytes() == b"data" -def test_lock_deadline_leaves_download_to_the_holder(tmp_path: Path) -> None: - """A lock held past the deadline means another process is fetching the - same file; skipping cleanly beats a misleading failure warning. The - tracker is still polled so a parked worker observes cancellation.""" +@pytest.mark.parametrize("staged", [b"", b"ab"]) +def test_lock_deadline_leaves_download_to_the_holder( + tmp_path: Path, staged: bytes +) -> None: + """A lock held past the deadline is another process's download; skip + cleanly, polling the tracker with what the holder has staged so far.""" dl_path = tmp_path / "archive" + (tmp_path / "archive.prefetch.part").write_bytes(staged) ticks: list[int] = [] with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) mock_download.assert_not_called() - assert ticks == [0] + assert ticks == [len(staged)] assert not dl_path.exists() +@pytest.mark.parametrize( + ("job", "part_name", "chunks", "expected"), + [ + ( + lambda dl_path: pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + ), + "archive.part", + [b"a", b"abc"], + [1, 3, 4], + ), + ( + lambda dl_path: pf._uri_fetch_job( + MagicMock(), "https://x/a.zip", dl_path, 4 + ), + "archive.prefetch.part", + [b"ab"], + [2, 4], + ), + ], + ids=["registry", "uri"], +) +def test_lock_wait_reports_the_holders_progress( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + held_lock, + job, + part_name: str, + chunks: list[bytes], + expected: list[int], +) -> None: + """A waiting job reports the holder's part file (the staging one for a + URL job), then the full size once the holder lands the archive.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / part_name, chunks, lambda: dl_path.write_bytes(b"abcd") + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + caplog.at_level(logging.INFO), + ): + job(dl_path)(ticks.append) + mock_download.assert_not_called() + assert ticks == expected + assert caplog.text.count("Waiting for another process downloading archive") == 1 + + +def test_uri_lock_wait_prefers_the_landed_archive(tmp_path: Path, held_lock) -> None: + """Between the holder's promotion rename and its release the staging + part is gone; the landed cache file is credited instead of 0.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / "archive.prefetch.part", + [b"ab", lambda: dl_path.write_bytes(b"abcd")], + lambda: None, + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) + mock_download.assert_not_called() + assert ticks == [2, 4, 4] + + def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: """A registry job that lost the download race to another process must not stamp a nonexistent archive into pio's usage.db.""" @@ -479,7 +552,7 @@ def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( lambda done: None diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 6ba8691c4e..9d5f6c4ce5 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -8,6 +8,7 @@ import os from pathlib import Path from unittest.mock import MagicMock, patch +from filelock import Timeout import pytest from esphome.core import EsphomeError @@ -540,16 +541,13 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: dest = tmp_path / "a" dest.mkdir() - from contextlib import contextmanager - - @contextmanager - def marker_appears_under_lock(path, **kwargs): + def marker_appears_under_lock(*args, **kwargs): # Simulates the concurrent build finishing while we waited (dest / ".esphome_extracted").touch() - yield with ( - patch("filelock.FileLock", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.acquire", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.release"), patch.object(registry, "download_with_resume") as mock_download, patch.object( registry, "registry_download", side_effect=_resolve_for({"a": 10}) @@ -559,6 +557,69 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: mock_download.assert_not_called() +def test_prefetch_packages_waits_with_the_holders_progress( + tmp_path: Path, held_lock +) -> None: + """A worker parked on another build's lock reports that build's part + file, then the full size once the marker appears.""" + dest = tmp_path / "a" + dest.mkdir() + ticks: list[int] = [] + part = tmp_path / "dl" / "a-1.0.part" + + def installed_and_pruned() -> None: + # install_package touches the marker, then unlinks the archive + (dest / ".esphome_extracted").touch() + part.unlink() + + acquire = held_lock( + part, + [lambda: None, b"abc", installed_and_pruned], + (dest / ".esphome_extracted").touch, + ) + + def fake_batch(header, jobs): + for _name, _size, fetch in jobs: + fetch(ticks.append) + return [] + + with ( + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + patch.object(registry, "run_batch_downloads", side_effect=fake_batch), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", dest, []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + assert ticks == [0, 3, 10, 10] + mock_download.assert_called_once() + + +def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( + tmp_path: Path, +) -> None: + """Past the deadline the worker skips; install_package waits on the same + lock later and verifies whatever the holder produced.""" + with ( + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", tmp_path / "a", []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + def test_already_installed_probe(tmp_path: Path) -> None: """Both arms of the marker probe the prefetch worker keys on.""" dest = tmp_path / "pkg" From 8434dc5474e433a61a250800b489674ec5116d84 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:29:20 +1200 Subject: [PATCH 32/53] [esp32_hosted] Add ESP-NOW-over-hosted shim for the ESP32-P4 (#17712) --- esphome/components/esp32_hosted/__init__.py | 36 ++ .../esp32_hosted/esp_now_hosted.cpp | 467 ++++++++++++++++++ .../esp32_hosted/esp_now_hosted_rpc.h | 128 +++++ esphome/components/espnow/__init__.py | 20 + esphome/core/defines.h | 1 + script/ci-custom.py | 17 +- .../test-espnow.esp32-p4-idf.yaml | 5 + tests/unit_tests/components/test_espnow.py | 48 ++ 8 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 esphome/components/esp32_hosted/esp_now_hosted.cpp create mode 100644 esphome/components/esp32_hosted/esp_now_hosted_rpc.h create mode 100644 tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml create mode 100644 tests/unit_tests/components/test_espnow.py diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index ab9455250c..21626e432b 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -37,6 +37,25 @@ CONF_HANDSHAKE_PIN = "handshake_pin" CONF_SDIO_FREQUENCY = "sdio_frequency" CONF_SPI_MODE = "spi_mode" +# ESP-NOW-over-hosted shim (esp_now_hosted.cpp). esp-hosted proxies esp_wifi.h +# but not esp_now.h (espressif/esp-hosted-mcu#19), and esp_wifi_remote injects +# the esp_now.h header on the ESP32-P4 host with no implementation, leaving the +# esp_now_* symbols undefined at link. On a P4 host, esp_now_hosted.cpp DEFINES +# those symbols and forwards each call to the co-processor over esp-hosted's +# CustomRpc "peer data transfer" channel, so ESPHome's `espnow` component links +# and runs unchanged (proven on a Tab5, 2026-07-20). The .cpp is guarded to +# CONFIG_IDF_TARGET_ESP32P4 so it compiles to nothing on hosts with a native +# ESP-NOW stack. CustomRpc needs these two host-side Kconfig options. Host +# registers 3 handlers (RESP, RECV, SEND); the coprocessor registers 1 (REQ); +# we ask for 8 to leave room for other CustomRpc extensions alongside. +# +# The coprocessor must run the matching custom firmware (a parallel effort in +# esphome/esp-hosted-firmware). esp_now_hosted_rpc.h here is the canonical copy +# of the wire contract and MUST stay byte-identical to the copy that coprocessor +# firmware uses — the packed structs are the on-wire layout, so any divergence +# silently corrupts every ESP-NOW frame. +_MAX_CUSTOM_MSG_HANDLERS = 8 + # Shared fields for both transport modes BASE_SCHEMA = cv.Schema( { @@ -262,6 +281,23 @@ async def to_code(config: ConfigType) -> None: else: _configure_spi(config) + # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see + # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not + # gated on the `espnow` component being present: the shim is tiny and the + # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options + # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so + # coupling the two keeps the build consistent. When `espnow` is absent the + # symbols are simply unused and never register a callback at runtime. + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + add_define("USE_ESP_NOW_HOSTED") + # esp-hosted's CustomRpc ("peer data transfer") path — off by default. + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS + ) + # Place the transport mempool in PSRAM. Required on memory-tight host # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM # mempool allocation fails at boot with `sdio_mempool_create` assert. diff --git a/esphome/components/esp32_hosted/esp_now_hosted.cpp b/esphome/components/esp32_hosted/esp_now_hosted.cpp new file mode 100644 index 0000000000..ad29b208fe --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted.cpp @@ -0,0 +1,467 @@ +/* + * esp_now_hosted — host-side shim implementing over esp-hosted + * CustomRpc, so ESPHome's `espnow` component can run on a radio-less host + * (e.g. the ESP32-P4) whose radio lives on an esp-hosted co-processor. + * + * A radio-less host has no native ESP-NOW. esp_wifi_remote INJECTS the full + * esp_now.h header (types + declarations) but ships NO implementation, so every + * esp_now_* symbol is an undefined reference at link time. This translation + * unit provides those definitions; each forwards to the co-processor over + * CustomRpc (see esphome/esp-hosted-firmware for the matching coprocessor + * handlers). No esp-hosted or esp_wifi_remote source is patched, and there is no + * duplicate-symbol clash because nothing else defines these symbols here. + * + * See esp_now_hosted_rpc.h for the wire protocol. + */ + +#include "sdkconfig.h" + +// Only build the shim on the radio-less host. On chips with a native ESP-NOW +// stack (S3, C6, …) the real symbols exist and this file must stay empty to +// avoid duplicate definitions. +#if defined(CONFIG_IDF_TARGET_ESP32P4) + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" + +#include "esp_idf_version.h" +#include "esp_log.h" +#include "esp_timer.h" + +#include // injected declarations we are now DEFINING +#include // wifi_pkt_rx_ctrl_t, wifi_tx_info_t + +// esp_hosted_misc.h (host) ships WITHOUT an extern "C" guard, so including it +// from C++ would give its declarations C++ linkage and the real C symbols in +// libesp_hosted would go unresolved at link. Wrap it. (Verified vs +// esp_hosted 2.12.9.) +extern "C" { +#include "esp_hosted_misc.h" // esp_hosted_{send_custom_data,register_custom_callback} +} + +#include "esp_now_hosted_rpc.h" + +namespace { + +const char *const TAG = "esp_now_hosted"; + +// One outstanding request at a time. ESPHome drives esp_now_* from the main +// loop; the matching response and the async RECV/SEND events all arrive on the +// single esp-hosted RPC RX thread. Serializing requests keeps the shared +// response slot race-free; a sequence number stops a late/stale response from +// being mistaken for ours. +SemaphoreHandle_t g_req_mutex = nullptr; +SemaphoreHandle_t g_resp_sem = nullptr; // given when the matching RESP lands +bool g_setup_done = false; // set only after setup fully succeeds +uint8_t g_seq = 0; +volatile uint8_t g_expect_seq = 0; +volatile int32_t g_resp_status = 0; +uint8_t g_resp_ret[16]; +volatile uint16_t g_resp_ret_len = 0; + +// Written from the main loop (register/unregister/deinit), read from the +// esp-hosted RX thread (on_recv/on_send). volatile for the same reason the +// g_resp_* globals are: force the RX thread to observe an updated pointer +// (e.g. a nulling by esp_now_deinit) rather than a cached one. +volatile esp_now_recv_cb_t g_recv_cb = nullptr; +volatile esp_now_send_cb_t g_send_cb = nullptr; + +// Local mirror of the co-processor's peer table. ESPHome's espnow component +// calls esp_now_is_peer_exist() on the main loop for every received frame +// (twice) and every send; forwarding each as a blocking RPC round-trip stalls +// the loop. The shim is the only path that mutates the co-processor peer table +// (add/del/deinit all go through here), so this mirror is authoritative and +// esp_now_is_peer_exist() can answer from it with no round-trip. +// +// esp_now_* are public C symbols: any component or user lambda may call them, +// and although ESPHome's espnow touches peers only from the main loop today +// (its RX/TX callbacks merely enqueue), the shim cannot rely on that. A short +// spinlock keeps the mirror consistent from any task/core, matching native +// esp_now_*'s own internal thread-safety. The critical sections are a bounded +// (<=20-entry) scan, so they stay tiny. ESP_NOW_MAX_TOTAL_PEER_NUM is 20. +constexpr size_t ESP_NOW_HOSTED_MAX_PEERS = 20; +uint8_t g_peer_cache[ESP_NOW_HOSTED_MAX_PEERS][6]; +size_t g_peer_count = 0; +portMUX_TYPE g_peer_lock = portMUX_INITIALIZER_UNLOCKED; + +// Caller must hold g_peer_lock. +int peer_cache_find_locked(const uint8_t *mac) { + for (size_t i = 0; i < g_peer_count; i++) { + if (memcmp(g_peer_cache[i], mac, 6) == 0) + return static_cast(i); + } + return -1; +} + +bool peer_cache_contains(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const bool found = peer_cache_find_locked(mac) >= 0; + portEXIT_CRITICAL(&g_peer_lock); + return found; +} + +void peer_cache_add(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + if (peer_cache_find_locked(mac) < 0 && g_peer_count < ESP_NOW_HOSTED_MAX_PEERS) + memcpy(g_peer_cache[g_peer_count++], mac, 6); + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_remove(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const int idx = peer_cache_find_locked(mac); + if (idx >= 0) { + g_peer_count--; + if (static_cast(idx) != g_peer_count) // move the last entry into the gap + memcpy(g_peer_cache[idx], g_peer_cache[g_peer_count], 6); + } + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_clear() { + portENTER_CRITICAL(&g_peer_lock); + g_peer_count = 0; + portEXIT_CRITICAL(&g_peer_lock); +} + +// ── CustomRpc event handlers (run on the esp-hosted RPC RX thread) ────────── +// Keep them short and non-blocking. In particular they MUST NOT call back into +// any esp_now_* shim function: that would try to take g_req_mutex / wait on the +// RX thread that delivers the response, and deadlock. + +void on_resp(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + if (len < sizeof(esp_now_hosted_resp_t)) { + ESP_LOGW(TAG, "RESP too short: %u bytes", static_cast(len)); + return; + } + const auto *r = reinterpret_cast(data); + if (r->seq != g_expect_seq) { // late response from a timed-out request (expected) + ESP_LOGV(TAG, "dropping stale RESP seq %u (want %u)", r->seq, g_expect_seq); + return; + } + g_resp_status = r->status; + uint16_t rl = r->ret_len; + if (rl > sizeof(g_resp_ret)) { + // Larger than any real opcode return — a likely wire-format drift signal. + ESP_LOGW(TAG, "RESP ret_len %u exceeds buffer, clamping (wire drift?)", rl); + rl = sizeof(g_resp_ret); + } + if (len >= sizeof(esp_now_hosted_resp_t) + rl) { + memcpy(g_resp_ret, r->ret, rl); + } else { + // Truncated frame: fail closed. Never hand the caller stale bytes left in + // g_resp_ret by a previous response, and don't let request() report a + // zeroed payload as success — override the status to an error. + ESP_LOGW(TAG, "RESP truncated: claims %u ret bytes, frame too short", rl); + rl = 0; + g_resp_status = ESP_ERR_INVALID_RESPONSE; + } + g_resp_ret_len = rl; + xSemaphoreGive(g_resp_sem); +} + +void on_recv(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once: esp_now_unregister_recv_cb()/deinit() (via + // the espnow component's disable()) can null it on the main loop between the + // guard and the call, which would otherwise turn the call into a null-deref. + const esp_now_recv_cb_t cb = g_recv_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_recv_evt_t)) { + ESP_LOGW(TAG, "RECV too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); + if (len < sizeof(esp_now_hosted_recv_evt_t) + e->data_len) { + ESP_LOGW(TAG, "RECV data_len %u exceeds frame", e->data_len); + return; + } + + // ESPHome dereferences info->rx_ctrl->{rssi,timestamp}; give it a real one. + wifi_pkt_rx_ctrl_t rx_ctrl; + memset(&rx_ctrl, 0, sizeof(rx_ctrl)); + rx_ctrl.rssi = e->rssi; + rx_ctrl.channel = e->channel; + rx_ctrl.timestamp = static_cast(esp_timer_get_time()); + + esp_now_recv_info_t info; + info.src_addr = const_cast(e->src_addr); + info.des_addr = const_cast(e->des_addr); + info.rx_ctrl = &rx_ctrl; + cb(&info, e->data, static_cast(e->data_len)); +} + +void on_send(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once (see on_recv): disable()/deinit() can null it + // on the main loop concurrently with this RX-thread callback. + const esp_now_send_cb_t cb = g_send_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_send_evt_t)) { + ESP_LOGW(TAG, "SEND evt too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0) + // IDF >= 5.5: esp_now_send_cb_t takes esp_now_send_info_t (== wifi_tx_info_t), + // whose des_addr is a POINTER (not an inline array). Point it at the event's + // MAC (valid for this callback) — do NOT memcpy into it (that writes NULL and + // faults). ESPHome reads only info->des_addr. + esp_now_send_info_t si; + memset(&si, 0, sizeof(si)); + si.des_addr = const_cast(e->des_addr); + cb(&si, static_cast(e->status)); +#else + cb(e->des_addr, static_cast(e->status)); +#endif +} + +esp_err_t ensure_setup() { + // Gate on g_setup_done, not on g_req_mutex: a failure part-way through (a + // semaphore that did not allocate, a callback that did not register) must not + // leave a later call thinking setup completed. Semaphore creation is guarded + // so a retry after a partial failure does not leak the earlier handles. + if (g_setup_done) + return ESP_OK; + if (g_req_mutex == nullptr) + g_req_mutex = xSemaphoreCreateMutex(); + if (g_resp_sem == nullptr) + g_resp_sem = xSemaphoreCreateBinary(); + if (g_req_mutex == nullptr || g_resp_sem == nullptr) + return ESP_ERR_NO_MEM; + esp_err_t err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RESP, on_resp, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RECV, on_recv, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_SEND, on_send, nullptr)) != ESP_OK) + return err; + g_setup_done = true; + return ESP_OK; +} + +// Send one request envelope. With wait=true (default) block until the matching +// response (or timeout); with wait=false return as soon as the frame is handed +// to the transport (fire-and-forget, used by esp_now_send). +// +// `tail` is an optional second chunk written straight after `payload`. Callers +// with a fixed header plus a bulk body (esp_now_send) pass the two separately +// so they never need a build buffer of their own: both chunks are laid into the +// request buffer here, under g_req_mutex, which keeps concurrent callers from +// racing and saves a full copy of the body on every transmit. +esp_err_t request(uint8_t opcode, const void *payload, uint16_t plen, void *ret, uint16_t ret_cap, uint16_t *ret_len, + bool wait = true, const void *tail = nullptr, uint16_t tail_len = 0) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + if (plen > ESP_NOW_HOSTED_MAX_PAYLOAD || tail_len > ESP_NOW_HOSTED_MAX_PAYLOAD - plen) + return ESP_ERR_INVALID_SIZE; + const uint16_t total_len = static_cast(plen + tail_len); + + if (xSemaphoreTake(g_req_mutex, portMAX_DELAY) != pdTRUE) + return ESP_FAIL; + + static uint8_t buf[sizeof(esp_now_hosted_req_t) + ESP_NOW_HOSTED_MAX_PAYLOAD]; // guarded by g_req_mutex + auto *req = reinterpret_cast(buf); + req->opcode = opcode; + req->seq = ++g_seq; + req->payload_len = total_len; + if (plen != 0) + memcpy(req->payload, payload, plen); + if (tail_len != 0) + memcpy(req->payload + plen, tail, tail_len); + g_expect_seq = req->seq; + + xSemaphoreTake(g_resp_sem, 0); // drain any stale signal before sending + err = esp_hosted_send_custom_data(ESP_NOW_HOSTED_MSG_REQ, buf, sizeof(esp_now_hosted_req_t) + total_len); + if (err != ESP_OK) { + xSemaphoreGive(g_req_mutex); + return err; + } + if (!wait) { + // Fire-and-forget (esp_now_send): the co-processor enqueues the frame and + // reports the real TX result later via the async SEND event, exactly like + // native esp_now_send. Returning here keeps the main loop off the ~100 ms+ + // RPC round-trip. The matching RESP is ignored (seq won't match the next + // waited request, so on_resp drops it). + xSemaphoreGive(g_req_mutex); + return ESP_OK; + } + if (xSemaphoreTake(g_resp_sem, pdMS_TO_TICKS(ESP_NOW_HOSTED_TIMEOUT_MS)) != pdTRUE) { + ESP_LOGW(TAG, "opcode %u timed out", opcode); + xSemaphoreGive(g_req_mutex); + return ESP_ERR_TIMEOUT; + } + + const int32_t status = g_resp_status; + if (ret != nullptr && ret_cap != 0) { + uint16_t n = g_resp_ret_len < ret_cap ? g_resp_ret_len : ret_cap; + memcpy(ret, const_cast(g_resp_ret), n); + if (ret_len != nullptr) + *ret_len = n; + } + xSemaphoreGive(g_req_mutex); + return static_cast(status); +} + +} // namespace + +// ── The surface, defined for the radio-less host ──────────────── +extern "C" { + +esp_err_t esp_now_init(void) { return request(ESP_NOW_HOSTED_OP_INIT, nullptr, 0, nullptr, 0, nullptr); } + +esp_err_t esp_now_deinit(void) { + g_recv_cb = nullptr; + g_send_cb = nullptr; + peer_cache_clear(); // the co-processor drops all peers on deinit + return request(ESP_NOW_HOSTED_OP_DEINIT, nullptr, 0, nullptr, 0, nullptr); +} + +esp_err_t esp_now_get_version(uint32_t *version) { + uint32_t v = 0; + uint16_t rl = 0; + esp_err_t err = request(ESP_NOW_HOSTED_OP_GET_VERSION, nullptr, 0, &v, sizeof(v), &rl); + if (version != nullptr) + *version = v; + return err; +} + +esp_err_t esp_now_register_recv_cb(esp_now_recv_cb_t cb) { + // Only arm the callback once the CustomRpc handlers are actually registered, + // so a failed setup leaves g_recv_cb null rather than falsely "registered". + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_recv_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_recv_cb(void) { + g_recv_cb = nullptr; + return ESP_OK; +} +esp_err_t esp_now_register_send_cb(esp_now_send_cb_t cb) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_send_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_send_cb(void) { + g_send_cb = nullptr; + return ESP_OK; +} + +static esp_err_t add_or_mod_peer(uint8_t opcode, const esp_now_peer_info_t *peer, bool wait) { + if (peer == nullptr) + return ESP_ERR_ESPNOW_ARG; + esp_now_hosted_peer_t p; + memset(&p, 0, sizeof(p)); + memcpy(p.peer_addr, peer->peer_addr, 6); + memcpy(p.lmk, peer->lmk, 16); + p.channel = peer->channel; + p.ifidx = static_cast(peer->ifidx); + p.encrypt = peer->encrypt ? 1 : 0; + return request(opcode, &p, sizeof(p), nullptr, 0, nullptr, wait); +} +esp_err_t esp_now_add_peer(const esp_now_peer_info_t *peer) { + // Fire-and-forget (wait=false): adding a peer is a blocking RPC round-trip, + // and ESPHome's espnow calls it on the main loop when a device joins the mesh + // — under co-processor load that stalls the UI (peer-churn stutter). Issue it + // without waiting and mirror it locally. Safe against a following + // esp_now_send to the same peer: both ride the same in-order CustomRpc + // channel (mutex-serialized on the host) and the co-processor processes REQs + // FIFO, so ADD_PEER is applied before the SEND. Trade-off: a co-processor-side + // failure (e.g. peer table full) is no longer reported synchronously — the + // same limitation as esp_now_send — but ESPHome only adds peers it validated. + esp_err_t err = add_or_mod_peer(ESP_NOW_HOSTED_OP_ADD_PEER, peer, /*wait=*/false); + if (err == ESP_OK) + peer_cache_add(peer->peer_addr); // keep the local mirror in sync + return err; +} +esp_err_t esp_now_mod_peer(const esp_now_peer_info_t *peer) { + // mod_peer changes a peer's parameters, not its existence, so the cache is + // unaffected. Kept synchronous — it is not on any hot path (espnow never + // calls it), so the extra round-trip does not matter and the status is useful. + return add_or_mod_peer(ESP_NOW_HOSTED_OP_MOD_PEER, peer, /*wait=*/true); +} + +esp_err_t esp_now_del_peer(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return ESP_ERR_ESPNOW_ARG; + // Fire-and-forget for the same reason as add_peer (peer churn on the main + // loop). Removal is order-independent, so this is strictly safe. + esp_err_t err = request(ESP_NOW_HOSTED_OP_DEL_PEER, peer_addr, 6, nullptr, 0, nullptr, /*wait=*/false); + if (err == ESP_OK) + peer_cache_remove(peer_addr); // keep the local mirror in sync + return err; +} + +bool esp_now_is_peer_exist(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return false; + // Answered from the local mirror — no RPC round-trip. ESPHome's espnow calls + // this on the main loop for every received frame and every send, so a + // blocking round-trip here would stall rendering under mesh traffic. + return peer_cache_contains(peer_addr); +} + +esp_err_t esp_now_send(const uint8_t *peer_addr, const uint8_t *data, size_t len) { + if (len > ESP_NOW_HOSTED_MAX_FRAME) + return ESP_ERR_ESPNOW_ARG; + if (data == nullptr && len != 0) // native esp_now_send treats this as an arg error + return ESP_ERR_ESPNOW_ARG; + // Only the small fixed header is built here; the caller's frame goes over as + // the request tail, so request() lays both into its own buffer under + // g_req_mutex. esp_now_send is a public C symbol and may be called from any + // task, and a shared build buffer here would let two callers corrupt each + // other's frame. Passing the body through also drops a full-frame copy per + // transmit, on the path this shim exists to keep quick. + uint8_t hdr[sizeof(esp_now_hosted_send_req_t)]; + auto *s = reinterpret_cast(hdr); + s->has_addr = peer_addr != nullptr ? 1 : 0; + if (peer_addr != nullptr) + memcpy(s->peer_addr, peer_addr, 6); + else + memset(s->peer_addr, 0, 6); + s->data_len = static_cast(len); + // Fire-and-forget (wait=false): native esp_now_send returns once the frame is + // queued, with the real TX result delivered later through the send callback. + // The co-processor mirrors that — it acks enqueue immediately and reports the + // outcome via the async SEND event (on_send -> on_send_report). Waiting for + // the RPC RESP here would block the main loop for the full round-trip on + // every transmit. + return request(ESP_NOW_HOSTED_OP_SEND, hdr, sizeof(hdr), nullptr, 0, nullptr, /*wait=*/false, data, + static_cast(len)); +} + +esp_err_t esp_now_set_pmk(const uint8_t *pmk) { + if (pmk == nullptr) + return ESP_ERR_ESPNOW_ARG; + return request(ESP_NOW_HOSTED_OP_SET_PMK, pmk, 16, nullptr, 0, nullptr); +} + +// Remainder of the surface. Not used by ESPHome's espnow component +// today; provided so the whole header links and future callers get a defined +// (if unimplemented) symbol rather than a link error. Wire them through +// CustomRpc if a use case appears. +esp_err_t esp_now_get_peer(const uint8_t * /*peer_addr*/, esp_now_peer_info_t * /*peer*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_now_fetch_peer(bool /*from_head*/, esp_now_peer_info_t * /*peer*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_get_peer_num(esp_now_peer_num_t * /*num*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_set_wake_window(uint16_t /*window*/) { + return ESP_ERR_NOT_SUPPORTED; // power-save wake window is not forwarded; don't claim success +} +esp_err_t esp_now_set_peer_rate_config(const uint8_t * /*peer_addr*/, esp_now_rate_config_t * /*cfg*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_wifi_config_espnow_rate(wifi_interface_t /*ifx*/, wifi_phy_rate_t /*rate*/) { + return ESP_ERR_NOT_SUPPORTED; +} + +} // extern "C" + +#endif // CONFIG_IDF_TARGET_ESP32P4 diff --git a/esphome/components/esp32_hosted/esp_now_hosted_rpc.h b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h new file mode 100644 index 0000000000..bf68c759ee --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h @@ -0,0 +1,128 @@ +/* + * esp_now_hosted — ESP-NOW-over-CustomRpc wire protocol. + * + * Shared, byte-for-byte-identical contract between: + * - the host shim (esphome/components/esp32_hosted/esp_now_hosted.cpp) + * - the coprocessor firmware (esphome/esp-hosted-firmware) + * + * It rides esp-hosted's CustomRpc channel (RPC ID 388, "peer data transfer", + * available since esp-hosted v2.8.1), teaching the radio-less host <-> radio + * co-processor link to carry esp_now.h, which esp-hosted itself does not proxy + * (Espressif issue espressif/esp-hosted-mcu#19). + * + * KEEP THE TWO COPIES IN SYNC. The canonical copy lives here; the coprocessor + * firmware uses a verbatim copy. Both sides are little-endian, so these packed + * structs are wire-compatible with no byte-swapping. + */ + +#ifndef ESP_NOW_HOSTED_RPC_H +#define ESP_NOW_HOSTED_RPC_H + +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* ── CustomRpc message IDs (any uint32_t except 0xFFFFFFFF) ────────────────── + * One REQ handler slot on the device; three event handler slots on the host. + * The bytes spell "now" + index, a private range unlikely to clash with other + * CustomRpc users (e.g. the stock peer_data_transfer example's 1..6). */ +#define ESP_NOW_HOSTED_MSG_REQ 0x6E6F7701u /* host -> device : request envelope */ +#define ESP_NOW_HOSTED_MSG_RESP 0x6E6F7702u /* device -> host : reply to a REQ */ +#define ESP_NOW_HOSTED_MSG_RECV 0x6E6F7703u /* device -> host : async RX frame */ +#define ESP_NOW_HOSTED_MSG_SEND 0x6E6F7704u /* device -> host : async TX status */ + +/* ── Request opcodes ────────────────────────────────────────────────────── */ +enum { + ESP_NOW_HOSTED_OP_INIT = 1, /* esp_now_init + register device recv/send cbs */ + ESP_NOW_HOSTED_OP_DEINIT = 2, /* unregister cbs + esp_now_deinit */ + ESP_NOW_HOSTED_OP_ADD_PEER = 3, /* payload: esp_now_hosted_peer_t */ + ESP_NOW_HOSTED_OP_DEL_PEER = 4, /* payload: 6-byte peer MAC */ + ESP_NOW_HOSTED_OP_IS_PEER_EXIST = 5, /* payload: 6-byte MAC; ret: 1 byte bool */ + ESP_NOW_HOSTED_OP_SEND = 6, /* payload: esp_now_hosted_send_req_t */ + ESP_NOW_HOSTED_OP_GET_VERSION = 7, /* ret: uint32 version */ + ESP_NOW_HOSTED_OP_SET_PMK = 8, /* payload: 16-byte PMK */ + ESP_NOW_HOSTED_OP_MOD_PEER = 9, /* payload: esp_now_hosted_peer_t */ +}; + +/* Largest ESP-NOW payload we forward. ESP-NOW v2 (IDF >= 5.4) is 1470 B; well + * under esp-hosted's 8166 B CustomRpc cap, so the shim never truncates. */ +#define ESP_NOW_HOSTED_MAX_FRAME 1470u +/* Envelope slack for the largest opcode payload (a SEND req wrapping a frame). */ +#define ESP_NOW_HOSTED_MAX_PAYLOAD (ESP_NOW_HOSTED_MAX_FRAME + 16u) +/* Host request/response round-trip timeout over the transport. Generous: + * normal RTT is sub-millisecond, but Wi-Fi/BLE contention on the co-processor + * can stall the RX thread. */ +#define ESP_NOW_HOSTED_TIMEOUT_MS 2000 + +/* ── Envelopes ──────────────────────────────────────────────────────────── */ + +/* These payloads are shared verbatim with the C co-processor firmware, so they + * use C's `typedef struct {...} name;` idiom rather than C++ `using` aliases, + * which would not compile there. Silence clang-tidy's modernize-use-using for + * the shared struct block. */ +// NOLINTBEGIN(modernize-use-using) +typedef struct { + uint8_t opcode; /* one of ESP_NOW_HOSTED_OP_* */ + uint8_t seq; /* wraps 0..255; echoed in the response for matching */ + uint16_t payload_len; /* bytes of opcode-specific payload that follow */ + uint8_t payload[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_req_t; + +typedef struct { + uint8_t opcode; /* echoes the request opcode */ + uint8_t seq; /* echoes the request seq */ + int32_t status; /* esp_err_t from the native call on the co-processor */ + uint16_t ret_len; /* bytes of return payload that follow */ + uint8_t ret[]; /* flexible (e.g. version u32, is_peer_exist bool) */ +} __attribute__((packed)) esp_now_hosted_resp_t; + +/* ── Opcode payloads ────────────────────────────────────────────────────── */ + +/* esp_now_peer_info_t minus the host-only `priv` pointer, which is meaningless + * across the transport and never set by ESPHome's espnow component. */ +typedef struct { + uint8_t peer_addr[6]; + uint8_t lmk[16]; + uint8_t channel; /* 0 = current channel */ + uint8_t ifidx; /* wifi_interface_t (0=STA, 1=AP) */ + uint8_t encrypt; /* bool */ +} __attribute__((packed)) esp_now_hosted_peer_t; + +typedef struct { + uint8_t has_addr; /* 0 => peer_addr is NULL (broadcast to all peers) */ + uint8_t peer_addr[6]; + uint16_t data_len; + uint8_t data[]; /* flexible, up to ESP_NOW_HOSTED_MAX_FRAME */ +} __attribute__((packed)) esp_now_hosted_send_req_t; + +/* ── Async events (device -> host) ──────────────────────────────────────── */ + +/* Reconstructed on the host into an esp_now_recv_info_t + a minimal + * wifi_pkt_rx_ctrl_t. ESPHome's espnow reads info->src_addr, info->des_addr, + * info->rx_ctrl->rssi and info->rx_ctrl->timestamp. */ +typedef struct { + uint8_t src_addr[6]; + uint8_t des_addr[6]; + int8_t rssi; + uint8_t channel; + uint16_t data_len; + uint8_t data[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_recv_evt_t; + +typedef struct { + uint8_t des_addr[6]; + uint8_t status; /* esp_now_send_status_t (0 = success) */ +} __attribute__((packed)) esp_now_hosted_send_evt_t; +// NOLINTEND(modernize-use-using) + +#ifdef __cplusplus +} +#endif + +#endif /* ESP_NOW_HOSTED_RPC_H */ diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 5541a6ee97..14d099ec06 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -3,6 +3,7 @@ from typing import Any from esphome import automation, core import esphome.codegen as cg from esphome.components import wifi +from esphome.components.esp32 import VARIANT_ESP32P4, get_esp32_variant from esphome.components.udp import CONF_ON_RECEIVE import esphome.config_validation as cv from esphome.const import ( @@ -17,6 +18,7 @@ from esphome.const import ( ) from esphome.core import CORE, HexInt from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] @@ -132,6 +134,24 @@ CONFIG_SCHEMA = cv.All( ) +def _validate_variant(config: ConfigType) -> ConfigType: + # ESP-NOW rides the Wi-Fi PHY. Radio-less esp32 variants have no native + # ESP-NOW; only the ESP32-P4 has a path, via the esp32_hosted shim that + # supplies the esp_now_* symbols. Fail here with a clear message instead of + # letting the build reach an "undefined reference to esp_now_*" link error. + variant = get_esp32_variant() + if wifi.variant_has_wifi(variant): + return config + if variant != VARIANT_ESP32P4: + raise cv.Invalid(f"ESP-NOW is not supported on {variant} (no Wi-Fi radio)") + if "esp32_hosted" not in fv.full_config.get(): + raise cv.Invalid(f"ESP-NOW on {variant} requires the esp32_hosted component") + return config + + +FINAL_VALIDATE_SCHEMA = _validate_variant + + async def _trigger_to_code(config: ConfigType) -> MockObj: if address := config.get(CONF_ADDRESS): address = address.parts diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9dd1e0ced6..eaece6d5ff 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_ESP32_HOSTED #define USE_ESP32_HOSTED_HTTP_UPDATE #define USE_ESP32_IMPROV_STATE_CALLBACK +#define USE_ESP_NOW_HOSTED #define USE_EVENT #define USE_FAN #define USE_GPIO_BINARY_SENSOR_INTERRUPT diff --git a/script/ci-custom.py b/script/ci-custom.py index f481fda860..e2b7cd8d37 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -294,6 +294,9 @@ def highlight(s): "esphome/components/socket/headers.h", "esphome/core/defines.h", "esphome/components/http_request/httplib.h", + # Shared C wire header (byte-identical with the co-processor firmware); + # these are protocol constants and constexpr is C++-only. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_no_defines(fname, match): @@ -816,6 +819,10 @@ def lint_relative_py_import(fname: Path, line, col, content): "esphome/components/host/helpers.cpp", "esphome/components/zephyr/helpers.cpp", "esphome/components/http_request/httplib.h", + # Global extern "C" esp_now_* linker symbols + shared C wire header; + # neither can live in a C++ namespace. + "esphome/components/esp32_hosted/esp_now_hosted.cpp", + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: @@ -841,7 +848,15 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) +@lint_content_check( + include=["*.h"], + exclude=[ + "esphome/core/entity_types.h", + # Shared C wire header; uses a classic #ifndef guard for portability + # across the co-processor firmware repo it stays byte-identical with. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + ], +) def lint_pragma_once(fname, content): if "#pragma once" not in content: return ( diff --git a/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml new file mode 100644 index 0000000000..fab0a64ab8 --- /dev/null +++ b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml @@ -0,0 +1,5 @@ +# Exercises the ESP-NOW-over-hosted shim: on the ESP32-P4 host, esp32_hosted +# supplies the esp_now_* symbols that the espnow component links against. +packages: + esp32_hosted: !include common.yaml + espnow: !include ../espnow/common.yaml diff --git a/tests/unit_tests/components/test_espnow.py b/tests/unit_tests/components/test_espnow.py new file mode 100644 index 0000000000..21305c2b33 --- /dev/null +++ b/tests/unit_tests/components/test_espnow.py @@ -0,0 +1,48 @@ +"""Tests for the espnow component's final validation.""" + +import pytest + +from esphome.components.esp32.const import ( + VARIANT_ESP32C3, + VARIANT_ESP32H2, + VARIANT_ESP32P4, +) +from esphome.components.espnow import _validate_variant +import esphome.config_validation as cv +import esphome.final_validate as fv +from esphome.types import ConfigType + + +def _run( + monkeypatch, variant: str, full_config: dict, config: ConfigType +) -> ConfigType: + monkeypatch.setattr("esphome.components.espnow.get_esp32_variant", lambda: variant) + token = fv.full_config.set(full_config) + try: + return _validate_variant(config) + finally: + fv.full_config.reset(token) + + +def test_variant_with_native_wifi_passes(monkeypatch) -> None: + """A variant with a native Wi-Fi PHY needs no shim; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32C3, {}, config) is config + + +def test_radioless_non_p4_variant_rejected(monkeypatch) -> None: + """Radio-less variants without any ESP-NOW path are rejected outright.""" + with pytest.raises(cv.Invalid, match="not supported"): + _run(monkeypatch, VARIANT_ESP32H2, {}, {}) + + +def test_p4_without_esp32_hosted_rejected(monkeypatch) -> None: + """The P4 needs the esp32_hosted shim to supply the esp_now_* symbols.""" + with pytest.raises(cv.Invalid, match="esp32_hosted"): + _run(monkeypatch, VARIANT_ESP32P4, {}, {}) + + +def test_p4_with_esp32_hosted_passes(monkeypatch) -> None: + """The P4 with esp32_hosted present validates; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32P4, {"esp32_hosted": {}}, config) is config From 9ba4477ada0f207b7426fe83b37fde69c9ed9947 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:46:22 +1200 Subject: [PATCH 33/53] Bump version to 2026.9.0b2 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 7b2d21027a..060de51d3a 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0b1 +PROJECT_NUMBER = 2026.9.0b2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 378da14197..287804ace3 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0b1" +__version__ = "2026.9.0b2" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 5e37872da24f4626dc7ea8bdd61f4c5534f6c0ee Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:32:18 +1200 Subject: [PATCH 34/53] [ci] Sync pre-commit revs and prek version from requirements files (#19026) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 13 +- .../workflows/sync-dependency-versions.yml | 94 ++++++++ .pre-commit-config.yaml | 5 +- AGENTS.md | 2 +- requirements_dev.txt | 4 +- requirements_test.txt | 9 +- script/sync_dependency_versions.py | 164 +++++++++++++ tests/script/test_sync_dependency_versions.py | 219 ++++++++++++++++++ 8 files changed, 498 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/sync-dependency-versions.yml create mode 100755 script/sync_dependency_versions.py create mode 100644 tests/script/test_sync_dependency_versions.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7c93b3b86..173d2c227a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,11 +244,20 @@ jobs: steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Read prek version from requirements_test.txt + id: prek + # requirements_test.txt is the only place the version is pinned, so a + # Dependabot bump there is picked up here without a second edit. + run: | + if ! version=$(sed -nE 's/^prek==([^[:space:]#]+).*/\1/p' requirements_test.txt) || [ -z "$version" ]; then + echo "::error::No prek== pin found in requirements_test.txt." + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" - name: Run prek uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 with: - # Keep in sync with requirements_test.txt. - prek-version: "0.4.11" + prek-version: ${{ steps.prek.outputs.version }} # This job only runs on pull requests, so nothing ever populates # the cache on dev. Every run would miss and then write a per-pull # request copy, which is what the old seed-cache job existed to diff --git a/.github/workflows/sync-dependency-versions.yml b/.github/workflows/sync-dependency-versions.yml new file mode 100644 index 0000000000..5599691ed1 --- /dev/null +++ b/.github/workflows/sync-dependency-versions.yml @@ -0,0 +1,94 @@ +# Keeps pre-commit hook revs in sync with the requirements files. +# +# Dependabot only bumps the pins in requirements*.txt. Some of those tools +# are pinned again as hook revs in .pre-commit-config.yaml. This workflow +# runs script/sync_dependency_versions.py against the pull request branch +# and pushes a commit with the revs updated. + +name: Sync dependency versions + +on: + # pull_request_target rather than pull_request so the App secret is + # available on Dependabot pull requests (pull_request runs opened by + # Dependabot only see Dependabot secrets). The job below only touches + # branches in this repository and only ever executes the script from the + # base branch checkout, so fork code never runs with the token. + pull_request_target: + types: [opened, synchronize, reopened] + paths: + - requirements_dev.txt + - requirements_test.txt + - .pre-commit-config.yaml + - script/sync_dependency_versions.py + +# The push to the pull request branch uses the App token minted below, so +# the workflow's GITHUB_TOKEN does not need any scopes. +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + sync: + name: Sync pinned versions + runs-on: ubuntu-latest + # Same-repository branches only: a push to a fork is not possible with + # this token, and it keeps untrusted heads out of a privileged job. + if: >- + github.repository == 'esphome/esphome' + && github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} + # A push made with the workflow's own GITHUB_TOKEN would not start + # CI on the new commit; a push with the App token does. + permission-contents: write # git push of the sync commit to the pull request branch + + - name: Check out base branch + # Provides the script that runs below. Deliberately the base branch + # so the pull request cannot change what executes here. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + + - name: Check out pull request branch + # No allow-unsafe-pr-checkout here on purpose: checkout v7 only + # refuses heads that live in a different repository, and the job + # condition above already limits runs to same-repository branches. + # Leaving it off keeps that refusal as a backstop for fork heads. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.ref }} + path: pull-request + token: ${{ steps.generate-token.outputs.token }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install yamlrocks + # The script edits YAML through yamlrocks. Take the pin from the + # base branch requirements so this workflow has no copy of its own. + run: pip install "$(grep -E '^yamlrocks==' requirements_test.txt | cut -d'#' -f1)" + + - name: Sync pinned versions + run: python script/sync_dependency_versions.py --root pull-request + + - name: Push changes + working-directory: pull-request + run: | + if git diff --quiet; then + echo "All pinned versions already match the requirements files." + exit 0 + fi + git config user.name "esphome[bot]" + git config user.email "115708604+esphome[bot]@users.noreply.github.com" + git commit -am "Sync pinned tool versions with requirements files" + git push diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ea799aa4d..1af0e19273 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ --- # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks - ci: autoupdate_commit_msg: 'pre-commit: autoupdate' autoupdate_schedule: off # Disabled until ruff versions are synced between deps and pre-commit @@ -11,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.3 + rev: v0.16.5 hooks: # Run the linter. - id: ruff @@ -42,7 +41,7 @@ repos: - id: pyupgrade args: [--py312-plus] - repo: https://github.com/adrienverge/yamllint.git - rev: v1.37.1 + rev: v1.38.0 hooks: - id: yamllint exclude: ^(\.clang-format|\.clang-tidy)$ diff --git a/AGENTS.md b/AGENTS.md index 15b92c4deb..98bdd58ec5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -840,7 +840,7 @@ file does, and it is the authority when they disagree. The most useful starting cv.rename_key( CONF_OLD_KEY, CONF_NEW_KEY, removed_in="2026.6.0", component="my_component" ), - cv.Schema({ ... }), + cv.Schema({...}), ) ``` For other deprecations, warn manually during validation: diff --git a/requirements_dev.txt b/requirements_dev.txt index f2cf855d6b..ee94a2401a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ # Useful stuff when working in a development environment -clang-format==13.0.1 # also change in .pre-commit-config.yaml and Dockerfile when updating +clang-format==13.0.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py clang-tidy==22.1.8 -yamllint==1.38.0 # also change in .pre-commit-config.yaml when updating +yamllint==1.38.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py diff --git a/requirements_test.txt b/requirements_test.txt index 897445a4cb..ef70a5ac0c 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,8 +1,9 @@ pylint==4.0.8 -flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.16.5 # also change in .pre-commit-config.yaml when updating -pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.5.1 # also change in .github/workflows/ci.yml when updating +flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.5 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +prek==0.5.1 # .github/workflows/ci.yml reads this pin +yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests pytest==9.1.1 diff --git a/script/sync_dependency_versions.py b/script/sync_dependency_versions.py new file mode 100755 index 0000000000..a97a58b3b0 --- /dev/null +++ b/script/sync_dependency_versions.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Keep pre-commit hook revs in sync with the requirements files. + +Dependabot only bumps the ``package==version`` pins in ``requirements*.txt``. +Some of those tools are pinned a second time as hook ``rev`` values in +``.pre-commit-config.yaml``. This script treats the requirements files as +the source of truth and rewrites the revs to match, editing the config +through yamlrocks so comments and layout survive. + +Run without arguments to apply the changes in place, or with ``--check`` to +only report drift (exit status 1 when anything is out of sync). +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path +import re +import sys +from typing import Any + +import yamlrocks + +REPO_ROOT = Path(__file__).resolve().parent.parent +PRECOMMIT_CONFIG = ".pre-commit-config.yaml" + + +class SyncError(Exception): + """A pin could not be located in a requirements file or the config.""" + + +@dataclass(frozen=True) +class SyncTarget: + """A requirements pin and the pre-commit repo whose rev mirrors it.""" + + package: str + requirements_file: str + repo: str + + +SYNC_TARGETS: tuple[SyncTarget, ...] = ( + SyncTarget( + "ruff", "requirements_test.txt", "https://github.com/astral-sh/ruff-pre-commit" + ), + SyncTarget("flake8", "requirements_test.txt", "https://github.com/PyCQA/flake8"), + SyncTarget( + "pyupgrade", "requirements_test.txt", "https://github.com/asottile/pyupgrade" + ), + SyncTarget( + "clang-format", + "requirements_dev.txt", + "https://github.com/pre-commit/mirrors-clang-format", + ), + SyncTarget( + "yamllint", + "requirements_dev.txt", + "https://github.com/adrienverge/yamllint.git", + ), +) + + +def read_requirement_version(requirements: str, package: str) -> str | None: + """Return the ``==`` pin for ``package`` or None when it is not pinned.""" + pattern = re.compile( + rf"^{re.escape(package)}==(?P[^\s#]+)", + re.MULTILINE | re.IGNORECASE, + ) + match = pattern.search(requirements) + return match.group("version") if match else None + + +def find_repo_entry(doc: Any, repo: str) -> Any: + """Return the single ``- repo:`` block for ``repo`` in a pre-commit doc.""" + try: + entries = [entry for entry in doc["repos"] if entry["repo"] == repo] + except KeyError as err: + raise SyncError(f"malformed pre-commit config, missing key {err}") from None + if len(entries) != 1: + raise SyncError( + f"expected exactly one block for repo {repo}, found {len(entries)}" + ) + return entries[0] + + +def current_rev(entry: Any, repo: str) -> tuple[str, str]: + """Split the block's rev into its tag prefix (``v`` or empty) and version.""" + if "rev" not in entry: + raise SyncError(f"repo {repo} has no rev") + rev = entry["rev"] + if not isinstance(rev, str): + # A rev such as ``1.0`` parses as a number and cannot be compared or + # rewritten safely; quote it in the config instead. + raise SyncError(f"rev of repo {repo} is not a string: {rev!r}") + prefix = "v" if rev.startswith("v") else "" + return prefix, rev.removeprefix("v") + + +def sync(root: Path, *, write: bool) -> list[str]: + """Bring every hook rev in line with its requirements pin. + + Returns one description per rev that was (or, when ``write`` is False, + would be) changed. Raises SyncError when a pin cannot be found, which + means SYNC_TARGETS has gone stale and needs updating by hand. + """ + config_path = root / PRECOMMIT_CONFIG + doc = yamlrocks.loads(config_path.read_bytes(), option=yamlrocks.OPT_ROUND_TRIP) + requirements: dict[str, str] = {} + changes: list[str] = [] + for target in SYNC_TARGETS: + if target.requirements_file not in requirements: + requirements[target.requirements_file] = ( + root / target.requirements_file + ).read_text() + version = read_requirement_version( + requirements[target.requirements_file], target.package + ) + if version is None: + raise SyncError( + f"{target.requirements_file}: no '{target.package}==' pin found" + ) + + entry = find_repo_entry(doc, target.repo) + prefix, current = current_rev(entry, target.repo) + if current == version: + continue + changes.append(f"{target.package}: {current} -> {version}") + entry["rev"] = f"{prefix}{version}" + + if changes and write: + config_path.write_bytes(doc.to_yaml()) + return changes + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--check", + action="store_true", + help="report drift without modifying any file; exit 1 if out of sync", + ) + parser.add_argument( + "--root", + type=Path, + default=REPO_ROOT, + help="repository checkout to operate on (default: this checkout)", + ) + args = parser.parse_args(argv) + + try: + changes = sync(args.root, write=not args.check) + except SyncError as err: + print(f"error: {err}", file=sys.stderr) + return 1 + + for change in changes: + print(change) + if args.check and changes: + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/tests/script/test_sync_dependency_versions.py b/tests/script/test_sync_dependency_versions.py new file mode 100644 index 0000000000..787c8112d9 --- /dev/null +++ b/tests/script/test_sync_dependency_versions.py @@ -0,0 +1,219 @@ +"""Unit tests for script/sync_dependency_versions.py.""" + +from pathlib import Path +import subprocess +import sys + +import pytest +import yamlrocks + +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) + +import sync_dependency_versions as sync_mod # noqa: E402 + +PRECOMMIT = """\ +# See https://pre-commit.com for more information +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.1.0 + hooks: + - id: ruff + - repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + - repo: https://github.com/asottile/pyupgrade + rev: v3.0.0 + hooks: + - id: pyupgrade + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v13.0.1 + hooks: + - id: clang-format + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.0.0 + hooks: + - id: yamllint + - repo: local + hooks: + - id: pylint +""" + +REQ_TEST = """\ +pylint==4.0.8 +flake8==7.1.0 +ruff==0.2.0 # comment +pyupgrade==3.0.0 +""" + +REQ_DEV = """\ +clang-format==13.0.1 +yamllint==1.0.0 +""" + +RUFF_REPO = "https://github.com/astral-sh/ruff-pre-commit" +DUPLICATE_RUFF_BLOCK = f" - repo: {RUFF_REPO}\n rev: v0.3.0\n hooks: []\n" + +EXPECTED_DRIFT = ["ruff: 0.1.0 -> 0.2.0", "flake8: 7.0.0 -> 7.1.0"] +EXPECTED_PRECOMMIT = PRECOMMIT.replace("rev: v0.1.0", "rev: v0.2.0").replace( + "rev: 7.0.0", "rev: 7.1.0" +) + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + """A fake checkout where ruff (v-prefixed) and flake8 (bare) have drifted.""" + (tmp_path / ".pre-commit-config.yaml").write_text(PRECOMMIT) + (tmp_path / "requirements_test.txt").write_text(REQ_TEST) + (tmp_path / "requirements_dev.txt").write_text(REQ_DEV) + return tmp_path + + +def _load(text: str) -> object: + return yamlrocks.loads(text.encode(), option=yamlrocks.OPT_ROUND_TRIP) + + +@pytest.mark.parametrize( + ("requirements", "expected"), + [ + ("prek==0.5.1 # comment\n", "0.5.1"), + ("Prek==0.5.1\n", "0.5.1"), + ("other==1.0\nprek==0.5.1\n", "0.5.1"), + ("prek>=0.5.1\n", None), + ("prek-extra==0.5.1\n", None), + ("", None), + ], +) +def test_read_requirement_version(requirements: str, expected: str | None) -> None: + assert sync_mod.read_requirement_version(requirements, "prek") == expected + + +def test_find_repo_entry() -> None: + entry = sync_mod.find_repo_entry(_load(PRECOMMIT), RUFF_REPO) + assert entry["rev"] == "v0.1.0" + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ("hooks: []\n", "missing key 'repos'"), + ("repos:\n - rev: 1.0.0\n", "missing key 'repo'"), + (PRECOMMIT + DUPLICATE_RUFF_BLOCK, "found 2"), + ("repos:\n - repo: other\n rev: 1.0.0\n", "found 0"), + ], +) +def test_find_repo_entry_errors(text: str, message: str) -> None: + with pytest.raises(sync_mod.SyncError, match=message): + sync_mod.find_repo_entry(_load(text), RUFF_REPO) + + +@pytest.mark.parametrize( + ("rev", "expected"), + [("v0.1.0", ("v", "0.1.0")), ("7.0.0", ("", "7.0.0")), ("'1.0'", ("", "1.0"))], +) +def test_current_rev(rev: str, expected: tuple[str, str]) -> None: + doc = _load(f"repos:\n - repo: {RUFF_REPO}\n rev: {rev}\n") + assert sync_mod.current_rev(doc["repos"][0], RUFF_REPO) == expected + + +@pytest.mark.parametrize( + ("block", "message"), + [(" hooks: []\n", "has no rev"), (" rev: 1.0\n", "not a string: 1.0")], +) +def test_current_rev_errors(block: str, message: str) -> None: + doc = _load(f"repos:\n - repo: {RUFF_REPO}\n{block}") + with pytest.raises(sync_mod.SyncError, match=message): + sync_mod.current_rev(doc["repos"][0], RUFF_REPO) + + +def test_sync_reports_without_writing(root: Path) -> None: + assert sync_mod.sync(root, write=False) == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == PRECOMMIT + + +def test_sync_writes_keeps_layout_and_is_idempotent(root: Path) -> None: + assert sync_mod.sync(root, write=True) == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == EXPECTED_PRECOMMIT + assert sync_mod.sync(root, write=True) == [] + + +def test_sync_does_not_touch_a_config_that_matches(root: Path) -> None: + (root / ".pre-commit-config.yaml").write_text(EXPECTED_PRECOMMIT) + before = (root / ".pre-commit-config.yaml").stat().st_mtime_ns + assert sync_mod.sync(root, write=True) == [] + assert (root / ".pre-commit-config.yaml").stat().st_mtime_ns == before + + +def test_sync_missing_requirement_pin(root: Path) -> None: + (root / "requirements_dev.txt").write_text("") + with pytest.raises(sync_mod.SyncError, match="no 'clang-format==' pin"): + sync_mod.sync(root, write=True) + + +def test_sync_propagates_config_errors(root: Path) -> None: + (root / ".pre-commit-config.yaml").write_text(PRECOMMIT + DUPLICATE_RUFF_BLOCK) + with pytest.raises(sync_mod.SyncError, match="found 2"): + sync_mod.sync(root, write=True) + + +def test_main_check_reports_drift( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert sync_mod.main(["--check", "--root", str(root)]) == 1 + assert capsys.readouterr().out.splitlines() == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == PRECOMMIT + + +def test_main_writes_then_check_is_clean( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert sync_mod.main(["--root", str(root)]) == 0 + assert capsys.readouterr().out.splitlines() == EXPECTED_DRIFT + assert sync_mod.main(["--check", "--root", str(root)]) == 0 + assert capsys.readouterr().out == "" + + +def test_main_reports_sync_error( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (root / "requirements_dev.txt").write_text("") + assert sync_mod.main(["--root", str(root)]) == 1 + assert ( + "error: requirements_dev.txt: no 'clang-format==' pin" + in capsys.readouterr().err + ) + + +def test_main_defaults_to_repo_root(monkeypatch: pytest.MonkeyPatch) -> None: + seen: dict[str, object] = {} + + def fake_sync(root: Path, *, write: bool) -> list[str]: + seen["root"] = root + seen["write"] = write + return [] + + monkeypatch.setattr(sync_mod, "sync", fake_sync) + assert sync_mod.main([]) == 0 + assert seen == {"root": sync_mod.REPO_ROOT, "write": True} + + +def test_repository_is_in_sync() -> None: + """The real checkout must match; a failure here means a rev has drifted. + + Also proves every SYNC_TARGETS entry still resolves in the real files. + """ + assert sync_mod.sync(sync_mod.REPO_ROOT, write=False) == [] + + +def test_cli_entry_point(root: Path) -> None: + """Run the script the way the workflow does, as a subprocess.""" + script = Path(sync_mod.__file__) + result = subprocess.run( + [sys.executable, str(script), "--check", "--root", str(root)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 1 + assert result.stdout.splitlines() == EXPECTED_DRIFT From 390742cf9ba113ea89bc88a05582c4c409a1bd63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 23:47:46 +0000 Subject: [PATCH 35/53] Bump ruff from 0.16.5 to 0.16.6 (#19022) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- esphome/api_client.py | 4 +--- esphome/components/debug/sensor.py | 6 +----- esphome/components/debug/text_sensor.py | 6 +----- esphome/components/esp32/const.py | 11 ++--------- esphome/components/nextion/display.py | 7 +------ esphome/happy_eyeballs.py | 5 +---- requirements_test.txt | 2 +- 8 files changed, 9 insertions(+), 34 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1af0e19273..95e6f0f73e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.5 + rev: v0.16.6 hooks: # Run the linter. - id: ruff diff --git a/esphome/api_client.py b/esphome/api_client.py index fb41075de8..2b93b4790f 100644 --- a/esphome/api_client.py +++ b/esphome/api_client.py @@ -23,9 +23,7 @@ from esphome.util import safe_print if TYPE_CHECKING: from collections.abc import Callable - from aioesphomeapi.api_pb2 import ( - SubscribeLogsResponse, # pylint: disable=no-name-in-module - ) + from aioesphomeapi.api_pb2 import SubscribeLogsResponse # pylint: disable=no-name-in-module _LOGGER = logging.getLogger(__name__) diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index e53cb0d1e4..80d1daa81f 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -23,11 +23,7 @@ from esphome.const import ( ) from esphome.types import ConfigType -from . import ( # noqa: F401 pylint: disable=unused-import - CONF_DEBUG_ID, - FILTER_SOURCE_FILES, - DebugComponent, -) +from . import CONF_DEBUG_ID, FILTER_SOURCE_FILES, DebugComponent # noqa: F401 pylint: disable=unused-import DEPENDENCIES = ["debug"] diff --git a/esphome/components/debug/text_sensor.py b/esphome/components/debug/text_sensor.py index 9d4fcc1b42..2e02af67cb 100644 --- a/esphome/components/debug/text_sensor.py +++ b/esphome/components/debug/text_sensor.py @@ -9,11 +9,7 @@ from esphome.const import ( ) from esphome.types import ConfigType -from . import ( # noqa: F401 pylint: disable=unused-import - CONF_DEBUG_ID, - FILTER_SOURCE_FILES, - DebugComponent, -) +from . import CONF_DEBUG_ID, FILTER_SOURCE_FILES, DebugComponent # noqa: F401 pylint: disable=unused-import DEPENDENCIES = ["debug"] diff --git a/esphome/components/esp32/const.py b/esphome/components/esp32/const.py index e7d8a66e7a..a0c9809c50 100644 --- a/esphome/components/esp32/const.py +++ b/esphome/components/esp32/const.py @@ -3,18 +3,11 @@ import esphome.codegen as cg # Re-exported for the many esp32-side users; defined in esphome.const # and esphome.espidf so the upload/logs fast path can use them without # importing this package. -from esphome.const import ( # noqa: F401 # pylint: disable=unused-import - KEY_ESP32, - KEY_FLASH_SIZE, - KEY_IDF_VERSION, - KEY_VARIANT, -) +from esphome.const import KEY_ESP32, KEY_FLASH_SIZE, KEY_IDF_VERSION, KEY_VARIANT # noqa: F401 # pylint: disable=unused-import # Back compat for external components only; in-tree callers import it # from esphome.espidf directly. -from esphome.espidf import ( # noqa: F401 # pylint: disable=unused-import - variant_to_idf_target, -) +from esphome.espidf import variant_to_idf_target # noqa: F401 # pylint: disable=unused-import KEY_BOARD = "board" KEY_SDKCONFIG_OPTIONS = "sdkconfig_options" diff --git a/esphome/components/nextion/display.py b/esphome/components/nextion/display.py index 3f5ba94b40..a5894bdaf7 100644 --- a/esphome/components/nextion/display.py +++ b/esphome/components/nextion/display.py @@ -14,12 +14,7 @@ from esphome.const import ( ) from esphome.core import CORE, TimePeriod -from . import ( # noqa: F401 pylint: disable=unused-import - FILTER_SOURCE_FILES, - Nextion, - nextion_ns, - nextion_ref, -) +from . import FILTER_SOURCE_FILES, Nextion, nextion_ns, nextion_ref # noqa: F401 pylint: disable=unused-import from .base_component import ( CONF_AUTO_WAKE_ON_TOUCH, CONF_COMMAND_SPACING, diff --git a/esphome/happy_eyeballs.py b/esphome/happy_eyeballs.py index 35092e7daa..8b0d020862 100644 --- a/esphome/happy_eyeballs.py +++ b/esphome/happy_eyeballs.py @@ -69,10 +69,7 @@ def _make_create_connection() -> Callable[..., socket.socket]: from aiohappyeyeballs import start_connection from urllib3.exceptions import LocationParseError - from urllib3.util.connection import ( # noqa: PLC2701 - _set_socket_options, - allowed_gai_family, - ) + from urllib3.util.connection import _set_socket_options, allowed_gai_family # noqa: PLC2701 from urllib3.util.timeout import _DEFAULT_TIMEOUT # noqa: PLC2701 from esphome import async_thread diff --git a/requirements_test.txt b/requirements_test.txt index ef70a5ac0c..9fd82b7509 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.8 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -ruff==0.16.5 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.6 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py prek==0.5.1 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py From 89a56298c231a138080a8604deea8ebb5a630369 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:00:28 +0000 Subject: [PATCH 36/53] Bump prek from 0.5.1 to 0.5.2 (#19021) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 9fd82b7509..cd0427f33e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.8 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.6 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -prek==0.5.1 # .github/workflows/ci.yml reads this pin +prek==0.5.2 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests From 50ca38119873fc717db2ec00ef9b614d5539921b Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:10:55 +0200 Subject: [PATCH 37/53] [i2s_audio] Keep a start request that arrives while the speaker task stops (#19027) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../components/i2s_audio/speaker/i2s_audio_speaker.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 5e271e671e..1c2eb12904 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -91,7 +91,14 @@ void I2SAudioSpeakerBase::loop() { this->speaker_task_handle_ = nullptr; this->stop_i2s_driver_(); - xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + // ALL_BITS includes COMMAND_START. Take the bits from the clear itself, not from the snapshot at + // the top of loop(): the audio source's task can raise a start at any point above, including + // during stop_i2s_driver_(), and nothing would ever re-issue it. + const EventBits_t bits_before_clear = xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + if (bits_before_clear & SpeakerEventGroupBits::COMMAND_START) { + ESP_LOGD(TAG, "Start requested while stopping; keeping the request"); + xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); + } this->status_clear_error(); this->on_task_stopped(); From 56c3361b9adafd3f1f433987d04f27f549a4d965 Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:13:03 +0200 Subject: [PATCH 38/53] [audio] Do not treat MP3_STREAM_INFO_CHANGED as a fatal decoder error (#19028) Co-authored-by: Claude --- esphome/components/audio/audio_decoder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index fe9ad9c9ad..051395606c 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -313,9 +313,10 @@ FileDecoderState AudioDecoder::decode_mp3_() { this->output_transfer_buffer_->increase_buffer_length( this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); } - } else if (result == micro_mp3::MP3_STREAM_INFO_READY) { - // First successful header parse: capture stream info and resize the output buffer to fit one full frame. - // microMP3 always outputs 16-bit PCM. + } else if (result == micro_mp3::MP3_STREAM_INFO_READY || result == micro_mp3::MP3_STREAM_INFO_CHANGED) { + // Header parsed: capture stream info and resize the output buffer to fit one full frame. + // microMP3 always outputs 16-bit PCM. MP3_STREAM_INFO_CHANGED is handled identically: despite its + // negative value it is documented as recoverable, so it must not reach the catch-all below. this->audio_stream_info_ = audio::AudioStreamInfo(16, this->mp3_decoder_->get_channels(), this->mp3_decoder_->get_sample_rate()); this->free_buffer_required_ = From 62eafc477d9ab731b1ce5b8d493d5b69c7e5b468 Mon Sep 17 00:00:00 2001 From: Ryan Ronnander <61520+ryan-ronnander@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:09:02 -0400 Subject: [PATCH 39/53] [mqtt] Restore brightness flag in light discovery (#18950) --- esphome/components/mqtt/mqtt_light.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/mqtt/mqtt_light.cpp b/esphome/components/mqtt/mqtt_light.cpp index aa47bdf996..a8b52a3839 100644 --- a/esphome/components/mqtt/mqtt_light.cpp +++ b/esphome/components/mqtt/mqtt_light.cpp @@ -67,6 +67,9 @@ void MQTTJSONLightComponent::send_discovery(JsonObject root, mqtt::SendDiscovery if (traits.supports_color_mode(ColorMode::RGB_COLD_WARM_WHITE)) color_modes.add(ESPHOME_F("rgbww")); + if (traits.supports_color_capability(ColorCapability::BRIGHTNESS)) + root[ESPHOME_F("brightness")] = true; + if (traits.supports_color_mode(ColorMode::COLOR_TEMPERATURE) || traits.supports_color_mode(ColorMode::COLD_WARM_WHITE)) { root[MQTT_MIN_MIREDS] = traits.get_min_mireds(); From 639ce609bf70332146f25d04cdf5ac6a15bb4ee2 Mon Sep 17 00:00:00 2001 From: AndreKR Date: Tue, 8 Sep 2026 03:13:51 +0200 Subject: [PATCH 40/53] [logger] Fix garbled stack traces (#17939) --- esphome/components/logger/logger_esp32.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index 05fc959ceb..c3d777299d 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -5,6 +5,7 @@ #include #include +#include #ifdef USE_LOGGER_UART_SELECTION_USB_SERIAL_JTAG #include @@ -76,7 +77,11 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { uart_config.parity = UART_PARITY_DISABLE; uart_config.stop_bits = UART_STOP_BITS_1; uart_config.flow_ctrl = UART_HW_FLOWCTRL_DISABLE; +#if SOC_UART_SUPPORT_XTAL_CLK + uart_config.source_clk = UART_SCLK_XTAL; +#else uart_config.source_clk = UART_SCLK_DEFAULT; +#endif uart_param_config(uart_num, &uart_config); // The logger only writes to UART, never reads, so use the minimum RX buffer. // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). From e6aa575f2e960f406cc8edaccb9719dc11f2a92b Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Mon, 7 Sep 2026 18:27:49 -0700 Subject: [PATCH 41/53] [dallas_temp] filter 85 temp from sensor reset (#17877) Co-authored-by: Samuel Sieb --- esphome/components/dallas_temp/dallas_temp.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esphome/components/dallas_temp/dallas_temp.cpp b/esphome/components/dallas_temp/dallas_temp.cpp index ab4a8c458f..c418362ced 100644 --- a/esphome/components/dallas_temp/dallas_temp.cpp +++ b/esphome/components/dallas_temp/dallas_temp.cpp @@ -6,6 +6,7 @@ namespace esphome::dallas_temp { static const char *const TAG = "dallas.temp.sensor"; static const uint8_t DALLAS_MODEL_DS18S20 = 0x10; +static const uint8_t DALLAS_MODEL_DS18B20 = 0x28; static const uint8_t DALLAS_COMMAND_START_CONVERSION = 0x44; static const uint8_t DALLAS_COMMAND_READ_SCRATCH_PAD = 0xBE; static const uint8_t DALLAS_COMMAND_WRITE_SCRATCH_PAD = 0x4E; @@ -154,7 +155,14 @@ float DallasTemperatureSensor::get_temp_c_() { default: break; } - + // undocumented test for powerup measurement of 85 + // https://github.com/cpetrich/counterfeit_DS18B20#solution-to-the-85-c-problem + if ((this->address_ & 0xff) == DALLAS_MODEL_DS18B20) { + if ((temp == 85 * 16) && (this->scratch_pad_[6] == 0xc)) { + ESP_LOGD(TAG, "dropping reading caused by sensor reset"); + return NAN; + } + } return temp / 16.0f; } From d34ffaf3928ef4a5fdaccc94f30b3ab59f6b75c4 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 7 Sep 2026 18:57:50 -0700 Subject: [PATCH 42/53] [ble_client] Report Established from nodes that never read services (#17920) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/ble_client/automation.h | 34 +++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/esphome/components/ble_client/automation.h b/esphome/components/ble_client/automation.h index 94eeb83b3e..93aae23b6a 100644 --- a/esphome/components/ble_client/automation.h +++ b/esphome/components/ble_client/automation.h @@ -22,6 +22,23 @@ class Automation { static const char *const TAG; }; +// Base for nodes that never read the parent's services. +// The parent releases its services only once every node reports Established, so a node that never +// reports it keeps that memory allocated for the life of the connection. +class BLEClientServicelessNode : public BLEClientNode { + public: + // Final so that Established is always reported on SEARCH_CMPL, before the derived node sees the event. + void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) final { + if (event == ESP_GATTC_SEARCH_CMPL_EVT) + this->node_state = espbt::ClientState::ESTABLISHED; + this->on_gattc_event(event, gattc_if, param); + } + + protected: + // Derived nodes handle GATT events here rather than by overriding the handler above. + virtual void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) {} +}; + // implement on_connect automation. class BLEClientConnectTrigger final : public Trigger<>, public BLEClientNode { public: @@ -61,7 +78,7 @@ class BLEClientDisconnectTrigger final : public Trigger<>, public BLEClientNode } }; -class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientNode { +class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientServicelessNode { public: explicit BLEClientPasskeyRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -71,7 +88,7 @@ class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientN } }; -class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientNode { +class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientPasskeyNotificationTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -82,7 +99,7 @@ class BLEClientPasskeyNotificationTrigger final : public Trigger, publ } }; -class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientNode { +class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientNumericComparisonRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -315,19 +332,17 @@ template class BLEClientRemoveBondAction final : public Action class BLEClientConnectAction final : public Action, public BLEClientNode { +template class BLEClientConnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientConnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { case ESP_GATTC_SEARCH_CMPL_EVT: - this->node_state = espbt::ClientState::ESTABLISHED; this->parent()->run_later([this]() { this->play_next_tuple_(this->var_); }); break; // if the connection is closed, terminate the automation chain. @@ -364,14 +379,13 @@ template class BLEClientConnectAction final : public Action var_{}; }; -template class BLEClientDisconnectAction final : public Action, public BLEClientNode { +template class BLEClientDisconnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientDisconnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { From 574762f07861b7008225bb7de89164ed697836da Mon Sep 17 00:00:00 2001 From: Davide D M Date: Tue, 8 Sep 2026 03:59:05 +0200 Subject: [PATCH 43/53] [debug] Check reboot source pref on ESP_RST_WDT and guard against empty source (#17537) --- esphome/components/debug/debug_esp32.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esphome/components/debug/debug_esp32.cpp b/esphome/components/debug/debug_esp32.cpp index 969cd840cf..8e1a67224e 100644 --- a/esphome/components/debug/debug_esp32.cpp +++ b/esphome/components/debug/debug_esp32.cpp @@ -66,11 +66,15 @@ const char *DebugComponent::get_reset_reason_(std::spanmake_preference(REBOOT_MAX_LEN, fnv1_hash_extend(fnv1_hash(REBOOT_KEY), App.get_name().c_str())); char reboot_source[REBOOT_MAX_LEN]{}; - if (pref.load(&reboot_source)) { + if (pref.load(&reboot_source) && reboot_source[0] != '\0') { reboot_source[REBOOT_MAX_LEN - 1] = '\0'; snprintf(buf, size, "Reboot request from %s", reboot_source); } else { From 94e5c3839d8372e95a320cd1796ca500de75be91 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:17:22 +1200 Subject: [PATCH 44/53] [udp] Use cv.invalid for relocated packet_transport options (#19032) --- esphome/components/udp/__init__.py | 16 +++------ tests/unit_tests/components/udp/__init__.py | 0 tests/unit_tests/components/udp/test_init.py | 37 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 tests/unit_tests/components/udp/__init__.py create mode 100644 tests/unit_tests/components/udp/test_init.py diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index a782d875b9..d96a731e9c 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -1,5 +1,4 @@ -from collections.abc import Callable -from typing import Any, NoReturn +from typing import Any from esphome import automation from esphome.automation import Trigger @@ -48,17 +47,10 @@ UDP_SCHEMA = cv.Schema( ) -def is_relocated(option: str) -> Callable[[Any], NoReturn]: - def validator(value: Any) -> NoReturn: - raise cv.Invalid( - f"The '{option}' option should now be configured in the 'packet_transport' component" - ) - - return validator - - RELOCATED = { - cv.Optional(x): is_relocated(x) + cv.Optional(x): cv.invalid( + f"The '{x}' option should now be configured in the 'packet_transport' component" + ) for x in ( CONF_PROVIDERS, CONF_ENCRYPTION, diff --git a/tests/unit_tests/components/udp/__init__.py b/tests/unit_tests/components/udp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/udp/test_init.py b/tests/unit_tests/components/udp/test_init.py new file mode 100644 index 0000000000..5afc92e9c6 --- /dev/null +++ b/tests/unit_tests/components/udp/test_init.py @@ -0,0 +1,37 @@ +"""Tests for the udp component configuration schema.""" + +from __future__ import annotations + +import pytest + +from esphome.components import udp +from esphome.components.packet_transport import ( + CONF_BINARY_SENSORS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_PROVIDERS, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, +) +import esphome.config_validation as cv + + +@pytest.mark.parametrize( + "option", + [ + CONF_PROVIDERS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, + CONF_BINARY_SENSORS, + ], +) +def test_relocated_option_rejected(option: str) -> None: + """Options that moved to packet_transport raise a pointing error.""" + with pytest.raises(cv.Invalid) as exc_info: + udp.CONFIG_SCHEMA({option: True}) + assert ( + f"The '{option}' option should now be configured in the 'packet_transport' component" + in str(exc_info.value) + ) From 5722ccba372857c175e75de217b2e455ca57f88d Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:36:04 -0400 Subject: [PATCH 45/53] [tuya] Build without a network component (#18948) --- esphome/components/tuya/tuya.cpp | 17 +++++++++-- .../tuya/test-no-network.bk72xx-ard.yaml | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/components/tuya/test-no-network.bk72xx-ard.yaml diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index 82fb96d787..f9b4fe2453 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -1,10 +1,13 @@ #include "tuya.h" -#include "esphome/components/network/util.h" #include "esphome/core/gpio.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/util.h" +#ifdef USE_NETWORK +#include "esphome/components/network/util.h" +#endif + #ifdef USE_WIFI #include "esphome/components/wifi/wifi_component.h" #endif @@ -22,6 +25,14 @@ static const int MAX_RETRIES = 5; // Max bytes to log for datapoint values (larger values are truncated) static constexpr size_t MAX_DATAPOINT_LOG_BYTES = 16; +static bool network_is_connected() { +#ifdef USE_NETWORK + return network::is_connected(); +#else + return false; +#endif +} + void Tuya::setup() { this->set_interval("heartbeat", 15000, [this] { this->send_empty_command_(TuyaCommandType::HEARTBEAT); }); if (this->status_pin_ != nullptr) { @@ -554,14 +565,14 @@ void Tuya::send_empty_command_(TuyaCommandType command) { } void Tuya::set_status_pin_() { - bool is_network_ready = network::is_connected() && remote_is_connected(); + bool is_network_ready = network_is_connected() && remote_is_connected(); this->status_pin_->digital_write(is_network_ready); } uint8_t Tuya::get_wifi_status_code_() { uint8_t status = 0x02; - if (network::is_connected()) { + if (network_is_connected()) { status = 0x03; // Protocol version 3 also supports specifying when connected to "the cloud" diff --git a/tests/components/tuya/test-no-network.bk72xx-ard.yaml b/tests/components/tuya/test-no-network.bk72xx-ard.yaml new file mode 100644 index 0000000000..64207e94e3 --- /dev/null +++ b/tests/components/tuya/test-no-network.bk72xx-ard.yaml @@ -0,0 +1,29 @@ +# Tuya without any network component (no wifi/ethernet/api), as used on +# serial-only or BLE-only Tuya MCU boards. Regression test for +# https://github.com/esphome/esphome/issues/18942 +substitutions: + status_pin: P6 + +packages: + uart: !include ../../test_build_components/common/uart/bk72xx-ard.yaml + +tuya: + status_pin: ${status_pin} + +binary_sensor: + - platform: tuya + id: tuya_presence + sensor_datapoint: 101 + +sensor: + - platform: tuya + id: tuya_light_intensity + sensor_datapoint: 103 + +number: + - platform: tuya + id: tuya_far_detection + number_datapoint: 109 + min_value: 0 + max_value: 600 + step: 1 From 53075e41391a706a52d69885f70057cc9616c675 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 06:42:30 +0200 Subject: [PATCH 46/53] [core] Skip PlatformIO's private-package authorization probe (#18823) --- esphome/platformio/library.py | 6 ++- esphome/platformio/prefetch.py | 2 + esphome/platformio/runner.py | 14 ++++++- tests/unit_tests/test_platformio_library.py | 19 ++++++++++ tests/unit_tests/test_platformio_prefetch.py | 14 +++++++ tests/unit_tests/test_platformio_runner.py | 40 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 3ff60f8aaa..fb6779b807 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -616,11 +616,15 @@ def _make_registry_client() -> Any: elsewhere, not by the PlatformIO registry. """ from platformio.package.manager._registry import PackageManagerRegistryMixin + from platformio.registry.client import RegistryClient class _Registry(PackageManagerRegistryMixin): def __init__(self) -> None: - self._registry_client = None self.pkg_type = "library" + self._registry_client = RegistryClient() + # The probe sleeps ~500 ms per lookup (see runner.patch_registry_private_packages); + # instance-level so the ESPHome process never patches PlatformIO's class + self._registry_client.allowed_private_packages = lambda: False @staticmethod def is_system_compatible(value: Any, custom_system: Any = None) -> bool: diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 17a06cb9c1..e648192b73 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -951,8 +951,10 @@ def main(argv: list[str]) -> int: """Subprocess entry point: ``prefetch ``.""" from esphome.core import CORE from esphome.log import setup_log + from esphome.platformio.runner import patch_registry_private_packages signal.signal(signal.SIGTERM, _sigterm) + patch_registry_private_packages() raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") try: level = int(raw_level) if raw_level is not None else logging.INFO diff --git a/esphome/platformio/runner.py b/esphome/platformio/runner.py index 9bb2205a90..b9fbdec38d 100644 --- a/esphome/platformio/runner.py +++ b/esphome/platformio/runner.py @@ -2,7 +2,8 @@ Invoked via ``python -m esphome.platformio.runner`` instead of ``python -m platformio`` so that the patches (incremental rebuild -preservation, download retries) apply inside the subprocess. Running +preservation, download retries, skipping the private-package probe) apply +inside the subprocess. Running PlatformIO in a subprocess keeps its ``sys.path`` mutations and other global state from leaking into the ESPHome process. """ @@ -105,6 +106,16 @@ def patch_file_downloader() -> None: FileDownloader.__init__ = patched_init +def patch_registry_private_packages() -> None: + """Skip PlatformIO's private-package probe; it sleeps ~500 ms per lookup. + + ESPHome never uses private packages, so the answer is always False. + """ + from platformio.registry.client import RegistryClient + + RegistryClient.allowed_private_packages = staticmethod(lambda: False) # type: ignore[method-assign] + + _IGNORE_LIB_WARNINGS = "(?:Hash|Update)" # Regex patterns matched against each line of PlatformIO output. Lines that # match are dropped by RedirectText before they reach the parent process. @@ -152,6 +163,7 @@ FILTER_PLATFORMIO_LINES = [ def main() -> int: patch_structhash() patch_file_downloader() + patch_registry_private_packages() # Wrap stdout/stderr with RedirectText before PlatformIO runs: # diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 3bae39b3c1..512c883c37 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -7,6 +7,7 @@ exercised in their own test modules).""" import json import logging from pathlib import Path +from unittest.mock import Mock import pytest @@ -228,6 +229,24 @@ def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): _resolve_registry_version("owner", "pkg", set()) +def test_make_registry_client_skips_private_package_probe(monkeypatch): + """Our client answers the probe locally without patching PlatformIO's class.""" + from platformio.account.client import AccountClient + from platformio.registry.client import RegistryClient + + pio_probe = RegistryClient.__dict__["allowed_private_packages"] + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + client = lib._make_registry_client().get_registry_client_instance() + + assert client.allowed_private_packages() is False + assert RegistryClient.__dict__["allowed_private_packages"] is pio_probe + + def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: """Stub the registry lookup so tests never touch the network.""" monkeypatch.setattr( diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 77490fd861..14c52dda8d 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1225,6 +1225,20 @@ def test_main_runs_prefetch(tmp_path: Path) -> None: mock_prefetch.assert_called_once_with(tmp_path, "testenv") +def test_main_skips_private_package_probe_before_prefetch(tmp_path: Path) -> None: + """The registry probe patch is applied before any package manager runs.""" + order: list[str] = [] + with ( + patch.object(pf, "_prefetch", side_effect=lambda *_: order.append("prefetch")), + patch( + "esphome.platformio.runner.patch_registry_private_packages", + side_effect=lambda: order.append("patch"), + ), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + assert order == ["patch", "prefetch"] + + def test_main_bad_argv_is_a_distinct_exit( caplog: pytest.LogCaptureFixture, ) -> None: diff --git a/tests/unit_tests/test_platformio_runner.py b/tests/unit_tests/test_platformio_runner.py index f375aa457a..007455f45a 100644 --- a/tests/unit_tests/test_platformio_runner.py +++ b/tests/unit_tests/test_platformio_runner.py @@ -6,7 +6,9 @@ from collections.abc import Callable import io import sys from types import ModuleType +from unittest.mock import Mock +from platformio.registry.client import RegistryClient import pytest from esphome.platformio import runner @@ -30,6 +32,7 @@ def _prepare_main( monkeypatch.setattr(sys, "stderr", stream) monkeypatch.setattr(runner, "patch_structhash", lambda: None) monkeypatch.setattr(runner, "patch_file_downloader", lambda: None) + monkeypatch.setattr(runner, "patch_registry_private_packages", lambda: None) platformio = ModuleType("platformio") platformio_main = ModuleType("platformio.__main__") @@ -91,3 +94,40 @@ def test_main_still_filters_a_drained_partial_line( assert runner.main() == 0 assert buf.getvalue() == b"" + + +def test_main_applies_registry_private_packages_patch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The probe is patched before PlatformIO runs.""" + order: list[str] = [] + _prepare_main(monkeypatch, lambda: order.append("pio") or 0) + monkeypatch.setattr( + runner, "patch_registry_private_packages", lambda: order.append("patch") + ) + + assert runner.main() == 0 + assert order == ["patch", "pio"] + + +# Snapshot PlatformIO's own probe at import, before any test can patch it +_PIO_PROBE = RegistryClient.__dict__["allowed_private_packages"] + + +def test_patch_registry_private_packages_skips_account_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Answers False without touching the account client.""" + from platformio.account.client import AccountClient + + monkeypatch.setattr(RegistryClient, "allowed_private_packages", _PIO_PROBE) + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + runner.patch_registry_private_packages() + + assert RegistryClient.allowed_private_packages() is False + assert RegistryClient().allowed_private_packages() is False From a23f7bb5693a3ef0ec23e0bcc49f6e30561ae986 Mon Sep 17 00:00:00 2001 From: Gytis Date: Tue, 8 Sep 2026 08:31:30 +0200 Subject: [PATCH 47/53] [lvgl] Add missing label dependency to qrcode, keyboard and tabview (#18387) --- esphome/components/lvgl/widgets/keyboard.py | 3 +- esphome/components/lvgl/widgets/qrcode.py | 3 +- esphome/components/lvgl/widgets/tabview.py | 3 +- .../lvgl/config/keyboard_no_label.yaml | 32 +++++++++++++++++ .../lvgl/config/qrcode_no_label.yaml | 34 ++++++++++++++++++ .../lvgl/config/tabview_no_label.yaml | 35 +++++++++++++++++++ .../lvgl/test_widget_label_dependency.py | 32 +++++++++++++++++ 7 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/config/keyboard_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/qrcode_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/tabview_no_label.yaml create mode 100644 tests/component_tests/lvgl/test_widget_label_dependency.py diff --git a/esphome/components/lvgl/widgets/keyboard.py b/esphome/components/lvgl/widgets/keyboard.py index bcd2d2ae59..65516513a6 100644 --- a/esphome/components/lvgl/widgets/keyboard.py +++ b/esphome/components/lvgl/widgets/keyboard.py @@ -15,6 +15,7 @@ from ..defines import ( from ..types import LvCompound, LvType from . import Widget, WidgetType, get_widgets from .buttonmatrix import CONF_BUTTONMATRIX +from .label import CONF_LABEL from .textarea import CONF_TEXTAREA, lv_textarea_t CONF_KEYBOARD = "keyboard" @@ -49,7 +50,7 @@ class KeyboardType(WidgetType): ) def get_uses(self): - return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX + return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX, CONF_LABEL async def to_code(self, w: Widget, config: dict): add_lv_use("KEY_LISTENER") diff --git a/esphome/components/lvgl/widgets/qrcode.py b/esphome/components/lvgl/widgets/qrcode.py index df76ab6bb0..59af9168aa 100644 --- a/esphome/components/lvgl/widgets/qrcode.py +++ b/esphome/components/lvgl/widgets/qrcode.py @@ -10,6 +10,7 @@ from ..types import lv_obj_t from . import Widget, WidgetType from .canvas import CONF_CANVAS from .img import CONF_IMAGE +from .label import CONF_LABEL CONF_QRCODE = "qrcode" CONF_DARK_COLOR = "dark_color" @@ -41,7 +42,7 @@ class QrCodeType(WidgetType): ) def get_uses(self): - return CONF_CANVAS, CONF_IMAGE + return CONF_CANVAS, CONF_IMAGE, CONF_LABEL async def to_code(self, w: Widget, config): await w.set_property( diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index ee252ecf0b..77c88c48ff 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -28,6 +28,7 @@ from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t, lv_obj_t_ptr from . import Widget, WidgetType, add_widgets, get_widgets, set_obj_properties from .button import button_spec from .buttonmatrix import CONF_BUTTONMATRIX, buttonmatrix_spec +from .label import CONF_LABEL from .obj import obj_spec CONF_TABVIEW = "tabview" @@ -74,7 +75,7 @@ class TabviewType(WidgetType): ) def get_uses(self): - return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON + return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON, CONF_LABEL async def to_code(self, w: Widget, config: dict): await w.set_property( diff --git a/tests/component_tests/lvgl/config/keyboard_no_label.yaml b/tests/component_tests/lvgl/config/keyboard_no_label.yaml new file mode 100644 index 0000000000..7a45a537d3 --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_no_label.yaml @@ -0,0 +1,32 @@ +esphome: + name: test-keyboard-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - keyboard: + id: keyboard_widget diff --git a/tests/component_tests/lvgl/config/qrcode_no_label.yaml b/tests/component_tests/lvgl/config/qrcode_no_label.yaml new file mode 100644 index 0000000000..8bb1aafdd6 --- /dev/null +++ b/tests/component_tests/lvgl/config/qrcode_no_label.yaml @@ -0,0 +1,34 @@ +esphome: + name: test-qrcode-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - qrcode: + id: qr_widget + size: 100 + text: "esphome.io" diff --git a/tests/component_tests/lvgl/config/tabview_no_label.yaml b/tests/component_tests/lvgl/config/tabview_no_label.yaml new file mode 100644 index 0000000000..a3c16ab347 --- /dev/null +++ b/tests/component_tests/lvgl/config/tabview_no_label.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-tabview-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - tabview: + id: tabview_widget + tabs: + - name: "Tab 1" + id: tab_1 diff --git a/tests/component_tests/lvgl/test_widget_label_dependency.py b/tests/component_tests/lvgl/test_widget_label_dependency.py new file mode 100644 index 0000000000..9d3e24c8c5 --- /dev/null +++ b/tests/component_tests/lvgl/test_widget_label_dependency.py @@ -0,0 +1,32 @@ +"""Widgets whose LVGL C implementation creates or references labels +internally (tab titles, key legends, the QR canvas fallback) must declare +the label dependency in ``get_uses()``. Otherwise a config that contains +no ``label`` widget of its own compiles LVGL without ``LV_USE_LABEL`` and +fails at C compile time with undefined ``lv_label_*`` symbols. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.lvgl import defines as df + + +@pytest.mark.parametrize( + "yaml_file", + [ + "qrcode_no_label.yaml", + "keyboard_no_label.yaml", + "tabview_no_label.yaml", + ], +) +def test_label_less_config_enables_lv_use_label( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + yaml_file: str, +) -> None: + generate_main(component_config_path(yaml_file)) + assert "LV_USE_LABEL" in df.get_defines() From 10a9baff746613b52df73ff07895e177cf1a0f81 Mon Sep 17 00:00:00 2001 From: Bryan Li Date: Mon, 7 Sep 2026 23:36:42 -0700 Subject: [PATCH 48/53] [rf_bridge] Fix bucket sniffing with Portisch firmware (#17683) Co-authored-by: Bryan Li Co-authored-by: Claude Fable 5 --- esphome/components/rf_bridge/rf_bridge.cpp | 109 +++++++++++++++++---- esphome/components/rf_bridge/rf_bridge.h | 13 +++ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/esphome/components/rf_bridge/rf_bridge.cpp b/esphome/components/rf_bridge/rf_bridge.cpp index 549cce72df..a4a4da5d8c 100644 --- a/esphome/components/rf_bridge/rf_bridge.cpp +++ b/esphome/components/rf_bridge/rf_bridge.cpp @@ -18,6 +18,16 @@ void RFBridgeComponent::ack_() { } bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { + if (this->bucket_frame_candidate_ && byte == RF_CODE_START) { + // A queued next frame proves the trailing 0x55 really was the bucket + // frame's terminator: Portisch builds pulse entries from alternating + // signal edges, so the two level bits inside one pulse byte are always + // opposite — 0xAA (two high-level nibbles) cannot occur in pulse data. + // Finalize before this byte starts the new frame, so back-to-back + // deliveries are split even when loop() never observed a quiet gap + // between them. + this->finish_bucket_frame_(); + } size_t at = this->rx_buffer_.size(); this->rx_buffer_.push_back(byte); const uint8_t *raw = &this->rx_buffer_[0]; @@ -84,26 +94,21 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { break; } case RF_CODE_RFIN_BUCKET: { - if (byte != RF_CODE_STOP) { - return true; + if (at == 2) { + // The count byte: Portisch sends at most 7 buckets + sync, so 0 or + // >8 cannot be a genuine capture — reject before it can occupy the + // buffer for a full frame timeout. + return byte != 0 && byte <= B1_MAX_BUCKET_COUNT; } - - uint8_t buckets = raw[2] << 1; - std::string str; - char next_byte[3]; // 2 hex chars + null - - for (uint32_t i = 0; i <= at; i++) { - buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); - str += next_byte; - if ((i > 3) && buckets) { - buckets--; - } - if ((i < 3) || (buckets % 2) || (i == at - 1)) { - str += " "; - } - } - ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); - break; + // 0x55 is legal DATA inside a B1 frame: bucket durations are sent + // with only their HIGH byte masked to 7 bits, so a duration such as + // 0x0155 puts a raw 0x55 low byte inside the table — the first 0x55 + // must therefore not end the capture. The header declares the table + // length (raw[2] pairs), so a 0x55 there is always data; one at or + // past the first pulse index is a terminator CANDIDATE, confirmed + // once the UART goes quiet (finish_bucket_frame_ in loop()). + this->bucket_frame_candidate_ = byte == RF_CODE_STOP && at >= 3 + static_cast(raw[2]) * 2; + return true; } default: ESP_LOGW(TAG, "Unknown action: 0x%02X", action); @@ -119,6 +124,47 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { return false; } +void RFBridgeComponent::finish_bucket_frame_() { + if (this->rx_buffer_.size() < 4) { + // The candidate flag requires a header + non-empty bucket table, so + // this cannot happen while flag and buffer stay consistent; guard the + // raw[2] / size-1 reads against any future divergence anyway. + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + return; + } + const uint8_t *raw = this->rx_buffer_.data(); + const size_t at = this->rx_buffer_.size() - 1; + + uint8_t buckets = raw[2] << 1; + std::string str; + char next_byte[3]; // 2 hex chars + null + + for (uint32_t i = 0; i <= at; i++) { + buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); + str += next_byte; + if ((i > 3) && buckets) { + buckets--; + } + if ((i < 3) || (buckets % 2) || (i == at - 1)) { + str += " "; + } + } + ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); + + // Deliberately NOT ACKed: Portisch's B1 command handler leaves its + // last_sniffing_command at the previous mode (RF_CODE_RFIN), and its + // host-ACK handler re-arms sniffing from that stale value — so ACKing a + // bucket delivery silently reverts the radio to standard sniffing and + // ends bucket capture. Its delivery path is fire-and-forget and never + // waits for a host ACK. Stock Itead firmware never sends B1 frames, so + // suppressing this ACK cannot change stock-firmware behavior. + // https://github.com/esphome/esphome/issues/17682 + + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; +} + void RFBridgeComponent::write_byte_str_(const std::string &codes) { uint8_t code; int size = codes.length(); @@ -130,12 +176,31 @@ void RFBridgeComponent::write_byte_str_(const std::string &codes) { void RFBridgeComponent::loop() { const uint32_t now = App.get_loop_component_start_time(); - if (now - this->last_bridge_byte_ > 50) { + size_t avail = this->available(); + if (avail == 0 && this->bucket_frame_candidate_ && now - this->last_bridge_byte_ > BUCKET_CANDIDATE_QUIET_MS) { + // The trailing 0x55 was followed by UART quiet, so it really was the + // frame terminator and not an interior data byte. + this->finish_bucket_frame_(); + this->last_bridge_byte_ = now; + } + const bool receiving_bucket = this->rx_buffer_.size() >= 2 && this->rx_buffer_[1] == RF_CODE_RFIN_BUCKET; + if (receiving_bucket) { + // Never declare an in-progress bucket frame dead while its continuation + // bytes are already queued: a stalled loop() otherwise discards a live + // frame that the UART buffer proves is still arriving. + if (avail == 0 && now - this->last_bridge_byte_ > BUCKET_FRAME_TIMEOUT_MS) { + ESP_LOGD(TAG, "Discarding incomplete RFBridge Bucket frame (%u bytes)", + static_cast(this->rx_buffer_.size())); + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + this->last_bridge_byte_ = now; + } + } else if (now - this->last_bridge_byte_ > 50) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; this->last_bridge_byte_ = now; } - size_t avail = this->available(); while (avail > 0) { uint8_t buf[64]; size_t to_read = std::min(avail, sizeof(buf)); @@ -146,12 +211,14 @@ void RFBridgeComponent::loop() { for (size_t i = 0; i < to_read; i++) { if (this->rx_buffer_.size() > MAX_RX_BUFFER_SIZE) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } if (this->parse_bridge_byte_(buf[i])) { ESP_LOGVV(TAG, "Parsed: 0x%02X", buf[i]); this->last_bridge_byte_ = now; } else { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } } } diff --git a/esphome/components/rf_bridge/rf_bridge.h b/esphome/components/rf_bridge/rf_bridge.h index 5ad75650ab..cbb1880ec5 100644 --- a/esphome/components/rf_bridge/rf_bridge.h +++ b/esphome/components/rf_bridge/rf_bridge.h @@ -30,6 +30,17 @@ static const uint8_t RF_CODE_BEEP = 0xC0; static const uint8_t RF_CODE_STOP = 0x55; static const uint8_t RF_DEBOUNCE = 200; static const size_t MAX_RX_BUFFER_SIZE = 512; +// ~10 byte times at 19200 baud: long enough to prove the UART went quiet +// after a possible bucket-frame terminator, short enough to finish well +// before the next radio capture can be delivered. +static const uint32_t BUCKET_CANDIDATE_QUIET_MS = 5; +// Portisch drains a B1 frame's header, bucket table, and pulse data as +// separate UART writes, so an in-progress bucket frame tolerates a longer +// inter-region gap than the generic 50 ms inter-byte timeout. +static const uint32_t BUCKET_FRAME_TIMEOUT_MS = 250; +// Portisch's uart_put_RF_buckets sends at most 7 buckets plus the sync +// bucket, so a B1 count byte above 8 (or 0) is malformed for any protocol. +static const uint8_t B1_MAX_BUCKET_COUNT = 8; struct RFBridgeData { uint16_t sync; @@ -67,10 +78,12 @@ class RFBridgeComponent final : public uart::UARTDevice, public Component { void ack_(); void decode_(); bool parse_bridge_byte_(uint8_t byte); + void finish_bucket_frame_(); void write_byte_str_(const std::string &codes); std::vector rx_buffer_; uint32_t last_bridge_byte_{0}; + bool bucket_frame_candidate_{false}; CallbackManager data_callback_; CallbackManager advanced_data_callback_; From 28588310e74f93140e020fcaf6f95584412f671a Mon Sep 17 00:00:00 2001 From: raykholo Date: Tue, 8 Sep 2026 02:57:33 -0400 Subject: [PATCH 49/53] [anova] Re-assert temperature unit on every poll cycle (#17141) --- esphome/components/anova/anova.cpp | 107 ++++++++++++++--------------- esphome/components/anova/anova.h | 13 +++- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/esphome/components/anova/anova.cpp b/esphome/components/anova/anova.cpp index 6e382872e2..b0769bb622 100644 --- a/esphome/components/anova/anova.cpp +++ b/esphome/components/anova/anova.cpp @@ -13,7 +13,7 @@ void Anova::dump_config() { LOG_CLIMATE("", "Anova BLE Cooker", this); } void Anova::setup() { this->codec_ = make_unique(); - this->current_request_ = 0; + this->poll_step_ = PollStep::IDLE; } void Anova::loop() { @@ -22,6 +22,15 @@ void Anova::loop() { this->disable_loop(); } +void Anova::write_request_(AnovaPacket *pkt) { + auto status = + esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, + pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); + if (status) { + ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } +} + void Anova::control(const ClimateCall &call) { auto mode_val = call.get_mode(); if (mode_val.has_value()) { @@ -38,22 +47,11 @@ void Anova::control(const ClimateCall &call) { ESP_LOGW(TAG, "Unsupported mode: %d", mode); return; } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(pkt); } auto target_temp = call.get_target_temperature(); if (target_temp.has_value()) { - auto *pkt = this->codec_->get_set_target_temp_request(*target_temp); - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(this->codec_->get_set_target_temp_request(*target_temp)); } } @@ -62,6 +60,7 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ case ESP_GATTC_DISCONNECT_EVT: { this->current_temperature = NAN; this->target_temperature = NAN; + this->poll_step_ = PollStep::IDLE; this->publish_state(); break; } @@ -83,8 +82,8 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } case ESP_GATTC_REG_FOR_NOTIFY_EVT: { this->node_state = espbt::ClientState::ESTABLISHED; - this->current_request_ = 0; - this->update(); + this->poll_step_ = PollStep::IDLE; + this->update(); // begin the first poll cycle immediately break; } case ESP_GATTC_NOTIFY_EVT: { @@ -101,33 +100,30 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ this->mode = this->codec_->running_ ? climate::CLIMATE_MODE_HEAT : climate::CLIMATE_MODE_OFF; } if (this->codec_->has_unit()) { - this->fahrenheit_ = (this->codec_->unit_ == 'f'); - ESP_LOGD(TAG, "Anova units is %s", this->fahrenheit_ ? "fahrenheit" : "celsius"); - this->current_request_++; + ESP_LOGD(TAG, "Anova units is %s", (this->codec_->unit_ == 'f') ? "fahrenheit" : "celsius"); } this->publish_state(); - if (this->current_request_ > 1) { - AnovaPacket *pkt = nullptr; - switch (this->current_request_++) { - case 2: - pkt = this->codec_->get_read_target_temp_request(); - break; - case 3: - pkt = this->codec_->get_read_current_temp_request(); - break; - default: - this->current_request_ = 1; - break; - } - if (pkt != nullptr) { - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - } + // Advance the poll cycle to its next request based on the reply we got. + switch (this->poll_step_) { + case PollStep::SET_UNIT: + this->poll_step_ = PollStep::STATUS; + this->write_request_(this->codec_->get_read_device_status_request()); + break; + case PollStep::STATUS: + this->poll_step_ = PollStep::TARGET; + this->write_request_(this->codec_->get_read_target_temp_request()); + break; + case PollStep::TARGET: + this->poll_step_ = PollStep::CURRENT; + this->write_request_(this->codec_->get_read_current_temp_request()); + break; + case PollStep::CURRENT: + this->poll_step_ = PollStep::IDLE; // full cycle complete + break; + default: + // A reply to an ad-hoc control() write, outside a managed cycle. + break; } break; } @@ -136,27 +132,26 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } } -void Anova::set_unit_of_measurement(const char *unit) { this->fahrenheit_ = !strncmp(unit, "f", 1); } +void Anova::set_unit_of_measurement(const char *unit) { this->want_fahrenheit_ = !strncmp(unit, "f", 1); } void Anova::update() { if (this->node_state != espbt::ClientState::ESTABLISHED) return; - - if (this->current_request_ < 2) { - AnovaPacket *pkt; - if (this->current_request_ == 0) { - pkt = this->codec_->get_set_unit_request(this->fahrenheit_ ? 'f' : 'c'); - } else { - pkt = this->codec_->get_read_device_status_request(); - } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - this->current_request_++; + if (this->poll_step_ != PollStep::IDLE) { + // The previous cycle never finished within a full polling interval -- a + // reply was missed or a write failed. Restart the cycle rather than stall; + // the polling interval itself acts as the timeout. A late reply from the + // abandoned cycle is harmless: state decoding happens on every notify + // regardless of step, and each notify sends at most one follow-up request. + ESP_LOGW(TAG, "[%s] Poll cycle incomplete (step %u); restarting cycle", this->parent_->address_str(), + static_cast(this->poll_step_)); } + // Re-assert the configured unit at the start of every poll cycle, then fall + // through the status/temperature reads via the notification handler. Always + // command the configured unit (want_fahrenheit_) -- never the last value the + // device reported, or a drift to 'c' would lock itself in. + this->poll_step_ = PollStep::SET_UNIT; + this->write_request_(this->codec_->get_set_unit_request(this->want_fahrenheit_ ? 'f' : 'c')); } } // namespace esphome::anova diff --git a/esphome/components/anova/anova.h b/esphome/components/anova/anova.h index 49b1100c37..a0fa03df01 100644 --- a/esphome/components/anova/anova.h +++ b/esphome/components/anova/anova.h @@ -37,11 +37,20 @@ class Anova final : public climate::Climate, public esphome::ble_client::BLEClie void set_unit_of_measurement(const char *unit); protected: + // A poll cycle re-asserts the configured unit, then reads device state. + // Re-asserting every cycle prevents the cooker from silently reverting to + // its default (Celsius); previously the unit was only set once on + // connection, so a drift persisted (and corrupted the F/C interpretation of + // subsequent readings) until the BLE link was re-established. + enum class PollStep : uint8_t { SET_UNIT, STATUS, TARGET, CURRENT, IDLE }; + + void write_request_(AnovaPacket *pkt); + std::unique_ptr codec_; void control(const climate::ClimateCall &call) override; uint16_t char_handle_; - uint8_t current_request_; - bool fahrenheit_; + bool want_fahrenheit_{true}; // configured target unit; never overwritten by device replies + PollStep poll_step_{PollStep::IDLE}; }; } // namespace esphome::anova From 1700a40b7cc0cb4a033ba7f2d7c27c1ba65ea0f7 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:59:55 +1200 Subject: [PATCH 50/53] [core] Restore the shared git hooks after post-checkout runs script/setup (#19036) --- script/git-hooks/post-checkout | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/script/git-hooks/post-checkout b/script/git-hooks/post-checkout index 8f4085ae6e..853c2b0352 100755 --- a/script/git-hooks/post-checkout +++ b/script/git-hooks/post-checkout @@ -14,7 +14,31 @@ top=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0 [ -x "$top/venv/bin/python" ] && exit 0 [ -x "$top/script/setup" ] || exit 0 +# Every worktree shares the hooks directory of the checkout it was created +# from, and the script/setup run below is the one from whichever branch was just +# checked out. Older branches install their own pre-commit hook without checking +# for a worktree: that moves the shared hook aside as pre-commit.legacy and +# replaces it with one tied to this worktree's virtual environment, so commits +# break in every checkout. To rule that out, the hooks directory is copied +# before script/setup runs and put back exactly as it was afterwards, including +# removing any file script/setup added. +hooks=$(git rev-parse --path-format=absolute --git-path hooks 2>/dev/null) || exit 0 +snap=$(mktemp -d "$hooks/.post-checkout.XXXXXX") || exit 0 +cp -p "$hooks"/* "$snap"/ 2>/dev/null + # Clear VIRTUAL_ENV so a checkout made from a shell with an environment already # activated still gets its own, rather than having the active one repointed at # this working tree. -exec env -u VIRTUAL_ENV "$top/script/setup" +env -u VIRTUAL_ENV "$top/script/setup" +status=$? + +for f in "$hooks"/*; do + [ -e "$snap/${f##*/}" ] || rm -f "$f" +done +# Files are moved rather than copied so a hook that is still running, such as +# this one, is swapped out atomically instead of being rewritten in place. +for f in "$snap"/*; do + cmp -s "$f" "$hooks/${f##*/}" 2>/dev/null || mv -f "$f" "$hooks/${f##*/}" +done +rm -rf "$snap" +exit $status From 227ca90aad10d3084e3e8a45a13c46aea65a6c3f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:00:08 +1200 Subject: [PATCH 51/53] [core] Restore the shared git hooks after post-checkout runs script/setup (#19036) From f191d5e0c384ca785e562e652cc03cdaf21a99d4 Mon Sep 17 00:00:00 2001 From: John <34163498+CircuitSetup@users.noreply.github.com> Date: Tue, 8 Sep 2026 04:02:49 -0400 Subject: [PATCH 52/53] [atm90e32] Verify offset calibration writes (#18701) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/atm90e32/atm90e32.cpp | 360 ++++++++++-------- esphome/components/atm90e32/atm90e32.h | 71 ++-- tests/components/atm90e32/__init__.py | 5 + .../offset_register_verification_test.cpp | 62 +++ 4 files changed, 322 insertions(+), 176 deletions(-) create mode 100644 tests/components/atm90e32/__init__.py create mode 100644 tests/components/atm90e32/offset_register_verification_test.cpp diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp index d948b3741d..23701e7834 100644 --- a/esphome/components/atm90e32/atm90e32.cpp +++ b/esphome/components/atm90e32/atm90e32.cpp @@ -9,6 +9,10 @@ namespace esphome::atm90e32 { static const char *const TAG = "atm90e32"; +static const LogString *offset_calibration_name(bool power_offsets) { + return power_offsets ? LOG_STR("Power offset") : LOG_STR("Offset"); +} + static uint32_t pref_hash(const char *prefix, const char *name_space) { auto hash = fnv1_hash(prefix); return fnv1_hash_extend(hash, name_space); @@ -203,13 +207,12 @@ void ATM90E32Component::setup() { // Initialize flash storage for power offset calibrations uint32_t po_hash = pref_hash("_power_offset_calibration_", cs); - this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); + this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); bool migrated_power_offset = false; if (has_distinct_legacy_namespace) { uint32_t legacy_po_hash = pref_hash("_power_offset_calibration_", legacy_cs); - auto legacy_power_offset_pref = - global_preferences->make_preference(legacy_po_hash, true); - PowerOffsetCalibration power_offset_data[3]{}; + auto legacy_power_offset_pref = global_preferences->make_preference(legacy_po_hash, true); + OffsetCalibration power_offset_data[3]{}; int migration_status = migrate_legacy_pref_if_needed(this->power_offset_pref_, legacy_power_offset_pref, &power_offset_data); migrated_power_offset = migration_status > 0; @@ -224,20 +227,20 @@ void ATM90E32Component::setup() { global_preferences->sync(); } - this->restore_offset_calibrations_(); - this->restore_power_offset_calibrations_(); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } else { ESP_LOGI(TAG, "[CALIBRATION][%s] Power & Voltage/Current offset calibration is disabled. Using config file values.", cs); for (uint8_t phase = 0; phase < 3; ++phase) { this->write16_(this->voltage_offset_registers[phase], - static_cast(this->offset_phase_[phase].voltage_offset_)); + static_cast(this->offset_phase_[phase].first_offset)); this->write16_(this->current_offset_registers[phase], - static_cast(this->offset_phase_[phase].current_offset_)); + static_cast(this->offset_phase_[phase].second_offset)); this->write16_(this->power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].active_power_offset)); + static_cast(this->power_offset_phase_[phase].first_offset)); this->write16_(this->reactive_power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].reactive_power_offset)); + static_cast(this->power_offset_phase_[phase].second_offset)); } } @@ -317,8 +320,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].voltage_offset_, - this->config_offset_phase_[phase].current_offset_, this->offset_phase_[phase].current_offset_); + this->config_offset_phase_[phase].first_offset, this->offset_phase_[phase].first_offset, + this->config_offset_phase_[phase].second_offset, this->offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -335,10 +338,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].active_power_offset, - this->config_power_offset_phase_[phase].reactive_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->config_power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].first_offset, + this->config_power_offset_phase_[phase].second_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -372,7 +373,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\\n", cs); } @@ -385,8 +386,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); } @@ -756,36 +756,68 @@ void ATM90E32Component::save_gain_calibration_to_memory_() { } } -void ATM90E32Component::save_offset_calibration_to_memory_() { +void ATM90E32Component::finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); - bool success = this->offset_pref_.save(&this->offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_offset_calibration_ = true; - for (bool &phase : this->offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save offset calibration to memory!", cs); - } -} + const LogString *name = offset_calibration_name(power_offsets); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; -void ATM90E32Component::save_power_offset_calibration_to_memory_() { - const char *cs = this->get_calibration_id_(); - bool success = this->power_offset_pref_.save(&this->power_offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_power_offset_calibration_ = true; - for (bool &phase : this->power_offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Power offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save power offset calibration to memory!", cs); + const bool writes_verified = this->verify_offset_writes_(type); + bool saved = false; + bool synced = false; + if (writes_verified) { + saved = preference->save(offsets); + synced = global_preferences->sync(); } + + if (writes_verified && saved && synced) { + this->using_saved_calibrations_ = true; + *has_stored = true; + *restored = true; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration saved to memory. %s calibration completed and verified.", cs, + LOG_STR_ARG(name), LOG_STR_ARG(name)); + return; + } + + if (writes_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save %s calibration to memory!", cs, LOG_STR_ARG(name)); + } + + for (uint8_t phase = 0; phase < 3; phase++) { + this->write_offsets_to_registers_(phase, previous[phase].first_offset, previous[phase].second_offset, type); + } + const bool rollback_verified = this->verify_offset_writes_(type); + + bool rollback_persisted = false; + if (writes_verified) { + OffsetCalibration rollback[3]{}; + prepare_offset_rollback(previous, previous_restored, rollback); + const bool rollback_saved = preference->save(&rollback); + const bool rollback_synced = global_preferences->sync(); + rollback_persisted = rollback_saved && rollback_synced; + if (!rollback_saved || !rollback_synced) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to persist restored %s calibration values!", cs, LOG_STR_ARG(name)); + } + } + + *restored = previous_restored; + if (rollback_persisted) + *has_stored = previous_restored; + this->using_saved_calibrations_ = previous_using_saved; + if (!rollback_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; rollback readback verification failed.", cs, + LOG_STR_ARG(name)); + return; + } + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; previous values restored.", cs, LOG_STR_ARG(name)); } void ATM90E32Component::run_offset_calibrations() { @@ -803,11 +835,16 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->offset_phase_[0], this->offset_phase_[1], this->offset_phase_[2]}; + const bool previous_restored = this->restored_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = calibrate_offset(phase, true); int16_t current_offset = calibrate_offset(phase, false); - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); @@ -815,7 +852,8 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] ==================================================================\n", cs); - this->save_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); } void ATM90E32Component::run_power_offset_calibrations() { @@ -834,18 +872,25 @@ void ATM90E32Component::run_power_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->power_offset_phase_[0], this->power_offset_phase_[1], + this->power_offset_phase_[2]}; + const bool previous_restored = this->restored_power_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; ++phase) { int16_t active_offset = calibrate_power_offset(phase, false); int16_t reactive_offset = calibrate_power_offset(phase, true); - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - this->save_power_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } void ATM90E32Component::write_gains_to_registers_() { @@ -859,35 +904,26 @@ void ATM90E32Component::write_gains_to_registers_() { this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } -void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset) { - // Save to runtime - this->offset_phase_[phase].voltage_offset_ = voltage_offset; - this->phase_[phase].voltage_offset_ = voltage_offset; +void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + OffsetCalibration &offsets = power_offsets ? this->power_offset_phase_[phase] : this->offset_phase_[phase]; + offsets.first_offset = first_offset; + offsets.second_offset = second_offset; + if (power_offsets) { + this->phase_[phase].active_power_offset_ = first_offset; + this->phase_[phase].reactive_power_offset_ = second_offset; + } else { + this->phase_[phase].voltage_offset_ = first_offset; + this->phase_[phase].current_offset_ = second_offset; + } - // Save to flash-storable struct - this->offset_phase_[phase].current_offset_ = current_offset; - this->phase_[phase].current_offset_ = current_offset; - - // Write to registers + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(voltage_offset_registers[phase], static_cast(voltage_offset)); - this->write16_(current_offset_registers[phase], static_cast(current_offset)); - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); -} - -void ATM90E32Component::write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset) { - // Save to runtime - this->phase_[phase].active_power_offset_ = p_offset; - this->phase_[phase].reactive_power_offset_ = q_offset; - - // Save to flash-storable struct - this->power_offset_phase_[phase].active_power_offset = p_offset; - this->power_offset_phase_[phase].reactive_power_offset = q_offset; - - // Write to registers - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(this->power_offset_registers[phase], static_cast(p_offset)); - this->write16_(this->reactive_power_offset_registers[phase], static_cast(q_offset)); + this->write16_(first_registers[phase], static_cast(first_offset)); + this->write16_(second_registers[phase], static_cast(second_offset)); this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } @@ -947,89 +983,78 @@ void ATM90E32Component::restore_gain_calibrations_() { ESP_LOGW(TAG, "[CALIBRATION][%s] No stored gain calibrations found. Using config file values.", cs); } -void ATM90E32Component::restore_offset_calibrations_() { +void ATM90E32Component::restore_offset_calibrations_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); + const LogString *name = power_offsets ? LOG_STR("power offset") : LOG_STR("offset"); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + OffsetCalibration(*config_offsets)[3] = + power_offsets ? &this->config_power_offset_phase_ : &this->config_offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; + const bool *has_first = power_offsets ? this->has_config_active_power_offset_ : this->has_config_voltage_offset_; + const bool *has_second = power_offsets ? this->has_config_reactive_power_offset_ : this->has_config_current_offset_; + for (uint8_t i = 0; i < 3; ++i) - this->config_offset_phase_[i] = this->offset_phase_[i]; - - bool have_data = this->offset_pref_.load(&this->offset_phase_); + (*config_offsets)[i] = (*offsets)[i]; + const bool have_data = preference->load(offsets); bool all_zero = true; if (have_data) { - for (auto &phase : this->offset_phase_) { - if (phase.voltage_offset_ != 0 || phase.current_offset_ != 0) { + for (const auto &phase : *offsets) { + if (phase.first_offset != 0 || phase.second_offset != 0) { all_zero = false; break; } } } - if (have_data && !all_zero) { - this->restored_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; phase++) { - auto &offset = this->offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_voltage_offset_[phase] && - offset.voltage_offset_ != this->config_offset_phase_[phase].voltage_offset_) - mismatch = true; - if (this->has_config_current_offset_[phase] && - offset.current_offset_ != this->config_offset_phase_[phase].current_offset_) - mismatch = true; - if (mismatch) - this->offset_calibration_mismatch_[phase] = true; + *has_stored = have_data && !all_zero; + *restored = false; + for (uint8_t phase = 0; phase < 3; phase++) { + mismatches[phase] = false; + if (*has_stored) { + mismatches[phase] = + (has_first[phase] && (*offsets)[phase].first_offset != (*config_offsets)[phase].first_offset) || + (has_second[phase] && (*offsets)[phase].second_offset != (*config_offsets)[phase].second_offset); } - } else { + } + + if (!*has_stored) { for (uint8_t phase = 0; phase < 3; phase++) - this->offset_phase_[phase] = this->config_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored offset calibrations found. Using default values.", cs); + (*offsets)[phase] = (*config_offsets)[phase]; + ESP_LOGW(TAG, "[CALIBRATION][%s] No stored %s calibrations found. Using default values.", cs, LOG_STR_ARG(name)); } for (uint8_t phase = 0; phase < 3; phase++) { - write_offsets_to_registers_(phase, this->offset_phase_[phase].voltage_offset_, - this->offset_phase_[phase].current_offset_); + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); } -} - -void ATM90E32Component::restore_power_offset_calibrations_() { - const char *cs = this->get_calibration_id_(); - for (uint8_t i = 0; i < 3; ++i) - this->config_power_offset_phase_[i] = this->power_offset_phase_[i]; - - bool have_data = this->power_offset_pref_.load(&this->power_offset_phase_); - - bool all_zero = true; - if (have_data) { - for (auto &phase : this->power_offset_phase_) { - if (phase.active_power_offset != 0 || phase.reactive_power_offset != 0) { - all_zero = false; - break; - } - } + const bool initial_values_verified = this->verify_offset_writes_(type); + if (initial_values_verified) { + const auto state = resolve_offset_restore_state(*has_stored, true, false); + *restored = state.restored; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration values verified.", cs, LOG_STR_ARG(name)); + return; } - if (have_data && !all_zero) { - this->restored_power_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; ++phase) { - auto &offset = this->power_offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_active_power_offset_[phase] && - offset.active_power_offset != this->config_power_offset_phase_[phase].active_power_offset) - mismatch = true; - if (this->has_config_reactive_power_offset_[phase] && - offset.reactive_power_offset != this->config_power_offset_phase_[phase].reactive_power_offset) - mismatch = true; - if (mismatch) - this->power_offset_calibration_mismatch_[phase] = true; - } + this->using_saved_calibrations_ = false; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + for (uint8_t phase = 0; phase < 3; phase++) { + (*offsets)[phase] = (*config_offsets)[phase]; + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); + } + const auto state = resolve_offset_restore_state(*has_stored, false, this->verify_offset_writes_(type)); + *restored = state.restored; + if (state.values_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore failed verification; config values verified.", cs, + LOG_STR_ARG(name)); } else { - for (uint8_t phase = 0; phase < 3; ++phase) - this->power_offset_phase_[phase] = this->config_power_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored power offsets found. Using default values.", cs); - } - - for (uint8_t phase = 0; phase < 3; ++phase) { - write_power_offsets_to_registers_(phase, this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore and config fallback both failed verification.", cs, + LOG_STR_ARG(name)); } } @@ -1084,14 +1109,14 @@ void ATM90E32Component::clear_gain_calibrations() { void ATM90E32Component::clear_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_offset_calibration_) { + if (!this->has_stored_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored offset calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\n", cs); return; @@ -1104,10 +1129,11 @@ void ATM90E32Component::clear_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = - this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].voltage_offset_ : 0; + this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].first_offset : 0; int16_t current_offset = - this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].current_offset_ : 0; - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); } @@ -1117,6 +1143,7 @@ void ATM90E32Component::clear_offset_calibrations() { this->offset_pref_.save(&zero_offsets); // Clear stored values in flash global_preferences->sync(); + this->has_stored_offset_calibration_ = false; this->restored_offset_calibration_ = false; for (bool &phase : this->offset_calibration_mismatch_) phase = false; @@ -1126,15 +1153,14 @@ void ATM90E32Component::clear_offset_calibrations() { void ATM90E32Component::clear_power_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_power_offset_calibration_) { + if (!this->has_stored_power_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored power offsets to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); return; @@ -1147,20 +1173,21 @@ void ATM90E32Component::clear_power_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t active_offset = - this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].active_power_offset : 0; - int16_t reactive_offset = this->has_config_reactive_power_offset_[phase] - ? this->config_power_offset_phase_[phase].reactive_power_offset - : 0; - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].first_offset : 0; + int16_t reactive_offset = + this->has_config_reactive_power_offset_[phase] ? this->config_power_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - PowerOffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; + OffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; this->power_offset_pref_.save(&zero_power_offsets); global_preferences->sync(); + this->has_stored_power_offset_calibration_ = false; this->restored_power_offset_calibration_ = false; for (bool &phase : this->power_offset_calibration_mismatch_) phase = false; @@ -1215,6 +1242,31 @@ bool ATM90E32Component::verify_gain_writes_() { return success; // Return true if all writes were successful, false otherwise } +bool ATM90E32Component::verify_offset_writes_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + const char *cs = this->get_calibration_id_(); + const LogString *name = offset_calibration_name(power_offsets); + const LogString *first_name = power_offsets ? LOG_STR("active") : LOG_STR("voltage"); + const LogString *second_name = power_offsets ? LOG_STR("reactive") : LOG_STR("current"); + const OffsetCalibration *offsets = power_offsets ? this->power_offset_phase_ : this->offset_phase_; + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; + bool success = true; + for (uint8_t phase = 0; phase < 3; phase++) { + const uint16_t first = this->read16_(first_registers[phase]); + const uint16_t second = this->read16_(second_registers[phase]); + if (!offset_register_value_matches(first, offsets[phase].first_offset) || + !offset_register_value_matches(second, offsets[phase].second_offset)) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s readback failed for Phase %s: %s %d/%d, %s %d/%d.", cs, LOG_STR_ARG(name), + phase_labels[phase], LOG_STR_ARG(first_name), static_cast(first), offsets[phase].first_offset, + LOG_STR_ARG(second_name), static_cast(second), offsets[phase].second_offset); + success = false; + } + } + return success; +} + #ifdef USE_TEXT_SENSOR void ATM90E32Component::check_phase_status() { uint16_t state0 = this->read16_(ATM90E32_REGISTER_EMMSTATE0); diff --git a/esphome/components/atm90e32/atm90e32.h b/esphome/components/atm90e32/atm90e32.h index c636e5065a..fe7d903962 100644 --- a/esphome/components/atm90e32/atm90e32.h +++ b/esphome/components/atm90e32/atm90e32.h @@ -13,6 +13,40 @@ namespace esphome::atm90e32 { +inline bool offset_register_value_matches(uint16_t actual, int16_t expected) { + return actual == static_cast(expected); +} + +struct OffsetCalibration { + int16_t first_offset{0}; + int16_t second_offset{0}; +}; + +static_assert(sizeof(OffsetCalibration[3]) == 12, "Offset calibration preference layout must remain compatible"); + +enum class OffsetCalibrationType : uint8_t { + OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT, + OFFSET_CALIBRATION_TYPE_POWER, +}; + +struct OffsetRestoreState { + bool restored; + bool values_verified; +}; + +inline OffsetRestoreState resolve_offset_restore_state(bool has_stored_values, bool initial_values_verified, + bool fallback_values_verified) { + if (initial_values_verified) + return {has_stored_values, true}; + return {false, fallback_values_verified}; +} + +inline void prepare_offset_rollback(const OffsetCalibration (&previous)[3], bool had_stored_values, + OffsetCalibration (&rollback)[3]) { + for (uint8_t phase = 0; phase < 3; phase++) + rollback[phase] = had_stored_values ? previous[phase] : OffsetCalibration{}; +} + class ATM90E32Component final : public PollingComponent, public spi::SPIDevice { @@ -71,19 +105,19 @@ class ATM90E32Component final : public PollingComponent, this->has_config_current_gain_[phase] = true; } void set_voltage_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].voltage_offset_ = offset; + this->offset_phase_[phase].first_offset = offset; this->has_config_voltage_offset_[phase] = true; } void set_current_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].current_offset_ = offset; + this->offset_phase_[phase].second_offset = offset; this->has_config_current_offset_[phase] = true; } void set_active_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].active_power_offset = offset; + this->power_offset_phase_[phase].first_offset = offset; this->has_config_active_power_offset_[phase] = true; } void set_reactive_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].reactive_power_offset = offset; + this->power_offset_phase_[phase].second_offset = offset; this->has_config_reactive_power_offset_[phase] = true; } void set_freq_sensor(sensor::Sensor *freq_sensor) { freq_sensor_ = freq_sensor; } @@ -171,16 +205,16 @@ class ATM90E32Component final : public PollingComponent, float get_chip_temperature_(); bool get_publish_interval_flag_() { return publish_interval_flag_; }; void set_publish_interval_flag_(bool flag) { publish_interval_flag_ = flag; }; - void restore_offset_calibrations_(); - void restore_power_offset_calibrations_(); + void restore_offset_calibrations_(OffsetCalibrationType type); void restore_gain_calibrations_(); - void save_offset_calibration_to_memory_(); void save_gain_calibration_to_memory_(); - void save_power_offset_calibration_to_memory_(); - void write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset); - void write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset); + void finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type); + void write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type); void write_gains_to_registers_(); bool verify_gain_writes_(); + bool verify_offset_writes_(OffsetCalibrationType type); bool validate_spi_read_(uint16_t expected, const char *context = nullptr); void log_calibration_status_(); const char *get_calibration_id_(); @@ -219,19 +253,10 @@ class ATM90E32Component final : public PollingComponent, uint32_t cumulative_reverse_active_energy_{0}; } phase_[3]; - struct OffsetCalibration { - int16_t voltage_offset_{0}; - int16_t current_offset_{0}; - } offset_phase_[3]; - + OffsetCalibration offset_phase_[3]; OffsetCalibration config_offset_phase_[3]; - - struct PowerOffsetCalibration { - int16_t active_power_offset{0}; - int16_t reactive_power_offset{0}; - } power_offset_phase_[3]; - - PowerOffsetCalibration config_power_offset_phase_[3]; + OffsetCalibration power_offset_phase_[3]; + OffsetCalibration config_power_offset_phase_[3]; struct GainCalibration { uint16_t voltage_gain{1}; @@ -265,6 +290,8 @@ class ATM90E32Component final : public PollingComponent, bool enable_offset_calibration_{false}; bool enable_gain_calibration_{false}; const char *instance_id_{nullptr}; + bool has_stored_offset_calibration_{false}; + bool has_stored_power_offset_calibration_{false}; bool restored_offset_calibration_{false}; bool restored_power_offset_calibration_{false}; bool restored_gain_calibration_{false}; diff --git a/tests/components/atm90e32/__init__.py b/tests/components/atm90e32/__init__.py new file mode 100644 index 0000000000..37d6797e2d --- /dev/null +++ b/tests/components/atm90e32/__init__.py @@ -0,0 +1,5 @@ +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + manifest.dependencies = manifest.dependencies + ["sensor", "spi"] diff --git a/tests/components/atm90e32/offset_register_verification_test.cpp b/tests/components/atm90e32/offset_register_verification_test.cpp new file mode 100644 index 0000000000..3bb3eb76ea --- /dev/null +++ b/tests/components/atm90e32/offset_register_verification_test.cpp @@ -0,0 +1,62 @@ +#include + +#include "esphome/components/atm90e32/atm90e32.h" + +namespace esphome::atm90e32::testing { + +TEST(ATM90E32OffsetRegisterVerification, AcceptsExactSignedReadback) { + EXPECT_TRUE(offset_register_value_matches(0x007B, 123)); + EXPECT_TRUE(offset_register_value_matches(0xFF85, -123)); +} + +TEST(ATM90E32OffsetRegisterVerification, RejectsMismatchedReadback) { + EXPECT_FALSE(offset_register_value_matches(0x007C, 123)); + EXPECT_FALSE(offset_register_value_matches(0xFF84, -123)); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedStoredValuesAsRestored) { + const auto state = resolve_offset_restore_state(true, true, false); + + EXPECT_TRUE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedConfigFallbackAsNotRestored) { + const auto state = resolve_offset_restore_state(true, false, true); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsFailedConfigFallbackAsUnverified) { + const auto state = resolve_offset_restore_state(true, false, false); + + EXPECT_FALSE(state.restored); + EXPECT_FALSE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsConfigWithoutStoredValuesAsNotRestored) { + const auto state = resolve_offset_restore_state(false, true, false); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetPersistence, RollsBackStoredValuesOrZeroSentinel) { + const OffsetCalibration previous[3]{{1, -1}, {2, -2}, {3, -3}}; + OffsetCalibration rollback[3]{}; + + prepare_offset_rollback(previous, true, rollback); + for (uint8_t phase = 0; phase < 3; phase++) { + EXPECT_EQ(rollback[phase].first_offset, previous[phase].first_offset); + EXPECT_EQ(rollback[phase].second_offset, previous[phase].second_offset); + } + + prepare_offset_rollback(previous, false, rollback); + for (const auto &phase : rollback) { + EXPECT_EQ(phase.first_offset, 0); + EXPECT_EQ(phase.second_offset, 0); + } +} + +} // namespace esphome::atm90e32::testing From f91486305ff20743d086651517d652360ab58ff7 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:21:48 +0200 Subject: [PATCH 53/53] Bump bundled esphome-device-builder to 1.14.5 (#19040) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index da76ab7b6a..ac84ee4689 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 RUN \ platformio settings set enable_telemetry No \