diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 689baa1292e..bb461678cb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -321,7 +321,7 @@ jobs: . venv/bin/activate pytest -vv --cov-report=xml --tb=native --durations=30 -n auto tests --ignore=tests/integration/ - name: Upload coverage to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@0b35c9ecc4f0529d0eb674914510c22f85b196b4 # v7.1.0 - name: Save Python virtual environment cache if: github.ref == 'refs/heads/dev' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -350,7 +350,7 @@ jobs: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Report empty upload to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@0b35c9ecc4f0529d0eb674914510c22f85b196b4 # v7.1.0 with: run_command: empty-upload force: true diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 67e03f4a8df..e6dc797e4c8 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -29,6 +29,29 @@ static const char *const TAG = "api"; // APIServer APIServer *global_api_server = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +#ifdef USE_API_NOISE +static constexpr uint32_t NOISE_PSK_PREF_HASH = 88491486UL; +#endif + +#if defined(USE_API_NOISE) && defined(USE_OTA_ENCRYPTION_PROVISIONED) +bool load_saved_noise_psk(noise::psk_t &out) { + SavedNoisePsk saved; +#ifdef USE_PREFERENCE_KEY_LOOKUP + const bool loaded = + global_preferences->load_from_key(NOISE_PSK_PREF_HASH, reinterpret_cast(&saved), sizeof(saved)); +#else + // Slot backends need the reservation walk; it only lands on the record when the reservations before + // it match a normal boot, otherwise the type checked checksum fails the load + const bool loaded = global_preferences->make_preference(NOISE_PSK_PREF_HASH, true).load(&saved); +#endif + // The all-zeros record means no key + if (!loaded || noise::NoiseContext::is_all_zeros(saved.psk)) + return false; + out = saved.psk; + return true; +} +#endif + APIServer::APIServer() { global_api_server = this; } void APIServer::socket_failed_(const LogString *msg) { @@ -43,8 +66,7 @@ void APIServer::setup() { #ifdef USE_API_NOISE // Always reserve the slot: flash preferences are positional on esp8266, so // a yaml key build must keep the layout of a runtime key build - uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); + this->noise_pref_ = global_preferences->make_preference(NOISE_PSK_PREF_HASH, true); #ifndef USE_API_NOISE_PSK_FROM_YAML // A cleared record loads fine but holds no key if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index fa9ebd61a99..c4421e4bc29 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -43,6 +43,11 @@ struct SavedNoisePsk { noise::psk_t psk; } PACKED; // NOLINT #endif +#if defined(USE_API_NOISE) && defined(USE_OTA_ENCRYPTION_PROVISIONED) +/// One-shot read of the provisioned key for a boot without an api server (safe mode); false when +/// there is no key +bool load_saved_noise_psk(noise::psk_t &out); +#endif class APIServer final : public Component, public Controller diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 748be9ae4cf..ae1f821c178 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -262,7 +262,7 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "esp_lcd", # LCD controller drivers - only needed by display component "esp_local_ctrl", # Local control over HTTPS/BLE - ESPHome has native API "esp_phy", # RF PHY - re-included by internal_temperature on the original ESP32; esp_wifi/bt/ieee802154 pull it back - "esp_wifi", # WiFi stack - re-included by request_wifi(), espnow; bt pulls it back for BLE builds + "esp_wifi", # WiFi stack - re-included by request_wifi(), espnow, esp32_hosted; bt pulls it back for BLE builds "espcoredump", # Core dump support - ESPHome has its own debug component "fatfs", # FAT filesystem - ESPHome doesn't use filesystem storage "ieee802154", # 802.15.4 radio - IDF openthread and the Zigbee libs pull it back diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index 21626e432b6..6943efc3cb8 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -290,6 +290,8 @@ async def to_code(config: ConfigType) -> None: # symbols are simply unused and never register a callback at runtime. if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: add_define("USE_ESP_NOW_HOSTED") + # esp_now_hosted.cpp includes esp_now.h, which esp_wifi provides + esp32.include_builtin_idf_component("esp_wifi") # esp-hosted's CustomRpc ("peer data transfer") path — off by default. esp32.add_idf_sdkconfig_option( "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index bcf2a2271cb..bace13c17fc 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -316,20 +316,16 @@ async def to_code(config: ConfigType) -> None: # One key per device: an api encryption block supplies it (static or # runtime) and offers; the ota block only adds the requirement api_conf = CORE.config.get(CONF_API) or {} - encryption_conf = config.get(CONF_ENCRYPTION) - own_key = None - if encryption_conf is not None and static_encryption_key(api_conf) is None: - own_key = encryption_conf[CONF_KEY] - if own_key is not None: + if key := static_encryption_key(config) or static_encryption_key(api_conf): + # Build time key: the ota keeps its own pointer so safe mode, which + # has no api server, still has it cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) elif CONF_ENCRYPTION in api_conf: + # Runtime key: found in the api server, or in preferences in safe mode cg.add_define("USE_OTA_ENCRYPTION") - cg.add_define("USE_OTA_ENCRYPTION_FROM_API") - if static_encryption_key(api_conf) is None: - # The key arrives at runtime, so the offer has to look for it - cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") - if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if CONF_ENCRYPTION in config: cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 3010df10561..d353d01d205 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,5 +1,5 @@ #include "ota_esphome.h" -#ifdef USE_OTA_ENCRYPTION_FROM_API +#ifdef USE_OTA_ENCRYPTION_PROVISIONED #include "esphome/components/api/api_server.h" #endif #ifdef USE_OTA @@ -32,11 +32,13 @@ static const char *const TAG = "esphome.ota"; #ifdef USE_OTA_ENCRYPTION const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { -#ifdef USE_OTA_ENCRYPTION_FROM_API - return api::global_api_server->get_noise_ctx(); -#else - return this->noise_ctx_; +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // The api server holds the live key; safe mode never constructs it, and then + // noise_ctx_ holds the saved key setup() found, if any + if (api::global_api_server != nullptr) + return api::global_api_server->get_noise_ctx(); #endif + return this->noise_ctx_; } #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; @@ -58,6 +60,16 @@ extern "C" void esphome_wake_ota_component_any_context() { } void ESPHomeOTAComponent::setup() { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // Safe mode never constructs the api server, so read the key it saved + noise::psk_t psk; + if (api::global_api_server == nullptr && api::load_saved_noise_psk(psk)) { + this->saved_psk_ = RAMAllocator().make_unique(psk); + if (this->saved_psk_ != nullptr) { + this->noise_ctx_.set_psk(this->saved_psk_->data()); + } + } +#endif this->server_ = socket::socket_ip_loop_monitored(SOCK_STREAM, 0).release(); // monitored for incoming connections if (this->server_ == nullptr) { this->server_failed_(LOG_STR("creation")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index 68dd0ffb9ef..92ba094c8de 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,7 +44,7 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) +#ifdef USE_OTA_ENCRYPTION /// psk points at 32 bytes that live in flash for the life of the program void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif @@ -86,7 +86,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; - // The api server's live context when the api has encryption, else our own + // The api server's live context when it exists, otherwise our own (a build + // time key, or the saved key loaded in safe mode) const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); @@ -148,8 +149,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { RAMUniquePtr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION -#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // Backs noise_ctx_ in safe mode, where no api server holds the saved key + RAMUniquePtr saved_psk_; #endif RAMUniquePtr noise_; #endif // USE_OTA_ENCRYPTION diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 8a3943c672f..c894c147cdf 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -5,11 +5,12 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import CONF_ENCRYPTION, CONF_KEY -from esphome.core import ID +from esphome.core import CORE, ID from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] +DOMAIN = "noise" noise_ns = cg.esphome_ns.namespace("noise") @@ -70,11 +71,16 @@ def static_encryption_key(conf: ConfigType) -> str | None: def new_psk_progmem(parent_id: ID, key: str) -> MockObj: """Emit the decoded key as a PROGMEM array; the component keeps a pointer - so the key never occupies RAM.""" - return cg.progmem_array( - ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), - list(decode_encryption_key(key)), - ) + so the key never occupies RAM. Components sharing one key (api and ota) + share the array.""" + decoded = decode_encryption_key(key) + arrays: dict[bytes, MockObj] = CORE.data.setdefault(DOMAIN, {}) + if (array := arrays.get(decoded)) is None: + array = arrays[decoded] = cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decoded), + ) + return array def encryption_schema(config: ConfigType | None) -> ConfigType: diff --git a/esphome/components/snapshot/snapshot.cpp b/esphome/components/snapshot/snapshot.cpp index 995f87710e9..5aa0bcd0e13 100644 --- a/esphome/components/snapshot/snapshot.cpp +++ b/esphome/components/snapshot/snapshot.cpp @@ -211,7 +211,7 @@ bool Snapshot::take_snapshot(const char *filename) { exact = !name.empty(); if (name_changed) { ESP_LOGW(TAG, "Requested snapshot name '%s' is not an acceptable file name, using '%s' instead", filename, - name.empty() ? "a name made from the time" : name.c_str()); + name.empty() ? LOG_STR_LITERAL("a name made from the time") : name.c_str()); } } if (name.empty()) { diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp index 8b0dfe166fb..bad3d050f3a 100644 --- a/esphome/components/web_server/web_server.cpp +++ b/esphome/components/web_server/web_server.cpp @@ -200,10 +200,10 @@ void DeferredUpdateEventSource::process_deferred_queue_() { deferred_queue_.erase(deferred_queue_.begin()); this->consecutive_send_failures_ = 0; // Reset failure count on successful send } else { - // NOTE: Similar logic exists in web_server_idf/web_server_idf.cpp in AsyncEventSourceResponse::process_buffer_() - // The implementations differ due to platform-specific APIs (DISCARDED vs HTTPD_SOCK_ERR_TIMEOUT, close() vs - // fd_.store(0)), but the failure counting and timeout logic should be kept in sync. If you change this logic, - // also update the ESP-IDF implementation. + // NOTE: Similar logic exists in web_server_idf/web_server_idf.cpp in AsyncEventSourceResponse::process_buffer_(). + // The close mechanisms are platform-specific (this path calls close() directly; the IDF path is time-based and + // closes through HTTPD to preserve session ownership), but both drop a client after roughly 20 seconds without + // send progress. Keep that stall policy in sync when changing either side. this->consecutive_send_failures_++; if (this->consecutive_send_failures_ >= MAX_CONSECUTIVE_SEND_FAILURES) { // Too many failures, connection is likely dead diff --git a/esphome/components/web_server_idf/web_server_idf.cpp b/esphome/components/web_server_idf/web_server_idf.cpp index 9550570cdcf..0062556063f 100644 --- a/esphome/components/web_server_idf/web_server_idf.cpp +++ b/esphome/components/web_server_idf/web_server_idf.cpp @@ -6,6 +6,7 @@ #include #include +#include "esphome/core/application.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -728,7 +729,7 @@ bool AsyncEventSource::loop() { for (size_t i = 0; i < this->sessions_.size();) { auto *ses = this->sessions_[i]; // If the session has a dead socket (marked by destroy callback) - if (ses->fd_.load() == 0) { + if (ses->safe_to_delete_()) { // destroy() already logged the close with the fd; don't double-log here. delete ses; // NOLINT(cppcoreguidelines-owning-memory) // Remove by swapping with last element (O(1) removal, order doesn't matter for sessions) @@ -751,7 +752,7 @@ void AsyncEventSource::adopt_pending_sessions_main_loop_() { } for (auto *rsp : incoming) { // Already disconnected? Drop it; skip on_connect_/session start on a dead session. - if (rsp->fd_.load() == 0) { + if (rsp->safe_to_delete_()) { delete rsp; // NOLINT(cppcoreguidelines-owning-memory) continue; } @@ -865,10 +866,16 @@ void AsyncEventSourceResponse::deq_push_back_with_dedup_(void *source, message_g } void AsyncEventSourceResponse::process_deferred_queue_() { + if (this->close_requested_) { + return; + } while (!deferred_queue_.empty()) { DeferredEvent &de = deferred_queue_.front(); auto message = de.message_generator_(web_server_, de.source_); if (this->try_send_nodefer(message.c_str(), message.size(), "state")) { + if (this->close_requested_ || deferred_queue_.empty()) { + return; + } // O(n) but memory efficiency is more important than speed here which is why std::vector was chosen deferred_queue_.erase(deferred_queue_.begin()); } else { @@ -877,8 +884,64 @@ void AsyncEventSourceResponse::process_deferred_queue_() { } } +void AsyncEventSourceResponse::request_close_() { + if (!this->close_requested_) { + this->close_requested_ = true; + this->deferred_queue_.clear(); + this->event_buffer_.clear(); + this->event_bytes_sent_ = 0; + this->next_close_attempt_ms_ = App.get_loop_component_start_time(); + } + + this->process_close_(); +} + +void AsyncEventSourceResponse::process_close_() { + if (!this->close_requested_ || this->close_work_queued_.load(std::memory_order_acquire)) { + return; + } + const int fd = this->fd_.load(); + if (fd == 0) { + return; + } + + const uint32_t now = App.get_loop_component_start_time(); + if (static_cast(now - this->next_close_attempt_ms_) < 0) { + return; + } + + // Queue an identity-checked shutdown on the HTTPD task. The public + // httpd_sess_trigger_close() queues only a reusable fd/session slot and can + // therefore close a new client if the original peer disconnects meanwhile. + this->close_work_queued_.store(true, std::memory_order_release); + const esp_err_t err = httpd_queue_work(this->hd_, &AsyncEventSourceResponse::close_session_work, this); + this->next_close_attempt_ms_ = now + (err == ESP_OK ? CLOSE_CONFIRM_INTERVAL_MS : CLOSE_RETRY_INTERVAL_MS); + if (err == ESP_OK) { + return; + } + + this->close_work_queued_.store(false, std::memory_order_release); + if (!this->close_retry_warning_logged_) { + ESP_LOGW(TAG, "Failed to queue EventSource close (%s); retrying", esp_err_to_name(err)); + this->close_retry_warning_logged_ = true; + } +} + +void AsyncEventSourceResponse::close_session_work(void *arg) { + auto *response = static_cast(arg); + const int fd = response->fd_.load(); + if (fd != 0 && httpd_sess_get_ctx(response->hd_, fd) == response) { + // The HTTPD task remains the session owner. Shutting the socket down makes + // its next select/recv path delete the session and invoke destroy(). + shutdown(fd, SHUT_RDWR); + } + + // Release self only after the HTTPD-task callback has finished every access. + response->close_work_queued_.store(false, std::memory_order_release); +} + void AsyncEventSourceResponse::process_buffer_() { - if (event_buffer_.empty()) { + if (this->close_requested_ || event_buffer_.empty()) { return; } if (event_bytes_sent_ == event_buffer_.size()) { @@ -892,32 +955,33 @@ void AsyncEventSourceResponse::process_buffer_() { httpd_socket_send(this->hd_, this->fd_.load(), event_buffer_.c_str() + event_bytes_sent_, remaining, 0); if (bytes_sent == HTTPD_SOCK_ERR_TIMEOUT) { // EAGAIN/EWOULDBLOCK - socket buffer full, try again later - // NOTE: Similar logic exists in web_server/web_server.cpp in DeferredUpdateEventSource::process_deferred_queue_() - // The implementations differ due to platform-specific APIs (HTTPD_SOCK_ERR_TIMEOUT vs DISCARDED, fd_.store(0) vs - // close()), but the failure counting and timeout logic should be kept in sync. If you change this logic, also - // update the Arduino implementation. - this->consecutive_send_failures_++; - if (this->consecutive_send_failures_ >= MAX_CONSECUTIVE_SEND_FAILURES) { - // Too many failures, connection is likely dead - ESP_LOGW(TAG, "Closing stuck EventSource connection after %" PRIu16 " failed sends", - this->consecutive_send_failures_); - this->fd_.store(0); // Mark for cleanup - this->deferred_queue_.clear(); + // NOTE: Similar logic exists in web_server/web_server.cpp in DeferredUpdateEventSource::process_deferred_queue_(). + // The IDF path is intentionally time-based and closes through HTTPD to preserve session ownership. + const uint32_t now = App.get_loop_component_start_time(); + if (this->send_failure_started_ms_ == 0) { + this->send_failure_started_ms_ = now != 0 ? now : 1; // Reserve zero for no stall. + } + if (static_cast(now - (this->send_failure_started_ms_ + SEND_STALL_TIMEOUT_MS)) >= 0) { + ESP_LOGW(TAG, "Closing stuck EventSource connection after %" PRIu32 " ms without send progress", + now - this->send_failure_started_ms_); + this->request_close_(); } return; } if (bytes_sent == HTTPD_SOCK_ERR_FAIL) { - // Real socket error - connection will be closed by httpd and destroy callback will be called + // Low-level asynchronous sends do not make HTTPD close the session automatically. + this->request_close_(); return; } if (bytes_sent <= 0) { // Unexpected error or zero bytes sent ESP_LOGW(TAG, "Unexpected send result: %d", bytes_sent); + this->request_close_(); return; } - // Successful send - reset failure counter - this->consecutive_send_failures_ = 0; + // Successful send - reset stall tracking + this->send_failure_started_ms_ = 0; event_bytes_sent_ += bytes_sent; // Log partial sends for debugging @@ -933,20 +997,26 @@ void AsyncEventSourceResponse::process_buffer_() { } void AsyncEventSourceResponse::loop() { + if (this->close_requested_) { + this->process_close_(); + return; + } process_buffer_(); process_deferred_queue_(); + if (this->close_requested_) + return; // One step per loop; refusals retry next pass this->entities_iterator_.try_advance(1); } bool AsyncEventSourceResponse::try_send_nodefer(const char *message, size_t message_len, const char *event, uint32_t id, uint32_t reconnect) { - if (this->fd_.load() == 0) { + if (this->fd_.load() == 0 || this->close_requested_) { return false; } process_buffer_(); - if (!event_buffer_.empty()) { + if (this->close_requested_ || !event_buffer_.empty()) { // there is still pending event data to send first return false; } @@ -1098,6 +1168,10 @@ void AsyncEventSourceResponse::deferrable_send_state(void *source, const char *e process_buffer_(); process_deferred_queue_(); + if (this->close_requested_) { + return; + } + if (!event_buffer_.empty() || !deferred_queue_.empty()) { // outgoing event buffer or deferred queue still not empty which means downstream tcp send buffer full, no point // trying to send first diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h index 743d296d73a..7894206102f 100644 --- a/esphome/components/web_server_idf/web_server_idf.h +++ b/esphome/components/web_server_idf/web_server_idf.h @@ -301,6 +301,14 @@ class AsyncEventSourceResponse { void deq_push_back_with_dedup_(void *source, message_generator_t *message_generator); void process_deferred_queue_(); void process_buffer_(); + void request_close_(); + void process_close_(); + static void close_session_work(void *arg); + + // Deletable only after destroy() zeroed fd_ and no queued HTTPD close work still references this object. + bool safe_to_delete_() const { + return this->fd_.load() == 0 && !this->close_work_queued_.load(std::memory_order_acquire); + } static void destroy(void *p); AsyncEventSource *server_; @@ -311,8 +319,16 @@ class AsyncEventSourceResponse { esphome::web_server::ListEntitiesIterator entities_iterator_; std::string event_buffer_; size_t event_bytes_sent_; - uint16_t consecutive_send_failures_{0}; - static constexpr uint16_t MAX_CONSECUTIVE_SEND_FAILURES = 2500; // ~20 seconds at 125Hz loop rate + uint32_t send_failure_started_ms_{0}; // Zero means no send stall in progress. + uint32_t next_close_attempt_ms_{0}; + // Main-loop only; the HTTPD task never reads or writes this flag. + bool close_requested_{false}; + bool close_retry_warning_logged_{false}; + // Set on the main loop before queueing close work, cleared by the HTTPD-task callback when done. + std::atomic close_work_queued_{false}; + static constexpr uint32_t SEND_STALL_TIMEOUT_MS = 20000; + static constexpr uint32_t CLOSE_RETRY_INTERVAL_MS = 250; + static constexpr uint32_t CLOSE_CONFIRM_INTERVAL_MS = 1000; }; using AsyncEventSourceClient = AsyncEventSourceResponse; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index f59662b319d..9ae83514a64 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -284,7 +284,6 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION -#define USE_OTA_ENCRYPTION_FROM_API #define USE_OTA_ENCRYPTION_PROVISIONED #define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index 7ed11d75543..bf563a4f438 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -74,6 +74,64 @@ FILTER_IDF_LINES: list[str] = [ r"Stopping at filesystem boundary", ] +# Windows code page identifier for UTF-8, as used by ``chcp 65001``. +UTF8_CODEPAGE = 65001 + + +def _get_kernel32(): + """Return the Windows kernel32 module, or None on any other platform.""" + if sys.platform != "win32": + return None + import ctypes + + return ctypes.windll.kernel32 + + +class _Utf8Console: + """Keep an attached Windows console on UTF-8 for the length of the build. + + The build tree runs in UTF-8 mode, so esp_idf_size draws its table with + Unicode box characters. ``idf.py size`` reaches it through ``cmake -P``, + and CMake re-decodes the child's output with the console code page before + printing it, which turns the table into mojibake on any code page but + UTF-8. Every process in the build shares this console, so switching it + here covers CMake too. The old code pages go back on exit so the user's + terminal is left as it was. + + A console that is already on UTF-8 is left alone. The code page belongs to + the console, not to this process, so a build that overlaps another one + must not save UTF-8 as the page to go back to. + """ + + def __init__(self, kernel32) -> None: + self._kernel32 = kernel32 + self._codepages: tuple[int, int] | None = None + + def __enter__(self) -> None: + kernel32 = self._kernel32 + if kernel32 is None: + return + old_in = kernel32.GetConsoleCP() + old_out = kernel32.GetConsoleOutputCP() + # Both calls return 0 when no console is attached. + if not old_in or not old_out: + return + if old_in == UTF8_CODEPAGE and old_out == UTF8_CODEPAGE: + return + # Record the old pages first so a switch that fails part way through + # still gets put back on exit. + self._codepages = (old_in, old_out) + kernel32.SetConsoleCP(UTF8_CODEPAGE) + kernel32.SetConsoleOutputCP(UTF8_CODEPAGE) + + def __exit__(self, *exc_info: object) -> None: + if self._codepages is None: + return + old_in, old_out = self._codepages + self._codepages = None + self._kernel32.SetConsoleCP(old_in) + self._kernel32.SetConsoleOutputCP(old_out) + def main() -> int: # ---- sys.path fix-up --------------------------------------------------- @@ -269,8 +327,29 @@ def main() -> int: is_verbose = any(arg in ("-v", "--verbose") for arg in sys.argv[2:]) filter_lines = None if is_verbose else FILTER_IDF_LINES or None - stdout_shim = sys.stdout = _FilteringTTYStream(sys.stdout, filter_lines) # type: ignore[assignment] - stderr_shim = sys.stderr = _FilteringTTYStream(sys.stderr, filter_lines) # type: ignore[assignment] + class _FilteredStreams: + """Route ``sys.stdout`` and ``sys.stderr`` through the filtering shims. + + On exit each shim releases a last line that never got its + terminator. The shims made here are drained rather than whatever + ``sys.stdout`` holds by then, which the script is free to replace. + """ + + def __init__(self, filter_lines: list[str] | None) -> None: + self._stdout = _FilteringTTYStream(sys.stdout, filter_lines) + self._stderr = _FilteringTTYStream(sys.stderr, filter_lines) + + def __enter__(self) -> None: + sys.stdout = self._stdout # type: ignore[assignment] + sys.stderr = self._stderr # type: ignore[assignment] + + def __exit__(self, *exc_info: object) -> None: + # Drain stderr from a finally so a surprise from the first one + # cannot strand the second. + try: + self._stdout.drain() + finally: + self._stderr.drain() # Shift argv so the target script sees its own path as argv[0] and # its own arguments starting at argv[1]. runpy.run_path does not @@ -288,19 +367,11 @@ def main() -> int: # If idf.py calls sys.exit(), SystemExit propagates out of run_path # and carries the exit code back to our caller. For normal returns, - # fall through and exit with 0. Either way the streams get a chance to - # release a last line that never got its terminator. Drain the shims we - # made rather than sys.stdout, which the script is free to replace, and - # report instead of raising so cleanup cannot bury the real exit code. - try: + # fall through and exit with 0. Either way the context managers drain + # the streams and put the console back on the way out, and they report + # instead of raising so cleanup cannot bury the real exit code. + with _FilteredStreams(filter_lines), _Utf8Console(_get_kernel32()): runpy.run_path(script_path, run_name="__main__") - finally: - # Drain stderr from a finally so a surprise from the first one cannot - # strand the second. - try: - stdout_shim.drain() - finally: - stderr_shim.drain() return 0 diff --git a/script/ci-custom.py b/script/ci-custom.py index 286dda85b9e..6672760b06d 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -3,6 +3,7 @@ import argparse import codecs import collections +from collections.abc import Iterator import fnmatch import functools import os.path @@ -1138,7 +1139,56 @@ def lint_no_std_bind(fname, match): ) -LOG_MULTILINE_RE = re.compile(r"ESP_LOG\w+\s*\(.*?;", re.DOTALL) +LOG_CALL_START_RE = re.compile(r"ESP_LOG\w+\s*\(") +# Comments, raw/plain string literals and single char literals are consumed whole so ; ( ) ? : +# inside them are never seen. A char literal is exactly one (escaped) char so a digit separator +# like 1'000'000 cannot open one. +CPP_COMMENT_RE = r"//[^\n]*|/\*.*?\*/" +CPP_SKIP_RE = ( + CPP_COMMENT_RE + + r'|R"(?P[^(\s]*)\(.*?\)(?P=raw_delim)"|"(?:[^"\\]|\\.)*"|\'(?:[^\'\\\n]|\\.)\'' +) +LOG_CALL_TOKEN_RE = re.compile(CPP_SKIP_RE + r"|[()]", re.DOTALL) +# The last alternative matches a ? or : followed (after spaces or comments) by an opening quote, +# i.e. a string literal used as a ternary branch. +LOG_TERNARY_LITERAL_RE = re.compile( + CPP_SKIP_RE + r"|[?:](?:\s|" + CPP_COMMENT_RE + r')*(?=")', re.DOTALL +) +# A bare NOLINT; a clang-tidy NOLINT(check-name) is aimed at a different tool. +NOLINT_RE = re.compile(r"\bNOLINT\b(?!\()") + + +def _line_col(content: str, pos: int) -> tuple[int, int]: + """1-based line and column of an offset in content.""" + return content.count("\n", 0, pos) + 1, pos - content.rfind("\n", 0, pos) + + +def _iter_log_calls(content: str) -> Iterator[tuple[int, str | None]]: + """Yield (start, text) for every ESP_LOG*(...) call, text running to the matching close paren. + text is None when no matching paren exists so callers can report the call instead of skipping it.""" + for head in LOG_CALL_START_RE.finditer(content): + depth = 1 + for tok in LOG_CALL_TOKEN_RE.finditer(content, head.end()): + if tok.group(0) == "(": + depth += 1 + elif tok.group(0) == ")": + depth -= 1 + if depth == 0: + yield head.start(), content[head.start() : tok.end()] + break + else: + yield head.start(), None + + +def _unbalanced_log_call_error(content: str, pos: int) -> tuple[int, int, str]: + lineno, col = _line_col(content, pos) + return ( + lineno, + col, + "ESP_LOG call has no matching closing parenthesis, so it cannot be checked.", + ) + + LOG_BAD_CONTINUATION_RE = re.compile(r'\\n(?:[^ \\"\r\n\t]|"\s*\n\s*"[^ \\])') LOG_PERCENT_S_CONTINUATION_RE = re.compile(r'\\n(?:%s|"\s*\n\s*"%s)') @@ -1146,16 +1196,16 @@ LOG_PERCENT_S_CONTINUATION_RE = re.compile(r'\\n(?:%s|"\s*\n\s*"%s)') @lint_content_check(include=cpp_include) def lint_log_multiline_continuation(fname, content): errs = [] - for log_match in LOG_MULTILINE_RE.finditer(content): - log_text = log_match.group(0) + for log_start, log_text in _iter_log_calls(content): + if log_text is None: + errs.append(_unbalanced_log_call_error(content, log_start)) + continue for bad_match in LOG_BAD_CONTINUATION_RE.finditer(log_text): # %s may expand to a whitespace prefix at runtime, skip those if LOG_PERCENT_S_CONTINUATION_RE.match(log_text, bad_match.start()): continue # Calculate line number from position in full content - abs_pos = log_match.start() + bad_match.start() - lineno = content.count("\n", 0, abs_pos) + 1 - col = abs_pos - content.rfind("\n", 0, abs_pos) + lineno, col = _line_col(content, log_start + bad_match.start()) errs.append( ( lineno, @@ -1173,6 +1223,90 @@ def lint_log_multiline_continuation(fname, content): return errs +def _find_ternary_literals(text: str) -> Iterator[tuple[int, str]]: + """Yield (offset, literal) for every string literal used as a ternary branch.""" + branch = False + for m in LOG_TERNARY_LITERAL_RE.finditer(text): + tok = m.group(0) + # An empty literal is merged with every other string's terminator, so it costs no RAM, + # while a PSTR("") would add its own flash array; leave it alone. + if branch and tok[0] == '"' and tok != '""': + yield m.start(), tok + branch = tok[0] in "?:" + + +# LOG_STR_LITERAL is a no op everywhere except ESP8266, so code that never builds there is skipped +# to avoid churn: platform specific sources and components for ESP32, LibreTiny, RP2 and Zephyr only. +# A component belongs here only if it has no tests/components//test.esp8266-ard.yaml. +LOG_LITERAL_LINT_EXCLUDE = [ + "*_esp32.cpp", + "*_esp32_*.cpp", + "*_esp_idf.cpp", + "*_rmt.cpp", + "*_zephyr.cpp", + "*_bk72xx.cpp", + "*_libretiny.cpp", + "*_pico_w.cpp", + "*_host.cpp", + "esphome/components/esp32*/*", + "esphome/components/bk72xx*/*", + "esphome/components/ln882h*/*", + "esphome/components/ln882x*/*", + "esphome/components/rp2*/*", + "esphome/components/zephyr*/*", + "esphome/components/host/*", + "esphome/components/libretiny*/*", + "esphome/components/bluetooth_proxy/*", + "esphome/components/bluetooth_connection/*", + "esphome/components/ble_client/*", + "esphome/components/bedjet/*", + "esphome/components/anova/*", + "esphome/components/xiaomi_ble/*", + "esphome/components/bthome_mithermometer/*", + "esphome/components/usb_host/*", + "esphome/components/zigbee/*", + "esphome/components/lvgl/*", + # Test fixtures and host only unit tests - not production embedded code + "tests/integration/fixtures/*", + "tests/components/*", +] + + +@lint_content_check(include=cpp_include, exclude=LOG_LITERAL_LINT_EXCLUDE) +def lint_log_no_bare_literal_ternary( + fname: Path, content: str +) -> list[tuple[int, int, str]]: + errs = [] + for log_start, log_text in _iter_log_calls(content): + if log_text is None: + continue # reported by lint_log_multiline_continuation, which sees every file + # A NOLINT anywhere on the lines the call spans silences every branch in it + first_line = content.rfind("\n", 0, log_start) + 1 + last_line = content.find("\n", log_start + len(log_text)) + if NOLINT_RE.search( + content[first_line : last_line if last_line != -1 else None] + ): + continue + for offset, literal in _find_ternary_literals(log_text): + lineno, col = _line_col(content, log_start + offset) + errs.append( + ( + lineno, + col, + ( + "String literal used as a ternary branch in a log call. On ESP8266 the " + "log macro moves the format string to flash, but bare literal arguments " + "stay in RAM. Wrap each branch passed straight to the log call in " + f"{highlight('LOG_STR_LITERAL(...)')}:\n" + f" Before: {highlight(literal)}\n" + f" After: {highlight(f'LOG_STR_LITERAL({literal})')}\n" + f"(If strictly necessary, add `{highlight('// NOLINT')}` to the end of the line)" + ), + ) + ) + return errs + + @lint_content_find_check( "ESP_LOG", include=["*.h", "*.tcc"], diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 235ad902dbc..d8bcd7e2758 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -476,43 +476,36 @@ def test_static_encryption_key() -> None: ("yaml_name", "defines_present", "defines_absent"), [ # An api key alone compiles the transport in without requiring it; - # the device uses the api server's key, not a copy + # the ota keeps its own pointer to the key so safe mode, which never + # constructs the api server, can still use it ( "api_key_offer", - {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION"}, {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, ), # A password still guards plaintext uploads on an offering device ( "api_key_offer_password", - {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION", "USE_OTA_PASSWORD"}, {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, ), # The ota encryption block is what makes the device refuse plaintext ( "encryption_required", - { - "USE_OTA_ENCRYPTION", - "USE_OTA_ENCRYPTION_REQUIRED", - "USE_OTA_ENCRYPTION_FROM_API", - }, + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, {"USE_OTA_ENCRYPTION_PROVISIONED"}, ), # Without api encryption the ota key is the device's own ( "own_key", {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, - {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, ), # A key provisioned at runtime lives in the api server; the device # offers with it once provisioned and never requires it ( "runtime_api_key", - { - "USE_OTA_ENCRYPTION", - "USE_OTA_ENCRYPTION_FROM_API", - "USE_OTA_ENCRYPTION_PROVISIONED", - }, + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_PROVISIONED"}, {"USE_OTA_ENCRYPTION_REQUIRED"}, ), # No api encryption at all keeps the noise glue out of the build @@ -522,7 +515,6 @@ def test_static_encryption_key() -> None: { "USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED", - "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED", }, ), @@ -541,8 +533,10 @@ def test_encryption_offer_codegen( assert defines_present <= defines assert not (defines_absent & defines) encrypted = "USE_OTA_ENCRYPTION" in defines_present - own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_PROVISIONED" not in defines_present assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + # The api shares the ota's array instead of emitting the same key twice + assert main_cpp.count("_psk[] PROGMEM") == (1 if own_key else 0) assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) # The noise transport source compiles only when the define is set assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) diff --git a/tests/components/esp32_hosted/test-no-wifi.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-no-wifi.esp32-p4-idf.yaml new file mode 100644 index 00000000000..2f57abd296e --- /dev/null +++ b/tests/components/esp32_hosted/test-no-wifi.esp32-p4-idf.yaml @@ -0,0 +1,12 @@ +# No wifi, espnow or BLE: nothing else re-includes esp_wifi for the ESP-NOW shim. +esp32_hosted: + variant: ESP32C6 + slot: 1 + active_high: true + reset_pin: GPIO15 + cmd_pin: GPIO13 + clk_pin: GPIO12 + d0_pin: GPIO11 + d1_pin: GPIO10 + d2_pin: GPIO9 + d3_pin: GPIO8 diff --git a/tests/integration/fixtures/host_ota_encrypted_safe_mode.yaml b/tests/integration/fixtures/host_ota_encrypted_safe_mode.yaml new file mode 100644 index 00000000000..57f3d57a010 --- /dev/null +++ b/tests/integration/fixtures/host_ota_encrypted_safe_mode.yaml @@ -0,0 +1,13 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + encryption: +safe_mode: +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key_safe_mode.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key_safe_mode.yaml new file mode 100644 index 00000000000..c00954515b1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key_safe_mode.yaml @@ -0,0 +1,11 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +safe_mode: +logger: + level: DEBUG diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index 5f526dce5f4..b160f307628 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -15,6 +15,12 @@ import os from pathlib import Path import struct +_ENTRY = struct.Struct(" Path: """Return the on-disk prefs file path for a host-platform device. @@ -42,16 +48,33 @@ def write_host_prefs(device_name: str, entries: dict[int, bytes]) -> Path: for key, data in entries.items(): if len(data) > 255: raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") - payload += struct.pack(" Path: - """Write a single preference entry, replacing the file's contents. +def read_host_prefs(device_name: str) -> dict[int, bytes]: + """Read the preference entries of a host-platform device; empty when + the file does not exist.""" + path = host_prefs_path(device_name) + if not path.exists(): + return {} + payload = path.read_bytes() + entries: dict[int, bytes] = {} + pos = 0 + while pos < len(payload): + key, length = _ENTRY.unpack_from(payload, pos) + pos += _ENTRY.size + entries[key] = payload[pos : pos + length] + pos += length + return entries - Returns the path that was written. - """ - return write_host_prefs(device_name, {key: data}) + +def force_safe_mode(device_name: str) -> None: + """Make the next boot of a host-platform device enter safe mode; other + saved preferences are kept.""" + entries = read_host_prefs(device_name) + entries[_SAFE_MODE_RTC_KEY] = struct.pack(" None: + """Provision PROVISIONING_PSK over the api and wait for it to activate.""" + async with api_client_connected(port=dev.api_port, noise_psk=ZERO_PSK) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -227,6 +237,31 @@ async def test_host_ota_encrypted( await dev.ota(None, API_KEY, "encrypted OTA reported failure") +@pytest.mark.asyncio +async def test_host_ota_encrypted_safe_mode( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> None: + """Safe mode never constructs the api server, so an encrypted OTA with the + api key has to run on the ota component's own copy of that key.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # The api port never opens in safe mode, so wait for the log line instead + force_safe_mode(DEVICE_NAME) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT) + await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT) + # The safe mode boot clears the counter, so the re-exec boots normally + await dev.ota(None, API_KEY, "encrypted OTA in safe mode reported failure") + + @pytest.mark.asyncio async def test_host_ota_api_key_offer_with_password( yaml_config: str, @@ -305,11 +340,7 @@ async def test_host_ota_provisioned_api_key( None, None, "plaintext upload to an unprovisioned device must succeed" ) - async with api_client_connected( - port=dev.api_port, noise_psk=ZERO_PSK - ) as client: - assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True - await asyncio.sleep(KEY_ACTIVATION_DELAY) + await _provision_key(dev, api_client_connected) key = PROVISIONING_PSK.decode() await dev.ota( @@ -319,6 +350,45 @@ async def test_host_ota_provisioned_api_key( await dev.ota(None, None, "plaintext must stay accepted on an offering device") +@pytest.mark.asyncio +async def test_host_ota_provisioned_api_key_safe_mode( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """Safe mode never constructs the api server, so the OTA has to load the + provisioned key from preferences itself to keep encrypting there.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _provision_key(dev, api_client_connected) + + # The saved key is already on disk; a host reboot outside an OTA just + # exits, so safe mode takes a second start + force_safe_mode(DEVICE_NAME) + key = PROVISIONING_PSK.decode() + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT) + await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT) + await dev.ota( + None, + key, + "encrypted upload with the provisioned key must succeed in safe mode", + ) + # The re-exec boots normally and the api reads the same record + async with api_client_connected(port=dev.api_port, noise_psk=key): + pass + + @pytest.mark.asyncio async def test_host_ota_rejects_garbage( yaml_config: str, diff --git a/tests/integration/test_safe_mode_loop_runs.py b/tests/integration/test_safe_mode_loop_runs.py index 652877fc092..55756fca13d 100644 --- a/tests/integration/test_safe_mode_loop_runs.py +++ b/tests/integration/test_safe_mode_loop_runs.py @@ -25,19 +25,13 @@ from __future__ import annotations import asyncio import re -import struct import pytest from .conftest import run_binary -from .host_prefs import clear_host_prefs, write_host_pref +from .host_prefs import clear_host_prefs, force_safe_mode from .types import CompileFunction, ConfigWriter -# Must match esphome::safe_mode::RTC_KEY in safe_mode.h -SAFE_MODE_RTC_KEY = 233825507 -# Must match esphome::safe_mode::SafeModeComponent::ENTER_SAFE_MODE_MAGIC -ENTER_SAFE_MODE_MAGIC = 0x5AFE5AFE - DEVICE_NAME = "safe-mode-loop-runs" THREAD_LOG_MARKER = "looping component ran in safe mode" @@ -56,9 +50,7 @@ async def test_safe_mode_loop_runs( # Compile finished successfully; pre-populate prefs so the *next* run # enters safe mode immediately. - write_host_pref( - DEVICE_NAME, SAFE_MODE_RTC_KEY, struct.pack(" None: def test_nolint_on_control_line_suppresses() -> None: assert not _lint("if (x) // NOLINT\n ESP_LOGD(t);\n") + + +# --- ESP_LOG call scanner and bare-literal-ternary lint --- + + +def _calls(content: str) -> list[str | None]: + return [text for _, text in ci_custom._iter_log_calls(content)] + + +def _ternary_errors(content: str) -> list[tuple[int, int]]: + errs = ci_custom.lint_log_no_bare_literal_ternary(Path("x.cpp"), content) + return [(line, col) for line, col, _ in errs] + + +@pytest.mark.parametrize( + "content", + [ + 'ESP_LOGD(TAG, "a ) b ( c; d")', + 'ESP_LOGD(TAG, "quote \\" inside")', + "ESP_LOGD(TAG, \"%s\", format_hex_pretty(x, '-', false).c_str())", + "ESP_LOGD(TAG, \"%c%c\", '(', ')')", + "ESP_LOGD(TAG, \"%d\", 1'000'000)", + 'ESP_LOGD(TAG, // it\'s a comment with ) and (\n "x")', + 'ESP_LOGD(TAG, /* :) */ "x")', + 'ESP_LOGD(TAG, "%s", R"(say "hi" :) )")', + 'ESP_LOGD(TAG, "%s", R"x(a)"b)x")', + ], +) +def test_iter_log_calls_spans_whole_call(content: str) -> None: + calls = _calls(content + ";\nint other = (1);") + assert calls == [content] + + +def test_iter_log_calls_reports_unbalanced_call_once() -> None: + content = 'ESP_LOGD(TAG, "x";\nvoid f();' + assert _calls(content) == [None] + errs = ci_custom.lint_log_multiline_continuation(Path("x.cpp"), content) + assert len(errs) == 1 + assert errs[0][:2] == (1, 1) + assert "no matching closing parenthesis" in errs[0][2] + assert _ternary_errors(content) == [] + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + # A ; inside the format string no longer cuts the call short + ('ESP_LOGD(TAG, "a; b\\nc %s", x);', [(1, 20)]), + # A \n%s continuation is exempt since %s may expand to leading whitespace + ('ESP_LOGD(TAG, "a\\n%s", x);', []), + ('ESP_LOGD(TAG, "a\\n b");', []), + ], +) +def test_multiline_continuation_detection( + content: str, expected: list[tuple[int, int]] +) -> None: + errs = ci_custom.lint_log_multiline_continuation(Path("x.cpp"), content) + assert [(line, col) for line, col, _ in errs] == expected + + +def test_exclusion_list_only_names_components_without_esp8266_tests() -> None: + root = Path(__file__).parent / ".." / ".." + for pattern in ci_custom.LOG_LITERAL_LINT_EXCLUDE: + if not pattern.startswith("esphome/components/"): + continue + prefix = pattern.removeprefix("esphome/components/").split("/")[0] + comps = list((root / "esphome" / "components").glob(prefix)) + assert comps, f"{pattern!r} matches no component" + for comp in comps: + test = root / "tests" / "components" / comp.name / "test.esp8266-ard.yaml" + assert not test.exists(), ( + f"{comp.name} builds for ESP8266, drop {pattern!r}" + ) + + +def test_unbalanced_calls_are_reported_by_a_check_that_sees_every_file() -> None: + # lint_log_no_bare_literal_ternary skips unbalanced calls and relies on this + checks = {c["func"].__name__: c for c in ci_custom.LINT_CONTENT_CHECKS} + continuation = checks["lint_log_multiline_continuation"] + ternary = checks["lint_log_no_bare_literal_ternary"] + assert continuation["exclude"] == [] + assert continuation["include"] == ternary["include"] + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + ('ESP_LOGD(TAG, "%s", x ? "on" : "off");', [(1, 25), (1, 32)]), + ( + 'ESP_LOGD(TAG, "%s", x ? LOG_STR_LITERAL("on") : LOG_STR_LITERAL("off"));', + [], + ), + ('ESP_LOGD(TAG, "%s", x ? LOG_STR_LITERAL("on") : "off");', [(1, 49)]), + ('ESP_LOGD(TAG, "%s", x ? "on" : "");', [(1, 25)]), + ( + 'ESP_LOGD(TAG, "%s",\n x ? "yes"\n : "no");', + [(2, 14), (3, 14)], + ), + ("ESP_LOGD(TAG, \"%c\", x ? '1' : '0');", []), + ('ESP_LOGD(TAG, "a ? b : c %s", x ? "on" : "off");', [(1, 35), (1, 42)]), + ('ESP_LOGD(TAG, "x:" "y %s", p);', []), + ('ESP_LOGD(TAG, "%s", x ? "on" : "off"); // NOLINT', []), + ('ESP_LOGD(TAG, "%s",\n x ? "yes"\n : "no"); // NOLINT', []), + ('ESP_LOGD(TAG, "%s", x ? /* c */ "on" : "off");', [(1, 33), (1, 40)]), + ( + 'ESP_LOGD(TAG, "%s",\n x ? "on" // NOLINT(some-clang-check)\n : "off");', + [(2, 14), (3, 14)], + ), + ], +) +def test_ternary_literal_detection( + content: str, expected: list[tuple[int, int]] +) -> None: + assert _ternary_errors(content) == expected + + +def test_ternary_error_message_names_the_literal() -> None: + errs = ci_custom.lint_log_no_bare_literal_ternary( + Path("x.cpp"), 'ESP_LOGD(TAG, "%s", x ? "enabled" : LOG_STR_LITERAL("off"));' + ) + assert len(errs) == 1 + assert 'LOG_STR_LITERAL("enabled")' in errs[0][2] diff --git a/tests/unit_tests/test_espidf_runner.py b/tests/unit_tests/test_espidf_runner.py index e4cc6e137e7..71ccfee1f2c 100644 --- a/tests/unit_tests/test_espidf_runner.py +++ b/tests/unit_tests/test_espidf_runner.py @@ -209,3 +209,107 @@ def test_runner_streams_output_before_the_build_finishes( # Join before leaving the block, so the reader is done rather than # racing ``Popen`` closing the pipe under it. reader.join(1.0) + + +class _FakeKernel32: + """Stand-in for the Windows kernel32 console code page calls.""" + + def __init__(self, input_cp: int, output_cp: int) -> None: + self.input_cp = input_cp + self.output_cp = output_cp + self.calls: list[tuple[str, int]] = [] + + def GetConsoleCP(self) -> int: # noqa: N802 + return self.input_cp + + def GetConsoleOutputCP(self) -> int: # noqa: N802 + return self.output_cp + + def SetConsoleCP(self, codepage: int) -> int: # noqa: N802 + self.calls.append(("SetConsoleCP", codepage)) + self.input_cp = codepage + return 1 + + def SetConsoleOutputCP(self, codepage: int) -> int: # noqa: N802 + self.calls.append(("SetConsoleOutputCP", codepage)) + self.output_cp = codepage + return 1 + + +def test_main_runs_the_build_with_a_utf8_console( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """An attached console is switched to UTF-8 and then put back.""" + kernel32 = _FakeKernel32(850, 850) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.calls == [ + ("SetConsoleCP", runner.UTF8_CODEPAGE), + ("SetConsoleOutputCP", runner.UTF8_CODEPAGE), + ("SetConsoleCP", 850), + ("SetConsoleOutputCP", 850), + ] + + +def test_main_restores_the_console_when_the_build_dies( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """A failing build must not leave the user's console on UTF-8.""" + kernel32 = _FakeKernel32(437, 437) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + _prepare_main(monkeypatch, fixture_path / "espidf" / "crashing_probe.py") + + with pytest.raises(SystemExit): + runner.main() + + assert (kernel32.input_cp, kernel32.output_cp) == (437, 437) + + +def test_main_restores_the_console_when_the_switch_fails_part_way( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """A failed output page switch must not strand the changed input page.""" + kernel32 = _FakeKernel32(850, 850) + + def _refuse(codepage: int) -> int: + kernel32.calls.append(("SetConsoleOutputCP", codepage)) + return 0 + + kernel32.SetConsoleOutputCP = _refuse # type: ignore[method-assign] + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.input_cp == 850 + assert kernel32.calls[-2:] == [("SetConsoleCP", 850), ("SetConsoleOutputCP", 850)] + + +def test_main_leaves_the_console_alone_when_there_is_none( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """Without a console the code page calls return 0 and nothing is set.""" + kernel32 = _FakeKernel32(0, 0) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.calls == [] + + +def test_main_leaves_a_console_already_on_utf8_alone( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """An overlapping build must not save UTF-8 as the page to restore.""" + kernel32 = _FakeKernel32(runner.UTF8_CODEPAGE, runner.UTF8_CODEPAGE) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.calls == [] + + +@pytest.mark.skipif(sys.platform == "win32", reason="kernel32 exists on Windows") +def test_get_kernel32_is_none_off_windows() -> None: + assert runner._get_kernel32() is None